stmt.c (expand_asm_operands): Treat EXTRA_CONSTRAINT like g or X.

	* stmt.c (expand_asm_operands): Treat EXTRA_CONSTRAINT like g or X.

	* config/ia64/ia64-protos.h: Update.
	* config/ia64/ia64.c (gr_register_operand): New.
	(fr_register_operand, grfr_register_operand): New.
	(gr_nonimmediate_operand, grfr_nonimmediate_operand): New.
	(grfr_reg_or_8bit_operand): New.
	(gr_reg_or_0_operand): Rename from reg_or_0_operand and
	use gr_register_operand.
	(gr_reg_or_5bit_operand, gr_reg_or_6bit_operand): Likewise.
	(gr_reg_or_8bit_operand, gr_reg_or_8bit_adjusted_operand): Likewise.
	(gr_reg_or_8bit_and_adjusted_operand): Likewise.
	(gr_reg_or_14bit_operand, gr_reg_or_22bit_operand): Likewise.
	(fr_reg_or_fp01_operand): Likewise.
	(not_postinc_memory_operand): New.
	(ia64_split_timode): Remove unused variables.
	(rtx_needs_barrier): Check arguments to cmpxchg.
	(builtin_description): Remove.
	(bdesc_2argsi, bdesc_2argdi): Remove.
	(ia64_init_builtins): Declare all builtins directly.
	(ia64_expand_fetch_and_op): Rewrite to be called from
	ia64_expand_builtin directly.  Use expand_binop and co.
	(ia64_expand_op_and_fetch): Likewise.
	(ia64_expand_compare_and_swap): Likewise.
	(ia64_expand_binop_builtin): Remove.
	(ia64_expand_lock_test_and_set): New.
	(ia64_expand_lock_release): New.
	(ia64_expand_builtin): Use them.
	* config/ia64/ia64.h (CONSTRAINT_OK_FOR_S): New.
	(EXTRA_CONSTRAINT): Use it.
	(PREDICATE_CODES): Update.
	* config/ia64/ia64.md (*): Use gr_register_operand and co.
	(mf): Indicate that we set memory as well as use it.
	(fetchadd_acq_si): Show memory being modified as well.
	(fetchadd_acq_di, cmpxchg_acq_si, cmpxchg_acq_di): Likewise.
	(val_compare_and_swap_si, val_compare_and_swap_di): Remove.
	(lock_test_and_set_si, lock_test_and_set_di): Remove.
	(fetch_and_add_si, fetch_and_sub_si, fetch_and_or_si): Remove.
	(fetch_and_and_si, fetch_and_xor_si, fetch_and_nand_si): Remove.
	(fetch_and_add_di, fetch_and_sub_di, fetch_and_or_di): Remove.
	(fetch_and_and_di, fetch_and_xor_di, fetch_and_nand_di): Remove.
	(add_and_fetch_di, sub_and_fetch_di, or_and_fetch_di): Remove.
	(and_and_fetch_di, xor_and_fetch_di, nand_and_fetch_di): Remove.
	(add_and_fetch_si, sub_and_fetch_si, or_and_fetch_si): Remove.
	(and_and_fetch_si, xor_and_fetch_si, nand_and_fetch_si): Remove.
	* config/ia64/ia64intrin.h (*): Cast result to the appropriate
	return type.  Pretty print definitions.

From-SVN: r35956

7 files changed, 1116 insertions, 1208 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 3a3ed67..f346a88 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,53 @@
+2000-08-24  Richard Henderson  <rth@cygnus.com>
+
+	* stmt.c (expand_asm_operands): Treat EXTRA_CONSTRAINT like g or X.
+
+	* config/ia64/ia64-protos.h: Update.
+	* config/ia64/ia64.c (gr_register_operand): New.
+	(fr_register_operand, grfr_register_operand): New.
+	(gr_nonimmediate_operand, grfr_nonimmediate_operand): New.
+	(grfr_reg_or_8bit_operand): New.
+	(gr_reg_or_0_operand): Rename from reg_or_0_operand and 
+	use gr_register_operand.
+	(gr_reg_or_5bit_operand, gr_reg_or_6bit_operand): Likewise.
+	(gr_reg_or_8bit_operand, gr_reg_or_8bit_adjusted_operand): Likewise.
+	(gr_reg_or_8bit_and_adjusted_operand): Likewise.
+	(gr_reg_or_14bit_operand, gr_reg_or_22bit_operand): Likewise.
+	(fr_reg_or_fp01_operand): Likewise.
+	(not_postinc_memory_operand): New.
+	(ia64_split_timode): Remove unused variables.
+	(rtx_needs_barrier): Check arguments to cmpxchg.
+	(builtin_description): Remove.
+	(bdesc_2argsi, bdesc_2argdi): Remove.
+	(ia64_init_builtins): Declare all builtins directly.
+	(ia64_expand_fetch_and_op): Rewrite to be called from
+	ia64_expand_builtin directly.  Use expand_binop and co.
+	(ia64_expand_op_and_fetch): Likewise.
+	(ia64_expand_compare_and_swap): Likewise.
+	(ia64_expand_binop_builtin): Remove.
+	(ia64_expand_lock_test_and_set): New.
+	(ia64_expand_lock_release): New.
+	(ia64_expand_builtin): Use them.
+	* config/ia64/ia64.h (CONSTRAINT_OK_FOR_S): New.
+	(EXTRA_CONSTRAINT): Use it.
+	(PREDICATE_CODES): Update.
+	* config/ia64/ia64.md (*): Use gr_register_operand and co.
+	(mf): Indicate that we set memory as well as use it.
+	(fetchadd_acq_si): Show memory being modified as well.
+	(fetchadd_acq_di, cmpxchg_acq_si, cmpxchg_acq_di): Likewise. 
+	(val_compare_and_swap_si, val_compare_and_swap_di): Remove.
+	(lock_test_and_set_si, lock_test_and_set_di): Remove.
+	(fetch_and_add_si, fetch_and_sub_si, fetch_and_or_si): Remove.
+	(fetch_and_and_si, fetch_and_xor_si, fetch_and_nand_si): Remove.
+	(fetch_and_add_di, fetch_and_sub_di, fetch_and_or_di): Remove.
+	(fetch_and_and_di, fetch_and_xor_di, fetch_and_nand_di): Remove.
+	(add_and_fetch_di, sub_and_fetch_di, or_and_fetch_di): Remove.
+	(and_and_fetch_di, xor_and_fetch_di, nand_and_fetch_di): Remove.
+	(add_and_fetch_si, sub_and_fetch_si, or_and_fetch_si): Remove.
+	(and_and_fetch_si, xor_and_fetch_si, nand_and_fetch_si): Remove.
+	* config/ia64/ia64intrin.h (*): Cast result to the appropriate
+	return type.  Pretty print definitions.
+
 2000-08-24  Jim Wilson  <wilson@cygnus.com>
 
 	* config/ia64/ia64.md (movdi): Don't call gen_movdi_symbolic if
diff --git a/gcc/config/ia64/ia64-protos.h b/gcc/config/ia64/ia64-protos.h
index 180418d..8aead0a 100644
--- a/gcc/config/ia64/ia64-protos.h
+++ b/gcc/config/ia64/ia64-protos.h
@@ -34,23 +34,30 @@ extern int symbolic_operand PARAMS((rtx, enum machine_mode));
 extern int function_operand PARAMS((rtx, enum machine_mode));
 extern int setjmp_operand PARAMS((rtx, enum machine_mode));
 extern int move_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_0_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_5bit_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_6bit_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_8bit_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_8bit_adjusted_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_8bit_and_adjusted_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_14bit_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_22bit_operand PARAMS((rtx, enum machine_mode));
+extern int gr_register_operand PARAMS((rtx, enum machine_mode));
+extern int fr_register_operand PARAMS((rtx, enum machine_mode));
+extern int grfr_register_operand PARAMS((rtx, enum machine_mode));
+extern int gr_nonimmediate_operand PARAMS((rtx, enum machine_mode));
+extern int grfr_nonimmediate_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_0_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_5bit_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_6bit_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_8bit_operand PARAMS((rtx, enum machine_mode));
+extern int grfr_reg_or_8bit_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_8bit_adjusted_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_8bit_and_adjusted_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_14bit_operand PARAMS((rtx, enum machine_mode));
+extern int gr_reg_or_22bit_operand PARAMS((rtx, enum machine_mode));
 extern int shift_count_operand PARAMS((rtx, enum machine_mode));
 extern int shift_32bit_count_operand PARAMS((rtx, enum machine_mode));
 extern int shladd_operand PARAMS((rtx, enum machine_mode));
 extern int fetchadd_operand PARAMS((rtx, enum machine_mode));
-extern int reg_or_fp01_operand PARAMS((rtx, enum machine_mode));
+extern int fr_reg_or_fp01_operand PARAMS((rtx, enum machine_mode));
 extern int normal_comparison_operator PARAMS((rtx, enum machine_mode));
 extern int adjusted_comparison_operator PARAMS((rtx, enum machine_mode));
 extern int call_multiple_values_operation PARAMS((rtx, enum machine_mode));
 extern int destination_operand PARAMS((rtx, enum machine_mode));
+extern int not_postinc_memory_operand PARAMS((rtx, enum machine_mode));
 extern int predicate_operator PARAMS((rtx, enum machine_mode));
 extern int ar_lc_reg_operand PARAMS((rtx, enum machine_mode));
 extern int ar_ccv_reg_operand PARAMS((rtx, enum machine_mode));
@@ -72,10 +79,6 @@ extern void ia64_function_epilogue PARAMS((FILE *, int));
 
 extern int ia64_direct_return PARAMS((void));
 extern void ia64_expand_load_address PARAMS((rtx, rtx));
-extern void ia64_expand_fetch_and_op PARAMS ((enum fetchop_code,
-					      enum machine_mode, rtx []));
-extern void ia64_expand_op_and_fetch PARAMS ((enum fetchop_code,
-					      enum machine_mode, rtx []));
 
 extern void ia64_initialize_trampoline PARAMS((rtx, rtx, rtx));
 extern void ia64_print_operand_address PARAMS((FILE *, rtx));
diff --git a/gcc/config/ia64/ia64.c b/gcc/config/ia64/ia64.c
index f0ad8ba4..f739729 100644
--- a/gcc/config/ia64/ia64.c
+++ b/gcc/config/ia64/ia64.c
@@ -105,6 +105,9 @@ static void finish_spill_pointers PARAMS ((void));
 static rtx spill_restore_mem PARAMS ((rtx, HOST_WIDE_INT));
 static void do_spill PARAMS ((rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT, rtx));
 static void do_restore PARAMS ((rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT));
+static rtx gen_movdi_x PARAMS ((rtx, rtx, rtx));
+static rtx gen_fr_spill_x PARAMS ((rtx, rtx, rtx));
+static rtx gen_fr_restore_x PARAMS ((rtx, rtx, rtx));
 
 static enum machine_mode hfa_element_mode PARAMS ((tree, int));
 static void fix_range PARAMS ((const char *));
@@ -114,9 +117,17 @@ static void ia64_mark_machine_status PARAMS ((struct function *));
 static void emit_insn_group_barriers PARAMS ((rtx));
 static void emit_predicate_relation_info PARAMS ((rtx));
 static int process_set PARAMS ((FILE *, rtx));
-static rtx ia64_expand_compare_and_swap PARAMS ((enum insn_code, tree,
-						 rtx, int));
-static rtx ia64_expand_binop_builtin PARAMS ((enum insn_code, tree, rtx));
+
+static rtx ia64_expand_fetch_and_op PARAMS ((optab, enum machine_mode,
+					     tree, rtx));
+static rtx ia64_expand_op_and_fetch PARAMS ((optab, enum machine_mode,
+					     tree, rtx));
+static rtx ia64_expand_compare_and_swap PARAMS ((enum machine_mode, int,
+						 tree, rtx));
+static rtx ia64_expand_lock_test_and_set PARAMS ((enum machine_mode,
+						  tree, rtx));
+static rtx ia64_expand_lock_release PARAMS ((enum machine_mode, tree, rtx));
+
 
 /* Return 1 if OP is a valid operand for the MEM of a CALL insn.  */
 
@@ -306,64 +317,175 @@ move_operand (op, mode)
   return general_operand (op, mode);
 }
 
-/* Return 1 if OP is a register operand, or zero.  */
+/* Return 1 if OP is a register operand that is (or could be) a GR reg.  */
+
+int
+gr_register_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  if (! register_operand (op, mode))
+    return 0;
+  if (GET_CODE (op) == SUBREG)
+    op = SUBREG_REG (op);
+  if (GET_CODE (op) == REG)
+    {
+      unsigned int regno = REGNO (op);
+      if (regno < FIRST_PSEUDO_REGISTER)
+	return GENERAL_REGNO_P (regno);
+    }
+  return 1;
+}
+
+/* Return 1 if OP is a register operand that is (or could be) an FR reg.  */
+
+int
+fr_register_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  if (! register_operand (op, mode))
+    return 0;
+  if (GET_CODE (op) == SUBREG)
+    op = SUBREG_REG (op);
+  if (GET_CODE (op) == REG)
+    {
+      unsigned int regno = REGNO (op);
+      if (regno < FIRST_PSEUDO_REGISTER)
+	return FR_REGNO_P (regno);
+    }
+  return 1;
+}
+
+/* Return 1 if OP is a register operand that is (or could be) a GR/FR reg.  */
+
+int
+grfr_register_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  if (! register_operand (op, mode))
+    return 0;
+  if (GET_CODE (op) == SUBREG)
+    op = SUBREG_REG (op);
+  if (GET_CODE (op) == REG)
+    {
+      unsigned int regno = REGNO (op);
+      if (regno < FIRST_PSEUDO_REGISTER)
+	return GENERAL_REGNO_P (regno) || FR_REGNO_P (regno);
+    }
+  return 1;
+}
+
+/* Return 1 if OP is a nonimmediate operand that is (or could be) a GR reg.  */
+
+int
+gr_nonimmediate_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  if (! nonimmediate_operand (op, mode))
+    return 0;
+  if (GET_CODE (op) == SUBREG)
+    op = SUBREG_REG (op);
+  if (GET_CODE (op) == REG)
+    {
+      unsigned int regno = REGNO (op);
+      if (regno < FIRST_PSEUDO_REGISTER)
+	return GENERAL_REGNO_P (regno);
+    }
+  return 1;
+}
+
+/* Return 1 if OP is a nonimmediate operand that is a GR/FR reg.  */
+
+int
+grfr_nonimmediate_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  if (! nonimmediate_operand (op, mode))
+    return 0;
+  if (GET_CODE (op) == SUBREG)
+    op = SUBREG_REG (op);
+  if (GET_CODE (op) == REG)
+    {
+      unsigned int regno = REGNO (op);
+      if (regno < FIRST_PSEUDO_REGISTER)
+	return GENERAL_REGNO_P (regno) || FR_REGNO_P (regno);
+    }
+  return 1;
+}
+
+/* Return 1 if OP is a GR register operand, or zero.  */
 
 int
-reg_or_0_operand (op, mode)
+gr_reg_or_0_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
-  return (op == const0_rtx || register_operand (op, mode));
+  return (op == const0_rtx || gr_register_operand (op, mode));
 }
 
-/* Return 1 if OP is a register operand, or a 5 bit immediate operand.  */
+/* Return 1 if OP is a GR register operand, or a 5 bit immediate operand.  */
 
 int
-reg_or_5bit_operand (op, mode)
+gr_reg_or_5bit_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && INTVAL (op) >= 0 && INTVAL (op) < 32)
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
-/* Return 1 if OP is a register operand, or a 6 bit immediate operand.  */
+/* Return 1 if OP is a GR register operand, or a 6 bit immediate operand.  */
 
 int
-reg_or_6bit_operand (op, mode)
+gr_reg_or_6bit_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_M (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
-/* Return 1 if OP is a register operand, or an 8 bit immediate operand.  */
+/* Return 1 if OP is a GR register operand, or an 8 bit immediate operand.  */
 
 int
-reg_or_8bit_operand (op, mode)
+gr_reg_or_8bit_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_K (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
+/* Return 1 if OP is a GR/FR register operand, or an 8 bit immediate.  */
+
+int
+grfr_reg_or_8bit_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_K (INTVAL (op)))
+	  || GET_CODE (op) == CONSTANT_P_RTX
+	  || grfr_register_operand (op, mode));
+}
 
 /* Return 1 if OP is a register operand, or an 8 bit adjusted immediate
    operand.  */
 
 int
-reg_or_8bit_adjusted_operand (op, mode)
+gr_reg_or_8bit_adjusted_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_L (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
 /* Return 1 if OP is a register operand, or is valid for both an 8 bit
@@ -372,38 +494,38 @@ reg_or_8bit_adjusted_operand (op, mode)
    so we need the union of the immediates accepted by GT and LT.  */
 
 int
-reg_or_8bit_and_adjusted_operand (op, mode)
+gr_reg_or_8bit_and_adjusted_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_K (INTVAL (op))
 	   && CONST_OK_FOR_L (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
 /* Return 1 if OP is a register operand, or a 14 bit immediate operand.  */
 
 int
-reg_or_14bit_operand (op, mode)
+gr_reg_or_14bit_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_I (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
 /* Return 1 if OP is a register operand, or a 22 bit immediate operand.  */
 
 int
-reg_or_22bit_operand (op, mode)
+gr_reg_or_22bit_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_INT && CONST_OK_FOR_J (INTVAL (op)))
 	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || gr_register_operand (op, mode));
 }
 
 /* Return 1 if OP is a 6 bit immediate operand.  */
@@ -458,13 +580,12 @@ fetchadd_operand (op, mode)
 /* Return 1 if OP is a floating-point constant zero, one, or a register.  */
 
 int
-reg_or_fp01_operand (op, mode)
+fr_reg_or_fp01_operand (op, mode)
      rtx op;
      enum machine_mode mode;
 {
   return ((GET_CODE (op) == CONST_DOUBLE && CONST_DOUBLE_OK_FOR_G (op))
-	  || GET_CODE (op) == CONSTANT_P_RTX
-	  || register_operand (op, mode));
+	  || fr_register_operand (op, mode));
 }
 
 /* Like nonimmediate_operand, but don't allow MEMs that try to use a
@@ -484,6 +605,17 @@ destination_operand (op, mode)
   return 1;
 }
 
+/* Like memory_operand, but don't allow post-increments.  */
+
+int
+not_postinc_memory_operand (op, mode)
+     rtx op;
+     enum machine_mode mode;
+{
+  return (memory_operand (op, mode)
+	  && GET_RTX_CLASS (GET_CODE (XEXP (op, 0))) != 'a');
+}
+
 /* Return 1 if this is a comparison operator, which accepts an normal 8-bit
    signed immediate operand.  */
 
@@ -620,7 +752,7 @@ tfreg_or_fp01_operand (op, mode)
 {
   if (GET_CODE (op) == SUBREG)
     return 0;
-  return reg_or_fp01_operand (op, mode);
+  return fr_reg_or_fp01_operand (op, mode);
 }
 
 /* Return 1 if the operands of a move are ok.  */
@@ -767,9 +899,7 @@ ia64_split_timode (out, in, scratch)
 
     case MEM:
       {
-	HOST_WIDE_INT offset;
 	rtx base = XEXP (in, 0);
-	rtx offset_rtx;
 
 	switch (GET_CODE (base))
 	  {
@@ -3832,12 +3962,16 @@ rtx_needs_barrier (x, flags, pred)
 	  break;
 
         case 12: /* mf */
-        case 13: /* cmpxchg_acq */
         case 19: /* fetchadd_acq */
 	case 20: /* mov = ar.bsp */
 	case 21: /* flushrs */
           break;
 
+        case 13: /* cmpxchg_acq */
+	  need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 1), flags, pred);
+	  need_barrier |= rtx_needs_barrier (XVECEXP (x, 0, 2), flags, pred);
+	  break;
+
 	default:
 	  abort ();
 	}
@@ -4569,81 +4703,10 @@ process_for_unwind_directive (asm_out_file, insn)
     }
 }
 
-#define def_builtin(name, type, code) \
-  builtin_function ((name), (type), (code), BUILT_IN_MD, NULL_PTR)
-
-struct builtin_description
-{
-  enum insn_code icode;
-  const char *name;
-  enum ia64_builtins code;
-  enum rtx_code comparison;
-  unsigned int flag;
-};
-
-/* All 32 bit intrinsics that take 2 arguments. */
-static struct builtin_description bdesc_2argsi[] =
-{
-  { CODE_FOR_fetch_and_add_si, "__sync_fetch_and_add_si",
-    IA64_BUILTIN_FETCH_AND_ADD_SI, 0, 0 },
-  { CODE_FOR_fetch_and_sub_si, "__sync_fetch_and_sub_si",
-    IA64_BUILTIN_FETCH_AND_SUB_SI, 0, 0 },
-  { CODE_FOR_fetch_and_or_si, "__sync_fetch_and_or_si",
-    IA64_BUILTIN_FETCH_AND_OR_SI, 0, 0 },
-  { CODE_FOR_fetch_and_and_si, "__sync_fetch_and_and_si",
-    IA64_BUILTIN_FETCH_AND_AND_SI, 0, 0 },
-  { CODE_FOR_fetch_and_xor_si, "__sync_fetch_and_xor_si",
-    IA64_BUILTIN_FETCH_AND_XOR_SI, 0, 0 },
-  { CODE_FOR_fetch_and_nand_si, "__sync_fetch_and_nand_si",
-    IA64_BUILTIN_FETCH_AND_NAND_SI, 0, 0 },
-  { CODE_FOR_add_and_fetch_si, "__sync_add_and_fetch_si",
-    IA64_BUILTIN_ADD_AND_FETCH_SI, 0, 0 },
-  { CODE_FOR_sub_and_fetch_si, "__sync_sub_and_fetch_si",
-    IA64_BUILTIN_SUB_AND_FETCH_SI, 0, 0 },
-  { CODE_FOR_or_and_fetch_si, "__sync_or_and_fetch_si",
-    IA64_BUILTIN_OR_AND_FETCH_SI, 0, 0 },
-  { CODE_FOR_and_and_fetch_si, "__sync_and_and_fetch_si",
-    IA64_BUILTIN_AND_AND_FETCH_SI, 0, 0 },
-  { CODE_FOR_xor_and_fetch_si, "__sync_xor_and_fetch_si",
-    IA64_BUILTIN_XOR_AND_FETCH_SI, 0, 0 },
-  { CODE_FOR_nand_and_fetch_si, "__sync_nand_and_fetch_si",
-    IA64_BUILTIN_NAND_AND_FETCH_SI, 0, 0 }
-};
-
-/* All 64 bit intrinsics that take 2 arguments. */
-static struct builtin_description bdesc_2argdi[] =
-{
-  { CODE_FOR_fetch_and_add_di, "__sync_fetch_and_add_di",
-    IA64_BUILTIN_FETCH_AND_ADD_DI, 0, 0 },
-  { CODE_FOR_fetch_and_sub_di, "__sync_fetch_and_sub_di",
-    IA64_BUILTIN_FETCH_AND_SUB_DI, 0, 0 },
-  { CODE_FOR_fetch_and_or_di, "__sync_fetch_and_or_di",
-    IA64_BUILTIN_FETCH_AND_OR_DI, 0, 0 },
-  { CODE_FOR_fetch_and_and_di, "__sync_fetch_and_and_di",
-    IA64_BUILTIN_FETCH_AND_AND_DI, 0, 0 },
-  { CODE_FOR_fetch_and_xor_di, "__sync_fetch_and_xor_di",
-    IA64_BUILTIN_FETCH_AND_XOR_DI, 0, 0 },
-  { CODE_FOR_fetch_and_nand_di, "__sync_fetch_and_nand_di",
-    IA64_BUILTIN_FETCH_AND_NAND_DI, 0, 0 },
-  { CODE_FOR_add_and_fetch_di, "__sync_add_and_fetch_di",
-    IA64_BUILTIN_ADD_AND_FETCH_DI, 0, 0 },
-  { CODE_FOR_sub_and_fetch_di, "__sync_sub_and_fetch_di",
-    IA64_BUILTIN_SUB_AND_FETCH_DI, 0, 0 },
-  { CODE_FOR_or_and_fetch_di, "__sync_or_and_fetch_di",
-    IA64_BUILTIN_OR_AND_FETCH_DI, 0, 0 },
-  { CODE_FOR_and_and_fetch_di, "__sync_and_and_fetch_di",
-    IA64_BUILTIN_AND_AND_FETCH_DI, 0, 0 },
-  { CODE_FOR_xor_and_fetch_di, "__sync_xor_and_fetch_di",
-    IA64_BUILTIN_XOR_AND_FETCH_DI, 0, 0 },
-  { CODE_FOR_nand_and_fetch_di, "__sync_nand_and_fetch_di",
-    IA64_BUILTIN_NAND_AND_FETCH_DI, 0, 0 }
-};
-
+
 void
 ia64_init_builtins ()
 {
-  size_t i;
-
   tree psi_type_node = build_pointer_type (integer_type_node);
   tree pdi_type_node = build_pointer_type (long_integer_type_node);
   tree endlink = tree_cons (NULL_TREE, void_type_node, NULL_TREE);
@@ -4664,8 +4727,8 @@ ia64_init_builtins ()
                                       tree_cons (NULL_TREE,
 						 long_integer_type_node,
                                                  tree_cons (NULL_TREE,
-							    long_integer_type_node,
-                                                            endlink))));
+							long_integer_type_node,
+							endlink))));
   /* __sync_synchronize */
   tree void_ftype_void
     = build_function_type (void_type_node, endlink);
@@ -4693,15 +4756,15 @@ ia64_init_builtins ()
     = build_function_type (void_type_node, tree_cons (NULL_TREE, pdi_type_node,
 						      endlink));
 
+#define def_builtin(name, type, code) \
+  builtin_function ((name), (type), (code), BUILT_IN_MD, NULL_PTR)
+
   def_builtin ("__sync_val_compare_and_swap_si", si_ftype_psi_si_si,
 	       IA64_BUILTIN_VAL_COMPARE_AND_SWAP_SI);
-
   def_builtin ("__sync_val_compare_and_swap_di", di_ftype_pdi_di_di,
 	       IA64_BUILTIN_VAL_COMPARE_AND_SWAP_DI);
-
   def_builtin ("__sync_bool_compare_and_swap_si", si_ftype_psi_si_si,
 	       IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_SI);
-
   def_builtin ("__sync_bool_compare_and_swap_di", di_ftype_pdi_di_di,
 	       IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_DI);
 
@@ -4710,13 +4773,10 @@ ia64_init_builtins ()
 
   def_builtin ("__sync_lock_test_and_set_si", si_ftype_psi_si,
 	       IA64_BUILTIN_LOCK_TEST_AND_SET_SI);
-
   def_builtin ("__sync_lock_test_and_set_di", di_ftype_pdi_di,
 	       IA64_BUILTIN_LOCK_TEST_AND_SET_DI);
-
   def_builtin ("__sync_lock_release_si", void_ftype_psi,
 	       IA64_BUILTIN_LOCK_RELEASE_SI);
-
   def_builtin ("__sync_lock_release_di", void_ftype_pdi,
 	       IA64_BUILTIN_LOCK_RELEASE_DI);
 
@@ -4728,191 +4788,201 @@ ia64_init_builtins ()
 	       build_function_type (void_type_node, endlink), 
 	       IA64_BUILTIN_FLUSHRS);
 
-  /* Add all builtins that are operations on two args. */
-  for (i = 0; i < sizeof(bdesc_2argsi) / sizeof *bdesc_2argsi; i++)
-    def_builtin (bdesc_2argsi[i].name, si_ftype_psi_si, bdesc_2argsi[i].code);
-  for (i = 0; i < sizeof(bdesc_2argdi) / sizeof *bdesc_2argdi; i++)
-    def_builtin (bdesc_2argdi[i].name, si_ftype_psi_si, bdesc_2argdi[i].code);
+  def_builtin ("__sync_fetch_and_add_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_ADD_SI);
+  def_builtin ("__sync_fetch_and_sub_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_SUB_SI);
+  def_builtin ("__sync_fetch_and_or_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_OR_SI);
+  def_builtin ("__sync_fetch_and_and_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_AND_SI);
+  def_builtin ("__sync_fetch_and_xor_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_XOR_SI);
+  def_builtin ("__sync_fetch_and_nand_si", si_ftype_psi_si,
+	       IA64_BUILTIN_FETCH_AND_NAND_SI);
+
+  def_builtin ("__sync_add_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_ADD_AND_FETCH_SI);
+  def_builtin ("__sync_sub_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_SUB_AND_FETCH_SI);
+  def_builtin ("__sync_or_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_OR_AND_FETCH_SI);
+  def_builtin ("__sync_and_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_AND_AND_FETCH_SI);
+  def_builtin ("__sync_xor_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_XOR_AND_FETCH_SI);
+  def_builtin ("__sync_nand_and_fetch_si", si_ftype_psi_si,
+	       IA64_BUILTIN_NAND_AND_FETCH_SI);
+
+  def_builtin ("__sync_fetch_and_add_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_ADD_DI);
+  def_builtin ("__sync_fetch_and_sub_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_SUB_DI);
+  def_builtin ("__sync_fetch_and_or_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_OR_DI);
+  def_builtin ("__sync_fetch_and_and_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_AND_DI);
+  def_builtin ("__sync_fetch_and_xor_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_XOR_DI);
+  def_builtin ("__sync_fetch_and_nand_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_FETCH_AND_NAND_DI);
+
+  def_builtin ("__sync_add_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_ADD_AND_FETCH_DI);
+  def_builtin ("__sync_sub_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_SUB_AND_FETCH_DI);
+  def_builtin ("__sync_or_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_OR_AND_FETCH_DI);
+  def_builtin ("__sync_and_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_AND_AND_FETCH_DI);
+  def_builtin ("__sync_xor_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_XOR_AND_FETCH_DI);
+  def_builtin ("__sync_nand_and_fetch_di", di_ftype_pdi_di,
+	       IA64_BUILTIN_NAND_AND_FETCH_DI);
+
+#undef def_builtin
 }
 
 /* Expand fetch_and_op intrinsics.  The basic code sequence is:
 
      mf
-     ldsz return = [ptr];
-     tmp = return;
+     tmp = [ptr];
      do {
-       oldval = tmp;
+       ret = tmp;
        ar.ccv = tmp;
        tmp <op>= value;
        cmpxchgsz.acq tmp = [ptr], tmp
-       cmpxchgsz.acq tmp = [ptr], tmp
-     } while (tmp != oldval)
+     } while (tmp != ret)
 */
-void
-ia64_expand_fetch_and_op (code, mode, operands)
-     enum fetchop_code code;
+
+static rtx
+ia64_expand_fetch_and_op (binoptab, mode, arglist, target)
+     optab binoptab;
      enum machine_mode mode;
-     rtx operands[];
+     tree arglist;
+     rtx target;
 {
-  rtx mfreg = gen_rtx_MEM (BLKmode, gen_rtx_REG (mode, GR_REG (1)));
-  rtx oldval, newlabel, tmp_reg, ccv;
+  rtx ret, label, tmp, ccv, insn, mem, value;
+  tree arg0, arg1;
 
-  emit_insn (gen_mf (mfreg));
-  tmp_reg = gen_reg_rtx (mode);
-  oldval = gen_reg_rtx (mode);
-  ccv = gen_rtx_REG (mode, AR_CCV_REGNUM);
+  arg0 = TREE_VALUE (arglist);
+  arg1 = TREE_VALUE (TREE_CHAIN (arglist));
+  mem = expand_expr (arg0, NULL_RTX, Pmode, 0);
+  value = expand_expr (arg1, NULL_RTX, mode, 0);
 
-  emit_move_insn (operands[0], operands[1]);
-  emit_move_insn (tmp_reg, operands[0]);
+  mem = gen_rtx_MEM (mode, force_reg (Pmode, mem));
+  MEM_VOLATILE_P (mem) = 1;
 
-  newlabel = gen_label_rtx ();
-  emit_label (newlabel);
-  emit_move_insn (oldval, tmp_reg);
-  emit_move_insn (ccv, tmp_reg);
+  if (target && register_operand (target, mode))
+    ret = target;
+  else
+    ret = gen_reg_rtx (mode);
 
-  /* Perform the specific operation. */
-  switch (code)
-  {
-  case IA64_ADD_OP:
+  emit_insn (gen_mf ());
+
+  /* Special case for fetchadd instructions.  */
+  if (binoptab == add_optab && fetchadd_operand (value, VOIDmode))
     {
-      rtx reg;
-      if (GET_CODE (operands[2]) == CONST_INT)
-	reg = gen_reg_rtx (mode);
-      else
-        reg = operands[2];
       if (mode == SImode)
-	{
-	  if (reg != operands[2])
-	    emit_insn (gen_movsi (reg, operands[2]));
-	  emit_insn (gen_addsi3 (tmp_reg, tmp_reg, reg));
-	}
+        insn = gen_fetchadd_acq_si (ret, mem, value);
       else
-        {
-	  if (reg != operands[2])
-	    emit_insn (gen_movdi (reg, operands[2]));
-	  emit_insn (gen_adddi3 (tmp_reg, tmp_reg, reg));
-	}
-      break;
+        insn = gen_fetchadd_acq_di (ret, mem, value);
+      emit_insn (insn);
+      return ret;
     }
 
-  case IA64_SUB_OP:
-    if (mode == SImode)
-      emit_insn (gen_subsi3 (tmp_reg, tmp_reg, operands[2]));
-    else
-      emit_insn (gen_subdi3 (tmp_reg, tmp_reg, operands[2]));
-    break;
-
-  case IA64_OR_OP:
-    emit_insn (gen_iordi3 (tmp_reg, tmp_reg, operands[2]));
-    break;
-
-  case IA64_AND_OP:
-    emit_insn (gen_anddi3 (tmp_reg, tmp_reg, operands[2]));
-    break;
-
-  case IA64_XOR_OP:
-    emit_insn (gen_xordi3 (tmp_reg, tmp_reg, operands[2]));
-    break;
-
-  case IA64_NAND_OP:
-    emit_insn (gen_anddi3 (tmp_reg, tmp_reg, operands[2]));
-    if (mode == SImode)
-      emit_insn (gen_one_cmplsi2 (tmp_reg, operands[0]));
-    else
-      emit_insn (gen_one_cmpldi2 (tmp_reg, operands[0]));
-    break;
-
-  default:
-    break;
-  }
+  tmp = gen_reg_rtx (mode);
+  ccv = gen_rtx_REG (mode, AR_CCV_REGNUM);
+  emit_move_insn (tmp, mem);
+
+  label = gen_label_rtx ();
+  emit_label (label);
+  emit_move_insn (ret, tmp);
+  emit_move_insn (ccv, tmp);
+
+  /* Perform the specific operation.  Special case NAND by noticing
+     one_cmpl_optab instead.  */
+  if (binoptab == one_cmpl_optab)
+    {
+      tmp = expand_unop (mode, binoptab, tmp, NULL, OPTAB_WIDEN);
+      binoptab = and_optab;
+    }
+  tmp = expand_binop (mode, binoptab, tmp, value, tmp, 1, OPTAB_WIDEN);
 
   if (mode == SImode)
-    emit_insn (gen_cmpxchg_acq_si (tmp_reg, operands[1], tmp_reg, ccv));
+    insn = gen_cmpxchg_acq_si (tmp, mem, tmp, ccv);
   else
-    emit_insn (gen_cmpxchg_acq_di (tmp_reg, operands[1], tmp_reg, ccv));
+    insn = gen_cmpxchg_acq_di (tmp, mem, tmp, ccv);
+  emit_insn (insn);
+
+  emit_cmp_and_jump_insns (tmp, ret, NE, 0, mode, 1, 0, label);
 
-  emit_cmp_and_jump_insns (tmp_reg, oldval, NE, 0, mode, 1, 0, newlabel);
+  return ret;
 }
 
 /* Expand op_and_fetch intrinsics.  The basic code sequence is:
 
      mf
-     ldsz return = [ptr];
+     tmp = [ptr];
      do {
-       oldval = tmp;
+       old = tmp;
        ar.ccv = tmp;
-       return = tmp + value;
-       cmpxchgsz.acq tmp = [ptr], return
-     } while (tmp != oldval)
+       ret = tmp + value;
+       cmpxchgsz.acq tmp = [ptr], ret
+     } while (tmp != old)
 */
-void
-ia64_expand_op_and_fetch (code, mode, operands)
-     enum fetchop_code code;
+
+static rtx
+ia64_expand_op_and_fetch (binoptab, mode, arglist, target)
+     optab binoptab;
      enum machine_mode mode;
-     rtx operands[];
+     tree arglist;
+     rtx target;
 {
-  rtx mfreg = gen_rtx_MEM (BLKmode, gen_rtx_REG (mode, GR_REG (1)));
-  rtx oldval, newlabel, tmp_reg, ccv;
+  rtx old, label, tmp, ret, ccv, insn, mem, value;
+  tree arg0, arg1;
+
+  arg0 = TREE_VALUE (arglist);
+  arg1 = TREE_VALUE (TREE_CHAIN (arglist));
+  mem = expand_expr (arg0, NULL_RTX, Pmode, 0);
+  value = expand_expr (arg1, NULL_RTX, mode, 0);
 
-  emit_insn (gen_mf (mfreg));
-  tmp_reg = gen_reg_rtx (mode);
+  mem = gen_rtx_MEM (mode, force_reg (Pmode, mem));
+  MEM_VOLATILE_P (mem) = 1;
+
+  if (target && ! register_operand (target, mode))
+    target = NULL_RTX;
+
+  emit_insn (gen_mf ());
+  tmp = gen_reg_rtx (mode);
+  old = gen_reg_rtx (mode);
   ccv = gen_rtx_REG (mode, AR_CCV_REGNUM);
 
-  emit_move_insn (tmp_reg, operands[1]);  
+  emit_move_insn (tmp, mem);
 
-  newlabel = gen_label_rtx ();
-  emit_label (newlabel);
-  oldval = gen_reg_rtx (mode);
-  emit_move_insn (oldval, tmp_reg);
-  emit_move_insn (ccv, tmp_reg);
+  label = gen_label_rtx ();
+  emit_label (label);
+  emit_move_insn (old, tmp);
+  emit_move_insn (ccv, tmp);
 
-  /* Perform the specific operation. */
-  switch (code)
-  {
-  case IA64_ADD_OP:
-    if (mode == SImode)
-      emit_insn (gen_addsi3 (operands[0], tmp_reg, operands[2]));
-    else
-      emit_insn (gen_adddi3 (operands[0], tmp_reg, operands[2]));
-    break;
-
-  case IA64_SUB_OP:
-    if (mode == SImode)
-      emit_insn (gen_subsi3 (operands[0], tmp_reg, operands[2]));
-    else
-      emit_insn (gen_subdi3 (operands[0], tmp_reg, operands[2]));
-    break;
-
-  case IA64_OR_OP:
-    emit_insn (gen_iordi3 (operands[0], tmp_reg, operands[2]));
-    break;
-
-  case IA64_AND_OP:
-    emit_insn (gen_anddi3 (operands[0], tmp_reg, operands[2]));
-    break;
-
-  case IA64_XOR_OP:
-    emit_insn (gen_xordi3 (operands[0], tmp_reg, operands[2]));
-    break;
-
-  case IA64_NAND_OP:
-    emit_insn (gen_anddi3 (operands[0], tmp_reg, operands[2]));
-    if (mode == SImode)
-      emit_insn (gen_one_cmplsi2 (operands[0], operands[0]));
-    else
-      emit_insn (gen_one_cmpldi2 (operands[0], operands[0]));
-    break;
-
-  default:
-    break;
-  }
+  /* Perform the specific operation.  Special case NAND by noticing
+     one_cmpl_optab instead.  */
+  if (binoptab == one_cmpl_optab)
+    {
+      tmp = expand_unop (mode, binoptab, tmp, NULL, OPTAB_WIDEN);
+      binoptab = and_optab;
+    }
+  ret = expand_binop (mode, binoptab, tmp, value, target, 1, OPTAB_WIDEN);
 
   if (mode == SImode)
-    emit_insn (gen_cmpxchg_acq_si (tmp_reg, operands[1], operands[0], ccv));
+    insn = gen_cmpxchg_acq_si (tmp, mem, ret, ccv);
   else
-    emit_insn (gen_cmpxchg_acq_di (tmp_reg, operands[1], operands[0], ccv));
+    insn = gen_cmpxchg_acq_di (tmp, mem, ret, ccv);
+  emit_insn (insn);
+
+  emit_cmp_and_jump_insns (tmp, old, NE, 0, mode, 1, 0, label);
 
-  emit_cmp_and_jump_insns (tmp_reg, oldval, NE, 0, mode, 1, 0, newlabel);
+  return ret;
 }
 
 /* Expand val_ and bool_compare_and_swap.  For val_ we want:
@@ -4924,88 +4994,111 @@ ia64_expand_op_and_fetch (code, mode, operands)
 
    For bool_ it's the same except return ret == oldval.
 */
+
 static rtx
-ia64_expand_compare_and_swap (icode, arglist, target, boolcode)
-     enum insn_code icode;
+ia64_expand_compare_and_swap (mode, boolp, arglist, target)
+     enum machine_mode mode;
+     int boolp;
      tree arglist;
      rtx target;
-     int boolcode;
 {
   tree arg0, arg1, arg2;
-  rtx op0, op1, op2, pat;
-  enum machine_mode tmode, mode0, mode1, mode2;
+  rtx mem, old, new, ccv, tmp, insn;
 
   arg0 = TREE_VALUE (arglist);
   arg1 = TREE_VALUE (TREE_CHAIN (arglist));
   arg2 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (arglist)));
-  op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-  op1 = expand_expr (arg1, NULL_RTX, VOIDmode, 0);
-  op2 = expand_expr (arg2, NULL_RTX, VOIDmode, 0);
-  tmode = insn_data[icode].operand[0].mode;
-  mode0 = insn_data[icode].operand[1].mode;
-  mode1 = insn_data[icode].operand[2].mode;
-  mode2 = insn_data[icode].operand[3].mode;
-
-  op0 = gen_rtx_MEM (mode0, copy_to_mode_reg (Pmode, op0));
-  if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
-    op1 = copy_to_mode_reg (mode1, op1);
-  if (! (*insn_data[icode].operand[3].predicate) (op2, mode2))
-    op2 = copy_to_mode_reg (mode2, op2);
-  if (target == 0
-      || GET_MODE (target) != tmode
-      || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
-    target = gen_reg_rtx (tmode);
-
-  pat = GEN_FCN (icode) (target, op0, op1, op2);
-  if (! pat)
-    return 0;
-  emit_insn (pat);
-  if (boolcode)
+  mem = expand_expr (arg0, NULL_RTX, Pmode, 0);
+  old = expand_expr (arg1, NULL_RTX, mode, 0);
+  new = expand_expr (arg2, NULL_RTX, mode, 0);
+
+  mem = gen_rtx_MEM (mode, force_reg (Pmode, mem));
+  MEM_VOLATILE_P (mem) = 1;
+
+  if (! register_operand (old, mode))
+    old = copy_to_mode_reg (mode, old);
+  if (! register_operand (new, mode))
+    new = copy_to_mode_reg (mode, new);
+
+  if (! boolp && target && register_operand (target, mode))
+    tmp = target;
+  else
+    tmp = gen_reg_rtx (mode);
+
+  ccv = gen_rtx_REG (mode, AR_CCV_REGNUM);
+  emit_move_insn (ccv, old);
+  emit_insn (gen_mf ());
+  if (mode == SImode)
+    insn = gen_cmpxchg_acq_si (tmp, mem, new, ccv);
+  else
+    insn = gen_cmpxchg_acq_di (tmp, mem, new, ccv);
+  emit_insn (insn);
+
+  if (boolp)
     {
-      if (tmode == SImode)
-        {
-          emit_insn (gen_cmpsi (target, op1));
-          emit_insn (gen_seq (gen_lowpart (DImode, target)));
-        }
-      else
-        {
-          emit_insn (gen_cmpdi (target, op1));
-          emit_insn (gen_seq (target));
-        }
+      if (! target)
+	target = gen_reg_rtx (mode);
+      return emit_store_flag_force (target, EQ, tmp, old, mode, 1, 1);
     }
-  return target;
+  else
+    return tmp;
 }
 
-/* Expand all intrinsics that take 2 arguments. */
+/* Expand lock_test_and_set.  I.e. `xchgsz ret = [ptr], new'.  */
+
 static rtx
-ia64_expand_binop_builtin (icode, arglist, target)
-     enum insn_code icode;
+ia64_expand_lock_test_and_set (mode, arglist, target)
+     enum machine_mode mode;
      tree arglist;
      rtx target;
 {
-  rtx pat;
-  tree arg0 = TREE_VALUE (arglist);
-  tree arg1 = TREE_VALUE (TREE_CHAIN (arglist));
-  rtx op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-  rtx op1 = expand_expr (arg1, NULL_RTX, VOIDmode, 0);
-  enum machine_mode tmode = insn_data[icode].operand[0].mode;
-  enum machine_mode mode0 = insn_data[icode].operand[1].mode;
-  enum machine_mode mode1 = insn_data[icode].operand[2].mode;
-
-  if (! target
-      || GET_MODE (target) != tmode
-      || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
-    target = gen_reg_rtx (tmode);
-
-  op0 = gen_rtx_MEM (mode0, copy_to_mode_reg (Pmode, op0));
-  if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
-    op1 = copy_to_mode_reg (mode1, op1);
-
-  pat = GEN_FCN (icode) (target, op0, op1);
-  if (! pat)
-    return 0;
-  emit_insn (pat);
-  return target;
+  tree arg0, arg1;
+  rtx mem, new, ret, insn;
+
+  arg0 = TREE_VALUE (arglist);
+  arg1 = TREE_VALUE (TREE_CHAIN (arglist));
+  mem = expand_expr (arg0, NULL_RTX, Pmode, 0);
+  new = expand_expr (arg1, NULL_RTX, mode, 0);
+
+  mem = gen_rtx_MEM (mode, force_reg (Pmode, mem));
+  MEM_VOLATILE_P (mem) = 1;
+  if (! register_operand (new, mode))
+    new = copy_to_mode_reg (mode, new);
+
+  if (target && register_operand (target, mode))
+    ret = target;
+  else
+    ret = gen_reg_rtx (mode);
+
+  if (mode == SImode)
+    insn = gen_xchgsi (ret, mem, new);
+  else
+    insn = gen_xchgdi (ret, mem, new);
+  emit_insn (insn);
+
+  return ret;
+}
+
+/* Expand lock_release.  I.e. `stsz.rel [ptr] = r0'.  */
+
+static rtx
+ia64_expand_lock_release (mode, arglist, target)
+     enum machine_mode mode;
+     tree arglist;
+     rtx target ATTRIBUTE_UNUSED;
+{
+  tree arg0;
+  rtx mem;
+
+  arg0 = TREE_VALUE (arglist);
+  mem = expand_expr (arg0, NULL_RTX, Pmode, 0);
+
+  mem = gen_rtx_MEM (mode, force_reg (Pmode, mem));
+  MEM_VOLATILE_P (mem) = 1;
+
+  emit_move_insn (mem, const0_rtx);
+
+  return const0_rtx;
 }
 
 rtx
@@ -5016,126 +5109,137 @@ ia64_expand_builtin (exp, target, subtarget, mode, ignore)
      enum machine_mode mode ATTRIBUTE_UNUSED;
      int ignore ATTRIBUTE_UNUSED;
 {
-  rtx op0, op1, pat;
-  rtx tmp_reg;
-  tree arg0, arg1;
-  tree arglist = TREE_OPERAND (exp, 1);
   tree fndecl = TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
   unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
-  enum machine_mode tmode, mode0, mode1;
-  enum insn_code icode;
-  size_t i;
-  struct builtin_description *d;
+  tree arglist = TREE_OPERAND (exp, 1);
 
   switch (fcode)
     {
     case IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_SI:
-      return ia64_expand_compare_and_swap (CODE_FOR_val_compare_and_swap_si,
-					   arglist, target, 1);
-
     case IA64_BUILTIN_VAL_COMPARE_AND_SWAP_SI:
-      return ia64_expand_compare_and_swap (CODE_FOR_val_compare_and_swap_si,
-					   arglist, target, 0);
+    case IA64_BUILTIN_LOCK_TEST_AND_SET_SI:
+    case IA64_BUILTIN_LOCK_RELEASE_SI:
+    case IA64_BUILTIN_FETCH_AND_ADD_SI:
+    case IA64_BUILTIN_FETCH_AND_SUB_SI:
+    case IA64_BUILTIN_FETCH_AND_OR_SI:
+    case IA64_BUILTIN_FETCH_AND_AND_SI:
+    case IA64_BUILTIN_FETCH_AND_XOR_SI:
+    case IA64_BUILTIN_FETCH_AND_NAND_SI:
+    case IA64_BUILTIN_ADD_AND_FETCH_SI:
+    case IA64_BUILTIN_SUB_AND_FETCH_SI:
+    case IA64_BUILTIN_OR_AND_FETCH_SI:
+    case IA64_BUILTIN_AND_AND_FETCH_SI:
+    case IA64_BUILTIN_XOR_AND_FETCH_SI:
+    case IA64_BUILTIN_NAND_AND_FETCH_SI:
+      mode = SImode;
+      break;
 
     case IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_DI:
-      return ia64_expand_compare_and_swap (CODE_FOR_val_compare_and_swap_di,
-					   arglist, target, 1);
+    case IA64_BUILTIN_VAL_COMPARE_AND_SWAP_DI:
+    case IA64_BUILTIN_LOCK_TEST_AND_SET_DI:
+    case IA64_BUILTIN_LOCK_RELEASE_DI:
+    case IA64_BUILTIN_FETCH_AND_ADD_DI:
+    case IA64_BUILTIN_FETCH_AND_SUB_DI:
+    case IA64_BUILTIN_FETCH_AND_OR_DI:
+    case IA64_BUILTIN_FETCH_AND_AND_DI:
+    case IA64_BUILTIN_FETCH_AND_XOR_DI:
+    case IA64_BUILTIN_FETCH_AND_NAND_DI:
+    case IA64_BUILTIN_ADD_AND_FETCH_DI:
+    case IA64_BUILTIN_SUB_AND_FETCH_DI:
+    case IA64_BUILTIN_OR_AND_FETCH_DI:
+    case IA64_BUILTIN_AND_AND_FETCH_DI:
+    case IA64_BUILTIN_XOR_AND_FETCH_DI:
+    case IA64_BUILTIN_NAND_AND_FETCH_DI:
+      mode = DImode;
+      break;
 
+    default:
+      break;
+    }
+
+  switch (fcode)
+    {
+    case IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_SI:
+    case IA64_BUILTIN_BOOL_COMPARE_AND_SWAP_DI:
+      return ia64_expand_compare_and_swap (mode, 1, arglist, target);
+
+    case IA64_BUILTIN_VAL_COMPARE_AND_SWAP_SI:
     case IA64_BUILTIN_VAL_COMPARE_AND_SWAP_DI:
-      return ia64_expand_compare_and_swap (CODE_FOR_val_compare_and_swap_di,
-					   arglist, target, 0);
+      return ia64_expand_compare_and_swap (mode, 0, arglist, target);
 
     case IA64_BUILTIN_SYNCHRONIZE:
-      /* Pass a volatile memory operand. */
-      tmp_reg = gen_rtx_REG (DImode, GR_REG(0));
-      target = gen_rtx_MEM (BLKmode, tmp_reg);
-      emit_insn (gen_mf (target));
+      emit_insn (gen_mf ());
       return const0_rtx;
 
     case IA64_BUILTIN_LOCK_TEST_AND_SET_SI:
-      icode = CODE_FOR_lock_test_and_set_si;
-      arg0 = TREE_VALUE (arglist);
-      arg1 = TREE_VALUE (TREE_CHAIN (arglist));
-      op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-      op1 = expand_expr (arg1, NULL_RTX, VOIDmode, 0);
-      tmode = insn_data[icode].operand[0].mode;
-      mode0 = insn_data[icode].operand[1].mode;
-      mode1 = insn_data[icode].operand[2].mode;
-      op0 = gen_rtx_MEM (mode0, copy_to_mode_reg (Pmode, op0));
-      if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
-        op1 = copy_to_mode_reg (mode1, op1);
-      if (target == 0
-          || GET_MODE (target) != tmode
-          || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
-        target = gen_reg_rtx (tmode);
-      pat = GEN_FCN (icode) (target, op0, op1);
-      if (! pat)
-        return 0;
-      emit_insn (pat);
-      return target;
-
     case IA64_BUILTIN_LOCK_TEST_AND_SET_DI:
-      icode = CODE_FOR_lock_test_and_set_di;
-      arg0 = TREE_VALUE (arglist);
-      arg1 = TREE_VALUE (TREE_CHAIN (arglist));
-      op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-      op1 = expand_expr (arg1, NULL_RTX, VOIDmode, 0);
-      tmode = insn_data[icode].operand[0].mode;
-      mode0 = insn_data[icode].operand[1].mode;
-      mode1 = insn_data[icode].operand[2].mode;
-      op0 = gen_rtx_MEM (mode0, copy_to_mode_reg (Pmode, op0));
-      if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
-        op1 = copy_to_mode_reg (mode1, op1);
-      if (target == 0
-          || GET_MODE (target) != tmode
-          || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
-        target = gen_reg_rtx (tmode);
-      pat = GEN_FCN (icode) (target, op0, op1);
-      if (! pat)
-        return 0;
-      emit_insn (pat);
-      return target;
+      return ia64_expand_lock_test_and_set (mode, arglist, target);
 
     case IA64_BUILTIN_LOCK_RELEASE_SI:
-      arg0 = TREE_VALUE (arglist);
-      op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-      op0 = gen_rtx_MEM (SImode, copy_to_mode_reg (Pmode, op0));
-      MEM_VOLATILE_P (op0) = 1;
-      emit_insn (gen_movsi (op0, GEN_INT(0)));
-      return const0_rtx;
-
     case IA64_BUILTIN_LOCK_RELEASE_DI:
-      arg0 = TREE_VALUE (arglist);
-      op0 = expand_expr (arg0, NULL_RTX, VOIDmode, 0);
-      op0 = gen_rtx_MEM (DImode, copy_to_mode_reg (Pmode, op0));
-      MEM_VOLATILE_P (op0) = 1;
-      emit_insn (gen_movdi (op0, GEN_INT(0)));
-      return const0_rtx;
+      return ia64_expand_lock_release (mode, arglist, target);
 
     case IA64_BUILTIN_BSP:
-      {
-	rtx reg = gen_reg_rtx (DImode);
-	emit_insn (gen_bsp_value (reg));
-	return reg;
-      }
+      if (! target || ! register_operand (target, DImode))
+	target = gen_reg_rtx (DImode);
+      emit_insn (gen_bsp_value (target));
+      return target;
 
     case IA64_BUILTIN_FLUSHRS:
       emit_insn (gen_flushrs ());
       return const0_rtx;
 
+    case IA64_BUILTIN_FETCH_AND_ADD_SI:
+    case IA64_BUILTIN_FETCH_AND_ADD_DI:
+      return ia64_expand_fetch_and_op (add_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_FETCH_AND_SUB_SI:
+    case IA64_BUILTIN_FETCH_AND_SUB_DI:
+      return ia64_expand_fetch_and_op (sub_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_FETCH_AND_OR_SI:
+    case IA64_BUILTIN_FETCH_AND_OR_DI:
+      return ia64_expand_fetch_and_op (ior_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_FETCH_AND_AND_SI:
+    case IA64_BUILTIN_FETCH_AND_AND_DI:
+      return ia64_expand_fetch_and_op (and_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_FETCH_AND_XOR_SI:
+    case IA64_BUILTIN_FETCH_AND_XOR_DI:
+      return ia64_expand_fetch_and_op (xor_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_FETCH_AND_NAND_SI:
+    case IA64_BUILTIN_FETCH_AND_NAND_DI:
+      return ia64_expand_fetch_and_op (one_cmpl_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_ADD_AND_FETCH_SI:
+    case IA64_BUILTIN_ADD_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (add_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_SUB_AND_FETCH_SI:
+    case IA64_BUILTIN_SUB_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (sub_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_OR_AND_FETCH_SI:
+    case IA64_BUILTIN_OR_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (ior_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_AND_AND_FETCH_SI:
+    case IA64_BUILTIN_AND_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (and_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_XOR_AND_FETCH_SI:
+    case IA64_BUILTIN_XOR_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (xor_optab, mode, arglist, target);
+
+    case IA64_BUILTIN_NAND_AND_FETCH_SI:
+    case IA64_BUILTIN_NAND_AND_FETCH_DI:
+      return ia64_expand_op_and_fetch (one_cmpl_optab, mode, arglist, target);
+
     default:
       break;
     }
 
-  /* Expand all 32 bit intrinsics that take 2 arguments. */
-  for (i=0, d = bdesc_2argsi; i < sizeof (bdesc_2argsi) / sizeof *d; i++, d++)
-    if (d->code == fcode)
-      return ia64_expand_binop_builtin (d->icode, arglist, target);
-
-  /* Expand all 64 bit intrinsics that take 2 arguments. */
-  for (i=0, d = bdesc_2argdi; i < sizeof (bdesc_2argdi) / sizeof *d; i++, d++)
-    if (d->code == fcode)
-      return ia64_expand_binop_builtin (d->icode, arglist, target);
-
-  return 0;
+  return NULL_RTX;
 }
diff --git a/gcc/config/ia64/ia64.h b/gcc/config/ia64/ia64.h
index 629f847..2410ecd 100644
--- a/gcc/config/ia64/ia64.h
+++ b/gcc/config/ia64/ia64.h
@@ -1114,10 +1114,16 @@ enum reg_class
 /* 1..4 for shladd arguments.  */
 #define CONSTRAINT_OK_FOR_R(VALUE) \
   (GET_CODE (VALUE) == CONST_INT && INTVAL (VALUE) >= 1 && INTVAL (VALUE) <= 4)
+/* Non-post-inc memory for asms and other unsavory creatures.  */
+#define CONSTRAINT_OK_FOR_S(VALUE)				\
+  (GET_CODE (VALUE) == MEM					\
+   && GET_RTX_CLASS (GET_CODE (XEXP ((VALUE), 0))) != 'a'	\
+   && (reload_in_progress || memory_operand ((VALUE), VOIDmode)))
 
 #define EXTRA_CONSTRAINT(VALUE, C) \
   ((C) == 'Q' ? CONSTRAINT_OK_FOR_Q (VALUE)	\
    : (C) == 'R' ? CONSTRAINT_OK_FOR_R (VALUE)	\
+   : (C) == 'S' ? CONSTRAINT_OK_FOR_S (VALUE)	\
    : 0)
 
 /* Basic Stack Layout */
@@ -2642,24 +2648,31 @@ do {									\
 { "function_operand", {SYMBOL_REF}},					\
 { "setjmp_operand", {SYMBOL_REF}},					\
 { "destination_operand", {SUBREG, REG, MEM}},				\
+{ "not_postinc_memory_operand", {MEM}},					\
 { "move_operand", {SUBREG, REG, MEM, CONST_INT, CONST_DOUBLE,		\
 		     CONSTANT_P_RTX, SYMBOL_REF, CONST, LABEL_REF}},	\
-{ "reg_or_0_operand", {SUBREG, REG, CONST_INT}},			\
-{ "reg_or_5bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
-{ "reg_or_6bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
-{ "reg_or_8bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
-{ "reg_or_8bit_adjusted_operand", {SUBREG, REG, CONST_INT,		\
+{ "gr_register_operand", {SUBREG, REG}},				\
+{ "fr_register_operand", {SUBREG, REG}},				\
+{ "grfr_register_operand", {SUBREG, REG}},				\
+{ "gr_nonimmediate_operand", {SUBREG, REG, MEM}},			\
+{ "grfr_nonimmediate_operand", {SUBREG, REG, MEM}},			\
+{ "gr_reg_or_0_operand", {SUBREG, REG, CONST_INT}},			\
+{ "gr_reg_or_5bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
+{ "gr_reg_or_6bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
+{ "gr_reg_or_8bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
+{ "grfr_reg_or_8bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}}, \
+{ "gr_reg_or_8bit_adjusted_operand", {SUBREG, REG, CONST_INT,		\
 				     CONSTANT_P_RTX}},			\
-{ "reg_or_8bit_and_adjusted_operand", {SUBREG, REG, CONST_INT,		\
+{ "gr_reg_or_8bit_and_adjusted_operand", {SUBREG, REG, CONST_INT,	\
 					 CONSTANT_P_RTX}},		\
-{ "reg_or_14bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
-{ "reg_or_22bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
+{ "gr_reg_or_14bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}}, \
+{ "gr_reg_or_22bit_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}}, \
 { "shift_count_operand", {SUBREG, REG, CONST_INT, CONSTANT_P_RTX}},	\
 { "shift_32bit_count_operand", {SUBREG, REG, CONST_INT,			\
 				  CONSTANT_P_RTX}},			\
 { "shladd_operand", {CONST_INT}},					\
 { "fetchadd_operand", {CONST_INT}},					\
-{ "reg_or_fp01_operand", {SUBREG, REG, CONST_DOUBLE}},			\
+{ "fr_reg_or_fp01_operand", {SUBREG, REG, CONST_DOUBLE}},		\
 { "normal_comparison_operator", {EQ, NE, GT, LE, GTU, LEU}},		\
 { "adjusted_comparison_operator", {LT, GE, LTU, GEU}},			\
 { "call_multiple_values_operation", {PARALLEL}},			\
diff --git a/gcc/config/ia64/ia64.md b/gcc/config/ia64/ia64.md
index 5057ba0..b8cd92c 100644
--- a/gcc/config/ia64/ia64.md
+++ b/gcc/config/ia64/ia64.md
@@ -931,22 +931,22 @@
 ;; Signed conversions from a smaller integer to a larger integer
 
 (define_insn "extendqidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(sign_extend:DI (match_operand:QI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(sign_extend:DI (match_operand:QI 1 "gr_register_operand" "r")))]
   ""
   "sxt1 %0 = %1"
   [(set_attr "type" "I")])
 
 (define_insn "extendhidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(sign_extend:DI (match_operand:HI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(sign_extend:DI (match_operand:HI 1 "gr_register_operand" "r")))]
   ""
   "sxt2 %0 = %1"
   [(set_attr "type" "I")])
 
 (define_insn "extendsidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-	(sign_extend:DI (match_operand:SI 1 "register_operand" "r,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+	(sign_extend:DI (match_operand:SI 1 "grfr_register_operand" "r,*f")))]
   ""
   "@
    sxt4 %0 = %1
@@ -956,8 +956,8 @@
 ;; Unsigned conversions from a smaller integer to a larger integer
 
 (define_insn "zero_extendqidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
-	(zero_extend:DI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+	(zero_extend:DI (match_operand:QI 1 "gr_nonimmediate_operand" "r,m")))]
   ""
   "@
    zxt1 %0 = %1
@@ -965,8 +965,8 @@
   [(set_attr "type" "I,M")])
 
 (define_insn "zero_extendhidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
-	(zero_extend:DI (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+	(zero_extend:DI (match_operand:HI 1 "gr_nonimmediate_operand" "r,m")))]
   ""
   "@
    zxt2 %0 = %1
@@ -974,8 +974,9 @@
   [(set_attr "type" "I,M")])
 
 (define_insn "zero_extendsidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r,*f")
-	(zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,r,*f")
+	(zero_extend:DI
+	  (match_operand:SI 1 "grfr_nonimmediate_operand" "r,m,*f")))]
   ""
   "@
    zxt4 %0 = %1
@@ -991,8 +992,8 @@
 ;; would let combine merge the thing into adjacent insns.
 
 (define_insn_and_split "extendsfdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-	(float_extend:DF (match_operand:SF 1 "register_operand" "0,f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f,f")
+	(float_extend:DF (match_operand:SF 1 "fr_register_operand" "0,f")))]
   ""
   "mov %0 = %1"
   "reload_completed"
@@ -1001,8 +1002,8 @@
   [(set_attr "type" "F")])
 
 (define_insn_and_split "extendsftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-	(float_extend:TF (match_operand:SF 1 "register_operand" "0,f")))]
+  [(set (match_operand:TF 0 "fr_register_operand" "=f,f")
+	(float_extend:TF (match_operand:SF 1 "fr_register_operand" "0,f")))]
   ""
   "mov %0 = %1"
   "reload_completed"
@@ -1011,8 +1012,8 @@
   [(set_attr "type" "F")])
 
 (define_insn_and_split "extenddftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-	(float_extend:TF (match_operand:DF 1 "register_operand" "0,f")))]
+  [(set (match_operand:TF 0 "fr_register_operand" "=f,f")
+	(float_extend:TF (match_operand:DF 1 "fr_register_operand" "0,f")))]
   ""
   "mov %0 = %1"
   "reload_completed"
@@ -1021,22 +1022,22 @@
   [(set_attr "type" "F")])
 
 (define_insn "truncdfsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(float_truncate:SF (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(float_truncate:SF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
   "fnorm.s %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "trunctfsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(float_truncate:SF (match_operand:TF 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(float_truncate:SF (match_operand:TF 1 "fr_register_operand" "f")))]
   ""
   "fnorm.s %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "trunctfdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(float_truncate:DF (match_operand:TF 1 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(float_truncate:DF (match_operand:TF 1 "fr_register_operand" "f")))]
   ""
   "fnorm.d %0 = %1%B0"
   [(set_attr "type" "F")])
@@ -1044,29 +1045,29 @@
 ;; Convert between signed integer types and floating point.
 
 (define_insn "floatditf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-	(float:TF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
+	(float:TF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
   "fcvt.xf %0 = %1"
   [(set_attr "type" "F")])
 
 (define_insn "fix_truncsfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(fix:DI (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fx.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "fix_truncdfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(fix:DI (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fx.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "fix_trunctfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(fix:DI (match_operand:TF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(fix:DI (match_operand:TF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fx.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
@@ -1074,43 +1075,43 @@
 ;; Convert between unsigned integer types and floating point.
 
 (define_insn "floatunsdisf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(unsigned_float:SF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(unsigned_float:SF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
   "fcvt.xuf.s %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "floatunsdidf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(unsigned_float:DF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(unsigned_float:DF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
   "fcvt.xuf.d %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "floatunsditf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-	(unsigned_float:TF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
+	(unsigned_float:TF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
   "fcvt.xuf %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "fixuns_truncsfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(unsigned_fix:DI (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(unsigned_fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fxu.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "fixuns_truncdfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(unsigned_fix:DI (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(unsigned_fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fxu.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "fixuns_trunctfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(unsigned_fix:DI (match_operand:TF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(unsigned_fix:DI (match_operand:TF 1 "fr_register_operand" "f")))]
   ""
   "fcvt.fxu.trunc %0 = %1%B0"
   [(set_attr "type" "F")])
@@ -1122,8 +1123,8 @@
 ;; ::::::::::::::::::::
 
 (define_insn "extv"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(sign_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(sign_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
 			 (match_operand:DI 2 "const_int_operand" "n")
 			 (match_operand:DI 3 "const_int_operand" "n")))]
   ""
@@ -1131,8 +1132,8 @@
   [(set_attr "type" "I")])
 
 (define_insn "extzv"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
 			 (match_operand:DI 2 "const_int_operand" "n")
 			 (match_operand:DI 3 "const_int_operand" "n")))]
   ""
@@ -1149,7 +1150,7 @@
 ;; where source2 != dest.
 
 (define_expand "insv"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "")
 			 (match_operand:DI 1 "const_int_operand" "")
 			 (match_operand:DI 2 "const_int_operand" ""))
 	(match_operand:DI 3 "nonmemory_operand" ""))]
@@ -1225,19 +1226,19 @@
 }")
 
 (define_insn "*insv_internal"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
 			 (match_operand:DI 1 "const_int_operand" "n")
 			 (match_operand:DI 2 "const_int_operand" "n"))
 	(match_operand:DI 3 "nonmemory_operand" "rP"))]
-  "(register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
+  "(gr_register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
    || operands[3] == const0_rtx || operands[3] == constm1_rtx"
   "dep %0 = %3, %0, %2, %1"
   [(set_attr "type" "I")])
 
 ;; Combine doesn't like to create bitfield insertions into zero.
 (define_insn "*depz_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(and:DI (ashift:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
 			   (match_operand:DI 2 "const_int_operand" "n"))
 		(match_operand:DI 3 "const_int_operand" "n")))]
   "CONST_OK_FOR_M (INTVAL (operands[2]))
@@ -1250,10 +1251,10 @@
   [(set_attr "type" "I")])
 
 (define_insn "shift_mix4left"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
 			 (const_int 32) (const_int 0))
-	(match_operand:DI 1 "register_operand" "r"))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
+	(match_operand:DI 1 "gr_register_operand" "r"))
+   (clobber (match_operand:DI 2 "gr_register_operand" "=r"))]
   ""
   "#"
   [(set_attr "type" "unknown")])
@@ -1282,18 +1283,18 @@
   "operands[3] = operands[2];")
 
 (define_insn "*mix4left"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
 			 (const_int 32) (const_int 0))
-	(lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
+	(lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
 		     (const_int 32)))]
   ""
   "mix4.l %0 = %0, %r1"
   [(set_attr "type" "I")])
 
 (define_insn "mix4right"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
 			 (const_int 32) (const_int 32))
-	(match_operand:DI 1 "reg_or_0_operand" "rO"))]
+	(match_operand:DI 1 "gr_reg_or_0_operand" "rO"))]
   ""
   "mix4.r %0 = %r1, %0"
   [(set_attr "type" "I")])
@@ -1301,9 +1302,10 @@
 ;; This is used by the rotrsi3 pattern.
 
 (define_insn "*mix4right_3op"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "r"))
-		(ashift:DI (zero_extend:DI (match_operand:SI 2 "register_operand" "r"))
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(ior:DI (zero_extend:DI (match_operand:SI 1 "gr_register_operand" "r"))
+		(ashift:DI (zero_extend:DI
+			     (match_operand:SI 2 "gr_register_operand" "r"))
 			   (const_int 32))))]
   ""
   "mix4.r %0 = %2, %1"
@@ -1317,9 +1319,9 @@
 ;; ::::::::::::::::::::
 
 (define_insn "addsi3"
-  [(set (match_operand:SI 0 "register_operand" "=r,r,r")
-	(plus:SI (match_operand:SI 1 "register_operand" "%r,r,a")
-		 (match_operand:SI 2 "reg_or_22bit_operand" "r,I,J")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
+	(plus:SI (match_operand:SI 1 "gr_register_operand" "%r,r,a")
+		 (match_operand:SI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
   ""
   "@
   add %0 = %1, %2
@@ -1328,17 +1330,17 @@
   [(set_attr "type" "A")])
 
 (define_insn "*addsi3_plus1"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(plus:SI (plus:SI (match_operand:SI 1 "register_operand" "r")
-			  (match_operand:SI 2 "register_operand" "r"))
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(plus:SI (plus:SI (match_operand:SI 1 "gr_register_operand" "r")
+			  (match_operand:SI 2 "gr_register_operand" "r"))
 		 (const_int 1)))]
   ""
   "add %0 = %1, %2, 1"
   [(set_attr "type" "A")])
 
 (define_insn "*addsi3_plus1_alt"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
 			  (const_int 2))
 		 (const_int 1)))]
   ""
@@ -1346,26 +1348,26 @@
   [(set_attr "type" "A")])
 
 (define_insn "*addsi3_shladd"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
 			  (match_operand:SI 2 "shladd_operand" "n"))
-		 (match_operand:SI 3 "register_operand" "r")))]
+		 (match_operand:SI 3 "gr_register_operand" "r")))]
   ""
   "shladd %0 = %1, %S2, %3"
   [(set_attr "type" "A")])
 
 (define_insn "subsi3"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(minus:SI (match_operand:SI 1 "reg_or_8bit_operand" "rK")
-		  (match_operand:SI 2 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(minus:SI (match_operand:SI 1 "gr_reg_or_8bit_operand" "rK")
+		  (match_operand:SI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %1, %2"
   [(set_attr "type" "A")])
 
 (define_insn "*subsi3_minus1"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(plus:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
-		 (match_operand:SI 2 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(plus:SI (not:SI (match_operand:SI 1 "gr_register_operand" "r"))
+		 (match_operand:SI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %2, %1, 1"
   [(set_attr "type" "A")])
@@ -1373,24 +1375,24 @@
 ;; ??? Could add maddsi3 patterns patterned after the madddi3 patterns.
 
 (define_insn "mulsi3"
-  [(set (match_operand:SI 0 "register_operand" "=f")
-	(mult:SI (match_operand:SI 1 "register_operand" "f")
+  [(set (match_operand:SI 0 "fr_register_operand" "=f")
+	(mult:SI (match_operand:SI 1 "fr_register_operand" "f")
 		 (match_operand:SI 2 "nonmemory_operand" "f")))]
   ""
   "xma.l %0 = %1, %2, f0%B0"
   [(set_attr "type" "F")])
 
 (define_insn "negsi2"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(neg:SI (match_operand:SI 1 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(neg:SI (match_operand:SI 1 "gr_register_operand" "r")))]
   ""
   "sub %0 = r0, %1"
   [(set_attr "type" "A")])
 
 (define_expand "abssi2"
   [(set (match_dup 2)
-	(ge:CC (match_operand:SI 1 "register_operand" "") (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "")
+	(ge:CC (match_operand:SI 1 "gr_register_operand" "") (const_int 0)))
+   (set (match_operand:SI 0 "gr_register_operand" "")
 	(if_then_else:SI (eq:CC (match_dup 2) (const_int 0))
 			 (neg:SI (match_dup 1))
 			 (match_dup 1)))]
@@ -1402,9 +1404,9 @@
 
 (define_expand "sminsi3"
   [(set (match_dup 3)
-	(ge:CC (match_operand:SI 1 "register_operand" "")
-	       (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
+	(ge:CC (match_operand:SI 1 "gr_register_operand" "")
+	       (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
 	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 2) (match_dup 1)))]
   ""
@@ -1415,9 +1417,9 @@
 
 (define_expand "smaxsi3"
   [(set (match_dup 3)
-	(ge:CC (match_operand:SI 1 "register_operand" "")
-	       (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
+	(ge:CC (match_operand:SI 1 "gr_register_operand" "")
+	       (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
 	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 1) (match_dup 2)))]
   ""
@@ -1428,9 +1430,9 @@
 
 (define_expand "uminsi3"
   [(set (match_dup 3)
-	(geu:CC (match_operand:SI 1 "register_operand" "")
-		(match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
+	(geu:CC (match_operand:SI 1 "gr_register_operand" "")
+		(match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
 	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 2) (match_dup 1)))]
   ""
@@ -1441,9 +1443,9 @@
 
 (define_expand "umaxsi3"
   [(set (match_dup 3)
-	(geu:CC (match_operand:SI 1 "register_operand" "")
-		(match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
+	(geu:CC (match_operand:SI 1 "gr_register_operand" "")
+		(match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
 	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 1) (match_dup 2)))]
   ""
@@ -1460,9 +1462,9 @@
 ;; ::::::::::::::::::::
 
 (define_insn "adddi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
-	(plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
-		 (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r,r")
+	(plus:DI (match_operand:DI 1 "gr_register_operand" "%r,r,a")
+		 (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
   ""
   "@
   add %0 = %1, %2
@@ -1471,9 +1473,9 @@
   [(set_attr "type" "A")])
 
 (define_insn "*adddi3_plus1"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(plus:DI (plus:DI (match_operand:DI 1 "register_operand" "r")
-			  (match_operand:DI 2 "register_operand" "r"))
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(plus:DI (plus:DI (match_operand:DI 1 "gr_register_operand" "r")
+			  (match_operand:DI 2 "gr_register_operand" "r"))
 		 (const_int 1)))]
   ""
   "add %0 = %1, %2, 1"
@@ -1483,8 +1485,8 @@
 ;; eliminator hack handle it, which results in the 1 being forced into
 ;; a register, but not more ugliness here.
 (define_insn "*adddi3_plus1_alt"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
 			  (const_int 2))
 		 (const_int 1)))]
   ""
@@ -1492,25 +1494,25 @@
   [(set_attr "type" "A")])
 
 (define_insn "subdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(minus:DI (match_operand:DI 1 "reg_or_8bit_operand" "rK")
-		  (match_operand:DI 2 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(minus:DI (match_operand:DI 1 "gr_reg_or_8bit_operand" "rK")
+		  (match_operand:DI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %1, %2"
   [(set_attr "type" "A")])
 
 (define_insn "*subdi3_minus1"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(plus:DI (not:DI (match_operand:DI 1 "register_operand" "r"))
-		 (match_operand:DI 2 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(plus:DI (not:DI (match_operand:DI 1 "gr_register_operand" "r"))
+		 (match_operand:DI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %2, %1, 1"
   [(set_attr "type" "A")])
 
 (define_insn "muldi3"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(mult:DI (match_operand:DI 1 "register_operand" "f")
-		 (match_operand:DI 2 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(mult:DI (match_operand:DI 1 "fr_register_operand" "f")
+		 (match_operand:DI 2 "fr_register_operand" "f")))]
   ""
   "xma.l %0 = %1, %2, f0%B0"
   [(set_attr "type" "F")])
@@ -1527,10 +1529,10 @@
 ;; ??? Maybe we should change how adds are canonicalized.
 
 (define_insn "*madddi3"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-	(plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
-			  (match_operand:DI 2 "register_operand" "f"))
-		 (match_operand:DI 3 "register_operand" "f")))
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+	(plus:DI (mult:DI (match_operand:DI 1 "fr_register_operand" "f")
+			  (match_operand:DI 2 "fr_register_operand" "f"))
+		 (match_operand:DI 3 "fr_register_operand" "f")))
    (clobber (match_scratch:DI 4 "=X"))]
   ""
   "xma.l %0 = %1, %2, %3%B0"
@@ -1563,7 +1565,7 @@
 	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
 				   (match_operand:DI 2 "register_operand" ""))
 			  (match_operand:DI 3 "register_operand" ""))
-		 (match_operand:DI 4 "reg_or_14bit_operand" "")))
+		 (match_operand:DI 4 "gr_reg_or_14bit_operand" "")))
    (clobber (match_scratch:DI 5 ""))]
   "reload_completed"
   [(parallel [(set (match_dup 5) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
@@ -1579,38 +1581,42 @@
 ;; to generate them.
 
 (define_insn "smuldi3_highpart"
-  [(set (match_operand:DI 0 "register_operand" "=f")
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
 	(truncate:DI
 	 (lshiftrt:TI
-	  (mult:TI (sign_extend:TI (match_operand:DI 1 "register_operand" "f"))
-		   (sign_extend:TI (match_operand:DI 2 "register_operand" "f")))
+	  (mult:TI (sign_extend:TI
+		     (match_operand:DI 1 "fr_register_operand" "f"))
+		   (sign_extend:TI
+		     (match_operand:DI 2 "fr_register_operand" "f")))
 	  (const_int 64))))]
   ""
   "xma.h %0 = %1, %2, f0%B0"
   [(set_attr "type" "F")])
 
 (define_insn "umuldi3_highpart"
-  [(set (match_operand:DI 0 "register_operand" "=f")
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
 	(truncate:DI
 	 (lshiftrt:TI
-	  (mult:TI (zero_extend:TI (match_operand:DI 1 "register_operand" "f"))
-		   (zero_extend:TI (match_operand:DI 2 "register_operand" "f")))
+	  (mult:TI (zero_extend:TI
+		     (match_operand:DI 1 "fr_register_operand" "f"))
+		   (zero_extend:TI
+		     (match_operand:DI 2 "fr_register_operand" "f")))
 	  (const_int 64))))]
   ""
   "xma.hu %0 = %1, %2, f0%B0"
   [(set_attr "type" "F")])
 
 (define_insn "negdi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(neg:DI (match_operand:DI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(neg:DI (match_operand:DI 1 "gr_register_operand" "r")))]
   ""
   "sub %0 = r0, %1"
   [(set_attr "type" "A")])
 
 (define_expand "absdi2"
   [(set (match_dup 2)
-	(ge:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "")
+	(ge:CC (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (eq:CC (match_dup 2) (const_int 0))
 			 (neg:DI (match_dup 1))
 			 (match_dup 1)))]
@@ -1622,9 +1628,9 @@
 
 (define_expand "smindi3"
   [(set (match_dup 3)
-	(ge:CC (match_operand:DI 1 "register_operand" "")
-	       (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
+	(ge:CC (match_operand:DI 1 "gr_register_operand" "")
+	       (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 2) (match_dup 1)))]
   ""
@@ -1635,9 +1641,9 @@
 
 (define_expand "smaxdi3"
   [(set (match_dup 3)
-	(ge:CC (match_operand:DI 1 "register_operand" "")
-	       (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
+	(ge:CC (match_operand:DI 1 "gr_register_operand" "")
+	       (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 1) (match_dup 2)))]
   ""
@@ -1648,9 +1654,9 @@
 
 (define_expand "umindi3"
   [(set (match_dup 3)
-	(geu:CC (match_operand:DI 1 "register_operand" "")
-		(match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
+	(geu:CC (match_operand:DI 1 "gr_register_operand" "")
+		(match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 2) (match_dup 1)))]
   ""
@@ -1661,9 +1667,9 @@
 
 (define_expand "umaxdi3"
   [(set (match_dup 3)
-	(geu:CC (match_operand:DI 1 "register_operand" "")
-		(match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
+	(geu:CC (match_operand:DI 1 "gr_register_operand" "")
+		(match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
 			 (match_dup 1) (match_dup 2)))]
   ""
@@ -1674,12 +1680,12 @@
 
 (define_expand "ffsdi2"
   [(set (match_dup 6)
-	(eq:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
+	(eq:CC (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
    (set (match_dup 2) (plus:DI (match_dup 1) (const_int -1)))
    (set (match_dup 5) (const_int 0))
    (set (match_dup 3) (xor:DI (match_dup 1) (match_dup 2)))
    (set (match_dup 4) (unspec:DI [(match_dup 3)] 8))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(if_then_else:DI (ne:CC (match_dup 6) (const_int 0))
 			 (match_dup 5) (match_dup 4)))]
   ""
@@ -1693,8 +1699,8 @@
 }")
 
 (define_insn "*popcnt"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(unspec:DI [(match_operand:DI 1 "register_operand" "r")] 8))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(unspec:DI [(match_operand:DI 1 "gr_register_operand" "r")] 8))]
   ""
   "popcnt %0 = %1"
   [(set_attr "type" "I")])
@@ -1707,88 +1713,88 @@
 ;; ::::::::::::::::::::
 
 (define_insn "addsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(plus:SF (match_operand:SF 1 "register_operand" "%f")
-		 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(plus:SF (match_operand:SF 1 "fr_register_operand" "%f")
+		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fadd.s %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "subsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(minus:SF (match_operand:SF 1 "reg_or_fp01_operand" "fG")
-		  (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(minus:SF (match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+		  (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fsub.s %0 = %F1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "mulsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(mult:SF (match_operand:SF 1 "register_operand" "%f")
-		 (match_operand:SF 2 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(mult:SF (match_operand:SF 1 "fr_register_operand" "%f")
+		 (match_operand:SF 2 "fr_register_operand" "f")))]
   ""
   "fmpy.s %0 = %1, %2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "abssf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(abs:SF (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(abs:SF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
   "fabs %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "negsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(neg:SF (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(neg:SF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
   "fneg %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*nabssf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(neg:SF (abs:SF (match_operand:SF 1 "register_operand" "f"))))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(neg:SF (abs:SF (match_operand:SF 1 "fr_register_operand" "f"))))]
   ""
   "fnegabs %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "minsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(smin:SF (match_operand:SF 1 "register_operand" "f")
-		 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(smin:SF (match_operand:SF 1 "fr_register_operand" "f")
+		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fmin %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "maxsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(smax:SF (match_operand:SF 1 "register_operand" "f")
-		 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(smax:SF (match_operand:SF 1 "fr_register_operand" "f")
+		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fmax %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*maddsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(plus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-			  (match_operand:SF 2 "register_operand" "f"))
-		 (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(plus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+			  (match_operand:SF 2 "fr_register_operand" "f"))
+		 (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fma.s %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*msubsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(minus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-			   (match_operand:SF 2 "register_operand" "f"))
-		  (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(minus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+			   (match_operand:SF 2 "fr_register_operand" "f"))
+		  (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fms.s %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*nmulsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-			 (match_operand:SF 2 "register_operand" "f"))))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(neg:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+			 (match_operand:SF 2 "fr_register_operand" "f"))))]
   ""
   "fnmpy.s %0 = %1, %2%B0"
   [(set_attr "type" "F")])
@@ -1796,10 +1802,10 @@
 ;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
 
 (define_insn "*nmaddsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(plus:SF (neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-				  (match_operand:SF 2 "register_operand" "f")))
-		 (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+	(plus:SF (neg:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+				  (match_operand:SF 2 "fr_register_operand" "f")))
+		 (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fnma.s %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
@@ -1812,88 +1818,88 @@
 ;; ::::::::::::::::::::
 
 (define_insn "adddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(plus:DF (match_operand:DF 1 "register_operand" "%f")
-		 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(plus:DF (match_operand:DF 1 "fr_register_operand" "%f")
+		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fadd.d %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "subdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(minus:DF (match_operand:DF 1 "reg_or_fp01_operand" "fG")
-		  (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(minus:DF (match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+		  (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fsub.d %0 = %F1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "muldf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(mult:DF (match_operand:DF 1 "register_operand" "f")
-		 (match_operand:DF 2 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+		 (match_operand:DF 2 "fr_register_operand" "f")))]
   ""
   "fmpy.d %0 = %1, %2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "absdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(abs:DF (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(abs:DF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
   "fabs %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "negdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(neg:DF (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(neg:DF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
   "fneg %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*nabsdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(neg:DF (abs:DF (match_operand:DF 1 "register_operand" "f"))))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(neg:DF (abs:DF (match_operand:DF 1 "fr_register_operand" "f"))))]
   ""
   "fnegabs %0 = %1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "mindf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(smin:DF (match_operand:DF 1 "register_operand" "f")
-		 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(smin:DF (match_operand:DF 1 "fr_register_operand" "f")
+		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fmin %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "maxdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(smax:DF (match_operand:DF 1 "register_operand" "f")
-		 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(smax:DF (match_operand:DF 1 "fr_register_operand" "f")
+		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fmax %0 = %1, %F2%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*madddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(plus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-			  (match_operand:DF 2 "register_operand" "f"))
-		 (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(plus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+			  (match_operand:DF 2 "fr_register_operand" "f"))
+		 (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fma.d %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*msubdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(minus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-			   (match_operand:DF 2 "register_operand" "f"))
-		  (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(minus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+			   (match_operand:DF 2 "fr_register_operand" "f"))
+		  (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fms.d %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*nmuldf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-			 (match_operand:DF 2 "register_operand" "f"))))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+			 (match_operand:DF 2 "fr_register_operand" "f"))))]
   ""
   "fnmpy.d %0 = %1, %2%B0"
   [(set_attr "type" "F")])
@@ -1901,10 +1907,10 @@
 ;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
 
 (define_insn "*nmadddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(plus:DF (neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-				  (match_operand:DF 2 "register_operand" "f")))
-		 (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+	(plus:DF (neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+				  (match_operand:DF 2 "fr_register_operand" "f")))
+		 (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
   "fnma.d %0 = %1, %2, %F3%B0"
   [(set_attr "type" "F")])
@@ -1916,7 +1922,7 @@
 ;; ::::::::::::::::::::
 
 (define_insn "addtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(plus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
   ""
@@ -1924,7 +1930,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "subtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(minus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 		  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
   ""
@@ -1932,7 +1938,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "multf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
   ""
@@ -1940,28 +1946,28 @@
   [(set_attr "type" "F")])
 
 (define_insn "abstf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
   ""
   "fabs %0 = %F1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "negtf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(neg:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
   ""
   "fneg %0 = %F1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "*nabstf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(neg:TF (abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG"))))]
   ""
   "fnegabs %0 = %F1%B0"
   [(set_attr "type" "F")])
 
 (define_insn "mintf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(smin:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
   ""
@@ -1969,7 +1975,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "maxtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(smax:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
   ""
@@ -1977,7 +1983,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "*maddtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(plus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 			  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
 		 (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
@@ -1986,7 +1992,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "*msubtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(minus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 			   (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
 		  (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
@@ -1995,7 +2001,7 @@
   [(set_attr "type" "F")])
 
 (define_insn "*nmultf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(neg:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 			 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))))]
   ""
@@ -2005,7 +2011,7 @@
 ;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
 
 (define_insn "*nmaddtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
+  [(set (match_operand:TF 0 "fr_register_operand" "=f")
 	(plus:TF (neg:TF (mult:TF
 			  (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
 			  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))
@@ -2021,9 +2027,9 @@
 ;; ::::::::::::::::::::
 
 (define_expand "ashlsi3"
-  [(set (match_operand:SI 0 "register_operand" "")
-	(ashift:SI (match_operand:SI 1 "register_operand" "")
-		   (match_operand:SI 2 "reg_or_5bit_operand" "")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+	(ashift:SI (match_operand:SI 1 "gr_register_operand" "")
+		   (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
   "
 {
@@ -2038,9 +2044,9 @@
 }")
 
 (define_insn "*ashlsi3_internal"
-  [(set (match_operand:SI 0 "register_operand" "=r,r,r")
-	(ashift:SI (match_operand:SI 1 "register_operand" "r,r,r")
-		   (match_operand:DI 2 "reg_or_5bit_operand" "R,n,r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
+	(ashift:SI (match_operand:SI 1 "gr_register_operand" "r,r,r")
+		   (match_operand:DI 2 "gr_reg_or_5bit_operand" "R,n,r")))]
   ""
   "@
    shladd %0 = %1, %2, r0
@@ -2049,9 +2055,9 @@
   [(set_attr "type" "A,I,I")])
 
 (define_expand "ashrsi3"
-  [(set (match_operand:SI 0 "register_operand" "")
-	(ashiftrt:SI (match_operand:SI 1 "register_operand" "")
-		     (match_operand:SI 2 "reg_or_5bit_operand" "")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+	(ashiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
+		     (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
   "
 {
@@ -2071,9 +2077,9 @@
 }")
 
 (define_expand "lshrsi3"
-  [(set (match_operand:SI 0 "register_operand" "")
-	(lshiftrt:SI (match_operand:SI 1 "register_operand" "")
-		     (match_operand:SI 2 "reg_or_5bit_operand" "")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+	(lshiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
+		     (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
   "
 {
@@ -2097,12 +2103,12 @@
 
 (define_expand "rotrsi3"
   [(set (match_dup 3)
-	(ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" ""))
+	(ior:DI (zero_extend:DI (match_operand:SI 1 "gr_register_operand" ""))
 		(ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
    (set (match_dup 3)
 	(lshiftrt:DI (match_dup 3)
 		     (match_operand:DI 2 "nonmemory_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
+   (set (match_operand:SI 0 "gr_register_operand" "") (match_dup 4))]
   ""
   "
 {
@@ -2119,9 +2125,9 @@
 ;; ::::::::::::::::::::
 
 (define_insn "ashldi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
-	(ashift:DI (match_operand:DI 1 "register_operand" "r,r")
-		   (match_operand:DI 2 "reg_or_6bit_operand" "R,rM")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+	(ashift:DI (match_operand:DI 1 "gr_register_operand" "r,r")
+		   (match_operand:DI 2 "gr_reg_or_6bit_operand" "R,rM")))]
   ""
   "@
    shladd %0 = %1, %2, r0
@@ -2131,10 +2137,10 @@
 ;; ??? Maybe combine this with the multiply and add instruction?
 
 (define_insn "*shladd"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
 			  (match_operand:DI 2 "shladd_operand" "n"))
-		 (match_operand:DI 3 "register_operand" "r")))]
+		 (match_operand:DI 3 "gr_register_operand" "r")))]
   ""
   "shladd %0 = %1, %S2, %3"
   [(set_attr "type" "A")])
@@ -2148,8 +2154,8 @@
 ;; incorrectly.
 
 (define_insn_and_split "*shladd_elim"
-  [(set (match_operand:DI 0 "register_operand" "=&r")
-	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=&r")
+	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
 				   (match_operand:DI 2 "shladd_operand" "n"))
 			  (match_operand:DI 3 "nonmemory_operand" "r"))
 		 (match_operand:DI 4 "nonmemory_operand" "rI")))]
@@ -2163,17 +2169,17 @@
   [(set_attr "type" "unknown")])
 
 (define_insn "ashrdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(ashiftrt:DI (match_operand:DI 1 "register_operand" "r")
-		     (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(ashiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
+		     (match_operand:DI 2 "gr_reg_or_6bit_operand" "rM")))]
   ""
   "shr %0 = %1, %2"
   [(set_attr "type" "I")])
 
 (define_insn "lshrdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
-		     (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
+		     (match_operand:DI 2 "gr_reg_or_6bit_operand" "rM")))]
   ""
   "shr.u %0 = %1, %2"
   [(set_attr "type" "I")])
@@ -2184,8 +2190,8 @@
 ;; verify that we have an appropriate constant in the expander.
 
 (define_expand "rotrdi3"
-  [(set (match_operand:DI 0 "register_operand" "")
-	(rotatert:DI (match_operand:DI 1 "register_operand" "")
+  [(set (match_operand:DI 0 "gr_register_operand" "")
+	(rotatert:DI (match_operand:DI 1 "gr_register_operand" "")
 		     (match_operand:DI 2 "nonmemory_operand" "")))]
   ""
   "
@@ -2195,8 +2201,8 @@
 }")
 
 (define_insn "*rotrdi3_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(rotatert:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(rotatert:DI (match_operand:DI 1 "gr_register_operand" "r")
 		     (match_operand:DI 2 "shift_count_operand" "M")))]
   ""
   "shrp %0 = %1, %1, %2"
@@ -2224,8 +2230,8 @@
 ;; one_cmplsi2 pattern.
 
 (define_insn "one_cmplsi2"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-	(not:SI (match_operand:SI 1 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(not:SI (match_operand:SI 1 "gr_register_operand" "r")))]
   ""
   "andcm %0 = -1, %1"
   [(set_attr "type" "A")])
@@ -2237,9 +2243,9 @@
 ;; ::::::::::::::::::::
 
 (define_insn "anddi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-	(and:DI (match_operand:DI 1 "register_operand" "%r,*f")
-		(match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+	(and:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    and %0 = %2, %1
@@ -2247,9 +2253,9 @@
   [(set_attr "type" "A,F")])
 
 (define_insn "*andnot"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-	(and:DI (not:DI (match_operand:DI 1 "register_operand" "r,*f"))
-		(match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+	(and:DI (not:DI (match_operand:DI 1 "grfr_register_operand" "r,*f"))
+		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    andcm %0 = %2, %1
@@ -2257,9 +2263,9 @@
   [(set_attr "type" "A,F")])
 
 (define_insn "iordi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-	(ior:DI (match_operand:DI 1 "register_operand" "%r,*f")
-		(match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+	(ior:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    or %0 = %2, %1
@@ -2267,9 +2273,9 @@
   [(set_attr "type" "A,F")])
 
 (define_insn "xordi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-	(xor:DI (match_operand:DI 1 "register_operand" "%r,*f")
-		(match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+	(xor:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    xor %0 = %2, %1
@@ -2277,8 +2283,8 @@
   [(set_attr "type" "A,F")])
 
 (define_insn "one_cmpldi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(not:DI (match_operand:DI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(not:DI (match_operand:DI 1 "gr_register_operand" "r")))]
   ""
   "andcm %0 = -1, %1"
   [(set_attr "type" "A")])
@@ -2291,8 +2297,8 @@
 
 (define_expand "cmpsi"
   [(set (cc0)
-        (compare (match_operand:SI 0 "register_operand" "")
-  		 (match_operand:SI 1 "reg_or_8bit_and_adjusted_operand" "")))]
+        (compare (match_operand:SI 0 "gr_register_operand" "")
+  		 (match_operand:SI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
   ""
   "
 {
@@ -2303,8 +2309,8 @@
 
 (define_expand "cmpdi"
   [(set (cc0)
-        (compare (match_operand:DI 0 "register_operand" "")
-  		 (match_operand:DI 1 "reg_or_8bit_and_adjusted_operand" "")))]
+        (compare (match_operand:DI 0 "gr_register_operand" "")
+  		 (match_operand:DI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
   ""
   "
 {
@@ -2315,8 +2321,8 @@
 
 (define_expand "cmpsf"
   [(set (cc0)
-        (compare (match_operand:SF 0 "reg_or_fp01_operand" "")
-  		 (match_operand:SF 1 "reg_or_fp01_operand" "")))]
+        (compare (match_operand:SF 0 "fr_reg_or_fp01_operand" "")
+  		 (match_operand:SF 1 "fr_reg_or_fp01_operand" "")))]
   ""
   "
 {
@@ -2327,8 +2333,8 @@
 
 (define_expand "cmpdf"
   [(set (cc0)
-        (compare (match_operand:DF 0 "reg_or_fp01_operand" "")
-  		 (match_operand:DF 1 "reg_or_fp01_operand" "")))]
+        (compare (match_operand:DF 0 "fr_reg_or_fp01_operand" "")
+  		 (match_operand:DF 1 "fr_reg_or_fp01_operand" "")))]
   ""
   "
 {
@@ -2352,8 +2358,8 @@
 (define_insn "*cmpsi_normal"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "normal_comparison_operator"
-			   [(match_operand:SI 2 "register_operand" "r")
-			    (match_operand:SI 3 "reg_or_8bit_operand" "rK")]))]
+	   [(match_operand:SI 2 "gr_register_operand" "r")
+	    (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")]))]
   ""
   "cmp4.%C1 %0, %I0 = %3, %2"
   [(set_attr "type" "A")])
@@ -2361,9 +2367,8 @@
 (define_insn "*cmpsi_adjusted"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "adjusted_comparison_operator"
-			   [(match_operand:SI 2 "register_operand" "r")
-			    (match_operand:SI 3 "reg_or_8bit_adjusted_operand"
-					      "rL")]))]
+	   [(match_operand:SI 2 "gr_register_operand" "r")
+	    (match_operand:SI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
   ""
   "cmp4.%C1 %0, %I0 = %3, %2"
   [(set_attr "type" "A")])
@@ -2371,8 +2376,8 @@
 (define_insn "*cmpdi_normal"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "normal_comparison_operator"
-			   [(match_operand:DI 2 "register_operand" "r")
-			    (match_operand:DI 3 "reg_or_8bit_operand" "rK")]))]
+	   [(match_operand:DI 2 "gr_register_operand" "r")
+	    (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")]))]
   ""
   "cmp.%C1 %0, %I0 = %3, %2"
   [(set_attr "type" "A")])
@@ -2380,9 +2385,8 @@
 (define_insn "*cmpdi_adjusted"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "adjusted_comparison_operator"
-			   [(match_operand:DI 2 "register_operand" "r")
-			    (match_operand:DI 3 "reg_or_8bit_adjusted_operand"
-					      "rL")]))]
+	   [(match_operand:DI 2 "gr_register_operand" "r")
+	    (match_operand:DI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
   ""
   "cmp.%C1 %0, %I0 = %3, %2"
   [(set_attr "type" "A")])
@@ -2390,8 +2394,8 @@
 (define_insn "*cmpsf_internal"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "comparison_operator"
-			   [(match_operand:SF 2 "reg_or_fp01_operand" "fG")
-			    (match_operand:SF 3 "reg_or_fp01_operand" "fG")]))]
+	   [(match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")
+	    (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")]))]
   ""
   "fcmp.%D1 %0, %I0 = %F2, %F3"
   [(set_attr "type" "F")])
@@ -2399,8 +2403,8 @@
 (define_insn "*cmpdf_internal"
   [(set (match_operand:CC 0 "register_operand" "=c")
 	(match_operator:CC 1 "comparison_operator"
-			   [(match_operand:DF 2 "reg_or_fp01_operand" "fG")
-			    (match_operand:DF 3 "reg_or_fp01_operand" "fG")]))]
+	   [(match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")
+	    (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")]))]
   ""
   "fcmp.%D1 %0, %I0 = %F2, %F3"
   [(set_attr "type" "F")])
@@ -2418,7 +2422,7 @@
 
 (define_insn "*bit_zero"
   [(set (match_operand:CC 0 "register_operand" "=c")
-	(eq:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+	(eq:CC (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
 				(const_int 1)
 				(match_operand:DI 2 "immediate_operand" "n"))
 	       (const_int 0)))]
@@ -2428,7 +2432,7 @@
 
 (define_insn "*bit_one"
   [(set (match_operand:CC 0 "register_operand" "=c")
-	(ne:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+	(ne:CC (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
 				(const_int 1)
 				(match_operand:DI 2 "immediate_operand" "n"))
 	       (const_int 0)))]
@@ -2767,7 +2771,7 @@
   [(set (match_dup 1)
 	(eq:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2781,7 +2785,7 @@
   [(set (match_dup 1)
 	(ne:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2795,7 +2799,7 @@
   [(set (match_dup 1)
 	(lt:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2809,7 +2813,7 @@
   [(set (match_dup 1)
 	(le:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2823,7 +2827,7 @@
   [(set (match_dup 1)
 	(gt:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2837,7 +2841,7 @@
   [(set (match_dup 1)
 	(ge:CC (match_dup 2)
 	       (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2851,7 +2855,7 @@
   [(set (match_dup 1)
 	(ltu:CC (match_dup 2)
 		(match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2865,7 +2869,7 @@
   [(set (match_dup 1)
 	(leu:CC (match_dup 2)
 		(match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2879,7 +2883,7 @@
   [(set (match_dup 1)
 	(gtu:CC (match_dup 2)
 		(match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2893,7 +2897,7 @@
   [(set (match_dup 1)
 	(geu:CC (match_dup 2)
 		(match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2907,7 +2911,7 @@
   [(set (match_dup 1)
 	(unordered:CC (match_dup 2)
 		      (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2921,7 +2925,7 @@
   [(set (match_dup 1)
 	(ordered:CC (match_dup 2)
 		      (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
+   (set (match_operand:DI 0 "gr_register_operand" "")
 	(ne:DI (match_dup 1) (const_int 0)))]
   ""
   "
@@ -2934,18 +2938,12 @@
 ;; Don't allow memory as destination here, because cmov/cmov/st is more
 ;; efficient than mov/mov/cst/cst.
 
-(define_insn "*sne_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
+(define_insn_and_split "*sne_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
 	(ne:DI (match_operand:CC 1 "register_operand" "c")
 	       (const_int 0)))]
   ""
   "#"
-  [(set_attr "type" "unknown")])
-
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-	(ne:DI (match_operand:CC 1 "register_operand" "")
-	       (const_int 0)))]
   "reload_completed"
   [(set (match_dup 0)
 	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
@@ -2955,24 +2953,17 @@
 	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
 			 (match_dup 0)
 			 (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "unknown")])
 
 ;; ??? Unknown if this can be matched.
 
-(define_insn "*seq_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
+(define_insn_and_split "*seq_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
 	(eq:DI (match_operand:CC 1 "register_operand" "c")
 	       (const_int 0)))]
   ""
   "#"
-  [(set_attr "type" "unknown")])
-
-;; ??? Unknown if this can be matched.
-
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-	(eq:DI (match_operand:CC 1 "register_operand" "")
-	       (const_int 0)))]
   "reload_completed"
   [(set (match_dup 0)
 	(if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
@@ -2982,7 +2973,8 @@
 	(if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
 			 (match_dup 0)
 			 (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "unknown")])
 
 
 ;; ::::::::::::::::::::
@@ -3064,13 +3056,13 @@
 ;; Absolute value pattern.
 
 (define_insn "*absdi2_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
 	(if_then_else:DI
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" "rI,rI"))
-	  (match_operand:DI 3 "reg_or_22bit_operand" "0,rI")))]
+	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" "rI,rI"))
+	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "0,rI")))]
   ""
   "#"
   [(set_attr "type" "A,unknown")
@@ -3082,8 +3074,8 @@
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
-	  (match_operand:DI 3 "reg_or_22bit_operand" "")))]
+	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
+	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed && rtx_equal_p (operands[0], operands[3])"
   [(cond_exec
      (match_dup 4)
@@ -3097,8 +3089,8 @@
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
-	  (match_operand:DI 3 "reg_or_22bit_operand" "")))]
+	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
+	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed"
   [(cond_exec
      (match_dup 4)
@@ -3145,13 +3137,13 @@
   [(set_attr "predicable" "no")])
 
 (define_insn "*abssi2_internal"
-  [(set (match_operand:SI 0 "register_operand" "=r,r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r")
 	(if_then_else:SI
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:SI (match_operand:SI 3 "reg_or_22bit_operand" "rI,rI"))
-	  (match_operand:SI 2 "reg_or_22bit_operand" "0,rI")))]
+	  (neg:SI (match_operand:SI 3 "gr_reg_or_22bit_operand" "rI,rI"))
+	  (match_operand:SI 2 "gr_reg_or_22bit_operand" "0,rI")))]
   ""
   "#"
   [(set_attr "type" "A,unknown")
@@ -3163,8 +3155,8 @@
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
-	  (match_operand:SI 3 "reg_or_22bit_operand" "")))]
+	  (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
+	  (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed && rtx_equal_p (operands[0], operands[3])"
   [(cond_exec
      (match_dup 4)
@@ -3178,8 +3170,8 @@
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand" "c,c")
 	     (const_int 0)])
-	  (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
-	  (match_operand:SI 3 "reg_or_22bit_operand" "")))]
+	  (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
+	  (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed"
   [(cond_exec
      (match_dup 4)
@@ -3617,7 +3609,7 @@
 (define_insn "prologue_allocate_stack"
   [(set (match_operand:DI 0 "register_operand" "=r,r,r")
 	(plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
-		 (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))
+		 (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))
    (set (match_operand:DI 3 "register_operand" "=r,r,r")
 	(match_dup 3))]
   ""
@@ -3860,409 +3852,82 @@
 
 ;;; Intrinsics support.
 
-(define_insn "mf"
-  [(unspec [(match_operand:BLK 0 "memory_operand" "m")] 12)]
+(define_expand "mf"
+  [(set (mem:BLK (match_dup 0))
+	(unspec:BLK [(mem:BLK (match_dup 0))] 12))]
+  ""
+  "
+{
+  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (DImode));
+  MEM_VOLATILE_P (operands[0]) = 1;
+}")
+
+(define_insn "*mf_internal"
+  [(set (match_operand:BLK 0 "" "")
+	(unspec:BLK [(match_operand:BLK 1 "" "")] 12))]
   ""
   "mf"
   [(set_attr "type" "M")])
 
 (define_insn "fetchadd_acq_si"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
-                    (match_operand:SI 2 "fetchadd_operand" "n")] 19))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(match_dup 1))
+   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
+	(unspec:SI [(match_dup 1)
+		    (match_operand:SI 2 "fetchadd_operand" "n")] 19))]
   ""
   "fetchadd4.acq %0 = %1, %2"
   [(set_attr "type" "M")])
 
 (define_insn "fetchadd_acq_di"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                    (match_operand:DI 2 "fetchadd_operand" "n")] 19))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(match_dup 1))
+   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
+	(unspec:DI [(match_dup 1)
+		    (match_operand:DI 2 "fetchadd_operand" "n")] 19))]
   ""
   "fetchadd8.acq %0 = %1, %2"
   [(set_attr "type" "M")])
 
 (define_insn "cmpxchg_acq_si"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
-                    (match_operand:SI 2 "register_operand" "r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+	(match_dup 1))
+   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
+        (unspec:SI [(match_dup 1)
+                    (match_operand:SI 2 "gr_register_operand" "r")
 		    (match_operand:SI 3 "ar_ccv_reg_operand" "")] 13))]
   ""
   "cmpxchg4.acq %0 = %1, %2, %3"
   [(set_attr "type" "M")])
 
 (define_insn "cmpxchg_acq_di"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                    (match_operand:DI 2 "register_operand" "r")
-                    (match_operand:DI 3 "ar_ccv_reg_operand" "")] 13))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+	(match_dup 1))
+   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
+        (unspec:DI [(match_dup 1)
+                    (match_operand:DI 2 "gr_register_operand" "r")
+		    (match_operand:DI 3 "ar_ccv_reg_operand" "")] 13))]
   ""
   "cmpxchg8.acq %0 = %1, %2, %3"
   [(set_attr "type" "M")])
 
-(define_expand "val_compare_and_swap_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")
-   (match_operand:SI 3 "register_operand" "")]
-  ""
-  "
-{
-  rtx target = gen_rtx_MEM (BLKmode, gen_rtx_REG (DImode, GR_REG (1)));
-  rtx ccv = gen_rtx_REG (SImode, AR_CCV_REGNUM);
-  emit_move_insn (ccv, operands[2]);
-  emit_insn (gen_mf (target));
-  emit_insn (gen_cmpxchg_acq_si (operands[0], operands[1], operands[3], ccv));
-  DONE;
-}")
-
-(define_expand "val_compare_and_swap_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")
-   (match_operand:DI 3 "register_operand" "")]
-  ""
-  "
-{
-  rtx target = gen_rtx_MEM (BLKmode, gen_rtx_REG (DImode, GR_REG (1)));
-  rtx ccv = gen_rtx_REG (DImode, AR_CCV_REGNUM);
-  emit_move_insn (ccv, operands[2]);
-  emit_insn (gen_mf (target));
-  emit_insn (gen_cmpxchg_acq_di (operands[0], operands[1], operands[3], ccv));
-  DONE;
-}")
-
 (define_insn "xchgsi"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (match_operand:SI 1 "memory_operand" "+m"))
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+        (match_operand:SI 1 "not_postinc_memory_operand" "+S"))
    (set (match_dup 1)
-        (match_operand:SI 2 "register_operand" "r"))]
+        (match_operand:SI 2 "gr_register_operand" "r"))]
   ""
   "xchg4 %0 = %1, %2"
   [(set_attr "type" "M")])
 
 (define_insn "xchgdi"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (match_operand:DI 1 "memory_operand" "+m"))
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+        (match_operand:DI 1 "not_postinc_memory_operand" "+S"))
    (set (match_dup 1)
-        (match_operand:DI 2 "register_operand" "r"))]
+        (match_operand:DI 2 "gr_register_operand" "r"))]
   ""
   "xchg8 %0 = %1, %2"
   [(set_attr "type" "M")])
-
-(define_expand "lock_test_and_set_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  emit_insn (gen_xchgsi (operands[0], operands[1], operands[2]));
-  DONE;
-}")
-
-(define_expand "lock_test_and_set_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  emit_insn (gen_xchgdi (operands[0], operands[1], operands[2]));
-  DONE;
-}")
-
-(define_expand "fetch_and_add_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "nonmemory_operand" "")]
-  ""
-  "
-{
-  int x;
-
-  if (GET_CODE (operands[2]) == CONST_INT)
-    {
-      x = INTVAL(operands[2]);
-      if (x == -16 || x == -8 || x == -4 || x == -1 ||
-          x ==  16 || x ==  8 || x ==  4 || x ==  1)
-        {
-          emit_insn (gen_fetchadd_acq_si (operands[0], operands[1],
-					  operands[2]));
-          DONE;
-        }
-    }
-
-  ia64_expand_fetch_and_op (IA64_ADD_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_sub_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_SUB_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_or_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_OR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_and_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_AND_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_xor_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_XOR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_nand_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_NAND_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_add_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "nonmemory_operand" "")]
-  ""
-  "
-{
-  int x;
-
-  if (GET_CODE (operands[2]) == CONST_INT)
-    {
-      x = INTVAL(operands[2]);
-      if (x == -16 || x == -8 || x == -4 || x == -1 ||
-          x ==  16 || x ==  8 || x ==  4 || x ==  1)
-        {
-          emit_insn (gen_fetchadd_acq_di (operands[0], operands[1],
-					  operands[2]));
-          DONE;
-        }
-    }
-
-  ia64_expand_fetch_and_op (IA64_ADD_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_sub_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_SUB_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_or_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_OR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_and_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_AND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_xor_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_XOR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_nand_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_NAND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "add_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_ADD_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "sub_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_SUB_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "or_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_OR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "and_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_AND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "xor_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_XOR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "nand_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_NAND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "add_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_ADD_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "sub_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_SUB_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "or_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_OR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "and_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_AND_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "xor_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_XOR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "nand_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_NAND_OP, SImode, operands);
-  DONE;
-}")
 
 ;; Predication.
 
diff --git a/gcc/config/ia64/ia64intrin.h b/gcc/config/ia64/ia64intrin.h
index c75f185..0910ed8 100644
--- a/gcc/config/ia64/ia64intrin.h
+++ b/gcc/config/ia64/ia64intrin.h
@@ -1,60 +1,132 @@
 #ifndef _IA64INTRIN_H_INCLUDED
 #define _IA64INTRIN_H_INCLUDED
 
-void __sync_synchronize (void);
-
-int  __sync_val_compare_and_swap_si (int *, int, int);
-long __sync_val_compare_and_swap_di (long *, long, long);
-#define __sync_val_compare_and_swap(A,B,C) ((sizeof (*(A)) == sizeof(int)) ? __sync_val_compare_and_swap_si((int *)(A),(int)(B),(int)(C)) : __sync_val_compare_and_swap_di((long *)(A),(long)(B),(long)(C)))
-
-int  __sync_bool_compare_and_swap_si (int *, int, int);
-long __sync_bool_compare_and_swap_di (long *, long, long);
-#define __sync_bool_compare_and_swap(A,B,C) ((sizeof (*(A)) == sizeof(int)) ? __sync_bool_compare_and_swap_si((int *)(A),(int)(B),(int)(C)) : __sync_bool_compare_and_swap_di((long *)(A),(long)(B),(long)(C)))
-
-void __sync_lock_release_si (int *);
-void __sync_lock_release_di (long *);
-#define __sync_lock_release(A) ((sizeof (*(A)) == sizeof(int)) ? __sync_lock_release_si((int *)(A)) : __sync_lock_release_di((long *)(A)))
-
-int  __sync_lock_test_and_set_si (int *, int);
-long __sync_lock_test_and_set_di (long *, long);
-#define __sync_lock_test_and_set(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_lock_test_and_set_si((int *)(A),(int)(B)) : __sync_lock_test_and_set_di((long *)(A),(long)(B)))
-
-int  __sync_fetch_and_add_si  (int *, int);
-int  __sync_fetch_and_sub_si  (int *, int);
-int  __sync_fetch_and_and_si  (int *, int);
-int  __sync_fetch_and_or_si   (int *, int);
-int  __sync_fetch_and_xor_si  (int *, int);
-int  __sync_fetch_and_nand_si (int *, int);
-long __sync_fetch_and_add_di  (long *, long);
-long __sync_fetch_and_sub_di  (long *, long);
-long __sync_fetch_and_and_di  (long *, long);
-long __sync_fetch_and_or_di   (long *, long);
-long __sync_fetch_and_xor_di  (long *, long);
-long __sync_fetch_and_nand_di (long *, long);
-#define __sync_fetch_and_add(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_add_si((int *)(A),(int)(B)) : __sync_fetch_and_add_di((long *)(A),(long)(B)))
-#define __sync_fetch_and_sub(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_sub_si((int *)(A),(int)(B)) : __sync_fetch_and_sub_di((long *)(A),(long)(B)))
-#define __sync_fetch_and_and(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_and_si((int *)(A),(int)(B)) : __sync_fetch_and_and_di((long *)(A),(long)(B)))
-#define __sync_fetch_and_or(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_or_si((int *)(A),(int)(B)) : __sync_fetch_and_or_di((long *)(A),(long)(B)))
-#define __sync_fetch_and_xor(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_xor_si((int *)(A),(int)(B)) : __sync_fetch_and_xor_di((long *)(A),(long)(B)))
-#define __sync_fetch_and_nand(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_fetch_and_nand_si((int *)(A),(int)(B)) : __sync_fetch_and_nand_di((long *)(A),(long)(B)))
-
-int  __sync_add_and_fetch_si  (int *, int);
-int  __sync_sub_and_fetch_si  (int *, int);
-int  __sync_and_and_fetch_si  (int *, int);
-int  __sync_or_and_fetch_si   (int *, int);
-int  __sync_xor_and_fetch_si  (int *, int);
-int  __sync_nand_and_fetch_si (int *, int);
-long __sync_add_and_fetch_di  (long *, long);
-long __sync_sub_and_fetch_di  (long *, long);
-long __sync_and_and_fetch_di  (long *, long);
-long __sync_or_and_fetch_di   (long *, long);
-long __sync_xor_and_fetch_di  (long *, long);
-long __sync_nand_and_fetch_di (long *, long);
-#define __sync_add_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_add_and_fetch_si((int *)(A),(int)(B)) : __sync_add_and_fetch_di((long *)(A),(long)(B)))
-#define __sync_sub_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_sub_and_fetch_si((int *)(A),(int)(B)) : __sync_sub_and_fetch_di((long *)(A),(long)(B)))
-#define __sync_and_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_and_and_fetch_si((int *)(A),(int)(B)) : __sync_and_and_fetch_di((long *)(A),(long)(B)))
-#define __sync_or_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_or_and_fetch_si((int *)(A),(int)(B)) : __sync_or_and_fetch_di((long *)(A),(long)(B)))
-#define __sync_xor_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_xor_and_fetch_si((int *)(A),(int)(B)) : __sync_xor_and_fetch_di((long *)(A),(long)(B)))
-#define __sync_nand_and_fetch(A,B) ((sizeof (*(A)) == sizeof(int)) ? __sync_nand_and_fetch_si((int *)(A),(int)(B)) : __sync_nand_and_fetch_di((long *)(A),(long)(B)))
+/* Actually, everything is a compiler builtin, but just so
+   there's no confusion... */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void __sync_synchronize (void);
+
+extern int __sync_val_compare_and_swap_si (int *, int, int);
+extern long __sync_val_compare_and_swap_di (long *, long, long);
+#define __sync_val_compare_and_swap(PTR, OLD, NEW)			\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR)))						\
+    __sync_val_compare_and_swap_si((int *)(PTR),(int)(OLD),(int)(NEW))	\
+  : (__typeof__(*(PTR)))						\
+    __sync_val_compare_and_swap_di((long *)(PTR),(long)(OLD),(long)(NEW)))
+
+extern int __sync_bool_compare_and_swap_si (int *, int, int);
+extern long __sync_bool_compare_and_swap_di (long *, long, long);
+#define __sync_bool_compare_and_swap(PTR, OLD, NEW)			\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR)))						\
+    __sync_bool_compare_and_swap_si((int *)(PTR),(int)(OLD),(int)(NEW))	\
+  : (__typeof__(*(PTR)))						\
+    __sync_bool_compare_and_swap_di((long *)(PTR),(long)(OLD),(long)(NEW)))
+
+extern void __sync_lock_release_si (int *);
+extern void __sync_lock_release_di (long *);
+#define __sync_lock_release(PTR)		\
+  ((sizeof (*(PTR)) == sizeof(int))		\
+   ? __sync_lock_release_si((int *)(PTR))	\
+   : __sync_lock_release_di((long *)(PTR)))
+
+extern int __sync_lock_test_and_set_si (int *, int);
+extern long __sync_lock_test_and_set_di (long *, long);
+#define __sync_lock_test_and_set(PTR,VAL)				\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_lock_test_and_set_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_lock_test_and_set_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_add_si (int *, int);
+extern long __sync_fetch_and_add_di (long *, long);
+#define __sync_fetch_and_add(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_add_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_add_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_sub_si (int *, int);
+extern long __sync_fetch_and_sub_di (long *, long);
+#define __sync_fetch_and_sub(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_sub_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_sub_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_and_si (int *, int);
+extern long __sync_fetch_and_and_di (long *, long);
+#define __sync_fetch_and_and(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_and_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_and_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_or_si (int *, int);
+extern long __sync_fetch_and_or_di (long *, long);
+#define __sync_fetch_and_or(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_or_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_or_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_xor_si (int *, int);
+extern long __sync_fetch_and_xor_di (long *, long);
+#define __sync_fetch_and_xor(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_xor_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_xor_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_fetch_and_nand_si (int *, int);
+extern long __sync_fetch_and_nand_di (long *, long);
+#define __sync_fetch_and_nand(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_fetch_and_nand_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_fetch_and_nand_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_add_and_fetch_si (int *, int);
+extern long __sync_add_and_fetch_di (long *, long);
+#define __sync_add_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_add_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_add_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_sub_and_fetch_si (int *, int);
+extern long __sync_sub_and_fetch_di (long *, long);
+#define __sync_sub_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_sub_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_sub_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_and_and_fetch_si (int *, int);
+extern long __sync_and_and_fetch_di (long *, long);
+#define __sync_and_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_and_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_and_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_or_and_fetch_si (int *, int);
+extern long __sync_or_and_fetch_di (long *, long);
+#define __sync_or_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_or_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_or_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_xor_and_fetch_si (int *, int);
+extern long __sync_xor_and_fetch_di (long *, long);
+#define __sync_xor_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_xor_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_xor_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+extern int __sync_nand_and_fetch_si (int *, int);
+extern long __sync_nand_and_fetch_di (long *, long);
+#define __sync_nand_and_fetch(PTR,VAL)					\
+ ((sizeof (*(PTR)) == sizeof(int))					\
+  ? (__typeof__(*(PTR))) __sync_nand_and_fetch_si((int *)(PTR),(int)(VAL)) \
+  : (__typeof__(*(PTR))) __sync_nand_and_fetch_di((long *)(PTR),(long)(VAL)))
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif
diff --git a/gcc/stmt.c b/gcc/stmt.c
index 2d7566e..d0dbb9e 100644
--- a/gcc/stmt.c
+++ b/gcc/stmt.c
@@ -1489,9 +1489,6 @@ expand_asm_operands (string, outputs, inputs, clobbers, vol, filename, line)
 	  case 's':  case 'i':  case 'n':
 	  case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
 	  case 'N':  case 'O':  case 'P':  case ',':
-#ifdef EXTRA_CONSTRAINT
-	  case 'Q':  case 'R':  case 'S':  case 'T':  case 'U':
-#endif
 	    break;
 
 	  case '0':  case '1':  case '2':  case '3':  case '4':
@@ -1511,6 +1508,9 @@ expand_asm_operands (string, outputs, inputs, clobbers, vol, filename, line)
 	    break;
 
 	  case 'g':  case 'X':
+#ifdef EXTRA_CONSTRAINT
+	  case 'Q':  case 'R':  case 'S':  case 'T':  case 'U':
+#endif
 	    allows_reg = 1;
 	    allows_mem = 1;
 	    break;
@@ -1643,13 +1643,10 @@ expand_asm_operands (string, outputs, inputs, clobbers, vol, filename, line)
 
 	  case '<':  case '>':
 	  case '?':  case '!':  case '*':
-	  case 'E':  case 'F':  case 'G':  case 'H':  case 'X':
+	  case 'E':  case 'F':  case 'G':  case 'H':
 	  case 's':  case 'i':  case 'n':
 	  case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
 	  case 'N':  case 'O':  case 'P':  case ',':
-#ifdef EXTRA_CONSTRAINT
-	  case 'Q':  case 'R':  case 'S':  case 'T':  case 'U':
-#endif
 	    break;
 
 	    /* Whether or not a numeric constraint allows a register is
@@ -1688,7 +1685,11 @@ expand_asm_operands (string, outputs, inputs, clobbers, vol, filename, line)
 	    allows_reg = 1;
 	    break;
 
-	  case 'g':
+	  case 'g':  case 'X':
+#ifdef EXTRA_CONSTRAINT
+	  case 'Q':  case 'R':  case 'S':  case 'T':  case 'U':
+#endif
+
 	    allows_reg = 1;
 	    allows_mem = 1;
 	    break;