re PR target/64661 ([SH] Allow @(disp,reg) address mode for atomics)

gcc/
	PR target/64661
	* config/sh/sh-protos.h (TARGET_ATOMIC_ANY, TARGET_ATOMIC_STRICT,
	TARGET_ATOMIC_SOFT_GUSA, TARGET_ATOMIC_HARD_LLCS,
	TARGET_ATOMIC_SOFT_TCB, TARGET_ATOMIC_SOFT_IMASK): Add parentheses.
	* config/sh/constraints.md (Ara, Add): New constraints.
	* config/sh/sync.md (atomic_mem_operand_0, atomic_mem_operand_1): New
	predicates.
	(atomic_compare_and_swap<mode>, atomic_exchange<mode>): Use
	atomic_mem_operand_0.  Don't use force_reg on the memory address.
	(atomic_compare_and_swapsi_hard): Use atomic_mem_operand_0 predicate and
	Sra constraint.  Convert to insn_and_split.  Add workaround for
	PR 64974.
	(atomic_compare_and_swap<mode>_hard): Copy to
	atomic_compare_and_swap<mode>_hard_1.  Convert to insn_and_split.
	Use atomic_mem_operand_0 predicate.
	(atomic_compare_and_swap<mode>_soft_gusa,
	atomic_exchange<mode>_soft_gusa): Use atomic_mem_operand_0 predicate and
	AraAdd constraints.
	(atomic_compare_and_swap<mode>_soft_tcb,
	atomic_compare_and_swap<mode>_soft_imask,
	atomic_exchange<mode>_soft_tcb, atomic_exchange<mode>_soft_imask): Use
	atomic_mem_operand_0 predicate and SraSdd constraints.
	(atomic_exchangesi_hard) Use atomic_mem_operand_0 predicate and Sra
	constraint.
	(atomic_exchange<mode>_hard): Copy to atomic_exchange<mode>_hard_1.
	Convert to insn_and_split.  Use atomic_mem_operand_0 predicate.
	(atomic_fetch_<fetchop_name><mode>, atomic_fetch_nand<mode>,
	atomic_<fetchop_name>_fetch<mode>): Use atomic_mem_operand_1.  Don't use
	force_reg on the memory address.
	(atomic_fetch_<fetchop_name>si_hard, atomic_fetch_notsi_hard,
	atomic_fetch_nandsi_hard, atomic_<fetchop_name>_fetchsi_hard,
	atomic_not_fetchsi_hard, atomic_nand_fetchsi_hard): Use
	atomic_mem_operand_1 predicate and Sra constraint.
	(atomic_fetch_<fetchop_name><mode>_hard): Copy to
	atomic_fetch_<fetchop_name><mode>_hard_1.  Convert to insn_and_split.
	Use atomic_mem_operand_1 predicate.
	(atomic_<fetchop_name><mode>_hard): Copy to
	atomic_<fetchop_name><mode>_hard_1.  Convert to insn_and_split.
	Use atomic_mem_operand_1 predicate.
	(atomic_fetch_nand<mode>_hard): Copy to atomic_fetch_nand<mode>_hard_1.
	Convert to insn_and_split.  Use atomic_mem_operand_1 predicate.
	(atomic_nand<mode>_hard): Copy to atomic_nand<mode>_hard_1.  Convert to
	insn_and_split.  Use atomic_mem_operand_1 predicate.
	(atomic_<fetchop_name>_fetch<mode>_hard): Copy to
	atomic_<fetchop_name>_fetch<mode>_hard_1.  Convert to insn_and_split.
	Use atomic_mem_operand_1 predicate.
	(atomic_nand_fetch<mode>_hard): Copy to atomic_nand_fetch<mode>_hard_1.
	Convert to insn_and_split.  Use atomic_mem_operand_1 predicate.
	(atomic_fetch_not<mode>_hard, atomic_not_fetch<mode>_hard): Replace mems
	in generated insn with original mem operand before emitting the insn.
	(atomic_fetch_<fetchop_name><mode>_soft_gusa,
	atomic_fetch_not<mode>_soft_gusa, atomic_fetch_nand<mode>_soft_gusa,
	atomic_<fetchop_name>_fetch<mode>_soft_gusa,
	atomic_not_fetch<mode>_soft_gusa, atomic_nand_fetch<mode>_soft_gusa):
	Use atomic_mem_operand_1 predicate and AraAdd constraints.
	(atomic_fetch_<fetchop_name><mode>_soft_tcb,
	atomic_<fetchop_name><mode>_soft_tcb, atomic_fetch_not<mode>_soft_tcb,
	atomic_not<mode>_soft_tcb, atomic_fetch_<fetchop_name><mode>_soft_imask,
	atomic_fetch_not<mode>_soft_imask, atomic_fetch_nand<mode>_soft_tcb,
	atomic_nand<mode>_soft_tcb, atomic_fetch_nand<mode>_soft_imask,
	atomic_<fetchop_name>_fetch<mode>_soft_tcb,
	atomic_not_fetch<mode>_soft_tcb,
	atomic_<fetchop_name>_fetch<mode>_soft_imask,
	atomic_not_fetch<mode>_soft_imask, atomic_nand_fetch<mode>,
	atomic_nand_fetch<mode>_soft_tcb, atomic_nand_fetch<mode>_soft_imask):
	Use atomic_mem_operand_1 predicate and SraSdd constraints.

gcc/testsuite/
	PR target/64661
	* gcc.taget/sh/pr64661-0.h: New.
	* gcc.taget/sh/pr64661-1.c: New.
	* gcc.taget/sh/pr64661-2.c: New.
	* gcc.taget/sh/pr64661-3.c: New.
	* gcc.taget/sh/pr64661-4.c: New.

From-SVN: r220594

3 files changed, 560 insertions, 240 deletions

diff --git a/gcc/config/sh/constraints.md b/gcc/config/sh/constraints.md
index 5ef3db6..0ac5db5 100644
--- a/gcc/config/sh/constraints.md
+++ b/gcc/config/sh/constraints.md
@@ -18,6 +18,9 @@
 ;; <http://www.gnu.org/licenses/>.
 
 ;; Overview of uppercase letter constraints:
+;; Axx: atomic memory operand constraints
+;;  Ara: Same as Sra but disallows r15
+;;  Add: Same as Sdd but disallows r15
 ;; Bxx: miscellaneous constraints
 ;;  Bsc: SCRATCH - for the scratch register in movsi_ie in the
 ;;       fldi0 / fldi0 cases
@@ -322,3 +325,19 @@
   (and (match_test "MEM_P (op)")
        (match_test "REG_P (XEXP (op, 0))")))
 
+(define_memory_constraint "Ara"
+  "A memory reference that uses simple register addressing suitable for
+   gusa atomic operations."
+  (and (match_code "mem")
+       (match_code "reg" "0")
+       (match_test "REGNO (XEXP (op, 0)) != SP_REG")))
+
+(define_memory_constraint "Add"
+  "A memory reference that uses displacement addressing suitable for
+   gusa atomic operations."
+  (and (match_code "mem")
+       (match_test "GET_MODE (op) == SImode")
+       (match_code "plus" "0")
+       (match_code "reg" "00")
+       (match_code "const_int" "01")
+       (match_test "REGNO (XEXP (XEXP (op, 0), 0)) != SP_REG")))
diff --git a/gcc/config/sh/sh-protos.h b/gcc/config/sh/sh-protos.h
index 3f72b07..1d8ba1d 100644
--- a/gcc/config/sh/sh-protos.h
+++ b/gcc/config/sh/sh-protos.h
@@ -68,25 +68,25 @@ extern const sh_atomic_model& selected_atomic_model (void);
 
 /* Shortcuts to check the currently selected atomic model.  */
 #define TARGET_ATOMIC_ANY \
-  selected_atomic_model ().type != sh_atomic_model::none
+  (selected_atomic_model ().type != sh_atomic_model::none)
 
 #define TARGET_ATOMIC_STRICT \
-  selected_atomic_model ().strict
+  (selected_atomic_model ().strict)
 
 #define TARGET_ATOMIC_SOFT_GUSA \
-  selected_atomic_model ().type == sh_atomic_model::soft_gusa
+  (selected_atomic_model ().type == sh_atomic_model::soft_gusa)
 
 #define TARGET_ATOMIC_HARD_LLCS \
-  selected_atomic_model ().type == sh_atomic_model::hard_llcs
+  (selected_atomic_model ().type == sh_atomic_model::hard_llcs)
 
 #define TARGET_ATOMIC_SOFT_TCB \
-  selected_atomic_model ().type == sh_atomic_model::soft_tcb
+  (selected_atomic_model ().type == sh_atomic_model::soft_tcb)
 
 #define TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX \
   GEN_INT (selected_atomic_model ().tcb_gbr_offset)
 
 #define TARGET_ATOMIC_SOFT_IMASK \
-  selected_atomic_model ().type == sh_atomic_model::soft_imask
+  (selected_atomic_model ().type == sh_atomic_model::soft_imask)
 
 #ifdef RTX_CODE
 extern rtx sh_fsca_sf2int (void);
diff --git a/gcc/config/sh/sync.md b/gcc/config/sh/sync.md
index d1413c5..6e890a7 100644
--- a/gcc/config/sh/sync.md
+++ b/gcc/config/sh/sync.md
@@ -209,10 +209,21 @@
 		      (match_test "TARGET_ATOMIC_ANY && TARGET_SH4A
 				   && !TARGET_ATOMIC_STRICT"))))))
 
+;; Displacement addressing can be used for all SImode atomic patterns, except
+;; llcs.
+(define_predicate "atomic_mem_operand_0"
+  (and (match_code "mem")
+       (ior (match_operand 0 "simple_mem_operand")
+	    (and (match_test "mode == SImode")
+		 (and (match_test "!TARGET_ATOMIC_HARD_LLCS")
+		      (match_test "!TARGET_SH4A || TARGET_ATOMIC_STRICT"))
+		 (match_operand 0 "displacement_mem_operand")
+		 (match_operand 0 "short_displacement_mem_operand")))))
+
 (define_expand "atomic_compare_and_swap<mode>"
   [(match_operand:SI 0 "arith_reg_dest")		;; bool success output
    (match_operand:QIHISI 1 "arith_reg_dest")		;; oldval output
-   (match_operand:QIHISI 2 "memory_operand")		;; memory
+   (match_operand:QIHISI 2 "atomic_mem_operand_0")	;; memory
    (match_operand:QIHISI 3 "atomic_arith_operand_0")	;; expected input
    (match_operand:QIHISI 4 "atomic_arith_operand_0")	;; newval input
    (match_operand:SI 5 "const_int_operand")		;; is_weak
@@ -220,7 +231,7 @@
    (match_operand:SI 7 "const_int_operand")]		;; failure model
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[2], 0));
+  rtx mem = operands[2];
   rtx old_val = gen_lowpart (SImode, operands[1]);
   rtx exp_val = operands[3];
   rtx new_val = operands[4];
@@ -228,16 +239,16 @@
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_compare_and_swap<mode>_hard (old_val, addr,
+    atomic_insn = gen_atomic_compare_and_swap<mode>_hard (old_val, mem,
 							  exp_val, new_val);
   else if (TARGET_ATOMIC_SOFT_GUSA)
-    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_gusa (old_val, addr,
+    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_gusa (old_val, mem,
 		      exp_val, new_val);
   else if (TARGET_ATOMIC_SOFT_TCB)
-    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_tcb (old_val, addr,
+    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_tcb (old_val, mem,
 		      exp_val, new_val, TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
-    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_imask (old_val, addr,
+    atomic_insn = gen_atomic_compare_and_swap<mode>_soft_imask (old_val, mem,
 		      exp_val, new_val);
   else
     FAIL;
@@ -254,14 +265,14 @@
   DONE;
 })
 
-(define_insn "atomic_compare_and_swapsi_hard"
+(define_insn_and_split "atomic_compare_and_swapsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
 	(unspec_volatile:SI
-	  [(mem:SI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  [(match_operand:SI 1 "atomic_mem_operand_0" "=Sra")
 	   (match_operand:SI 2 "arith_operand" "rI08")
 	   (match_operand:SI 3 "arith_operand" "rI08")]
 	  UNSPECV_CMPXCHG_1))
-   (set (mem:SI (match_dup 1))
+   (set (match_dup 1)
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_2))
    (set (reg:SI T_REG)
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_3))
@@ -269,18 +280,66 @@
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,r0"		"\n"
+  return "\r0:	movli.l	%1,r0"		"\n"
 	 "	cmp/eq	%2,r0"		"\n"
 	 "	bf{.|/}s	0f"	"\n"
 	 "	mov	r0,%0"		"\n"
 	 "	mov	%3,r0"		"\n"
-	 "	movco.l	r0,@%1"		"\n"
+	 "	movco.l	r0,%1"		"\n"
 	 "	bf	0b"		"\n"
 	 "0:";
 }
+  "&& can_create_pseudo_p () && !satisfies_constraint_I08 (operands[2])"
+  [(const_int 0)]
+{
+  /* FIXME: Sometimes the 'expected value' operand is not propagated as
+     immediate value.  See PR 64974.  */
+  set_of_reg op2 = sh_find_set_of_reg (operands[2], curr_insn,
+				       prev_nonnote_insn_bb);
+  if (op2.set_src != NULL && satisfies_constraint_I08 (op2.set_src))
+    {
+      rtx* r = &XVECEXP (XEXP (XVECEXP (PATTERN (curr_insn), 0, 0), 1), 0, 1);
+      validate_change (curr_insn, r, op2.set_src, false);
+      DONE;
+    }
+  else
+    FAIL;
+}
   [(set_attr "length" "14")])
 
-(define_insn "atomic_compare_and_swap<mode>_hard"
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
+(define_insn_and_split "atomic_compare_and_swap<mode>_hard"
+  [(set (match_operand:SI 0 "arith_reg_dest")
+	(unspec_volatile:SI
+	  [(match_operand:QIHI 1 "atomic_mem_operand_0")
+	   (match_operand:QIHI 2 "arith_reg_operand")
+	   (match_operand:QIHI 3 "arith_reg_operand")]
+	  UNSPECV_CMPXCHG_1))
+   (set (match_dup 1)
+	(unspec_volatile:QIHI [(const_int 0)] UNSPECV_CMPXCHG_2))
+   (set (reg:SI T_REG)
+	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_3))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  rtx i = gen_atomic_compare_and_swap<mode>_hard_1 (
+		operands[0], XEXP (operands[1], 0), operands[2], operands[3]);
+
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  XVECEXP (XEXP (XVECEXP (i, 0, 0), 1), 0, 0) = operands[1];
+  XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+  emit_insn (i);
+})
+
+(define_insn "atomic_compare_and_swap<mode>_hard_1"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
 	(unspec_volatile:SI
 	  [(mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r"))
@@ -319,11 +378,11 @@
 (define_insn "atomic_compare_and_swap<mode>_soft_gusa"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&u")
 	(unspec_volatile:SI
-	  [(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u"))
+	  [(match_operand:QIHISI 1 "atomic_mem_operand_0" "=AraAdd")
 	   (match_operand:QIHISI 2 "arith_reg_operand" "u")
 	   (match_operand:QIHISI 3 "arith_reg_operand" "u")]
 	  UNSPECV_CMPXCHG_1))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec_volatile:QIHISI [(const_int 0)] UNSPECV_CMPXCHG_2))
    (set (reg:SI T_REG)
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_3))
@@ -337,10 +396,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	r15,r1"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	cmp/eq	%0,%4"			"\n"
 	 "	bf	1f"			"\n"
-	 "	mov.<bwl>	%3,@%1"		"\n"
+	 "	mov.<bwl>	%3,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   [(set_attr "length" "20")])
@@ -348,11 +407,11 @@
 (define_insn "atomic_compare_and_swap<mode>_soft_tcb"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
 	(unspec_volatile:SI
-	  [(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  [(match_operand:QIHISI 1 "atomic_mem_operand_0" "=SraSdd")
 	   (match_operand:QIHISI 2 "arith_reg_operand" "r")
 	   (match_operand:QIHISI 3 "arith_reg_operand" "r")]
 	  UNSPECV_CMPXCHG_1))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec_volatile:QIHISI [(const_int 0)] UNSPECV_CMPXCHG_2))
    (set (reg:SI T_REG)
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_3))
@@ -367,11 +426,11 @@
 	 "	<i124extend_insn>	%2,%5"	"\n"
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	mov.l	r0,@(%O4,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	mov	#0,r0"			"\n"
 	 "	cmp/eq	%0,%5"			"\n"
 	 "	bf	1f"			"\n"
-	 "	mov.<bwl>	%3,@%1"		"\n"
+	 "	mov.<bwl>	%3,%1"		"\n"
 	 "1:	mov.l	r0,@(%O4,gbr)";
 }
   [(set_attr "length" "22")])
@@ -379,11 +438,11 @@
 (define_insn "atomic_compare_and_swap<mode>_soft_imask"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&z")
 	(unspec_volatile:SI
-	  [(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  [(match_operand:QIHISI 1 "atomic_mem_operand_0" "=SraSdd")
 	   (match_operand:QIHISI 2 "arith_reg_operand" "r")
 	   (match_operand:QIHISI 3 "arith_reg_operand" "r")]
 	  UNSPECV_CMPXCHG_1))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec_volatile:QIHISI [(const_int 0)] UNSPECV_CMPXCHG_2))
    (set (reg:SI T_REG)
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_CMPXCHG_3))
@@ -402,10 +461,10 @@
 	   "	or	#0xF0,%0"		"\n"
 	   "	shlr	%5"			"\n"
 	   "	ldc	%0,sr"			"\n"
-	   "	mov.<bwl>	@%1,%0"		"\n"
+	   "	mov.<bwl>	%1,%0"		"\n"
 	   "	cmp/eq	%4,%0"			"\n"
 	   "	bf	1f"			"\n"
-	   "	mov.<bwl>	%3,@%1"		"\n"
+	   "	mov.<bwl>	%3,%1"		"\n"
 	   "1:	rotcl	%5"			"\n"
 	   "	ldc	%5,sr";
   else
@@ -414,11 +473,11 @@
 	   "	mov	%0,%5"			"\n"
 	   "	or	#0xF0,%0"		"\n"
 	   "	ldc	%0,sr"			"\n"
-	   "	mov.<bwl>	@%1,%0"		"\n"
+	   "	mov.<bwl>	%1,%0"		"\n"
 	   "	cmp/eq	%4,%0"			"\n"
 	   "	bst	#0,%5"			"\n"
 	   "	bf	1f"			"\n"
-	   "	mov.<bwl>	%3,@%1"		"\n"
+	   "	mov.<bwl>	%3,%1"		"\n"
 	   "1:	ldc	%5,sr";
 }
   [(set (attr "length") (if_then_else (match_test "!TARGET_SH2A")
@@ -430,25 +489,25 @@
 
 (define_expand "atomic_exchange<mode>"
   [(match_operand:QIHISI 0 "arith_reg_dest")		;; oldval output
-   (match_operand:QIHISI 1 "memory_operand")		;; memory
+   (match_operand:QIHISI 1 "atomic_mem_operand_0")	;; memory
    (match_operand:QIHISI 2 "atomic_arith_operand_0")	;; newval input
    (match_operand:SI 3 "const_int_operand")]		;; memory model
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[1], 0));
+  rtx mem = operands[1];
   rtx val = operands[2];
   rtx atomic_insn;
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_exchange<mode>_hard (operands[0], addr, val);
+    atomic_insn = gen_atomic_exchange<mode>_hard (operands[0], mem, val);
   else if (TARGET_ATOMIC_SOFT_GUSA)
-    atomic_insn = gen_atomic_exchange<mode>_soft_gusa (operands[0], addr, val);
+    atomic_insn = gen_atomic_exchange<mode>_soft_gusa (operands[0], mem, val);
   else if (TARGET_ATOMIC_SOFT_TCB)
-    atomic_insn = gen_atomic_exchange<mode>_soft_tcb (operands[0], addr, val,
+    atomic_insn = gen_atomic_exchange<mode>_soft_tcb (operands[0], mem, val,
 		      TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
-    atomic_insn = gen_atomic_exchange<mode>_soft_imask (operands[0], addr, val);
+    atomic_insn = gen_atomic_exchange<mode>_soft_imask (operands[0], mem, val);
   else
     FAIL;
 
@@ -465,8 +524,8 @@
 
 (define_insn "atomic_exchangesi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
-	(mem:SI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:SI (match_dup 1))
+	(match_operand:SI 1 "atomic_mem_operand_0" "=Sra"))
+   (set (match_dup 1)
 	(unspec:SI
 	  [(match_operand:SI 2 "arith_operand" "rI08")] UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))
@@ -474,15 +533,43 @@
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,r0"		"\n"
+  return "\r0:	movli.l	%1,r0"		"\n"
 	 "	mov	r0,%0"		"\n"
 	 "	mov	%2,r0"		"\n"
-	 "	movco.l r0,@%1"		"\n"
+	 "	movco.l r0,%1"		"\n"
 	 "	bf	0b";
 }
   [(set_attr "length" "10")])
 
-(define_insn "atomic_exchange<mode>_hard"
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
+(define_insn_and_split "atomic_exchange<mode>_hard"
+  [(set (match_operand:QIHI 0 "arith_reg_dest")
+	(match_operand:QIHI 1 "atomic_mem_operand_0"))
+   (set (match_dup 1)
+	(unspec:QIHI
+	  [(match_operand:QIHI 2 "arith_reg_operand")] UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  rtx i = gen_atomic_exchange<mode>_hard_1 (operands[0], XEXP (operands[1], 0),
+					    operands[2]);
+
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  XEXP (XVECEXP (i, 0, 0), 1) = operands[1];
+  XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+  emit_insn (i);
+})
+
+(define_insn "atomic_exchange<mode>_hard_1"
   [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
 	(mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r")))
    (set (mem:QIHI (match_dup 1))
@@ -511,8 +598,8 @@
 
 (define_insn "atomic_exchange<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_0" "=AraAdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(match_operand:QIHISI 2 "arith_reg_operand" "u")] UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
@@ -523,16 +610,16 @@
 	 "	.align 2"			"\n"
 	 "	mov	r15,r1"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
-	 "	mov.<bwl>	%2,@%1"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
+	 "	mov.<bwl>	%2,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   [(set_attr "length" "14")])
 
 (define_insn "atomic_exchange<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_0" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(match_operand:QIHISI 2 "arith_reg_operand" "r")] UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
@@ -544,17 +631,17 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	mov	#0,r0"			"\n"
-	 "	mov.<bwl>	%2,@%1"		"\n"
+	 "	mov.<bwl>	%2,%1"		"\n"
 	 "1:	mov.l	r0,@(%O3,gbr)";
 }
   [(set_attr "length" "16")])
 
 (define_insn "atomic_exchange<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&z")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_0" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(match_operand:QIHISI 2 "arith_reg_operand" "r")] UNSPEC_ATOMIC))
    (clobber (match_scratch:SI 3 "=&r"))]
@@ -564,8 +651,8 @@
 	 "	mov	%0,%3"			"\n"
 	 "	or	#0xF0,%0"		"\n"
 	 "	ldc	%0,sr"			"\n"
-	 "	mov.<bwl>	@%1,%0"		"\n"
-	 "	mov.<bwl>	%2,@%1"		"\n"
+	 "	mov.<bwl>	%1,%0"		"\n"
+	 "	mov.<bwl>	%2,%1"		"\n"
 	 "	ldc	%3,sr";
 }
   [(set_attr "length" "14")])
@@ -610,9 +697,32 @@
 (define_code_attr fetchop_constraint_1_imask
   [(plus "rI08") (minus "r") (ior "rK08") (xor "rK08") (and "rK08")])
 
+;; Displacement addressing mode (incl. GBR relative) can be used by tcb and
+;; imask atomic patterns in any mode, since all the patterns use R0 as the
+;; register operand for memory loads/stores.  gusa and llcs patterns can only
+;; use displacement addressing for SImode.
+(define_predicate "atomic_mem_operand_1"
+  (and (match_code "mem")
+       (ior (match_operand 0 "simple_mem_operand")
+	    (and (match_test "mode == SImode")
+		 (match_test "TARGET_ATOMIC_SOFT_GUSA
+			      && (!TARGET_SH4A || TARGET_ATOMIC_STRICT)")
+		 (match_operand 0 "displacement_mem_operand")
+		 (match_operand 0 "short_displacement_mem_operand"))
+	    (and (ior (match_test "(TARGET_ATOMIC_SOFT_TCB
+				    || TARGET_ATOMIC_SOFT_IMASK)
+				   && (!TARGET_SH4A || TARGET_ATOMIC_STRICT)")
+		      (match_test "(TARGET_ATOMIC_SOFT_TCB
+				    || TARGET_ATOMIC_SOFT_IMASK)
+				   && TARGET_SH4A && !TARGET_ATOMIC_STRICT
+				   && mode != SImode"))
+		 (ior (and (match_operand 0 "displacement_mem_operand")
+			   (match_operand 0 "short_displacement_mem_operand"))
+		      (match_operand 0 "gbr_address_mem"))))))
+
 (define_expand "atomic_fetch_<fetchop_name><mode>"
   [(set (match_operand:QIHISI 0 "arith_reg_dest")
-	(match_operand:QIHISI 1 "memory_operand"))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1"))
    (set (match_dup 1)
 	(unspec:QIHISI
 	  [(FETCHOP:QIHISI (match_dup 1)
@@ -621,22 +731,22 @@
    (match_operand:SI 3 "const_int_operand")]
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[1], 0));
+  rtx mem = operands[1];
   rtx atomic_insn;
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_fetch_<fetchop_name><mode>_hard (operands[0], addr,
+    atomic_insn = gen_atomic_fetch_<fetchop_name><mode>_hard (operands[0], mem,
 							      operands[2]);
   else if (TARGET_ATOMIC_SOFT_GUSA)
     atomic_insn = gen_atomic_fetch_<fetchop_name><mode>_soft_gusa (operands[0],
-		      addr, operands[2]);
+		      mem, operands[2]);
   else if (TARGET_ATOMIC_SOFT_TCB)
     atomic_insn = gen_atomic_fetch_<fetchop_name><mode>_soft_tcb (operands[0],
-		      addr, operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
+		      mem, operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
     atomic_insn = gen_atomic_fetch_<fetchop_name><mode>_soft_imask (operands[0],
-		      addr, operands[2]);
+		      mem, operands[2]);
   else
     FAIL;
 
@@ -653,10 +763,10 @@
 
 (define_insn_and_split "atomic_fetch_<fetchop_name>si_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
-	(mem:SI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:SI (match_dup 1))
+	(match_operand:SI 1 "atomic_mem_operand_1" "=Sra"))
+   (set (match_dup 1)
 	(unspec:SI
-	  [(FETCHOP:SI (mem:SI (match_dup 1))
+	  [(FETCHOP:SI (match_dup 1)
 		       (match_operand:SI 2 "<fetchop_predicate_1>"
 					   "<fetchop_constraint_1_llcs>"))]
 	  UNSPEC_ATOMIC))
@@ -665,10 +775,10 @@
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,r0"		"\n"
+  return "\r0:	movli.l	%1,r0"		"\n"
 	 "	mov	r0,%0"		"\n"
 	 "	<fetchop_name>	%2,r0"	"\n"
-	 "	movco.l	r0,@%1"		"\n"
+	 "	movco.l	r0,%1"		"\n"
 	 "	bf	0b";
 }
   "&& can_create_pseudo_p () && optimize
@@ -683,18 +793,18 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn_and_split "atomic_fetch_notsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
-	(mem:SI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:SI (match_dup 1))
-	(unspec:SI [(not:SI (mem:SI (match_dup 1)))] UNSPEC_ATOMIC))
+	(match_operand:SI 1 "atomic_mem_operand_1" "=Sra"))
+   (set (match_dup 1)
+	(unspec:SI [(not:SI (match_dup 1))] UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))
    (clobber (reg:SI R0_REG))]
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,r0"		"\n"
+  return "\r0:	movli.l	%1,r0"		"\n"
 	 "	mov	r0,%0"		"\n"
 	 "	not	r0,r0"		"\n"
-	 "	movco.l	r0,@%1"		"\n"
+	 "	movco.l	r0,%1"		"\n"
 	 "	bf	0b";
 }
   "&& can_create_pseudo_p () && optimize
@@ -705,7 +815,44 @@
 }
   [(set_attr "length" "10")])
 
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
 (define_insn_and_split "atomic_fetch_<fetchop_name><mode>_hard"
+  [(set (match_operand:QIHI 0 "arith_reg_dest")
+	(match_operand:QIHI 1 "atomic_mem_operand_1"))
+   (set (match_dup 1)
+	(unspec:QIHI
+	  [(FETCHOP:QIHI (match_dup 1)
+			 (match_operand:QIHI 2 "<fetchop_predicate_1>"))]
+	  UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  if (optimize
+      && sh_reg_dead_or_unused_after_insn (curr_insn, REGNO (operands[0])))
+    emit_insn (gen_atomic_<fetchop_name><mode>_hard (operands[1], operands[2]));
+  else
+    {
+      rtx i = gen_atomic_fetch_<fetchop_name><mode>_hard_1 (
+			operands[0], XEXP (operands[1], 0), operands[2]);
+
+      /* Replace the new mems in the new insn with the old mem to preserve
+	 aliasing info.  */
+      XEXP (XVECEXP (i, 0, 0), 1) = operands[1];
+      XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+      XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 1), 1), 0, 0), 0) = operands[1];
+      emit_insn (i);
+    }
+})
+
+(define_insn "atomic_fetch_<fetchop_name><mode>_hard_1"
   [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
 	(mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r")))
    (set (mem:QIHI (match_dup 1))
@@ -735,15 +882,36 @@
 	 "	movco.l	r0,@%3"			"\n"
 	 "	bf	0b";
 }
-  "&& can_create_pseudo_p () && optimize
-   && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
+  [(set_attr "length" "28")])
+
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
+(define_insn_and_split "atomic_<fetchop_name><mode>_hard"
+  [(set (match_operand:QIHI 0 "atomic_mem_operand_1")
+	(unspec:QIHI
+	  [(FETCHOP:QIHI (match_dup 0)
+			 (match_operand:QIHI 1 "<fetchop_predicate_1>"))]
+	  UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
   [(const_int 0)]
 {
-  emit_insn (gen_atomic_<fetchop_name><mode>_hard (operands[1], operands[2]));
-}
-  [(set_attr "length" "28")])
+  rtx i = gen_atomic_<fetchop_name><mode>_hard_1 (XEXP (operands[0], 0),
+						  operands[1]);
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  XEXP (XVECEXP (i, 0, 0), 0) = operands[0];
+  XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 0), 1), 0, 0), 0) = operands[0];
+  emit_insn (i);
+})
 
-(define_insn "atomic_<fetchop_name><mode>_hard"
+(define_insn "atomic_<fetchop_name><mode>_hard_1"
   [(set (mem:QIHI (match_operand:SI 0 "arith_reg_operand" "r"))
 	(unspec:QIHI
 	  [(FETCHOP:QIHI (mem:QIHI (match_dup 0))
@@ -802,7 +970,14 @@
    && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
   [(const_int 0)]
 {
-  emit_insn (gen_atomic_not<mode>_hard (operands[1]));
+  rtx i = gen_atomic_not<mode>_hard (operands[1]);
+
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  rtx m = XEXP (XVECEXP (PATTERN (curr_insn), 0, 0), 1);
+  XEXP (XVECEXP (i, 0, 0), 0) = m;
+  XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 0), 1), 0, 0), 0) = m;
+  emit_insn (i);
 }
   [(set_attr "length" "26")])
 
@@ -833,11 +1008,11 @@
 
 (define_insn_and_split "atomic_fetch_<fetchop_name><mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(FETCHOP:QIHISI
-		(mem:QIHISI (match_dup 1))
+		(match_dup 1)
 		(match_operand:QIHISI 2 "<fetchop_predicate_1>"
 					"<fetchop_constraint_1_gusa>"))]
 	  UNSPEC_ATOMIC))
@@ -850,10 +1025,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	r15,r1"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	mov	%0,%3"			"\n"
 	 "	<fetchop_name>	%2,%3"		"\n"
-	 "	mov.<bwl>	%3,@%1"		"\n"
+	 "	mov.<bwl>	%3,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   "&& can_create_pseudo_p () && optimize
@@ -868,9 +1043,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn_and_split "atomic_fetch_not<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u")))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd"))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (clobber (match_scratch:QIHISI 2 "=&u"))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
@@ -880,9 +1055,9 @@
 	 "	mov	r15,r1"			"\n"
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	not	%0,%2"			"\n"
-	 "	mov.<bwl>	%2,@%1"		"\n"
+	 "	mov.<bwl>	%2,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   "&& can_create_pseudo_p () && optimize
@@ -896,11 +1071,11 @@
 
 (define_insn_and_split "atomic_fetch_<fetchop_name><mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(FETCHOP:QIHISI
-		(mem:QIHISI (match_dup 1))
+		(match_dup 1)
 		(match_operand:QIHISI 2 "<fetchop_predicate_1>"
 					"<fetchop_constraint_1_tcb>"))]
 	  UNSPEC_ATOMIC))
@@ -913,10 +1088,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	mov.l	r0,@(%O3,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	<fetchop_name>	%2,r0"		"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)";
 }
@@ -930,10 +1105,10 @@
   [(set_attr "length" "20")])
 
 (define_insn "atomic_<fetchop_name><mode>_soft_tcb"
-  [(set (mem:QIHISI (match_operand:SI 0 "arith_reg_operand" "r"))
+  [(set (match_operand:QIHISI 0 "atomic_mem_operand_1" "=SraSdd")
 	(unspec:QIHISI
 	  [(FETCHOP:QIHISI
-		(mem:QIHISI (match_dup 0))
+		(match_dup 0)
 		(match_operand:QIHISI 1 "<fetchop_predicate_1>"
 					"<fetchop_constraint_1_tcb>"))]
 	  UNSPEC_ATOMIC))
@@ -946,9 +1121,9 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%0,r0"		"\n"
+	 "0:	mov.<bwl>	%0,r0"		"\n"
 	 "	<fetchop_name>	%1,r0"		"\n"
-	 "	mov.<bwl>	r0,@%0"		"\n"
+	 "	mov.<bwl>	r0,%0"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)";
 }
@@ -957,9 +1132,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn_and_split "atomic_fetch_not<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (use (match_operand:SI 2 "gbr_displacement"))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
@@ -969,10 +1144,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	mov.l	r0,@(%O2,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)";
 }
@@ -985,8 +1160,8 @@
   [(set_attr "length" "20")])
 
 (define_insn "atomic_not<mode>_soft_tcb"
-  [(set (mem:QIHISI (match_operand:SI 0 "arith_reg_operand" "r"))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 0)))] UNSPEC_ATOMIC))
+  [(set (match_operand:QIHISI 0 "atomic_mem_operand_1" "=SraSdd")
+	(unspec:QIHISI [(not:QIHISI (match_dup 0))] UNSPEC_ATOMIC))
    (use (match_operand:SI 1 "gbr_displacement"))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
@@ -996,9 +1171,9 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O1,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%0,r0"		"\n"
+	 "0:	mov.<bwl>	%0,r0"		"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%0"		"\n"
+	 "	mov.<bwl>	r0,%0"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O1,gbr)";
 }
@@ -1006,11 +1181,11 @@
 
 (define_insn_and_split "atomic_fetch_<fetchop_name><mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(FETCHOP:QIHISI
-		(mem:QIHISI (match_dup 1))
+		(match_dup 1)
 		(match_operand:QIHISI 2 "<fetchop_predicate_1>"
 					"<fetchop_constraint_1_imask>"))]
 	  UNSPEC_ATOMIC))
@@ -1022,10 +1197,10 @@
 	 "	mov	r0,%3"			"\n"
 	 "	or	#0xF0,r0"		"\n"
 	 "	ldc	r0,sr"			"\n"
-	 "	mov.<bwl>	@%1,r0"		"\n"
+	 "	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	<fetchop_name>	%2,r0"		"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "	ldc	%3,sr";
 }
   "&& can_create_pseudo_p () && optimize
@@ -1040,9 +1215,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn_and_split "atomic_fetch_not<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (match_scratch:QIHISI 2 "=&r"))]
   "TARGET_ATOMIC_SOFT_IMASK"
@@ -1051,10 +1226,10 @@
 	 "	mov	r0,%2"			"\n"
 	 "	or	#0xF0,r0"		"\n"
 	 "	ldc	r0,sr"			"\n"
-	 "	mov.<bwl>	@%1,r0"		"\n"
+	 "	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "	ldc	%2,sr";
 }
   "&& can_create_pseudo_p () && optimize
@@ -1068,7 +1243,7 @@
 
 (define_expand "atomic_fetch_nand<mode>"
   [(set (match_operand:QIHISI 0 "arith_reg_dest")
-	(match_operand:QIHISI 1 "memory_operand"))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1"))
    (set (match_dup 1)
 	(unspec:QIHISI
 	  [(not:QIHISI (and:QIHISI (match_dup 1)
@@ -1077,21 +1252,21 @@
    (match_operand:SI 3 "const_int_operand")]
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[1], 0));
+  rtx mem = operands[1];
   rtx atomic_insn;
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_fetch_nand<mode>_hard (operands[0], addr,
+    atomic_insn = gen_atomic_fetch_nand<mode>_hard (operands[0], mem,
 						    operands[2]);
   else if (TARGET_ATOMIC_SOFT_GUSA)
-    atomic_insn = gen_atomic_fetch_nand<mode>_soft_gusa (operands[0], addr,
+    atomic_insn = gen_atomic_fetch_nand<mode>_soft_gusa (operands[0], mem,
 							 operands[2]);
   else if (TARGET_ATOMIC_SOFT_TCB)
-    atomic_insn = gen_atomic_fetch_nand<mode>_soft_tcb (operands[0], addr,
+    atomic_insn = gen_atomic_fetch_nand<mode>_soft_tcb (operands[0], mem,
 		      operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
-    atomic_insn = gen_atomic_fetch_nand<mode>_soft_imask (operands[0], addr,
+    atomic_insn = gen_atomic_fetch_nand<mode>_soft_imask (operands[0], mem,
 							  operands[2]);
   else
     FAIL;
@@ -1109,10 +1284,10 @@
 
 (define_insn_and_split "atomic_fetch_nandsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&r")
-	(mem:SI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:SI (match_dup 1))
+	(match_operand:SI 1 "atomic_mem_operand_1" "=Sra"))
+   (set (match_dup 1)
 	(unspec:SI
-	  [(not:SI (and:SI (mem:SI (match_dup 1))
+	  [(not:SI (and:SI (match_dup 1)
 		   (match_operand:SI 2 "logical_operand" "rK08")))]
 	  UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))
@@ -1120,11 +1295,11 @@
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,r0"		"\n"
+  return "\r0:	movli.l	%1,r0"		"\n"
 	 "	mov	r0,%0"		"\n"
 	 "	and	%2,r0"		"\n"
 	 "	not	r0,r0"		"\n"
-	 "	movco.l	r0,@%1"		"\n"
+	 "	movco.l	r0,%1"		"\n"
 	 "	bf	0b";
 }
   "&& can_create_pseudo_p () && optimize
@@ -1136,7 +1311,45 @@
 }
   [(set_attr "length" "12")])
 
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
 (define_insn_and_split "atomic_fetch_nand<mode>_hard"
+  [(set (match_operand:QIHI 0 "arith_reg_dest")
+	(match_operand:QIHI 1 "atomic_mem_operand_1"))
+   (set (match_dup 1)
+	(unspec:QIHI
+	  [(not:QIHI (and:QIHI (match_dup 1)
+		     (match_operand:QIHI 2 "logical_operand" "rK08")))]
+	  UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  if (optimize
+      && sh_reg_dead_or_unused_after_insn (curr_insn, REGNO (operands[0])))
+    emit_insn (gen_atomic_nand<mode>_hard (operands[1], operands[2]));
+  else
+    {
+      rtx i = gen_atomic_fetch_nand<mode>_hard_1 (
+			operands[0], XEXP (operands[1], 0), operands[2]);
+
+      /* Replace the new mems in the new insn with the old mem to preserve
+	 aliasing info.  */
+      XEXP (XVECEXP (i, 0, 0), 1) = operands[1];
+      XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+      XEXP (XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 1), 1), 0, 0), 0),
+	    0) = operands[1];
+      emit_insn (i);
+    }
+})
+
+(define_insn "atomic_fetch_nand<mode>_hard_1"
   [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
 	(mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r")))
    (set (mem:QIHI (match_dup 1))
@@ -1166,15 +1379,36 @@
 	 "	movco.l	r0,@%3"			"\n"
 	 "	bf	0b";
 }
-  "&& can_create_pseudo_p () && optimize
-   && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
+  [(set_attr "length" "30")])
+
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
+(define_insn_and_split "atomic_nand<mode>_hard"
+  [(set (match_operand:QIHI 0 "atomic_mem_operand_1")
+	(unspec:QIHI
+	  [(not:QIHI (and:QIHI (match_dup 0)
+			       (match_operand:QIHI 1 "logical_operand")))]
+	  UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
   [(const_int 0)]
 {
-  emit_insn (gen_atomic_nand<mode>_hard (operands[1], operands[2]));
-}
-  [(set_attr "length" "30")])
+  rtx i = gen_atomic_nand<mode>_hard_1 (XEXP (operands[0], 0), operands[1]);
 
-(define_insn "atomic_nand<mode>_hard"
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  XEXP (XVECEXP (i, 0, 0), 0) = operands[0];
+  XEXP (XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 0), 1), 0, 0), 0), 0) = operands[0];
+  emit_insn (i);
+})
+
+(define_insn "atomic_nand<mode>_hard_1"
   [(set (mem:QIHI (match_operand:SI 0 "arith_reg_operand" "r"))
 	(unspec:QIHI
 	  [(not:QIHI (and:QIHI (mem:QIHI (match_dup 0))
@@ -1205,11 +1439,11 @@
 
 (define_insn_and_split "atomic_fetch_nand<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(not:QIHISI
-	     (and:QIHISI (mem:QIHISI (match_dup 1))
+	     (and:QIHISI (match_dup 1)
 			 (match_operand:QIHISI 2 "arith_reg_operand" "u")))]
 	  UNSPEC_ATOMIC))
    (clobber (match_scratch:QIHISI 3 "=&u"))
@@ -1221,11 +1455,11 @@
 	 "	mov	r15,r1"			"\n"
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	mov	%2,%3"			"\n"
 	 "	and	%0,%3"			"\n"
 	 "	not	%3,%3"			"\n"
-	 "	mov.<bwl>	%3,@%1"		"\n"
+	 "	mov.<bwl>	%3,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   "&& can_create_pseudo_p () && optimize
@@ -1239,11 +1473,11 @@
 
 (define_insn_and_split "atomic_fetch_nand<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(not:QIHISI
-	     (and:QIHISI (mem:QIHISI (match_dup 1))
+	     (and:QIHISI (match_dup 1)
 			 (match_operand:QIHISI 2 "logical_operand" "rK08")))]
 	  UNSPEC_ATOMIC))
    (use (match_operand:SI 3 "gbr_displacement"))
@@ -1255,11 +1489,11 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	and	%2,r0"			"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)";
 }
@@ -1273,10 +1507,10 @@
   [(set_attr "length" "22")])
 
 (define_insn "atomic_nand<mode>_soft_tcb"
-  [(set (mem:QIHISI (match_operand:SI 0 "arith_reg_operand" "r"))
+  [(set (match_operand:QIHISI 0 "atomic_mem_operand_1" "=SraSdd")
 	(unspec:QIHISI
 	  [(not:QIHISI
-	     (and:QIHISI (mem:QIHISI (match_dup 0))
+	     (and:QIHISI (match_dup 0)
 			 (match_operand:QIHISI 1 "logical_operand" "rK08")))]
 	  UNSPEC_ATOMIC))
    (use (match_operand:SI 2 "gbr_displacement"))
@@ -1288,10 +1522,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	mov.l	r0,@(%O2,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%0,r0"		"\n"
+	 "0:	mov.<bwl>	%0,r0"		"\n"
 	 "	and	%1,r0"			"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%0"		"\n"
+	 "	mov.<bwl>	r0,%0"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)";
 }
@@ -1299,11 +1533,11 @@
 
 (define_insn_and_split "atomic_fetch_nand<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r")))
-   (set (mem:QIHISI (match_dup 1))
+	(match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd"))
+   (set (match_dup 1)
 	(unspec:QIHISI
 	  [(not:QIHISI
-	     (and:QIHISI (mem:QIHISI (match_dup 1))
+	     (and:QIHISI (match_dup 1)
 			 (match_operand:QIHISI 2 "logical_operand" "rK08")))]
 	  UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
@@ -1314,11 +1548,11 @@
 	 "	mov	r0,%3"			"\n"
 	 "	or	#0xF0,r0"		"\n"
 	 "	ldc	r0,sr"			"\n"
-	 "	mov.<bwl>	@%1,r0"		"\n"
+	 "	mov.<bwl>	%1,r0"		"\n"
 	 "	mov	r0,%0"			"\n"
 	 "	and	%2,r0"			"\n"
 	 "	not	r0,r0"			"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "	ldc	%3,sr";
 }
   "&& can_create_pseudo_p () && optimize
@@ -1336,7 +1570,7 @@
 (define_expand "atomic_<fetchop_name>_fetch<mode>"
   [(set (match_operand:QIHISI 0 "arith_reg_dest")
 	(FETCHOP:QIHISI
-	  (match_operand:QIHISI 1 "memory_operand")
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1")
 	  (match_operand:QIHISI 2 "<fetchop_predicate_1>")))
    (set (match_dup 1)
 	(unspec:QIHISI
@@ -1345,22 +1579,22 @@
    (match_operand:SI 3 "const_int_operand" "")]
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[1], 0));
+  rtx mem = operands[1];
   rtx atomic_insn;
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_<fetchop_name>_fetch<mode>_hard (operands[0], addr,
+    atomic_insn = gen_atomic_<fetchop_name>_fetch<mode>_hard (operands[0], mem,
 							      operands[2]);
   else if (TARGET_ATOMIC_SOFT_GUSA)
     atomic_insn = gen_atomic_<fetchop_name>_fetch<mode>_soft_gusa (operands[0],
-		      addr, operands[2]);
+		      mem, operands[2]);
   else if (TARGET_ATOMIC_SOFT_TCB)
     atomic_insn = gen_atomic_<fetchop_name>_fetch<mode>_soft_tcb (operands[0],
-		      addr, operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
+		      mem, operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
     atomic_insn = gen_atomic_<fetchop_name>_fetch<mode>_soft_imask (operands[0],
-		      addr, operands[2]);
+		      mem, operands[2]);
   else
     FAIL;
 
@@ -1378,20 +1612,20 @@
 (define_insn "atomic_<fetchop_name>_fetchsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&z")
 	(FETCHOP:SI
-	  (mem:SI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  (match_operand:SI 1 "atomic_mem_operand_1" "=Sra")
 	  (match_operand:SI 2 "<fetchop_predicate_1>"
 			      "<fetchop_constraint_1_llcs>")))
-   (set (mem:SI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:SI
-	  [(FETCHOP:SI (mem:SI (match_dup 1)) (match_dup 2))]
+	  [(FETCHOP:SI (match_dup 1) (match_dup 2))]
 	  UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))]
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,%0"		"\n"
+  return "\r0:	movli.l	%1,%0"		"\n"
 	 "	<fetchop_name>	%2,%0"	"\n"
-	 "	movco.l	%0,@%1"		"\n"
+	 "	movco.l	%0,%1"		"\n"
 	 "	bf	0b";
 }
   [(set_attr "length" "8")])
@@ -1399,22 +1633,57 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn "atomic_not_fetchsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&z")
-	(not:SI (mem:SI (match_operand:SI 1 "arith_reg_operand" "r"))))
-   (set (mem:SI (match_dup 1))
-	(unspec:SI [(not:SI (mem:SI (match_dup 1)))] UNSPEC_ATOMIC))
+	(not:SI (match_operand:SI 1 "atomic_mem_operand_1" "=Sra")))
+   (set (match_dup 1)
+	(unspec:SI [(not:SI (match_dup 1))] UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))]
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,%0"		"\n"
+  return "\r0:	movli.l	%1,%0"		"\n"
 	 "	not	%0,%0"		"\n"
-	 "	movco.l	%0,@%1"		"\n"
+	 "	movco.l	%0,%1"		"\n"
 	 "	bf	0b";
 }
   [(set_attr "length" "8")])
 
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
 (define_insn_and_split "atomic_<fetchop_name>_fetch<mode>_hard"
   [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
+	(FETCHOP:QIHI (match_operand:QIHI 1 "atomic_mem_operand_1")
+		      (match_operand:QIHI 2 "<fetchop_predicate_1>")))
+   (set (match_dup 1) (unspec:QIHI [(FETCHOP:QIHI (match_dup 1) (match_dup 2))]
+				   UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  if (optimize
+      && sh_reg_dead_or_unused_after_insn (curr_insn, REGNO (operands[0])))
+    emit_insn (gen_atomic_<fetchop_name><mode>_hard (operands[1], operands[2]));
+  else
+    {
+      rtx i = gen_atomic_<fetchop_name>_fetch<mode>_hard_1 (
+			operands[0], XEXP (operands[1], 0), operands[2]);
+
+      /* Replace the new mems in the new insn with the old mem to preserve
+	 aliasing info.  */
+      XEXP (XEXP (XVECEXP (i, 0, 0), 1), 0) = operands[1];
+      XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+      XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 1), 1), 0, 0), 0) = operands[1];
+      emit_insn (i);
+    }
+})
+
+(define_insn "atomic_<fetchop_name>_fetch<mode>_hard_1"
+  [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
 	(FETCHOP:QIHI
 	  (mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r"))
 	  (match_operand:QIHI 2 "<fetchop_predicate_1>"
@@ -1444,12 +1713,6 @@
 	 "	movco.l	r0,@%3"			"\n"
 	 "	bf	0b";
 }
-  "&& can_create_pseudo_p () && optimize
-   && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
-  [(const_int 0)]
-{
-  emit_insn (gen_atomic_<fetchop_name><mode>_hard (operands[1], operands[2]));
-}
   [(set_attr "length" "28")])
 
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
@@ -1483,19 +1746,26 @@
    && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
   [(const_int 0)]
 {
-  emit_insn (gen_atomic_not<mode>_hard (operands[1]));
+  rtx i = gen_atomic_not<mode>_hard (operands[1]);
+
+  /* Replace the new mems in the new insn with the old mem to preserve
+     aliasing info.  */
+  rtx m = XEXP (XEXP (XVECEXP (PATTERN (curr_insn), 0, 0), 1), 0);
+  XEXP (XVECEXP (i, 0, 0), 0) = m;
+  XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 0), 1), 0, 0), 0) = m;
+  emit_insn (i);
 }
   [(set_attr "length" "28")])
 
 (define_insn "atomic_<fetchop_name>_fetch<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
 	(FETCHOP:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd")
 	  (match_operand:QIHISI 2 "<fetchop_predicate_1>"
 				  "<fetchop_constraint_1_gusa>")))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(FETCHOP:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2))]
+	  [(FETCHOP:QIHISI (match_dup 1) (match_dup 2))]
 	  UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
@@ -1505,9 +1775,9 @@
 	 "	mov	r15,r1"			"\n"
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	<fetchop_name>	%2,%0"		"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   [(set_attr "length" "16")])
@@ -1515,9 +1785,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn "atomic_not_fetch<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
-	(not:QIHISI (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u"))))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(not:QIHISI (match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd")))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
   "TARGET_ATOMIC_SOFT_GUSA"
@@ -1526,9 +1796,9 @@
 	 "	mov	r15,r1"			"\n"
 	 "	.align 2"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
-	 "	not	%0,%0"		"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
+	 "	not	%0,%0"			"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   [(set_attr "length" "16")])
@@ -1536,12 +1806,12 @@
 (define_insn_and_split "atomic_<fetchop_name>_fetch<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
 	(FETCHOP:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")
 	  (match_operand:QIHISI 2 "<fetchop_predicate_1>"
 				  "<fetchop_constraint_1_tcb>")))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(FETCHOP:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2))]
+	  [(FETCHOP:QIHISI (match_dup 1) (match_dup 2))]
 	  UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))
@@ -1552,9 +1822,9 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
 	 "	<fetchop_name>	%2,r0"		"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	r0,%0"			"\n"
 	 "	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)";
@@ -1571,9 +1841,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn_and_split "atomic_not_fetch<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
-	(not:QIHISI (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(not:QIHISI (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))
    (use (match_operand:SI 2 "gbr_displacement"))]
@@ -1583,9 +1853,9 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
-	 "	not	r0,r0"		"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
+	 "	not	r0,r0"			"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	r0,%0"			"\n"
 	 "	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O2,gbr)";
@@ -1601,12 +1871,12 @@
 (define_insn "atomic_<fetchop_name>_fetch<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&z")
 	(FETCHOP:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")
 	  (match_operand:QIHISI 2 "<fetchop_predicate_1>"
 				  "<fetchop_constraint_1_imask>")))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(FETCHOP:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2))]
+	  [(FETCHOP:QIHISI (match_dup 1) (match_dup 2))]
 	  UNSPEC_ATOMIC))
    (clobber (match_scratch:SI 3 "=&r"))]
   "TARGET_ATOMIC_SOFT_IMASK"
@@ -1615,9 +1885,9 @@
 	 "	mov	%0,%3"			"\n"
 	 "	or	#0xF0,%0"		"\n"
 	 "	ldc	%0,sr"			"\n"
-	 "	mov.<bwl>	@%1,%0"		"\n"
+	 "	mov.<bwl>	%1,%0"		"\n"
 	 "	<fetchop_name>	%2,%0"		"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "	ldc	%3,sr";
 }
   [(set_attr "length" "16")])
@@ -1625,9 +1895,9 @@
 ;; Combine pattern for xor (val, -1) / nand (val, -1).
 (define_insn "atomic_not_fetch<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&z")
-	(not:QIHISI (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))))
-   (set (mem:QIHISI (match_dup 1))
-	(unspec:QIHISI [(not:QIHISI (mem:QIHISI (match_dup 1)))] UNSPEC_ATOMIC))
+	(not:QIHISI (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")))
+   (set (match_dup 1)
+	(unspec:QIHISI [(not:QIHISI (match_dup 1))] UNSPEC_ATOMIC))
    (clobber (match_scratch:SI 2 "=&r"))]
   "TARGET_ATOMIC_SOFT_IMASK"
 {
@@ -1635,9 +1905,9 @@
 	 "	mov	%0,%2"			"\n"
 	 "	or	#0xF0,%0"		"\n"
 	 "	ldc	%0,sr"			"\n"
-	 "	mov.<bwl>	@%1,%0"		"\n"
-	 "	not	%0,%0"		"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "	mov.<bwl>	%1,%0"		"\n"
+	 "	not	%0,%0"			"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "	ldc	%2,sr";
 }
   [(set_attr "length" "16")])
@@ -1645,7 +1915,7 @@
 (define_expand "atomic_nand_fetch<mode>"
   [(set (match_operand:QIHISI 0 "arith_reg_dest")
 	(not:QIHISI (and:QIHISI
-	  (match_operand:QIHISI 1 "memory_operand")
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1")
 	  (match_operand:QIHISI 2 "atomic_logical_operand_1"))))
    (set (match_dup 1)
 	(unspec:QIHISI
@@ -1654,21 +1924,21 @@
    (match_operand:SI 3 "const_int_operand")]
   "TARGET_ATOMIC_ANY"
 {
-  rtx addr = force_reg (Pmode, XEXP (operands[1], 0));
+  rtx mem = operands[1];
   rtx atomic_insn;
 
   if (TARGET_ATOMIC_HARD_LLCS
       || (TARGET_SH4A && <MODE>mode == SImode && !TARGET_ATOMIC_STRICT))
-    atomic_insn = gen_atomic_nand_fetch<mode>_hard (operands[0], addr,
+    atomic_insn = gen_atomic_nand_fetch<mode>_hard (operands[0], mem,
 						    operands[2]);
   else if (TARGET_ATOMIC_SOFT_GUSA)
-    atomic_insn = gen_atomic_nand_fetch<mode>_soft_gusa (operands[0], addr,
+    atomic_insn = gen_atomic_nand_fetch<mode>_soft_gusa (operands[0], mem,
 							 operands[2]);
   else if (TARGET_ATOMIC_SOFT_TCB)
-    atomic_insn = gen_atomic_nand_fetch<mode>_soft_tcb (operands[0], addr,
+    atomic_insn = gen_atomic_nand_fetch<mode>_soft_tcb (operands[0], mem,
 		      operands[2], TARGET_ATOMIC_SOFT_TCB_GBR_OFFSET_RTX);
   else if (TARGET_ATOMIC_SOFT_IMASK)
-    atomic_insn = gen_atomic_nand_fetch<mode>_soft_imask (operands[0], addr,
+    atomic_insn = gen_atomic_nand_fetch<mode>_soft_imask (operands[0], mem,
 							  operands[2]);
   else
     FAIL;
@@ -1686,26 +1956,63 @@
 
 (define_insn "atomic_nand_fetchsi_hard"
   [(set (match_operand:SI 0 "arith_reg_dest" "=&z")
-	(not:SI (and:SI (mem:SI (match_operand:SI 1 "arith_reg_operand" "r"))
+	(not:SI (and:SI (match_operand:SI 1 "atomic_mem_operand_1" "=Sra")
 			(match_operand:SI 2 "logical_operand" "rK08"))))
-   (set (mem:SI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:SI
-	  [(not:SI (and:SI (mem:SI (match_dup 1)) (match_dup 2)))]
+	  [(not:SI (and:SI (match_dup 1) (match_dup 2)))]
 	  UNSPEC_ATOMIC))
    (set (reg:SI T_REG) (const_int 1))]
   "TARGET_ATOMIC_HARD_LLCS
    || (TARGET_SH4A && TARGET_ATOMIC_ANY && !TARGET_ATOMIC_STRICT)"
 {
-  return "\r0:	movli.l	@%1,%0"		"\n"
+  return "\r0:	movli.l	%1,%0"		"\n"
 	 "	and	%2,%0"		"\n"
 	 "	not	%0,%0"		"\n"
-	 "	movco.l	%0,@%1"		"\n"
+	 "	movco.l	%0,%1"		"\n"
 	 "	bf	0b";
 }
   [(set_attr "length" "10")])
 
+;; The QIHImode llcs patterns modify the address register of the memory
+;; operand.  In order to express that, we have to open code the memory
+;; operand.  Initially the insn is expanded like every other atomic insn
+;; using the memory operand.  In split1 the insn is converted and the
+;; memory operand's address register is exposed.
 (define_insn_and_split "atomic_nand_fetch<mode>_hard"
   [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
+	(not:QIHI (and:QIHI (match_operand:QIHI 1 "atomic_mem_operand_1")
+			    (match_operand:QIHI 2 "logical_operand"))))
+   (set (match_dup 1)
+	(unspec:QIHI [(not:QIHI (and:QIHI (match_dup 1) (match_dup 2)))]
+		     UNSPEC_ATOMIC))
+   (set (reg:SI T_REG) (const_int 1))
+   (clobber (reg:SI R0_REG))]
+  "TARGET_ATOMIC_HARD_LLCS && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+{
+  if (optimize
+      && sh_reg_dead_or_unused_after_insn (curr_insn, REGNO (operands[0])))
+    emit_insn (gen_atomic_nand<mode>_hard (operands[1], operands[2]));
+  else
+    {
+      rtx i = gen_atomic_nand_fetch<mode>_hard_1 (
+			operands[0], XEXP (operands[1], 0), operands[2]);
+      
+      /* Replace the new mems in the new insn with the old mem to preserve
+	 aliasing info.  */
+      XEXP (XEXP (XEXP (XVECEXP (i, 0, 0), 1), 0), 0) = operands[1];
+      XEXP (XVECEXP (i, 0, 1), 0) = operands[1];
+      XEXP (XEXP (XVECEXP (XEXP (XVECEXP (i, 0, 1), 1), 0, 0), 0),
+	    0) = operands[1];
+      emit_insn (i);
+    }
+})
+
+(define_insn "atomic_nand_fetch<mode>_hard_1"
+  [(set (match_operand:QIHI 0 "arith_reg_dest" "=&r")
 	(not:QIHI
 	  (and:QIHI (mem:QIHI (match_operand:SI 1 "arith_reg_operand" "r"))
 		    (match_operand:QIHI 2 "logical_operand" "rK08"))))
@@ -1734,22 +2041,16 @@
 	 "	movco.l	r0,@%3"			"\n"
 	 "	bf	0b";
 }
-  "&& can_create_pseudo_p () && optimize
-   && sh_reg_dead_or_unused_after_insn (insn, REGNO (operands[0]))"
-  [(const_int 0)]
-{
-  emit_insn (gen_atomic_nand<mode>_hard (operands[1], operands[2]));
-}
   [(set_attr "length" "28")])
 
 (define_insn "atomic_nand_fetch<mode>_soft_gusa"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&u")
 	(not:QIHISI (and:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "u"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=AraAdd")
 	  (match_operand:QIHISI 2 "arith_reg_operand" "u"))))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(not:QIHISI (and:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2)))]
+	  [(not:QIHISI (and:QIHISI (match_dup 1) (match_dup 2)))]
 	  UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))]
@@ -1759,10 +2060,10 @@
 	 "	.align 2"			"\n"
 	 "	mov	r15,r1"			"\n"
 	 "	mov	#(0f-1f),r15"		"\n"
-	 "0:	mov.<bwl>	@%1,%0"		"\n"
+	 "0:	mov.<bwl>	%1,%0"		"\n"
 	 "	and	%2,%0"			"\n"
 	 "	not	%0,%0"			"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "1:	mov	r1,r15";
 }
   [(set_attr "length" "18")])
@@ -1770,11 +2071,11 @@
 (define_insn_and_split "atomic_nand_fetch<mode>_soft_tcb"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&r")
 	(not:QIHISI (and:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")
 	  (match_operand:QIHISI 2 "logical_operand" "rK08"))))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(not:QIHISI (and:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2)))]
+	  [(not:QIHISI (and:QIHISI (match_dup 1) (match_dup 2)))]
 	  UNSPEC_ATOMIC))
    (clobber (reg:SI R0_REG))
    (clobber (reg:SI R1_REG))
@@ -1785,11 +2086,11 @@
 	 "	mov	#(0f-1f),r1"		"\n"
 	 "	.align 2"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)"		"\n"
-	 "0:	mov.<bwl>	@%1,r0"		"\n"
+	 "0:	mov.<bwl>	%1,r0"		"\n"
 	 "	and	%2,r0"			"\n"
 	 "	not	r0,r0"			"\n"
 	 "	mov	r0,%0"			"\n"
-	 "	mov.<bwl>	r0,@%1"		"\n"
+	 "	mov.<bwl>	r0,%1"		"\n"
 	 "1:	mov	#0,r0"			"\n"
 	 "	mov.l	r0,@(%O3,gbr)";
 }
@@ -1805,11 +2106,11 @@
 (define_insn "atomic_nand_fetch<mode>_soft_imask"
   [(set (match_operand:QIHISI 0 "arith_reg_dest" "=&z")
 	(not:QIHISI (and:QIHISI
-	  (mem:QIHISI (match_operand:SI 1 "arith_reg_operand" "r"))
+	  (match_operand:QIHISI 1 "atomic_mem_operand_1" "=SraSdd")
 	  (match_operand:QIHISI 2 "logical_operand" "rK08"))))
-   (set (mem:QIHISI (match_dup 1))
+   (set (match_dup 1)
 	(unspec:QIHISI
-	  [(not:QIHISI (and:QIHISI (mem:QIHISI (match_dup 1)) (match_dup 2)))]
+	  [(not:QIHISI (and:QIHISI (match_dup 1) (match_dup 2)))]
 	  UNSPEC_ATOMIC))
    (clobber (match_scratch:SI 3 "=&r"))]
   "TARGET_ATOMIC_SOFT_IMASK"
@@ -1818,10 +2119,10 @@
 	 "	mov	%0,%3"			"\n"
 	 "	or	#0xF0,%0"		"\n"
 	 "	ldc	%0,sr"			"\n"
-	 "	mov.<bwl>	@%1,%0"		"\n"
+	 "	mov.<bwl>	%1,%0"		"\n"
 	 "	and	%2,%0"			"\n"
 	 "	not	%0,%0"			"\n"
-	 "	mov.<bwl>	%0,@%1"		"\n"
+	 "	mov.<bwl>	%0,%1"		"\n"
 	 "	ldc	%3,sr";
 }
   [(set_attr "length" "18")])