Make ivopts handle calls to internal functions

ivopts previously treated pointer arguments to internal functions
like IFN_MASK_LOAD and IFN_MASK_STORE as normal gimple values.
This patch makes it treat them as addresses instead.  This makes
a significant difference to the code quality for SVE loops,
since we can then use loads and stores with scaled indices.

2018-01-13  Richard Sandiford  <richard.sandiford@linaro.org>
	    Alan Hayward  <alan.hayward@arm.com>
	    David Sherwood  <david.sherwood@arm.com>

gcc/
	* tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into...
	(USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types.
	(dump_groups): Update accordingly.
	(iv_use::mem_type): New member variable.
	(address_p): New function.
	(record_use): Add a mem_type argument and initialize the new
	mem_type field.
	(record_group_use): Add a mem_type argument.  Use address_p.
	Remove obsolete null checks of base_object.  Update call to record_use.
	(find_interesting_uses_op): Update call to record_group_use.
	(find_interesting_uses_cond): Likewise.
	(find_interesting_uses_address): Likewise.
	(get_mem_type_for_internal_fn): New function.
	(find_address_like_use): Likewise.
	(find_interesting_uses_stmt): Try find_address_like_use before
	calling find_interesting_uses_op.
	(addr_offset_valid_p): Use the iv mem_type field as the type
	of the addressed memory.
	(add_autoinc_candidates): Likewise.
	(get_address_cost): Likewise.
	(split_small_address_groups_p): Use address_p.
	(split_address_groups): Likewise.
	(add_iv_candidate_for_use): Likewise.
	(autoinc_possible_for_pair): Likewise.
	(rewrite_groups): Likewise.
	(get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS.
	(determine_group_iv_cost): Update after split of USE_ADDRESS.
	(get_alias_ptr_type_for_ptr_address): New function.
	(rewrite_use_address): Rewrite address uses in calls that were
	identified by find_address_like_use.

gcc/testsuite/
	* gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS
	instead of just ADDRESS.
	* gcc.dg/tree-ssa/scev-10.c: Likewise.
	* gcc.dg/tree-ssa/scev-11.c: Likewise.
	* gcc.dg/tree-ssa/scev-12.c: Likewise.
	* gcc.target/aarch64/sve/index_offset_1.c: New test.
	* gcc.target/aarch64/sve/index_offset_1_run.c: Likewise.
	* gcc.target/aarch64/sve/loop_add_2.c: Likewise.
	* gcc.target/aarch64/sve/loop_add_3.c: Likewise.
	* gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes.
	* gcc.target/aarch64/sve/while_2.c: Likewise.
	* gcc.target/aarch64/sve/while_3.c: Likewise.
	* gcc.target/aarch64/sve/while_4.c: Likewise.

Co-Authored-By: Alan Hayward <alan.hayward@arm.com>
Co-Authored-By: David Sherwood <david.sherwood@arm.com>

From-SVN: r256628

15 files changed, 354 insertions, 39 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 79ddaf3..6d198be 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -2,6 +2,41 @@
 	    Alan Hayward  <alan.hayward@arm.com>
 	    David Sherwood  <david.sherwood@arm.com>
 
+	* tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into...
+	(USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types.
+	(dump_groups): Update accordingly.
+	(iv_use::mem_type): New member variable.
+	(address_p): New function.
+	(record_use): Add a mem_type argument and initialize the new
+	mem_type field.
+	(record_group_use): Add a mem_type argument.  Use address_p.
+	Remove obsolete null checks of base_object.  Update call to record_use.
+	(find_interesting_uses_op): Update call to record_group_use.
+	(find_interesting_uses_cond): Likewise.
+	(find_interesting_uses_address): Likewise.
+	(get_mem_type_for_internal_fn): New function.
+	(find_address_like_use): Likewise.
+	(find_interesting_uses_stmt): Try find_address_like_use before
+	calling find_interesting_uses_op.
+	(addr_offset_valid_p): Use the iv mem_type field as the type
+	of the addressed memory.
+	(add_autoinc_candidates): Likewise.
+	(get_address_cost): Likewise.
+	(split_small_address_groups_p): Use address_p.
+	(split_address_groups): Likewise.
+	(add_iv_candidate_for_use): Likewise.
+	(autoinc_possible_for_pair): Likewise.
+	(rewrite_groups): Likewise.
+	(get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS.
+	(determine_group_iv_cost): Update after split of USE_ADDRESS.
+	(get_alias_ptr_type_for_ptr_address): New function.
+	(rewrite_use_address): Rewrite address uses in calls that were
+	identified by find_address_like_use.
+
+2018-01-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
 	* expr.c (expand_expr_addr_expr_1): Handle ADDR_EXPRs of
 	TARGET_MEM_REFs.
 	* gimple-expr.h (is_gimple_addressable: Likewise.
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 7363c0d..c59919a 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -2,6 +2,24 @@
 	    Alan Hayward  <alan.hayward@arm.com>
 	    David Sherwood  <david.sherwood@arm.com>
 
+	* gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS
+	instead of just ADDRESS.
+	* gcc.dg/tree-ssa/scev-10.c: Likewise.
+	* gcc.dg/tree-ssa/scev-11.c: Likewise.
+	* gcc.dg/tree-ssa/scev-12.c: Likewise.
+	* gcc.target/aarch64/sve/index_offset_1.c: New test.
+	* gcc.target/aarch64/sve/index_offset_1_run.c: Likewise.
+	* gcc.target/aarch64/sve/loop_add_2.c: Likewise.
+	* gcc.target/aarch64/sve/loop_add_3.c: Likewise.
+	* gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes.
+	* gcc.target/aarch64/sve/while_2.c: Likewise.
+	* gcc.target/aarch64/sve/while_3.c: Likewise.
+	* gcc.target/aarch64/sve/while_4.c: Likewise.
+
+2018-01-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
 	* gcc.dg/vect/pr60482.c: Remove XFAIL for variable-length vectors.
 	* gcc.target/aarch64/sve/reduc_1.c: Expect the loop operations
 	to be predicated.
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c
index e402bf9..6c4fca7 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c
@@ -18,5 +18,5 @@ foo (signed char s, signed char l)
 }
 
 /* Address of array reference is scev.  */
-/* { dg-final { scan-tree-dump-times "  Type:\\tADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times "  Type:\\tREFERENCE ADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c
index 088771e..c665fd7 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c
@@ -23,4 +23,4 @@ foo (int n)
 }
 
 /* Address of array reference to b is scev.  */
-/* { dg-final { scan-tree-dump-times "  Type:\\tADDRESS\n  Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times "  Type:\\tREFERENCE ADDRESS\n  Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c
index c112639..f598c7b 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c
@@ -24,4 +24,4 @@ foo (int x, int n)
 }
 
 /* Address of array reference to b is not scev.  */
-/* { dg-final { scan-tree-dump-times "  Type:\\tADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times "  Type:\\tREFERENCE ADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c
index b11b2f1..2863918 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c
@@ -18,5 +18,5 @@ foo (unsigned char s, unsigned char l)
 }
 
 /* Address of array reference is scev.  */
-/* { dg-final { scan-tree-dump-times "  Type:\\tADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times "  Type:\\tREFERENCE ADDRESS\n  Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c
new file mode 100644
index 0000000..31d46aa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c
@@ -0,0 +1,54 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" } */
+
+#define SIZE (15 * 8 + 3)
+
+#define DEF_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE)			\
+void __attribute__ ((noinline, noclone))				\
+set_##SIGNED##_##TYPE##_##ITERTYPE (SIGNED TYPE *restrict out,		\
+				    SIGNED TYPE *restrict in)		\
+{									\
+  SIGNED ITERTYPE i;							\
+  for (i = 0; i < SIZE; i++)						\
+  {									\
+    out[i] = in[i];							\
+  }									\
+}									\
+void __attribute__ ((noinline, noclone))				\
+set_##SIGNED##_##TYPE##_##ITERTYPE##_var (SIGNED TYPE *restrict out,	\
+					  SIGNED TYPE *restrict in,	\
+					  SIGNED ITERTYPE n)		\
+{									\
+  SIGNED ITERTYPE i;							\
+  for (i = 0; i < n; i++)						\
+  {									\
+    out[i] = in[i];							\
+  }									\
+}
+
+#define TEST_TYPE(T, SIGNED, TYPE)		\
+  T (SIGNED, TYPE, char)			\
+  T (SIGNED, TYPE, short)			\
+  T (SIGNED, TYPE, int)				\
+  T (SIGNED, TYPE, long)
+
+#define TEST_ALL(T)				\
+  TEST_TYPE (T, signed, long)			\
+  TEST_TYPE (T, unsigned, long)			\
+  TEST_TYPE (T, signed, int)			\
+  TEST_TYPE (T, unsigned, int)			\
+  TEST_TYPE (T, signed, short)			\
+  TEST_TYPE (T, unsigned, short)		\
+  TEST_TYPE (T, signed, char)			\
+  TEST_TYPE (T, unsigned, char)
+
+TEST_ALL (DEF_INDEX_OFFSET)
+
+/* { dg-final { scan-assembler-times "ld1d\\tz\[0-9\]+.d, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1d\\tz\[0-9\]+.d, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1w\\tz\[0-9\]+.s, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1w\\tz\[0-9\]+.s, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1h\\tz\[0-9\]+.h, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1h\\tz\[0-9\]+.h, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1b\\tz\[0-9\]+.b, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1b\\tz\[0-9\]+.b, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c
new file mode 100644
index 0000000..00114ac
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c
@@ -0,0 +1,34 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" { target aarch64_sve256_hw } } */
+
+#include "index_offset_1.c"
+
+#define TEST_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE)		\
+{								\
+  SIGNED TYPE out[SIZE + 1];					\
+  SIGNED TYPE in1[SIZE + 1];					\
+  SIGNED TYPE in2[SIZE + 1];					\
+  for (int i = 0; i < SIZE + 1; ++i)				\
+    {								\
+      in1[i] = (i * 4) ^ i;					\
+      in2[i] = (i * 2) ^ i;					\
+      asm volatile ("" ::: "memory");				\
+    }								\
+  out[SIZE] = 42;						\
+  set_##SIGNED##_##TYPE##_##ITERTYPE (out, in1);		\
+  if (0 != __builtin_memcmp (out, in1, SIZE * sizeof (TYPE)))	\
+    __builtin_abort ();						\
+  set_##SIGNED##_##TYPE##_##ITERTYPE##_var (out, in2, SIZE);	\
+  if (0 != __builtin_memcmp (out, in2, SIZE * sizeof (TYPE)))	\
+    __builtin_abort ();						\
+  if (out[SIZE] != 42)						\
+    __builtin_abort ();						\
+}
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+  TEST_ALL (TEST_INDEX_OFFSET);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c
new file mode 100644
index 0000000..a58ba0b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c99 -O3" } */
+
+void
+foo (int *__restrict a, int *__restrict b)
+{
+  for (int i = 0; i < 512; ++i)
+    a[i] += b[i];
+}
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c
new file mode 100644
index 0000000..9d2c598
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-std=c99 -O3" } */
+
+void
+f (int *__restrict a,
+   int *__restrict b,
+   int *__restrict c,
+   int *__restrict d,
+   int *__restrict e,
+   int *__restrict f,
+   int *__restrict g,
+   int *__restrict h,
+   int count)
+{
+  for (int i = 0; i < count; ++i)
+    a[i] = b[i] + c[i] + d[i] + e[i] + f[i] + g[i] + h[i];
+}
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 7 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_1.c b/gcc/testsuite/gcc.target/aarch64/sve/while_1.c
index 70ceec8..a93a04b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/while_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_1.c
@@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP)
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_2.c b/gcc/testsuite/gcc.target/aarch64/sve/while_2.c
index 651bb57..69bb6cb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/while_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_2.c
@@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP)
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_3.c b/gcc/testsuite/gcc.target/aarch64/sve/while_3.c
index 68d524e..f805c2f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/while_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_3.c
@@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP)
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_4.c b/gcc/testsuite/gcc.target/aarch64/sve/while_4.c
index d8ec536..e7aa99f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/while_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_4.c
@@ -35,3 +35,11 @@ TEST_ALL (ADD_LOOP)
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
 /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
diff --git a/gcc/tree-ssa-loop-ivopts.c b/gcc/tree-ssa-loop-ivopts.c
index 63f01d2..b313571 100644
--- a/gcc/tree-ssa-loop-ivopts.c
+++ b/gcc/tree-ssa-loop-ivopts.c
@@ -166,7 +166,11 @@ struct version_info
 enum use_type
 {
   USE_NONLINEAR_EXPR,	/* Use in a nonlinear expression.  */
-  USE_ADDRESS,		/* Use in an address.  */
+  USE_REF_ADDRESS,	/* Use is an address for an explicit memory
+			   reference.  */
+  USE_PTR_ADDRESS,	/* Use is a pointer argument to a function in
+			   cases where the expansion of the function
+			   will turn the argument into a normal address.  */
   USE_COMPARE		/* Use is a compare.  */
 };
 
@@ -362,6 +366,9 @@ struct iv_use
   unsigned id;		/* The id of the use.  */
   unsigned group_id;	/* The group id the use belongs to.  */
   enum use_type type;	/* Type of the use.  */
+  tree mem_type;	/* The memory type to use when testing whether an
+			   address is legitimate, and what the address's
+			   cost is.  */
   struct iv *iv;	/* The induction variable it is based on.  */
   gimple *stmt;		/* Statement in that it occurs.  */
   tree *op_p;		/* The place where it occurs.  */
@@ -506,6 +513,14 @@ struct iv_inv_expr_hasher : free_ptr_hash <iv_inv_expr_ent>
   static inline bool equal (const iv_inv_expr_ent *, const iv_inv_expr_ent *);
 };
 
+/* Return true if uses of type TYPE represent some form of address.  */
+
+inline bool
+address_p (use_type type)
+{
+  return type == USE_REF_ADDRESS || type == USE_PTR_ADDRESS;
+}
+
 /* Hash function for loop invariant expressions.  */
 
 inline hashval_t
@@ -768,8 +783,10 @@ dump_groups (FILE *file, struct ivopts_data *data)
       fprintf (file, "Group %d:\n", group->id);
       if (group->type == USE_NONLINEAR_EXPR)
 	fprintf (file, "  Type:\tGENERIC\n");
-      else if (group->type == USE_ADDRESS)
-	fprintf (file, "  Type:\tADDRESS\n");
+      else if (group->type == USE_REF_ADDRESS)
+	fprintf (file, "  Type:\tREFERENCE ADDRESS\n");
+      else if (group->type == USE_PTR_ADDRESS)
+	fprintf (file, "  Type:\tPOINTER ARGUMENT ADDRESS\n");
       else
 	{
 	  gcc_assert (group->type == USE_COMPARE);
@@ -1502,19 +1519,21 @@ find_induction_variables (struct ivopts_data *data)
 
 /* Records a use of TYPE at *USE_P in STMT whose value is IV in GROUP.
    For address type use, ADDR_BASE is the stripped IV base, ADDR_OFFSET
-   is the const offset stripped from IV base; for other types use, both
-   are zero by default.  */
+   is the const offset stripped from IV base and MEM_TYPE is the type
+   of the memory being addressed.  For uses of other types, ADDR_BASE
+   and ADDR_OFFSET are zero by default and MEM_TYPE is NULL_TREE.  */
 
 static struct iv_use *
 record_use (struct iv_group *group, tree *use_p, struct iv *iv,
-	    gimple *stmt, enum use_type type, tree addr_base,
-	    poly_uint64 addr_offset)
+	    gimple *stmt, enum use_type type, tree mem_type,
+	    tree addr_base, poly_uint64 addr_offset)
 {
   struct iv_use *use = XCNEW (struct iv_use);
 
   use->id = group->vuses.length ();
   use->group_id = group->id;
   use->type = type;
+  use->mem_type = mem_type;
   use->iv = iv;
   use->stmt = stmt;
   use->op_p = use_p;
@@ -1569,18 +1588,21 @@ record_group (struct ivopts_data *data, enum use_type type)
 }
 
 /* Record a use of TYPE at *USE_P in STMT whose value is IV in a group.
-   New group will be created if there is no existing group for the use.  */
+   New group will be created if there is no existing group for the use.
+   MEM_TYPE is the type of memory being addressed, or NULL if this
+   isn't an address reference.  */
 
 static struct iv_use *
 record_group_use (struct ivopts_data *data, tree *use_p,
-		  struct iv *iv, gimple *stmt, enum use_type type)
+		  struct iv *iv, gimple *stmt, enum use_type type,
+		  tree mem_type)
 {
   tree addr_base = NULL;
   struct iv_group *group = NULL;
   poly_uint64 addr_offset = 0;
 
   /* Record non address type use in a new group.  */
-  if (type == USE_ADDRESS && iv->base_object)
+  if (address_p (type))
     {
       unsigned int i;
 
@@ -1591,7 +1613,7 @@ record_group_use (struct ivopts_data *data, tree *use_p,
 
 	  group = data->vgroups[i];
 	  use = group->vuses[0];
-	  if (use->type != USE_ADDRESS || !use->iv->base_object)
+	  if (!address_p (use->type))
 	    continue;
 
 	  /* Check if it has the same stripped base and step.  */
@@ -1607,7 +1629,8 @@ record_group_use (struct ivopts_data *data, tree *use_p,
   if (!group)
     group = record_group (data, type);
 
-  return record_use (group, use_p, iv, stmt, type, addr_base, addr_offset);
+  return record_use (group, use_p, iv, stmt, type, mem_type,
+		     addr_base, addr_offset);
 }
 
 /* Checks whether the use OP is interesting and if so, records it.  */
@@ -1641,7 +1664,7 @@ find_interesting_uses_op (struct ivopts_data *data, tree op)
   stmt = SSA_NAME_DEF_STMT (op);
   gcc_assert (gimple_code (stmt) == GIMPLE_PHI || is_gimple_assign (stmt));
 
-  use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR);
+  use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR, NULL_TREE);
   iv->nonlin_use = use;
   return use;
 }
@@ -1757,10 +1780,10 @@ find_interesting_uses_cond (struct ivopts_data *data, gimple *stmt)
       return;
     }
 
-  record_group_use (data, var_p, var_iv, stmt, USE_COMPARE);
+  record_group_use (data, var_p, var_iv, stmt, USE_COMPARE, NULL_TREE);
   /* Record compare type iv_use for iv on the other side of comparison.  */
   if (ret == COMP_IV_EXPR_2)
-    record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE);
+    record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE, NULL_TREE);
 }
 
 /* Returns the outermost loop EXPR is obviously invariant in
@@ -2375,7 +2398,7 @@ find_interesting_uses_address (struct ivopts_data *data, gimple *stmt,
   if (civ->base_object == NULL_TREE)
     goto fail;
 
-  record_group_use (data, op_p, civ, stmt, USE_ADDRESS);
+  record_group_use (data, op_p, civ, stmt, USE_REF_ADDRESS, TREE_TYPE (*op_p));
   return;
 
 fail:
@@ -2398,6 +2421,55 @@ find_invariants_stmt (struct ivopts_data *data, gimple *stmt)
     }
 }
 
+/* CALL calls an internal function.  If operand *OP_P will become an
+   address when the call is expanded, return the type of the memory
+   being addressed, otherwise return null.  */
+
+static tree
+get_mem_type_for_internal_fn (gcall *call, tree *op_p)
+{
+  switch (gimple_call_internal_fn (call))
+    {
+    case IFN_MASK_LOAD:
+      if (op_p == gimple_call_arg_ptr (call, 0))
+	return TREE_TYPE (gimple_call_lhs (call));
+      return NULL_TREE;
+
+    case IFN_MASK_STORE:
+      if (op_p == gimple_call_arg_ptr (call, 0))
+	return TREE_TYPE (gimple_call_arg (call, 3));
+      return NULL_TREE;
+
+    default:
+      return NULL_TREE;
+    }
+}
+
+/* IV is a (non-address) iv that describes operand *OP_P of STMT.
+   Return true if the operand will become an address when STMT
+   is expanded and record the associated address use if so.  */
+
+static bool
+find_address_like_use (struct ivopts_data *data, gimple *stmt, tree *op_p,
+		       struct iv *iv)
+{
+  /* Fail if base object of this memory reference is unknown.  */
+  if (iv->base_object == NULL_TREE)
+    return false;
+
+  tree mem_type = NULL_TREE;
+  if (gcall *call = dyn_cast <gcall *> (stmt))
+    if (gimple_call_internal_p (call))
+      mem_type = get_mem_type_for_internal_fn (call, op_p);
+  if (mem_type)
+    {
+      iv = alloc_iv (data, iv->base, iv->step);
+      record_group_use (data, op_p, iv, stmt, USE_PTR_ADDRESS, mem_type);
+      return true;
+    }
+  return false;
+}
+
 /* Finds interesting uses of induction variables in the statement STMT.  */
 
 static void
@@ -2482,7 +2554,8 @@ find_interesting_uses_stmt (struct ivopts_data *data, gimple *stmt)
       if (!iv)
 	continue;
 
-      find_interesting_uses_op (data, op);
+      if (!find_address_like_use (data, stmt, use_p->use, iv))
+	find_interesting_uses_op (data, op);
     }
 }
 
@@ -2516,7 +2589,7 @@ addr_offset_valid_p (struct iv_use *use, poly_int64 offset)
   rtx reg, addr;
   unsigned list_index;
   addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
-  machine_mode addr_mode, mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+  machine_mode addr_mode, mem_mode = TYPE_MODE (use->mem_type);
 
   list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
   if (list_index >= vec_safe_length (addr_list))
@@ -2573,7 +2646,7 @@ split_small_address_groups_p (struct ivopts_data *data)
       if (group->vuses.length () == 1)
 	continue;
 
-      gcc_assert (group->type == USE_ADDRESS);
+      gcc_assert (address_p (group->type));
       if (group->vuses.length () == 2)
 	{
 	  if (compare_sizes_for_sort (group->vuses[0]->addr_offset,
@@ -2625,7 +2698,7 @@ split_address_groups (struct ivopts_data *data)
       if (group->vuses.length () == 1)
 	continue;
 
-      gcc_assert (group->type == USE_ADDRESS);
+      gcc_assert (address_p (use->type));
 
       for (j = 1; j < group->vuses.length ();)
 	{
@@ -3145,7 +3218,7 @@ add_autoinc_candidates (struct ivopts_data *data, tree base, tree step,
 
   cstepi = int_cst_value (step);
 
-  mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+  mem_mode = TYPE_MODE (use->mem_type);
   if (((USE_LOAD_PRE_INCREMENT (mem_mode)
 	|| USE_STORE_PRE_INCREMENT (mem_mode))
        && known_eq (GET_MODE_SIZE (mem_mode), cstepi))
@@ -3436,7 +3509,7 @@ add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use)
   /* At last, add auto-incremental candidates.  Make such variables
      important since other iv uses with same base object may be based
      on it.  */
-  if (use != NULL && use->type == USE_ADDRESS)
+  if (use != NULL && address_p (use->type))
     add_autoinc_candidates (data, iv->base, iv->step, true, use);
 }
 
@@ -3903,7 +3976,7 @@ get_use_type (struct iv_use *use)
   tree base_type = TREE_TYPE (use->iv->base);
   tree type;
 
-  if (use->type == USE_ADDRESS)
+  if (use->type == USE_REF_ADDRESS)
     {
       /* The base_type may be a void pointer.  Create a pointer type based on
 	 the mem_ref instead.  */
@@ -4331,7 +4404,7 @@ get_address_cost (struct ivopts_data *data, struct iv_use *use,
   struct mem_address parts = {NULL_TREE, integer_one_node,
 			      NULL_TREE, NULL_TREE, NULL_TREE};
   machine_mode addr_mode = TYPE_MODE (type);
-  machine_mode mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+  machine_mode mem_mode = TYPE_MODE (use->mem_type);
   addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
   /* Only true if ratio != 1.  */
   bool ok_with_ratio_p = false;
@@ -5220,7 +5293,8 @@ determine_group_iv_cost (struct ivopts_data *data,
     case USE_NONLINEAR_EXPR:
       return determine_group_iv_cost_generic (data, group, cand);
 
-    case USE_ADDRESS:
+    case USE_REF_ADDRESS:
+    case USE_PTR_ADDRESS:
       return determine_group_iv_cost_address (data, group, cand);
 
     case USE_COMPARE:
@@ -5238,7 +5312,7 @@ static bool
 autoinc_possible_for_pair (struct ivopts_data *data, struct iv_use *use,
 			   struct iv_cand *cand)
 {
-  if (use->type != USE_ADDRESS)
+  if (!address_p (use->type))
     return false;
 
   bool can_autoinc = false;
@@ -6997,6 +7071,27 @@ adjust_iv_update_pos (struct iv_cand *cand, struct iv_use *use)
   cand->incremented_at = use->stmt;
 }
 
+/* Return the alias pointer type that should be used for a MEM_REF
+   associated with USE, which has type USE_PTR_ADDRESS.  */
+
+static tree
+get_alias_ptr_type_for_ptr_address (iv_use *use)
+{
+  gcall *call = as_a <gcall *> (use->stmt);
+  switch (gimple_call_internal_fn (call))
+    {
+    case IFN_MASK_LOAD:
+    case IFN_MASK_STORE:
+      /* The second argument contains the correct alias type.  */
+      gcc_assert (use->op_p = gimple_call_arg_ptr (call, 0));
+      return TREE_TYPE (gimple_call_arg (call, 1));
+
+    default:
+      gcc_unreachable ();
+    }
+}
+
+
 /* Rewrites USE (address that is an iv) using candidate CAND.  */
 
 static void
@@ -7025,16 +7120,31 @@ rewrite_use_address (struct ivopts_data *data,
   tree iv = var_at_stmt (data->current_loop, cand, use->stmt);
   tree base_hint = (cand->iv->base_object) ? iv : NULL_TREE;
   gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
-  tree type = TREE_TYPE (*use->op_p);
-  unsigned int align = get_object_alignment (*use->op_p);
-  if (align != TYPE_ALIGN (type))
-    type = build_aligned_type (type, align);
-
-  tree ref = create_mem_ref (&bsi, type, &aff,
-			     reference_alias_ptr_type (*use->op_p),
+  tree type = use->mem_type;
+  tree alias_ptr_type;
+  if (use->type == USE_PTR_ADDRESS)
+    alias_ptr_type = get_alias_ptr_type_for_ptr_address (use);
+  else
+    {
+      gcc_assert (type == TREE_TYPE (*use->op_p));
+      unsigned int align = get_object_alignment (*use->op_p);
+      if (align != TYPE_ALIGN (type))
+	type = build_aligned_type (type, align);
+      alias_ptr_type = reference_alias_ptr_type (*use->op_p);
+    }
+  tree ref = create_mem_ref (&bsi, type, &aff, alias_ptr_type,
 			     iv, base_hint, data->speed);
 
-  copy_ref_info (ref, *use->op_p);
+  if (use->type == USE_PTR_ADDRESS)
+    {
+      ref = fold_build1 (ADDR_EXPR, build_pointer_type (use->mem_type), ref);
+      ref = fold_convert (get_use_type (use), ref);
+      ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
+				      true, GSI_SAME_STMT);
+    }
+  else
+    copy_ref_info (ref, *use->op_p);
+
   *use->op_p = ref;
 }
 
@@ -7110,7 +7220,7 @@ rewrite_groups (struct ivopts_data *data)
 	      update_stmt (group->vuses[j]->stmt);
 	    }
 	}
-      else if (group->type == USE_ADDRESS)
+      else if (address_p (group->type))
 	{
 	  for (j = 0; j < group->vuses.length (); j++)
 	    {