middle-end/99122 - more VLA inlining fixes

This avoids declaring a function with VLA arguments or return values
as inlineable.  IPA CP still ICEs, so the testcase has that disabled.

2021-02-19  Richard Biener  <rguenther@suse.de>

	PR middle-end/99122
	* tree-inline.c (inline_forbidden_p): Do not inline functions
	with VLA arguments or return value.

	* gcc.dg/pr99122-3.c: New testcase.

2 files changed, 29 insertions, 0 deletions

diff --git a/gcc/testsuite/gcc.dg/pr99122-3.c b/gcc/testsuite/gcc.dg/pr99122-3.c
new file mode 100644
index 0000000..6aa5b29
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr99122-3.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -g -fno-ipa-cp -w" } */
+
+static int foo ();
+
+int
+bar (int n)
+{
+  return foo (n, 2.0);
+}
+
+static inline int
+foo (int n, struct T { char a[n]; } b)
+{
+  int r = 0, i;
+  for (i = 0; i < n; i++)
+    r += b.a[i];
+  return r;
+}
diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 01a08cf..c993b1f 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -4022,6 +4022,16 @@ inline_forbidden_p (tree fndecl)
   wi.info = (void *) fndecl;
   wi.pset = &visited_nodes;
 
+  /* We cannot inline a function with a VLA typed argument or result since
+     we have no implementation materializing a variable of such type in
+     the caller.  */
+  if (COMPLETE_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
+      && !poly_int_tree_p (TYPE_SIZE (TREE_TYPE (TREE_TYPE (fndecl)))))
+    return true;
+  for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
+    if (!poly_int_tree_p (DECL_SIZE (parm)))
+      return true;
+
   FOR_EACH_BB_FN (bb, fun)
     {
       gimple *ret;