tailc: Don't reject all tail calls if param has addr taken [PR119616]

Before my PR119376 r15-9145 changes, suitable_for_tail_call_opt_p would
return the same value in the same caller, regardless of the calls in it.
If it fails, the caller clears opt_tailcalls which is a reference and
therefore shared by all calls in the caller and we only do tail recursion,
all non-recursive or tail recursion non-optimizable calls are not
tail call optimized.

For musttail calls we want to allow address taken parameters, but the
r15-9145 change effectively resulted in the behavior where if there
are just musttail calls considered, they will be tail call optimized,
and if there are also other tail call candidates (without musttail),
we clear opt_tailcall and then error out on all the musttail calls.

The following patch fixes that by moving the address taken parameter
discovery from suitable_for_tail_call_opt_p to its single caller.
If there are addressable parameters, if !cfun->has_musttail it will
work as before, disable all tail calls in the caller but possibly
allow tail recursions.  If cfun->has_musttail, it will set a new
bool automatic flag and reject non-tail recursions.  This way musttail
calls can be still accepted and normal tail call candidates rejected
(and tail recursions accepted).

2025-04-04  Jakub Jelinek  <jakub@redhat.com>

	PR tree-optimization/119616
	* tree-tailcall.cc (suitable_for_tail_call_opt_p): Move checking
	for addressable parameters from here ...
	(find_tail_calls): ... here.  If cfun->has_musttail, don't clear
	opt_tailcalls for it, instead set a local flag and punt if we can't
	tail recurse optimize it.

	* c-c++-common/pr119616.c: New test.

2 files changed, 46 insertions, 14 deletions

diff --git a/gcc/testsuite/c-c++-common/pr119616.c b/gcc/testsuite/c-c++-common/pr119616.c
new file mode 100644
index 0000000..5ffdb8c
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/pr119616.c
@@ -0,0 +1,23 @@
+/* PR tree-optimization/119616 */
+/* { dg-do compile { target external_musttail } } */
+/* { dg-options "-O2" } */
+
+int foo (int *);
+int bar (int);
+
+int
+baz (int x)
+{
+  if (!x)
+    [[gnu::musttail]] return bar (x);
+  return foo (&x);
+}
+
+int
+qux (int x)
+{
+  if (!x)
+    [[gnu::musttail]] return bar (x);
+  foo (&x);
+  return 1;
+}
diff --git a/gcc/tree-tailcall.cc b/gcc/tree-tailcall.cc
index c5fac51..c8740f9 100644
--- a/gcc/tree-tailcall.cc
+++ b/gcc/tree-tailcall.cc
@@ -165,8 +165,6 @@ suitable_for_tail_opt_p (gcall *call, bool diag_musttail)
 static bool
 suitable_for_tail_call_opt_p (gcall *call, bool diag_musttail)
 {
-  tree param;
-
   /* alloca (until we have stack slot life analysis) inhibits
      sibling call optimizations, but not tail recursion.  */
   if (cfun->calls_alloca)
@@ -204,18 +202,6 @@ suitable_for_tail_call_opt_p (gcall *call, bool diag_musttail)
       return false;
     }
 
-  /* ??? It is OK if the argument of a function is taken in some cases,
-     but not in all cases.  See PR15387 and PR19616.  Revisit for 4.1.  */
-  if (!diag_musttail || !gimple_call_must_tail_p (call))
-    for (param = DECL_ARGUMENTS (current_function_decl);
-	 param; param = DECL_CHAIN (param))
-      if (TREE_ADDRESSABLE (param))
-	{
-	  maybe_error_musttail (call, _("address of caller arguments taken"),
-				diag_musttail);
-	  return false;
-	}
-
   if (diag_musttail
       && gimple_call_must_tail_p (call)
       && warn_musttail_local_addr)
@@ -565,6 +551,7 @@ find_tail_calls (basic_block bb, struct tailcall **ret, bool only_musttail,
   basic_block abb;
   size_t idx;
   tree var;
+  bool only_tailr = false;
 
   if (!single_succ_p (bb)
       && (EDGE_COUNT (bb->succs) || !cfun->has_musttail || !diag_musttail))
@@ -660,6 +647,25 @@ find_tail_calls (basic_block bb, struct tailcall **ret, bool only_musttail,
   if (!suitable_for_tail_call_opt_p (call, diag_musttail))
     opt_tailcalls = false;
 
+  /* ??? It is OK if the argument of a function is taken in some cases,
+     but not in all cases.  See PR15387 and PR19616.  Revisit for 4.1.  */
+  if (!diag_musttail || !gimple_call_must_tail_p (call))
+    for (param = DECL_ARGUMENTS (current_function_decl);
+	 param; param = DECL_CHAIN (param))
+      if (TREE_ADDRESSABLE (param))
+	{
+	  maybe_error_musttail (call, _("address of caller arguments taken"),
+				diag_musttail);
+	  /* If current function has musttail calls, we can't disable tail
+	     calls altogether for the whole caller, because those might be
+	     actually fine.  So just punt if this exact call is not
+	     a tail recursion.  */
+	  if (cfun->has_musttail)
+	    only_tailr = true;
+	  else
+	    opt_tailcalls = false;
+	}
+
   /* If the LHS of our call is not just a simple register or local
      variable, we can't transform this into a tail or sibling call.
      This situation happens, in (e.g.) "*p = foo()" where foo returns a
@@ -794,6 +800,9 @@ find_tail_calls (basic_block bb, struct tailcall **ret, bool only_musttail,
 	tail_recursion = true;
     }
 
+  if (only_tailr && !tail_recursion)
+    return;
+
   /* Compute live vars if not computed yet.  */
   if (live_vars == NULL)
     {