coroutines: Correct frame capture of compiler temps [PR95591+4].

When a full expression contains a co_await (or co_yield), this means
that it might be suspended; which would destroy temporary variables
(on a stack for example).  However the language guarantees that such
temporaries are live until the end of the expression.

In order to preserve this, we 'promote' temporaries where necessary
so that they are saved in the coroutine frame (which allows them to
remain live potentially until the frame is destroyed).  In addition,
sub-expressions that produce control flow (such as TRUTH_AND/OR_IF
or COND_EXPR) must be handled specifically to ensure that await
expressions are properly expanded.

This patch corrects two mistakes in which we were (a) failing to
promote some temporaries and (b) we we failing to sequence DTORs for
the captures properly. This manifests in a number of related (but not
exact duplicate) PRs.

The revised code collects the actions into one place and maps all the
control flow into one form - a benefit of this is that co_returns are
now expanded earlier (which provides an opportunity to address PR95517
in some future patch).

We replace a statement that contains await expression(s) with a bind
scope that has a variable list describing the temporaries that have
been 'promoted' and a statement list that contains a series of cleanup
expressions for each of those.  Where we encounter nested conditional
expressions, these are wrapped in a try-finally block with a guard var
for each sub-expression variable that needs a DTOR.  The guards are all
declared and initialized to false before the first conditional sub-
expression.  The 'finally' block contains a series of if blocks (one
per guard variable) enclosing the relevant DTOR.

Variables listed in a bind scope in this manner are automatically moved
to a coroutine frame version by existing code (so we re-use that rather
than having a separate mechanism).

gcc/cp/ChangeLog:

	PR c++/95591
	PR c++/95599
	PR c++/95823
	PR c++/95824
	PR c++/95895
	* coroutines.cc (struct coro_ret_data): Delete.
	(coro_maybe_expand_co_return): Delete.
	(co_return_expander): Delete.
	(expand_co_returns): Delete.
	(co_await_find_in_subtree): Remove unused name.
	(build_actor_fn): Remove unused parm, remove handling
	for co_return expansion.
	(register_await_info): Demote duplicate info message to a
	warning.
	(coro_make_frame_entry): Move closer to use site.
	(struct susp_frame_data): Add fields for final suspend label
	and a flag to indicate await expressions with initializers.
	(captures_temporary): Delete.
	(register_awaits): Remove unused code, update comments.
	(find_any_await): New.
	(tmp_target_expr_p): New.
	(struct interesting): New.
	(find_interesting_subtree): New.
	(struct var_nest_node): New.
	(flatten_await_stmt): New.
	(handle_nested_conditionals): New.
	(process_conditional): New.
	(replace_statement_captures): Rename to...
	(maybe_promote_temps): ... this.
	(maybe_promote_captured_temps): Delete.
	(analyze_expression_awaits): Check for await expressions with
	initializers.  Simplify handling for truth-and/or-if.
	(expand_one_truth_if): Simplify (map cases that need expansion
	to COND_EXPR).
	(await_statement_walker): Handle CO_RETURN_EXPR. Simplify the
	handling for truth-and/or-if expressions.
	(register_local_var_uses): Ensure that we create names in the
	implementation namespace.
	(morph_fn_to_coro): Add final suspend label to suspend frame
	callback data and remove it from the build_actor_fn call.

gcc/testsuite/ChangeLog:

	PR c++/95591
	PR c++/95599
	PR c++/95823
	PR c++/95824
	PR c++/95895
	* g++.dg/coroutines/pr95591.C: New test.
	* g++.dg/coroutines/pr95599.C: New test.
	* g++.dg/coroutines/pr95823.C: New test.
	* g++.dg/coroutines/pr95824.C: New test.

1 files changed, 654 insertions, 474 deletions

diff --git a/gcc/cp/coroutines.cc b/gcc/cp/coroutines.cc
index d6b584e..84b6a4e 100644
--- a/gcc/cp/coroutines.cc
+++ b/gcc/cp/coroutines.cc
@@ -1390,119 +1390,6 @@ replace_proxy (tree *here, int *do_subtree, void *d)
   return NULL_TREE;
 }
 
-/* Support for expansion of co_return statements.  */
-
-struct coro_ret_data
-{
-  tree promise_proxy;
-  tree real_promise;
-  tree fs_label;
-};
-
-/* If this is a coreturn statement (or one wrapped in a cleanup) then
-   return the list of statements to replace it.  */
-
-static tree
-coro_maybe_expand_co_return (tree co_ret_expr, coro_ret_data *data)
-{
-  /* Look inside <(void) (expr)> cleanup */
-  if (TREE_CODE (co_ret_expr) == CLEANUP_POINT_EXPR)
-    co_ret_expr = TREE_OPERAND (co_ret_expr, 0);
-
-  if (TREE_CODE (co_ret_expr) != CO_RETURN_EXPR)
-    return NULL_TREE;
-
-  location_t loc = EXPR_LOCATION (co_ret_expr);
-  tree expr = TREE_OPERAND (co_ret_expr, 0);
-  tree call = TREE_OPERAND (co_ret_expr, 1);
-  tree stmt_list = NULL;
-  if (expr && VOID_TYPE_P (TREE_TYPE (expr)))
-    {
-       /* [stmt.return.coroutine], 2.2 
-	  If expr is present and void, it is placed immediately before
-	  the call for return_void;  */
-      expr = maybe_cleanup_point_expr_void (expr);
-      append_to_statement_list (expr, &stmt_list);
-    }
-
-  /* Now replace the promise proxy with its real value.  */
-  proxy_replace p_data;
-  p_data.from = data->promise_proxy;
-  p_data.to = data->real_promise;
-  cp_walk_tree (&call, replace_proxy, &p_data, NULL);
-
-  /* The types of p.return_void and p.return_value are not explicitly stated
-     at least in n4835, it is expected that they will return void.  */
-  call = maybe_cleanup_point_expr_void (call);
-  append_to_statement_list (call, &stmt_list);
-  tree r = build1_loc (loc, GOTO_EXPR, void_type_node, data->fs_label);
-  append_to_statement_list (r, &stmt_list);
-  return stmt_list;
-}
-
-/* Callback that rewrites co_return as per [stmt.return.coroutine]
-   - for co_return;
-   { p.return_void (); goto final_suspend; }
-   - for co_return [void expr];
-   { expr; p.return_void(); goto final_suspend;}
-   - for co_return [non void expr];
-   { p.return_value(expr); goto final_suspend; }  */
-
-static tree
-co_return_expander (tree *stmt, int *do_subtree, void *d)
-{
-  coro_ret_data *data = (coro_ret_data *) d;
-
-  /* To avoid nesting statement lists, walk them and insert as needed.  */
-  if (TREE_CODE (*stmt) == STATEMENT_LIST)
-    {
-      tree_stmt_iterator i;
-      for (i = tsi_start (*stmt); !tsi_end_p (i); tsi_next (&i))
-	{
-	  tree *new_stmt = tsi_stmt_ptr (i);
-	  tree replace = coro_maybe_expand_co_return (*new_stmt, data);
-	  /* If we got something, it will be list and we want to splice
-	     it in.  */
-	  if (replace != NULL_TREE)
-	    {
-	      /* Splice it in ... */
-	      tsi_link_before (&i, replace, TSI_SAME_STMT);
-	      /* ... and delete what we expanded.  */
-	      tsi_delink (&i);
-	      /* Maybe, even likely, we replaced the last in the list.  */
-	      if (tsi_end_p (i))
-		break;
-	    }
-	  else /* Continue the walk.  */
-	    cp_walk_tree (new_stmt, co_return_expander, d, NULL);
-	}
-      *do_subtree = 0; /* Done subtrees.  */
-    }
-  else
-    {
-      /* We might have a single co_return statement, in which case, we do
-	 have to replace it with a list.  */
-      tree replace = coro_maybe_expand_co_return (*stmt, data);
-      if (replace != NULL_TREE)
-	{
-	  *stmt = replace;
-	  *do_subtree = 0; /* Done here.  */
-	}
-    }
-  return NULL_TREE;
-}
-
-/* Walk the original function body, rewriting co_returns.  */
-
-static tree
-expand_co_returns (tree *fnbody, tree promise_proxy, tree promise,
-		   tree fs_label)
-{
-  coro_ret_data data = {promise_proxy, promise, fs_label};
-  cp_walk_tree (fnbody, co_return_expander, &data, NULL);
-  return *fnbody;
-}
-
 /* Support for expansion of co_await statements.  */
 
 struct coro_aw_data
@@ -1527,7 +1414,7 @@ struct coro_aw_data
    awaits.  */
 
 static tree
-co_await_find_in_subtree (tree *stmt, int *do_subtree ATTRIBUTE_UNUSED, void *d)
+co_await_find_in_subtree (tree *stmt, int *, void *d)
 {
   tree **p = (tree **) d;
   if (TREE_CODE (*stmt) == CO_AWAIT_EXPR)
@@ -2090,8 +1977,7 @@ static void
 build_actor_fn (location_t loc, tree coro_frame_type, tree actor, tree fnbody,
 		tree orig, hash_map<tree, param_info> *param_uses,
 		hash_map<tree, local_var_info> *local_var_uses,
-		vec<tree, va_gc> *param_dtor_list,
-		tree fs_label, tree resume_fn_field,
+		vec<tree, va_gc> *param_dtor_list, tree resume_fn_field,
 		unsigned body_count, tree frame_size)
 {
   verify_stmt_tree (fnbody);
@@ -2307,9 +2193,6 @@ build_actor_fn (location_t loc, tree coro_frame_type, tree actor, tree fnbody,
   await_xform_data xform
     = {actor, actor_frame, promise_proxy, ap, self_h_proxy, ash};
 
-  /* Expand co_returns in the saved function body  */
-  fnbody = expand_co_returns (&fnbody, promise_proxy, ap, fs_label);
-
   /* Transform the await expressions in the function body.  Only do each
      await tree once!  */
   hash_set<tree> pset;
@@ -2645,7 +2528,7 @@ register_await_info (tree await_expr, tree aw_type, tree aw_nam)
     = suspend_points->get_or_insert (await_expr, &seen);
   if (seen)
     {
-      error_at (EXPR_LOCATION (await_expr), "duplicate info for %qE",
+      warning_at (EXPR_LOCATION (await_expr), 0, "duplicate info for %qE",
 		await_expr);
       return false;
     }
@@ -2654,26 +2537,14 @@ register_await_info (tree await_expr, tree aw_type, tree aw_nam)
   return true;
 }
 
-/* Small helper for the repetitive task of adding a new field to the coro
-   frame type.  */
-
-static tree
-coro_make_frame_entry (tree *field_list, const char *name, tree fld_type,
-		       location_t loc)
-{
-  tree id = get_identifier (name);
-  tree decl = build_decl (loc, FIELD_DECL, id, fld_type);
-  DECL_CHAIN (decl) = *field_list;
-  *field_list = decl;
-  return id;
-}
-
 /* This data set is used when analyzing statements for await expressions.  */
+
 struct susp_frame_data
 {
   /* Function-wide.  */
   tree *field_list; /* The current coroutine frame field list.  */
   tree handle_type; /* The self-handle type for this coroutine.  */
+  tree fs_label;    /* The destination for co_returns.  */
   vec<tree, va_gc> *block_stack; /* Track block scopes.  */
   vec<tree, va_gc> *bind_stack;  /* Track current bind expr.  */
   unsigned await_number;	 /* Which await in the function.  */
@@ -2685,275 +2556,575 @@ struct susp_frame_data
   unsigned saw_awaits;		 /* Count of awaits in this statement  */
   bool captures_temporary;	 /* This expr captures temps by ref.  */
   bool needs_truth_if_exp;	 /* We must expand a truth_if expression.  */
+  bool has_awaiter_init;	 /* We must handle initializing an awaiter.  */
 };
 
-/* Walk the sub-tree looking for call expressions that both capture
-   references and have compiler-temporaries as parms.  */
+/* If this is an await expression, then count it (both uniquely within the
+   function and locally within a single statement).  */
 
 static tree
-captures_temporary (tree *stmt, int *do_subtree, void *d)
+register_awaits (tree *stmt, int *, void *d)
 {
+  tree aw_expr = *stmt;
+
   /* We should have already lowered co_yields to their co_await.  */
-  gcc_checking_assert (TREE_CODE (*stmt) != CO_YIELD_EXPR);
+  gcc_checking_assert (TREE_CODE (aw_expr) != CO_YIELD_EXPR);
+
+  if (TREE_CODE (aw_expr) != CO_AWAIT_EXPR)
+    return NULL_TREE;
 
-  /* Stop recursing if we see an await expression, the subtrees
-     of that will be handled when it is processed.  */
+  /* Count how many awaits the current expression contains.  */
+  susp_frame_data *data = (susp_frame_data *) d;
+  data->saw_awaits++;
+  /* Each await suspend context is unique, this is a function-wide value.  */
+  data->await_number++;
+
+  /* Awaitables should either be user-locals or promoted to coroutine frame
+     entries at this point, and their initializers should have been broken
+     out.  */
+  tree aw = TREE_OPERAND (aw_expr, 1);
+  gcc_checking_assert (!TREE_OPERAND (aw_expr, 2));
+
+  tree aw_field_type = TREE_TYPE (aw);
+  tree aw_field_nam = NULL_TREE;
+  register_await_info (aw_expr, aw_field_type, aw_field_nam);
+
+  /* Rewrite target expressions on the await_suspend () to remove extraneous
+     cleanups for the awaitables, which are now promoted to frame vars and
+     managed via that.  */
+  tree v = TREE_OPERAND (aw_expr, 3);
+  tree o = TREE_VEC_ELT (v, 1);
+  if (TREE_CODE (o) == TARGET_EXPR)
+    TREE_VEC_ELT (v, 1) = get_target_expr (TREE_OPERAND (o, 1));
+  return NULL_TREE;
+}
+
+/* There are cases where any await expression is relevant.  */
+static tree
+find_any_await (tree *stmt, int *dosub, void *d)
+{
   if (TREE_CODE (*stmt) == CO_AWAIT_EXPR)
     {
-      *do_subtree = 0;
-      return NULL_TREE;
+      *dosub = 0; /* We don't need to consider this any further.  */
+      tree **p = (tree **) d;
+      *p = stmt;
+      return *stmt;
     }
+  return NULL_TREE;
+}
 
-  /* We're only interested in calls.  */
-  if (TREE_CODE (*stmt) != CALL_EXPR)
-    return NULL_TREE;
+static bool
+tmp_target_expr_p (tree t)
+{
+  if (TREE_CODE (t) != TARGET_EXPR)
+    return false;
+  tree v = TREE_OPERAND (t, 0);
+  if (!DECL_ARTIFICIAL (v))
+    return false;
+  if (DECL_NAME (v))
+    return false;
+  return true;
+}
+
+/* Structure to record sub-expressions that need to be handled by the
+   statement flattener.  */
 
-  /* Does this call capture references?
-     Strip the ADDRESS_EXPR to get the fn decl and inspect it.  */
-  tree fn = TREE_OPERAND (CALL_EXPR_FN (*stmt), 0);
-  bool is_meth = TREE_CODE (TREE_TYPE (fn)) == METHOD_TYPE;
-  tree arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
-  unsigned offset = 3;
-  for (unsigned anum = 0; arg != NULL; arg = TREE_CHAIN (arg), anum++)
+struct coro_interesting_subtree
+{
+  tree* entry;
+  hash_set<tree> *temps_used;
+};
+
+/* tree-walk callback that returns the first encountered sub-expression of
+   a kind that needs to be handled specifically by the statement flattener.  */
+
+static tree
+find_interesting_subtree (tree *expr_p, int *dosub, void *d)
+{
+  tree expr = *expr_p;
+  coro_interesting_subtree *p = (coro_interesting_subtree *)d;
+  if (TREE_CODE (expr) == CO_AWAIT_EXPR)
     {
-      tree parm_type = TREE_VALUE (arg);
-      if (anum == 0 && is_meth && INDIRECT_TYPE_P (parm_type))
+      *dosub = 0; /* We don't need to consider this any further.  */
+      if (TREE_OPERAND (expr, 2))
 	{
-	  /* Account for 'this' when the fn is a method.  Unless it
-	     belongs to a CTOR or DTOR.  */
-	  if (DECL_CONSTRUCTOR_P (fn) || DECL_DESTRUCTOR_P (fn))
-	    continue;
+	  p->entry = expr_p;
+	  return expr;
 	}
-      else if (!TYPE_REF_P (parm_type))
-	/* If it's not a reference, we don't care.  */
-	continue;
+    }
+  else if (tmp_target_expr_p (expr)
+	   && !p->temps_used->contains (expr))
+    {
+      p->entry = expr_p;
+      return expr;
+    }
 
-      /* Fetch the value presented to the fn.  */
-      tree parm = TREE_OPERAND (*stmt, anum + offset);
+  return NULL_TREE;
+}
 
-      while (TREE_CODE (parm) == NOP_EXPR)
-	parm = TREE_OPERAND (parm, 0);
+/* Node for a doubly-linked list of promoted variables and their
+   initializers.  When the initializer is a conditional expression
+   the 'then' and 'else' clauses are represented by a linked list
+   attached to then_cl and else_cl respectively.  */
 
-      /* We only care if we're taking the addr of a temporary.  */
-      if (TREE_CODE (parm) != ADDR_EXPR)
-	continue;
+struct var_nest_node
+{
+  var_nest_node () = default;
+  var_nest_node (tree v, tree i, var_nest_node *p, var_nest_node *n)
+    : var(v), init(i), prev(p), next(n)
+    {
+      if (p)
+	p->next = this;
+      if (n)
+	n->prev = this;
+    }
+  tree var;
+  tree init;
+  var_nest_node *prev;
+  var_nest_node *next;
+  var_nest_node *then_cl;
+  var_nest_node *else_cl;
+};
 
-      parm = TREE_OPERAND (parm, 0);
+/* This is called for single statements from the co-await statement walker.
+   It checks to see if the statement contains any initializers for awaitables
+   and if any of these capture items by reference.  */
 
-      /* In case of component_ref, we need to capture the object of base
-	 class as if it is temporary object.  There are two possibilities:
-	 (*base).field and base->field.  */
-      while (TREE_CODE (parm) == COMPONENT_REF)
+static void
+flatten_await_stmt (var_nest_node *n, hash_set<tree> *promoted,
+		    hash_set<tree> *temps_used, tree *replace_in)
+{
+  bool init_expr = false;
+  switch (TREE_CODE (n->init))
+    {
+      default: break;
+      /* Compound expressions must be flattened specifically.  */
+      case COMPOUND_EXPR:
 	{
-	  parm = TREE_OPERAND (parm, 0);
-	  if (TREE_CODE (parm) == INDIRECT_REF)
-	    parm = TREE_OPERAND (parm, 0);
-	  STRIP_NOPS (parm);
+	  tree first = TREE_OPERAND (n->init, 0);
+	  n->init = TREE_OPERAND (n->init, 1);
+	  var_nest_node *ins
+	    = new var_nest_node(NULL_TREE, first, n->prev, n);
+	  /* The compiler (but not the user) can generate temporaries with
+	     uses in the second arm of a compound expr.  */
+	  flatten_await_stmt (ins, promoted, temps_used, &n->init);
+	  flatten_await_stmt (n, promoted, temps_used, NULL);
+	  /* The two arms have been processed separately.  */
+	  return;
 	}
+	break;
+      /* Handle conditional expressions.  */
+      case INIT_EXPR:
+	init_expr = true;
+	/* FALLTHROUGH */
+      case MODIFY_EXPR:
+	{
+	  tree old_expr = TREE_OPERAND (n->init, 1);
+	  if (TREE_CODE (old_expr) == COMPOUND_EXPR)
+	    {
+	      tree first = TREE_OPERAND (old_expr, 0);
+	      TREE_OPERAND (n->init, 1) = TREE_OPERAND (old_expr, 1);
+	      var_nest_node *ins
+		= new var_nest_node(NULL_TREE, first, n->prev, n);
+	      flatten_await_stmt (ins, promoted, temps_used,
+				  &TREE_OPERAND (n->init, 1));
+	      flatten_await_stmt (n, promoted, temps_used, NULL);
+	      return;
+	    }
+	  if (TREE_CODE (old_expr) != COND_EXPR)
+	    break;
+	  /* Reconstruct x = t ? y : z;
+	     as (void) t ? x = y : x = z;  */
+	  tree var = TREE_OPERAND (n->init, 0);
+	  tree var_type = TREE_TYPE (var);
+	  tree cond = COND_EXPR_COND (old_expr);
+	  /* We are allowed a void type throw in one or both of the cond
+	     expr arms.  */
+	  tree then_cl = COND_EXPR_THEN (old_expr);
+	  if (!VOID_TYPE_P (TREE_TYPE (then_cl)))
+	    {
+	      gcc_checking_assert (TREE_CODE (then_cl) != STATEMENT_LIST);
+	      then_cl
+		= build2 (init_expr ? INIT_EXPR : MODIFY_EXPR, var_type,
+			  var, then_cl);
+	    }
+	  tree else_cl = COND_EXPR_ELSE (old_expr);
+	  if (!VOID_TYPE_P (TREE_TYPE (else_cl)))
+	    {
+	      gcc_checking_assert (TREE_CODE (then_cl) != STATEMENT_LIST);
+	      else_cl
+		= build2 (init_expr ? INIT_EXPR : MODIFY_EXPR, var_type,
+			  var, else_cl);
+	    }
+	  n->init = build3 (COND_EXPR, var_type, cond, then_cl, else_cl);
+	}
+	/* FALLTHROUGH */
+      case COND_EXPR:
+	{
+	  tree *found;
+	  tree cond = COND_EXPR_COND (n->init);
+	  /* If the condition contains an await expression, then we need to
+	     set that first and use a separate var.  */
+	  if (cp_walk_tree (&cond, find_any_await, &found, NULL))
+	    {
+	      tree cond_type = TREE_TYPE (cond);
+	      tree cond_var  = build_lang_decl (VAR_DECL, NULL_TREE, cond_type);
+	      DECL_ARTIFICIAL (cond_var) = true;
+	      layout_decl (cond_var, 0);
+	      gcc_checking_assert (!TYPE_NEEDS_CONSTRUCTING (cond_type));
+	      cond = build2 (INIT_EXPR, cond_type, cond_var, cond);
+	      var_nest_node *ins
+		= new var_nest_node (cond_var, cond, n->prev, n);
+	      COND_EXPR_COND (n->init) = cond_var;
+	      flatten_await_stmt (ins, promoted, temps_used, NULL);
+	    }
 
-      /* This isn't a temporary.  */
-      if ((VAR_P (parm)
-	   && (!DECL_ARTIFICIAL (parm) || DECL_HAS_VALUE_EXPR_P (parm)))
-	  || TREE_CODE (parm) == PARM_DECL
-	  || TREE_CODE (parm) == NON_LVALUE_EXPR)
-	continue;
-
-      if (TREE_CODE (parm) == TARGET_EXPR)
+	  n->then_cl
+	    = new var_nest_node (n->var, COND_EXPR_THEN (n->init), NULL, NULL);
+	  n->else_cl
+	    = new var_nest_node (n->var, COND_EXPR_ELSE (n->init), NULL, NULL);
+	  flatten_await_stmt (n->then_cl, promoted, temps_used, NULL);
+	  /* Point to the start of the flattened code.  */
+	  while (n->then_cl->prev)
+	    n->then_cl = n->then_cl->prev;
+	  flatten_await_stmt (n->else_cl, promoted, temps_used, NULL);
+	  while (n->else_cl->prev)
+	    n->else_cl = n->else_cl->prev;
+	  return;
+	}
+	break;
+    }
+  coro_interesting_subtree v = { NULL, temps_used };
+  tree t = cp_walk_tree (&n->init, find_interesting_subtree, (void *)&v, NULL);
+  if (!t)
+    return;
+  switch (TREE_CODE (t))
+    {
+      default: break;
+      case CO_AWAIT_EXPR:
 	{
-	  /* We're taking the address of a temporary and using it as a ref.  */
-	  tree tvar = TREE_OPERAND (parm, 0);
-	  gcc_checking_assert (DECL_ARTIFICIAL (tvar));
-
-	  susp_frame_data *data = (susp_frame_data *) d;
-	  data->captures_temporary = true;
-	  /* Record this one so we don't duplicate, and on the first
-	     occurrence note the target expr to be replaced.  */
-	  if (!data->captured_temps.add (tvar))
-	    vec_safe_push (data->to_replace, parm);
-	  /* Now see if the initializer contains any more cases.  */
-	  hash_set<tree> visited;
-	  tree res = cp_walk_tree (&TREE_OPERAND (parm, 1),
-				   captures_temporary, d, &visited);
-	  if (res)
-	    return res;
-	  /* Otherwise, we're done with sub-trees for this.  */
+	  /* Await expressions with initializers have a compiler-temporary
+	     as the awaitable.  'promote' this.  */
+	  tree var = TREE_OPERAND (t, 1);
+	  bool already_present = promoted->add (var);
+	  gcc_checking_assert (!already_present);
+	  tree init = TREE_OPERAND (t, 2);
+	  switch (TREE_CODE (init))
+	    {
+	      default: break;
+	      case INIT_EXPR:
+	      case MODIFY_EXPR:
+		{
+		  tree inner = TREE_OPERAND (init, 1);
+		  /* We can have non-lvalue-expressions here, but when we see
+		     a target expression, mark it as already used.  */
+		  if (TREE_CODE (inner) == TARGET_EXPR)
+		    {
+		      temps_used->add (inner);
+		      gcc_checking_assert
+			(TREE_CODE (TREE_OPERAND (inner, 1)) != COND_EXPR);
+		    }
+		}
+		break;
+	      case CALL_EXPR:
+		/* If this is a call and not a CTOR, then we didn't expect it.  */
+		gcc_checking_assert
+		  (DECL_CONSTRUCTOR_P (TREE_OPERAND (CALL_EXPR_FN (init), 0)));
+		break;
+	    }
+	  var_nest_node *ins = new var_nest_node (var, init, n->prev, n);
+	  TREE_OPERAND (t, 2) = NULL_TREE;
+	  flatten_await_stmt (ins, promoted, temps_used, NULL);
+	  flatten_await_stmt (n, promoted, temps_used, NULL);
+	  return;
 	}
-      else if (TREE_CODE (parm) == CO_AWAIT_EXPR)
+	break;
+      case TARGET_EXPR:
 	{
-	  /* CO_AWAIT expressions behave in a similar manner to target
-	     expressions when the await_resume call is contained in one.  */
-	  tree awr = TREE_OPERAND (parm, 3); /* call vector.  */
-	  awr = TREE_VEC_ELT (awr, 2); /* resume call.  */
-	  if (TREE_CODE (awr) == TARGET_EXPR)
+	  /* We have a temporary; promote it.  */
+	  tree init = t;
+	  temps_used->add (init);
+	  tree var_type = TREE_TYPE (init);
+	  char *buf = xasprintf ("D.%d", DECL_UID (TREE_OPERAND (init, 0)));
+	  tree var = build_lang_decl (VAR_DECL, get_identifier (buf), var_type);
+	  DECL_ARTIFICIAL (var) = true;
+	  free (buf);
+	  bool already_present = promoted->add (var);
+	  gcc_checking_assert (!already_present);
+	  tree inner = TREE_OPERAND (init, 1);
+	  gcc_checking_assert (TREE_CODE (inner) != COND_EXPR);
+	  if (TYPE_NEEDS_CONSTRUCTING (var_type))
 	    {
-	      tree tvar = TREE_OPERAND (awr, 0);
-	      gcc_checking_assert (DECL_ARTIFICIAL (tvar));
-
-	      susp_frame_data *data = (susp_frame_data *) d;
-	      data->captures_temporary = true;
-	      /* Use this as a place-holder.  */
-	      if (!data->captured_temps.add (tvar))
-		vec_safe_push (data->to_replace, parm);
+	      releasing_vec p_in (make_tree_vector_single (init));
+	      init = build_special_member_call (var, complete_ctor_identifier,
+						&p_in, var_type, LOOKUP_NORMAL,
+						tf_warning_or_error);
 	    }
-	/* We will walk the sub-trees of this co_await separately.  */
+	  else
+	    init = build2 (INIT_EXPR, var_type, var, init);
+	  var_nest_node *ins
+	    = new var_nest_node (var, init, n->prev, n);
+	  /* We have to replace the target expr... */
+	  proxy_replace pr = {TREE_OPERAND (t, 0), var};
+	  *v.entry = var;
+	  /* ... and any uses of its var.  */
+	  cp_walk_tree (&n->init, replace_proxy, &pr, NULL);
+	  /* Compiler-generated temporaries can also have uses in following
+	     arms of compound expressions, which will be listed in 'replace_in'
+	     if present.  */
+	  if (replace_in)
+	    cp_walk_tree (replace_in, replace_proxy, &pr, NULL);
+	  flatten_await_stmt (ins, promoted, temps_used, NULL);
+	  flatten_await_stmt (n, promoted, temps_used, NULL);
+	  return;
 	}
-      else
-	gcc_unreachable ();
+	break;
     }
-  /* As far as it's necessary, we've walked the subtrees of the call
-     expr.  */
-  *do_subtree = 0;
-  return NULL_TREE;
 }
 
-/* If this is an await, then register it and decide on what coro
-   frame storage is needed.
-   If this is a co_yield (which embeds an await), drop the yield
-   and record the await (the yield was kept for diagnostics only).  */
+/* Helper for 'process_conditional' that handles recursion into nested
+   conditionals.  */
 
-static tree
-register_awaits (tree *stmt, int *do_subtree ATTRIBUTE_UNUSED, void *d)
+static void
+handle_nested_conditionals (var_nest_node *n, vec<tree>& list,
+			    hash_map<tree, tree>& map)
 {
-  susp_frame_data *data = (susp_frame_data *) d;
+  do
+    {
+      if (n->var && DECL_NAME (n->var))
+	{
+	  list.safe_push (n->var);
+	  if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (TREE_TYPE (n->var)))
+	    {
+	      bool existed;
+	      tree& flag = map.get_or_insert (n->var, &existed);
+	      if (!existed)
+		{
+		  /* We didn't see this var before and it needs a DTOR, so
+		     build a guard variable for it.  */
+		  char *nam
+		    = xasprintf ("%s_guard",
+				 IDENTIFIER_POINTER (DECL_NAME (n->var)));
+		  flag = build_lang_decl (VAR_DECL, get_identifier (nam),
+					  boolean_type_node);
+		  free (nam);
+		  DECL_ARTIFICIAL (flag) = true;
+		}
+
+	      /* The initializer for this variable is replaced by a compound
+		 expression that performs the init and then records that the
+		 variable is live (and the DTOR should be run at the scope
+		 exit.  */
+	      tree set_flag = build2 (INIT_EXPR, boolean_type_node,
+				      flag, boolean_true_node);
+	      n->init
+		= build2 (COMPOUND_EXPR, boolean_type_node, n->init, set_flag);
+	}
+	}
+      if (TREE_CODE (n->init) == COND_EXPR)
+	{
+	  tree new_then = push_stmt_list ();
+	  handle_nested_conditionals (n->then_cl, list, map);
+	  new_then = pop_stmt_list (new_then);
+	  tree new_else = push_stmt_list ();
+	  handle_nested_conditionals (n->else_cl, list, map);
+	  new_else = pop_stmt_list (new_else);
+	  tree new_if
+	    = build4 (IF_STMT, void_type_node, COND_EXPR_COND (n->init),
+		      new_then, new_else, NULL_TREE);
+	  add_stmt (new_if);
+	}
+      else
+	finish_expr_stmt (n->init);
+      n = n->next;
+    } while (n);
+}
 
-  /* We should have already lowered co_yields to their co_await.  */
-  gcc_checking_assert (TREE_CODE (*stmt) != CO_YIELD_EXPR);
+/* helper for 'maybe_promote_temps'.
 
-  if (TREE_CODE (*stmt) != CO_AWAIT_EXPR)
-    return NULL_TREE;
+   When we have a conditional expression which might embed await expressions
+   and/or promoted variables, we need to handle it appropriately.
 
-  tree aw_expr = *stmt;
-  location_t aw_loc = EXPR_LOCATION (aw_expr); /* location of the co_xxxx.  */
+   The linked lists for the 'then' and 'else' clauses in a conditional node
+   identify the promoted variables (but these cannot be wrapped in a regular
+   cleanup).
 
-  /* If the awaitable is a parm or a local variable, then we already have
-     a frame copy, so don't make a new one.  */
-  tree aw = TREE_OPERAND (aw_expr, 1);
-  tree o = TREE_OPERAND (aw_expr, 2); /* Initializer for the frame var.  */
-  /* If we have an initializer, then the var is a temp and we need to make
-     it in the frame.  */
-  tree aw_field_type = TREE_TYPE (aw);
-  tree aw_field_nam = NULL_TREE;
-  if (o)
+   So recurse through the lists and build up a composite list of captured vars.
+   Declare these and any guard variables needed to decide if a DTOR should be
+   run.  Then embed the conditional into a try-finally expression that handles
+   running each DTOR conditionally on its guard variable.  */
+
+static void
+process_conditional (var_nest_node *n, tree& vlist)
+{
+  tree init = n->init;
+  hash_map<tree, tree> var_flags;
+  vec<tree> var_list = vNULL;
+  tree new_then = push_stmt_list ();
+  handle_nested_conditionals (n->then_cl, var_list, var_flags);
+  new_then = pop_stmt_list (new_then);
+  tree new_else = push_stmt_list ();
+  handle_nested_conditionals (n->else_cl, var_list, var_flags);
+  new_else = pop_stmt_list (new_else);
+  /* Declare the vars.  There are two loops so that the boolean flags are
+     grouped in the frame.  */
+  for (unsigned i = 0; i < var_list.length(); i++)
     {
-      /* The required field has the same type as the proxy stored in the
-	 await expr.  */
-      char *nam = xasprintf ("__aw_s.%d", data->await_number);
-      aw_field_nam = coro_make_frame_entry (data->field_list, nam,
-					    aw_field_type, aw_loc);
-      free (nam);
-
-      /* If the init is a target expression, then we need to remake it to
-	 strip off any extra cleanups added.  */
-      if (o && TREE_CODE (o) == TARGET_EXPR)
-	TREE_OPERAND (aw_expr, 2) = get_target_expr (TREE_OPERAND (o, 1));
+      tree var = var_list[i];
+      DECL_CHAIN (var) = vlist;
+      vlist = var;
+      add_decl_expr (var);
     }
+  /* Define the guard flags for variables that need a DTOR.  */
+  for (unsigned i = 0; i < var_list.length(); i++)
+    {
+      tree *flag = var_flags.get (var_list[i]);
+      if (flag)
+	{
+	  DECL_INITIAL (*flag) = boolean_false_node;
+	  DECL_CHAIN (*flag) = vlist;
+	  vlist = *flag;
+	  add_decl_expr (*flag);
+	}
+    }
+  tree new_if
+    = build4 (IF_STMT, void_type_node, COND_EXPR_COND (init),
+	      new_then, new_else, NULL_TREE);
+  /* Build a set of conditional DTORs.  */
+  tree final_actions = push_stmt_list ();
+  while (!var_list.is_empty())
+    {
+      tree var = var_list.pop ();
+      tree *flag = var_flags.get (var);
+      if (!flag)
+	continue;
+      tree var_type = TREE_TYPE (var);
+      tree cleanup
+	= build_special_member_call (var, complete_dtor_identifier,
+				     NULL, var_type, LOOKUP_NORMAL,
+				     tf_warning_or_error);
+      tree cond_cleanup = begin_if_stmt ();
+      finish_if_stmt_cond (*flag, cond_cleanup);
+      finish_expr_stmt (cleanup);
+      finish_then_clause (cond_cleanup);
+      finish_if_stmt (cond_cleanup);
+    }
+  final_actions = pop_stmt_list (final_actions);
+  tree try_finally
+    = build2 (TRY_FINALLY_EXPR, void_type_node, new_if, final_actions);
+  add_stmt (try_finally);
+}
 
-  tree v = TREE_OPERAND (aw_expr, 3);
-  o = TREE_VEC_ELT (v, 1);
-  if (TREE_CODE (o) == TARGET_EXPR)
-    TREE_VEC_ELT (v, 1) = get_target_expr (TREE_OPERAND (o, 1));
-
-  register_await_info (aw_expr, aw_field_type, aw_field_nam);
+/* Given *STMT, that contains at least one await expression.
 
-  /* Count how many awaits the current expression contains.  */
-  data->saw_awaits++;
-  /* Each await suspend context is unique, this is a function-wide value.  */
-  data->await_number++;
+   The full expression represented in the original source code will contain
+   suspension points, but it is still required that the lifetime of temporary
+   values extends to the end of the expression.
 
-  /* We now need to know if to take special action on lifetime extension
-     of temporaries captured by reference.  This can only happen if such
-     a case appears in the initializer for the awaitable.  The callback
-     records captured temporaries including subtrees of initializers.  */
-  hash_set<tree> visited;
-  tree res = cp_walk_tree (&TREE_OPERAND (aw_expr, 2), captures_temporary, d,
-			   &visited);
-  return res;
-}
+   We already have a mechanism to 'promote' user-authored local variables
+   to a coroutine frame counterpart (which allows explicit management of the
+   lifetime across suspensions).  The transform here re-writes STMT into
+   a bind expression, promotes temporary values into local variables in that
+   and flattens the statement into a series of cleanups.
 
-/* The gimplifier correctly extends the lifetime of temporaries captured
-   by reference (per. [class.temporary] (6.9) "A temporary object bound
-   to a reference parameter in a function call persists until the completion
-   of the full-expression containing the call").  However, that is not
-   sufficient to work across a suspension - and we need to promote such
-   temporaries to be regular vars that will then get a coro frame slot.
-   We don't want to incur the effort of checking for this unless we have
-   an await expression in the current full expression.  */
-
-/* This takes the statement which contains one or more temporaries that have
-   been 'captured' by reference in the initializer(s) of co_await(s).
-   The statement is replaced by a bind expression that has actual variables
-   to replace the temporaries.  These variables will be added to the coro-
-   frame in the same manner as user-authored ones.  */
+   Conditional expressions are re-written to regular 'if' statements.
+   The cleanups for variables initialized inside a conditional (including
+   nested cases) are wrapped in a try-finally clause, with guard variables
+   to determine which DTORs need to be run.  */
 
-static void
-replace_statement_captures (tree *stmt, void *d)
+static tree
+maybe_promote_temps (tree *stmt, void *d)
 {
   susp_frame_data *awpts = (susp_frame_data *) d;
+
   location_t sloc = EXPR_LOCATION (*stmt);
-  tree aw_bind
-    = build3_loc (sloc, BIND_EXPR, void_type_node, NULL, NULL, NULL);
-
-  /* Any cleanup point expression might no longer be necessary, since we
-     are removing one or more temporaries.  */
-  tree aw_statement_current = *stmt;
-  if (TREE_CODE (aw_statement_current) == CLEANUP_POINT_EXPR)
-    aw_statement_current = TREE_OPERAND (aw_statement_current, 0);
-
-  /* Collected the scope vars we need move the temps to regular. */
-  tree aw_bind_body = push_stmt_list ();
-  tree varlist = NULL_TREE;
-  int vnum = -1;
-  while (!awpts->to_replace->is_empty ())
+  tree expr = *stmt;
+  /* Strip off uninteresting wrappers.  */
+  if (TREE_CODE (expr) == CLEANUP_POINT_EXPR)
+    expr = TREE_OPERAND (expr, 0);
+  if (TREE_CODE (expr) == EXPR_STMT)
+    expr = EXPR_STMT_EXPR (expr);
+  if (TREE_CODE (expr) == CONVERT_EXPR
+      && VOID_TYPE_P (TREE_TYPE (expr)))
+    expr = TREE_OPERAND (expr, 0);
+  STRIP_NOPS (expr);
+
+  /* We walk the statement trees, flattening it into an ordered list of
+     variables with initializers and fragments corresponding to compound
+     expressions, truth or/and if and ternary conditionals.  Conditional
+     expressions carry a nested list of fragments for the then and else
+     clauses.  We anchor to the 'bottom' of the fragment list; we will write
+     a cleanup nest with one shell for each variable initialized.  */
+  var_nest_node *root = new var_nest_node (NULL_TREE, expr, NULL, NULL);
+  /* Check to see we didn't promote one twice.  */
+  hash_set<tree> promoted_vars;
+  hash_set<tree> used_temps;
+  flatten_await_stmt (root, &promoted_vars, &used_temps, NULL);
+
+  gcc_checking_assert (root->next == NULL);
+  tree vlist = NULL_TREE;
+  var_nest_node *t = root;
+  gcc_checking_assert (!t->var);
+  /* We build the bind scope expression from the bottom-up.
+     EXPR_LIST holds the inner expression nest at the current cleanup
+     level (becoming the final expression list when we've exhausted the
+     number of sub-expression fragments).  */
+  tree expr_list = NULL_TREE;
+  do
     {
-      tree to_replace = awpts->to_replace->pop ();
-      tree orig_temp;
-      if (TREE_CODE (to_replace) == CO_AWAIT_EXPR)
+      tree new_list = push_stmt_list ();
+      /* When we have a promoted variable, then add that to the bind scope
+	 and initialize it.  When there's no promoted variable, we just need
+	 to run the initializer.
+	 If the initializer is a conditional expression, we need to collect
+	 and declare any promoted variables nested within it.  DTORs for such
+	 variables must be run conditionally too.  */
+      if (t->var && DECL_NAME (t->var))
 	{
-	  orig_temp = TREE_OPERAND (to_replace, 3);
-	  orig_temp = TREE_VEC_ELT (orig_temp, 2);
-	  orig_temp = TREE_OPERAND (orig_temp, 0);
+	  tree var = t->var;
+	  DECL_CHAIN (var) = vlist;
+	  vlist = var;
+	  add_decl_expr (var);
+	  if (TREE_CODE (t->init) == COND_EXPR)
+	    process_conditional (t, vlist);
+	  else
+	    finish_expr_stmt (t->init);
+	  tree var_type = TREE_TYPE (var);
+	  if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (var_type))
+	    {
+	      tree cleanup
+		= build_special_member_call (var, complete_dtor_identifier,
+					     NULL, var_type, LOOKUP_NORMAL,
+					     tf_warning_or_error);
+	      tree cl = build_stmt (sloc, CLEANUP_STMT, expr_list, cleanup, var);
+	      add_stmt (cl); /* push this onto the level above.  */
+	    }
+	  else if (expr_list)
+	    add_stmt (expr_list);
+	  else
+	    gcc_unreachable ();
 	}
       else
-	orig_temp = TREE_OPERAND (to_replace, 0);
-
-      tree var_type = TREE_TYPE (orig_temp);
-      gcc_checking_assert (same_type_p (TREE_TYPE (to_replace), var_type));
-      /* Build a variable to hold the captured value, this will be included
-	 in the frame along with any user-authored locals.  */
-      char *nam = xasprintf ("aw_%d.tmp.%d", awpts->await_number, ++vnum);
-      tree newvar = build_lang_decl (VAR_DECL, get_identifier (nam), var_type);
-      free (nam);
-      /* If we have better location than the whole expression use that, else
-	 fall back to the expression loc.  */
-      DECL_CONTEXT (newvar) = DECL_CONTEXT (orig_temp);
-      if (DECL_SOURCE_LOCATION (orig_temp))
-	sloc = DECL_SOURCE_LOCATION (orig_temp);
-     else
-	sloc = EXPR_LOCATION (*stmt);
-      DECL_SOURCE_LOCATION (newvar) = sloc;
-      DECL_CHAIN (newvar) = varlist;
-      varlist = newvar; /* Chain it onto the list for the bind expr.  */
-      /* Declare and initialize it in the new bind scope.  */
-      add_decl_expr (newvar);
-      tree new_s = build2_loc (sloc, INIT_EXPR, var_type, newvar, to_replace);
-      new_s = coro_build_cvt_void_expr_stmt (new_s, sloc);
-      add_stmt (new_s);
-
-     /* Replace all instances of that temp in the original expr.  */
-      proxy_replace pr = {to_replace, newvar};
-       cp_walk_tree (&aw_statement_current, replace_proxy, &pr, NULL);
-    }
-
-  /* What's left should be the original statement with any co_await captured
-     temporaries broken out.  Other temporaries might remain so see if we
-     need to wrap the revised statement in a cleanup.  */
-  aw_statement_current = maybe_cleanup_point_expr_void (aw_statement_current);
-  add_stmt (aw_statement_current);
-
-  BIND_EXPR_BODY (aw_bind) = pop_stmt_list (aw_bind_body);
-  awpts->captured_temps.empty ();
-
-  BIND_EXPR_VARS (aw_bind) = nreverse (varlist);
+	{
+	  if (TREE_CODE (t->init) == COND_EXPR)
+	    process_conditional (t, vlist);
+	  else
+	    finish_expr_stmt (t->init);
+	  if (expr_list)
+	    add_stmt (expr_list);
+	}
+      expr_list = pop_stmt_list (new_list);
+      var_nest_node *old = t;
+      t = t->prev;
+      delete old;
+    } while (t);
+
+  /* Now produce the bind expression containing the 'promoted' temporaries
+     as its variable list, and the cleanup nest as the statement.  */
+  tree await_bind = build3_loc (sloc, BIND_EXPR, void_type_node,
+				NULL, NULL, NULL);
+  BIND_EXPR_BODY (await_bind) = expr_list;
+  BIND_EXPR_VARS (await_bind) = nreverse (vlist);
   tree b_block = make_node (BLOCK);
   if (!awpts->block_stack->is_empty ())
     {
@@ -2965,37 +3136,12 @@ replace_statement_captures (tree *stmt, void *d)
 	BLOCK_SUBBLOCKS (s_block) = b_block;
 	}
     }
-  BIND_EXPR_BLOCK (aw_bind) = b_block;
-  TREE_SIDE_EFFECTS (aw_bind) = TREE_SIDE_EFFECTS (BIND_EXPR_BODY (aw_bind));
-  *stmt = aw_bind;
-}
-
-/* This is called for single statements from the co-await statement walker.
-   It checks to see if the statement contains any co-awaits and, if so,
-   whether any of these 'capture' a temporary by reference.  */
-
-static tree
-maybe_promote_captured_temps (tree *stmt, void *d)
-{
-  susp_frame_data *awpts = (susp_frame_data *) d;
+  BLOCK_VARS (b_block) = BIND_EXPR_VARS (await_bind) ;
+  BIND_EXPR_BLOCK (await_bind) = b_block;
+  TREE_SIDE_EFFECTS (await_bind) = TREE_SIDE_EFFECTS (BIND_EXPR_BODY (await_bind));
+  *stmt = await_bind;
   hash_set<tree> visited;
-  awpts->saw_awaits = 0;
-
-  /* When register_awaits sees an await, it walks the initializer for
-     that await looking for temporaries captured by reference and notes
-     them in awpts->captured_temps.  */
-
-  if (tree res = cp_walk_tree (stmt, register_awaits, d, &visited))
-    return res; /* We saw some reason to abort the tree walk.  */
-
-  /* We only need to take any action here if the statement contained any
-     awaits and any of those had temporaries captured by reference in their
-     initializers. */
-
-  if (awpts->saw_awaits > 0 && !awpts->captured_temps.is_empty ())
-    replace_statement_captures (stmt, d);
-
-  return NULL_TREE;
+  return cp_walk_tree (stmt, register_awaits, d, &visited);
 }
 
 /* Lightweight callback to determine two key factors:
@@ -3004,6 +3150,7 @@ maybe_promote_captured_temps (tree *stmt, void *d)
       TRUTH_{AND,OR}IF_EXPRs since, in most cases, they will need expansion
       so that the await expressions are not processed in the case of the
       short-circuit arm.
+
    CO_YIELD expressions are re-written to their underlying co_await.  */
 
 static tree
@@ -3020,6 +3167,9 @@ analyze_expression_awaits (tree *stmt, int *do_subtree, void *d)
 	/* FALLTHROUGH */
       case CO_AWAIT_EXPR:
 	awpts->saw_awaits++;
+	/* A non-null initializer for the awaiter means we need to expand.  */
+	if (TREE_OPERAND (*stmt, 2))
+	  awpts->has_awaiter_init = true;
 	break;
       case TRUTH_ANDIF_EXPR:
       case TRUTH_ORIF_EXPR:
@@ -3053,15 +3203,13 @@ analyze_expression_awaits (tree *stmt, int *do_subtree, void *d)
 
 /* Given *EXPR
    If EXPR contains a TRUTH_{AND,OR}IF_EXPR, TAOIE with an await expr on
-   the conditional branch expand this to:
+   the conditionally executed branch, change this in a ternary operator.
 
    bool not_expr = TAOIE == TRUTH_ORIF_EXPR ? NOT : NOP;
-   A) bool t = always exec expr
-      if (not_expr (t))
-   B)   t = conditionally exec expr
-   c) EXPR' = EXPR with  TAOIE replaced by t.
+   not_expr (always-exec expr) ? conditionally-exec expr : not_expr;
 
-   Then repeat this for A, B and C.  */
+   Apply this recursively to the condition and the conditionally-exec
+   branch.  */
 
 struct truth_if_transform {
   tree *orig_stmt;
@@ -3087,47 +3235,26 @@ expand_one_truth_if (tree *expr, int *do_subtree, void *d)
 	    break;
 
 	  location_t sloc = EXPR_LOCATION (*expr);
-	  tree type = TREE_TYPE (xform->scratch_var);
-	  gcc_checking_assert (TREE_CODE (type) == BOOLEAN_TYPE);
-	  tree new_list = push_stmt_list ();
-	  /* Init our scratch with the unconditionally-evaluated expr.  */
-	  tree new_s = build2_loc (sloc, INIT_EXPR, boolean_type_node,
-				   xform->scratch_var,
-				   TREE_OPERAND (*expr, 0));
-	  finish_expr_stmt (new_s);
-	  tree *pre = tsi_stmt_ptr (tsi_last (new_list));
-	  tree if_cond = xform->scratch_var;
+	  /* Transform truth expression into a cond expression with
+	     * the always-executed arm as the condition.
+	     * the conditionally-executed arm as the then clause.
+	     * the 'else' clause is fixed: 'true' for ||,'false' for &&.  */
+	  tree cond = TREE_OPERAND (*expr, 0);
+	  tree test1 = TREE_OPERAND (*expr, 1);
+	  tree fixed = needs_not ? boolean_true_node : boolean_false_node;
 	  if (needs_not)
-	    if_cond = build1 (TRUTH_NOT_EXPR, boolean_type_node, if_cond);
-	  tree if_stmt = begin_if_stmt ();
-	  finish_if_stmt_cond (if_cond, if_stmt);
-	  /* If we take the if branch, then overwrite scratch with the cond
-	     executed branch.  */
-	  new_s = build2 (INIT_EXPR, boolean_type_node,
-			  xform->scratch_var, TREE_OPERAND (*expr, 1));
-	  finish_expr_stmt (new_s);
-	  finish_then_clause (if_stmt);
-	  finish_if_stmt (if_stmt);
-	  *expr = xform->scratch_var; /* now contains the result.  */
-	  /* So now we've got a statement list expanding one TAOIe.  */
-	  add_stmt (*xform->orig_stmt);
-	  tree *post = tsi_stmt_ptr (tsi_last (new_list));
-	  *xform->orig_stmt = pop_stmt_list (new_list);
-	  /* Now recurse into the pre, if and post parts.  */
-	  truth_if_transform sub_data = {pre, xform->scratch_var,
-					 xform->truth_aoif_to_expand};
-	  if (tree res = cp_walk_tree (pre, expand_one_truth_if, &sub_data,
-				       NULL))
-	    return res;
-	  sub_data.orig_stmt = &THEN_CLAUSE (if_stmt);
-	  if (tree res = cp_walk_tree (&THEN_CLAUSE (if_stmt),
-				       expand_one_truth_if, &sub_data, NULL))
+	    cond = build1 (TRUTH_NOT_EXPR, boolean_type_node, cond);
+	  tree cond_expr
+	    = build3_loc (sloc, COND_EXPR, boolean_type_node,
+			  cond, test1, fixed);
+	  *expr = cond_expr;
+	  if (tree res = cp_walk_tree (&COND_EXPR_COND (*expr),
+				       expand_one_truth_if, d, NULL))
 	    return res;
-	  sub_data.orig_stmt = post;
-	  if (tree res = cp_walk_tree (post, expand_one_truth_if, &sub_data,
-				       NULL))
+	  if (tree res = cp_walk_tree (&COND_EXPR_THEN (*expr),
+				       expand_one_truth_if, d, NULL))
 	    return res;
-	  /* We've done the sub-trees here.  */
+	  /* We've manually processed necessary sub-trees here.  */
 	  *do_subtree = 0;
 	}
 	break;
@@ -3142,7 +3269,6 @@ static tree
 add_var_to_bind (tree& bind, tree var_type,
 		 const char *nam_root, unsigned nam_vers)
 {
-
   tree b_vars = BIND_EXPR_VARS (bind);
   /* Build a variable to hold the condition, this will be included in the
      frame as a local var.  */
@@ -3208,14 +3334,20 @@ await_statement_walker (tree *stmt, int *do_subtree, void *d)
     }
 
   /* We have something to be handled as a single statement.  */
+  bool has_cleanup_wrapper = TREE_CODE (*stmt) == CLEANUP_POINT_EXPR;
   hash_set<tree> visited;
   awpts->saw_awaits = 0;
   hash_set<tree> truth_aoif_to_expand;
   awpts->truth_aoif_to_expand = &truth_aoif_to_expand;
   awpts->needs_truth_if_exp = false;
-
-  if (STATEMENT_CLASS_P (*stmt))
-    switch (TREE_CODE (*stmt))
+  awpts->has_awaiter_init = false;
+  tree expr = *stmt;
+  if (has_cleanup_wrapper)
+    expr = TREE_OPERAND (expr, 0);
+  STRIP_NOPS (expr);
+
+  if (STATEMENT_CLASS_P (expr))
+    switch (TREE_CODE (expr))
       {
 	/* Unless it's a special case, just walk the subtrees as usual.  */
 	default: return NULL_TREE;
@@ -3255,13 +3387,6 @@ await_statement_walker (tree *stmt, int *do_subtree, void *d)
 	    tree new_s = build2_loc (sloc, MODIFY_EXPR, boolean_type_node,
 				     newvar, cond_inner);
 	    finish_expr_stmt (new_s);
-	    if (awpts->needs_truth_if_exp)
-	      {
-		tree *sp = tsi_stmt_ptr (tsi_last (insert_list));
-		truth_if_transform xf = {sp, newvar, &truth_aoif_to_expand};
-		if ((res = cp_walk_tree (sp, expand_one_truth_if, &xf, NULL)))
-		  return res;
-	      }
 	    IF_COND (if_stmt) = newvar;
 	    add_stmt (if_stmt);
 	    *stmt = pop_stmt_list (insert_list);
@@ -3327,7 +3452,6 @@ await_statement_walker (tree *stmt, int *do_subtree, void *d)
 				d, NULL);
 	    *do_subtree = 0; /* Done subtrees.  */
 	    return res;
-
 	  }
 	  break;
 	case SWITCH_STMT:
@@ -3371,8 +3495,57 @@ await_statement_walker (tree *stmt, int *do_subtree, void *d)
 	    return res;
 	  }
 	  break;
+	case CO_RETURN_EXPR:
+	  {
+	    /* Expand the co_return as per [stmt.return.coroutine]
+	       - for co_return;
+		{ p.return_void (); goto final_suspend; }
+	       - for co_return [void expr];
+		{ expr; p.return_void(); goto final_suspend;}
+	       - for co_return [non void expr];
+		{ p.return_value(expr); goto final_suspend; }  */
+	    if ((res = cp_walk_tree (stmt, analyze_expression_awaits,
+		 d, &visited)))
+	      return res;
+	    location_t loc = EXPR_LOCATION (expr);
+	    tree call = TREE_OPERAND (expr, 1);
+	    expr = TREE_OPERAND (expr, 0);
+	    tree ret_list = push_stmt_list ();
+	    /* [stmt.return.coroutine], 2.2
+	       If expr is present and void, it is placed immediately before
+	       the call for return_void;  */
+	    tree *maybe_await_stmt = NULL;
+	    if (expr && VOID_TYPE_P (TREE_TYPE (expr)))
+	      {
+		finish_expr_stmt (expr);
+		/* If the return argument was a void expression, then any
+		   awaits must be contained in that.  */
+		maybe_await_stmt = tsi_stmt_ptr (tsi_last (ret_list));
+	      }
+	    /* Insert p.return_{void,value(expr)}.  */
+	    finish_expr_stmt (call);
+	    /* Absent a return of a void expression, any awaits must be in
+	       the parameter to return_value().  */
+	    if (!maybe_await_stmt)
+	      maybe_await_stmt = tsi_stmt_ptr (tsi_last (ret_list));
+	    expr = build1_loc (loc, GOTO_EXPR, void_type_node, awpts->fs_label);
+	    finish_expr_stmt (expr);
+	    *stmt = pop_stmt_list (ret_list);
+	    /* Once this is complete, we will have processed subtrees.  */
+	    *do_subtree = 0;
+	    if (awpts->saw_awaits)
+	      {
+		gcc_checking_assert (maybe_await_stmt);
+		res = cp_walk_tree (maybe_await_stmt, await_statement_walker,
+				    d, NULL);
+		if (res)
+		  return res;
+	      }
+	    return NULL_TREE; /* Done.  */
+	  }
+	break;
       }
-  else if (EXPR_P (*stmt))
+  else if (EXPR_P (expr))
     {
       if ((res = cp_walk_tree (stmt, analyze_expression_awaits, d, &visited)))
 	return res;
@@ -3380,31 +3553,19 @@ await_statement_walker (tree *stmt, int *do_subtree, void *d)
       if (!awpts->saw_awaits)
 	return NULL_TREE; /* Nothing special to do here.  */
 
-      /* Unless we need to expand any truth-and/or-if expressions, then the
-	 remaining action is to check for temporaries to await expressions
-	 captured by refence.  */
-      if (!awpts->needs_truth_if_exp)
-	return maybe_promote_captured_temps (stmt, d);
-
-      gcc_checking_assert (!awpts->bind_stack->is_empty());
-      tree& bind_expr = awpts->bind_stack->last ();
-      /* Build a variable to hold the condition, this will be
-      included in the frame as a local var.  */
-      tree newvar = add_var_to_bind (bind_expr, boolean_type_node,
-				     "taoi", awpts->cond_number++);
-      tree insert_list = push_stmt_list ();
-      add_decl_expr (newvar);
-      add_stmt (*stmt);
-      tree *sp = tsi_stmt_ptr (tsi_last (insert_list));
-      *stmt = pop_stmt_list (insert_list);
-
-      truth_if_transform xf = {sp, newvar, &truth_aoif_to_expand};
-      if ((res = cp_walk_tree (sp, expand_one_truth_if, &xf, NULL)))
-	return res;
-      /* Process the expanded trees.  */
-      return cp_walk_tree (stmt, await_statement_walker, d, NULL);
+      if (awpts->needs_truth_if_exp)
+	{
+	  /* If a truth-and/or-if expression has an await expression in the
+	     conditionally-taken branch, then it must be rewritten into a
+	     regular conditional.  */
+	  truth_if_transform xf = {stmt, NULL_TREE, &truth_aoif_to_expand};
+	  if ((res = cp_walk_tree (stmt, expand_one_truth_if, &xf, NULL)))
+	    return res;
+	}
+      /* Process this statement, which contains at least one await expression
+	 to 'promote' temporary values to a coroutine frame slot.  */
+      return maybe_promote_temps (stmt, d);
     }
- 
   /* Continue recursion, if needed.  */
   return res;
 }
@@ -3420,6 +3581,9 @@ struct param_frame_data
   bool param_seen;
 };
 
+/* A tree-walk callback that records the use of parameters (to allow for
+   optimizations where handling unused parameters may be omitted).  */
+
 static tree
 register_param_uses (tree *stmt, int *do_subtree ATTRIBUTE_UNUSED, void *d)
 {
@@ -3455,7 +3619,21 @@ register_param_uses (tree *stmt, int *do_subtree ATTRIBUTE_UNUSED, void *d)
   return NULL_TREE;
 }
 
-/* For figuring out what local variable usage we have.  */
+/* Small helper for the repetitive task of adding a new field to the coro
+   frame type.  */
+
+static tree
+coro_make_frame_entry (tree *field_list, const char *name, tree fld_type,
+		       location_t loc)
+{
+  tree id = get_identifier (name);
+  tree decl = build_decl (loc, FIELD_DECL, id, fld_type);
+  DECL_CHAIN (decl) = *field_list;
+  *field_list = decl;
+  return id;
+}
+
+/* For recording local variable usage.  */
 
 struct local_vars_frame_data
 {
@@ -3467,6 +3645,10 @@ struct local_vars_frame_data
   bool local_var_seen;
 };
 
+/* A tree-walk callback that processes one bind expression noting local
+   variables, and making a coroutine frame slot available for those that
+   need it, so that they can be 'promoted' across suspension points.  */
+
 static tree
 register_local_var_uses (tree *stmt, int *do_subtree, void *d)
 {
@@ -3523,11 +3705,11 @@ register_local_var_uses (tree *stmt, int *do_subtree, void *d)
 	  tree lvname = DECL_NAME (lvar);
 	  char *buf;
 	  if (lvname != NULL_TREE)
-	    buf = xasprintf ("__lv.%u.%u.%s", lvd->bind_indx, lvd->nest_depth,
-			     IDENTIFIER_POINTER (lvname));
+	    buf = xasprintf ("__%s.%u.%u", IDENTIFIER_POINTER (lvname),
+			     lvd->nest_depth, lvd->bind_indx);
 	  else
-	    buf = xasprintf ("__lv.%u.%u.D%u", lvd->bind_indx, lvd->nest_depth,
-			     DECL_UID (lvar));
+	    buf = xasprintf ("_D%u.%u.%u", DECL_UID (lvar), lvd->nest_depth,
+			     lvd->bind_indx);
 	  /* TODO: Figure out if we should build a local type that has any
 	     excess alignment or size from the original decl.  */
 	  local_var.field_id
@@ -3631,7 +3813,7 @@ coro_rewrite_function_body (location_t fn_start, tree fnbody,
 					 fn_start, NULL, /*musthave=*/true);
       /* Create and initialize the initial-await-resume-called variable per
 	 [dcl.fct.def.coroutine] / 5.3.  */
-      tree i_a_r_c = build_lang_decl (VAR_DECL, get_identifier ("__i_a_r_c"),
+      tree i_a_r_c = build_lang_decl (VAR_DECL, get_identifier ("i_a_r_c"),
 				      boolean_type_node);
       DECL_ARTIFICIAL (i_a_r_c) = true;
       DECL_CHAIN (i_a_r_c) = var_list;
@@ -3760,9 +3942,7 @@ coro_rewrite_function_body (location_t fn_start, tree fnbody,
   short __resume_at;
   handle_type self_handle;
   (maybe) parameter copies.
-  coro1::suspend_never_prt __is;
-  coro1::suspend_always_prt __fs;
-  (maybe) local variables saved
+  (maybe) local variables saved (including awaitables)
   (maybe) trailing space.
  };  */
 
@@ -3966,8 +4146,8 @@ morph_fn_to_coro (tree orig, tree *resumer, tree *destroyer)
      to promote any temporaries that are captured by reference (to regular
      vars) they will get added to the coro frame along with other locals.  */
   susp_frame_data body_aw_points
-    = {&field_list, handle_type, NULL, NULL, 0, 0,
-       hash_set<tree> (), NULL, NULL, 0, false, false};
+    = {&field_list, handle_type, fs_label, NULL, NULL, 0, 0,
+       hash_set<tree> (), NULL, NULL, 0, false, false, false};
   body_aw_points.block_stack = make_tree_vector ();
   body_aw_points.bind_stack = make_tree_vector ();
   body_aw_points.to_replace = make_tree_vector ();
@@ -4555,7 +4735,7 @@ morph_fn_to_coro (tree orig, tree *resumer, tree *destroyer)
 
   /* Build the actor...  */
   build_actor_fn (fn_start, coro_frame_type, actor, fnbody, orig, param_uses,
-		  &local_var_uses, param_dtor_list, fs_label, resume_fn_field,
+		  &local_var_uses, param_dtor_list, resume_fn_field,
 		  body_aw_points.await_number, frame_size);
 
   /* Destroyer ... */