path: root/gcc/omp-oacc-neuter-broadcast.cc

/* OpenACC worker partitioning via middle end neutering/broadcasting scheme

   Copyright (C) 2015-2021 Free Software Foundation, Inc.

   This file is part of GCC.

   GCC is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published
   by the Free Software Foundation; either version 3, or (at your
   option) any later version.

   GCC is distributed in the hope that it will be useful, but WITHOUT
   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
   License for more details.

   You should have received a copy of the GNU General Public License
   along with GCC; see the file COPYING3.  If not see
   <http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "tree-pass.h"
#include "ssa.h"
#include "cgraph.h"
#include "pretty-print.h"
#include "fold-const.h"
#include "gimplify.h"
#include "gimple-iterator.h"
#include "gimple-walk.h"
#include "tree-inline.h"
#include "langhooks.h"
#include "omp-general.h"
#include "omp-low.h"
#include "gimple-pretty-print.h"
#include "cfghooks.h"
#include "insn-config.h"
#include "recog.h"
#include "internal-fn.h"
#include "bitmap.h"
#include "tree-nested.h"
#include "stor-layout.h"
#include "tree-ssa-threadupdate.h"
#include "tree-into-ssa.h"
#include "splay-tree.h"
#include "target.h"
#include "cfgloop.h"
#include "tree-cfg.h"
#include "omp-offload.h"
#include "attribs.h"

/* Loop structure of the function.  The entire function is described as
   a NULL loop.  */
/* Adapted from 'gcc/config/nvptx/nvptx.c:struct parallel'.  */

struct parallel_g
{
  /* Parent parallel.  */
  parallel_g *parent;

  /* Next sibling parallel.  */
  parallel_g *next;

  /* First child parallel.  */
  parallel_g *inner;

  /* Partitioning mask of the parallel.  */
  unsigned mask;

  /* Partitioning used within inner parallels. */
  unsigned inner_mask;

  /* Location of parallel forked and join.  The forked is the first
     block in the parallel and the join is the first block after of
     the partition.  */
  basic_block forked_block;
  basic_block join_block;

  gimple *forked_stmt;
  gimple *join_stmt;

  gimple *fork_stmt;
  gimple *joining_stmt;

  /* Basic blocks in this parallel, but not in child parallels.  The
     FORKED and JOINING blocks are in the partition.  The FORK and JOIN
     blocks are not.  */
  auto_vec<basic_block> blocks;

  tree record_type;
  tree sender_decl;
  tree receiver_decl;

public:
  parallel_g (parallel_g *parent, unsigned mode);
  ~parallel_g ();
};

/* Constructor links the new parallel into it's parent's chain of
   children.  */

parallel_g::parallel_g (parallel_g *parent_, unsigned mask_)
  :parent (parent_), next (0), inner (0), mask (mask_), inner_mask (0)
{
  forked_block = join_block = 0;
  forked_stmt = join_stmt = NULL;
  fork_stmt = joining_stmt = NULL;

  record_type = NULL_TREE;
  sender_decl = NULL_TREE;
  receiver_decl = NULL_TREE;

  if (parent)
    {
      next = parent->inner;
      parent->inner = this;
    }
}

parallel_g::~parallel_g ()
{
  delete inner;
  delete next;
}

static bool
local_var_based_p (tree decl)
{
  switch (TREE_CODE (decl))
    {
    case VAR_DECL:
      return !is_global_var (decl);

    case COMPONENT_REF:
    case BIT_FIELD_REF:
    case ARRAY_REF:
      return local_var_based_p (TREE_OPERAND (decl, 0));

    default:
      return false;
    }
}

/* Map of basic blocks to gimple stmts.  */
typedef hash_map<basic_block, gimple *> bb_stmt_map_t;

/* Calls to OpenACC routines are made by all workers/wavefronts/warps, since
   the routine likely contains partitioned loops (else will do its own
   neutering and variable propagation). Return TRUE if a function call CALL
   should be made in (worker) single mode instead, rather than redundant
   mode.  */

static bool
omp_sese_active_worker_call (gcall *call)
{
#define GOMP_DIM_SEQ GOMP_DIM_MAX
  tree fndecl = gimple_call_fndecl (call);

  if (!fndecl)
    return true;

  tree attrs = oacc_get_fn_attrib (fndecl);

  if (!attrs)
    return true;

  int level = oacc_fn_attrib_level (attrs);

  /* Neither regular functions nor "seq" routines should be run by all threads
     in worker-single mode.  */
  return level == -1 || level == GOMP_DIM_SEQ;
#undef GOMP_DIM_SEQ
}

/* Split basic blocks such that each forked and join unspecs are at
   the start of their basic blocks.  Thus afterwards each block will
   have a single partitioning mode.  We also do the same for return
   insns, as they are executed by every thread.  Return the
   partitioning mode of the function as a whole.  Populate MAP with
   head and tail blocks.  We also clear the BB visited flag, which is
   used when finding partitions.  */
/* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_split_blocks'.  */

static void
omp_sese_split_blocks (bb_stmt_map_t *map)
{
  auto_vec<gimple *> worklist;
  basic_block block;

  /* Locate all the reorg instructions of interest.  */
  FOR_ALL_BB_FN (block, cfun)
    {
      /* Clear visited flag, for use by parallel locator  */
      block->flags &= ~BB_VISITED;

      for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	   !gsi_end_p (gsi);
	   gsi_next (&gsi))
	{
	  gimple *stmt = gsi_stmt (gsi);

	  if (gimple_call_internal_p (stmt, IFN_UNIQUE))
	    {
	      enum ifn_unique_kind k = ((enum ifn_unique_kind)
		TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));

	      if (k == IFN_UNIQUE_OACC_JOIN)
		worklist.safe_push (stmt);
	      else if (k == IFN_UNIQUE_OACC_FORK)
		{
		  gcc_assert (gsi_one_before_end_p (gsi));
		  basic_block forked_block = single_succ (block);
		  gimple_stmt_iterator gsi2 = gsi_start_bb (forked_block);

		  /* We push a NOP as a placeholder for the "forked" stmt.
		     This is then recognized in omp_sese_find_par.  */
		  gimple *nop = gimple_build_nop ();
		  gsi_insert_before (&gsi2, nop, GSI_SAME_STMT);

		  worklist.safe_push (nop);
		}
	    }
	  else if (gimple_code (stmt) == GIMPLE_RETURN
		   || gimple_code (stmt) == GIMPLE_COND
		   || gimple_code (stmt) == GIMPLE_SWITCH
		   || (gimple_code (stmt) == GIMPLE_CALL
		       && !gimple_call_internal_p (stmt)
		       && !omp_sese_active_worker_call (as_a <gcall *> (stmt))))
	    worklist.safe_push (stmt);
	  else if (is_gimple_assign (stmt))
	    {
	      tree lhs = gimple_assign_lhs (stmt);

	      /* Force assignments to components/fields/elements of local
		 aggregates into fully-partitioned (redundant) mode.  This
		 avoids having to broadcast the whole aggregate.  The RHS of
		 the assignment will be propagated using the normal
		 mechanism.  */

	      switch (TREE_CODE (lhs))
		{
		case COMPONENT_REF:
		case BIT_FIELD_REF:
		case ARRAY_REF:
		  {
		    tree aggr = TREE_OPERAND (lhs, 0);

		    if (local_var_based_p (aggr))
		      worklist.safe_push (stmt);
		  }
		  break;

		default:
		  ;
		}
	    }
	}
    }

  /* Split blocks on the worklist.  */
  unsigned ix;
  gimple *stmt;

  for (ix = 0; worklist.iterate (ix, &stmt); ix++)
    {
      basic_block block = gimple_bb (stmt);

      if (gimple_code (stmt) == GIMPLE_COND)
	{
	  gcond *orig_cond = as_a <gcond *> (stmt);
	  tree_code code = gimple_expr_code (orig_cond);
	  tree pred = make_ssa_name (boolean_type_node);
	  gimple *asgn = gimple_build_assign (pred, code,
			   gimple_cond_lhs (orig_cond),
			   gimple_cond_rhs (orig_cond));
	  gcond *new_cond
	    = gimple_build_cond (NE_EXPR, pred, boolean_false_node,
				 gimple_cond_true_label (orig_cond),
				 gimple_cond_false_label (orig_cond));

	  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
	  gsi_insert_before (&gsi, asgn, GSI_SAME_STMT);
	  gsi_replace (&gsi, new_cond, true);

	  edge e = split_block (block, asgn);
	  block = e->dest;
	  map->get_or_insert (block) = new_cond;
	}
      else if ((gimple_code (stmt) == GIMPLE_CALL
		&& !gimple_call_internal_p (stmt))
	       || is_gimple_assign (stmt))
	{
	  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
	  gsi_prev (&gsi);

	  edge call = split_block (block, gsi_stmt (gsi));

	  gimple *call_stmt = gsi_stmt (gsi_start_bb (call->dest));

	  edge call_to_ret = split_block (call->dest, call_stmt);

	  map->get_or_insert (call_to_ret->src) = call_stmt;
	}
      else
	{
	  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
	  gsi_prev (&gsi);

	  if (gsi_end_p (gsi))
	    map->get_or_insert (block) = stmt;
	  else
	    {
	      /* Split block before insn. The insn is in the new block.  */
	      edge e = split_block (block, gsi_stmt (gsi));

	      block = e->dest;
	      map->get_or_insert (block) = stmt;
	    }
	}
    }
}

static const char *
mask_name (unsigned mask)
{
  switch (mask)
    {
    case 0: return "gang redundant";
    case 1: return "gang partitioned";
    case 2: return "worker partitioned";
    case 3: return "gang+worker partitioned";
    case 4: return "vector partitioned";
    case 5: return "gang+vector partitioned";
    case 6: return "worker+vector partitioned";
    case 7: return "fully partitioned";
    default: return "<illegal>";
    }
}

/* Dump this parallel and all its inner parallels.  */
/* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_dump_pars'.  */

static void
omp_sese_dump_pars (parallel_g *par, unsigned depth)
{
  fprintf (dump_file, "%u: mask %d (%s) head=%d, tail=%d\n",
	   depth, par->mask, mask_name (par->mask),
	   par->forked_block ? par->forked_block->index : -1,
	   par->join_block ? par->join_block->index : -1);

  fprintf (dump_file, "    blocks:");

  basic_block block;
  for (unsigned ix = 0; par->blocks.iterate (ix, &block); ix++)
    fprintf (dump_file, " %d", block->index);
  fprintf (dump_file, "\n");
  if (par->inner)
    omp_sese_dump_pars (par->inner, depth + 1);

  if (par->next)
    omp_sese_dump_pars (par->next, depth);
}

/* If BLOCK contains a fork/join marker, process it to create or
   terminate a loop structure.  Add this block to the current loop,
   and then walk successor blocks.   */
/* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_find_par'.  */

static parallel_g *
omp_sese_find_par (bb_stmt_map_t *map, parallel_g *par, basic_block block)
{
  if (block->flags & BB_VISITED)
    return par;
  block->flags |= BB_VISITED;

  if (gimple **stmtp = map->get (block))
    {
      gimple *stmt = *stmtp;

      if (gimple_code (stmt) == GIMPLE_COND
	  || gimple_code (stmt) == GIMPLE_SWITCH
	  || gimple_code (stmt) == GIMPLE_RETURN
	  || (gimple_code (stmt) == GIMPLE_CALL
	      && !gimple_call_internal_p (stmt))
	  || is_gimple_assign (stmt))
	{
	  /* A single block that is forced to be at the maximum partition
	     level.  Make a singleton par for it.  */
	  par = new parallel_g (par, GOMP_DIM_MASK (GOMP_DIM_GANG)
				   | GOMP_DIM_MASK (GOMP_DIM_WORKER)
				   | GOMP_DIM_MASK (GOMP_DIM_VECTOR));
	  par->forked_block = block;
	  par->forked_stmt = stmt;
	  par->blocks.safe_push (block);
	  par = par->parent;
	  goto walk_successors;
	}
      else if (gimple_nop_p (stmt))
	{
	  basic_block pred = single_pred (block);
	  gcc_assert (pred);
	  gimple_stmt_iterator gsi = gsi_last_bb (pred);
	  gimple *final_stmt = gsi_stmt (gsi);

	  if (gimple_call_internal_p (final_stmt, IFN_UNIQUE))
	    {
	      gcall *call = as_a <gcall *> (final_stmt);
	      enum ifn_unique_kind k = ((enum ifn_unique_kind)
		TREE_INT_CST_LOW (gimple_call_arg (call, 0)));

	      if (k == IFN_UNIQUE_OACC_FORK)
		{
		  HOST_WIDE_INT dim
		    = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
		  unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;

		  par = new parallel_g (par, mask);
		  par->forked_block = block;
		  par->forked_stmt = final_stmt;
		  par->fork_stmt = stmt;
		}
	      else
		gcc_unreachable ();
	    }
	  else
	    gcc_unreachable ();
	}
      else if (gimple_call_internal_p (stmt, IFN_UNIQUE))
	{
	  gcall *call = as_a <gcall *> (stmt);
	  enum ifn_unique_kind k = ((enum ifn_unique_kind)
	    TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
	  if (k == IFN_UNIQUE_OACC_JOIN)
	    {
	      HOST_WIDE_INT dim = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
	      unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;

	      gcc_assert (par->mask == mask);
	      par->join_block = block;
	      par->join_stmt = stmt;
	      par = par->parent;
	    }
	  else
	    gcc_unreachable ();
	}
      else
	gcc_unreachable ();
    }

  if (par)
    /* Add this block onto the current loop's list of blocks.  */
    par->blocks.safe_push (block);
  else
    /* This must be the entry block.  Create a NULL parallel.  */
    par = new parallel_g (0, 0);

walk_successors:
  /* Walk successor blocks.  */
  edge e;
  edge_iterator ei;

  FOR_EACH_EDGE (e, ei, block->succs)
    omp_sese_find_par (map, par, e->dest);

  return par;
}

/* DFS walk the CFG looking for fork & join markers.  Construct
   loop structures as we go.  MAP is a mapping of basic blocks
   to head & tail markers, discovered when splitting blocks.  This
   speeds up the discovery.  We rely on the BB visited flag having
   been cleared when splitting blocks.  */
/* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_discover_pars'.  */

static parallel_g *
omp_sese_discover_pars (bb_stmt_map_t *map)
{
  basic_block block;

  /* Mark exit blocks as visited.  */
  block = EXIT_BLOCK_PTR_FOR_FN (cfun);
  block->flags |= BB_VISITED;

  /* And entry block as not.  */
  block = ENTRY_BLOCK_PTR_FOR_FN (cfun);
  block->flags &= ~BB_VISITED;

  parallel_g *par = omp_sese_find_par (map, 0, block);

  if (dump_file)
    {
      fprintf (dump_file, "\nLoops\n");
      omp_sese_dump_pars (par, 0);
      fprintf (dump_file, "\n");
    }

  return par;
}

static void
populate_single_mode_bitmaps (parallel_g *par, bitmap worker_single,
			      bitmap vector_single, unsigned outer_mask,
			      int depth)
{
  unsigned mask = outer_mask | par->mask;

  basic_block block;

  for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
    {
      if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
	bitmap_set_bit (worker_single, block->index);

      if ((mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR)) == 0)
	bitmap_set_bit (vector_single, block->index);
    }

  if (par->inner)
    populate_single_mode_bitmaps (par->inner, worker_single, vector_single,
				  mask, depth + 1);
  if (par->next)
    populate_single_mode_bitmaps (par->next, worker_single, vector_single,
				  outer_mask, depth);
}

/* A map from SSA names or var decls to record fields.  */

typedef hash_map<tree, tree> field_map_t;

/* For each propagation record type, this is a map from SSA names or var decls
   to propagate, to the field in the record type that should be used for
   transmission and reception.  */

typedef hash_map<tree, field_map_t *> record_field_map_t;

static void
install_var_field (tree var, tree record_type, field_map_t *fields)
{
  tree name;
  char tmp[20];

  if (TREE_CODE (var) == SSA_NAME)
    {
      name = SSA_NAME_IDENTIFIER (var);
      if (!name)
	{
	  sprintf (tmp, "_%u", (unsigned) SSA_NAME_VERSION (var));
	  name = get_identifier (tmp);
	}
    }
  else if (TREE_CODE (var) == VAR_DECL)
    {
      name = DECL_NAME (var);
      if (!name)
	{
	  sprintf (tmp, "D_%u", (unsigned) DECL_UID (var));
	  name = get_identifier (tmp);
	}
    }
  else
    gcc_unreachable ();

  gcc_assert (!fields->get (var));

  tree type = TREE_TYPE (var);

  if (POINTER_TYPE_P (type)
      && TYPE_RESTRICT (type))
    type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);

  tree field = build_decl (BUILTINS_LOCATION, FIELD_DECL, name, type);

  if (TREE_CODE (var) == VAR_DECL && type == TREE_TYPE (var))
    {
      SET_DECL_ALIGN (field, DECL_ALIGN (var));
      DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
      TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
    }
  else
    SET_DECL_ALIGN (field, TYPE_ALIGN (type));

  fields->put (var, field);

  insert_field_into_struct (record_type, field);
}

/* Sets of SSA_NAMES or VAR_DECLs to propagate.  */
typedef hash_set<tree> propagation_set;

static void
find_ssa_names_to_propagate (parallel_g *par, unsigned outer_mask,
			     bitmap worker_single, bitmap vector_single,
			     vec<propagation_set *> *prop_set)
{
  unsigned mask = outer_mask | par->mask;

  if (par->inner)
    find_ssa_names_to_propagate (par->inner, mask, worker_single,
				 vector_single, prop_set);
  if (par->next)
    find_ssa_names_to_propagate (par->next, outer_mask, worker_single,
				 vector_single, prop_set);

  if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
    {
      basic_block block;
      int ix;

      for (ix = 0; par->blocks.iterate (ix, &block); ix++)
	{
	  for (gphi_iterator psi = gsi_start_phis (block);
	       !gsi_end_p (psi); gsi_next (&psi))
	    {
	      gphi *phi = psi.phi ();
	      use_operand_p use;
	      ssa_op_iter iter;

	      FOR_EACH_PHI_ARG (use, phi, iter, SSA_OP_USE)
		{
		  tree var = USE_FROM_PTR (use);

		  if (TREE_CODE (var) != SSA_NAME)
		    continue;

		  gimple *def_stmt = SSA_NAME_DEF_STMT (var);

		  if (gimple_nop_p (def_stmt))
		    continue;

		  basic_block def_bb = gimple_bb (def_stmt);

		  if (bitmap_bit_p (worker_single, def_bb->index))
		    {
		      if (!(*prop_set)[def_bb->index])
			(*prop_set)[def_bb->index] = new propagation_set;

		      propagation_set *ws_prop = (*prop_set)[def_bb->index];

		      ws_prop->add (var);
		    }
		}
	    }

	  for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	       !gsi_end_p (gsi); gsi_next (&gsi))
	    {
	      use_operand_p use;
	      ssa_op_iter iter;
	      gimple *stmt = gsi_stmt (gsi);

	      FOR_EACH_SSA_USE_OPERAND (use, stmt, iter, SSA_OP_USE)
		{
		  tree var = USE_FROM_PTR (use);

		  gimple *def_stmt = SSA_NAME_DEF_STMT (var);

		  if (gimple_nop_p (def_stmt))
		    continue;

		  basic_block def_bb = gimple_bb (def_stmt);

		  if (bitmap_bit_p (worker_single, def_bb->index))
		    {
		      if (!(*prop_set)[def_bb->index])
			(*prop_set)[def_bb->index] = new propagation_set;

		      propagation_set *ws_prop = (*prop_set)[def_bb->index];

		      ws_prop->add (var);
		    }
		}
	    }
	}
    }
}

/* Callback for walk_gimple_stmt to find RHS VAR_DECLs (uses) in a
   statement.  */

static tree
find_partitioned_var_uses_1 (tree *node, int *, void *data)
{
  walk_stmt_info *wi = (walk_stmt_info *) data;
  hash_set<tree> *partitioned_var_uses = (hash_set<tree> *) wi->info;

  if (!wi->is_lhs && VAR_P (*node))
    partitioned_var_uses->add (*node);

  return NULL_TREE;
}

static void
find_partitioned_var_uses (parallel_g *par, unsigned outer_mask,
			   hash_set<tree> *partitioned_var_uses)
{
  unsigned mask = outer_mask | par->mask;

  if (par->inner)
    find_partitioned_var_uses (par->inner, mask, partitioned_var_uses);
  if (par->next)
    find_partitioned_var_uses (par->next, outer_mask, partitioned_var_uses);

  if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
    {
      basic_block block;
      int ix;

      for (ix = 0; par->blocks.iterate (ix, &block); ix++)
	for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	     !gsi_end_p (gsi); gsi_next (&gsi))
	  {
	    walk_stmt_info wi;
	    memset (&wi, 0, sizeof (wi));
	    wi.info = (void *) partitioned_var_uses;
	    walk_gimple_stmt (&gsi, NULL, find_partitioned_var_uses_1, &wi);
	  }
    }
}

/* Gang-private variables (typically placed in a GPU's shared memory) do not
   need to be processed by the worker-propagation mechanism.  Populate the
   GANG_PRIVATE_VARS set with any such variables found in the current
   function.  */

static void
find_gang_private_vars (hash_set<tree> *gang_private_vars)
{
  basic_block block;

  FOR_EACH_BB_FN (block, cfun)
    {
      for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	   !gsi_end_p (gsi);
	   gsi_next (&gsi))
	{
	  gimple *stmt = gsi_stmt (gsi);

	  if (gimple_call_internal_p (stmt, IFN_UNIQUE))
	    {
	      enum ifn_unique_kind k = ((enum ifn_unique_kind)
		TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
	      if (k == IFN_UNIQUE_OACC_PRIVATE)
		{
		  HOST_WIDE_INT level
		    = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
		  if (level != GOMP_DIM_GANG)
		    continue;
		  for (unsigned i = 3; i < gimple_call_num_args (stmt); i++)
		    {
		      tree arg = gimple_call_arg (stmt, i);
		      gcc_assert (TREE_CODE (arg) == ADDR_EXPR);
		      tree decl = TREE_OPERAND (arg, 0);
		      gang_private_vars->add (decl);
		    }
		}
	    }
	}
    }
}

static void
find_local_vars_to_propagate (parallel_g *par, unsigned outer_mask,
			      hash_set<tree> *partitioned_var_uses,
			      hash_set<tree> *gang_private_vars,
			      vec<propagation_set *> *prop_set)
{
  unsigned mask = outer_mask | par->mask;

  if (par->inner)
    find_local_vars_to_propagate (par->inner, mask, partitioned_var_uses,
				  gang_private_vars, prop_set);
  if (par->next)
    find_local_vars_to_propagate (par->next, outer_mask, partitioned_var_uses,
				  gang_private_vars, prop_set);

  if (!(mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)))
    {
      basic_block block;
      int ix;

      for (ix = 0; par->blocks.iterate (ix, &block); ix++)
	{
	  for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	       !gsi_end_p (gsi); gsi_next (&gsi))
	    {
	      gimple *stmt = gsi_stmt (gsi);
	      tree var;
	      unsigned i;

	      FOR_EACH_LOCAL_DECL (cfun, i, var)
		{
		  if (!VAR_P (var)
		      || is_global_var (var)
		      || AGGREGATE_TYPE_P (TREE_TYPE (var))
		      || !partitioned_var_uses->contains (var)
		      || gang_private_vars->contains (var))
		    continue;

		  if (stmt_may_clobber_ref_p (stmt, var))
		    {
		      if (dump_file)
			{
			  fprintf (dump_file, "bb %u: local variable may be "
				   "clobbered in %s mode: ", block->index,
				   mask_name (mask));
			  print_generic_expr (dump_file, var, TDF_SLIM);
			  fprintf (dump_file, "\n");
			}

		      if (!(*prop_set)[block->index])
			(*prop_set)[block->index] = new propagation_set;

		      propagation_set *ws_prop
			= (*prop_set)[block->index];

		      ws_prop->add (var);
		    }
		}
	    }
	}
    }
}

/* Transform basic blocks FROM, TO (which may be the same block) into:
   if (GOACC_single_start ())
     BLOCK;
   GOACC_barrier ();
			      \  |  /
			      +----+
			      |    |        (new) predicate block
			      +----+--
   \  |  /   \  |  /	        |t    \
   +----+    +----+	      +----+  |
   |	|    |    |	===>  |    |  | f   (old) from block
   +----+    +----+	      +----+  |
     |       t/  \f	        |    /
			      +----+/
  (split  (split before       |    |        skip block
  at end)   condition)	      +----+
			      t/  \f
*/

static void
worker_single_simple (basic_block from, basic_block to,
		      hash_set<tree> *def_escapes_block)
{
  gimple *call, *cond;
  tree lhs, decl;
  basic_block skip_block;

  gimple_stmt_iterator gsi = gsi_last_bb (to);
  if (EDGE_COUNT (to->succs) > 1)
    {
      gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_COND);
      gsi_prev (&gsi);
    }
  edge e = split_block (to, gsi_stmt (gsi));
  skip_block = e->dest;

  gimple_stmt_iterator start = gsi_after_labels (from);

  decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_START);
  lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
  call = gimple_build_call (decl, 0);
  gimple_call_set_lhs (call, lhs);
  gsi_insert_before (&start, call, GSI_NEW_STMT);
  update_stmt (call);

  cond = gimple_build_cond (EQ_EXPR, lhs,
			    fold_convert_loc (UNKNOWN_LOCATION,
					      TREE_TYPE (lhs),
					      boolean_true_node),
			    NULL_TREE, NULL_TREE);
  gsi_insert_after (&start, cond, GSI_NEW_STMT);
  update_stmt (cond);

  edge et = split_block (from, cond);
  et->flags &= ~EDGE_FALLTHRU;
  et->flags |= EDGE_TRUE_VALUE;
  /* Make the active worker the more probable path so we prefer fallthrough
     (letting the idle workers jump around more).  */
  et->probability = profile_probability::likely ();

  edge ef = make_edge (from, skip_block, EDGE_FALSE_VALUE);
  ef->probability = et->probability.invert ();

  basic_block neutered = split_edge (ef);
  gimple_stmt_iterator neut_gsi = gsi_last_bb (neutered);

  for (gsi = gsi_start_bb (et->dest); !gsi_end_p (gsi); gsi_next (&gsi))
    {
      gimple *stmt = gsi_stmt (gsi);
      ssa_op_iter iter;
      tree var;

      FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
	{
	  if (def_escapes_block->contains (var))
	    {
	      gphi *join_phi = create_phi_node (NULL_TREE, skip_block);
	      create_new_def_for (var, join_phi,
				  gimple_phi_result_ptr (join_phi));
	      add_phi_arg (join_phi, var, e, UNKNOWN_LOCATION);

	      tree neutered_def = copy_ssa_name (var, NULL);
	      /* We really want "don't care" or some value representing
		 undefined here, but optimizers will probably get rid of the
		 zero-assignments anyway.  */
	      gassign *zero = gimple_build_assign (neutered_def,
				build_zero_cst (TREE_TYPE (neutered_def)));

	      gsi_insert_after (&neut_gsi, zero, GSI_CONTINUE_LINKING);
	      update_stmt (zero);

	      add_phi_arg (join_phi, neutered_def, single_succ_edge (neutered),
			   UNKNOWN_LOCATION);
	      update_stmt (join_phi);
	    }
	}
    }

  gsi = gsi_start_bb (skip_block);

  decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
  gimple *acc_bar = gimple_build_call (decl, 0);

  gsi_insert_before (&gsi, acc_bar, GSI_SAME_STMT);
  update_stmt (acc_bar);
}

/* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
   as appropriate.  */
/* Adapted from 'gcc/omp-low.c:omp_build_component_ref'.  */

static tree
oacc_build_component_ref (tree obj, tree field)
{
  tree field_type = TREE_TYPE (field);
  tree obj_type = TREE_TYPE (obj);
  if (!ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (obj_type)))
    field_type = build_qualified_type
			(field_type,
			 KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (obj_type)));

  tree ret = build3 (COMPONENT_REF, field_type, obj, field, NULL);
  if (TREE_THIS_VOLATILE (field))
    TREE_THIS_VOLATILE (ret) |= 1;
  if (TREE_READONLY (field))
    TREE_READONLY (ret) |= 1;
  return ret;
}

static tree
build_receiver_ref (tree var, tree receiver_decl, field_map_t *fields)
{
  tree x = build_simple_mem_ref (receiver_decl);
  tree field = *fields->get (var);
  TREE_THIS_NOTRAP (x) = 1;
  x = oacc_build_component_ref (x, field);
  return x;
}

static tree
build_sender_ref (tree var, tree sender_decl, field_map_t *fields)
{
  tree field = *fields->get (var);
  return oacc_build_component_ref (sender_decl, field);
}

static int
sort_by_ssa_version_or_uid (const void *p1, const void *p2)
{
  const tree t1 = *(const tree *)p1;
  const tree t2 = *(const tree *)p2;

  if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) == SSA_NAME)
    return SSA_NAME_VERSION (t1) - SSA_NAME_VERSION (t2);
  else if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) != SSA_NAME)
    return -1;
  else if (TREE_CODE (t1) != SSA_NAME && TREE_CODE (t2) == SSA_NAME)
    return 1;
  else
    return DECL_UID (t1) - DECL_UID (t2);
}

static int
sort_by_size_then_ssa_version_or_uid (const void *p1, const void *p2)
{
  const tree t1 = *(const tree *)p1;
  const tree t2 = *(const tree *)p2;
  unsigned HOST_WIDE_INT s1 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t1)));
  unsigned HOST_WIDE_INT s2 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t2)));
  if (s1 != s2)
    return s2 - s1;
  else
    return sort_by_ssa_version_or_uid (p1, p2);
}

static void
worker_single_copy (basic_block from, basic_block to,
		    hash_set<tree> *def_escapes_block,
		    hash_set<tree> *worker_partitioned_uses,
		    tree record_type, record_field_map_t *record_field_map)
{
  /* If we only have virtual defs, we'll have no record type, but we still want
     to emit single_copy_start and (particularly) single_copy_end to act as
     a vdef source on the neutered edge representing memory writes on the
     non-neutered edge.  */
  if (!record_type)
    record_type = char_type_node;

  tree sender_decl
    = targetm.goacc.create_worker_broadcast_record (record_type, true,
						    ".oacc_worker_o");
  tree receiver_decl
    = targetm.goacc.create_worker_broadcast_record (record_type, false,
						    ".oacc_worker_i");

  gimple_stmt_iterator gsi = gsi_last_bb (to);
  if (EDGE_COUNT (to->succs) > 1)
    gsi_prev (&gsi);
  edge e = split_block (to, gsi_stmt (gsi));
  basic_block barrier_block = e->dest;

  gimple_stmt_iterator start = gsi_after_labels (from);

  tree decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_START);

  tree lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));

  gimple *call = gimple_build_call (decl, 1,
				    build_fold_addr_expr (sender_decl));
  gimple_call_set_lhs (call, lhs);
  gsi_insert_before (&start, call, GSI_NEW_STMT);
  update_stmt (call);

  tree conv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));

  gimple *conv = gimple_build_assign (conv_tmp,
				      fold_convert (TREE_TYPE (receiver_decl),
						    lhs));
  update_stmt (conv);
  gsi_insert_after (&start, conv, GSI_NEW_STMT);
  gimple *asgn = gimple_build_assign (receiver_decl, conv_tmp);
  gsi_insert_after (&start, asgn, GSI_NEW_STMT);
  update_stmt (asgn);

  tree zero_ptr = build_int_cst (TREE_TYPE (receiver_decl), 0);

  tree recv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));
  asgn = gimple_build_assign (recv_tmp, receiver_decl);
  gsi_insert_after (&start, asgn, GSI_NEW_STMT);
  update_stmt (asgn);

  gimple *cond = gimple_build_cond (EQ_EXPR, recv_tmp, zero_ptr, NULL_TREE,
				    NULL_TREE);
  update_stmt (cond);

  gsi_insert_after (&start, cond, GSI_NEW_STMT);

  edge et = split_block (from, cond);
  et->flags &= ~EDGE_FALLTHRU;
  et->flags |= EDGE_TRUE_VALUE;
  /* Make the active worker the more probable path so we prefer fallthrough
     (letting the idle workers jump around more).  */
  et->probability = profile_probability::likely ();

  basic_block body = et->dest;

  edge ef = make_edge (from, barrier_block, EDGE_FALSE_VALUE);
  ef->probability = et->probability.invert ();

  decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
  gimple *acc_bar = gimple_build_call (decl, 0);

  gimple_stmt_iterator bar_gsi = gsi_start_bb (barrier_block);
  gsi_insert_before (&bar_gsi, acc_bar, GSI_NEW_STMT);

  cond = gimple_build_cond (NE_EXPR, recv_tmp, zero_ptr, NULL_TREE, NULL_TREE);
  gsi_insert_after (&bar_gsi, cond, GSI_NEW_STMT);

  edge et2 = split_block (barrier_block, cond);
  et2->flags &= ~EDGE_FALLTHRU;
  et2->flags |= EDGE_TRUE_VALUE;
  et2->probability = profile_probability::unlikely ();

  basic_block exit_block = et2->dest;

  basic_block copyout_block = split_edge (et2);
  edge ef2 = make_edge (barrier_block, exit_block, EDGE_FALSE_VALUE);
  ef2->probability = et2->probability.invert ();

  gimple_stmt_iterator copyout_gsi = gsi_start_bb (copyout_block);

  edge copyout_to_exit = single_succ_edge (copyout_block);

  gimple_seq sender_seq = NULL;

  /* Make sure we iterate over definitions in a stable order.  */
  auto_vec<tree> escape_vec (def_escapes_block->elements ());
  for (hash_set<tree>::iterator it = def_escapes_block->begin ();
       it != def_escapes_block->end (); ++it)
    escape_vec.quick_push (*it);
  escape_vec.qsort (sort_by_ssa_version_or_uid);

  for (unsigned i = 0; i < escape_vec.length (); i++)
    {
      tree var = escape_vec[i];

      if (TREE_CODE (var) == SSA_NAME && SSA_NAME_IS_VIRTUAL_OPERAND (var))
	continue;

      tree barrier_def = 0;

      if (TREE_CODE (var) == SSA_NAME)
	{
	  gimple *def_stmt = SSA_NAME_DEF_STMT (var);

	  if (gimple_nop_p (def_stmt))
	    continue;

	  /* The barrier phi takes one result from the actual work of the
	     block we're neutering, and the other result is constant zero of
	     the same type.  */

	  gphi *barrier_phi = create_phi_node (NULL_TREE, barrier_block);
	  barrier_def = create_new_def_for (var, barrier_phi,
			  gimple_phi_result_ptr (barrier_phi));

	  add_phi_arg (barrier_phi, var, e, UNKNOWN_LOCATION);
	  add_phi_arg (barrier_phi, build_zero_cst (TREE_TYPE (var)), ef,
		       UNKNOWN_LOCATION);

	  update_stmt (barrier_phi);
	}
      else
	gcc_assert (TREE_CODE (var) == VAR_DECL);

      /* If we had no record type, we will have no fields map.  */
      field_map_t **fields_p = record_field_map->get (record_type);
      field_map_t *fields = fields_p ? *fields_p : NULL;

      if (worker_partitioned_uses->contains (var)
	  && fields
	  && fields->get (var))
	{
	  tree neutered_def = make_ssa_name (TREE_TYPE (var));

	  /* Receive definition from shared memory block.  */

	  tree receiver_ref = build_receiver_ref (var, receiver_decl, fields);
	  gassign *recv = gimple_build_assign (neutered_def,
					       receiver_ref);
	  gsi_insert_after (&copyout_gsi, recv, GSI_CONTINUE_LINKING);
	  update_stmt (recv);

	  if (TREE_CODE (var) == VAR_DECL)
	    {
	      /* If it's a VAR_DECL, we only copied to an SSA temporary.  Copy
		 to the final location now.  */
	      gassign *asgn = gimple_build_assign (var, neutered_def);
	      gsi_insert_after (&copyout_gsi, asgn, GSI_CONTINUE_LINKING);
	      update_stmt (asgn);
	    }
	  else
	    {
	      /* If it's an SSA name, create a new phi at the join node to
		 represent either the output from the active worker (the
		 barrier) or the inactive workers (the copyout block).  */
	      gphi *join_phi = create_phi_node (NULL_TREE, exit_block);
	      create_new_def_for (barrier_def, join_phi,
				  gimple_phi_result_ptr (join_phi));
	      add_phi_arg (join_phi, barrier_def, ef2, UNKNOWN_LOCATION);
	      add_phi_arg (join_phi, neutered_def, copyout_to_exit,
			   UNKNOWN_LOCATION);
	      update_stmt (join_phi);
	    }

	  /* Send definition to shared memory block.  */

	  tree sender_ref = build_sender_ref (var, sender_decl, fields);

	  if (TREE_CODE (var) == SSA_NAME)
	    {
	      gassign *send = gimple_build_assign (sender_ref, var);
	      gimple_seq_add_stmt (&sender_seq, send);
	      update_stmt (send);
	    }
	  else if (TREE_CODE (var) == VAR_DECL)
	    {
	      tree tmp = make_ssa_name (TREE_TYPE (var));
	      gassign *send = gimple_build_assign (tmp, var);
	      gimple_seq_add_stmt (&sender_seq, send);
	      update_stmt (send);
	      send = gimple_build_assign (sender_ref, tmp);
	      gimple_seq_add_stmt (&sender_seq, send);
	      update_stmt (send);
	    }
	  else
	    gcc_unreachable ();
	}
    }

  /* It's possible for the ET->DEST block (the work done by the active thread)
     to finish with a control-flow insn, e.g. a UNIQUE function call.  Split
     the block and add SENDER_SEQ in the latter part to avoid having control
     flow in the middle of a BB.  */

  decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_END);
  call = gimple_build_call (decl, 1, build_fold_addr_expr (sender_decl));
  gimple_seq_add_stmt (&sender_seq, call);

  gsi = gsi_last_bb (body);
  gimple *last = gsi_stmt (gsi);
  basic_block sender_block = split_block (body, last)->dest;
  gsi = gsi_last_bb (sender_block);
  gsi_insert_seq_after (&gsi, sender_seq, GSI_CONTINUE_LINKING);
}

static void
neuter_worker_single (parallel_g *par, unsigned outer_mask,
		      bitmap worker_single, bitmap vector_single,
		      vec<propagation_set *> *prop_set,
		      hash_set<tree> *partitioned_var_uses,
		      record_field_map_t *record_field_map)
{
  unsigned mask = outer_mask | par->mask;

  if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
    {
      basic_block block;

      for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
	{
	  bool has_defs = false;
	  hash_set<tree> def_escapes_block;
	  hash_set<tree> worker_partitioned_uses;
	  unsigned j;
	  tree var;

	  FOR_EACH_SSA_NAME (j, var, cfun)
	    {
	      if (SSA_NAME_IS_VIRTUAL_OPERAND (var))
		{
		  has_defs = true;
		  continue;
		}

	      gimple *def_stmt = SSA_NAME_DEF_STMT (var);

	      if (gimple_nop_p (def_stmt))
		continue;

	      if (gimple_bb (def_stmt)->index != block->index)
		continue;

	      gimple *use_stmt;
	      imm_use_iterator use_iter;
	      bool uses_outside_block = false;
	      bool worker_partitioned_use = false;

	      FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, var)
		{
		  int blocknum = gimple_bb (use_stmt)->index;

		  /* Don't propagate SSA names that are only used in the
		     current block, unless the usage is in a phi node: that
		     means the name left the block, then came back in at the
		     top.  */
		  if (blocknum != block->index
		      || gimple_code (use_stmt) == GIMPLE_PHI)
		    uses_outside_block = true;
		  if (!bitmap_bit_p (worker_single, blocknum))
		    worker_partitioned_use = true;
		}

	      if (uses_outside_block)
		def_escapes_block.add (var);

	      if (worker_partitioned_use)
		{
		  worker_partitioned_uses.add (var);
		  has_defs = true;
		}
	    }

	  propagation_set *ws_prop = (*prop_set)[block->index];

	  if (ws_prop)
	    {
	      for (propagation_set::iterator it = ws_prop->begin ();
		   it != ws_prop->end ();
		   ++it)
		{
		  tree var = *it;
		  if (TREE_CODE (var) == VAR_DECL)
		    {
		      def_escapes_block.add (var);
		      if (partitioned_var_uses->contains (var))
			{
			  worker_partitioned_uses.add (var);
			  has_defs = true;
			}
		    }
		}

	      delete ws_prop;
	      (*prop_set)[block->index] = 0;
	    }

	  tree record_type = (tree) block->aux;

	  if (has_defs)
	    worker_single_copy (block, block, &def_escapes_block,
				&worker_partitioned_uses, record_type,
				record_field_map);
	  else
	    worker_single_simple (block, block, &def_escapes_block);
	}
    }

  if ((outer_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
    {
      basic_block block;

      for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
	for (gimple_stmt_iterator gsi = gsi_start_bb (block);
	     !gsi_end_p (gsi);
	     gsi_next (&gsi))
	  {
	    gimple *stmt = gsi_stmt (gsi);

	    if (gimple_code (stmt) == GIMPLE_CALL
		&& !gimple_call_internal_p (stmt)
		&& !omp_sese_active_worker_call (as_a <gcall *> (stmt)))
	      {
		/* If we have an OpenACC routine call in worker-single mode,
		   place barriers before and afterwards to prevent
		   clobbering re-used shared memory regions (as are used
		   for AMDGCN at present, for example).  */
		tree decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
		gsi_insert_before (&gsi, gimple_build_call (decl, 0),
				   GSI_SAME_STMT);
		gsi_insert_after (&gsi, gimple_build_call (decl, 0),
				  GSI_NEW_STMT);
	      }
	  }
    }

  if (par->inner)
    neuter_worker_single (par->inner, mask, worker_single, vector_single,
			  prop_set, partitioned_var_uses, record_field_map);
  if (par->next)
    neuter_worker_single (par->next, outer_mask, worker_single, vector_single,
			  prop_set, partitioned_var_uses, record_field_map);
}

static int
execute_omp_oacc_neuter_broadcast ()
{
  bb_stmt_map_t bb_stmt_map;
  auto_bitmap worker_single, vector_single;

  omp_sese_split_blocks (&bb_stmt_map);

  if (dump_file)
    {
      fprintf (dump_file, "\n\nAfter splitting:\n\n");
      dump_function_to_file (current_function_decl, dump_file, dump_flags);
    }

  unsigned mask = 0;

  /* If this is a routine, calculate MASK as if the outer levels are already
     partitioned.  */
  tree attr = oacc_get_fn_attrib (current_function_decl);
  if (attr)
    {
      tree dims = TREE_VALUE (attr);
      unsigned ix;
      for (ix = 0; ix != GOMP_DIM_MAX; ix++, dims = TREE_CHAIN (dims))
	{
	  tree allowed = TREE_PURPOSE (dims);
	  if (allowed && integer_zerop (allowed))
	    mask |= GOMP_DIM_MASK (ix);
	}
    }

  parallel_g *par = omp_sese_discover_pars (&bb_stmt_map);
  populate_single_mode_bitmaps (par, worker_single, vector_single, mask, 0);

  basic_block bb;
  FOR_ALL_BB_FN (bb, cfun)
    bb->aux = NULL;

  vec<propagation_set *> prop_set (vNULL);
  prop_set.safe_grow_cleared (last_basic_block_for_fn (cfun), true);

  find_ssa_names_to_propagate (par, mask, worker_single, vector_single,
			       &prop_set);

  hash_set<tree> partitioned_var_uses;
  hash_set<tree> gang_private_vars;

  find_gang_private_vars (&gang_private_vars);
  find_partitioned_var_uses (par, mask, &partitioned_var_uses);
  find_local_vars_to_propagate (par, mask, &partitioned_var_uses,
				&gang_private_vars, &prop_set);

  record_field_map_t record_field_map;

  FOR_ALL_BB_FN (bb, cfun)
    {
      propagation_set *ws_prop = prop_set[bb->index];
      if (ws_prop)
	{
	  tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
	  tree name = create_tmp_var_name (".oacc_ws_data_s");
	  name = build_decl (UNKNOWN_LOCATION, TYPE_DECL, name, record_type);
	  DECL_ARTIFICIAL (name) = 1;
	  DECL_NAMELESS (name) = 1;
	  TYPE_NAME (record_type) = name;
	  TYPE_ARTIFICIAL (record_type) = 1;

	  auto_vec<tree> field_vec (ws_prop->elements ());
	  for (hash_set<tree>::iterator it = ws_prop->begin ();
	       it != ws_prop->end (); ++it)
	    field_vec.quick_push (*it);

	  field_vec.qsort (sort_by_size_then_ssa_version_or_uid);

	  field_map_t *fields = new field_map_t;

	  bool existed;
	  existed = record_field_map.put (record_type, fields);
	  gcc_checking_assert (!existed);

	  /* Insert var fields in reverse order, so the last inserted element
	     is the first in the structure.  */
	  for (int i = field_vec.length () - 1; i >= 0; i--)
	    install_var_field (field_vec[i], record_type, fields);

	  layout_type (record_type);

	  bb->aux = (tree) record_type;
	}
    }

  neuter_worker_single (par, mask, worker_single, vector_single, &prop_set,
			&partitioned_var_uses, &record_field_map);

  for (auto it : record_field_map)
    delete it.second;
  record_field_map.empty ();

  /* These are supposed to have been 'delete'd by 'neuter_worker_single'.  */
  for (auto it : prop_set)
    gcc_checking_assert (!it);
  prop_set.release ();

  delete par;

  /* This doesn't seem to make a difference.  */
  loops_state_clear (LOOP_CLOSED_SSA);

  /* Neutering worker-single neutered blocks will invalidate dominance info.
     It may be possible to incrementally update just the affected blocks, but
     obliterate everything for now.  */
  free_dominance_info (CDI_DOMINATORS);
  free_dominance_info (CDI_POST_DOMINATORS);

  if (dump_file)
    {
      fprintf (dump_file, "\n\nAfter neutering:\n\n");
      dump_function_to_file (current_function_decl, dump_file, dump_flags);
    }

  return 0;
}

namespace {

const pass_data pass_data_omp_oacc_neuter_broadcast =
{
  GIMPLE_PASS, /* type */
  "omp_oacc_neuter_broadcast", /* name */
  OPTGROUP_OMP, /* optinfo_flags */
  TV_NONE, /* tv_id */
  PROP_cfg, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
};

class pass_omp_oacc_neuter_broadcast : public gimple_opt_pass
{
public:
  pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
    : gimple_opt_pass (pass_data_omp_oacc_neuter_broadcast, ctxt)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *)
  {
    return (flag_openacc
	    && targetm.goacc.create_worker_broadcast_record);
  };

  virtual unsigned int execute (function *)
    {
      return execute_omp_oacc_neuter_broadcast ();
    }

}; // class pass_omp_oacc_neuter_broadcast

} // anon namespace

gimple_opt_pass *
make_pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
{
  return new pass_omp_oacc_neuter_broadcast (ctxt);
}