OpenMP: middle-end support for dispatch + adjust_args

This patch adds middle-end support for the `dispatch` construct and the
`adjust_args` clause. The heavy lifting is done in `gimplify_omp_dispatch` and
`gimplify_call_expr` respectively. For `adjust_args`, this mostly consists in
emitting a call to `omp_get_mapped_ptr` for the adequate device.

For dispatch, the following steps are performed:

* Handle the device clause, if any: set the default-device ICV at the top of the
dispatch region and restore its previous value at the end.

* Handle novariants and nocontext clauses, if any. Evaluate compile-time
constants and select a variant, if possible. Otherwise, emit code to handle all
possible cases at run time.

gcc/ChangeLog:

	* builtins.cc (builtin_fnspec): Handle BUILT_IN_OMP_GET_MAPPED_PTR.
	* gimple-low.cc (lower_stmt): Handle GIMPLE_OMP_DISPATCH.
	* gimple-pretty-print.cc (dump_gimple_omp_dispatch): New function.
	(pp_gimple_stmt_1): Handle GIMPLE_OMP_DISPATCH.
	* gimple-walk.cc (walk_gimple_stmt): Likewise.
	* gimple.cc (gimple_build_omp_dispatch): New function.
	(gimple_copy): Handle GIMPLE_OMP_DISPATCH.
	* gimple.def (GIMPLE_OMP_DISPATCH): Define.
	* gimple.h (gimple_build_omp_dispatch): Declare.
	(gimple_has_substatements): Handle GIMPLE_OMP_DISPATCH.
	(gimple_omp_dispatch_clauses): New function.
	(gimple_omp_dispatch_clauses_ptr): Likewise.
	(gimple_omp_dispatch_set_clauses): Likewise.
	(gimple_return_set_retval): Handle GIMPLE_OMP_DISPATCH.
	* gimplify.cc (enum omp_region_type): Add ORT_DISPATCH.
	(struct gimplify_omp_ctx): Add in_call_args.
	(gimplify_call_expr): Handle need_device_ptr arguments.
	(is_gimple_stmt): Handle OMP_DISPATCH.
	(gimplify_scan_omp_clauses): Handle OMP_CLAUSE_DEVICE in a dispatch
	construct. Handle OMP_CLAUSE_NOVARIANTS and OMP_CLAUSE_NOCONTEXT.
	(omp_has_novariants): New function.
	(omp_has_nocontext): Likewise.
	(omp_construct_selector_matches): Handle OMP_DISPATCH with nocontext
	clause.
	(find_ifn_gomp_dispatch): New function.
	(gimplify_omp_dispatch): Likewise.
	(gimplify_expr): Handle OMP_DISPATCH.
	* gimplify.h (omp_has_novariants): Declare.
	* internal-fn.cc (expand_GOMP_DISPATCH): New function.
	* internal-fn.def (GOMP_DISPATCH): Define.
	* omp-builtins.def (BUILT_IN_OMP_GET_MAPPED_PTR): Define.
	(BUILT_IN_OMP_GET_DEFAULT_DEVICE): Define.
	(BUILT_IN_OMP_SET_DEFAULT_DEVICE): Define.
	* omp-general.cc (omp_construct_traits_to_codes): Add OMP_DISPATCH.
	(struct omp_ts_info): Add dispatch.
	(omp_resolve_declare_variant): Handle novariants. Adjust
	DECL_ASSEMBLER_NAME.
	* omp-low.cc (scan_omp_1_stmt): Handle GIMPLE_OMP_DISPATCH.
	(lower_omp_dispatch): New function.
	(lower_omp_1): Call it.
	* tree-inline.cc (remap_gimple_stmt): Handle GIMPLE_OMP_DISPATCH.
	(estimate_num_insns): Handle GIMPLE_OMP_DISPATCH.

15 files changed, 641 insertions, 22 deletions

diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index 7243c40..ce92b36 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -12583,6 +12583,8 @@ builtin_fnspec (tree callee)
 	 by its first argument.  */
       case BUILT_IN_POSIX_MEMALIGN:
 	return ".cOt";
+      case BUILT_IN_OMP_GET_MAPPED_PTR:
+	return ". R ";
 
       default:
 	return "";
diff --git a/gcc/gimple-low.cc b/gcc/gimple-low.cc
index 6475d7f..55d9dbf 100644
--- a/gcc/gimple-low.cc
+++ b/gcc/gimple-low.cc
@@ -746,6 +746,7 @@ lower_stmt (gimple_stmt_iterator *gsi, struct lower_data *data)
     case GIMPLE_EH_MUST_NOT_THROW:
     case GIMPLE_OMP_FOR:
     case GIMPLE_OMP_SCOPE:
+    case GIMPLE_OMP_DISPATCH:
     case GIMPLE_OMP_SECTIONS:
     case GIMPLE_OMP_SECTIONS_SWITCH:
     case GIMPLE_OMP_SECTION:
diff --git a/gcc/gimple-pretty-print.cc b/gcc/gimple-pretty-print.cc
index 8bd1a6c..dcd03f3 100644
--- a/gcc/gimple-pretty-print.cc
+++ b/gcc/gimple-pretty-print.cc
@@ -1727,6 +1727,35 @@ dump_gimple_omp_scope (pretty_printer *pp, const gimple *gs,
     }
 }
 
+/* Dump a GIMPLE_OMP_DISPATCH tuple on the pretty_printer BUFFER.  */
+
+static void
+dump_gimple_omp_dispatch (pretty_printer *buffer, const gimple *gs, int spc,
+			  dump_flags_t flags)
+{
+  if (flags & TDF_RAW)
+    {
+      dump_gimple_fmt (buffer, spc, flags, "%G <%+BODY <%S>%nCLAUSES <", gs,
+		       gimple_omp_body (gs));
+      dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+      dump_gimple_fmt (buffer, spc, flags, " >");
+    }
+  else
+    {
+      pp_string (buffer, "#pragma omp dispatch");
+      dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+      if (!gimple_seq_empty_p (gimple_omp_body (gs)))
+	{
+	  newline_and_indent (buffer, spc + 2);
+	  pp_left_brace (buffer);
+	  pp_newline (buffer);
+	  dump_gimple_seq (buffer, gimple_omp_body (gs), spc + 4, flags);
+	  newline_and_indent (buffer, spc + 2);
+	  pp_right_brace (buffer);
+	}
+    }
+}
+
 /* Dump a GIMPLE_OMP_TARGET tuple on the pretty_printer PP.  */
 
 static void
@@ -2806,6 +2835,10 @@ pp_gimple_stmt_1 (pretty_printer *pp, const gimple *gs, int spc,
       dump_gimple_omp_scope (pp, gs, spc, flags);
       break;
 
+    case GIMPLE_OMP_DISPATCH:
+      dump_gimple_omp_dispatch(pp, gs, spc, flags);
+      break;
+
     case GIMPLE_OMP_MASTER:
     case GIMPLE_OMP_SECTION:
     case GIMPLE_OMP_STRUCTURED_BLOCK:
diff --git a/gcc/gimple-walk.cc b/gcc/gimple-walk.cc
index 0052031..73346df 100644
--- a/gcc/gimple-walk.cc
+++ b/gcc/gimple-walk.cc
@@ -707,6 +707,7 @@ walk_gimple_stmt (gimple_stmt_iterator *gsi, walk_stmt_fn callback_stmt,
     case GIMPLE_OMP_PARALLEL:
     case GIMPLE_OMP_TASK:
     case GIMPLE_OMP_SCOPE:
+    case GIMPLE_OMP_DISPATCH:
     case GIMPLE_OMP_SECTIONS:
     case GIMPLE_OMP_SINGLE:
     case GIMPLE_OMP_TARGET:
diff --git a/gcc/gimple.cc b/gcc/gimple.cc
index f7b313b..add4a64 100644
--- a/gcc/gimple.cc
+++ b/gcc/gimple.cc
@@ -1239,6 +1239,21 @@ gimple_build_omp_scope (gimple_seq body, tree clauses)
   return p;
 }
 
+/* Build a GIMPLE_OMP_DISPATCH statement.
+
+   BODY is the target function call to be dispatched.
+   CLAUSES are any of the OMP dispatch construct's clauses.  */
+
+gimple *
+gimple_build_omp_dispatch (gimple_seq body, tree clauses)
+{
+  gimple *p = gimple_alloc (GIMPLE_OMP_DISPATCH, 0);
+  gimple_omp_dispatch_set_clauses (p, clauses);
+  if (body)
+    gimple_omp_set_body (p, body);
+
+  return p;
+}
 
 /* Build a GIMPLE_OMP_TARGET statement.
 
@@ -2152,6 +2167,11 @@ gimple_copy (gimple *stmt)
 	  gimple_omp_scope_set_clauses (copy, t);
 	  goto copy_omp_body;
 
+	case GIMPLE_OMP_DISPATCH:
+	  t = unshare_expr (gimple_omp_dispatch_clauses (stmt));
+	  gimple_omp_dispatch_set_clauses (copy, t);
+	  goto copy_omp_body;
+
 	case GIMPLE_OMP_TARGET:
 	  {
 	    gomp_target *omp_target_stmt = as_a <gomp_target *> (stmt);
diff --git a/gcc/gimple.def b/gcc/gimple.def
index fbcd727..21c7405 100644
--- a/gcc/gimple.def
+++ b/gcc/gimple.def
@@ -350,6 +350,11 @@ DEFGSCODE(GIMPLE_OMP_SCAN, "gimple_omp_scan", GSS_OMP_SINGLE_LAYOUT)
    CLAUSES is an OMP_CLAUSE chain holding the associated clauses.  */
 DEFGSCODE(GIMPLE_OMP_SCOPE, "gimple_omp_scope", GSS_OMP_SINGLE_LAYOUT)
 
+/* GIMPLE_OMP_DISPATCH <BODY, CLAUSES> represents #pragma omp dispatch
+   BODY is the target function call to be dispatched.
+   CLAUSES is an OMP_CLAUSE chain holding the associated clauses.  */
+DEFGSCODE(GIMPLE_OMP_DISPATCH, "gimple_omp_dispatch", GSS_OMP_SINGLE_LAYOUT)
+
 /* OMP_SECTION <BODY> represents #pragma omp section.
    BODY is the sequence of statements in the section body.  */
 DEFGSCODE(GIMPLE_OMP_SECTION, "gimple_omp_section", GSS_OMP)
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 4a6e0e9..b6967e6 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -742,7 +742,7 @@ struct GTY((tag("GSS_OMP_CONTINUE")))
 };
 
 /* GIMPLE_OMP_SINGLE, GIMPLE_OMP_ORDERED, GIMPLE_OMP_TASKGROUP,
-   GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE.  */
+   GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE, GIMPLE_OMP_DISPATCH. */
 
 struct GTY((tag("GSS_OMP_SINGLE_LAYOUT")))
   gimple_statement_omp_single_layout : public gimple_statement_omp
@@ -1591,6 +1591,7 @@ gomp_task *gimple_build_omp_task (gimple_seq, tree, tree, tree, tree,
 gimple *gimple_build_omp_section (gimple_seq);
 gimple *gimple_build_omp_structured_block (gimple_seq);
 gimple *gimple_build_omp_scope (gimple_seq, tree);
+gimple *gimple_build_omp_dispatch (gimple_seq, tree);
 gimple *gimple_build_omp_master (gimple_seq);
 gimple *gimple_build_omp_masked (gimple_seq, tree);
 gimple *gimple_build_omp_taskgroup (gimple_seq, tree);
@@ -1882,6 +1883,7 @@ gimple_has_substatements (gimple *g)
     case GIMPLE_OMP_PARALLEL:
     case GIMPLE_OMP_TASK:
     case GIMPLE_OMP_SCOPE:
+    case GIMPLE_OMP_DISPATCH:
     case GIMPLE_OMP_SECTIONS:
     case GIMPLE_OMP_SINGLE:
     case GIMPLE_OMP_TARGET:
@@ -5434,6 +5436,34 @@ gimple_omp_scope_set_clauses (gimple *gs, tree clauses)
     = clauses;
 }
 
+/* Return the clauses associated with OMP_DISPATCH statement GS.  */
+
+inline tree
+gimple_omp_dispatch_clauses (const gimple *gs)
+{
+  GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+  return static_cast<const gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Return a pointer to the clauses associated with OMP dispatch statement
+   GS.  */
+
+inline tree *
+gimple_omp_dispatch_clauses_ptr (gimple *gs)
+{
+  GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+  return &static_cast<gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Set CLAUSES to be the clauses associated with OMP dispatch statement
+   GS.  */
+
+inline void
+gimple_omp_dispatch_set_clauses (gimple *gs, tree clauses)
+{
+  GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+  static_cast<gimple_statement_omp_single_layout *> (gs)->clauses = clauses;
+}
 
 /* Return the kind of the OMP_FOR statemement G.  */
 
@@ -6768,6 +6798,7 @@ gimple_return_set_retval (greturn *gs, tree retval)
     case GIMPLE_OMP_TARGET:			\
     case GIMPLE_OMP_TEAMS:			\
     case GIMPLE_OMP_SCOPE:			\
+    case GIMPLE_OMP_DISPATCH:			\
     case GIMPLE_OMP_SECTION:			\
     case GIMPLE_OMP_STRUCTURED_BLOCK:		\
     case GIMPLE_OMP_MASTER:			\
diff --git a/gcc/gimplify.cc b/gcc/gimplify.cc
index 827941b..0899b279 100644
--- a/gcc/gimplify.cc
+++ b/gcc/gimplify.cc
@@ -162,7 +162,8 @@ enum omp_region_type
 {
   ORT_WORKSHARE = 0x00,
   ORT_TASKGROUP = 0x01,
-  ORT_SIMD 	= 0x04,
+  ORT_DISPATCH	= 0x02,
+  ORT_SIMD	= 0x04,
 
   ORT_PARALLEL	= 0x08,
   ORT_COMBINED_PARALLEL = ORT_PARALLEL | 1,
@@ -259,6 +260,7 @@ struct gimplify_omp_ctx
   bool order_concurrent;
   bool has_depend;
   bool in_for_exprs;
+  bool in_call_args;
   int defaultmap[5];
 };
 
@@ -4072,23 +4074,137 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
   /* Gimplify the function arguments.  */
   if (nargs > 0)
     {
+      tree device_num = NULL_TREE;
       for (i = (PUSH_ARGS_REVERSED ? nargs - 1 : 0);
-           PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
-           PUSH_ARGS_REVERSED ? i-- : i++)
-        {
-          enum gimplify_status t;
+	   PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
+	   PUSH_ARGS_REVERSED ? i-- : i++)
+	{
+	  enum gimplify_status t;
+
+	  /* Avoid gimplifying the second argument to va_start, which needs to
+	     be the plain PARM_DECL.  */
+	  if ((i != 1) || !builtin_va_start_p)
+	    {
+	      tree *arg_p = &CALL_EXPR_ARG (*expr_p, i);
+	      tree adjust_args_list;
+	      if (flag_openmp && gimplify_omp_ctxp != NULL
+		  && gimplify_omp_ctxp->code == OMP_DISPATCH
+		  && !gimplify_omp_ctxp->in_call_args
+		  && !integer_zerop (*arg_p)
+		  && EXPR_P (CALL_EXPR_FN (*expr_p))
+		  && DECL_P (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))
+		  && (adjust_args_list = lookup_attribute (
+			"omp declare variant variant adjust_args",
+			DECL_ATTRIBUTES (
+			  TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))))
+		      != NULL_TREE)
+		{
+		  tree arg_types = TYPE_ARG_TYPES (
+		    TREE_TYPE (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0)));
+
+		  if (arg_types != NULL_TREE)
+		    {
+		      for (int param_idx = 0; param_idx < i; param_idx++)
+			arg_types = TREE_CHAIN (arg_types);
+
+		      bool need_device_ptr = false;
+		      for (tree arg
+			   = TREE_PURPOSE (TREE_VALUE (adjust_args_list));
+			   arg != NULL; arg = TREE_CHAIN (arg))
+			{
+			  if (TREE_VALUE (arg)
+			      && TREE_CODE (TREE_VALUE (arg)) == INTEGER_CST
+			      && wi::eq_p (i, wi::to_wide (TREE_VALUE (arg))))
+			    {
+			      need_device_ptr = true;
+			      break;
+			    }
+			}
+
+		      if (need_device_ptr)
+			{
+			  bool is_device_ptr = false;
+			  for (tree c = gimplify_omp_ctxp->clauses; c;
+			       c = TREE_CHAIN (c))
+			    {
+			      if (OMP_CLAUSE_CODE (c)
+				  == OMP_CLAUSE_IS_DEVICE_PTR)
+				{
+				  tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c));
+				  tree decl2
+				    = tree_strip_nop_conversions (*arg_p);
+				  if (TREE_CODE (decl2) == ADDR_EXPR)
+				    decl2 = TREE_OPERAND (decl2, 0);
+				  if (VAR_P (decl2)
+				      || TREE_CODE (decl2) == PARM_DECL)
+				    {
+				      decl2 = DECL_NAME (decl2);
+				      if (decl1 == decl2)
+					is_device_ptr = true;
+				    }
+				}
+			      else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+				device_num = OMP_CLAUSE_OPERAND (c, 0);
+			    }
+
+			  if (!is_device_ptr)
+			    {
+			      if (device_num == NULL_TREE)
+				{
+				  // device_num = omp_get_default_device ()
+				  tree fn = builtin_decl_explicit (
+				    BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+				  gcall *call = gimple_build_call (fn, 0);
+				  device_num = create_tmp_var (
+				    gimple_call_return_type (call));
+				  gimple_call_set_lhs (call, device_num);
+				  gimplify_seq_add_stmt (pre_p, call);
+				}
+
+			      // mapped_arg = omp_get_mapped_ptr (arg,
+			      // device_num)
+			      tree fn = builtin_decl_explicit (
+				BUILT_IN_OMP_GET_MAPPED_PTR);
+			      gimplify_arg (arg_p, pre_p, loc);
+			      gimplify_arg (&device_num, pre_p, loc);
+			      call
+				= gimple_build_call (fn, 2, *arg_p, device_num);
+			      tree mapped_arg = create_tmp_var (
+				gimple_call_return_type (call));
+			      gimple_call_set_lhs (call, mapped_arg);
+			      gimplify_seq_add_stmt (pre_p, call);
+
+			      *arg_p = mapped_arg;
+
+			      // gimplify_call_expr might be called several
+			      // times on the same call, which would result in
+			      // duplicated calls to omp_get_default_device and
+			      // omp_get_mapped_ptr. To prevent that, we mark
+			      // already mapped arguments as device pointers.
+			      gcc_checking_assert (gimplify_omp_ctxp->code
+						   == OMP_DISPATCH);
+			      tree c
+				= build_omp_clause (input_location,
+						    OMP_CLAUSE_IS_DEVICE_PTR);
+			      OMP_CLAUSE_DECL (c) = *arg_p;
+			      OMP_CLAUSE_CHAIN (c) = gimplify_omp_ctxp->clauses;
+			      gimplify_omp_ctxp->clauses = c;
+			    }
+			}
+		    }
+		}
 
-          /* Avoid gimplifying the second argument to va_start, which needs to
-             be the plain PARM_DECL.  */
-          if ((i != 1) || !builtin_va_start_p)
-            {
-              t = gimplify_arg (&CALL_EXPR_ARG (*expr_p, i), pre_p,
-				EXPR_LOCATION (*expr_p), ! returns_twice);
+	      if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+		gimplify_omp_ctxp->in_call_args = true;
+	      t = gimplify_arg (arg_p, pre_p, EXPR_LOCATION (*expr_p),
+				!returns_twice);
+	      if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+		gimplify_omp_ctxp->in_call_args = false;
 
-              if (t == GS_ERROR)
-                ret = GS_ERROR;
-            }
-        }
+	      if (t == GS_ERROR)
+		ret = GS_ERROR;
+	    }
+	}
     }
 
   /* Gimplify the static chain.  */
@@ -6388,6 +6504,7 @@ is_gimple_stmt (tree t)
     case OACC_LOOP:
     case OMP_SCAN:
     case OMP_SCOPE:
+    case OMP_DISPATCH:
     case OMP_SECTIONS:
     case OMP_SECTION:
     case OMP_STRUCTURED_BLOCK:
@@ -13208,6 +13325,21 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
 		    break;
 		  }
 	    }
+	  else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE
+		   && code == OMP_DISPATCH)
+	    {
+	      bool saved_into_ssa = gimplify_ctxp->into_ssa;
+	      gimplify_ctxp->into_ssa = false;
+	      if (gimplify_expr (&OMP_CLAUSE_DEVICE_ID (c), pre_p, NULL,
+				 is_gimple_val, fb_rvalue)
+		  == GS_ERROR)
+		remove = true;
+	      else if (DECL_P (OMP_CLAUSE_DEVICE_ID (c)))
+		omp_add_variable (ctx, OMP_CLAUSE_DEVICE_ID (c),
+				  GOVD_SHARED | GOVD_SEEN);
+	      gimplify_ctxp->into_ssa = saved_into_ssa;
+	      break;
+	    }
 	  /* Fall through.  */
 
 	case OMP_CLAUSE_PRIORITY:
@@ -13448,6 +13580,14 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
 	  }
 	  break;
 
+	case OMP_CLAUSE_NOVARIANTS:
+	  OMP_CLAUSE_NOVARIANTS_EXPR (c)
+	    = gimple_boolify (OMP_CLAUSE_NOVARIANTS_EXPR (c));
+	  break;
+	case OMP_CLAUSE_NOCONTEXT:
+	  OMP_CLAUSE_NOCONTEXT_EXPR (c)
+	    = gimple_boolify (OMP_CLAUSE_NOCONTEXT_EXPR (c));
+	  break;
 	case OMP_CLAUSE_NOHOST:
 	default:
 	  gcc_unreachable ();
@@ -14775,6 +14915,54 @@ gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p,
   delete_omp_context (ctx);
 }
 
+/* Try to evaluate a novariants clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+int
+omp_has_novariants (void)
+{
+  struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+  if (ctx != NULL && ctx->code == OMP_DISPATCH && !ctx->in_call_args)
+    {
+      tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOVARIANTS);
+      if (c != NULL_TREE)
+	{
+	  if (integer_nonzerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+	    return 1;
+	  else if (integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+	    return 0;
+	  else
+	    return -1;
+	}
+      return 0;
+    }
+  return 0;
+}
+
+/* Try to evaluate a nocontext clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+static int
+omp_has_nocontext (void)
+{
+  struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+  if (ctx != NULL && ctx->code == OMP_DISPATCH)
+    {
+      tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOCONTEXT);
+      if (c != NULL_TREE)
+	{
+	  if (integer_nonzerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+	    return 1;
+	  else if (integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+	    return 0;
+	  else
+	    return -1;
+	}
+      return 0;
+    }
+  return 0;
+}
+
 /* Return 0 if CONSTRUCTS selectors don't match the OpenMP context,
    -1 if unknown yet (simd is involved, won't be known until vectorization)
    and 1 if they do.  If SCORES is non-NULL, it should point to an array
@@ -14802,9 +14990,9 @@ omp_construct_selector_matches (enum tree_code *constructs, int nconstructs,
 	      == ORT_TARGET && ctx->code == OMP_TARGET)
 	  || ((ctx->region_type & ORT_TEAMS) && ctx->code == OMP_TEAMS)
 	  || (ctx->region_type == ORT_WORKSHARE && ctx->code == OMP_FOR)
-	  || (ctx->region_type == ORT_SIMD
-	      && ctx->code == OMP_SIMD
-	      && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND)))
+	  || (ctx->region_type == ORT_SIMD && ctx->code == OMP_SIMD
+	      && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND))
+	  || (ctx->code == OMP_DISPATCH && omp_has_nocontext () != 1))
 	{
 	  ++cnt;
 	  if (scores)
@@ -17915,6 +18103,272 @@ gimplify_omp_ordered (tree expr, gimple_seq body)
   return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr));
 }
 
+/* Callback for walk_tree to find an IFN_GOMP_DISPATCH.  */
+
+static tree
+find_ifn_gomp_dispatch (tree *tp, int *, void *modify)
+{
+  tree t = *tp;
+
+  if (TREE_CODE (t) == CALL_EXPR && CALL_EXPR_IFN (t) == IFN_GOMP_DISPATCH)
+    {
+      *tp = CALL_EXPR_ARG (t, 0);
+      return *(tree *) modify ? *(tree *) modify : *tp;
+    }
+
+  if (TREE_CODE (t) == MODIFY_EXPR)
+    *(tree *) modify = *tp;
+
+  return NULL_TREE;
+}
+
+/* Gimplify an OMP_DISPATCH construct.  */
+
+static enum gimplify_status
+gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p)
+{
+  tree expr = *expr_p;
+  gimple_seq body = NULL;
+
+  gimplify_scan_omp_clauses (&OMP_DISPATCH_CLAUSES (expr), pre_p, ORT_DISPATCH,
+			     OMP_DISPATCH);
+  push_gimplify_context ();
+
+  // If device clause, adjust ICV
+  tree device
+    = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE);
+  tree saved_device_icv = NULL_TREE;
+  if (device
+      && (TREE_CODE (OMP_CLAUSE_DEVICE_ID (device)) != INTEGER_CST
+	  || !wi::eq_p (wi::to_wide (OMP_CLAUSE_DEVICE_ID (device)),
+			-1 /* omp_initial_device */)))
+    {
+      // Save current default-device-var ICV
+      saved_device_icv = create_tmp_var (integer_type_node);
+      tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+      gcall *call = gimple_build_call (fn, 0);
+      gimple_call_set_lhs (call, saved_device_icv);
+      gimplify_seq_add_stmt (&body, call);
+
+      // Set default device
+      fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+      call = gimple_build_call (fn, 1, OMP_CLAUSE_DEVICE_ID (device));
+      gimplify_seq_add_stmt (&body, call);
+    }
+
+  // If the novariants and nocontext clauses are not compile-time constants,
+  // we need to generate code for all possible cases:
+  //   if (novariants) // implies nocontext
+  //       base()
+  //   else if (nocontext)
+  //       variant1()
+  //   else
+  //       variant2()
+  tree *dispatch_body_p = &OMP_DISPATCH_BODY (expr);
+  if (TREE_CODE (*dispatch_body_p) == BIND_EXPR)
+    dispatch_body_p = &BIND_EXPR_BODY (*dispatch_body_p);
+  tree dispatch_body = *dispatch_body_p;
+
+  // Look for IFN_GOMP_DISPATCH and extract the base function call
+  tree base_call_expr = NULL_TREE;
+  if (TREE_CODE (dispatch_body) == STATEMENT_LIST)
+    for (tree_stmt_iterator tsi = tsi_start (dispatch_body); !tsi_end_p (tsi);
+	 tsi_next (&tsi))
+      {
+	tree modify = NULL_TREE;
+	tree stmt = tsi_stmt (tsi);
+	base_call_expr
+	  = walk_tree (&stmt, find_ifn_gomp_dispatch, &modify, NULL);
+	if (base_call_expr != NULL_TREE)
+	  {
+	    tsi_link_before (&tsi, base_call_expr, TSI_CONTINUE_LINKING);
+	    tsi_next (&tsi);
+	    tsi_delink (&tsi);
+	    break;
+	  }
+      }
+  else
+    {
+      tree modify = NULL_TREE;
+      base_call_expr
+	= walk_tree (dispatch_body_p, find_ifn_gomp_dispatch, &modify, NULL);
+    }
+  gcc_assert (base_call_expr != NULL_TREE);
+
+  tree dst = NULL_TREE;
+  if (TREE_CODE (base_call_expr) == MODIFY_EXPR)
+    {
+      dst = TREE_OPERAND (base_call_expr, 0);
+      base_call_expr = TREE_OPERAND (base_call_expr, 1);
+    }
+  while (TREE_CODE (base_call_expr) == FLOAT_EXPR
+	 || TREE_CODE (base_call_expr) == CONVERT_EXPR
+	 || TREE_CODE (base_call_expr) == COMPLEX_EXPR
+	 || TREE_CODE (base_call_expr) == INDIRECT_REF
+	 || TREE_CODE (base_call_expr) == NOP_EXPR)
+    base_call_expr = TREE_OPERAND (base_call_expr, 0);
+
+  tree base_fndecl = get_callee_fndecl (base_call_expr);
+  if (base_fndecl != NULL_TREE)
+    {
+      if (DECL_VIRTUAL_P (base_fndecl))
+	{
+	  error_at (
+	    EXPR_LOCATION (base_call_expr),
+	    "%qD is a virtual function but only a direct call is allowed "
+	    "in a dispatch construct",
+	    DECL_NAME (base_fndecl));
+	}
+
+      tree variant_fndecl = omp_resolve_declare_variant (base_fndecl);
+      if (base_fndecl != variant_fndecl
+	  && (omp_has_novariants () == -1 || omp_has_nocontext () == -1))
+	{
+	  tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE,
+	       novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE;
+	  for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c))
+	    {
+	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS
+		  && !integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+		{
+		  gcc_assert (novariants_cond == NULL_TREE);
+		  novariants_clause = c;
+		  novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c);
+		}
+	      else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT
+		       && !integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+		{
+		  gcc_assert (nocontext_cond == NULL_TREE);
+		  nocontext_clause = c;
+		  nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c);
+		}
+	    }
+	  gcc_assert (novariants_cond != NULL_TREE
+		      || nocontext_cond != NULL_TREE);
+
+	  enum gimplify_status ret
+	    = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val,
+			     fb_rvalue);
+	  if (ret == GS_ERROR || ret == GS_UNHANDLED)
+	    return ret;
+	  ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val,
+			       fb_rvalue);
+	  if (ret == GS_ERROR || ret == GS_UNHANDLED)
+	    return ret;
+
+	  tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+	  if (novariants_cond != NULL_TREE)
+	    {
+	      tree base_label = create_artificial_label (UNKNOWN_LOCATION);
+	      tree cond_label = create_artificial_label (UNKNOWN_LOCATION);
+	      gcond *novariants_cond_stmt
+		= gimple_build_cond_from_tree (novariants_cond, base_label,
+					       cond_label);
+	      gimplify_seq_add_stmt (&body, novariants_cond_stmt);
+
+	      gimplify_seq_add_stmt (&body, gimple_build_label (base_label));
+	      tree base_call_expr2 = copy_node (base_call_expr);
+	      if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+		{
+		  base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+					    base_call_expr2);
+		}
+	      OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+		= boolean_true_node;
+	      gimplify_and_add (base_call_expr2, &body);
+	      gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+
+	      OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+		= boolean_false_node;
+	      gimplify_seq_add_stmt (&body, gimple_build_label (cond_label));
+	    }
+
+	  if (nocontext_cond != NULL_TREE)
+	    {
+	      tree variant1_label = create_artificial_label (UNKNOWN_LOCATION);
+	      tree variant2_label = create_artificial_label (UNKNOWN_LOCATION);
+	      gcond *nocontext_cond_stmt
+		= gimple_build_cond_from_tree (nocontext_cond, variant1_label,
+					       variant2_label);
+	      gimplify_seq_add_stmt (&body, nocontext_cond_stmt);
+
+	      gimplify_seq_add_stmt (&body,
+				     gimple_build_label (variant1_label));
+	      tree variant_call_expr = copy_node (base_call_expr);
+	      if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+		{
+		  variant_call_expr = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+					      variant_call_expr);
+		}
+	      OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_true_node;
+	      gimplify_and_add (variant_call_expr, &body);
+	      gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+	      OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_false_node;
+	      gimplify_seq_add_stmt (&body,
+				     gimple_build_label (variant2_label));
+	    }
+
+	  tree variant_call_expr = base_call_expr;
+	  if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+	    {
+	      variant_call_expr
+		= build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr);
+	    }
+	  gimplify_and_add (variant_call_expr, &body);
+	  gimplify_seq_add_stmt (&body, gimple_build_label (end_label));
+	}
+      else
+	gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+    }
+  else
+    gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+
+  // Restore default-device-var ICV
+  if (saved_device_icv != NULL_TREE)
+    {
+      tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+      gcall *call = gimple_build_call (fn, 1, saved_device_icv);
+      gimplify_seq_add_stmt (&body, call);
+    }
+
+  // Wrap dispatch body into a bind
+  gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE);
+  pop_gimplify_context (bind);
+
+  // Manually tear down context created by gimplify_scan_omp_clauses to avoid a
+  // call to gimplify_adjust_omp_clauses
+  gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+  if (ctx != NULL)
+    {
+      gcc_assert (ctx->code == OMP_DISPATCH);
+      gimplify_omp_ctxp = ctx->outer_context;
+      delete_omp_context (ctx);
+    }
+
+  // Remove nowait as it has no effect on dispatch (OpenMP 5.2), device as it
+  // has been handled above, and depend as the front end handled it by inserting
+  // taskwait.
+  tree *dispatch_clauses_ptr = &OMP_DISPATCH_CLAUSES (expr);
+  for (tree c = *dispatch_clauses_ptr; c; c = *dispatch_clauses_ptr)
+    {
+      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT
+	  || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
+	  || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+	{
+	  *dispatch_clauses_ptr = OMP_CLAUSE_CHAIN (c);
+	  break;
+	}
+      else
+	dispatch_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
+    }
+
+  gimple *stmt = gimple_build_omp_dispatch (bind, OMP_DISPATCH_CLAUSES (expr));
+  gimplify_seq_add_stmt (pre_p, stmt);
+  *expr_p = NULL_TREE;
+  return GS_ALL_DONE;
+}
+
 /* Convert the GENERIC expression tree *EXPR_P to GIMPLE.  If the
    expression produces a value to be used as an operand inside a GIMPLE
    statement, the value will be stored back in *EXPR_P.  This value will
@@ -18853,6 +19307,10 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
 	  ret = gimplify_omp_atomic (expr_p, pre_p);
 	  break;
 
+	case OMP_DISPATCH:
+	  ret = gimplify_omp_dispatch (expr_p, pre_p);
+	  break;
+
 	case TRANSACTION_EXPR:
 	  ret = gimplify_transaction (expr_p, pre_p);
 	  break;
@@ -19178,7 +19636,8 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
 		  && code != OMP_SECTION
 		  && code != OMP_STRUCTURED_BLOCK
 		  && code != OMP_SINGLE
-		  && code != OMP_SCOPE);
+		  && code != OMP_SCOPE
+		  && code != OMP_DISPATCH);
     }
 #endif
 
diff --git a/gcc/gimplify.h b/gcc/gimplify.h
index ac3cc8e..2e91267 100644
--- a/gcc/gimplify.h
+++ b/gcc/gimplify.h
@@ -77,6 +77,7 @@ extern enum gimplify_status gimplify_expr (tree *, gimple_seq *, gimple_seq *,
 					   bool (*) (tree), fallback_t);
 
 int omp_construct_selector_matches (enum tree_code *, int, int *);
+int omp_has_novariants (void);
 
 extern void gimplify_type_sizes (tree, gimple_seq *);
 extern void gimplify_one_sizepos (tree *, gimple_seq *);
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index c522a53..a45b329 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -689,6 +689,14 @@ expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
   gcc_unreachable ();
 }
 
+/* This should get expanded in gimplify_omp_dispatch.  */
+
+static void
+expand_GOMP_DISPATCH (internal_fn, gcall *)
+{
+  gcc_unreachable ();
+}
+
 /* This should get expanded in the sanopt pass.  */
 
 static void
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 09b7844..e993c99 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -476,6 +476,7 @@ DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_START, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_END, ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (GOMP_DISPATCH, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (LOOP_VECTORIZED, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (LOOP_DIST_ALIAS, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (ANNOTATE,  ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
diff --git a/gcc/omp-builtins.def b/gcc/omp-builtins.def
index 7b49ef1..c70e0770 100644
--- a/gcc/omp-builtins.def
+++ b/gcc/omp-builtins.def
@@ -80,6 +80,12 @@ DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_TEAM_NUM, "omp_get_team_num",
 		  BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_NUM_TEAMS, "omp_get_num_teams",
 		  BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_MAPPED_PTR, "omp_get_mapped_ptr",
+		  BT_FN_PTR_CONST_PTR_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_DEFAULT_DEVICE, "omp_get_default_device",
+		  BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_SET_DEFAULT_DEVICE, "omp_set_default_device",
+		  BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
 
 DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ATOMIC_START, "GOMP_atomic_start",
 		  BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
diff --git a/gcc/omp-general.cc b/gcc/omp-general.cc
index 72fb7f9..adf6229 100644
--- a/gcc/omp-general.cc
+++ b/gcc/omp-general.cc
@@ -1063,7 +1063,7 @@ omp_construct_traits_to_codes (tree ctx, int nconstructs,
   /* Order must match the OMP_TRAIT_CONSTRUCT_* enumerators in
      enum omp_ts_code.  */
   static enum tree_code code_map[]
-    = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD };
+    = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD, OMP_DISPATCH };
 
   for (tree ts = ctx; ts; ts = TREE_CHAIN (ts), i--)
     {
@@ -1267,10 +1267,14 @@ struct omp_ts_info omp_ts_map[] =
      OMP_TRAIT_PROPERTY_CLAUSE_LIST,  false,
      NULL
    },
+   { "dispatch",
+     (1 << OMP_TRAIT_SET_CONSTRUCT),
+     OMP_TRAIT_PROPERTY_NONE,  false,
+     NULL
+   },
    { NULL, 0, OMP_TRAIT_PROPERTY_NONE, false, NULL }  /* OMP_TRAIT_LAST */
   };
 
-
 /* Return a name from PROP, a property in selectors accepting
    name lists.  */
 
@@ -2593,6 +2597,9 @@ omp_resolve_declare_variant (tree base)
   if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
     return omp_resolve_late_declare_variant (base);
 
+  if (omp_has_novariants () == 1)
+    return base;
+
   auto_vec <tree, 16> variants;
   auto_vec <bool, 16> defer;
   bool any_deferred = false;
@@ -2739,6 +2746,8 @@ omp_resolve_declare_variant (tree base)
       (*slot)->variants = entry.variants;
       tree alt = build_decl (DECL_SOURCE_LOCATION (base), FUNCTION_DECL,
 			     DECL_NAME (base), TREE_TYPE (base));
+      if (DECL_ASSEMBLER_NAME_SET_P (base))
+	SET_DECL_ASSEMBLER_NAME (alt, DECL_ASSEMBLER_NAME (base));
       DECL_ARTIFICIAL (alt) = 1;
       DECL_IGNORED_P (alt) = 1;
       TREE_STATIC (alt) = 1;
diff --git a/gcc/omp-low.cc b/gcc/omp-low.cc
index 70a2c10..32d252c 100644
--- a/gcc/omp-low.cc
+++ b/gcc/omp-low.cc
@@ -4206,6 +4206,11 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
       scan_omp (gimple_omp_body_ptr (stmt), ctx);
       break;
 
+    case GIMPLE_OMP_DISPATCH:
+      ctx = new_omp_context (stmt, ctx);
+      scan_omp (gimple_omp_body_ptr (stmt), ctx);
+      break;
+
     case GIMPLE_OMP_SECTIONS:
       scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
       break;
@@ -8948,6 +8953,31 @@ lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   if (BLOCK_VARS (block))
     TREE_USED (block) = 1;
 }
+
+/* Lower code for an OMP dispatch directive.  */
+
+static void
+lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
+{
+  tree block;
+  gimple *stmt = gsi_stmt (*gsi_p);
+  gbind *bind;
+
+  push_gimplify_context ();
+
+  block = make_node (BLOCK);
+  bind = gimple_build_bind (NULL, NULL, block);
+  gsi_replace (gsi_p, bind, true);
+
+  lower_omp (gimple_omp_body_ptr (stmt), ctx);
+  gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
+
+  pop_gimplify_context (bind);
+
+  gimple_bind_append_vars (bind, ctx->block_vars);
+  BLOCK_VARS (block) = ctx->block_vars;
+}
+
 /* Expand code for an OpenMP master or masked directive.  */
 
 static void
@@ -14579,6 +14609,11 @@ lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
       gcc_assert (ctx);
       lower_omp_scope (gsi_p, ctx);
       break;
+    case GIMPLE_OMP_DISPATCH:
+      ctx = maybe_lookup_ctx (stmt);
+      gcc_assert (ctx);
+      lower_omp_dispatch (gsi_p, ctx);
+      break;
     case GIMPLE_OMP_SINGLE:
       ctx = maybe_lookup_ctx (stmt);
       gcc_assert (ctx);
diff --git a/gcc/tree-inline.cc b/gcc/tree-inline.cc
index d16ad6a..d89b7ce 100644
--- a/gcc/tree-inline.cc
+++ b/gcc/tree-inline.cc
@@ -1680,6 +1680,12 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
 		   (s1, gimple_omp_scope_clauses (stmt));
 	  break;
 
+	case GIMPLE_OMP_DISPATCH:
+	  s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
+	  copy = gimple_build_omp_dispatch (s1,
+					    gimple_omp_dispatch_clauses (stmt));
+	  break;
+
 	case GIMPLE_OMP_TASKGROUP:
 	  s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
 	  copy = gimple_build_omp_taskgroup
@@ -4610,6 +4616,7 @@ estimate_num_insns (gimple *stmt, eni_weights *weights)
     case GIMPLE_OMP_MASTER:
     case GIMPLE_OMP_MASKED:
     case GIMPLE_OMP_SCOPE:
+    case GIMPLE_OMP_DISPATCH:
     case GIMPLE_OMP_TASKGROUP:
     case GIMPLE_OMP_ORDERED:
     case GIMPLE_OMP_SCAN: