openacc: Adjust dynamic reference count semantics

This patch adjusts how dynamic reference counts work so that they match
the semantics of the source program more closely, instead of representing
"excess" reference counts beyond those that represent pointers in the
internal libgomp splay-tree data structure. This allows some corner
cases to be handled more gracefully.

2020-07-10  Julian Brown  <julian@codesourcery.com>
	    Thomas Schwinge  <thomas@codesourcery.com>

	libgomp/
	* libgomp.h (struct splay_tree_key_s): Change virtual_refcount to
	dynamic_refcount.
	(struct gomp_device_descr): Remove GOMP_MAP_VARS_OPENACC_ENTER_DATA.
	* oacc-mem.c (acc_map_data): Substitute virtual_refcount for
	dynamic_refcount.
	(acc_unmap_data): Update comment.
	(goacc_map_var_existing, goacc_enter_datum): Adjust for
	dynamic_refcount semantics.
	(goacc_exit_datum_1, goacc_exit_datum): Re-add some error checking.
	Adjust for dynamic_refcount semantics.
	(goacc_enter_data_internal): Implement "present" case of dynamic
	memory-map handling here.  Update "non-present" case for
	dynamic_refcount semantics.
	(goacc_exit_data_internal): Use goacc_exit_datum_1.
	* target.c (gomp_map_vars_internal): Remove
	GOMP_MAP_VARS_OPENACC_ENTER_DATA handling.  Update for dynamic_refcount
	handling.
	(gomp_unmap_vars_internal): Remove virtual_refcount handling.
	(gomp_load_image_to_device): Substitute dynamic_refcount for
	virtual_refcount.
	* testsuite/libgomp.oacc-c-c++-common/pr92843-1.c: Remove XFAILs.
	* testsuite/libgomp.oacc-c-c++-common/refcounting-1.c: New test.
	* testsuite/libgomp.oacc-c-c++-common/refcounting-2.c: New test.
	* testsuite/libgomp.oacc-c-c++-common/struct-3-1-1.c: New test.
	* testsuite/libgomp.oacc-fortran/deep-copy-6.f90: Remove XFAILs and
	trace output.
	* testsuite/libgomp.oacc-fortran/deep-copy-6-no_finalize.F90: Remove
	trace output.
	* testsuite/libgomp.oacc-fortran/dynamic-incr-structural-1.f90: New
	test.
	* testsuite/libgomp.oacc-c-c++-common/structured-dynamic-lifetimes-4.c:
	Remove stale comment.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-1-1.f90: Remove XFAILs.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-1-2.F90: Likewise.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-2-1.f90: Likewise.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-2-2.f90: Likewise.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-3-1.f90: Likewise.
	* testsuite/libgomp.oacc-fortran/mdc-refcount-1-4-1.f90: Adjust XFAIL.

Co-Authored-By: Thomas Schwinge <thomas@codesourcery.com>

1 files changed, 155 insertions, 81 deletions

diff --git a/libgomp/oacc-mem.c b/libgomp/oacc-mem.c
index 34f519a..855cad8 100644
--- a/libgomp/oacc-mem.c
+++ b/libgomp/oacc-mem.c
@@ -409,7 +409,7 @@ acc_map_data (void *h, void *d, size_t s)
       splay_tree_key n = tgt->list[0].key;
       assert (n);
       assert (n->refcount == 1);
-      assert (n->virtual_refcount == 0);
+      assert (n->dynamic_refcount == 0);
       /* Special reference counting behavior.  */
       n->refcount = REFCOUNT_INFINITY;
 
@@ -456,7 +456,7 @@ acc_unmap_data (void *h)
 		  (void *) n->host_start, (int) host_size, (void *) h);
     }
   /* TODO This currently doesn't catch 'REFCOUNT_INFINITY' usage different from
-     'acc_map_data'.  Maybe 'virtual_refcount' can be used for disambiguating
+     'acc_map_data'.  Maybe 'dynamic_refcount' can be used for disambiguating
      the different 'REFCOUNT_INFINITY' cases, or simply separate
      'REFCOUNT_INFINITY' values per different usage ('REFCOUNT_ACC_MAP_DATA'
      etc.)?  */
@@ -520,10 +520,8 @@ goacc_map_var_existing (struct gomp_device_descr *acc_dev, void *hostaddr,
 
   assert (n->refcount != REFCOUNT_LINK);
   if (n->refcount != REFCOUNT_INFINITY)
-    {
-      n->refcount++;
-      n->virtual_refcount++;
-    }
+    n->refcount++;
+  n->dynamic_refcount++;
 
   return d;
 }
@@ -574,13 +572,14 @@ goacc_enter_datum (void **hostaddrs, size_t *sizes, void *kinds, int async)
 
       struct target_mem_desc *tgt
 	= gomp_map_vars_async (acc_dev, aq, mapnum, hostaddrs, NULL, sizes,
-			       kinds, true, GOMP_MAP_VARS_OPENACC_ENTER_DATA);
+			       kinds, true, GOMP_MAP_VARS_ENTER_DATA);
       assert (tgt);
       assert (tgt->list_count == 1);
       n = tgt->list[0].key;
       assert (n);
       assert (n->refcount == 1);
-      assert (n->virtual_refcount == 0);
+      assert (n->dynamic_refcount == 0);
+      n->dynamic_refcount++;
 
       d = (void *) tgt->tgt_start;
     }
@@ -676,24 +675,30 @@ goacc_exit_datum_1 (struct gomp_device_descr *acc_dev, void *h, size_t s,
 		  (void *) h, (int) s, (void *) n->host_start, (int) host_size);
     }
 
-  bool finalize = (kind == GOMP_MAP_DELETE
-		   || kind == GOMP_MAP_FORCE_FROM);
+  bool finalize = (kind == GOMP_MAP_FORCE_FROM
+		   || kind == GOMP_MAP_DELETE
+		   || kind == GOMP_MAP_FORCE_DETACH);
+
+  assert (n->refcount != REFCOUNT_LINK);
+  if (n->refcount != REFCOUNT_INFINITY
+      && n->refcount < n->dynamic_refcount)
+    {
+      gomp_mutex_unlock (&acc_dev->lock);
+      gomp_fatal ("Dynamic reference counting assert fail\n");
+    }
 
   if (finalize)
     {
       if (n->refcount != REFCOUNT_INFINITY)
-	n->refcount -= n->virtual_refcount;
-      n->virtual_refcount = 0;
+	n->refcount -= n->dynamic_refcount;
+      n->dynamic_refcount = 0;
     }
-
-  if (n->virtual_refcount > 0)
+  else if (n->dynamic_refcount)
     {
       if (n->refcount != REFCOUNT_INFINITY)
 	n->refcount--;
-      n->virtual_refcount--;
+      n->dynamic_refcount--;
     }
-  else if (n->refcount > 0 && n->refcount != REFCOUNT_INFINITY)
-    n->refcount--;
 
   if (n->refcount == 0)
     {
@@ -1068,18 +1073,144 @@ goacc_enter_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum,
 			   void **hostaddrs, size_t *sizes,
 			   unsigned short *kinds, goacc_aq aq)
 {
+  gomp_mutex_lock (&acc_dev->lock);
+
   for (size_t i = 0; i < mapnum; i++)
     {
-      int group_last = find_group_last (i, mapnum, sizes, kinds);
+      splay_tree_key n;
+      size_t group_last = find_group_last (i, mapnum, sizes, kinds);
+      bool struct_p = false;
+      size_t size, groupnum = (group_last - i) + 1;
+
+      switch (kinds[i] & 0xff)
+	{
+	case GOMP_MAP_STRUCT:
+	  {
+	    size = (uintptr_t) hostaddrs[group_last] + sizes[group_last]
+		   - (uintptr_t) hostaddrs[i];
+	    struct_p = true;
+	  }
+	  break;
+
+	case GOMP_MAP_ATTACH:
+	  size = sizeof (void *);
+	  break;
+
+	default:
+	  size = sizes[i];
+	}
 
-      gomp_map_vars_async (acc_dev, aq,
-			   (group_last - i) + 1,
-			   &hostaddrs[i], NULL,
-			   &sizes[i], &kinds[i], true,
-			   GOMP_MAP_VARS_OPENACC_ENTER_DATA);
+      n = lookup_host (acc_dev, hostaddrs[i], size);
+
+      if (n && struct_p)
+	{
+	  for (size_t j = i + 1; j <= group_last; j++)
+	    {
+	      struct splay_tree_key_s cur_node;
+	      cur_node.host_start = (uintptr_t) hostaddrs[j];
+	      cur_node.host_end = cur_node.host_start + sizes[j];
+	      splay_tree_key n2
+		= splay_tree_lookup (&acc_dev->mem_map, &cur_node);
+	      if (!n2
+		  || n2->tgt != n->tgt
+		  || n2->host_start - n->host_start
+		     != n2->tgt_offset - n->tgt_offset)
+		{
+		  gomp_mutex_unlock (&acc_dev->lock);
+		  gomp_fatal ("Trying to map into device [%p..%p) structure "
+			      "element when other mapped elements from the "
+			      "same structure weren't mapped together with "
+			      "it", (void *) cur_node.host_start,
+			      (void *) cur_node.host_end);
+		}
+	    }
+	  /* This is a special case because we must increment the refcount by
+	     the number of mapped struct elements, rather than by one.  */
+	  if (n->refcount != REFCOUNT_INFINITY)
+	    n->refcount += groupnum - 1;
+	  n->dynamic_refcount += groupnum - 1;
+	}
+      else if (n && groupnum == 1)
+	{
+	  void *h = hostaddrs[i];
+	  size_t s = sizes[i];
+
+	  /* A standalone attach clause.  */
+	  if ((kinds[i] & 0xff) == GOMP_MAP_ATTACH)
+	    gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, n,
+				 (uintptr_t) h, s, NULL);
+
+	  goacc_map_var_existing (acc_dev, h, s, n);
+	}
+      else if (n && groupnum > 1)
+	{
+	  assert (n->refcount != REFCOUNT_INFINITY
+		  && n->refcount != REFCOUNT_LINK);
+
+	  for (size_t j = i + 1; j <= group_last; j++)
+	    if ((kinds[j] & 0xff) == GOMP_MAP_ATTACH)
+	      {
+		splay_tree_key m
+		  = lookup_host (acc_dev, hostaddrs[j], sizeof (void *));
+		gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, m,
+				     (uintptr_t) hostaddrs[j], sizes[j], NULL);
+	      }
+
+	  bool processed = false;
+
+	  struct target_mem_desc *tgt = n->tgt;
+	  for (size_t j = 0; j < tgt->list_count; j++)
+	    if (tgt->list[j].key == n)
+	      {
+		/* We are processing a group of mappings (e.g.
+		   [GOMP_MAP_TO, GOMP_MAP_TO_PSET, GOMP_MAP_POINTER]).
+		   Find the right group in the target_mem_desc's variable
+		   list, and increment the refcounts for each item in that
+		   group.  */
+		for (size_t k = 0; k < groupnum; k++)
+		  if (j + k < tgt->list_count && tgt->list[j + k].key)
+		    {
+		      tgt->list[j + k].key->refcount++;
+		      tgt->list[j + k].key->dynamic_refcount++;
+		    }
+		processed = true;
+		break;
+	      }
+
+	  if (!processed)
+	    {
+	      gomp_mutex_unlock (&acc_dev->lock);
+	      gomp_fatal ("dynamic refcount incrementing failed for "
+			  "pointer/pset");
+	    }
+	}
+      else if (hostaddrs[i])
+	{
+	  /* The data is not mapped already.  Map it now, unless the first
+	     member in the group has a NULL pointer (e.g. a non-present
+	     optional parameter).  */
+	  gomp_mutex_unlock (&acc_dev->lock);
+
+	  struct target_mem_desc *tgt
+	    = gomp_map_vars_async (acc_dev, aq, groupnum, &hostaddrs[i], NULL,
+				   &sizes[i], &kinds[i], true,
+				   GOMP_MAP_VARS_ENTER_DATA);
+	  assert (tgt);
+
+	  gomp_mutex_lock (&acc_dev->lock);
+
+	  for (size_t j = 0; j < tgt->list_count; j++)
+	    {
+	      n = tgt->list[j].key;
+	      if (n)
+		n->dynamic_refcount++;
+	    }
+	}
 
       i = group_last;
     }
+
+  gomp_mutex_unlock (&acc_dev->lock);
 }
 
 /* Unmap variables for OpenACC "exit data".  */
@@ -1128,21 +1259,11 @@ goacc_exit_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum,
   for (size_t i = 0; i < mapnum; ++i)
     {
       unsigned char kind = kinds[i] & 0xff;
-      bool copyfrom = false;
-      bool finalize = false;
-
-      if (kind == GOMP_MAP_FORCE_FROM
-	  || kind == GOMP_MAP_DELETE
-	  || kind == GOMP_MAP_FORCE_DETACH)
-	finalize = true;
 
       switch (kind)
 	{
 	case GOMP_MAP_FROM:
 	case GOMP_MAP_FORCE_FROM:
-	  copyfrom = true;
-	  /* Fallthrough.  */
-
 	case GOMP_MAP_TO_PSET:
 	case GOMP_MAP_POINTER:
 	case GOMP_MAP_DELETE:
@@ -1166,54 +1287,7 @@ goacc_exit_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum,
 	    if (n == NULL)
 	      continue;
 
-	    if (finalize)
-	      {
-		if (n->refcount != REFCOUNT_INFINITY)
-		  n->refcount -= n->virtual_refcount;
-		n->virtual_refcount = 0;
-	      }
-
-	    if (n->virtual_refcount > 0)
-	      {
-		if (n->refcount != REFCOUNT_INFINITY)
-		  n->refcount--;
-		n->virtual_refcount--;
-	      }
-	    else if (n->refcount > 0 && n->refcount != REFCOUNT_INFINITY)
-	      n->refcount--;
-
-	    if (n->refcount == 0)
-	      {
-		if (copyfrom)
-		  {
-		    void *d = (void *) (n->tgt->tgt_start + n->tgt_offset
-					+ cur_node.host_start - n->host_start);
-		    gomp_copy_dev2host (acc_dev, aq,
-					(void *) cur_node.host_start, d,
-					cur_node.host_end - cur_node.host_start);
-		  }
-
-		if (aq)
-		  /* TODO We can't do the 'is_tgt_unmapped' checking -- see the
-		     'gomp_unref_tgt' comment in
-		     <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
-		     PR92881.  */
-		  gomp_remove_var_async (acc_dev, n, aq);
-		else
-		  {
-		    size_t num_mappings = 0;
-		    /* If the target_mem_desc represents a single data mapping,
-		       we can check that it is freed when this splay tree key's
-		       refcount reaches zero.  Otherwise (e.g. for a
-		       'GOMP_MAP_STRUCT' mapping with multiple members), fall
-		       back to skipping the test.  */
-		    for (size_t l_i = 0; l_i < n->tgt->list_count; ++l_i)
-		      if (n->tgt->list[l_i].key)
-			++num_mappings;
-		    bool is_tgt_unmapped = gomp_remove_var (acc_dev, n);
-		    assert (is_tgt_unmapped || num_mappings > 1);
-		  }
-	      }
+	    goacc_exit_datum_1 (acc_dev, hostaddrs[i], size, kind, n, aq);
 	  }
 	  break;