1 files changed, 294 insertions, 57 deletions

diff --git a/libgomp/config/linux/affinity.c b/libgomp/config/linux/affinity.c
index dc6c7e5..789cdce 100644
--- a/libgomp/config/linux/affinity.c
+++ b/libgomp/config/linux/affinity.c
@@ -29,90 +29,327 @@
 #endif
 #include "libgomp.h"
 #include "proc.h"
+#include <errno.h>
 #include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
 #include <unistd.h>
 
 #ifdef HAVE_PTHREAD_AFFINITY_NP
 
-static unsigned int affinity_counter;
+#ifndef CPU_ALLOC_SIZE
+#define CPU_ISSET_S(idx, size, set) CPU_ISSET(idx, set)
+#define CPU_ZERO_S(size, set) CPU_ZERO(set)
+#define CPU_SET_S(idx, size, set) CPU_SET(idx, set)
+#define CPU_CLR_S(idx, size, set) CPU_CLR(idx, set)
+#endif
 
 void
 gomp_init_affinity (void)
 {
-  cpu_set_t cpuset, cpusetnew;
-  size_t idx, widx;
-  unsigned long cpus = 0;
+  if (gomp_places_list == NULL)
+    {
+      if (!gomp_affinity_init_level (1, ULONG_MAX, true))
+	return;
+    }
+
+  struct gomp_thread *thr = gomp_thread ();
+  pthread_setaffinity_np (pthread_self (), gomp_cpuset_size,
+			  (cpu_set_t *) gomp_places_list[0]);
+  thr->place = 1;
+  thr->ts.place_partition_off = 0;
+  thr->ts.place_partition_len = gomp_places_list_len;
+}
+
+void
+gomp_init_thread_affinity (pthread_attr_t *attr, unsigned int place)
+{
+  pthread_attr_setaffinity_np (attr, gomp_cpuset_size,
+			       (cpu_set_t *) gomp_places_list[place]);
+}
+
+void **
+gomp_affinity_alloc (unsigned long count, bool quiet)
+{
+  unsigned long i;
+  void **ret;
+  char *p;
+
+  if (gomp_cpusetp == NULL)
+    {
+      if (!quiet)
+	gomp_error ("Could not get CPU affinity set");
+      return NULL;
+    }
 
-  if (pthread_getaffinity_np (pthread_self (), sizeof (cpuset), &cpuset))
+  ret = malloc (count * sizeof (void *) + count * gomp_cpuset_size);
+  if (ret == NULL)
     {
-      gomp_error ("could not get CPU affinity set");
-      free (gomp_cpu_affinity);
-      gomp_cpu_affinity = NULL;
-      gomp_cpu_affinity_len = 0;
-      return;
+      if (!quiet)
+	gomp_error ("Out of memory trying to allocate places list");
+      return NULL;
     }
 
-  CPU_ZERO (&cpusetnew);
-  if (gomp_cpu_affinity_len == 0)
+  p = (char *) (ret + count);
+  for (i = 0; i < count; i++, p += gomp_cpuset_size)
+    ret[i] = p;
+  return ret;
+}
+
+void
+gomp_affinity_init_place (void *p)
+{
+  cpu_set_t *cpusetp = (cpu_set_t *) p;
+  CPU_ZERO_S (gomp_cpuset_size, cpusetp);
+}
+
+bool
+gomp_affinity_add_cpus (void *p, unsigned long num,
+			unsigned long len, long stride, bool quiet)
+{
+  cpu_set_t *cpusetp = (cpu_set_t *) p;
+  unsigned long max = 8 * gomp_cpuset_size;
+  for (;;)
     {
-      unsigned long count = gomp_cpuset_popcount (&cpuset);
-      if (count >= 65536)
-	count = 65536;
-      gomp_cpu_affinity = malloc (count * sizeof (unsigned short));
-      if (gomp_cpu_affinity == NULL)
+      if (num >= max)
+	{
+	  if (!quiet)
+	    gomp_error ("Logical CPU number %lu out of range", num);
+	  return false;
+	}
+      CPU_SET_S (num, gomp_cpuset_size, cpusetp);
+      if (--len == 0)
+	return true;
+      if ((stride < 0 && num + stride > num)
+	  || (stride > 0 && num + stride < num))
 	{
-	  gomp_error ("not enough memory to store CPU affinity list");
-	  return;
+	  if (!quiet)
+	    gomp_error ("Logical CPU number %lu+%ld out of range",
+			num, stride);
+	  return false;
 	}
-      for (widx = idx = 0; widx < count && idx < 65536; idx++)
-	if (CPU_ISSET (idx, &cpuset))
+      num += stride;
+    }
+}
+
+bool
+gomp_affinity_remove_cpu (void *p, unsigned long num)
+{
+  cpu_set_t *cpusetp = (cpu_set_t *) p;
+  if (num >= 8 * gomp_cpuset_size)
+    {
+      gomp_error ("Logical CPU number %lu out of range", num);
+      return false;
+    }
+  if (!CPU_ISSET_S (num, gomp_cpuset_size, cpusetp))
+    {
+      gomp_error ("Logical CPU %lu to be removed is not in the set", num);
+      return false;
+    }
+  CPU_CLR_S (num, gomp_cpuset_size, cpusetp);
+  return true;
+}
+
+bool
+gomp_affinity_copy_place (void *p, void *q, long stride)
+{
+  unsigned long i, max = 8 * gomp_cpuset_size;
+  cpu_set_t *destp = (cpu_set_t *) p;
+  cpu_set_t *srcp = (cpu_set_t *) q;
+
+  CPU_ZERO_S (gomp_cpuset_size, destp);
+  for (i = 0; i < max; i++)
+    if (CPU_ISSET_S (i, gomp_cpuset_size, srcp))
+      {
+	if ((stride < 0 && i + stride > i)
+	    || (stride > 0 && (i + stride < i || i + stride >= max)))
+	  {
+	    gomp_error ("Logical CPU number %lu+%ld out of range", i, stride);
+	    return false;
+	  }
+	CPU_SET_S (i + stride, gomp_cpuset_size, destp);
+      }
+  return true;
+}
+
+bool
+gomp_affinity_same_place (void *p, void *q)
+{
+#ifdef CPU_EQUAL_S
+  return CPU_EQUAL_S (gomp_cpuset_size, (cpu_set_t *) p, (cpu_set_t *) q);
+#else
+  return memcmp (p, q, gomp_cpuset_size) == 0;
+#endif
+}
+
+bool
+gomp_affinity_finalize_place_list (bool quiet)
+{
+  unsigned long i, j;
+
+  for (i = 0, j = 0; i < gomp_places_list_len; i++)
+    {
+      cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[i];
+      bool nonempty = false;
+#ifdef CPU_AND_S
+      CPU_AND_S (gomp_cpuset_size, cpusetp, cpusetp, gomp_cpusetp);
+      nonempty = gomp_cpuset_popcount (gomp_cpuset_size, cpusetp) != 0;
+#else
+      unsigned long k, max = gomp_cpuset_size / sizeof (cpusetp->__bits[0]);
+      for (k = 0; k < max; k++)
+	if ((cpusetp->__bits[k] &= gomp_cpusetp->__bits[k]) != 0)
+	  nonempty = true;
+#endif
+      if (nonempty)
+	gomp_places_list[j++] = gomp_places_list[i];
+    }
+
+  if (j == 0)
+    {
+      if (!quiet)
+	gomp_error ("None of the places contain usable logical CPUs");
+      return false;
+    }
+  else if (j < gomp_places_list_len)
+    {
+      if (!quiet)
+	gomp_error ("Number of places reduced from %ld to %ld because some "
+		    "places didn't contain any usable logical CPUs",
+		    gomp_places_list_len, j);
+      gomp_places_list_len = j;
+    }
+  return true;
+}
+
+bool
+gomp_affinity_init_level (int level, unsigned long count, bool quiet)
+{
+  unsigned long i, max = 8 * gomp_cpuset_size;
+
+  if (gomp_cpusetp)
+    {
+      unsigned long maxcount
+	= gomp_cpuset_popcount (gomp_cpuset_size, gomp_cpusetp);
+      if (count > maxcount)
+	count = maxcount;
+    }
+  gomp_places_list = gomp_affinity_alloc (count, quiet);
+  gomp_places_list_len = 0;
+  if (gomp_places_list == NULL)
+    return false;
+  /* SMT (threads).  */
+  if (level == 1)
+    {
+      for (i = 0; i < max && gomp_places_list_len < count; i++)
+	if (CPU_ISSET_S (i, gomp_cpuset_size, gomp_cpusetp))
 	  {
-	    cpus++;
-	    gomp_cpu_affinity[widx++] = idx;
+	    gomp_affinity_init_place (gomp_places_list[gomp_places_list_len]);
+	    gomp_affinity_add_cpus (gomp_places_list[gomp_places_list_len],
+				    i, 1, 0, true);
+	    ++gomp_places_list_len;
 	  }
+      return true;
     }
   else
-    for (widx = idx = 0; idx < gomp_cpu_affinity_len; idx++)
-      if (gomp_cpu_affinity[idx] < CPU_SETSIZE
-	  && CPU_ISSET (gomp_cpu_affinity[idx], &cpuset))
+    {
+      char name[sizeof ("/sys/devices/system/cpu/cpu/topology/"
+			"thread_siblings_list") + 3 * sizeof (unsigned long)];
+      size_t prefix_len = sizeof ("/sys/devices/system/cpu/cpu") - 1;
+      cpu_set_t *copy = gomp_alloca (gomp_cpuset_size);
+      FILE *f;
+      char *line = NULL;
+      size_t linelen = 0;
+
+      memcpy (name, "/sys/devices/system/cpu/cpu", prefix_len);
+      memcpy (copy, gomp_cpusetp, gomp_cpuset_size);
+      for (i = 0; i < max && gomp_places_list_len < count; i++)
+	if (CPU_ISSET_S (i, gomp_cpuset_size, copy))
+	  {
+	    sprintf (name + prefix_len, "%lu/topology/%s_siblings_list",
+		     i, level == 2 ? "thread" : "core");
+	    f = fopen (name, "r");
+	    if (f != NULL)
+	      {
+		if (getline (&line, &linelen, f) > 0)
+		  {
+		    char *p = line;
+		    bool seen_i = false;
+		    void *pl = gomp_places_list[gomp_places_list_len];
+		    gomp_affinity_init_place (pl);
+		    while (*p && *p != '\n')
+		      {
+			unsigned long first, last;
+			errno = 0;
+			first = strtoul (p, &p, 10);
+			if (errno)
+			  break;
+			last = first;
+			if (*p == '-')
+			  {
+			    errno = 0;
+			    last = strtoul (p + 1, &p, 10);
+			    if (errno || last < first)
+			      break;
+			  }
+			for (; first <= last; first++)
+			  if (CPU_ISSET_S (first, gomp_cpuset_size, copy)
+			      && gomp_affinity_add_cpus (pl, first, 1, 0,
+							 true))
+			    {
+			      CPU_CLR_S (first, gomp_cpuset_size, copy);
+			      if (first == i)
+				seen_i = true;
+			    }
+			if (*p == ',')
+			  ++p;
+		      }
+		    if (seen_i)
+		      gomp_places_list_len++;
+		  }
+		fclose (f);
+	      }
+	  }
+      if (gomp_places_list == 0)
 	{
-	  if (! CPU_ISSET (gomp_cpu_affinity[idx], &cpusetnew))
-	    {
-	      cpus++;
-	      CPU_SET (gomp_cpu_affinity[idx], &cpusetnew);
-	    }
-	  gomp_cpu_affinity[widx++] = gomp_cpu_affinity[idx];
+	  if (!quiet)
+	    gomp_error ("Error reading %s topology",
+			level == 2 ? "core" : "socket");
+	  free (gomp_places_list);
+	  gomp_places_list = NULL;
+	  return false;
 	}
-
-  if (widx == 0)
-    {
-      gomp_error ("no CPUs left for affinity setting");
-      free (gomp_cpu_affinity);
-      gomp_cpu_affinity = NULL;
-      gomp_cpu_affinity_len = 0;
-      return;
+      return true;
     }
-
-  gomp_cpu_affinity_len = widx;
-  if (cpus < gomp_available_cpus)
-    gomp_available_cpus = cpus;
-  CPU_ZERO (&cpuset);
-  CPU_SET (gomp_cpu_affinity[0], &cpuset);
-  pthread_setaffinity_np (pthread_self (), sizeof (cpuset), &cpuset);
-  affinity_counter = 1;
+  return false;
 }
 
 void
-gomp_init_thread_affinity (pthread_attr_t *attr)
+gomp_affinity_print_place (void *p)
 {
-  unsigned int cpu;
-  cpu_set_t cpuset;
-
-  cpu = __atomic_fetch_add (&affinity_counter, 1, MEMMODEL_RELAXED);
-  cpu %= gomp_cpu_affinity_len;
-  CPU_ZERO (&cpuset);
-  CPU_SET (gomp_cpu_affinity[cpu], &cpuset);
-  pthread_attr_setaffinity_np (attr, sizeof (cpu_set_t), &cpuset);
+  unsigned long i, max = 8 * gomp_cpuset_size, len;
+  cpu_set_t *cpusetp = (cpu_set_t *) p;
+  bool notfirst = false;
+
+  for (i = 0, len = 0; i < max; i++)
+    if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
+      {
+	if (len == 0)
+	  {
+	    if (notfirst)
+	      fputc (',', stderr);
+	    notfirst = true;
+	    fprintf (stderr, "%lu", i);
+	  }
+	++len;
+      }
+    else
+      {
+	if (len > 1)
+	  fprintf (stderr, ":%lu", len);
+	len = 0;
+      }
+  if (len > 1)
+    fprintf (stderr, ":%lu", len);
 }
 
 #else