[gomp] Add thread attribute customization

libgomp/ChangeLog

	* config/posix/pool.h (gomp_adjust_thread_attr): New.
	* config/rtems/pool.h (gomp_adjust_thread_attr): Likewise.
	(gomp_thread_pool_reservoir): Add priority member.
	* confi/rtems/proc.c (allocate_thread_pool_reservoir): Add
	priority.
	(parse_thread_pools): Likewise.
	* team.c (gomp_team_start): Call configuration provided
	gomp_adjust_thread_attr(). Destroy thread attributes if
	necessary.
	* libgomp.texi: Document GOMP_RTEMS_THREAD_POOLS.

From-SVN: r227442

6 files changed, 128 insertions, 22 deletions

diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index e64155c..03ab332 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,5 +1,18 @@
 2015-09-03  Sebastian Huber  <sebastian.huber@embedded-brains.de>
 
+	* config/posix/pool.h (gomp_adjust_thread_attr): New.
+	* config/rtems/pool.h (gomp_adjust_thread_attr): Likewise.
+	(gomp_thread_pool_reservoir): Add priority member.
+	* confi/rtems/proc.c (allocate_thread_pool_reservoir): Add
+	priority.
+	(parse_thread_pools): Likewise.
+	* team.c (gomp_team_start): Call configuration provided
+	gomp_adjust_thread_attr(). Destroy thread attributes if
+	necessary.
+	* libgomp.texi: Document GOMP_RTEMS_THREAD_POOLS.
+
+2015-09-03  Sebastian Huber  <sebastian.huber@embedded-brains.de>
+
 	* config/posix/pool.h: New.
 	* config/rtems/pool.h: Likewise.
 	* config/rtems/proc.c: Likewise.
diff --git a/libgomp/config/posix/pool.h b/libgomp/config/posix/pool.h
index 6c4dc15..35ccc92 100644
--- a/libgomp/config/posix/pool.h
+++ b/libgomp/config/posix/pool.h
@@ -57,4 +57,11 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
   /* Do nothing in the default implementation.  */
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  /* Do nothing in the default implementation.  */
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/pool.h b/libgomp/config/rtems/pool.h
index 0ab68d9..8028b27 100644
--- a/libgomp/config/rtems/pool.h
+++ b/libgomp/config/rtems/pool.h
@@ -41,6 +41,7 @@ struct gomp_thread_pool_reservoir {
   gomp_sem_t available;
   gomp_mutex_t lock;
   size_t index;
+  int priority;
   struct gomp_thread_pool *pools[];
 };
 
@@ -125,4 +126,32 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
     }
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  struct gomp_thread_pool_reservoir *res = gomp_get_thread_pool_reservoir ();
+  if (res != NULL && res->priority > 0)
+    {
+      struct sched_param param;
+      int err;
+      if (attr != mutable_attr)
+	{
+	  attr = mutable_attr;
+	  pthread_attr_init (attr);
+	}
+      memset (&param, 0, sizeof (param));
+      param.sched_priority = res->priority;
+      err = pthread_attr_setschedparam (attr, &param);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler parameters failed: %s", strerror (err));
+      err = pthread_attr_setschedpolicy (attr, SCHED_FIFO);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler policy failed: %s", strerror (err));
+      err = pthread_attr_setinheritsched (attr, PTHREAD_EXPLICIT_SCHED);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set explicit scheduler failed: %s", strerror (err));
+    }
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/proc.c b/libgomp/config/rtems/proc.c
index e879a9d..0c3a79b 100644
--- a/libgomp/config/rtems/proc.c
+++ b/libgomp/config/rtems/proc.c
@@ -48,7 +48,8 @@ allocate_thread_pool_reservoirs (void)
 }
 
 static void
-allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
+allocate_thread_pool_reservoir (unsigned long count, unsigned long priority,
+				unsigned long scheduler)
 {
   struct gomp_thread_pool_reservoir *res;
   struct gomp_thread_pool *pools;
@@ -63,6 +64,7 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
   memset (pools, 0, size);
   res = (struct gomp_thread_pool_reservoir *) (pools + count);
   res->index = count;
+  res->priority = priority;
   gomp_sem_init (&res->available, count);
   gomp_mutex_init (&res->lock);
   for (i = 0; i < count; ++i)
@@ -71,7 +73,8 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
 }
 
 static char *
-parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
+parse_thread_pools (char *env, unsigned long *count, unsigned long *priority,
+		    unsigned long *scheduler)
 {
   size_t len;
   int i;
@@ -84,6 +87,17 @@ parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
   if (errno != 0)
     gomp_fatal ("Invalid thread pool count");
 
+  if (*env == '$')
+    {
+      ++env;
+      errno = 0;
+      *priority = strtoul (env, &env, 10);
+      if (errno != 0)
+	gomp_fatal ("Invalid thread pool priority");
+    }
+  else
+    *priority = -1;
+
   if (*env != '@')
     gomp_fatal ("Invalid thread pool scheduler prefix");
   ++env;
@@ -110,9 +124,10 @@ init_thread_pool_reservoirs (void)
       while (*env != '\0')
 	{
 	  unsigned long count;
+	  unsigned long priority;
 	  unsigned long scheduler;
-	  env = parse_thread_pools (env, &count, &scheduler);
-	  allocate_thread_pool_reservoir (count, scheduler);
+	  env = parse_thread_pools (env, &count, &priority, &scheduler);
+	  allocate_thread_pool_reservoir (count, priority, scheduler);
 	}
     }
 }
diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi
index 6c7f1ae..06b1c67 100644
--- a/libgomp/libgomp.texi
+++ b/libgomp/libgomp.texi
@@ -1306,23 +1306,24 @@ section 4 of the OpenMP specification in version 4.0, while those
 beginning with @env{GOMP_} are GNU extensions.
 
 @menu
-* OMP_CANCELLATION::      Set whether cancellation is activated
-* OMP_DISPLAY_ENV::       Show OpenMP version and environment variables
-* OMP_DEFAULT_DEVICE::    Set the device used in target regions
-* OMP_DYNAMIC::           Dynamic adjustment of threads
-* OMP_MAX_ACTIVE_LEVELS:: Set the maximum number of nested parallel regions
-* OMP_NESTED::            Nested parallel regions
-* OMP_NUM_THREADS::       Specifies the number of threads to use
-* OMP_PROC_BIND::         Whether theads may be moved between CPUs
-* OMP_PLACES::            Specifies on which CPUs the theads should be placed
-* OMP_STACKSIZE::         Set default thread stack size
-* OMP_SCHEDULE::          How threads are scheduled
-* OMP_THREAD_LIMIT::      Set the maximum number of threads
-* OMP_WAIT_POLICY::       How waiting threads are handled
-* GOMP_CPU_AFFINITY::     Bind threads to specific CPUs
-* GOMP_DEBUG::            Enable debugging output
-* GOMP_STACKSIZE::        Set default thread stack size
-* GOMP_SPINCOUNT::        Set the busy-wait spin count
+* OMP_CANCELLATION::        Set whether cancellation is activated
+* OMP_DISPLAY_ENV::         Show OpenMP version and environment variables
+* OMP_DEFAULT_DEVICE::      Set the device used in target regions
+* OMP_DYNAMIC::             Dynamic adjustment of threads
+* OMP_MAX_ACTIVE_LEVELS::   Set the maximum number of nested parallel regions
+* OMP_NESTED::              Nested parallel regions
+* OMP_NUM_THREADS::         Specifies the number of threads to use
+* OMP_PROC_BIND::           Whether theads may be moved between CPUs
+* OMP_PLACES::              Specifies on which CPUs the theads should be placed
+* OMP_STACKSIZE::           Set default thread stack size
+* OMP_SCHEDULE::            How threads are scheduled
+* OMP_THREAD_LIMIT::        Set the maximum number of threads
+* OMP_WAIT_POLICY::         How waiting threads are handled
+* GOMP_CPU_AFFINITY::       Bind threads to specific CPUs
+* GOMP_DEBUG::              Enable debugging output
+* GOMP_STACKSIZE::          Set default thread stack size
+* GOMP_SPINCOUNT::          Set the busy-wait spin count
+* GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools
 @end menu
 
 
@@ -1705,6 +1706,46 @@ or @env{OMP_WAIT_POLICY} is @code{PASSIVE}.
 
 
 
+@node GOMP_RTEMS_THREAD_POOLS
+@section @env{GOMP_RTEMS_THREAD_POOLS} -- Set the RTEMS specific thread pools
+@cindex Environment Variable
+@cindex Implementation specific setting
+@table @asis
+@item @emph{Description}:
+This environment variable is only used on the RTEMS real-time operating system.
+It determines the scheduler instance specific thread pools.  The format for
+@env{GOMP_RTEMS_THREAD_POOLS} is a list of optional
+@code{<thread-pool-count>[$<priority>]@@<scheduler-name>} configurations
+separated by @code{:} where:
+@itemize @bullet
+@item @code{<thread-pool-count>} is the thread pool count for this scheduler
+instance.
+@item @code{$<priority>} is an optional priority for the worker threads of a
+thread pool according to @code{pthread_setschedparam}.  In case a priority
+value is omitted, then a worker thread will inherit the priority of the OpenMP
+master thread that created it.  The priority of the worker thread is not
+changed after creation, even if a new OpenMP master thread using the worker has
+a different priority.
+@item @code{@@<scheduler-name>} is the scheduler instance name according to the
+RTEMS application configuration.
+@end itemize
+In case no thread pool configuration is specified for a scheduler instance,
+then each OpenMP master thread of this scheduler instance will use its own
+dynamically allocated thread pool.  To limit the worker thread count of the
+thread pools, each OpenMP master thread must call @code{omp_set_num_threads}.
+@item @emph{Example}:
+Lets suppose we have three scheduler instances @code{IO}, @code{WRK0}, and
+@code{WRK1} with @env{GOMP_RTEMS_THREAD_POOLS} set to
+@code{"1@@WRK0:3$4@@WRK1"}.  Then there are no thread pool restrictions for
+scheduler instance @code{IO}.  In the scheduler instance @code{WRK0} there is
+one thread pool available.  Since no priority is specified for this scheduler
+instance, the worker thread inherits the priority of the OpenMP master thread
+that created it.  In the scheduler instance @code{WRK1} there are three thread
+pools available and their worker threads run at priority four.
+@end table
+
+
+
 @c ---------------------------------------------------------------------
 @c The libgomp ABI
 @c ---------------------------------------------------------------------
diff --git a/libgomp/team.c b/libgomp/team.c
index 274f3ed..67e25b3 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -799,12 +799,13 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
       start_data->thread_pool = pool;
       start_data->nested = nested;
 
+      attr = gomp_adjust_thread_attr (attr, &thread_attr);
       err = pthread_create (&pt, attr, gomp_thread_start, start_data++);
       if (err != 0)
 	gomp_fatal ("Thread creation failed: %s", strerror (err));
     }
 
-  if (__builtin_expect (gomp_places_list != NULL, 0))
+  if (__builtin_expect (attr == &thread_attr, 0))
     pthread_attr_destroy (&thread_attr);
 
  do_release: