elf, nptl: Initialize static TLS directly in ld.so

The stack list is available in ld.so since commit
1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list
variables into _rtld_global"), so it's possible to walk the stack
list directly in ld.so and perform the initialization there.

This eliminates an unprotected function pointer from _rtld_global
and reduces the libpthread initialization code.

5 files changed, 46 insertions, 5 deletions

diff --git a/elf/dl-open.c b/elf/dl-open.c
index ab7aaa3..09f0df7 100644
--- a/elf/dl-open.c
+++ b/elf/dl-open.c
@@ -426,7 +426,7 @@ TLS generation counter wrapped!  Please report this."));
 	  _dl_update_slotinfo (imap->l_tls_modid);
 #endif
 
-	  GL(dl_init_static_tls) (imap);
+	  dl_init_static_tls (imap);
 	  assert (imap->l_need_tls_init == 0);
 	}
     }
diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c
index c2df26d..bb9ca1a 100644
--- a/elf/dl-reloc.c
+++ b/elf/dl-reloc.c
@@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional)
 	(void) _dl_update_slotinfo (map->l_tls_modid);
 #endif
 
-      GL(dl_init_static_tls) (map);
+      dl_init_static_tls (map);
     }
   else
     map->l_need_tls_init = 1;
@@ -141,6 +141,7 @@ cannot allocate memory in static TLS block"));
     }
 }
 
+#if !THREAD_GSCOPE_IN_TCB
 /* Initialize static TLS area and DTV for current (only) thread.
    libpthread implementations should provide their own hook
    to handle all threads.  */
@@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map)
   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 	  '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 }
-
+#endif /* !THREAD_GSCOPE_IN_TCB */
 
 void
 _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
diff --git a/elf/dl-support.c b/elf/dl-support.c
index 7fc2ee7..f966a2e 100644
--- a/elf/dl-support.c
+++ b/elf/dl-support.c
@@ -138,8 +138,6 @@ void *_dl_random;
 #include <dl-procruntime.c>
 #include <dl-procinfo.c>
 
-void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
-
 size_t _dl_pagesize = EXEC_PAGESIZE;
 
 size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ;
@@ -197,6 +195,7 @@ list_t _dl_stack_user;
 int _dl_stack_cache_lock;
 #else
 int _dl_thread_gscope_count;
+void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
 #endif
 struct dl_scope_free_list *_dl_scope_free_list;
 
diff --git a/elf/dl-tls.c b/elf/dl-tls.c
index f8b32b3..6baff0c 100644
--- a/elf/dl-tls.c
+++ b/elf/dl-tls.c
@@ -29,6 +29,10 @@
 #include <dl-tls.h>
 #include <ldsodefs.h>
 
+#if THREAD_GSCOPE_IN_TCB
+# include <list.h>
+#endif
+
 #define TUNABLE_NAMESPACE rtld
 #include <dl-tunables.h>
 
@@ -1005,3 +1009,38 @@ cannot create TLS data structures"));
       listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
     }
 }
+
+#if THREAD_GSCOPE_IN_TCB
+static inline void __attribute__((always_inline))
+init_one_static_tls (struct pthread *curp, struct link_map *map)
+{
+# if TLS_TCB_AT_TP
+  void *dest = (char *) curp - map->l_tls_offset;
+# elif TLS_DTV_AT_TP
+  void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
+# else
+#  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
+# endif
+
+  /* Initialize the memory.  */
+  memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
+	  '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
+}
+
+void
+_dl_init_static_tls (struct link_map *map)
+{
+  lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+
+  /* Iterate over the list with system-allocated threads first.  */
+  list_t *runp;
+  list_for_each (runp, &GL (dl_stack_used))
+    init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+  /* Now the list with threads using user-allocated stacks.  */
+  list_for_each (runp, &GL (dl_stack_user))
+    init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+  lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+}
+#endif /* THREAD_GSCOPE_IN_TCB */
diff --git a/elf/rtld.c b/elf/rtld.c
index 3487901..ad325d4 100644
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr,
   struct dl_main_state state;
   dl_main_state_init (&state);
 
+#if !THREAD_GSCOPE_IN_TCB
   GL(dl_init_static_tls) = &_dl_nothread_init_static_tls;
+#endif
 
 #if defined SHARED && defined _LIBC_REENTRANT \
     && defined __rtld_lock_default_lock_recursive