Update.

	* include/link.h (struct link_map): Add l_lookup_cache element.
	* elf/dl-reloc.c (RESOLVE): Add symbol caching here.
	(RESOLVE_MAP): Likewise.
	(_dl_relocate_object): Remove cache initialization.
	* elf/dl-lookup.c: Rip out cache handling code.
	* sysdeps/generic/ldsodefs.h: Remove lookup_cache struct and variable
	declarations.

	* elf/dl-lookup.c (_dl_lookup_symbol): Reorder some conditions and
	remove some __builtin_expect.
	(_dl_lookup_versioned_symbol): Likewise.

	R_386_RELATIVE handling for ld.so startup.  Reduce RTLD_BOOTSTRAP
	case to almost no code.

2 files changed, 49 insertions, 88 deletions

diff --git a/elf/dl-lookup.c b/elf/dl-lookup.c
index 047ce4b..71dda25 100644
--- a/elf/dl-lookup.c
+++ b/elf/dl-lookup.c
@@ -60,7 +60,6 @@ struct sym_val
 
 /* Statistics function.  */
 unsigned long int _dl_num_relocations;
-unsigned long int _dl_num_cache_relocations;
 
 
 /* We have two different situations when looking up a simple: with or
@@ -185,8 +184,6 @@ _dl_do_lookup_versioned (const char *undef_name, unsigned long int hash,
 			 const struct r_found_version *const version,
 			 struct link_map *skip, int noexec, int noplt);
 
-struct lookup_cache _dl_lookup_cache;
-struct lookup_cache _dl_lookup_cache_versioned;
 
 /* Search loaded objects' symbol tables for a definition of the symbol
    UNDEF_NAME.  */
@@ -197,29 +194,15 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 		   const ElfW(Sym) **ref, struct r_scope_elem *symbol_scope[],
 		   int reloc_type, int explicit)
 {
-  unsigned long int hash;
-  struct sym_val current_value;
+  unsigned long int hash = _dl_elf_hash (undef_name);
+  struct sym_val current_value = { NULL, NULL };
   struct r_scope_elem **scope;
   int protected;
   int noexec = elf_machine_lookup_noexec_p (reloc_type);
   int noplt = elf_machine_lookup_noplt_p (reloc_type);
 
-  /* First check if we can find it in the cache.  */
-  if (__builtin_expect (*ref == _dl_lookup_cache.sym, 0)
-      && _dl_lookup_cache.map == undef_map
-      && _dl_lookup_cache.noexec == noexec
-      && _dl_lookup_cache.noplt == noplt)
-    {
-      ++_dl_num_cache_relocations;
-      *ref = _dl_lookup_cache.ret;
-      return _dl_lookup_cache.value;
-    }
-
   ++_dl_num_relocations;
 
-  hash = _dl_elf_hash (undef_name);
-  current_value = ((struct sym_val) { NULL, NULL });
-
   /* Search the relevant loaded objects for a definition.  */
   for (scope = symbol_scope; *scope; ++scope)
     if (do_lookup (undef_name, hash, *ref, &current_value, *scope, 0, NULL,
@@ -229,13 +212,12 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 	   in the global scope which was dynamically loaded.  In this case
 	   we have to prevent the latter from being unloaded unless the
 	   UNDEF_MAP object is also unloaded.  */
-	if (__builtin_expect (current_value.m->l_global, 0)
-	    && (__builtin_expect (current_value.m->l_type, lt_library)
-		== lt_loaded)
+	if (__builtin_expect (current_value.m->l_type == lt_loaded, 0)
+	    && current_value.m->l_global
 	    && undef_map != current_value.m
 	    /* Don't do this for explicit lookups as opposed to implicit
 	       runtime lookups.  */
-	    && __builtin_expect (! explicit, 1)
+	    && ! explicit
 	    /* Add UNDEF_MAP to the dependencies.  */
 	    && add_dependency (undef_map, current_value.m) < 0)
 	  /* Something went wrong.  Perhaps the object we tried to reference
@@ -246,11 +228,6 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 	break;
       }
 
-  /* Update common information in the cache.  */
-  _dl_lookup_cache.sym = *ref;
-  _dl_lookup_cache.noexec = noexec;
-  _dl_lookup_cache.noplt = noplt;
-
   if (__builtin_expect (current_value.s == NULL, 0))
     {
       const char *reference_name = undef_map ? undef_map->l_name : NULL;
@@ -262,8 +239,6 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 			       ? reference_name
 			       : (_dl_argv[0] ?: "<main program>")),
 			   make_string (undefined_msg, undef_name));
-      _dl_lookup_cache.ret = NULL;
-      _dl_lookup_cache.value = 0;
       *ref = NULL;
       return 0;
     }
@@ -284,8 +259,6 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 
   if (__builtin_expect (protected == 0, 1))
     {
-      _dl_lookup_cache.ret = current_value.s;
-      _dl_lookup_cache.value = LOOKUP_VALUE (current_value.m);
       *ref = current_value.s;
       return LOOKUP_VALUE (current_value.m);
     }
@@ -302,13 +275,9 @@ _dl_lookup_symbol (const char *undef_name, struct link_map *undef_map,
 
       if (protected_value.s == NULL || protected_value.m == undef_map)
 	{
-	  _dl_lookup_cache.ret = current_value.s;
-	  _dl_lookup_cache.value = LOOKUP_VALUE (current_value.m);
 	  *ref = current_value.s;
 	  return LOOKUP_VALUE (current_value.m);
 	}
-      _dl_lookup_cache.ret = *ref;
-      _dl_lookup_cache.value = LOOKUP_VALUE (undef_map);
 
       return LOOKUP_VALUE (undef_map);
     }
@@ -407,30 +376,15 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 			     const struct r_found_version *version,
 			     int reloc_type, int explicit)
 {
-  unsigned long int hash;
-  struct sym_val current_value;
+  unsigned long int hash = _dl_elf_hash (undef_name);
+  struct sym_val current_value = { NULL, NULL };
   struct r_scope_elem **scope;
   int protected;
   int noexec = elf_machine_lookup_noexec_p (reloc_type);
   int noplt = elf_machine_lookup_noplt_p (reloc_type);
 
-  /* First check if we can find it in the cache.  */
-  if (__builtin_expect (*ref == _dl_lookup_cache_versioned.sym, 0)
-      && _dl_lookup_cache_versioned.map == undef_map
-      && _dl_lookup_cache_versioned.noexec == noexec
-      && _dl_lookup_cache_versioned.noplt == noplt
-      && _dl_lookup_cache_versioned.version == version)
-    {
-      ++_dl_num_cache_relocations;
-      *ref = _dl_lookup_cache_versioned.ret;
-      return _dl_lookup_cache_versioned.value;
-    }
-
   ++_dl_num_relocations;
 
-  hash = _dl_elf_hash (undef_name);
-  current_value = ((struct sym_val) { NULL, NULL });
-
   /* Search the relevant loaded objects for a definition.  */
   for (scope = symbol_scope; *scope; ++scope)
     {
@@ -442,13 +396,12 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 	     in the global scope which was dynamically loaded.  In this case
 	     we have to prevent the latter from being unloaded unless the
 	     UNDEF_MAP object is also unloaded.  */
-	  if (__builtin_expect (current_value.m->l_global, 0)
-	      && (__builtin_expect (current_value.m->l_type, lt_library)
-		  == lt_loaded)
+	  if (__builtin_expect (current_value.m->l_type == lt_loaded, 0)
+	      && current_value.m->l_global
 	      && undef_map != current_value.m
 	      /* Don't do this for explicit lookups as opposed to implicit
 		 runtime lookups.  */
-	      && __builtin_expect (! explicit, 1)
+	      && ! explicit
 	      /* Add UNDEF_MAP to the dependencies.  */
 	      && add_dependency (undef_map, current_value.m) < 0)
 	    /* Something went wrong.  Perhaps the object we tried to reference
@@ -482,12 +435,6 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 	}
     }
 
-  /* Update common information in the cache.  */
-  _dl_lookup_cache_versioned.sym = *ref;
-  _dl_lookup_cache_versioned.noexec = noexec;
-  _dl_lookup_cache_versioned.noplt = noplt;
-  _dl_lookup_cache_versioned.version = version;
-
   if (__builtin_expect (current_value.s == NULL, 0))
     {
       if (*ref == NULL || ELFW(ST_BIND) ((*ref)->st_info) != STB_WEAK)
@@ -503,9 +450,6 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 					  ", version ",
 					  version->name ?: NULL));
 	}
-
-      _dl_lookup_cache_versioned.ret = NULL;
-      _dl_lookup_cache_versioned.value = 0;
       *ref = NULL;
       return 0;
     }
@@ -527,8 +471,6 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 
   if (__builtin_expect (protected == 0, 1))
     {
-      _dl_lookup_cache_versioned.ret = current_value.s;
-      _dl_lookup_cache_versioned.value = LOOKUP_VALUE (current_value.m);
       *ref = current_value.s;
       return LOOKUP_VALUE (current_value.m);
     }
@@ -545,14 +487,10 @@ _dl_lookup_versioned_symbol (const char *undef_name,
 
       if (protected_value.s == NULL || protected_value.m == undef_map)
 	{
-	  _dl_lookup_cache_versioned.ret = current_value.s;
-	  _dl_lookup_cache_versioned.value = LOOKUP_VALUE (current_value.m);
 	  *ref = current_value.s;
 	  return LOOKUP_VALUE (current_value.m);
 	}
 
-      _dl_lookup_cache_versioned.ret = *ref;
-      _dl_lookup_cache_versioned.value = LOOKUP_VALUE (undef_map);
       return LOOKUP_VALUE (undef_map);
     }
 }
diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c
index 99cb189..276aea0 100644
--- a/elf/dl-reloc.c
+++ b/elf/dl-reloc.c
@@ -27,6 +27,9 @@
 #include <sys/types.h>
 #include "dynamic-link.h"
 
+/* Statistics function.  */
+unsigned long int _dl_num_cache_relocations;
+
 
 void
 _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
@@ -74,31 +77,51 @@ cannot make segment writable for relocation"));
     /* This macro is used as a callback from the ELF_DYNAMIC_RELOCATE code.  */
 #define RESOLVE_MAP(ref, version, flags) \
     (ELFW(ST_BIND) ((*ref)->st_info) != STB_LOCAL			      \
-     ? ((version) != NULL && (version)->hash != 0			      \
-	? _dl_lookup_versioned_symbol (strtab + (*ref)->st_name, l, (ref),    \
-				       scope, (version), (flags), 0)	      \
-	: _dl_lookup_symbol (strtab + (*ref)->st_name, l, (ref), scope,	      \
-			     (flags), 0))				      \
+     ? ((__builtin_expect ((*ref) == l->l_lookup_cache.sym, 0)		      \
+	 && elf_machine_lookup_noexec_p (flags) == l->l_lookup_cache.noexec   \
+	 && elf_machine_lookup_noplt_p (flags) == l->l_lookup_cache.noplt)    \
+	? (++_dl_num_cache_relocations,					      \
+	   (*ref) = l->l_lookup_cache.ret,				      \
+	   l->l_lookup_cache.value)					      \
+	: ({ lookup_t _lr;						      \
+	     l->l_lookup_cache.sym = (*ref);				      \
+	     l->l_lookup_cache.noexec = elf_machine_lookup_noexec_p (flags);  \
+	     l->l_lookup_cache.noplt = elf_machine_lookup_noplt_p (flags);    \
+	     _lr = ((version) != NULL && (version)->hash != 0		      \
+		    ? _dl_lookup_versioned_symbol (strtab + (*ref)->st_name,  \
+						   l, (ref), scope,	      \
+						   (version), (flags), 0)     \
+		    : _dl_lookup_symbol (strtab + (*ref)->st_name, l, (ref),  \
+					 scope, (flags), 0));		      \
+	     l->l_lookup_cache.ret = (*ref);				      \
+	     l->l_lookup_cache.value = _lr; }))				      \
      : l)
 #define RESOLVE(ref, version, flags) \
     (ELFW(ST_BIND) ((*ref)->st_info) != STB_LOCAL			      \
-     ? ((version) != NULL && (version)->hash != 0			      \
-	? _dl_lookup_versioned_symbol (strtab + (*ref)->st_name, l, (ref),    \
-				       scope, (version), (flags), 0)	      \
-	: _dl_lookup_symbol (strtab + (*ref)->st_name, l, (ref), scope,	      \
-			     (flags), 0))				      \
+     ? ((__builtin_expect ((*ref) == l->l_lookup_cache.sym, 0)		      \
+	 && elf_machine_lookup_noexec_p (flags) == l->l_lookup_cache.noexec   \
+	 && elf_machine_lookup_noplt_p (flags) == l->l_lookup_cache.noplt)    \
+	? (++_dl_num_cache_relocations,					      \
+	   (*ref) = l->l_lookup_cache.ret,				      \
+	   l->l_lookup_cache.value)					      \
+	: ({ lookup_t _lr;						      \
+	     l->l_lookup_cache.sym = (*ref);				      \
+	     l->l_lookup_cache.noexec = elf_machine_lookup_noexec_p (flags);  \
+	     l->l_lookup_cache.noplt = elf_machine_lookup_noplt_p (flags);    \
+	     _lr = ((version) != NULL && (version)->hash != 0		      \
+		    ? _dl_lookup_versioned_symbol (strtab + (*ref)->st_name,  \
+						   l, (ref), scope,	      \
+						   (version), (flags), 0)     \
+		    : _dl_lookup_symbol (strtab + (*ref)->st_name, l, (ref),  \
+					 scope, (flags), 0));		      \
+	     l->l_lookup_cache.ret = (*ref);				      \
+	     l->l_lookup_cache.value = _lr; }))				      \
      : l->l_addr)
 
 #include "dynamic-link.h"
-    /* Start symbol lookup caching for this object.  */
-    _dl_lookup_cache.map = l;
-    _dl_lookup_cache_versioned.map = l;
 
     ELF_DYNAMIC_RELOCATE (l, lazy, consider_profiling);
 
-    _dl_lookup_cache.map = NULL;
-    _dl_lookup_cache_versioned.map = NULL;
-
     if (__builtin_expect (consider_profiling, 0))
       {
 	/* Allocate the array which will contain the already found