If dynamically-loadable ENGINEs are linked against a shared-library version

of libcrypto, then it is possible that when they are loaded they will share
the same static data as the loading application/library. This means it will
be too late to set memory/ERR/ex_data/[etc] callbacks, but entirely
unnecessary to try. This change puts a static variable in the core ENGINE
code (contained in libcrypto) and a function returning a pointer to it. If
the loaded ENGINE's return value from this function matches the loading
application/library's return value - they share static data. If they don't
match, the loaded ENGINE has its own copy of libcrypto's static data and so
the callbacks need to be set.

Also, although 0.9.7 hasn't been released yet, it's clear this will
introduce a binary incompatibility between dynamic ENGINEs built for 0.9.7
and 0.9.8 (though others probably exist already from EC_*** hooks and
what-not) - so the version control values are correspondingly bumped.

3 files changed, 32 insertions, 3 deletions

diff --git a/crypto/engine/eng_dyn.c b/crypto/engine/eng_dyn.c
index 5803c01..61ae230 100644
--- a/crypto/engine/eng_dyn.c
+++ b/crypto/engine/eng_dyn.c
@@ -500,6 +500,7 @@ static int dynamic_load(ENGINE *e, dynamic_data_ctx *ctx)
 	 * engine.h, much of this would be simplified if each area of code
 	 * provided its own "summary" structure of all related callbacks. It
 	 * would also increase opaqueness. */
+	fns.static_state = ENGINE_get_static_state();
 	fns.err_fns = ERR_get_implementation();
 	fns.ex_data_fns = CRYPTO_get_ex_data_implementation();
 	CRYPTO_get_mem_functions(&fns.mem_fns.malloc_cb,
diff --git a/crypto/engine/eng_lib.c b/crypto/engine/eng_lib.c
index a66d0f0..999061a 100644
--- a/crypto/engine/eng_lib.c
+++ b/crypto/engine/eng_lib.c
@@ -319,3 +319,13 @@ const ENGINE_CMD_DEFN *ENGINE_get_cmd_defns(const ENGINE *e)
 	{
 	return e->cmd_defns;
 	}
+
+/* eng_lib.o is pretty much linked into anything that touches ENGINE already, so
+ * put the "static_state" hack here. */
+
+static int internal_static_hack = 0;
+
+void *ENGINE_get_static_state(void)
+	{
+	return &internal_static_hack;
+	}
diff --git a/crypto/engine/engine.h b/crypto/engine/engine.h
index 0d35b01..7766374 100644
--- a/crypto/engine/engine.h
+++ b/crypto/engine/engine.h
@@ -567,17 +567,20 @@ void ENGINE_add_conf_module(void);
 /**************************/
 
 /* Binary/behaviour compatibility levels */
-#define OSSL_DYNAMIC_VERSION		(unsigned long)0x00010100
+#define OSSL_DYNAMIC_VERSION		(unsigned long)0x00010200
 /* Binary versions older than this are too old for us (whether we're a loader or
  * a loadee) */
-#define OSSL_DYNAMIC_OLDEST		(unsigned long)0x00010100
+#define OSSL_DYNAMIC_OLDEST		(unsigned long)0x00010200
 
 /* When compiling an ENGINE entirely as an external shared library, loadable by
  * the "dynamic" ENGINE, these types are needed. The 'dynamic_fns' structure
  * type provides the calling application's (or library's) error functionality
  * and memory management function pointers to the loaded library. These should
  * be used/set in the loaded library code so that the loading application's
- * 'state' will be used/changed in all operations. */
+ * 'state' will be used/changed in all operations. The 'static_state' pointer
+ * allows the loaded library to know if it shares the same static data as the
+ * calling application (or library), and thus whether these callbacks need to be
+ * set or not. */
 typedef void *(*dyn_MEM_malloc_cb)(size_t);
 typedef void *(*dyn_MEM_realloc_cb)(void *, size_t);
 typedef void (*dyn_MEM_free_cb)(void *);
@@ -605,6 +608,7 @@ typedef struct st_dynamic_LOCK_fns {
 	} dynamic_LOCK_fns;
 /* The top-level structure */
 typedef struct st_dynamic_fns {
+	void 					*static_state;
 	const ERR_FNS				*err_fns;
 	const CRYPTO_EX_DATA_IMPL		*ex_data_fns;
 	dynamic_MEM_fns				mem_fns;
@@ -645,6 +649,7 @@ typedef int (*dynamic_bind_engine)(ENGINE *e, const char *id,
 				const dynamic_fns *fns);
 #define IMPLEMENT_DYNAMIC_BIND_FN(fn) \
 	int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns) { \
+		if(ENGINE_get_static_state() == fns->static_state) goto skip_cbs; \
 		if(!CRYPTO_set_mem_functions(fns->mem_fns.malloc_cb, \
 			fns->mem_fns.realloc_cb, fns->mem_fns.free_cb)) \
 			return 0; \
@@ -656,9 +661,22 @@ typedef int (*dynamic_bind_engine)(ENGINE *e, const char *id,
 		if(!CRYPTO_set_ex_data_implementation(fns->ex_data_fns)) \
 			return 0; \
 		if(!ERR_set_implementation(fns->err_fns)) return 0; \
+	skip_cbs: \
 		if(!fn(e,id)) return 0; \
 		return 1; }
 
+/* If the loading application (or library) and the loaded ENGINE library share
+ * the same static data (eg. they're both dynamically linked to the same
+ * libcrypto.so) we need a way to avoid trying to set system callbacks - this
+ * would fail, and for the same reason that it's unnecessary to try. If the
+ * loaded ENGINE has (or gets from through the loader) its own copy of the
+ * libcrypto static data, we will need to set the callbacks. The easiest way to
+ * detect this is to have a function that returns a pointer to some static data
+ * and let the loading application and loaded ENGINE compare their respective
+ * values. */
+void *ENGINE_get_static_state(void);
+
+
 /* BEGIN ERROR CODES */
 /* The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.