libsanitizer merge from upstream r175733

From-SVN: r196201

9 files changed, 412 insertions, 189 deletions

diff --git a/libsanitizer/asan/asan_flags.h b/libsanitizer/asan/asan_flags.h
index b05fdc3..b880896 100644
--- a/libsanitizer/asan/asan_flags.h
+++ b/libsanitizer/asan/asan_flags.h
@@ -15,11 +15,13 @@
 
 #include "sanitizer_common/sanitizer_internal_defs.h"
 
-// ASan flag values can be defined in three ways:
+// ASan flag values can be defined in four ways:
 // 1) initialized with default values at startup.
-// 2) overriden from string returned by user-specified function
+// 2) overriden during compilation of ASan runtime by providing
+//    compile definition ASAN_DEFAULT_OPTIONS.
+// 3) overriden from string returned by user-specified function
 //    __asan_default_options().
-// 3) overriden from env variable ASAN_OPTIONS.
+// 4) overriden from env variable ASAN_OPTIONS.
 
 namespace __asan {
 
diff --git a/libsanitizer/asan/asan_intercepted_functions.h b/libsanitizer/asan/asan_intercepted_functions.h
index 45c913c..ed75c42 100644
--- a/libsanitizer/asan/asan_intercepted_functions.h
+++ b/libsanitizer/asan/asan_intercepted_functions.h
@@ -77,9 +77,36 @@ using __sanitizer::uptr;
 # define ASAN_INTERCEPT___CXA_THROW 0
 #endif
 
+#define INTERPOSE_FUNCTION(function) \
+    { reinterpret_cast<const uptr>(WRAP(function)), \
+      reinterpret_cast<const uptr>(function) }
+
+#define INTERPOSE_FUNCTION_2(function, wrapper) \
+    { reinterpret_cast<const uptr>(wrapper), \
+      reinterpret_cast<const uptr>(function) }
+
+struct interpose_substitution {
+  const uptr replacement;
+  const uptr original;
+};
+
+#define INTERPOSER(func) __attribute__((used)) \
+const interpose_substitution substitution_##func[] \
+    __attribute__((section("__DATA, __interpose"))) = { \
+  INTERPOSE_FUNCTION(func), \
+}
+
+#define INTERPOSER_2(func, wrapper) __attribute__((used)) \
+const interpose_substitution substitution_##func[] \
+    __attribute__((section("__DATA, __interpose"))) = { \
+  INTERPOSE_FUNCTION_2(func, wrapper), \
+}
+
+
 #define DECLARE_FUNCTION_AND_WRAPPER(ret_type, func, ...) \
   ret_type func(__VA_ARGS__); \
-  ret_type WRAP(func)(__VA_ARGS__)
+  ret_type WRAP(func)(__VA_ARGS__); \
+  INTERPOSER(func)
 
 // Use extern declarations of intercepted functions on Mac and Windows
 // to avoid including system headers.
@@ -139,7 +166,8 @@ DECLARE_FUNCTION_AND_WRAPPER(char*, strdup, const char *s);
 DECLARE_FUNCTION_AND_WRAPPER(uptr, strnlen, const char *s, uptr maxlen);
 # endif
 # if ASAN_INTERCEPT_INDEX
-DECLARE_FUNCTION_AND_WRAPPER(char*, index, const char *string, int c);
+char* index(const char *string, int c);
+INTERPOSER_2(index, WRAP(strchr));
 # endif
 
 // stdlib.h
@@ -193,6 +221,20 @@ DECLARE_FUNCTION_AND_WRAPPER(int, pthread_create,
                              void *(*start_routine)(void*), void *arg);
 # endif
 
+# if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS
+DECLARE_FUNCTION_AND_WRAPPER(void *, localtime, unsigned long *timep);
+DECLARE_FUNCTION_AND_WRAPPER(void *, localtime_r, unsigned long *timep,
+                             void *result);
+DECLARE_FUNCTION_AND_WRAPPER(void *, gmtime, unsigned long *timep);
+DECLARE_FUNCTION_AND_WRAPPER(void *, gmtime_r, unsigned long *timep,
+                             void *result);
+DECLARE_FUNCTION_AND_WRAPPER(char *, ctime, unsigned long *timep);
+DECLARE_FUNCTION_AND_WRAPPER(char *, ctime_r, unsigned long *timep,
+                             char *result);
+DECLARE_FUNCTION_AND_WRAPPER(char *, asctime, void *tm);
+DECLARE_FUNCTION_AND_WRAPPER(char *, asctime_r, void *tm, char *result);
+# endif
+
 // stdio.h
 # if SANITIZER_INTERCEPT_SCANF
 DECLARE_FUNCTION_AND_WRAPPER(int, vscanf, const char *format, va_list ap);
@@ -205,17 +247,6 @@ DECLARE_FUNCTION_AND_WRAPPER(int, fscanf,
                              void* stream, const char *format, ...);
 DECLARE_FUNCTION_AND_WRAPPER(int, sscanf,  // NOLINT
                              const char *str, const char *format, ...);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_vscanf, const char *format,
-                             va_list ap);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_vsscanf, const char *str,
-                             const char *format, va_list ap);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_vfscanf, void *stream,
-                             const char *format, va_list ap);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_scanf, const char *format, ...);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_fscanf,
-                             void* stream, const char *format, ...);
-DECLARE_FUNCTION_AND_WRAPPER(int, __isoc99_sscanf,  // NOLINT
-                             const char *str, const char *format, ...);
 # endif
 
 # if defined(__APPLE__)
@@ -278,9 +309,11 @@ DECLARE_FUNCTION_AND_WRAPPER(void *, valloc, size_t size);
 DECLARE_FUNCTION_AND_WRAPPER(size_t, malloc_good_size, size_t size);
 DECLARE_FUNCTION_AND_WRAPPER(int, posix_memalign,
                              void **memptr, size_t alignment, size_t size);
+#if 0
 DECLARE_FUNCTION_AND_WRAPPER(void, _malloc_fork_prepare, void);
 DECLARE_FUNCTION_AND_WRAPPER(void, _malloc_fork_parent, void);
 DECLARE_FUNCTION_AND_WRAPPER(void, _malloc_fork_child, void);
+#endif
 
 
 
diff --git a/libsanitizer/asan/asan_interceptors.cc b/libsanitizer/asan/asan_interceptors.cc
index f4c5683..064fc62 100644
--- a/libsanitizer/asan/asan_interceptors.cc
+++ b/libsanitizer/asan/asan_interceptors.cc
@@ -24,6 +24,16 @@
 
 namespace __asan {
 
+// Return true if we can quickly decide that the region is unpoisoned.
+static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
+  if (size == 0) return true;
+  if (size <= 32)
+    return !AddressIsPoisoned(beg) &&
+           !AddressIsPoisoned(beg + size - 1) &&
+           !AddressIsPoisoned(beg + size / 2);
+  return false;
+}
+
 // We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE,
 // and ASAN_WRITE_RANGE as macro instead of function so
 // that no extra frames are created, and stack trace contains
@@ -32,7 +42,8 @@ namespace __asan {
 #define ACCESS_MEMORY_RANGE(offset, size, isWrite) do {                 \
     uptr __offset = (uptr)(offset);                                     \
     uptr __size = (uptr)(size);                                         \
-    if (__asan_region_is_poisoned(__offset, __size)) {                  \
+    if (!QuickCheckForUnpoisonedRegion(__offset, __size) &&             \
+        __asan_region_is_poisoned(__offset, __size)) {                  \
       GET_CURRENT_PC_BP_SP;                                             \
       __asan_report_error(pc, bp, sp, __offset, isWrite, __size);       \
     }                                                                   \
diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h
index 0fe620e..1ccbf10 100644
--- a/libsanitizer/asan/asan_internal.h
+++ b/libsanitizer/asan/asan_internal.h
@@ -52,7 +52,7 @@
 
 #define ASAN_POSIX (ASAN_LINUX || ASAN_MAC)
 
-#if __has_feature(address_sanitizer)
+#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
 # error "The AddressSanitizer run-time should not be"
         " instrumented by AddressSanitizer"
 #endif
@@ -89,6 +89,10 @@
 # endif
 #endif
 
+#ifndef ASAN_USE_PREINIT_ARRAY
+# define ASAN_USE_PREINIT_ARRAY (ASAN_LINUX && !ASAN_ANDROID)
+#endif
+
 // All internal functions in asan reside inside the __asan namespace
 // to avoid namespace collisions with the user programs.
 // Seperate namespace also makes it simpler to distinguish the asan run-time
diff --git a/libsanitizer/asan/asan_mac.cc b/libsanitizer/asan/asan_mac.cc
index c5fc7de..dd2657d 100644
--- a/libsanitizer/asan/asan_mac.cc
+++ b/libsanitizer/asan/asan_mac.cc
@@ -86,6 +86,39 @@ extern "C"
 void __asan_init();
 
 static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
+LowLevelAllocator allocator_for_env;
+
+// Change the value of the env var |name|, leaking the original value.
+// If |name_value| is NULL, the variable is deleted from the environment,
+// otherwise the corresponding "NAME=value" string is replaced with
+// |name_value|.
+void LeakyResetEnv(const char *name, const char *name_value) {
+  char ***env_ptr = _NSGetEnviron();
+  CHECK(env_ptr);
+  char **environ = *env_ptr;
+  CHECK(environ);
+  uptr name_len = internal_strlen(name);
+  while (*environ != 0) {
+    uptr len = internal_strlen(*environ);
+    if (len > name_len) {
+      const char *p = *environ;
+      if (!internal_memcmp(p, name, name_len) && p[name_len] == '=') {
+        // Match.
+        if (name_value) {
+          // Replace the old value with the new one.
+          *environ = const_cast<char*>(name_value);
+        } else {
+          // Shift the subsequent pointers back.
+          char **del = environ;
+          do {
+            del[0] = del[1];
+          } while (*del++);
+        }
+      }
+    }
+    environ++;
+  }
+}
 
 void MaybeReexec() {
   if (!flags()->allow_reexec) return;
@@ -94,7 +127,11 @@ void MaybeReexec() {
   // ourselves.
   Dl_info info;
   CHECK(dladdr((void*)((uptr)__asan_init), &info));
-  const char *dyld_insert_libraries = GetEnv(kDyldInsertLibraries);
+  char *dyld_insert_libraries =
+      const_cast<char*>(GetEnv(kDyldInsertLibraries));
+  uptr old_env_len = dyld_insert_libraries ?
+      internal_strlen(dyld_insert_libraries) : 0;
+  uptr fname_len = internal_strlen(info.dli_fname);
   if (!dyld_insert_libraries ||
       !REAL(strstr)(dyld_insert_libraries, info.dli_fname)) {
     // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
@@ -102,16 +139,79 @@ void MaybeReexec() {
     char program_name[1024];
     uint32_t buf_size = sizeof(program_name);
     _NSGetExecutablePath(program_name, &buf_size);
-    // Ok to use setenv() since the wrappers don't depend on the value of
-    // asan_inited.
-    setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0);
+    char *new_env = const_cast<char*>(info.dli_fname);
+    if (dyld_insert_libraries) {
+      // Append the runtime dylib name to the existing value of
+      // DYLD_INSERT_LIBRARIES.
+      new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2);
+      internal_strncpy(new_env, dyld_insert_libraries, old_env_len);
+      new_env[old_env_len] = ':';
+      // Copy fname_len and add a trailing zero.
+      internal_strncpy(new_env + old_env_len + 1, info.dli_fname,
+                       fname_len + 1);
+      // Ok to use setenv() since the wrappers don't depend on the value of
+      // asan_inited.
+      setenv(kDyldInsertLibraries, new_env, /*overwrite*/1);
+    } else {
+      // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name.
+      setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0);
+    }
     if (flags()->verbosity >= 1) {
       Report("exec()-ing the program with\n");
-      Report("%s=%s\n", kDyldInsertLibraries, info.dli_fname);
+      Report("%s=%s\n", kDyldInsertLibraries, new_env);
       Report("to enable ASan wrappers.\n");
       Report("Set ASAN_OPTIONS=allow_reexec=0 to disable this.\n");
     }
     execv(program_name, *_NSGetArgv());
+  } else {
+    // DYLD_INSERT_LIBRARIES is set and contains the runtime library.
+    if (old_env_len == fname_len) {
+      // It's just the runtime library name - fine to unset the variable.
+      LeakyResetEnv(kDyldInsertLibraries, NULL);
+    } else {
+      uptr env_name_len = internal_strlen(kDyldInsertLibraries);
+      // Allocate memory to hold the previous env var name, its value, the '='
+      // sign and the '\0' char.
+      char *new_env = (char*)allocator_for_env.Allocate(
+          old_env_len + 2 + env_name_len);
+      CHECK(new_env);
+      internal_memset(new_env, '\0', old_env_len + 2 + env_name_len);
+      internal_strncpy(new_env, kDyldInsertLibraries, env_name_len);
+      new_env[env_name_len] = '=';
+      char *new_env_pos = new_env + env_name_len + 1;
+
+      // Iterate over colon-separated pieces of |dyld_insert_libraries|.
+      char *piece_start = dyld_insert_libraries;
+      char *piece_end = NULL;
+      char *old_env_end = dyld_insert_libraries + old_env_len;
+      do {
+        if (piece_start[0] == ':') piece_start++;
+        piece_end =  REAL(strchr)(piece_start, ':');
+        if (!piece_end) piece_end = dyld_insert_libraries + old_env_len;
+        if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break;
+        uptr piece_len = piece_end - piece_start;
+
+        // If the current piece isn't the runtime library name,
+        // append it to new_env.
+        if ((piece_len != fname_len) ||
+            (internal_strncmp(piece_start, info.dli_fname, fname_len) != 0)) {
+          if (new_env_pos != new_env + env_name_len + 1) {
+            new_env_pos[0] = ':';
+            new_env_pos++;
+          }
+          internal_strncpy(new_env_pos, piece_start, piece_len);
+        }
+        // Move on to the next piece.
+        new_env_pos += piece_len;
+        piece_start = piece_end;
+      } while (piece_start < old_env_end);
+
+      // Can't use setenv() here, because it requires the allocator to be
+      // initialized.
+      // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in
+      // a separate function called after InitializeAllocator().
+      LeakyResetEnv(kDyldInsertLibraries, new_env);
+    }
   }
 }
 
diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h
index 48b2454..df95236 100644
--- a/libsanitizer/asan/asan_mapping.h
+++ b/libsanitizer/asan/asan_mapping.h
@@ -1,7 +1,5 @@
 //===-- asan_mapping.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
@@ -18,6 +16,37 @@
 
 // The full explanation of the memory mapping could be found here:
 // http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm
+//
+// Typical shadow mapping on Linux/x86_64 with SHADOW_OFFSET == 0x00007fff8000:
+// || `[0x10007fff8000, 0x7fffffffffff]` || HighMem    ||
+// || `[0x02008fff7000, 0x10007fff7fff]` || HighShadow ||
+// || `[0x00008fff7000, 0x02008fff6fff]` || ShadowGap  ||
+// || `[0x00007fff8000, 0x00008fff6fff]` || LowShadow  ||
+// || `[0x000000000000, 0x00007fff7fff]` || LowMem     ||
+//
+// When SHADOW_OFFSET is zero (-pie):
+// || `[0x100000000000, 0x7fffffffffff]` || HighMem    ||
+// || `[0x020000000000, 0x0fffffffffff]` || HighShadow ||
+// || `[0x000000040000, 0x01ffffffffff]` || ShadowGap  ||
+//
+// Special case when something is already mapped between
+// 0x003000000000 and 0x004000000000 (e.g. when prelink is installed):
+// || `[0x10007fff8000, 0x7fffffffffff]` || HighMem    ||
+// || `[0x02008fff7000, 0x10007fff7fff]` || HighShadow ||
+// || `[0x004000000000, 0x02008fff6fff]` || ShadowGap3 ||
+// || `[0x003000000000, 0x003fffffffff]` || MidMem     ||
+// || `[0x00087fff8000, 0x002fffffffff]` || ShadowGap2 ||
+// || `[0x00067fff8000, 0x00087fff7fff]` || MidShadow  ||
+// || `[0x00008fff7000, 0x00067fff7fff]` || ShadowGap  ||
+// || `[0x00007fff8000, 0x00008fff6fff]` || LowShadow  ||
+// || `[0x000000000000, 0x00007fff7fff]` || LowMem     ||
+//
+// Default Linux/i386 mapping:
+// || `[0x40000000, 0xffffffff]` || HighMem    ||
+// || `[0x28000000, 0x3fffffff]` || HighShadow ||
+// || `[0x24000000, 0x27ffffff]` || ShadowGap  ||
+// || `[0x20000000, 0x23ffffff]` || LowShadow  ||
+// || `[0x00000000, 0x1fffffff]` || LowMem     ||
 
 #if ASAN_FLEXIBLE_MAPPING_AND_OFFSET == 1
 extern SANITIZER_INTERFACE_ATTRIBUTE uptr __asan_mapping_scale;
@@ -36,7 +65,11 @@ extern SANITIZER_INTERFACE_ATTRIBUTE uptr __asan_mapping_offset;
 #   if defined(__powerpc64__)
 #    define SHADOW_OFFSET (1ULL << 41)
 #   else
-#    define SHADOW_OFFSET (1ULL << 44)
+#    if ASAN_MAC
+#     define SHADOW_OFFSET (1ULL << 44)
+#    else
+#     define SHADOW_OFFSET 0x7fff8000ULL
+#    endif
 #   endif
 #  endif
 # endif
@@ -57,49 +90,105 @@ extern SANITIZER_INTERFACE_ATTRIBUTE uptr __asan_mapping_offset;
 #define kHighShadowBeg  MEM_TO_SHADOW(kHighMemBeg)
 #define kHighShadowEnd  MEM_TO_SHADOW(kHighMemEnd)
 
+# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg)
+# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd)
+
 // With the zero shadow base we can not actually map pages starting from 0.
 // This constant is somewhat arbitrary.
 #define kZeroBaseShadowStart (1 << 18)
 
 #define kShadowGapBeg   (kLowShadowEnd ? kLowShadowEnd + 1 \
                                        : kZeroBaseShadowStart)
-#define kShadowGapEnd   (kHighShadowBeg - 1)
+#define kShadowGapEnd   ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1)
+
+#define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0)
+#define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0)
+
+#define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
+#define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
+
+#define DO_ASAN_MAPPING_PROFILE 0  // Set to 1 to profile the functions below.
+
+#if DO_ASAN_MAPPING_PROFILE
+# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
+#else
+# define PROFILE_ASAN_MAPPING()
+#endif
+
+// If 1, all shadow boundaries are constants.
+// Don't set to 1 other than for testing.
+#define ASAN_FIXED_MAPPING 0
 
 namespace __asan {
 
+extern uptr AsanMappingProfile[];
+
+#if ASAN_FIXED_MAPPING
+// Fixed mapping for 64-bit Linux. Mostly used for performance comparison
+// with non-fixed mapping. As of r175253 (Feb 2013) the performance
+// difference between fixed and non-fixed mapping is below the noise level.
+static uptr kHighMemEnd = 0x7fffffffffffULL;
+static uptr kMidMemBeg =    0x3000000000ULL;
+static uptr kMidMemEnd =    0x3fffffffffULL;
+#else
 SANITIZER_INTERFACE_ATTRIBUTE
-extern uptr kHighMemEnd;  // Initialized in __asan_init.
+extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;  // Initialized in __asan_init.
+#endif
 
 static inline bool AddrIsInLowMem(uptr a) {
+  PROFILE_ASAN_MAPPING();
   return a < kLowMemEnd;
 }
 
 static inline bool AddrIsInLowShadow(uptr a) {
+  PROFILE_ASAN_MAPPING();
   return a >= kLowShadowBeg && a <= kLowShadowEnd;
 }
 
 static inline bool AddrIsInHighMem(uptr a) {
+  PROFILE_ASAN_MAPPING();
   return a >= kHighMemBeg && a <= kHighMemEnd;
 }
 
+static inline bool AddrIsInMidMem(uptr a) {
+  PROFILE_ASAN_MAPPING();
+  return kMidMemBeg && a >= kMidMemBeg && a <= kMidMemEnd;
+}
+
 static inline bool AddrIsInMem(uptr a) {
-  return AddrIsInLowMem(a) || AddrIsInHighMem(a);
+  PROFILE_ASAN_MAPPING();
+  return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a);
 }
 
 static inline uptr MemToShadow(uptr p) {
+  PROFILE_ASAN_MAPPING();
   CHECK(AddrIsInMem(p));
   return MEM_TO_SHADOW(p);
 }
 
 static inline bool AddrIsInHighShadow(uptr a) {
-  return a >= kHighShadowBeg && a <=  kHighMemEnd;
+  PROFILE_ASAN_MAPPING();
+  return a >= kHighShadowBeg && a <= kHighMemEnd;
+}
+
+static inline bool AddrIsInMidShadow(uptr a) {
+  PROFILE_ASAN_MAPPING();
+  return kMidMemBeg && a >= kMidShadowBeg && a <= kMidMemEnd;
 }
 
 static inline bool AddrIsInShadow(uptr a) {
-  return AddrIsInLowShadow(a) || AddrIsInHighShadow(a);
+  PROFILE_ASAN_MAPPING();
+  return AddrIsInLowShadow(a) || AddrIsInMidShadow(a) || AddrIsInHighShadow(a);
 }
 
 static inline bool AddrIsInShadowGap(uptr a) {
+  PROFILE_ASAN_MAPPING();
+  if (kMidMemBeg) {
+    if (a <= kShadowGapEnd)
+      return SHADOW_OFFSET == 0 || a >= kShadowGapBeg;
+    return (a >= kShadowGap2Beg && a <= kShadowGap2End) ||
+           (a >= kShadowGap3Beg && a <= kShadowGap3End);
+  }
   // In zero-based shadow mode we treat addresses near zero as addresses
   // in shadow gap as well.
   if (SHADOW_OFFSET == 0)
@@ -108,12 +197,14 @@ static inline bool AddrIsInShadowGap(uptr a) {
 }
 
 static inline bool AddrIsAlignedByGranularity(uptr a) {
+  PROFILE_ASAN_MAPPING();
   return (a & (SHADOW_GRANULARITY - 1)) == 0;
 }
 
 static inline bool AddressIsPoisoned(uptr a) {
+  PROFILE_ASAN_MAPPING();
   const uptr kAccessSize = 1;
-  u8 *shadow_address = (u8*)MemToShadow(a);
+  u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
   s8 shadow_value = *shadow_address;
   if (shadow_value) {
     u8 last_accessed_byte = (a & (SHADOW_GRANULARITY - 1))
@@ -123,6 +214,9 @@ static inline bool AddressIsPoisoned(uptr a) {
   return false;
 }
 
+// Must be after all calls to PROFILE_ASAN_MAPPING().
+static const uptr kAsanMappingProfileSize = __LINE__;
+
 }  // namespace __asan
 
 #endif  // ASAN_MAPPING_H
diff --git a/libsanitizer/asan/asan_report.cc b/libsanitizer/asan/asan_report.cc
index 663e8f3..13e94c4 100644
--- a/libsanitizer/asan/asan_report.cc
+++ b/libsanitizer/asan/asan_report.cc
@@ -433,9 +433,9 @@ class ScopedInErrorReport {
         // an error report will finish doing it.
         SleepForSeconds(Max(100, flags()->sleep_before_dying + 1));
       }
-      // If we're still not dead for some reason, use raw Exit() instead of
+      // If we're still not dead for some reason, use raw _exit() instead of
       // Die() to bypass any additional checks.
-      Exit(flags()->exitcode);
+      internal__exit(flags()->exitcode);
     }
     ASAN_ON_ERROR();
     reporting_thread_tid = asanThreadRegistry().GetCurrentTidOrInvalid();
diff --git a/libsanitizer/asan/asan_rtl.cc b/libsanitizer/asan/asan_rtl.cc
index 175d377..e22fcd3 100644
--- a/libsanitizer/asan/asan_rtl.cc
+++ b/libsanitizer/asan/asan_rtl.cc
@@ -25,6 +25,8 @@
 
 namespace __asan {
 
+uptr AsanMappingProfile[kAsanMappingProfileSize];
+
 static void AsanDie() {
   static atomic_uint32_t num_calls;
   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
@@ -35,13 +37,19 @@ static void AsanDie() {
     Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
     SleepForSeconds(flags()->sleep_before_dying);
   }
-  if (flags()->unmap_shadow_on_exit)
-    UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+  if (flags()->unmap_shadow_on_exit) {
+    if (kMidMemBeg) {
+      UnmapOrDie((void*)kLowShadowBeg, kMidMemBeg - kLowShadowBeg);
+      UnmapOrDie((void*)kMidMemEnd, kHighShadowEnd - kMidMemEnd);
+    } else {
+      UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+    }
+  }
   if (death_callback)
     death_callback();
   if (flags()->abort_on_error)
     Abort();
-  Exit(flags()->exitcode);
+  internal__exit(flags()->exitcode);
 }
 
 static void AsanCheckFailed(const char *file, int line, const char *cond,
@@ -66,6 +74,17 @@ static const char *MaybeCallAsanDefaultOptions() {
   return (&__asan_default_options) ? __asan_default_options() : "";
 }
 
+static const char *MaybeUseAsanDefaultOptionsCompileDefiniton() {
+#ifdef ASAN_DEFAULT_OPTIONS
+// Stringize the macro value.
+# define ASAN_STRINGIZE(x) #x
+# define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options)
+  return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS);
+#else
+  return "";
+#endif
+}
+
 static void ParseFlagsFromString(Flags *f, const char *str) {
   ParseFlag(str, &f->quarantine_size, "quarantine_size");
   ParseFlag(str, &f->symbolize, "symbolize");
@@ -146,6 +165,9 @@ void InitializeFlags(Flags *f, const char *env) {
   f->alloc_dealloc_mismatch = true;
   f->use_stack_depot = true;  // Only affects allocator2.
 
+  // Override from compile definition.
+  ParseFlagsFromString(f, MaybeUseAsanDefaultOptionsCompileDefiniton());
+
   // Override from user-specified string.
   ParseFlagsFromString(f, MaybeCallAsanDefaultOptions());
   if (flags()->verbosity) {
@@ -161,7 +183,10 @@ void InitializeFlags(Flags *f, const char *env) {
 int asan_inited;
 bool asan_init_is_running;
 void (*death_callback)(void);
-uptr kHighMemEnd;
+
+#if !ASAN_FIXED_MAPPING
+uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;
+#endif
 
 // -------------------------- Misc ---------------- {{{1
 void ShowStatsAndAbort() {
@@ -209,6 +234,17 @@ ASAN_REPORT_ERROR(store, true, 4)
 ASAN_REPORT_ERROR(store, true, 8)
 ASAN_REPORT_ERROR(store, true, 16)
 
+#define ASAN_REPORT_ERROR_N(type, is_write)                    \
+extern "C" NOINLINE INTERFACE_ATTRIBUTE                        \
+void __asan_report_ ## type ## _n(uptr addr, uptr size);       \
+void __asan_report_ ## type ## _n(uptr addr, uptr size) {      \
+  GET_CALLER_PC_BP_SP;                                         \
+  __asan_report_error(pc, bp, sp, addr, is_write, size);       \
+}
+
+ASAN_REPORT_ERROR_N(load, false)
+ASAN_REPORT_ERROR_N(store, true)
+
 // Force the linker to keep the symbols for various ASan interface functions.
 // We want to keep those in the executable in order to let the instrumented
 // dynamic libraries access the symbol even if it is not used by the executable
@@ -259,9 +295,15 @@ static NOINLINE void force_interface_symbols() {
 static void asan_atexit() {
   Printf("AddressSanitizer exit stats:\n");
   __asan_print_accumulated_stats();
+  // Print AsanMappingProfile.
+  for (uptr i = 0; i < kAsanMappingProfileSize; i++) {
+    if (AsanMappingProfile[i] == 0) continue;
+    Printf("asan_mapping.h:%zd -- %zd\n", i, AsanMappingProfile[i]);
+  }
 }
 
 static void InitializeHighMemEnd() {
+#if !ASAN_FIXED_MAPPING
 #if SANITIZER_WORDSIZE == 64
 # if defined(__powerpc64__)
   // FIXME:
@@ -277,6 +319,58 @@ static void InitializeHighMemEnd() {
 #else  // SANITIZER_WORDSIZE == 32
   kHighMemEnd = (1ULL << 32) - 1;  // 0xffffffff;
 #endif  // SANITIZER_WORDSIZE
+#endif  // !ASAN_FIXED_MAPPING
+}
+
+static void ProtectGap(uptr a, uptr size) {
+  CHECK_EQ(a, (uptr)Mprotect(a, size));
+}
+
+static void PrintAddressSpaceLayout() {
+  Printf("|| `[%p, %p]` || HighMem    ||\n",
+         (void*)kHighMemBeg, (void*)kHighMemEnd);
+  Printf("|| `[%p, %p]` || HighShadow ||\n",
+         (void*)kHighShadowBeg, (void*)kHighShadowEnd);
+  if (kMidMemBeg) {
+    Printf("|| `[%p, %p]` || ShadowGap3 ||\n",
+           (void*)kShadowGap3Beg, (void*)kShadowGap3End);
+    Printf("|| `[%p, %p]` || MidMem     ||\n",
+           (void*)kMidMemBeg, (void*)kMidMemEnd);
+    Printf("|| `[%p, %p]` || ShadowGap2 ||\n",
+           (void*)kShadowGap2Beg, (void*)kShadowGap2End);
+    Printf("|| `[%p, %p]` || MidShadow  ||\n",
+           (void*)kMidShadowBeg, (void*)kMidShadowEnd);
+  }
+  Printf("|| `[%p, %p]` || ShadowGap  ||\n",
+         (void*)kShadowGapBeg, (void*)kShadowGapEnd);
+  if (kLowShadowBeg) {
+    Printf("|| `[%p, %p]` || LowShadow  ||\n",
+           (void*)kLowShadowBeg, (void*)kLowShadowEnd);
+    Printf("|| `[%p, %p]` || LowMem     ||\n",
+           (void*)kLowMemBeg, (void*)kLowMemEnd);
+  }
+  Printf("MemToShadow(shadow): %p %p %p %p",
+         (void*)MEM_TO_SHADOW(kLowShadowBeg),
+         (void*)MEM_TO_SHADOW(kLowShadowEnd),
+         (void*)MEM_TO_SHADOW(kHighShadowBeg),
+         (void*)MEM_TO_SHADOW(kHighShadowEnd));
+  if (kMidMemBeg) {
+    Printf(" %p %p",
+           (void*)MEM_TO_SHADOW(kMidShadowBeg),
+           (void*)MEM_TO_SHADOW(kMidShadowEnd));
+  }
+  Printf("\n");
+  Printf("red_zone=%zu\n", (uptr)flags()->redzone);
+  Printf("malloc_context_size=%zu\n", (uptr)flags()->malloc_context_size);
+
+  Printf("SHADOW_SCALE: %zx\n", (uptr)SHADOW_SCALE);
+  Printf("SHADOW_GRANULARITY: %zx\n", (uptr)SHADOW_GRANULARITY);
+  Printf("SHADOW_OFFSET: %zx\n", (uptr)SHADOW_OFFSET);
+  CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+  if (kMidMemBeg)
+    CHECK(kMidShadowBeg > kLowShadowEnd &&
+          kMidMemBeg > kMidShadowEnd &&
+          kHighShadowBeg > kMidMemEnd);
 }
 
 }  // namespace __asan
@@ -352,49 +446,48 @@ void __asan_init() {
   ReplaceSystemMalloc();
   ReplaceOperatorsNewAndDelete();
 
-  if (flags()->verbosity) {
-    Printf("|| `[%p, %p]` || HighMem    ||\n",
-           (void*)kHighMemBeg, (void*)kHighMemEnd);
-    Printf("|| `[%p, %p]` || HighShadow ||\n",
-           (void*)kHighShadowBeg, (void*)kHighShadowEnd);
-    Printf("|| `[%p, %p]` || ShadowGap  ||\n",
-           (void*)kShadowGapBeg, (void*)kShadowGapEnd);
-    Printf("|| `[%p, %p]` || LowShadow  ||\n",
-           (void*)kLowShadowBeg, (void*)kLowShadowEnd);
-    Printf("|| `[%p, %p]` || LowMem     ||\n",
-           (void*)kLowMemBeg, (void*)kLowMemEnd);
-    Printf("MemToShadow(shadow): %p %p %p %p\n",
-           (void*)MEM_TO_SHADOW(kLowShadowBeg),
-           (void*)MEM_TO_SHADOW(kLowShadowEnd),
-           (void*)MEM_TO_SHADOW(kHighShadowBeg),
-           (void*)MEM_TO_SHADOW(kHighShadowEnd));
-    Printf("red_zone=%zu\n", (uptr)flags()->redzone);
-    Printf("malloc_context_size=%zu\n", (uptr)flags()->malloc_context_size);
-
-    Printf("SHADOW_SCALE: %zx\n", (uptr)SHADOW_SCALE);
-    Printf("SHADOW_GRANULARITY: %zx\n", (uptr)SHADOW_GRANULARITY);
-    Printf("SHADOW_OFFSET: %zx\n", (uptr)SHADOW_OFFSET);
-    CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+  uptr shadow_start = kLowShadowBeg;
+  if (kLowShadowBeg) shadow_start -= GetMmapGranularity();
+  uptr shadow_end = kHighShadowEnd;
+  bool full_shadow_is_available =
+      MemoryRangeIsAvailable(shadow_start, shadow_end);
+
+#if ASAN_LINUX && defined(__x86_64__) && !ASAN_FIXED_MAPPING
+  if (!full_shadow_is_available) {
+    kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
+    kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x3fffffffffULL : 0;
   }
+#endif
+
+  if (flags()->verbosity)
+    PrintAddressSpaceLayout();
 
   if (flags()->disable_core) {
     DisableCoreDumper();
   }
 
-  uptr shadow_start = kLowShadowBeg;
-  if (kLowShadowBeg > 0) shadow_start -= GetMmapGranularity();
-  uptr shadow_end = kHighShadowEnd;
-  if (MemoryRangeIsAvailable(shadow_start, shadow_end)) {
-    if (kLowShadowBeg != kLowShadowEnd) {
-      // mmap the low shadow plus at least one page.
-      ReserveShadowMemoryRange(kLowShadowBeg - GetMmapGranularity(),
-                               kLowShadowEnd);
-    }
+  if (full_shadow_is_available) {
+    // mmap the low shadow plus at least one page at the left.
+    if (kLowShadowBeg)
+      ReserveShadowMemoryRange(shadow_start, kLowShadowEnd);
+    // mmap the high shadow.
+    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd);
+    // protect the gap.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+  } else if (kMidMemBeg &&
+      MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
+      MemoryRangeIsAvailable(kMidMemEnd + 1, shadow_end)) {
+    CHECK(kLowShadowBeg != kLowShadowEnd);
+    // mmap the low shadow plus at least one page at the left.
+    ReserveShadowMemoryRange(shadow_start, kLowShadowEnd);
+    // mmap the mid shadow.
+    ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd);
     // mmap the high shadow.
     ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd);
-    // protect the gap
-    void *prot = Mprotect(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
-    CHECK(prot == (void*)kShadowGapBeg);
+    // protect the gaps.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+    ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
+    ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
   } else {
     Report("Shadow memory range interleaves with an existing memory mapping. "
            "ASan cannot proceed correctly. ABORTING.\n");
@@ -427,12 +520,12 @@ void __asan_init() {
   }
 }
 
-#if defined(ASAN_USE_PREINIT_ARRAY)
+#if ASAN_USE_PREINIT_ARRAY
   // On Linux, we force __asan_init to be called before anyone else
   // by placing it into .preinit_array section.
   // FIXME: do we have anything like this on Mac?
   __attribute__((section(".preinit_array")))
-    typeof(__asan_init) *__asan_preinit =__asan_init;
+  void (*__asan_preinit)(void) =__asan_init;
 #elif defined(_WIN32) && defined(_DLL)
   // On Windows, when using dynamic CRT (/MD), we can put a pointer
   // to __asan_init into the global list of C initializers.
diff --git a/libsanitizer/asan/dynamic/asan_interceptors_dynamic.cc b/libsanitizer/asan/dynamic/asan_interceptors_dynamic.cc
deleted file mode 100644
index 727edf2..0000000
--- a/libsanitizer/asan/dynamic/asan_interceptors_dynamic.cc
+++ /dev/null
@@ -1,114 +0,0 @@
-//===-- asan_interceptors_dynamic.cc --------------------------------------===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of AddressSanitizer, an address sanity checker.
-//
-// __DATA,__interpose section of the dynamic runtime library for Mac OS.
-//===----------------------------------------------------------------------===//
-
-#if defined(__APPLE__)
-
-#include "../asan_interceptors.h"
-#include "../asan_intercepted_functions.h"
-
-namespace __asan {
-
-#define INTERPOSE_FUNCTION(function) \
-    { reinterpret_cast<const uptr>(WRAP(function)), \
-      reinterpret_cast<const uptr>(function) }
-
-#define INTERPOSE_FUNCTION_2(function, wrapper) \
-    { reinterpret_cast<const uptr>(wrapper), \
-      reinterpret_cast<const uptr>(function) }
-
-struct interpose_substitution {
-  const uptr replacement;
-  const uptr original;
-};
-
-__attribute__((used))
-const interpose_substitution substitutions[]
-    __attribute__((section("__DATA, __interpose"))) = {
-  INTERPOSE_FUNCTION(strlen),
-  INTERPOSE_FUNCTION(memcmp),
-  INTERPOSE_FUNCTION(memcpy),
-  INTERPOSE_FUNCTION(memmove),
-  INTERPOSE_FUNCTION(memset),
-  INTERPOSE_FUNCTION(strchr),
-  INTERPOSE_FUNCTION(strcat),
-  INTERPOSE_FUNCTION(strncat),
-  INTERPOSE_FUNCTION(strcpy),
-  INTERPOSE_FUNCTION(strncpy),
-  INTERPOSE_FUNCTION(pthread_create),
-  INTERPOSE_FUNCTION(longjmp),
-#if ASAN_INTERCEPT__LONGJMP
-  INTERPOSE_FUNCTION(_longjmp),
-#endif
-#if ASAN_INTERCEPT_SIGLONGJMP
-  INTERPOSE_FUNCTION(siglongjmp),
-#endif
-#if ASAN_INTERCEPT_STRDUP
-  INTERPOSE_FUNCTION(strdup),
-#endif
-#if ASAN_INTERCEPT_STRNLEN
-  INTERPOSE_FUNCTION(strnlen),
-#endif
-#if ASAN_INTERCEPT_INDEX
-  INTERPOSE_FUNCTION_2(index, WRAP(strchr)),
-#endif
-  INTERPOSE_FUNCTION(strcmp),
-  INTERPOSE_FUNCTION(strncmp),
-#if ASAN_INTERCEPT_STRCASECMP_AND_STRNCASECMP
-  INTERPOSE_FUNCTION(strcasecmp),
-  INTERPOSE_FUNCTION(strncasecmp),
-#endif
-  INTERPOSE_FUNCTION(atoi),
-  INTERPOSE_FUNCTION(atol),
-  INTERPOSE_FUNCTION(strtol),
-#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL
-  INTERPOSE_FUNCTION(atoll),
-  INTERPOSE_FUNCTION(strtoll),
-#endif
-#if ASAN_INTERCEPT_MLOCKX
-  INTERPOSE_FUNCTION(mlock),
-  INTERPOSE_FUNCTION(munlock),
-  INTERPOSE_FUNCTION(mlockall),
-  INTERPOSE_FUNCTION(munlockall),
-#endif
-  INTERPOSE_FUNCTION(dispatch_async_f),
-  INTERPOSE_FUNCTION(dispatch_sync_f),
-  INTERPOSE_FUNCTION(dispatch_after_f),
-  INTERPOSE_FUNCTION(dispatch_barrier_async_f),
-  INTERPOSE_FUNCTION(dispatch_group_async_f),
-#ifndef MISSING_BLOCKS_SUPPORT
-  INTERPOSE_FUNCTION(dispatch_group_async),
-  INTERPOSE_FUNCTION(dispatch_async),
-  INTERPOSE_FUNCTION(dispatch_after),
-  INTERPOSE_FUNCTION(dispatch_source_set_event_handler),
-  INTERPOSE_FUNCTION(dispatch_source_set_cancel_handler),
-#endif
-  INTERPOSE_FUNCTION(signal),
-  INTERPOSE_FUNCTION(sigaction),
-
-  INTERPOSE_FUNCTION(malloc_create_zone),
-  INTERPOSE_FUNCTION(malloc_default_zone),
-  INTERPOSE_FUNCTION(malloc_default_purgeable_zone),
-  INTERPOSE_FUNCTION(malloc_make_purgeable),
-  INTERPOSE_FUNCTION(malloc_make_nonpurgeable),
-  INTERPOSE_FUNCTION(malloc_set_zone_name),
-  INTERPOSE_FUNCTION(malloc),
-  INTERPOSE_FUNCTION(free),
-  INTERPOSE_FUNCTION(realloc),
-  INTERPOSE_FUNCTION(calloc),
-  INTERPOSE_FUNCTION(valloc),
-  INTERPOSE_FUNCTION(malloc_good_size),
-  INTERPOSE_FUNCTION(posix_memalign),
-};
-
-}  // namespace __asan
-
-#endif  // __APPLE__