Support lazy stat'ing of files referenced by module maps.

This patch adds support for a `header` declaration in a module map to specify
certain `stat` information (currently, size and mtime) about that header file.
This has two purposes:

- It removes the need to eagerly `stat` every file referenced by a module map.
  Instead, we track a list of unresolved header files with each size / mtime
  (actually, for simplicity, we track submodules with such headers), and when
  attempting to look up a header file based on a `FileEntry`, we check if there
  are any unresolved header directives with that `FileEntry`'s size / mtime and
  perform deferred `stat`s if so.

- It permits a preprocessed module to be compiled without the original files
  being present on disk. The only reason we used to need those files was to get
  the `stat` information in order to do header -> module lookups when using the
  module. If we're provided with the `stat` information in the preprocessed
  module, we can avoid requiring the files to exist.

Unlike most `header` directives, if a `header` directive with `stat`
information has no corresponding on-disk file the enclosing module is *not*
marked unavailable (so that behavior is consistent regardless of whether we've
resolved a header directive, and so that preprocessed modules don't get marked
unavailable). We could actually do this for all `header` directives: the only
reason we mark the module unavailable if headers are missing is to give a
diagnostic slightly earlier (rather than waiting until we actually try to build
the module / load and validate its .pcm file).

Differential Revision: https://reviews.llvm.org/D33703

llvm-svn: 304515

6 files changed, 408 insertions, 133 deletions

diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index ac3d7c5..83c5248 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -394,11 +394,30 @@ void Module::print(raw_ostream &OS, unsigned Indent) const {
                {"exclude ", HK_Excluded}};
 
   for (auto &K : Kinds) {
+    assert(&K == &Kinds[K.Kind] && "kinds in wrong order");
     for (auto &H : Headers[K.Kind]) {
       OS.indent(Indent + 2);
       OS << K.Prefix << "header \"";
       OS.write_escaped(H.NameAsWritten);
-      OS << "\"\n";
+      OS << "\" { size " << H.Entry->getSize()
+         << " mtime " << H.Entry->getModificationTime() << " }\n";
+    }
+  }
+  for (auto *Unresolved : {&UnresolvedHeaders, &MissingHeaders}) {
+    for (auto &U : *Unresolved) {
+      OS.indent(Indent + 2);
+      OS << Kinds[U.Kind].Prefix << "header \"";
+      OS.write_escaped(U.FileName);
+      OS << "\"";
+      if (U.Size || U.ModTime) {
+        OS << " {";
+        if (U.Size)
+          OS << " size " << *U.Size;
+        if (U.ModTime)
+          OS << " mtime " << *U.ModTime;
+        OS << " }";
+      }
+      OS << "\n";
     }
   }
 
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index cd67e46..e2fbe96 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -289,14 +289,28 @@ static void addHeaderInclude(StringRef HeaderName,
 ///
 /// \param Includes Will be augmented with the set of \#includes or \#imports
 /// needed to load all of the named headers.
-static std::error_code
-collectModuleHeaderIncludes(const LangOptions &LangOpts, FileManager &FileMgr,
-                            ModuleMap &ModMap, clang::Module *Module,
-                            SmallVectorImpl<char> &Includes) {
+static std::error_code collectModuleHeaderIncludes(
+    const LangOptions &LangOpts, FileManager &FileMgr, DiagnosticsEngine &Diag,
+    ModuleMap &ModMap, clang::Module *Module, SmallVectorImpl<char> &Includes) {
   // Don't collect any headers for unavailable modules.
   if (!Module->isAvailable())
     return std::error_code();
 
+  // Resolve all lazy header directives to header files.
+  ModMap.resolveHeaderDirectives(Module);
+
+  // If any headers are missing, we can't build this module. In most cases,
+  // diagnostics for this should have already been produced; we only get here
+  // if explicit stat information was provided.
+  // FIXME: If the name resolves to a file with different stat information,
+  // produce a better diagnostic.
+  if (!Module->MissingHeaders.empty()) {
+    auto &MissingHeader = Module->MissingHeaders.front();
+    Diag.Report(MissingHeader.FileNameLoc, diag::err_module_header_missing)
+      << MissingHeader.IsUmbrella << MissingHeader.FileName;
+    return std::error_code();
+  }
+
   // Add includes for each of these headers.
   for (auto HK : {Module::HK_Normal, Module::HK_Private}) {
     for (Module::Header &H : Module->Headers[HK]) {
@@ -367,7 +381,7 @@ collectModuleHeaderIncludes(const LangOptions &LangOpts, FileManager &FileMgr,
                                       SubEnd = Module->submodule_end();
        Sub != SubEnd; ++Sub)
     if (std::error_code Err = collectModuleHeaderIncludes(
-            LangOpts, FileMgr, ModMap, *Sub, Includes))
+            LangOpts, FileMgr, Diag, ModMap, *Sub, Includes))
       return Err;
 
   return std::error_code();
@@ -494,7 +508,7 @@ getInputBufferForModule(CompilerInstance &CI, Module *M) {
     addHeaderInclude(UmbrellaHeader.NameAsWritten, HeaderContents,
                      CI.getLangOpts(), M->IsExternC);
   Err = collectModuleHeaderIncludes(
-      CI.getLangOpts(), FileMgr,
+      CI.getLangOpts(), FileMgr, CI.getDiagnostics(),
       CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(), M,
       HeaderContents);
 
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 9084bc3..1ebcc0a 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -1114,6 +1114,8 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP,
   auto TryEnterImported = [&](void) -> bool {
     if (!ModulesEnabled)
       return false;
+    // Ensure FileInfo bits are up to date.
+    ModMap.resolveHeaderDirectives(File);
     // Modules with builtins are special; multiple modules use builtins as
     // modular headers, example:
     //
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 8c57931..018d59e 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -36,6 +36,37 @@
 #endif
 using namespace clang;
 
+Module::HeaderKind ModuleMap::headerRoleToKind(ModuleHeaderRole Role) {
+  switch ((int)Role) {
+  default: llvm_unreachable("unknown header role");
+  case NormalHeader:
+    return Module::HK_Normal;
+  case PrivateHeader:
+    return Module::HK_Private;
+  case TextualHeader:
+    return Module::HK_Textual;
+  case PrivateHeader | TextualHeader:
+    return Module::HK_PrivateTextual;
+  }
+}
+
+ModuleMap::ModuleHeaderRole
+ModuleMap::headerKindToRole(Module::HeaderKind Kind) {
+  switch ((int)Kind) {
+  case Module::HK_Normal:
+    return NormalHeader;
+  case Module::HK_Private:
+    return PrivateHeader;
+  case Module::HK_Textual:
+    return TextualHeader;
+  case Module::HK_PrivateTextual:
+    return ModuleHeaderRole(PrivateHeader | TextualHeader);
+  case Module::HK_Excluded:
+    llvm_unreachable("unexpected header kind");
+  }
+  llvm_unreachable("unknown header kind");
+}
+
 Module::ExportDecl 
 ModuleMap::resolveExport(Module *Mod, 
                          const Module::UnresolvedExportDecl &Unresolved,
@@ -104,12 +135,22 @@ static void appendSubframeworkPaths(Module *Mod,
 }
 
 const FileEntry *
-ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header,
-                         SmallVectorImpl<char> &RelativePathName) {
+ModuleMap::findHeader(Module *M,
+                      const Module::UnresolvedHeaderDirective &Header,
+                      SmallVectorImpl<char> &RelativePathName) {
+  auto GetFile = [&](StringRef Filename) -> const FileEntry * {
+    auto *File = SourceMgr.getFileManager().getFile(Filename);
+    if (!File ||
+        (Header.Size && File->getSize() != *Header.Size) ||
+        (Header.ModTime && File->getModificationTime() != *Header.ModTime))
+      return nullptr;
+    return File;
+  };
+
   if (llvm::sys::path::is_absolute(Header.FileName)) {
     RelativePathName.clear();
     RelativePathName.append(Header.FileName.begin(), Header.FileName.end());
-    return SourceMgr.getFileManager().getFile(Header.FileName);
+    return GetFile(Header.FileName);
   }
 
   // Search for the header file within the module's home directory.
@@ -124,7 +165,7 @@ ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header,
     // Check whether this file is in the public headers.
     llvm::sys::path::append(RelativePathName, "Headers", Header.FileName);
     llvm::sys::path::append(FullPathName, RelativePathName);
-    if (auto *File = SourceMgr.getFileManager().getFile(FullPathName))
+    if (auto *File = GetFile(FullPathName))
       return File;
 
     // Check whether this file is in the private headers.
@@ -141,31 +182,74 @@ ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header,
     llvm::sys::path::append(RelativePathName, "PrivateHeaders",
                             Header.FileName);
     llvm::sys::path::append(FullPathName, RelativePathName);
-    return SourceMgr.getFileManager().getFile(FullPathName);
+    return GetFile(FullPathName);
   }
 
   // Lookup for normal headers.
   llvm::sys::path::append(RelativePathName, Header.FileName);
   llvm::sys::path::append(FullPathName, RelativePathName);
-  return SourceMgr.getFileManager().getFile(FullPathName);
+  return GetFile(FullPathName);
 }
 
-const FileEntry *
-ModuleMap::resolveAsBuiltinHeader(Module *M,
-                                  Module::UnresolvedHeaderDirective Header,
-                                  SmallVectorImpl<char> &BuiltinPathName) {
-  if (llvm::sys::path::is_absolute(Header.FileName) || M->isPartOfFramework() ||
-      !M->IsSystem || Header.IsUmbrella || !BuiltinIncludeDir ||
-      BuiltinIncludeDir == M->Directory || !isBuiltinHeader(Header.FileName))
-    return nullptr;
+void ModuleMap::resolveHeader(Module *Mod,
+                              const Module::UnresolvedHeaderDirective &Header) {
+  SmallString<128> RelativePathName;
+  if (const FileEntry *File = findHeader(Mod, Header, RelativePathName)) {
+    if (Header.IsUmbrella) {
+      const DirectoryEntry *UmbrellaDir = File->getDir();
+      if (Module *UmbrellaMod = UmbrellaDirs[UmbrellaDir])
+        Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash)
+          << UmbrellaMod->getFullModuleName();
+      else
+        // Record this umbrella header.
+        setUmbrellaHeader(Mod, File, RelativePathName.str());
+    } else {
+      Module::Header H = {RelativePathName.str(), File};
+      if (Header.Kind == Module::HK_Excluded)
+        excludeHeader(Mod, H);
+      else
+        addHeader(Mod, H, headerKindToRole(Header.Kind));
+    }
+  } else if (Header.HasBuiltinHeader && !Header.Size && !Header.ModTime) {
+    // There's a builtin header but no corresponding on-disk header. Assume
+    // this was supposed to modularize the builtin header alone.
+  } else if (Header.Kind == Module::HK_Excluded) {
+    // Ignore missing excluded header files. They're optional anyway.
+  } else {
+    // If we find a module that has a missing header, we mark this module as
+    // unavailable and store the header directive for displaying diagnostics.
+    Mod->MissingHeaders.push_back(Header);
+    // A missing header with stat information doesn't make the module
+    // unavailable; this keeps our behavior consistent as headers are lazily
+    // resolved. (Such a module still can't be built though, except from
+    // preprocessed source.)
+    if (!Header.Size && !Header.ModTime)
+      Mod->markUnavailable();
+  }
+}
+
+bool ModuleMap::resolveAsBuiltinHeader(
+    Module *Mod, const Module::UnresolvedHeaderDirective &Header) {
+  if (Header.Kind == Module::HK_Excluded ||
+      llvm::sys::path::is_absolute(Header.FileName) ||
+      Mod->isPartOfFramework() || !Mod->IsSystem || Header.IsUmbrella ||
+      !BuiltinIncludeDir || BuiltinIncludeDir == Mod->Directory ||
+      !isBuiltinHeader(Header.FileName))
+    return false;
 
   // This is a system module with a top-level header. This header
   // may have a counterpart (or replacement) in the set of headers
   // supplied by Clang. Find that builtin header.
-  llvm::sys::path::append(BuiltinPathName, BuiltinIncludeDir->getName(),
-                          Header.FileName);
-  return SourceMgr.getFileManager().getFile(
-      StringRef(BuiltinPathName.data(), BuiltinPathName.size()));
+  SmallString<128> Path;
+  llvm::sys::path::append(Path, BuiltinIncludeDir->getName(), Header.FileName);
+  auto *File = SourceMgr.getFileManager().getFile(Path);
+  if (!File)
+    return false;
+
+  auto Role = headerKindToRole(Header.Kind);
+  Module::Header H = {Path.str(), File};
+  addHeader(Mod, H, Role);
+  return true;
 }
 
 ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags,
@@ -246,6 +330,7 @@ bool ModuleMap::isBuiltinHeader(StringRef FileName) {
 
 ModuleMap::HeadersMap::iterator
 ModuleMap::findKnownHeader(const FileEntry *File) {
+  resolveHeaderDirectives(File);
   HeadersMap::iterator Known = Headers.find(File);
   if (HeaderInfo.getHeaderSearchOpts().ImplicitModuleMaps &&
       Known == Headers.end() && File->getDir() == BuiltinIncludeDir &&
@@ -328,8 +413,10 @@ void ModuleMap::diagnoseHeaderInclusion(Module *RequestingModule,
   if (getTopLevelOrNull(RequestingModule) != getTopLevelOrNull(SourceModule))
     return;
 
-  if (RequestingModule)
+  if (RequestingModule) {
     resolveUses(RequestingModule, /*Complain=*/false);
+    resolveHeaderDirectives(RequestingModule);
+  }
 
   bool Excluded = false;
   Module *Private = nullptr;
@@ -511,6 +598,7 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(const FileEntry *File) {
 
 ArrayRef<ModuleMap::KnownHeader>
 ModuleMap::findAllModulesForHeader(const FileEntry *File) const {
+  resolveHeaderDirectives(File);
   auto It = Headers.find(File);
   if (It == Headers.end())
     return None;
@@ -524,6 +612,7 @@ bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) const {
 bool
 ModuleMap::isHeaderUnavailableInModule(const FileEntry *Header,
                                        const Module *RequestingModule) const {
+  resolveHeaderDirectives(Header);
   HeadersMap::const_iterator Known = Headers.find(Header);
   if (Known != Headers.end()) {
     for (SmallVectorImpl<KnownHeader>::const_iterator
@@ -896,20 +985,65 @@ void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir,
   UmbrellaDirs[UmbrellaDir] = Mod;
 }
 
-static Module::HeaderKind headerRoleToKind(ModuleMap::ModuleHeaderRole Role) {
-  switch ((int)Role) {
-  default: llvm_unreachable("unknown header role");
-  case ModuleMap::NormalHeader:
-    return Module::HK_Normal;
-  case ModuleMap::PrivateHeader:
-    return Module::HK_Private;
-  case ModuleMap::TextualHeader:
-    return Module::HK_Textual;
-  case ModuleMap::PrivateHeader | ModuleMap::TextualHeader:
-    return Module::HK_PrivateTextual;
+void ModuleMap::addUnresolvedHeader(Module *Mod,
+                                    Module::UnresolvedHeaderDirective Header) {
+  // If there is a builtin counterpart to this file, add it now so it can
+  // wrap the system header.
+  if (resolveAsBuiltinHeader(Mod, Header)) {
+    // If we have both a builtin and system version of the file, the
+    // builtin version may want to inject macros into the system header, so
+    // force the system header to be treated as a textual header in this
+    // case.
+    Header.Kind = headerRoleToKind(ModuleMap::ModuleHeaderRole(
+        headerKindToRole(Header.Kind) | ModuleMap::TextualHeader));
+    Header.HasBuiltinHeader = true;
+  }
+
+  // If possible, don't stat the header until we need to. This requires the
+  // user to have provided us with some stat information about the file.
+  // FIXME: Add support for lazily stat'ing umbrella headers and excluded
+  // headers.
+  if ((Header.Size || Header.ModTime) && !Header.IsUmbrella &&
+      Header.Kind != Module::HK_Excluded) {
+    // We expect more variation in mtime than size, so if we're given both,
+    // use the mtime as the key.
+    if (Header.ModTime)
+      LazyHeadersByModTime[*Header.ModTime].push_back(Mod);
+    else
+      LazyHeadersBySize[*Header.Size].push_back(Mod);
+    Mod->UnresolvedHeaders.push_back(Header);
+    return;
+  }
+
+  // We don't have stat information or can't defer looking this file up.
+  // Perform the lookup now.
+  resolveHeader(Mod, Header);
+}
+
+void ModuleMap::resolveHeaderDirectives(const FileEntry *File) const {
+  auto BySize = LazyHeadersBySize.find(File->getSize());
+  if (BySize != LazyHeadersBySize.end()) {
+    for (auto *M : BySize->second)
+      resolveHeaderDirectives(M);
+    LazyHeadersBySize.erase(BySize);
+  }
+
+  auto ByModTime = LazyHeadersByModTime.find(File->getModificationTime());
+  if (ByModTime != LazyHeadersByModTime.end()) {
+    for (auto *M : ByModTime->second)
+      resolveHeaderDirectives(M);
+    LazyHeadersByModTime.erase(ByModTime);
   }
 }
 
+void ModuleMap::resolveHeaderDirectives(Module *Mod) const {
+  for (auto &Header : Mod->UnresolvedHeaders)
+    // This operation is logically const; we're just changing how we represent
+    // the header information for this file.
+    const_cast<ModuleMap*>(this)->resolveHeader(Mod, Header);
+  Mod->UnresolvedHeaders.clear();
+}
+
 void ModuleMap::addHeader(Module *Mod, Module::Header Header,
                           ModuleHeaderRole Role, bool Imported) {
   KnownHeader KH(Mod, Role);
@@ -1063,6 +1197,7 @@ namespace clang {
       RequiresKeyword,
       Star,
       StringLiteral,
+      IntegerLiteral,
       TextualKeyword,
       LBrace,
       RBrace,
@@ -1072,7 +1207,12 @@ namespace clang {
     
     unsigned Location;
     unsigned StringLength;
-    const char *StringData;
+    union {
+      // If Kind != IntegerLiteral.
+      const char *StringData;
+      // If Kind == IntegerLiteral.
+      uint64_t IntegerValue;
+    };
     
     void clear() {
       Kind = EndOfFile;
@@ -1086,9 +1226,14 @@ namespace clang {
     SourceLocation getLocation() const {
       return SourceLocation::getFromRawEncoding(Location);
     }
+
+    uint64_t getInteger() const {
+      return Kind == IntegerLiteral ? IntegerValue : 0;
+    }
     
     StringRef getString() const {
-      return StringRef(StringData, StringLength);
+      return Kind == IntegerLiteral ? StringRef()
+                                    : StringRef(StringData, StringLength);
     }
   };
 
@@ -1278,6 +1423,25 @@ retry:
     Tok.StringLength = Length;
     break;
   }
+
+  case tok::numeric_constant: {
+    // We don't support any suffixes or other complications.
+    SmallString<32> SpellingBuffer;
+    SpellingBuffer.resize(LToken.getLength() + 1);
+    const char *Start = SpellingBuffer.data();
+    unsigned Length =
+        Lexer::getSpelling(LToken, Start, SourceMgr, L.getLangOpts());
+    uint64_t Value;
+    if (StringRef(Start, Length).getAsInteger(0, Value)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_unknown_token);
+      HadError = true;
+      goto retry;
+    }
+
+    Tok.Kind = MMToken::IntegerLiteral;
+    Tok.IntegerValue = Value;
+    break;
+  }
       
   case tok::comment:
     goto retry;
@@ -1904,6 +2068,9 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken,
   Header.FileName = Tok.getString();
   Header.FileNameLoc = consumeToken();
   Header.IsUmbrella = LeadingToken == MMToken::UmbrellaKeyword;
+  Header.Kind =
+      (LeadingToken == MMToken::ExcludeKeyword ? Module::HK_Excluded
+                                               : Map.headerRoleToKind(Role));
 
   // Check whether we already have an umbrella.
   if (Header.IsUmbrella && ActiveModule->Umbrella) {
@@ -1913,64 +2080,62 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken,
     return;
   }
 
-  // Look for this file by name if we don't have any stat information.
-  SmallString<128> RelativePathName, BuiltinPathName;
-  const FileEntry *File =
-      Map.resolveHeader(ActiveModule, Header, RelativePathName);
-  const FileEntry *BuiltinFile =
-      Map.resolveAsBuiltinHeader(ActiveModule, Header, BuiltinPathName);
+  // If we were given stat information, parse it so we can skip looking for
+  // the file.
+  if (Tok.is(MMToken::LBrace)) {
+    SourceLocation LBraceLoc = consumeToken();
+
+    while (!Tok.is(MMToken::RBrace) && !Tok.is(MMToken::EndOfFile)) {
+      enum Attribute { Size, ModTime, Unknown };
+      StringRef Str = Tok.getString();
+      SourceLocation Loc = consumeToken();
+      switch (llvm::StringSwitch<Attribute>(Str)
+                  .Case("size", Size)
+                  .Case("mtime", ModTime)
+                  .Default(Unknown)) {
+      case Size:
+        if (Header.Size)
+          Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str;
+        if (!Tok.is(MMToken::IntegerLiteral)) {
+          Diags.Report(Tok.getLocation(),
+                       diag::err_mmap_invalid_header_attribute_value) << Str;
+          skipUntil(MMToken::RBrace);
+          break;
+        }
+        Header.Size = Tok.getInteger();
+        consumeToken();
+        break;
 
-  // If Clang supplies this header but the underlying system does not,
-  // just silently swap in our builtin version. Otherwise, we'll end
-  // up adding both (later).
-  if (BuiltinFile && !File) {
-    RelativePathName = BuiltinPathName;
-    File = BuiltinFile;
-    BuiltinFile = nullptr;
-  }
+      case ModTime:
+        if (Header.ModTime)
+          Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str;
+        if (!Tok.is(MMToken::IntegerLiteral)) {
+          Diags.Report(Tok.getLocation(),
+                       diag::err_mmap_invalid_header_attribute_value) << Str;
+          skipUntil(MMToken::RBrace);
+          break;
+        }
+        Header.ModTime = Tok.getInteger();
+        consumeToken();
+        break;
 
-  // FIXME: We shouldn't be eagerly stat'ing every file named in a module map.
-  // Come up with a lazy way to do this.
-  if (File) {
-    if (Header.IsUmbrella) {
-      const DirectoryEntry *UmbrellaDir = File->getDir();
-      if (Module *UmbrellaModule = Map.UmbrellaDirs[UmbrellaDir]) {
-        Diags.Report(LeadingLoc, diag::err_mmap_umbrella_clash)
-          << UmbrellaModule->getFullModuleName();
-        HadError = true;
-      } else {
-        // Record this umbrella header.
-        Map.setUmbrellaHeader(ActiveModule, File, RelativePathName.str());
-      }
-    } else if (LeadingToken == MMToken::ExcludeKeyword) {
-      Module::Header H = {RelativePathName.str(), File};
-      Map.excludeHeader(ActiveModule, H);
-    } else {
-      // If there is a builtin counterpart to this file, add it now so it can
-      // wrap the system header.
-      if (BuiltinFile) {
-        Module::Header H = { BuiltinPathName.str(), BuiltinFile };
-        Map.addHeader(ActiveModule, H, Role);
-
-        // If we have both a builtin and system version of the file, the
-        // builtin version may want to inject macros into the system header, so
-        // force the system header to be treated as a textual header in this
-        // case.
-        Role = ModuleMap::ModuleHeaderRole(Role | ModuleMap::TextualHeader);
+      case Unknown:
+        Diags.Report(Loc, diag::err_mmap_expected_header_attribute);
+        skipUntil(MMToken::RBrace);
+        break;
       }
-
-      // Record this header.
-      Module::Header H = { RelativePathName.str(), File };
-      Map.addHeader(ActiveModule, H, Role);
     }
-  } else if (LeadingToken != MMToken::ExcludeKeyword) {
-    // Ignore excluded header files. They're optional anyway.
 
-    // If we find a module that has a missing header, we mark this module as
-    // unavailable and store the header directive for displaying diagnostics.
-    ActiveModule->markUnavailable();
-    ActiveModule->MissingHeaders.push_back(Header);
+    if (Tok.is(MMToken::RBrace))
+      consumeToken();
+    else {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace);
+      Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match);
+      HadError = true;
+    }
   }
+
+  Map.addUnresolvedHeader(ActiveModule, std::move(Header));
 }
 
 static int compareModuleHeaders(const Module::Header *A,
@@ -2521,6 +2686,7 @@ bool ModuleMapParser::parseModuleMapFile() {
     case MMToken::RequiresKeyword:
     case MMToken::Star:
     case MMToken::StringLiteral:
+    case MMToken::IntegerLiteral:
     case MMToken::TextualKeyword:
     case MMToken::UmbrellaKeyword:
     case MMToken::UseKeyword:
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 8b58779..2d3ad69 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -689,6 +689,8 @@ Preprocessor::getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
   while (!Loc.isInvalid() && !SM.isInMainFile(Loc)) {
     auto ID = SM.getFileID(SM.getExpansionLoc(Loc));
     auto *FE = SM.getFileEntryForID(ID);
+    if (!FE)
+      break;
 
     bool InTextualHeader = false;
     for (auto Header : HeaderInfo.getModuleMap().findAllModulesForHeader(FE)) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 95cb54f..f9e908e 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1856,24 +1856,31 @@ namespace {
   // Trait used for the on-disk hash table of header search information.
   class HeaderFileInfoTrait {
     ASTWriter &Writer;
-    const HeaderSearch &HS;
     
     // Keep track of the framework names we've used during serialization.
     SmallVector<char, 128> FrameworkStringData;
     llvm::StringMap<unsigned> FrameworkNameOffset;
     
   public:
-    HeaderFileInfoTrait(ASTWriter &Writer, const HeaderSearch &HS)
-      : Writer(Writer), HS(HS) { }
-    
+    HeaderFileInfoTrait(ASTWriter &Writer) : Writer(Writer) {}
+
     struct key_type {
-      const FileEntry *FE;
       StringRef Filename;
+      off_t Size;
+      time_t ModTime;
     };
     typedef const key_type &key_type_ref;
+
+    using UnresolvedModule =
+        llvm::PointerIntPair<Module *, 2, ModuleMap::ModuleHeaderRole>;
     
-    typedef HeaderFileInfo data_type;
+    struct data_type {
+      const HeaderFileInfo &HFI;
+      ArrayRef<ModuleMap::KnownHeader> KnownHeaders;
+      UnresolvedModule Unresolved;
+    };
     typedef const data_type &data_type_ref;
+
     typedef unsigned hash_value_type;
     typedef unsigned offset_type;
     
@@ -1881,8 +1888,7 @@ namespace {
       // The hash is based only on size/time of the file, so that the reader can
       // match even when symlinking or excess path elements ("foo/../", "../")
       // change the form of the name. However, complete path is still the key.
-      return llvm::hash_combine(key.FE->getSize(),
-                                Writer.getTimestampForOutput(key.FE));
+      return llvm::hash_combine(key.Size, key.ModTime);
     }
     
     std::pair<unsigned,unsigned>
@@ -1892,68 +1898,74 @@ namespace {
       unsigned KeyLen = key.Filename.size() + 1 + 8 + 8;
       LE.write<uint16_t>(KeyLen);
       unsigned DataLen = 1 + 2 + 4 + 4;
-      for (auto ModInfo : HS.getModuleMap().findAllModulesForHeader(key.FE))
+      for (auto ModInfo : Data.KnownHeaders)
         if (Writer.getLocalOrImportedSubmoduleID(ModInfo.getModule()))
           DataLen += 4;
+      if (Data.Unresolved.getPointer())
+        DataLen += 4;
       LE.write<uint8_t>(DataLen);
       return std::make_pair(KeyLen, DataLen);
     }
-    
+
     void EmitKey(raw_ostream& Out, key_type_ref key, unsigned KeyLen) {
       using namespace llvm::support;
       endian::Writer<little> LE(Out);
-      LE.write<uint64_t>(key.FE->getSize());
+      LE.write<uint64_t>(key.Size);
       KeyLen -= 8;
-      LE.write<uint64_t>(Writer.getTimestampForOutput(key.FE));
+      LE.write<uint64_t>(key.ModTime);
       KeyLen -= 8;
       Out.write(key.Filename.data(), KeyLen);
     }
-    
+
     void EmitData(raw_ostream &Out, key_type_ref key,
                   data_type_ref Data, unsigned DataLen) {
       using namespace llvm::support;
       endian::Writer<little> LE(Out);
       uint64_t Start = Out.tell(); (void)Start;
       
-      unsigned char Flags = (Data.isImport << 4)
-                          | (Data.isPragmaOnce << 3)
-                          | (Data.DirInfo << 1)
-                          | Data.IndexHeaderMapHeader;
+      unsigned char Flags = (Data.HFI.isImport << 4)
+                          | (Data.HFI.isPragmaOnce << 3)
+                          | (Data.HFI.DirInfo << 1)
+                          | Data.HFI.IndexHeaderMapHeader;
       LE.write<uint8_t>(Flags);
-      LE.write<uint16_t>(Data.NumIncludes);
+      LE.write<uint16_t>(Data.HFI.NumIncludes);
       
-      if (!Data.ControllingMacro)
-        LE.write<uint32_t>(Data.ControllingMacroID);
+      if (!Data.HFI.ControllingMacro)
+        LE.write<uint32_t>(Data.HFI.ControllingMacroID);
       else
-        LE.write<uint32_t>(Writer.getIdentifierRef(Data.ControllingMacro));
-      
+        LE.write<uint32_t>(Writer.getIdentifierRef(Data.HFI.ControllingMacro));
+
       unsigned Offset = 0;
-      if (!Data.Framework.empty()) {
+      if (!Data.HFI.Framework.empty()) {
         // If this header refers into a framework, save the framework name.
         llvm::StringMap<unsigned>::iterator Pos
-          = FrameworkNameOffset.find(Data.Framework);
+          = FrameworkNameOffset.find(Data.HFI.Framework);
         if (Pos == FrameworkNameOffset.end()) {
           Offset = FrameworkStringData.size() + 1;
-          FrameworkStringData.append(Data.Framework.begin(), 
-                                     Data.Framework.end());
+          FrameworkStringData.append(Data.HFI.Framework.begin(), 
+                                     Data.HFI.Framework.end());
           FrameworkStringData.push_back(0);
           
-          FrameworkNameOffset[Data.Framework] = Offset;
+          FrameworkNameOffset[Data.HFI.Framework] = Offset;
         } else
           Offset = Pos->second;
       }
       LE.write<uint32_t>(Offset);
 
-      // FIXME: If the header is excluded, we should write out some
-      // record of that fact.
-      for (auto ModInfo : HS.getModuleMap().findAllModulesForHeader(key.FE)) {
-        if (uint32_t ModID =
-                Writer.getLocalOrImportedSubmoduleID(ModInfo.getModule())) {
-          uint32_t Value = (ModID << 2) | (unsigned)ModInfo.getRole();
+      auto EmitModule = [&](Module *M, ModuleMap::ModuleHeaderRole Role) {
+        if (uint32_t ModID = Writer.getLocalOrImportedSubmoduleID(M)) {
+          uint32_t Value = (ModID << 2) | (unsigned)Role;
           assert((Value >> 2) == ModID && "overflow in header module info");
           LE.write<uint32_t>(Value);
         }
-      }
+      };
+
+      // FIXME: If the header is excluded, we should write out some
+      // record of that fact.
+      for (auto ModInfo : Data.KnownHeaders)
+        EmitModule(ModInfo.getModule(), ModInfo.getRole());
+      if (Data.Unresolved.getPointer())
+        EmitModule(Data.Unresolved.getPointer(), Data.Unresolved.getInt());
 
       assert(Out.tell() - Start == DataLen && "Wrong data length");
     }
@@ -1968,16 +1980,71 @@ namespace {
 ///
 /// \param HS The header search structure to save.
 void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
+  HeaderFileInfoTrait GeneratorTrait(*this);
+  llvm::OnDiskChainedHashTableGenerator<HeaderFileInfoTrait> Generator;
+  SmallVector<const char *, 4> SavedStrings;
+  unsigned NumHeaderSearchEntries = 0;
+
+  // Find all unresolved headers for the current module. We generally will
+  // have resolved them before we get here, but not necessarily: we might be
+  // compiling a preprocessed module, where there is no requirement for the
+  // original files to exist any more.
+  if (WritingModule) {
+    llvm::SmallVector<Module *, 16> Worklist(1, WritingModule);
+    while (!Worklist.empty()) {
+      Module *M = Worklist.pop_back_val();
+      if (!M->isAvailable())
+        continue;
+
+      // Map to disk files where possible, to pick up any missing stat
+      // information. This also means we don't need to check the unresolved
+      // headers list when emitting resolved headers in the first loop below.
+      // FIXME: It'd be preferable to avoid doing this if we were given
+      // sufficient stat information in the module map.
+      HS.getModuleMap().resolveHeaderDirectives(M);
+
+      // If the file didn't exist, we can still create a module if we were given
+      // enough information in the module map.
+      for (auto U : M->MissingHeaders) {
+        // Check that we were given enough information to build a module
+        // without this file existing on disk.
+        if (!U.Size || (!U.ModTime && IncludeTimestamps)) {
+          PP->Diag(U.FileNameLoc, diag::err_module_no_size_mtime_for_header)
+            << WritingModule->getFullModuleName() << U.Size.hasValue()
+            << U.FileName;
+          continue;
+        }
+
+        // Form the effective relative pathname for the file.
+        SmallString<128> Filename(M->Directory->getName());
+        llvm::sys::path::append(Filename, U.FileName);
+        PreparePathForOutput(Filename);
+
+        StringRef FilenameDup = strdup(Filename.c_str());
+        SavedStrings.push_back(FilenameDup.data());
+
+        HeaderFileInfoTrait::key_type Key = {
+          FilenameDup, *U.Size, IncludeTimestamps ? *U.ModTime : 0
+        };
+        HeaderFileInfoTrait::data_type Data = {
+          {}, {}, {M, ModuleMap::headerKindToRole(U.Kind)}
+        };
+        // FIXME: Deal with cases where there are multiple unresolved header
+        // directives in different submodules for the same header.
+        Generator.insert(Key, Data, GeneratorTrait);
+        ++NumHeaderSearchEntries;
+      }
+
+      Worklist.append(M->submodule_begin(), M->submodule_end());
+    }
+  }
+
   SmallVector<const FileEntry *, 16> FilesByUID;
   HS.getFileMgr().GetUniqueIDMapping(FilesByUID);
   
   if (FilesByUID.size() > HS.header_file_size())
     FilesByUID.resize(HS.header_file_size());
-  
-  HeaderFileInfoTrait GeneratorTrait(*this, HS);
-  llvm::OnDiskChainedHashTableGenerator<HeaderFileInfoTrait> Generator;
-  SmallVector<const char *, 4> SavedStrings;
-  unsigned NumHeaderSearchEntries = 0;
+
   for (unsigned UID = 0, LastUID = FilesByUID.size(); UID != LastUID; ++UID) {
     const FileEntry *File = FilesByUID[UID];
     if (!File)
@@ -2004,11 +2071,16 @@ void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
       SavedStrings.push_back(Filename.data());
     }
 
-    HeaderFileInfoTrait::key_type key = { File, Filename };
-    Generator.insert(key, *HFI, GeneratorTrait);
+    HeaderFileInfoTrait::key_type Key = {
+      Filename, File->getSize(), getTimestampForOutput(File)
+    };
+    HeaderFileInfoTrait::data_type Data = {
+      *HFI, HS.getModuleMap().findAllModulesForHeader(File), {}
+    };
+    Generator.insert(Key, Data, GeneratorTrait);
     ++NumHeaderSearchEntries;
   }
-  
+
   // Create the on-disk hash table in a buffer.
   SmallString<4096> TableData;
   uint32_t BucketOffset;