[𝘀𝗽𝗿] initial versionusers/MaskRay/spr/elf-add-thinlto-index

Created using spr 1.3.5-bogner

8 files changed, 184 insertions, 4 deletions

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 7cae867..a0a8f1c 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -215,6 +215,7 @@ struct Config {
   llvm::StringRef soName;
   llvm::StringRef sysroot;
   llvm::StringRef thinLTOCacheDir;
+  llvm::StringRef thinLTOIndex;
   llvm::StringRef thinLTOIndexOnlyArg;
   llvm::StringRef whyExtract;
   llvm::StringRef cmseInputLib;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index e25db0e..e7b3466 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1449,10 +1449,13 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)),
       "--thinlto-cache-policy: invalid cache policy");
   ctx.arg.thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
-  ctx.arg.thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) ||
+  ctx.arg.thinLTOIndex = args.getLastArgValue(OPT_thinlto_index);
+  ctx.arg.thinLTOEmitIndexFiles = ctx.arg.thinLTOIndex.size() ||
+                                  args.hasArg(OPT_thinlto_emit_index_files) ||
                                   args.hasArg(OPT_thinlto_index_only) ||
                                   args.hasArg(OPT_thinlto_index_only_eq);
-  ctx.arg.thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+  ctx.arg.thinLTOIndexOnly = ctx.arg.thinLTOIndex.size() ||
+                             args.hasArg(OPT_thinlto_index_only) ||
                              args.hasArg(OPT_thinlto_index_only_eq);
   ctx.arg.thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
   ctx.arg.thinLTOObjectSuffixReplace =
@@ -1469,7 +1472,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
             "--thinlto-emit-index-files");
   }
   if (!ctx.arg.thinLTOPrefixReplaceNativeObject.empty() &&
-      ctx.arg.thinLTOIndexOnlyArg.empty()) {
+      ctx.arg.thinLTOIndex.empty() && ctx.arg.thinLTOIndexOnlyArg.empty()) {
     error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with "
           "--thinlto-index-only=");
   }
@@ -2983,7 +2986,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Skip the normal linked output if some LTO options are specified.
   //
-  // For --thinlto-index-only, index file creation is performed in
+  // For --thinlto-index{,-only}, index file creation is performed in
   // compileBitcodeFiles, so we are done afterwards. --plugin-opt=emit-llvm and
   // --plugin-opt=emit-asm create output files in bitcode or assembly code,
   // respectively. When only certain thinLTO modules are specified for
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index f339f1c..2398b36 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -197,6 +197,28 @@ BitcodeCompiler::BitcodeCompiler() {
       createConfig(), backend, config->ltoPartitions,
       ltoModes[config->ltoKind]);
 
+  // Write a remapping file for the final native link. Non-lazy bitcode files
+  // are mapped to native object files. If lazy bitcode files is a minimized
+  // bitcode, it cannot participate the final link. Print /dev/null to ignore
+  // it.
+  if (!ctx.arg.thinLTOIndex.empty()) {
+    if (auto os = openFile(ctx.arg.thinLTOIndex)) {
+      for (BitcodeFile *file : ctx.bitcodeFiles) {
+        StringRef nativeDir = ctx.arg.thinLTOPrefixReplaceNativeObject.empty()
+                               ? ctx.arg.thinLTOPrefixReplaceNew
+                               : ctx.arg.thinLTOPrefixReplaceNativeObject;
+        *os << file->getName() << '='
+            << lto::getThinLTOOutputFile(replaceThinLTOSuffix(file->getName()),
+                                         ctx.arg.thinLTOPrefixReplaceOld,
+                                         nativeDir)
+            << '\n';
+      }
+      for (BitcodeFile *file : ctx.lazyBitcodeFiles)
+        if (file->lazy)
+          *os << file->getName() << "=/dev/null\n";
+    }
+  }
+
   // Initialize usedStartStop.
   if (ctx.bitcodeFiles.empty())
     return;
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c80c401..6de246fe 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -676,6 +676,7 @@ def thinlto_cache_dir: JJ<"thinlto-cache-dir=">,
 defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
 def thinlto_emit_imports_files: FF<"thinlto-emit-imports-files">;
 def thinlto_emit_index_files: FF<"thinlto-emit-index-files">;
+def thinlto_index: JJ<"thinlto-index=">;
 def thinlto_index_only: FF<"thinlto-index-only">;
 def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
 def thinlto_jobs_eq: JJ<"thinlto-jobs=">,
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 6d09de1..0f0396b 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -36,6 +36,10 @@ ELF Improvements
   increases the expressive power of linker scripts.
   (`#95323 <https://github.com/llvm/llvm-project/pull/95323>`_)
 
+* Experimental ``--thinlto-index=`` is added for distributed ThinLTO,
+  addressing some symbol resolution issues with the old ``--thinlto-index-only=``.
+  The ThinLTO indexing and the final link have very similar command lines.
+
 Breaking changes
 ----------------
 
diff --git a/lld/test/ELF/lto/thinlto-index-file.ll b/lld/test/ELF/lto/thinlto-index-file.ll
new file mode 100644
index 0000000..0384527
--- /dev/null
+++ b/lld/test/ELF/lto/thinlto-index-file.ll
@@ -0,0 +1,30 @@
+; REQUIRES: x86
+;; Test --thinlto-index-only= for distributed ThinLTO.
+;; This option is discouraged in favor of --thinlto-index=.
+
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: opt -module-summary %s -o 1.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o 2.o
+; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o 3.o
+
+;; Ensure lld writes linked files to linked objects file
+; RUN: ld.lld --plugin-opt=thinlto-index-only=1.txt -shared 1.o 2.o 3.o -o /dev/null
+; RUN: FileCheck %s < 1.txt
+; CHECK: 1.o
+; CHECK: 2.o
+; CHECK: 3.o
+
+;; Check that this also works without the --plugin-opt= prefix.
+; RUN: ld.lld --thinlto-index-only=2.txt -shared 1.o 2.o 3.o -o /dev/null
+; RUN: diff 1.txt 2.txt
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}
diff --git a/lld/test/ELF/lto/thinlto-index.ll b/lld/test/ELF/lto/thinlto-index.ll
new file mode 100644
index 0000000..68e6eb4
--- /dev/null
+++ b/lld/test/ELF/lto/thinlto-index.ll
@@ -0,0 +1,108 @@
+; REQUIRES: x86
+;; Test --thinlto-index= for distributed ThinLTO.
+
+; RUN: rm -rf %t && split-file %s %t && mkdir %t/thin %t/index && cd %t
+; RUN: llvm-mc -filetype=obj -triple=x86_64 main.s -o main.o
+; RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o
+; RUN: opt -thinlto-bc a.ll -o thin/a.o -thin-link-bitcode-file=thin/a.min.o
+; RUN: opt -thinlto-bc bc.ll -o thin/bc.o -thin-link-bitcode-file=thin/bc.min.o
+; RUN: opt -thinlto-bc %p/Inputs/thinlto_empty.ll -o thin/empty.o -thin-link-bitcode-file=thin/empty.min.o
+; RUN: opt -thinlto-bc %p/Inputs/thinlto_empty.ll -o thin/empty1.o -thin-link-bitcode-file=thin/empty1.min.o
+
+; RUN: ld.lld --thinlto-index=1.map --thinlto-emit-imports-files --thinlto-prefix-replace='thin;obj' \
+; RUN:   --thinlto-object-suffix-replace='.min.o;.o' main.o thin/a.min.o \
+; RUN:   --start-lib thin/bc.min.o thin/empty.min.o --end-lib thin/empty1.min.o --start-lib c.o --end-lib -o 1
+; RUN: FileCheck --input-file=1.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace
+
+;; No entry for empty.min.o which is not extracted. empty1.min.o is present,
+;; otherwise the final link may try in-process ThinLTO backend compilation.
+;      CHECK:thin/a.min.o=obj/a.o
+; CHECK-NEXT:thin/bc.min.o=obj/bc.o
+; CHECK-NEXT:thin/empty1.min.o=obj/empty1.o
+; CHECK-NEXT:thin/empty.min.o=/dev/null
+
+;; Nevertheless, empty.o.{imports,thinlto.bc} exist to meet the build system requirement.
+; RUN: ls obj/a.o.imports obj/bc.o.imports obj/empty.o.imports obj/empty1.o.imports
+
+; RUN: llvm-bcanalyzer -dump obj/a.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDA
+; RUN: llvm-bcanalyzer -dump obj/bc.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDB
+; RUN: llvm-bcanalyzer -dump obj/empty.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDE
+; RUN: llvm-bcanalyzer -dump obj/empty1.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDE1
+
+; BACKENDA:       <MODULE_STRTAB_BLOCK
+; BACKENDA-NEXT:  <ENTRY {{.*}} record string = 'thin/a.o'
+; BACKENDA-NEXT:  <HASH
+; BACKENDA-NEXT:  <ENTRY {{.*}} record string = 'thin/bc.o'
+; BACKENDA-NEXT:  <HASH
+; BACKENDA-NEXT:  </MODULE_STRTAB_BLOCK>
+
+; BACKENDB:       <MODULE_STRTAB_BLOCK
+; BACKENDB-NEXT:  <ENTRY {{.*}} record string = 'thin/bc.o'
+; BACKENDB-NEXT:  <HASH
+; BACKENDB-NEXT:  </MODULE_STRTAB_BLOCK>
+
+; BACKENDE:       <MODULE_STRTAB_BLOCK
+; BACKENDE-NEXT:  </MODULE_STRTAB_BLOCK>
+
+; BACKENDE1:      <MODULE_STRTAB_BLOCK
+; BACKENDE1-NEXT: <ENTRY {{.*}} record string = 'thin/empty1.o'
+; BACKENDE1-NEXT: <HASH
+; BACKENDE1-NEXT: </MODULE_STRTAB_BLOCK>
+
+;; Thin archives can be used as well.
+; RUN: llvm-ar rcTS thin.a thin/bc.min.o thin/empty.min.o
+; RUN: ld.lld --thinlto-index=2.map --thinlto-prefix-replace='thin;obj' \
+; RUN:   --thinlto-object-suffix-replace='.min.o;.o' main.o \
+; RUN:   thin/a.min.o thin.a thin/empty1.min.o -o 2
+; RUN: FileCheck --input-file=2.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace
+
+;; For regular archives, the filename may be insufficient to locate the archive and the particular member.
+; RUN: llvm-ar rcS bc.a thin/bc.min.o thin/empty.min.o
+; RUN: ld.lld --thinlto-index=3.map --thinlto-prefix-replace='thin;obj' \
+; RUN:   --thinlto-object-suffix-replace='.min.o;.o' main.o \
+; RUN:   thin/a.min.o bc.a thin/empty1.min.o -o 3
+; RUN: FileCheck --input-file=3.map %s --check-prefix=ARCHIVE --implicit-check-not={{.}} --match-full-lines --strict-whitespace
+
+;      ARCHIVE:thin/a.min.o=obj/a.o
+; ARCHIVE-NEXT:bc.min.o=bc.o
+; ARCHIVE-NEXT:thin/empty1.min.o=obj/empty1.o
+; ARCHIVE-NEXT:empty.min.o=/dev/null
+
+; RUN: ld.lld --thinlto-index=4.map --thinlto-emit-imports-files --thinlto-prefix-replace='thin;index;obj' \
+; RUN:   --thinlto-object-suffix-replace='.min.o;.o' main.o thin/a.min.o \
+; RUN:   --start-lib thin/bc.min.o thin/empty.min.o --end-lib thin/empty1.min.o --start-lib c.o --end-lib -o 4
+; RUN: FileCheck --input-file=4.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace
+; RUN: ls index/a.o.thinlto.bc index/a.o.imports
+
+;--- a.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @b()
+declare void @c()
+
+define void @a() {
+  call void () @b()
+  call void () @c()
+  ret void
+}
+
+;--- bc.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @b() {
+  ret void
+}
+define void @c() {
+  ret void
+}
+
+;--- main.s
+.globl _start
+_start:
+  call a
+
+;--- c.s
+.globl c
+c:
diff --git a/lld/test/ELF/lto/thinlto-single-module.ll b/lld/test/ELF/lto/thinlto-single-module.ll
index 0530213..9347664 100644
--- a/lld/test/ELF/lto/thinlto-single-module.ll
+++ b/lld/test/ELF/lto/thinlto-single-module.ll
@@ -48,9 +48,20 @@
 ; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null
 ; RUN: FileCheck %s --check-prefix=IDX < single5.idx
 ; RUN: count 1 < single5.idx
+; RUN: rm main.o.thinlto.bc
 
 ; IDX: main.o
 
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-index=single5.map
+; RUN: ls main.o.thinlto.bc
+; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null
+; RUN: FileCheck --input-file=single5.map %s --check-prefix=REMAP --implicit-check-not={{.}}
+
+;; Currently the --thinlto-index= file is not affected by --thinlto-single-module.
+; REMAP:      main.o=main.o
+; REMAP-NEXT: thin1.o=thin1.o
+; REMAP-NEXT: thin2.o=thin2.o
+
 ;; Check temporary output generated for main.o only.
 ; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --save-temps
 ; RUN: ls main.o.0.preopt.bc