From 11201315d5881a135faa5aa87f415ce03f99eb96 Mon Sep 17 00:00:00 2001 From: Jianzhou Zhao Date: Sat, 12 Sep 2020 19:35:17 +0000 Subject: Flush bitcode incrementally for LTO output Bitcode writer does not flush buffer until the end by default. This is fine to small bitcode files. When -flto,--plugin-opt=emit-llvm,-gmlt are used, the final bitcode file is large, for example, >8G. Keeping all data in memory consumes a lot of memory. This change allows bitcode writer flush data to disk early when buffered data size is above some threshold. This is only enabled when lld emits LLVM bitcode. One issue to address is backpatching bitcode: subblock length, function body indexes, meta data indexes need to backfill. If buffer can be flushed partially, we introduced raw_fd_stream that supports read/seek/write, and enables backpatching bitcode flushed in disk. Reviewed-by: tejohnson, MaskRay Differential Revision: https://reviews.llvm.org/D86905 --- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'llvm/lib/Bitcode/Writer/BitcodeWriter.cpp') diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 28384bc..26874c9 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -86,6 +86,9 @@ static cl::opt IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25), cl::desc("Number of metadatas above which we emit an index " "to enable lazy-loading")); +static cl::opt FlushThreshold( + "bitcode-flush-threshold", cl::Hidden, cl::init(512), + cl::desc("The threshold (unit M) for flushing LLVM bitcode.")); static cl::opt WriteRelBFToSummary( "write-relbf-to-summary", cl::Hidden, cl::init(false), @@ -4453,8 +4456,8 @@ static void writeBitcodeHeader(BitstreamWriter &Stream) { Stream.Emit(0xD, 4); } -BitcodeWriter::BitcodeWriter(SmallVectorImpl &Buffer) - : Buffer(Buffer), Stream(new BitstreamWriter(Buffer)) { +BitcodeWriter::BitcodeWriter(SmallVectorImpl &Buffer, raw_fd_stream *FS) + : Buffer(Buffer), Stream(new BitstreamWriter(Buffer, FS, FlushThreshold)) { writeBitcodeHeader(*Stream); } @@ -4565,7 +4568,7 @@ void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out, if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); - BitcodeWriter Writer(Buffer); + BitcodeWriter Writer(Buffer, dyn_cast(&Out)); Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, ModHash); Writer.writeSymtab(); @@ -4575,7 +4578,8 @@ void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out, emitDarwinBCHeaderAndTrailer(Buffer, TT); // Write the generated bitstream to "Out". - Out.write((char*)&Buffer.front(), Buffer.size()); + if (!Buffer.empty()) + Out.write((char *)&Buffer.front(), Buffer.size()); } void IndexBitcodeWriter::write() { -- cgit v1.1