From 2499fe1ac40f7a50e71bf93c26921f93df2bcc52 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 7 Oct 2025 10:52:45 -0500 Subject: [Offload] Rename and move 'clang-offload-packager' -> 'llvm-offload-binary' (#161438) Summary: This tool is pretty much a generic interface into creating and managing the offloading binary format. The binary format itself is just a fat binary block used to create heterogeneous objects. This should be made more general than just `clang` since it's likely going to be used for larger offloading projects and is the expected way to extract heterogeneous objects from offloading code. Relatively straightforward rename, a few tweaks and documentation changes. Kept in `clang-offload-packager` for legacy compatibility as we looked this tool up by name in places, will probably delete it next release. --- .../llvm-offload-binary/llvm-offload-binary.cpp | 259 +++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp (limited to 'llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp') diff --git a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp new file mode 100644 index 0000000..b1bc335 --- /dev/null +++ b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp @@ -0,0 +1,259 @@ +//===-- llvm-offload-binary.cpp - offload binary management utility -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tool takes several device object files and bundles them into a single +// binary image using a custom binary format. This is intended to be used to +// embed many device files into an application to create a fat binary. It also +// supports extracting these files from a known location. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringExtras.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/Object/ArchiveWriter.h" +#include "llvm/Object/OffloadBinary.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileOutputBuffer.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/WithColor.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +static cl::OptionCategory OffloadBinaryCategory("llvm-offload-binary options"); + +static cl::opt OutputFile("o", cl::desc("Write output to ."), + cl::value_desc("file"), + cl::cat(OffloadBinaryCategory)); + +static cl::opt InputFile(cl::Positional, + cl::desc("Extract from ."), + cl::value_desc("file"), + cl::cat(OffloadBinaryCategory)); + +static cl::list + DeviceImages("image", + cl::desc("List of key and value arguments. Required keywords " + "are 'file' and 'triple'."), + cl::value_desc("=,..."), + cl::cat(OffloadBinaryCategory)); + +static cl::opt + CreateArchive("archive", + cl::desc("Write extracted files to a static archive"), + cl::cat(OffloadBinaryCategory)); + +/// Path of the current binary. +static const char *PackagerExecutable; + +// Get a map containing all the arguments for the image. Repeated arguments will +// be placed in a comma separated list. +static DenseMap getImageArguments(StringRef Image, + StringSaver &Saver) { + DenseMap Args; + for (StringRef Arg : llvm::split(Image, ",")) { + auto [Key, Value] = Arg.split("="); + auto [It, Inserted] = Args.try_emplace(Key, Value); + if (!Inserted) + It->second = Saver.save(It->second + "," + Value); + } + + return Args; +} + +static Error writeFile(StringRef Filename, StringRef Data) { + Expected> OutputOrErr = + FileOutputBuffer::create(Filename, Data.size()); + if (!OutputOrErr) + return OutputOrErr.takeError(); + std::unique_ptr Output = std::move(*OutputOrErr); + llvm::copy(Data, Output->getBufferStart()); + if (Error E = Output->commit()) + return E; + return Error::success(); +} + +static Error bundleImages() { + SmallVector BinaryData; + raw_svector_ostream OS(BinaryData); + for (StringRef Image : DeviceImages) { + BumpPtrAllocator Alloc; + StringSaver Saver(Alloc); + DenseMap Args = getImageArguments(Image, Saver); + + if (!Args.count("triple") || !Args.count("file")) + return createStringError( + inconvertibleErrorCode(), + "'file' and 'triple' are required image arguments"); + + // Permit using multiple instances of `file` in a single string. + for (auto &File : llvm::split(Args["file"], ",")) { + OffloadBinary::OffloadingImage ImageBinary{}; + + llvm::ErrorOr> ObjectOrErr = + llvm::MemoryBuffer::getFileOrSTDIN(File); + if (std::error_code EC = ObjectOrErr.getError()) + return errorCodeToError(EC); + + // Clang uses the '.o' suffix for LTO bitcode. + if (identify_magic((*ObjectOrErr)->getBuffer()) == file_magic::bitcode) + ImageBinary.TheImageKind = object::IMG_Bitcode; + else if (sys::path::has_extension(File)) + ImageBinary.TheImageKind = + getImageKind(sys::path::extension(File).drop_front()); + else + ImageBinary.TheImageKind = IMG_None; + ImageBinary.Image = std::move(*ObjectOrErr); + for (const auto &[Key, Value] : Args) { + if (Key == "kind") { + ImageBinary.TheOffloadKind = getOffloadKind(Value); + } else if (Key != "file") { + ImageBinary.StringData[Key] = Value; + } + } + llvm::SmallString<0> Buffer = OffloadBinary::write(ImageBinary); + if (Buffer.size() % OffloadBinary::getAlignment() != 0) + return createStringError(inconvertibleErrorCode(), + "Offload binary has invalid size alignment"); + OS << Buffer; + } + } + + if (Error E = writeFile(OutputFile, + StringRef(BinaryData.begin(), BinaryData.size()))) + return E; + return Error::success(); +} + +static Error unbundleImages() { + ErrorOr> BufferOrErr = + MemoryBuffer::getFileOrSTDIN(InputFile); + if (std::error_code EC = BufferOrErr.getError()) + return createFileError(InputFile, EC); + std::unique_ptr Buffer = std::move(*BufferOrErr); + + // This data can be misaligned if extracted from an archive. + if (!isAddrAligned(Align(OffloadBinary::getAlignment()), + Buffer->getBufferStart())) + Buffer = MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), + Buffer->getBufferIdentifier()); + + SmallVector Binaries; + if (Error Err = extractOffloadBinaries(*Buffer, Binaries)) + return Err; + + // Try to extract each device image specified by the user from the input file. + for (StringRef Image : DeviceImages) { + BumpPtrAllocator Alloc; + StringSaver Saver(Alloc); + auto Args = getImageArguments(Image, Saver); + + SmallVector Extracted; + for (const OffloadFile &File : Binaries) { + const auto *Binary = File.getBinary(); + // We handle the 'file' and 'kind' identifiers differently. + bool Match = llvm::all_of(Args, [&](auto &Arg) { + const auto [Key, Value] = Arg; + if (Key == "file") + return true; + if (Key == "kind") + return Binary->getOffloadKind() == getOffloadKind(Value); + return Binary->getString(Key) == Value; + }); + if (Match) + Extracted.push_back(Binary); + } + + if (Extracted.empty()) + continue; + + if (CreateArchive) { + if (!Args.count("file")) + return createStringError(inconvertibleErrorCode(), + "Image must have a 'file' argument."); + + SmallVector Members; + for (const OffloadBinary *Binary : Extracted) + Members.emplace_back(MemoryBufferRef( + Binary->getImage(), + Binary->getMemoryBufferRef().getBufferIdentifier())); + + if (Error E = writeArchive( + Args["file"], Members, SymtabWritingMode::NormalSymtab, + Archive::getDefaultKind(), true, false, nullptr)) + return E; + } else if (auto It = Args.find("file"); It != Args.end()) { + if (Extracted.size() > 1) + WithColor::warning(errs(), PackagerExecutable) + << "Multiple inputs match to a single file, '" << It->second + << "'\n"; + if (Error E = writeFile(It->second, Extracted.back()->getImage())) + return E; + } else { + uint64_t Idx = 0; + for (const OffloadBinary *Binary : Extracted) { + StringRef Filename = + Saver.save(sys::path::stem(InputFile) + "-" + Binary->getTriple() + + "-" + Binary->getArch() + "." + std::to_string(Idx++) + + "." + getImageKindName(Binary->getImageKind())); + if (Error E = writeFile(Filename, Binary->getImage())) + return E; + } + } + } + + return Error::success(); +} + +int main(int argc, const char **argv) { + sys::PrintStackTraceOnErrorSignal(argv[0]); + cl::HideUnrelatedOptions(OffloadBinaryCategory); + cl::ParseCommandLineOptions( + argc, argv, + "A utility for bundling several object files into a single binary.\n" + "The output binary can then be embedded into the host section table\n" + "to create a fatbinary containing offloading code.\n"); + + if (sys::path::stem(argv[0]).ends_with("clang-offload-packager")) + WithColor::warning(errs(), PackagerExecutable) + << "'clang-offload-packager' is deprecated. Use 'llvm-offload-binary' " + "instead.\n"; + + if (Help || (OutputFile.empty() && InputFile.empty())) { + cl::PrintHelpMessage(); + return EXIT_SUCCESS; + } + + PackagerExecutable = argv[0]; + auto reportError = [argv](Error E) { + logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0])); + return EXIT_FAILURE; + }; + + if (!InputFile.empty() && !OutputFile.empty()) + return reportError( + createStringError(inconvertibleErrorCode(), + "Packaging to an output file and extracting from an " + "input file are mutually exclusive.")); + + if (!OutputFile.empty()) { + if (Error Err = bundleImages()) + return reportError(std::move(Err)); + } else if (!InputFile.empty()) { + if (Error Err = unbundleImages()) + return reportError(std::move(Err)); + } + + return EXIT_SUCCESS; +} -- cgit v1.1