From 047399c213a007f91b5d472cfe6742d5b7be70f3 Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:44:46 -0800 Subject: [PATCH 001/281] [mlir][sparse] cleanup of CodegenEnv reduction API (#75243) --- .../SparseTensor/Transforms/CodegenEnv.cpp | 27 +++++++++++-------- .../SparseTensor/Transforms/CodegenEnv.h | 9 ++++--- .../Transforms/Sparsification.cpp | 26 +++++++++--------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp index 312aefc0936c..4bd3af2d3f2f 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp @@ -115,10 +115,10 @@ std::optional CodegenEnv::genLoopBoundary( SmallVector params; if (isReduc()) { params.push_back(redVal); - if (redValidLexInsert) + if (isValidLexInsert()) params.push_back(redValidLexInsert); } else { - assert(!redValidLexInsert); + assert(!isValidLexInsert()); } if (isExpand()) params.push_back(expCount); @@ -128,8 +128,8 @@ std::optional CodegenEnv::genLoopBoundary( unsigned i = 0; if (isReduc()) { updateReduc(params[i++]); - if (redValidLexInsert) - setValidLexInsert(params[i++]); + if (isValidLexInsert()) + updateValidLexInsert(params[i++]); } if (isExpand()) updateExpandCount(params[i++]); @@ -235,14 +235,14 @@ void CodegenEnv::endExpand() { //===----------------------------------------------------------------------===// void CodegenEnv::startReduc(ExprId exp, Value val) { - assert(!isReduc() && exp != detail::kInvalidId); + assert(!isReduc() && exp != detail::kInvalidId && val); redExp = exp; redVal = val; latticeMerger.setExprValue(exp, val); } void CodegenEnv::updateReduc(Value val) { - assert(isReduc()); + assert(isReduc() && val); redVal = val; latticeMerger.clearExprValue(redExp); latticeMerger.setExprValue(redExp, val); @@ -257,13 +257,18 @@ Value CodegenEnv::endReduc() { return val; } -void CodegenEnv::setValidLexInsert(Value val) { - assert(isReduc() && val); +void CodegenEnv::startValidLexInsert(Value val) { + assert(!isValidLexInsert() && isReduc() && val); + redValidLexInsert = val; +} + +void CodegenEnv::updateValidLexInsert(Value val) { + assert(redValidLexInsert && isReduc() && val); redValidLexInsert = val; } -void CodegenEnv::clearValidLexInsert() { - assert(!isReduc()); +void CodegenEnv::endValidLexInsert() { + assert(isValidLexInsert() && !isReduc()); redValidLexInsert = Value(); } @@ -272,7 +277,7 @@ void CodegenEnv::startCustomReduc(ExprId exp) { redCustom = exp; } -Value CodegenEnv::getCustomRedId() { +Value CodegenEnv::getCustomRedId() const { assert(isCustomReduc()); return dyn_cast(exp(redCustom).op).getIdentity(); } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h index a1947f48393e..cd626041834b 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h @@ -150,13 +150,16 @@ public: void updateReduc(Value val); Value getReduc() const { return redVal; } Value endReduc(); - void setValidLexInsert(Value val); - void clearValidLexInsert(); + + void startValidLexInsert(Value val); + bool isValidLexInsert() const { return redValidLexInsert != nullptr; } + void updateValidLexInsert(Value val); Value getValidLexInsert() const { return redValidLexInsert; } + void endValidLexInsert(); void startCustomReduc(ExprId exp); bool isCustomReduc() const { return redCustom != detail::kInvalidId; } - Value getCustomRedId(); + Value getCustomRedId() const; void endCustomReduc(); private: diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 992be434fc62..2367d3b5f37a 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -415,9 +415,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t, SmallVector ivs = llvm::to_vector(llvm::drop_end( env.emitter().getLoopIVsRange(), env.getCurrentDepth() - numLoops)); Value chain = env.getInsertionChain(); - if (!env.getValidLexInsert()) { - env.updateInsertionChain(builder.create(loc, rhs, chain, ivs)); - } else { + if (env.isValidLexInsert()) { // Generates runtime check for a valid lex during reduction, // to avoid inserting the identity value for empty reductions. // if (validLexInsert) then @@ -438,6 +436,9 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t, // Value assignment. builder.setInsertionPointAfter(ifValidLexInsert); env.updateInsertionChain(ifValidLexInsert.getResult(0)); + } else { + // Generates regular insertion chain. + env.updateInsertionChain(builder.create(loc, rhs, chain, ivs)); } return; } @@ -688,12 +689,13 @@ static void genInvariants(CodegenEnv &env, OpBuilder &builder, ExprId exp, env.startReduc(exp, genTensorLoad(env, builder, exp)); } if (env.hasSparseOutput()) - env.setValidLexInsert(constantI1(builder, env.op().getLoc(), false)); + env.startValidLexInsert( + constantI1(builder, env.op().getLoc(), false)); } else { if (!env.isCustomReduc() || env.isReduc()) genTensorStore(env, builder, exp, env.endReduc()); if (env.hasSparseOutput()) - env.clearValidLexInsert(); + env.endValidLexInsert(); } } else { // Start or end loop invariant hoisting of a tensor load. @@ -846,9 +848,9 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder, if (env.isReduc()) { yields.push_back(env.getReduc()); env.updateReduc(ifOp.getResult(y++)); - if (env.getValidLexInsert()) { + if (env.isValidLexInsert()) { yields.push_back(env.getValidLexInsert()); - env.setValidLexInsert(ifOp.getResult(y++)); + env.updateValidLexInsert(ifOp.getResult(y++)); } } if (env.isExpand()) { @@ -904,7 +906,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr, }); if (env.isReduc()) { types.push_back(env.getReduc().getType()); - if (env.getValidLexInsert()) + if (env.isValidLexInsert()) types.push_back(env.getValidLexInsert().getType()); } if (env.isExpand()) @@ -924,10 +926,10 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp, if (env.isReduc()) { operands.push_back(env.getReduc()); env.updateReduc(redInput); - if (env.getValidLexInsert()) { + if (env.isValidLexInsert()) { // Any overlapping indices during a reduction creates a valid lex insert. operands.push_back(constantI1(builder, env.op().getLoc(), true)); - env.setValidLexInsert(validIns); + env.updateValidLexInsert(validIns); } } if (env.isExpand()) { @@ -1174,8 +1176,8 @@ static bool endLoop(CodegenEnv &env, RewriterBase &rewriter, Operation *loop, // Either a for-loop or a while-loop that iterates over a slice. if (isSingleCond) { // Any iteration creates a valid lex insert. - if (env.isReduc() && env.getValidLexInsert()) - env.setValidLexInsert(constantI1(rewriter, env.op().getLoc(), true)); + if (env.isReduc() && env.isValidLexInsert()) + env.updateValidLexInsert(constantI1(rewriter, env.op().getLoc(), true)); } else if (auto whileOp = dyn_cast(loop)) { // End a while-loop. finalizeWhileOp(env, rewriter, needsUniv); -- GitLab From 3959231695a088e1d8ca0bf8f9d2e5cf9c61a17a Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Tue, 12 Dec 2023 12:40:22 -0800 Subject: [PATCH 002/281] [X86][FastISel] Bail out on large objects when materializing a GlobalValue To avoid crashes with explicitly large objects. I will clean up fast-isel with large objects/medium code model soon. --- llvm/lib/Target/X86/X86FastISel.cpp | 6 ++++++ llvm/test/CodeGen/X86/fast-isel-large-object.ll | 13 +++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 llvm/test/CodeGen/X86/fast-isel-large-object.ll diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index 068119a6332a..bdc9d1d42dd1 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -714,6 +714,12 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) { if (TM.getCodeModel() != CodeModel::Small) return false; + // Can't handle large objects yet. + if (auto *GO = dyn_cast(GV)) { + if (TM.isLargeGlobalObject(GO)) + return false; + } + // Can't handle TLS yet. if (GV->isThreadLocal()) return false; diff --git a/llvm/test/CodeGen/X86/fast-isel-large-object.ll b/llvm/test/CodeGen/X86/fast-isel-large-object.ll new file mode 100644 index 000000000000..6ca2c4240723 --- /dev/null +++ b/llvm/test/CodeGen/X86/fast-isel-large-object.ll @@ -0,0 +1,13 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -fast-isel -mtriple=x86_64-unknown-unknown -relocation-model=pic < %s | FileCheck %s + +@g = external dso_local global i32, code_model "large" + +define ptr @f() { +; CHECK-LABEL: f: +; CHECK: # %bb.0: +; CHECK-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; CHECK-NEXT: leaq g@GOTOFF(%rax), %rax +; CHECK-NEXT: retq + ret ptr @g +} -- GitLab From 61ee9232569d84ae5572eafd2b8098e63a5c5c50 Mon Sep 17 00:00:00 2001 From: Abhina Sree <69635948+abhina-sree@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:49:11 -0500 Subject: [PATCH 003/281] [SystemZ][z/OS] Fix build errors on z/OS in the Unix .inc files (#74758) This patch resolves the following errors on z/OS: error: no member named 'wait4' in the global namespace error: no member named 'ru_maxrss' in 'rusage' error: use of undeclared identifier 'strsignal' error: Cannot get usage times on this platform error: Cannot get malloc info on this platform --- .../include/llvm/Support/SystemZ/zOSSupport.h | 39 +++++++++++++++++++ llvm/lib/Support/Unix/Process.inc | 4 ++ llvm/lib/Support/Unix/Program.inc | 3 +- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/Support/SystemZ/zOSSupport.h diff --git a/llvm/include/llvm/Support/SystemZ/zOSSupport.h b/llvm/include/llvm/Support/SystemZ/zOSSupport.h new file mode 100644 index 000000000000..ee78147cb215 --- /dev/null +++ b/llvm/include/llvm/Support/SystemZ/zOSSupport.h @@ -0,0 +1,39 @@ +//===- zOSSupport.h - Common z/OS Include File ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines z/OS implementations for common functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_ZOSSUPPORT_H +#define LLVM_SUPPORT_ZOSSUPPORT_H + +#ifdef __MVS__ +#include +#include + +// z/OS Unix System Services does not have strsignal() support, so the +// strsignal() function is implemented here. +inline char *strsignal(int sig) { + static char msg[256]; + sprintf(msg, "%d", sig); + return msg; +} + +// z/OS Unix System Services does not have wait4() support, so the wait4 +// function is implemented here. +inline pid_t wait4(pid_t pid, int *wstatus, int options, + struct rusage *rusage) { + pid_t Result = waitpid(pid, wstatus, options); + int GetrusageRC = getrusage(RUSAGE_CHILDREN, rusage); + assert(!GetrusageRC && "Must have valid measure of the resources!"); + return Result; +} + +#endif +#endif diff --git a/llvm/lib/Support/Unix/Process.inc b/llvm/lib/Support/Unix/Process.inc index 2babf07944bf..551f0d7f0f02 100644 --- a/llvm/lib/Support/Unix/Process.inc +++ b/llvm/lib/Support/Unix/Process.inc @@ -62,7 +62,9 @@ getRUsageTimes() { ::getrusage(RUSAGE_SELF, &RU); return {toDuration(RU.ru_utime), toDuration(RU.ru_stime)}; #else +#ifndef __MVS__ // Exclude for MVS in case -pedantic is used #warning Cannot get usage times on this platform +#endif return {std::chrono::microseconds::zero(), std::chrono::microseconds::zero()}; #endif } @@ -117,7 +119,9 @@ size_t Process::GetMallocUsage() { return EndOfMemory - StartOfMemory; return 0; #else +#ifndef __MVS__ // Exclude for MVS in case -pedantic is used #warning Cannot get malloc info on this platform +#endif return 0; #endif } diff --git a/llvm/lib/Support/Unix/Program.inc b/llvm/lib/Support/Unix/Program.inc index 9466d0f0ba85..895fdfc711e5 100644 --- a/llvm/lib/Support/Unix/Program.inc +++ b/llvm/lib/Support/Unix/Program.inc @@ -25,6 +25,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/SystemZ/zOSSupport.h" #include "llvm/Support/raw_ostream.h" #if HAVE_SYS_STAT_H #include @@ -466,7 +467,7 @@ ProcessInfo llvm::sys::Wait(const ProcessInfo &PI, std::chrono::microseconds UserT = toDuration(Info.ru_utime); std::chrono::microseconds KernelT = toDuration(Info.ru_stime); uint64_t PeakMemory = 0; -#ifndef __HAIKU__ +#if !defined(__HAIKU__) && !defined(__MVS__) PeakMemory = static_cast(Info.ru_maxrss); #endif *ProcStat = ProcessStatistics{UserT + KernelT, UserT, PeakMemory}; -- GitLab From fe6f137e48ceee094d0fa42ca54c7e1226b45fde Mon Sep 17 00:00:00 2001 From: Johannes Doerfert Date: Tue, 12 Dec 2023 12:49:46 -0800 Subject: [PATCH 004/281] [OpenMP][NFC] Move mapping related code into OpenMP/Mapping.cpp (#75239) DeviceTy provides an abstraction for "middle-level" operations that can be done with a offload device. Mapping was tied into it but is not strictly necessary. Other languages do not track mapping, and even OpenMP can be used completely without mapping. This simply moves the relevant code into the OpenMP/Mapping.cpp as part of a new class MappingInfoTy. Each device still has one, but it does not clutter the device.cpp anymore. --- openmp/libomptarget/include/ExclusiveAccess.h | 1 + openmp/libomptarget/include/OpenMP/Mapping.h | 91 ++++ openmp/libomptarget/include/device.h | 96 +--- openmp/libomptarget/src/OpenMP/API.cpp | 19 +- openmp/libomptarget/src/OpenMP/Mapping.cpp | 488 ++++++++++++++++- openmp/libomptarget/src/device.cpp | 507 +----------------- openmp/libomptarget/src/omptarget.cpp | 50 +- 7 files changed, 642 insertions(+), 610 deletions(-) diff --git a/openmp/libomptarget/include/ExclusiveAccess.h b/openmp/libomptarget/include/ExclusiveAccess.h index 33f59903637d..09b3aac6059d 100644 --- a/openmp/libomptarget/include/ExclusiveAccess.h +++ b/openmp/libomptarget/include/ExclusiveAccess.h @@ -11,6 +11,7 @@ #ifndef OMPTARGET_EXCLUSIVE_ACCESS #define OMPTARGET_EXCLUSIVE_ACCESS +#include #include #include #include diff --git a/openmp/libomptarget/include/OpenMP/Mapping.h b/openmp/libomptarget/include/OpenMP/Mapping.h index 9a1ecb808792..4bd676fc658a 100644 --- a/openmp/libomptarget/include/OpenMP/Mapping.h +++ b/openmp/libomptarget/include/OpenMP/Mapping.h @@ -13,6 +13,7 @@ #ifndef OMPTARGET_OPENMP_MAPPING_H #define OMPTARGET_OPENMP_MAPPING_H +#include "ExclusiveAccess.h" #include "Shared/EnvironmentVar.h" #include "omptarget.h" @@ -443,4 +444,94 @@ int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, void **ArgMappers, AsyncInfoTy &AsyncInfo, bool FromMapper = false); +struct MappingInfoTy { + MappingInfoTy(DeviceTy &Device) : Device(Device) {} + + /// Host data to device map type with a wrapper key indirection that allows + /// concurrent modification of the entries without invalidating the underlying + /// entries. + using HostDataToTargetListTy = + std::set>; + + /// The HDTTMap is a protected object that can only be accessed by one thread + /// at a time. + ProtectedObj HostDataToTargetMap; + + /// The type used to access the HDTT map. + using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy; + + /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures + /// exclusive access to the HDTT map. + LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, + int64_t Size, + HostDataToTargetTy *OwnedTPR = nullptr); + + /// Get the target pointer based on host pointer begin and base. If the + /// mapping already exists, the target pointer will be returned directly. In + /// addition, if required, the memory region pointed by \p HstPtrBegin of size + /// \p Size will also be transferred to the device. If the mapping doesn't + /// exist, and if unified shared memory is not enabled, a new mapping will be + /// created and the data will also be transferred accordingly. nullptr will be + /// returned because of any of following reasons: + /// - Data allocation failed; + /// - The user tried to do an illegal mapping; + /// - Data transfer issue fails. + TargetPointerResultTy getTargetPointer( + HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase, + int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, + bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, + bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier, + AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr, + bool ReleaseHDTTMap = true); + + /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor + /// ensures exclusive access to the HDTT map. + void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, + int64_t Size); + + /// Return the target pointer begin (where the data will be moved). + /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target. + /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry + /// reference counters will be decremented. + /// - \p MustContain enforces that the query must not extend beyond an already + /// mapped entry to be valid. + /// - \p ForceDelete deletes the entry regardless of its reference counting + /// (unless it is infinite). + /// - \p FromDataEnd tracks the number of threads referencing the entry at + /// targetDataEnd for delayed deletion purpose. + [[nodiscard]] TargetPointerResultTy + getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount, + bool UseHoldRefCount, bool MustContain = false, + bool ForceDelete = false, bool FromDataEnd = false); + + /// Remove the \p Entry from the data map. Expect the entry's total reference + /// count to be zero and the caller thread to be the last one using it. \p + /// HDTTMap ensure the caller holds exclusive access and can modify the map. + /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c + /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling + /// this function if the map entry is not expected to exist because \p + /// HstPtrBegin uses shared memory. + [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap, + HostDataToTargetTy *Entry, int64_t Size); + + /// Deallocate the \p Entry from the device memory and delete it. Return \c + /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and + /// return \c OFFLOAD_FAIL otherwise. + [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, + int64_t Size); + + int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size); + int disassociatePtr(void *HstPtrBegin); + + /// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice + /// versa if \p H2D is false). If there is an existing mapping, or if \p Entry + /// is set, the associated metadata will be printed as well. + void printCopyInfo(void *TgtPtr, void *HstPtr, int64_t Size, bool H2D, + HostDataToTargetTy *Entry, + MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr); + +private: + DeviceTy &Device; +}; + #endif // OMPTARGET_OPENMP_MAPPING_H diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h index a84551accaf9..d28d3c508faf 100644 --- a/openmp/libomptarget/include/device.h +++ b/openmp/libomptarget/include/device.h @@ -53,19 +53,6 @@ struct DeviceTy { bool HasMappedGlobalData = false; - /// Host data to device map type with a wrapper key indirection that allows - /// concurrent modification of the entries without invalidating the underlying - /// entries. - using HostDataToTargetListTy = - std::set>; - - /// The HDTTMap is a protected object that can only be accessed by one thread - /// at a time. - ProtectedObj HostDataToTargetMap; - - /// The type used to access the HDTT map. - using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy; - PendingCtorsDtorsPerLibrary PendingCtorsDtors; std::mutex PendingGlobalsMtx; @@ -80,71 +67,8 @@ struct DeviceTy { /// Try to initialize the device and return any failure. llvm::Error init(); - // Return true if data can be copied to DstDevice directly - bool isDataExchangable(const DeviceTy &DstDevice); - - /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures - /// exclusive access to the HDTT map. - LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, - int64_t Size, - HostDataToTargetTy *OwnedTPR = nullptr); - - /// Get the target pointer based on host pointer begin and base. If the - /// mapping already exists, the target pointer will be returned directly. In - /// addition, if required, the memory region pointed by \p HstPtrBegin of size - /// \p Size will also be transferred to the device. If the mapping doesn't - /// exist, and if unified shared memory is not enabled, a new mapping will be - /// created and the data will also be transferred accordingly. nullptr will be - /// returned because of any of following reasons: - /// - Data allocation failed; - /// - The user tried to do an illegal mapping; - /// - Data transfer issue fails. - TargetPointerResultTy getTargetPointer( - HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase, - int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, - bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, - bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier, - AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr, - bool ReleaseHDTTMap = true); - - /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor - /// ensures exclusive access to the HDTT map. - void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, - int64_t Size); - - /// Return the target pointer begin (where the data will be moved). - /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target. - /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry - /// reference counters will be decremented. - /// - \p MustContain enforces that the query must not extend beyond an already - /// mapped entry to be valid. - /// - \p ForceDelete deletes the entry regardless of its reference counting - /// (unless it is infinite). - /// - \p FromDataEnd tracks the number of threads referencing the entry at - /// targetDataEnd for delayed deletion purpose. - [[nodiscard]] TargetPointerResultTy - getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount, - bool UseHoldRefCount, bool MustContain = false, - bool ForceDelete = false, bool FromDataEnd = false); - - /// Remove the \p Entry from the data map. Expect the entry's total reference - /// count to be zero and the caller thread to be the last one using it. \p - /// HDTTMap ensure the caller holds exclusive access and can modify the map. - /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c - /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling - /// this function if the map entry is not expected to exist because \p - /// HstPtrBegin uses shared memory. - [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap, - HostDataToTargetTy *Entry, int64_t Size); - - /// Deallocate the \p Entry from the device memory and delete it. Return \c - /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and - /// return \c OFFLOAD_FAIL otherwise. - [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, - int64_t Size); - - int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size); - int disassociatePtr(void *HstPtrBegin); + /// Provide access to the mapping handler. + MappingInfoTy &getMappingInfo() { return MappingInfo; } __tgt_target_table *loadBinary(__tgt_device_image *Img); @@ -159,6 +83,7 @@ struct DeviceTy { /// be used (host, shared, device). void *allocData(int64_t Size, void *HstPtr = nullptr, int32_t Kind = TARGET_ALLOC_DEFAULT); + /// Deallocates memory which \p TgtPtrBegin points at and returns /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. p Kind dictates what /// allocator should be used (host, shared, device). @@ -170,12 +95,16 @@ struct DeviceTy { int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry = nullptr, - DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr); + MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr); + // Copy data from device back to host int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size, AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry = nullptr, - DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr); + MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr); + + // Return true if data can be copied to DstDevice directly + bool isDataExchangable(const DeviceTy &DstDevice); // Copy data from current device to destination device directly int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, @@ -240,7 +169,12 @@ private: void deinit(); /// All offload entries available on this device. - llvm::DenseMap DeviceOffloadEntries; + using DeviceOffloadEntriesMapTy = + llvm::DenseMap; + ProtectedObj DeviceOffloadEntries; + + /// Handler to collect and organize host-2-device mapping information. + MappingInfoTy MappingInfo; }; #endif diff --git a/openmp/libomptarget/src/OpenMP/API.cpp b/openmp/libomptarget/src/OpenMP/API.cpp index b73315f2b77a..1769404faf88 100644 --- a/openmp/libomptarget/src/OpenMP/API.cpp +++ b/openmp/libomptarget/src/OpenMP/API.cpp @@ -150,9 +150,9 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) { // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero // lengh array) is mapped instead of the referred storage. TargetPointerResultTy TPR = - DeviceOrErr->getTgtPtrBegin(const_cast(Ptr), 1, - /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false); + DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast(Ptr), 1, + /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); int Rc = TPR.isPresent(); DP("Call to omp_target_is_present returns %d\n", Rc); return Rc; @@ -544,8 +544,8 @@ EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr, FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset); - int Rc = DeviceOrErr->associatePtr(const_cast(HostPtr), - const_cast(DeviceAddr), Size); + int Rc = DeviceOrErr->getMappingInfo().associatePtr( + const_cast(HostPtr), const_cast(DeviceAddr), Size); DP("omp_target_associate_ptr returns %d\n", Rc); return Rc; } @@ -571,7 +571,8 @@ EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) { if (!DeviceOrErr) FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); - int Rc = DeviceOrErr->disassociatePtr(const_cast(HostPtr)); + int Rc = DeviceOrErr->getMappingInfo().disassociatePtr( + const_cast(HostPtr)); DP("omp_target_disassociate_ptr returns %d\n", Rc); return Rc; } @@ -603,9 +604,9 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) { FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); TargetPointerResultTy TPR = - DeviceOrErr->getTgtPtrBegin(const_cast(Ptr), 1, - /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false); + DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast(Ptr), 1, + /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); if (!TPR.isPresent()) { DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n", DPxPTR(Ptr), DeviceNum); diff --git a/openmp/libomptarget/src/OpenMP/Mapping.cpp b/openmp/libomptarget/src/OpenMP/Mapping.cpp index c7cb5b6b1e4c..a5c24810e0af 100644 --- a/openmp/libomptarget/src/OpenMP/Mapping.cpp +++ b/openmp/libomptarget/src/OpenMP/Mapping.cpp @@ -10,13 +10,15 @@ #include "OpenMP/Mapping.h" +#include "PluginManager.h" #include "Shared/Debug.h" +#include "Shared/Requirements.h" #include "device.h" /// Dump a table of all the host-target pointer pairs on failure void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) { - DeviceTy::HDTTMapAccessorTy HDTTMap = - Device.HostDataToTargetMap.getExclusiveAccessor(); + MappingInfoTy::HDTTMapAccessorTy HDTTMap = + Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor(); if (HDTTMap->empty()) return; @@ -38,3 +40,485 @@ void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) { Info.getLine(), Info.getColumn()); } } + +int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, + int64_t Size) { + HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); + + // Check if entry exists + auto It = HDTTMap->find(HstPtrBegin); + if (It != HDTTMap->end()) { + HostDataToTargetTy &HDTT = *It->HDTT; + std::lock_guard LG(HDTT); + // Mapping already exists + bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size && + HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin; + if (IsValid) { + DP("Attempt to re-associate the same device ptr+offset with the same " + "host ptr, nothing to do\n"); + return OFFLOAD_SUCCESS; + } + REPORT("Not allowed to re-associate a different device ptr+offset with " + "the same host ptr\n"); + return OFFLOAD_FAIL; + } + + // Mapping does not exist, allocate it with refCount=INF + const HostDataToTargetTy &NewEntry = + *HDTTMap + ->emplace(new HostDataToTargetTy( + /*HstPtrBase=*/(uintptr_t)HstPtrBegin, + /*HstPtrBegin=*/(uintptr_t)HstPtrBegin, + /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size, + /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin, + /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin, + /*UseHoldRefCount=*/false, /*Name=*/nullptr, + /*IsRefCountINF=*/true)) + .first->HDTT; + DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD + ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, " + "HoldRefCount=%s\n", + DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin), + DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin), + NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str()); + (void)NewEntry; + + // Notify the plugin about the new mapping. + return Device.notifyDataMapped(HstPtrBegin, Size); +} + +int MappingInfoTy::disassociatePtr(void *HstPtrBegin) { + HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); + + auto It = HDTTMap->find(HstPtrBegin); + if (It == HDTTMap->end()) { + REPORT("Association not found\n"); + return OFFLOAD_FAIL; + } + // Mapping exists + HostDataToTargetTy &HDTT = *It->HDTT; + std::lock_guard LG(HDTT); + + if (HDTT.getHoldRefCount()) { + // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657: + // "It is an error to call acc_unmap_data if the structured reference + // count for the pointer is not zero." + REPORT("Trying to disassociate a pointer with a non-zero hold reference " + "count\n"); + return OFFLOAD_FAIL; + } + + if (HDTT.isDynRefCountInf()) { + DP("Association found, removing it\n"); + void *Event = HDTT.getEvent(); + delete &HDTT; + if (Event) + Device.destroyEvent(Event); + HDTTMap->erase(It); + return Device.notifyDataUnmapped(HstPtrBegin); + } + + REPORT("Trying to disassociate a pointer which was not mapped via " + "omp_target_associate_ptr\n"); + return OFFLOAD_FAIL; +} + +LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap, + void *HstPtrBegin, int64_t Size, + HostDataToTargetTy *OwnedTPR) { + + uintptr_t HP = (uintptr_t)HstPtrBegin; + LookupResult LR; + + DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n", + DPxPTR(HP), Size); + + if (HDTTMap->empty()) + return LR; + + auto Upper = HDTTMap->upper_bound(HP); + + if (Size == 0) { + // specification v5.1 Pointer Initialization for Device Data Environments + // upper_bound satisfies + // std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin + if (Upper != HDTTMap->begin()) { + LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR); + // the left side of extended address range is satisified. + // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >= + // LR.TPR.getEntry()->HstPtrBase + LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd || + HP < LR.TPR.getEntry()->HstPtrBase; + } + + if (!LR.Flags.IsContained && Upper != HDTTMap->end()) { + LR.TPR.setEntry(Upper->HDTT, OwnedTPR); + // the right side of extended address range is satisified. + // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase + LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase; + } + } else { + // check the left bin + if (Upper != HDTTMap->begin()) { + LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR); + // Is it contained? + LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin && + HP < LR.TPR.getEntry()->HstPtrEnd && + (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd; + // Does it extend beyond the mapped region? + LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd && + (HP + Size) > LR.TPR.getEntry()->HstPtrEnd; + } + + // check the right bin + if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) && + Upper != HDTTMap->end()) { + LR.TPR.setEntry(Upper->HDTT, OwnedTPR); + // Does it extend into an already mapped region? + LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin && + (HP + Size) > LR.TPR.getEntry()->HstPtrBegin; + // Does it extend beyond the mapped region? + LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd && + (HP + Size) > LR.TPR.getEntry()->HstPtrEnd; + } + + if (LR.Flags.ExtendsBefore) { + DP("WARNING: Pointer is not mapped but section extends into already " + "mapped data\n"); + } + if (LR.Flags.ExtendsAfter) { + DP("WARNING: Pointer is already mapped but section extends beyond mapped " + "region\n"); + } + } + + return LR; +} + +TargetPointerResultTy MappingInfoTy::getTargetPointer( + HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase, + int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo, + bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, + bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier, + AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) { + + LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR); + LR.TPR.Flags.IsPresent = true; + + // Release the mapping table lock only after the entry is locked by + // attaching it to TPR. Once TPR is destroyed it will release the lock + // on entry. If it is returned the lock will move to the returned object. + // If LR.Entry is already owned/locked we avoid trying to lock it again. + + // Check if the pointer is contained. + // If a variable is mapped to the device manually by the user - which would + // lead to the IsContained flag to be true - then we must ensure that the + // device address is returned even under unified memory conditions. + if (LR.Flags.IsContained || + ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) { + const char *RefCountAction; + if (UpdateRefCount) { + // After this, reference count >= 1. If the reference count was 0 but the + // entry was still there we can reuse the data on the device and avoid a + // new submission. + LR.TPR.getEntry()->incRefCount(HasHoldModifier); + RefCountAction = " (incremented)"; + } else { + // It might have been allocated with the parent, but it's still new. + LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1; + RefCountAction = " (update suppressed)"; + } + const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction; + const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : ""; + uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin + + ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin); + INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID, + "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD + ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n", + (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr), + Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction, + LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction, + (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); + LR.TPR.TargetPointer = (void *)Ptr; + } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) { + // Explicit extension of mapped data - not allowed. + MESSAGE("explicit extension not allowed: host address specified is " DPxMOD + " (%" PRId64 + " bytes), but device allocation maps to host at " DPxMOD + " (%" PRId64 " bytes)", + DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin), + LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin); + if (HasPresentModifier) + MESSAGE("device mapping required by 'present' map type modifier does not " + "exist for host address " DPxMOD " (%" PRId64 " bytes)", + DPxPTR(HstPtrBegin), Size); + } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY && + !HasCloseModifier) { + // If unified shared memory is active, implicitly mapped variables that are + // not privatized use host address. Any explicitly mapped variables also use + // host address where correctness is not impeded. In all other cases maps + // are respected. + // In addition to the mapping rules above, the close map modifier forces the + // mapping of the variable to the device. + if (Size) { + DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " + "memory\n", + DPxPTR((uintptr_t)HstPtrBegin), Size); + LR.TPR.Flags.IsPresent = false; + LR.TPR.Flags.IsHostPointer = true; + LR.TPR.TargetPointer = HstPtrBegin; + } + } else if (HasPresentModifier) { + DP("Mapping required by 'present' map type modifier does not exist for " + "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n", + DPxPTR(HstPtrBegin), Size); + MESSAGE("device mapping required by 'present' map type modifier does not " + "exist for host address " DPxMOD " (%" PRId64 " bytes)", + DPxPTR(HstPtrBegin), Size); + } else if (Size) { + // If it is not contained and Size > 0, we should create a new entry for it. + LR.TPR.Flags.IsNewEntry = true; + uintptr_t TgtAllocBegin = + (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin); + uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding; + // Release the mapping table lock only after the entry is locked by + // attaching it to TPR. + LR.TPR.setEntry(HDTTMap + ->emplace(new HostDataToTargetTy( + (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin, + (uintptr_t)HstPtrBegin + Size, TgtAllocBegin, + TgtPtrBegin, HasHoldModifier, HstPtrName)) + .first->HDTT); + INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID, + "Creating new map entry with HstPtrBase=" DPxMOD + ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD + ", TgtPtrBegin=" DPxMOD + ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n", + DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin), + DPxPTR(TgtPtrBegin), Size, + LR.TPR.getEntry()->dynRefCountToStr().c_str(), + LR.TPR.getEntry()->holdRefCountToStr().c_str(), + (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); + LR.TPR.TargetPointer = (void *)TgtPtrBegin; + + // Notify the plugin about the new mapping. + if (Device.notifyDataMapped(HstPtrBegin, Size)) + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + nullptr /* Entry */, + nullptr /* TargetPointer */}; + } else { + // This entry is not present and we did not create a new entry for it. + LR.TPR.Flags.IsPresent = false; + } + + // All mapping table modifications have been made. If the user requested it we + // give up the lock. + if (ReleaseHDTTMap) + HDTTMap.destroy(); + + // If the target pointer is valid, and we need to transfer data, issue the + // data transfer. + if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo && + (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) { + DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size, + DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer)); + + int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size, + AsyncInfo, LR.TPR.getEntry()); + if (Ret != OFFLOAD_SUCCESS) { + REPORT("Copying data to device failed.\n"); + // We will also return nullptr if the data movement fails because that + // pointer points to a corrupted memory region so it doesn't make any + // sense to continue to use it. + LR.TPR.TargetPointer = nullptr; + } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) != + OFFLOAD_SUCCESS) + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + nullptr /* Entry */, + nullptr /* TargetPointer */}; + } else { + // If not a host pointer and no present modifier, we need to wait for the + // event if it exists. + // Note: Entry might be nullptr because of zero length array section. + if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer && + !HasPresentModifier) { + void *Event = LR.TPR.getEntry()->getEvent(); + if (Event) { + int Ret = Device.waitEvent(Event, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // If it fails to wait for the event, we need to return nullptr in + // case of any data race. + REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + nullptr /* Entry */, + nullptr /* TargetPointer */}; + } + } + } + } + + return std::move(LR.TPR); +} + +TargetPointerResultTy MappingInfoTy::getTgtPtrBegin( + void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount, + bool MustContain, bool ForceDelete, bool FromDataEnd) { + HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); + + LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size); + + LR.TPR.Flags.IsPresent = true; + + if (LR.Flags.IsContained || + (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) { + LR.TPR.Flags.IsLast = + LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete); + + if (ForceDelete) { + LR.TPR.getEntry()->resetRefCount(UseHoldRefCount); + assert(LR.TPR.Flags.IsLast == + LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) && + "expected correct IsLast prediction for reset"); + } + + // Increment the number of threads that is using the entry on a + // targetDataEnd, tracking the number of possible "deleters". A thread may + // come to own the entry deletion even if it was not the last one querying + // for it. Thus, we must track every query on targetDataEnds to ensure only + // the last thread that holds a reference to an entry actually deletes it. + if (FromDataEnd) + LR.TPR.getEntry()->incDataEndThreadCount(); + + const char *RefCountAction; + if (!UpdateRefCount) { + RefCountAction = " (update suppressed)"; + } else if (LR.TPR.Flags.IsLast) { + LR.TPR.getEntry()->decRefCount(UseHoldRefCount); + assert(LR.TPR.getEntry()->getTotalRefCount() == 0 && + "Expected zero reference count when deletion is scheduled"); + if (ForceDelete) + RefCountAction = " (reset, delayed deletion)"; + else + RefCountAction = " (decremented, delayed deletion)"; + } else { + LR.TPR.getEntry()->decRefCount(UseHoldRefCount); + RefCountAction = " (decremented)"; + } + const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction; + const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : ""; + uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin + + ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin); + INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID, + "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", " + "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n", + DPxPTR(HstPtrBegin), DPxPTR(TP), Size, + LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction, + LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction); + LR.TPR.TargetPointer = (void *)TP; + } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) { + // If the value isn't found in the mapping and unified shared memory + // is on then it means we have stumbled upon a value which we need to + // use directly from the host. + DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " + "memory\n", + DPxPTR((uintptr_t)HstPtrBegin), Size); + LR.TPR.Flags.IsPresent = false; + LR.TPR.Flags.IsHostPointer = true; + LR.TPR.TargetPointer = HstPtrBegin; + } else { + // OpenMP Specification v5.2: if a matching list item is not found, the + // pointer retains its original value as per firstprivate semantics. + LR.TPR.Flags.IsPresent = false; + LR.TPR.Flags.IsHostPointer = false; + LR.TPR.TargetPointer = HstPtrBegin; + } + + return std::move(LR.TPR); +} + +// Return the target pointer begin (where the data will be moved). +void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, + void *HstPtrBegin, int64_t Size) { + uintptr_t HP = (uintptr_t)HstPtrBegin; + LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size); + if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) { + uintptr_t TP = + LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin); + return (void *)TP; + } + + return NULL; +} + +int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap, + HostDataToTargetTy *Entry, int64_t Size) { + assert(Entry && "Trying to delete a null entry from the HDTT map."); + assert(Entry->getTotalRefCount() == 0 && + Entry->getDataEndThreadCount() == 0 && + "Trying to delete entry that is in use or owned by another thread."); + + INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID, + "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD + ", Size=%" PRId64 ", Name=%s\n", + DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size, + (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str() + : "unknown"); + + if (HDTTMap->erase(Entry) == 0) { + REPORT("Trying to remove a non-existent map entry\n"); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; +} + +int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, + int64_t Size) { + assert(Entry && "Trying to deallocate a null entry."); + + DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation " + "starting at " DPxMOD "\n", + DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin)); + + void *Event = Entry->getEvent(); + if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) { + REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event)); + return OFFLOAD_FAIL; + } + + int Ret = Device.deleteData((void *)Entry->TgtAllocBegin); + + // Notify the plugin about the unmapped memory. + Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin); + + delete Entry; + + return Ret; +} + +static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin, + void *DstPtrBegin, int64_t Size, + HostDataToTargetTy *HT) { + + INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId, + "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD + ", Size=%" PRId64 ", Name=%s\n", + H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt", + DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size, + (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str() + : "unknown"); +} + +void MappingInfoTy::printCopyInfo( + void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D, + HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) { + auto HDTTMap = + HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr); + LookupResult LR; + if (!Entry) { + LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size); + Entry = LR.TPR.getEntry(); + } + printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size, + Entry); +} diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp index 01fc32328876..dbad13b92bcc 100644 --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -12,6 +12,7 @@ #include "device.h" #include "OffloadEntry.h" +#include "OpenMP/Mapping.h" #include "OpenMP/OMPT/Callback.h" #include "OpenMP/OMPT/Interface.h" #include "PluginManager.h" @@ -65,7 +66,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID) : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID), - PendingCtorsDtors(), PendingGlobalsMtx() {} + PendingCtorsDtors(), PendingGlobalsMtx(), MappingInfo(*this) {} DeviceTy::~DeviceTy() { if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE)) @@ -75,460 +76,6 @@ DeviceTy::~DeviceTy() { dumpTargetPointerMappings(&Loc, *this); } -int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) { - HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); - - // Check if entry exists - auto It = HDTTMap->find(HstPtrBegin); - if (It != HDTTMap->end()) { - HostDataToTargetTy &HDTT = *It->HDTT; - std::lock_guard LG(HDTT); - // Mapping already exists - bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size && - HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin; - if (IsValid) { - DP("Attempt to re-associate the same device ptr+offset with the same " - "host ptr, nothing to do\n"); - return OFFLOAD_SUCCESS; - } - REPORT("Not allowed to re-associate a different device ptr+offset with " - "the same host ptr\n"); - return OFFLOAD_FAIL; - } - - // Mapping does not exist, allocate it with refCount=INF - const HostDataToTargetTy &NewEntry = - *HDTTMap - ->emplace(new HostDataToTargetTy( - /*HstPtrBase=*/(uintptr_t)HstPtrBegin, - /*HstPtrBegin=*/(uintptr_t)HstPtrBegin, - /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size, - /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin, - /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin, - /*UseHoldRefCount=*/false, /*Name=*/nullptr, - /*IsRefCountINF=*/true)) - .first->HDTT; - DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD - ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, " - "HoldRefCount=%s\n", - DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin), - DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin), - NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str()); - (void)NewEntry; - - // Notify the plugin about the new mapping. - return notifyDataMapped(HstPtrBegin, Size); -} - -int DeviceTy::disassociatePtr(void *HstPtrBegin) { - HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); - - auto It = HDTTMap->find(HstPtrBegin); - if (It == HDTTMap->end()) { - REPORT("Association not found\n"); - return OFFLOAD_FAIL; - } - // Mapping exists - HostDataToTargetTy &HDTT = *It->HDTT; - std::lock_guard LG(HDTT); - - if (HDTT.getHoldRefCount()) { - // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657: - // "It is an error to call acc_unmap_data if the structured reference - // count for the pointer is not zero." - REPORT("Trying to disassociate a pointer with a non-zero hold reference " - "count\n"); - return OFFLOAD_FAIL; - } - - if (HDTT.isDynRefCountInf()) { - DP("Association found, removing it\n"); - void *Event = HDTT.getEvent(); - delete &HDTT; - if (Event) - destroyEvent(Event); - HDTTMap->erase(It); - return notifyDataUnmapped(HstPtrBegin); - } - - REPORT("Trying to disassociate a pointer which was not mapped via " - "omp_target_associate_ptr\n"); - return OFFLOAD_FAIL; -} - -LookupResult DeviceTy::lookupMapping(HDTTMapAccessorTy &HDTTMap, - void *HstPtrBegin, int64_t Size, - HostDataToTargetTy *OwnedTPR) { - - uintptr_t HP = (uintptr_t)HstPtrBegin; - LookupResult LR; - - DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n", - DPxPTR(HP), Size); - - if (HDTTMap->empty()) - return LR; - - auto Upper = HDTTMap->upper_bound(HP); - - if (Size == 0) { - // specification v5.1 Pointer Initialization for Device Data Environments - // upper_bound satisfies - // std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin - if (Upper != HDTTMap->begin()) { - LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR); - // the left side of extended address range is satisified. - // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >= - // LR.TPR.getEntry()->HstPtrBase - LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd || - HP < LR.TPR.getEntry()->HstPtrBase; - } - - if (!LR.Flags.IsContained && Upper != HDTTMap->end()) { - LR.TPR.setEntry(Upper->HDTT, OwnedTPR); - // the right side of extended address range is satisified. - // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase - LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase; - } - } else { - // check the left bin - if (Upper != HDTTMap->begin()) { - LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR); - // Is it contained? - LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin && - HP < LR.TPR.getEntry()->HstPtrEnd && - (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd; - // Does it extend beyond the mapped region? - LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd && - (HP + Size) > LR.TPR.getEntry()->HstPtrEnd; - } - - // check the right bin - if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) && - Upper != HDTTMap->end()) { - LR.TPR.setEntry(Upper->HDTT, OwnedTPR); - // Does it extend into an already mapped region? - LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin && - (HP + Size) > LR.TPR.getEntry()->HstPtrBegin; - // Does it extend beyond the mapped region? - LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd && - (HP + Size) > LR.TPR.getEntry()->HstPtrEnd; - } - - if (LR.Flags.ExtendsBefore) { - DP("WARNING: Pointer is not mapped but section extends into already " - "mapped data\n"); - } - if (LR.Flags.ExtendsAfter) { - DP("WARNING: Pointer is already mapped but section extends beyond mapped " - "region\n"); - } - } - - return LR; -} - -TargetPointerResultTy DeviceTy::getTargetPointer( - HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase, - int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo, - bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, - bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier, - AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) { - - LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR); - LR.TPR.Flags.IsPresent = true; - - // Release the mapping table lock only after the entry is locked by - // attaching it to TPR. Once TPR is destroyed it will release the lock - // on entry. If it is returned the lock will move to the returned object. - // If LR.Entry is already owned/locked we avoid trying to lock it again. - - // Check if the pointer is contained. - // If a variable is mapped to the device manually by the user - which would - // lead to the IsContained flag to be true - then we must ensure that the - // device address is returned even under unified memory conditions. - if (LR.Flags.IsContained || - ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) { - const char *RefCountAction; - if (UpdateRefCount) { - // After this, reference count >= 1. If the reference count was 0 but the - // entry was still there we can reuse the data on the device and avoid a - // new submission. - LR.TPR.getEntry()->incRefCount(HasHoldModifier); - RefCountAction = " (incremented)"; - } else { - // It might have been allocated with the parent, but it's still new. - LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1; - RefCountAction = " (update suppressed)"; - } - const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction; - const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : ""; - uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin + - ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin); - INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID, - "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD - ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n", - (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr), - Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction, - LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction, - (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); - LR.TPR.TargetPointer = (void *)Ptr; - } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) { - // Explicit extension of mapped data - not allowed. - MESSAGE("explicit extension not allowed: host address specified is " DPxMOD - " (%" PRId64 - " bytes), but device allocation maps to host at " DPxMOD - " (%" PRId64 " bytes)", - DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin), - LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin); - if (HasPresentModifier) - MESSAGE("device mapping required by 'present' map type modifier does not " - "exist for host address " DPxMOD " (%" PRId64 " bytes)", - DPxPTR(HstPtrBegin), Size); - } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY && - !HasCloseModifier) { - // If unified shared memory is active, implicitly mapped variables that are - // not privatized use host address. Any explicitly mapped variables also use - // host address where correctness is not impeded. In all other cases maps - // are respected. - // In addition to the mapping rules above, the close map modifier forces the - // mapping of the variable to the device. - if (Size) { - DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " - "memory\n", - DPxPTR((uintptr_t)HstPtrBegin), Size); - LR.TPR.Flags.IsPresent = false; - LR.TPR.Flags.IsHostPointer = true; - LR.TPR.TargetPointer = HstPtrBegin; - } - } else if (HasPresentModifier) { - DP("Mapping required by 'present' map type modifier does not exist for " - "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n", - DPxPTR(HstPtrBegin), Size); - MESSAGE("device mapping required by 'present' map type modifier does not " - "exist for host address " DPxMOD " (%" PRId64 " bytes)", - DPxPTR(HstPtrBegin), Size); - } else if (Size) { - // If it is not contained and Size > 0, we should create a new entry for it. - LR.TPR.Flags.IsNewEntry = true; - uintptr_t TgtAllocBegin = - (uintptr_t)allocData(TgtPadding + Size, HstPtrBegin); - uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding; - // Release the mapping table lock only after the entry is locked by - // attaching it to TPR. - LR.TPR.setEntry(HDTTMap - ->emplace(new HostDataToTargetTy( - (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin, - (uintptr_t)HstPtrBegin + Size, TgtAllocBegin, - TgtPtrBegin, HasHoldModifier, HstPtrName)) - .first->HDTT); - INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID, - "Creating new map entry with HstPtrBase=" DPxMOD - ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD - ", TgtPtrBegin=" DPxMOD - ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n", - DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin), - DPxPTR(TgtPtrBegin), Size, - LR.TPR.getEntry()->dynRefCountToStr().c_str(), - LR.TPR.getEntry()->holdRefCountToStr().c_str(), - (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); - LR.TPR.TargetPointer = (void *)TgtPtrBegin; - - // Notify the plugin about the new mapping. - if (notifyDataMapped(HstPtrBegin, Size)) - return {{false /* IsNewEntry */, false /* IsHostPointer */}, - nullptr /* Entry */, - nullptr /* TargetPointer */}; - } else { - // This entry is not present and we did not create a new entry for it. - LR.TPR.Flags.IsPresent = false; - } - - // All mapping table modifications have been made. If the user requested it we - // give up the lock. - if (ReleaseHDTTMap) - HDTTMap.destroy(); - - // If the target pointer is valid, and we need to transfer data, issue the - // data transfer. - if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo && - (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) { - DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size, - DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer)); - - int Ret = submitData(LR.TPR.TargetPointer, HstPtrBegin, Size, AsyncInfo, - LR.TPR.getEntry()); - if (Ret != OFFLOAD_SUCCESS) { - REPORT("Copying data to device failed.\n"); - // We will also return nullptr if the data movement fails because that - // pointer points to a corrupted memory region so it doesn't make any - // sense to continue to use it. - LR.TPR.TargetPointer = nullptr; - } else if (LR.TPR.getEntry()->addEventIfNecessary(*this, AsyncInfo) != - OFFLOAD_SUCCESS) - return {{false /* IsNewEntry */, false /* IsHostPointer */}, - nullptr /* Entry */, - nullptr /* TargetPointer */}; - } else { - // If not a host pointer and no present modifier, we need to wait for the - // event if it exists. - // Note: Entry might be nullptr because of zero length array section. - if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer && - !HasPresentModifier) { - void *Event = LR.TPR.getEntry()->getEvent(); - if (Event) { - int Ret = waitEvent(Event, AsyncInfo); - if (Ret != OFFLOAD_SUCCESS) { - // If it fails to wait for the event, we need to return nullptr in - // case of any data race. - REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); - return {{false /* IsNewEntry */, false /* IsHostPointer */}, - nullptr /* Entry */, - nullptr /* TargetPointer */}; - } - } - } - } - - return std::move(LR.TPR); -} - -TargetPointerResultTy -DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount, - bool UseHoldRefCount, bool MustContain, - bool ForceDelete, bool FromDataEnd) { - HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); - - LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size); - - LR.TPR.Flags.IsPresent = true; - - if (LR.Flags.IsContained || - (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) { - LR.TPR.Flags.IsLast = - LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete); - - if (ForceDelete) { - LR.TPR.getEntry()->resetRefCount(UseHoldRefCount); - assert(LR.TPR.Flags.IsLast == - LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) && - "expected correct IsLast prediction for reset"); - } - - // Increment the number of threads that is using the entry on a - // targetDataEnd, tracking the number of possible "deleters". A thread may - // come to own the entry deletion even if it was not the last one querying - // for it. Thus, we must track every query on targetDataEnds to ensure only - // the last thread that holds a reference to an entry actually deletes it. - if (FromDataEnd) - LR.TPR.getEntry()->incDataEndThreadCount(); - - const char *RefCountAction; - if (!UpdateRefCount) { - RefCountAction = " (update suppressed)"; - } else if (LR.TPR.Flags.IsLast) { - LR.TPR.getEntry()->decRefCount(UseHoldRefCount); - assert(LR.TPR.getEntry()->getTotalRefCount() == 0 && - "Expected zero reference count when deletion is scheduled"); - if (ForceDelete) - RefCountAction = " (reset, delayed deletion)"; - else - RefCountAction = " (decremented, delayed deletion)"; - } else { - LR.TPR.getEntry()->decRefCount(UseHoldRefCount); - RefCountAction = " (decremented)"; - } - const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction; - const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : ""; - uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin + - ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin); - INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID, - "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", " - "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n", - DPxPTR(HstPtrBegin), DPxPTR(TP), Size, - LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction, - LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction); - LR.TPR.TargetPointer = (void *)TP; - } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) { - // If the value isn't found in the mapping and unified shared memory - // is on then it means we have stumbled upon a value which we need to - // use directly from the host. - DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " - "memory\n", - DPxPTR((uintptr_t)HstPtrBegin), Size); - LR.TPR.Flags.IsPresent = false; - LR.TPR.Flags.IsHostPointer = true; - LR.TPR.TargetPointer = HstPtrBegin; - } else { - // OpenMP Specification v5.2: if a matching list item is not found, the - // pointer retains its original value as per firstprivate semantics. - LR.TPR.Flags.IsPresent = false; - LR.TPR.Flags.IsHostPointer = false; - LR.TPR.TargetPointer = HstPtrBegin; - } - - return std::move(LR.TPR); -} - -// Return the target pointer begin (where the data will be moved). -void *DeviceTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, - int64_t Size) { - uintptr_t HP = (uintptr_t)HstPtrBegin; - LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size); - if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) { - uintptr_t TP = - LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin); - return (void *)TP; - } - - return NULL; -} - -int DeviceTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap, - HostDataToTargetTy *Entry, int64_t Size) { - assert(Entry && "Trying to delete a null entry from the HDTT map."); - assert(Entry->getTotalRefCount() == 0 && - Entry->getDataEndThreadCount() == 0 && - "Trying to delete entry that is in use or owned by another thread."); - - INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID, - "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD - ", Size=%" PRId64 ", Name=%s\n", - DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size, - (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str() - : "unknown"); - - if (HDTTMap->erase(Entry) == 0) { - REPORT("Trying to remove a non-existent map entry\n"); - return OFFLOAD_FAIL; - } - - return OFFLOAD_SUCCESS; -} - -int DeviceTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, int64_t Size) { - assert(Entry && "Trying to deallocate a null entry."); - - DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation " - "starting at " DPxMOD "\n", - DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin)); - - void *Event = Entry->getEvent(); - if (Event && destroyEvent(Event) != OFFLOAD_SUCCESS) { - REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event)); - return OFFLOAD_FAIL; - } - - int Ret = deleteData((void *)Entry->TgtAllocBegin); - - // Notify the plugin about the unmapped memory. - Ret |= notifyDataUnmapped((void *)Entry->HstPtrBegin); - - delete Entry; - - return Ret; -} - llvm::Error DeviceTy::init() { // Make call to init_requires if it exists for this plugin. int32_t Ret = 0; @@ -586,34 +133,13 @@ int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) { return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind); } -static void printCopyInfo(int DeviceId, bool H2D, void *SrcPtrBegin, - void *DstPtrBegin, int64_t Size, - HostDataToTargetTy *HT) { - - INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId, - "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD - ", Size=%" PRId64 ", Name=%s\n", - H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt", - DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size, - (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str() - : "unknown"); -} - // Submit data to device int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry, - DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) { - if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) { - HDTTMapAccessorTy HDTTMap = - HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr); - LookupResult LR; - if (!Entry) { - LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size); - Entry = LR.TPR.getEntry(); - } - printCopyInfo(DeviceID, /* H2D */ true, HstPtrBegin, TgtPtrBegin, Size, - Entry); - } + MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) { + if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) + MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true, + Entry, HDTTMapPtr); /// RAII to establish tool anchors before and after data submit OMPT_IF_BUILT( @@ -632,18 +158,10 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size, AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry, - DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) { - if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) { - HDTTMapAccessorTy HDTTMap = - HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr); - LookupResult LR; - if (!Entry) { - LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size); - Entry = LR.TPR.getEntry(); - } - printCopyInfo(DeviceID, /* H2D */ false, TgtPtrBegin, HstPtrBegin, Size, - Entry); - } + MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) { + if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) + MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false, + Entry, HDTTMapPtr); /// RAII to establish tool anchors before and after data retrieval OMPT_IF_BUILT( @@ -775,7 +293,8 @@ int32_t DeviceTy::destroyEvent(void *Event) { void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) { std::lock_guard Lock(PendingGlobalsMtx); - DeviceOffloadEntries[Entry.getName()] = &Entry; + DeviceOffloadEntries.getExclusiveAccessor()->insert( + {Entry.getName(), &Entry}); if (Entry.isGlobal()) return; @@ -808,7 +327,7 @@ void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) { void DeviceTy::dumpOffloadEntries() { fprintf(stderr, "Device %i offload entries:\n", DeviceID); - for (auto &It : DeviceOffloadEntries) { + for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) { const char *Kind = "kernel"; if (It.second->isCTor()) Kind = "constructor"; diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp index 2edbadaa6e02..0d16a41c7616 100644 --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -194,8 +194,8 @@ static int initLibrary(DeviceTy &Device) { break; } - DeviceTy::HDTTMapAccessorTy HDTTMap = - Device.HostDataToTargetMap.getExclusiveAccessor(); + MappingInfoTy::HDTTMapAccessorTy HDTTMap = + Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor(); __tgt_target_table *HostTable = &TransTable->HostTable; for (__tgt_offload_entry *CurrDeviceEntry = TargetTable->EntriesBegin, @@ -213,8 +213,8 @@ static int initLibrary(DeviceTy &Device) { // therefore we must allow for multiple weak symbols to be loaded from // the fat binary. Treat these mappings as any other "regular" // mapping. Add entry to map. - if (Device.getTgtPtrBegin(HDTTMap, CurrHostEntry->addr, - CurrHostEntry->size)) + if (Device.getMappingInfo().getTgtPtrBegin(HDTTMap, CurrHostEntry->addr, + CurrHostEntry->size)) continue; void *CurrDeviceEntryAddr = CurrDeviceEntry->addr; @@ -604,8 +604,8 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, bool UpdateRef = !(ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) && !(FromMapper && I == 0); - DeviceTy::HDTTMapAccessorTy HDTTMap = - Device.HostDataToTargetMap.getExclusiveAccessor(); + MappingInfoTy::HDTTMapAccessorTy HDTTMap = + Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor(); if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) { DP("Has a pointer entry: \n"); // Base is address of pointer. @@ -621,7 +621,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, // entry for a global that might not already be allocated by the time the // PTR_AND_OBJ entry is handled below, and so the allocation might fail // when HasPresentModifier. - PointerTpr = Device.getTargetPointer( + PointerTpr = Device.getMappingInfo().getTargetPointer( HDTTMap, HstPtrBase, HstPtrBase, /*TgtPadding=*/0, sizeof(void *), /*HstPtrName=*/nullptr, /*HasFlagTo=*/false, /*HasFlagAlways=*/false, IsImplicit, UpdateRef, @@ -651,7 +651,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, const bool HasFlagTo = ArgTypes[I] & OMP_TGT_MAPTYPE_TO; const bool HasFlagAlways = ArgTypes[I] & OMP_TGT_MAPTYPE_ALWAYS; // Note that HDTTMap will be released in getTargetPointer. - auto TPR = Device.getTargetPointer( + auto TPR = Device.getMappingInfo().getTargetPointer( HDTTMap, HstPtrBegin, HstPtrBase, TgtPadding, DataSize, HstPtrName, HasFlagTo, HasFlagAlways, IsImplicit, UpdateRef, HasCloseModifier, HasPresentModifier, HasHoldModifier, AsyncInfo, PointerTpr.getEntry()); @@ -768,8 +768,8 @@ postProcessingTargetDataEnd(DeviceTy *Device, // will avoid another thread reusing the entry now. Note that we do // not request (exclusive) access to the HDTT map if DelEntry is // not set. - DeviceTy::HDTTMapAccessorTy HDTTMap = - Device->HostDataToTargetMap.getExclusiveAccessor(); + MappingInfoTy::HDTTMapAccessorTy HDTTMap = + Device->getMappingInfo().HostDataToTargetMap.getExclusiveAccessor(); // We cannot use a lock guard because we may end up delete the mutex. // We also explicitly unlocked the entry after it was put in the EntriesInfo @@ -807,10 +807,10 @@ postProcessingTargetDataEnd(DeviceTy *Device, if (!DelEntry) continue; - Ret = Device->eraseMapEntry(HDTTMap, Entry, DataSize); + Ret = Device->getMappingInfo().eraseMapEntry(HDTTMap, Entry, DataSize); // Entry is already remove from the map, we can unlock it now. HDTTMap.destroy(); - Ret |= Device->deallocTgtPtrAndEntry(Entry, DataSize); + Ret |= Device->getMappingInfo().deallocTgtPtrAndEntry(Entry, DataSize); if (Ret != OFFLOAD_SUCCESS) { REPORT("Deallocating data from device failed.\n"); break; @@ -868,9 +868,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, bool HasHoldModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_OMPX_HOLD; // If PTR_AND_OBJ, HstPtrBegin is address of pointee - TargetPointerResultTy TPR = - Device.getTgtPtrBegin(HstPtrBegin, DataSize, UpdateRef, HasHoldModifier, - !IsImplicit, ForceDelete, /*FromDataEnd=*/true); + TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin( + HstPtrBegin, DataSize, UpdateRef, HasHoldModifier, !IsImplicit, + ForceDelete, /*FromDataEnd=*/true); void *TgtPtrBegin = TPR.TargetPointer; if (!TPR.isPresent() && !TPR.isHostPointer() && (DataSize || HasPresentModifier)) { @@ -965,9 +965,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, static int targetDataContiguous(ident_t *Loc, DeviceTy &Device, void *ArgsBase, void *HstPtrBegin, int64_t ArgSize, int64_t ArgType, AsyncInfoTy &AsyncInfo) { - TargetPointerResultTy TPR = - Device.getTgtPtrBegin(HstPtrBegin, ArgSize, /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false, /*MustContain=*/true); + TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin( + HstPtrBegin, ArgSize, /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false, /*MustContain=*/true); void *TgtPtrBegin = TPR.TargetPointer; if (!TPR.isPresent()) { DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin)); @@ -1445,9 +1445,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr, uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase; void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta); void *&PointerTgtPtrBegin = AsyncInfo.getVoidPtrLocation(); - TargetPointerResultTy TPR = DeviceOrErr->getTgtPtrBegin( - HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false); + TargetPointerResultTy TPR = + DeviceOrErr->getMappingInfo().getTgtPtrBegin( + HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); PointerTgtPtrBegin = TPR.TargetPointer; if (!TPR.isPresent()) { DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n", @@ -1503,9 +1504,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr, } else { if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) HstPtrBase = *reinterpret_cast(HstPtrBase); - TPR = DeviceOrErr->getTgtPtrBegin(HstPtrBegin, ArgSizes[I], - /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false); + TPR = DeviceOrErr->getMappingInfo().getTgtPtrBegin( + HstPtrBegin, ArgSizes[I], + /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); TgtPtrBegin = TPR.TargetPointer; TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin; #ifdef OMPTARGET_DEBUG -- GitLab From 0d1490f09f23bf204b714c3c6ba5e0aaf4eeed9a Mon Sep 17 00:00:00 2001 From: Benjamin Chetioui <3920784+bchetioui@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:00:23 +0100 Subject: [PATCH 005/281] [MLIR] Flatten fused locations when merging constants. (#75218) [PR 74670](https://github.com/llvm/llvm-project/pull/74670) added support for merging locations at constant folding time. We have discovered that in some cases, the number of locations grows so big as to cause a compilation process to OOM. In that case, many of the locations end up appearing several times in nested fused locations. We add here a helper that always flattens fused locations in order to eliminate duplicates in the case of nested fused locations. --- mlir/lib/Transforms/Utils/FoldUtils.cpp | 37 ++++++++++++++++++- .../Transforms/canonicalize-debuginfo.mlir | 13 +++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Transforms/Utils/FoldUtils.cpp b/mlir/lib/Transforms/Utils/FoldUtils.cpp index dfc63ed6c4a5..056a681718e1 100644 --- a/mlir/lib/Transforms/Utils/FoldUtils.cpp +++ b/mlir/lib/Transforms/Utils/FoldUtils.cpp @@ -331,6 +331,39 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants, return newIt.first->second; } +/// Helper that flattens nested fused locations to a single fused location. +/// Fused locations nested under non-fused locations are not flattened, and +/// calling this on non-fused locations is a no-op as a result. +/// +/// Fused locations are only flattened into parent fused locations if the +/// child fused location has no metadata, or if the metadata of the parent and +/// child fused locations are the same---this to avoid breaking cases where +/// metadata matter. +static Location FlattenFusedLocationRecursively(const Location loc) { + if (auto fusedLoc = dyn_cast(loc)) { + SetVector flattenedLocs; + Attribute metadata = fusedLoc.getMetadata(); + + for (const Location &unflattenedLoc : fusedLoc.getLocations()) { + Location flattenedLoc = FlattenFusedLocationRecursively(unflattenedLoc); + auto flattenedFusedLoc = dyn_cast(flattenedLoc); + + if (flattenedFusedLoc && (!flattenedFusedLoc.getMetadata() || + flattenedFusedLoc.getMetadata() == metadata)) { + ArrayRef nestedLocations = flattenedFusedLoc.getLocations(); + flattenedLocs.insert(nestedLocations.begin(), nestedLocations.end()); + } else { + flattenedLocs.insert(flattenedLoc); + } + } + + return FusedLoc::get(loc->getContext(), flattenedLocs.takeVector(), + fusedLoc.getMetadata()); + } + + return loc; +} + void OperationFolder::appendFoldedLocation(Operation *retainedOp, Location foldedLocation) { // Append into existing fused location if it has the same tag. @@ -344,7 +377,7 @@ void OperationFolder::appendFoldedLocation(Operation *retainedOp, locations.insert(foldedLocation); Location newFusedLoc = FusedLoc::get( retainedOp->getContext(), locations.takeVector(), existingMetadata); - retainedOp->setLoc(newFusedLoc); + retainedOp->setLoc(FlattenFusedLocationRecursively(newFusedLoc)); return; } } @@ -357,5 +390,5 @@ void OperationFolder::appendFoldedLocation(Operation *retainedOp, Location newFusedLoc = FusedLoc::get(retainedOp->getContext(), {retainedOp->getLoc(), foldedLocation}, fusedLocationTag); - retainedOp->setLoc(newFusedLoc); + retainedOp->setLoc(FlattenFusedLocationRecursively(newFusedLoc)); } diff --git a/mlir/test/Transforms/canonicalize-debuginfo.mlir b/mlir/test/Transforms/canonicalize-debuginfo.mlir index 034c9163a805..217cc29c0095 100644 --- a/mlir/test/Transforms/canonicalize-debuginfo.mlir +++ b/mlir/test/Transforms/canonicalize-debuginfo.mlir @@ -1,19 +1,26 @@ // RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize{test-convergence}))' -split-input-file -mlir-print-debuginfo | FileCheck %s // CHECK-LABEL: func @merge_constants -func.func @merge_constants() -> (index, index, index, index) { +func.func @merge_constants() -> (index, index, index, index, index, index, index) { // CHECK-NEXT: arith.constant 42 : index loc(#[[FusedLoc:.*]]) %0 = arith.constant 42 : index loc("merge_constants":0:0) %1 = arith.constant 42 : index loc("merge_constants":1:0) %2 = arith.constant 42 : index loc("merge_constants":2:0) %3 = arith.constant 42 : index loc("merge_constants":2:0) // repeated loc - return %0, %1, %2, %3: index, index, index, index + %4 = arith.constant 43 : index loc(fused<"some_label">["merge_constants":3:0]) + %5 = arith.constant 43 : index loc(fused<"some_label">["merge_constants":3:0]) + %6 = arith.constant 43 : index loc(fused<"some_other_label">["merge_constants":3:0]) + return %0, %1, %2, %3, %4, %5, %6 : index, index, index, index, index, index, index } // CHECK-DAG: #[[LocConst0:.*]] = loc("merge_constants":0:0) // CHECK-DAG: #[[LocConst1:.*]] = loc("merge_constants":1:0) // CHECK-DAG: #[[LocConst2:.*]] = loc("merge_constants":2:0) -// CHECK: #[[FusedLoc]] = loc(fused<"CSE">[#[[LocConst0]], #[[LocConst1]], #[[LocConst2]]]) +// CHECK-DAG: #[[LocConst3:.*]] = loc("merge_constants":3:0) +// CHECK-DAG: #[[FusedLoc_CSE_1:.*]] = loc(fused<"CSE">[#[[LocConst0]], #[[LocConst1]], #[[LocConst2]]]) +// CHECK-DAG: #[[FusedLoc_Some_Label:.*]] = loc(fused<"some_label">[#[[LocConst3]]]) +// CHECK-DAG: #[[FusedLoc_Some_Other_Label:.*]] = loc(fused<"some_other_label">[#[[LocConst3]]]) +// CHECK-DAG: #[[FusedLoc_CSE_2:.*]] = loc(fused<"CSE">[#[[FusedLoc_Some_Label]], #[[FusedLoc_Some_Other_Label]]]) // ----- -- GitLab From 7d34f8c09ee17327668c337ff3a7c30656f8daca Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Tue, 12 Dec 2023 22:17:02 +0100 Subject: [PATCH 006/281] [libc++][module] Fixes std::string UDL. (#75000) The fix changes the way the validation script determines the qualified name of a declaration. Inline namespaces without a reserved name are now always part of the name. The Clang code only does this when the names are ambigious. This method is often used for the operator""foo for UDLs. Adjusted the newly flagged issue and removed a work-around in the test code that is no longer required. Fixes https://github.com/llvm/llvm-project/issues/72427 --- libcxx/modules/std/string.inc | 7 +- .../header_exportable_declarations.cpp | 76 ++++++++----------- libcxx/utils/libcxx/test/modules.py | 8 -- 3 files changed, 33 insertions(+), 58 deletions(-) diff --git a/libcxx/modules/std/string.inc b/libcxx/modules/std/string.inc index 8366690fd9d3..c83ee7643f87 100644 --- a/libcxx/modules/std/string.inc +++ b/libcxx/modules/std/string.inc @@ -67,15 +67,10 @@ export namespace std { // [basic.string.hash], hash support using std::hash; - // TODO MODULES is this a bug? -#if _LIBCPP_STD_VER >= 23 - using std::operator""s; -#else inline namespace literals { inline namespace string_literals { // [basic.string.literals], suffix for basic_string literals using std::literals::string_literals::operator""s; } // namespace string_literals - } // namespace literals -#endif + } // namespace literals } // namespace std diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp index fcb5865adf0d..35f020da45c4 100644 --- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp +++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp @@ -166,7 +166,7 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match } } -/// Returns the qualified name of a declaration. +/// Returns the qualified name of a public declaration. /// /// There is a small issue with qualified names. Typically the name returned is /// in the namespace \c std instead of the namespace \c std::__1. Except when a @@ -182,14 +182,37 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match /// * exception has equality operators for the type \c exception_ptr /// * initializer_list has the functions \c begin and \c end /// -/// \warning In some cases the returned name can be an empty string. -/// The cause has not been investigated. +/// When the named declaration uses a reserved name the result is an +/// empty string. static std::string get_qualified_name(const clang::NamedDecl& decl) { - std::string result = decl.getQualifiedNameAsString(); - - if (result.starts_with("std::__1::")) - result.erase(5, 5); - + std::string result = decl.getNameAsString(); + // Reject reserved names (ignoring _ in global namespace). + if (result.size() >= 2 && result[0] == '_') + if (result[1] == '_' || std::isupper(result[1])) + if (result != "_Exit") + return ""; + + for (auto* context = llvm::dyn_cast_or_null(decl.getDeclContext()); // + context; + context = llvm::dyn_cast_or_null(context->getDeclContext())) { + std::string ns = std::string(context->getName()); + + if (ns.starts_with("__")) { + // When the reserved name is an inline namespace the namespace is + // not added to the qualified name instead of removed. Libc++ uses + // several inline namespace with reserved names. For example, + // __1 for every declaration, __cpo in range-based algorithms. + // + // Note other inline namespaces are expanded. This resolves + // ambiguity when two named declarations have the same name but in + // different inline namespaces. These typically are the literal + // conversion operators like operator""s which can be a + // std::string or std::chrono::seconds. + if (!context->isInline()) + return ""; + } else + result = ns + "::" + result; + } return result; } @@ -220,38 +243,6 @@ static bool is_viable_declaration(const clang::NamedDecl* decl) { return llvm::isa(decl); } -/// Returns the name is a reserved name. -/// -/// Detected reserved names are names starting with __ or _[A-Z]. -/// These names can be in the global namespace, std namespace or any namespace -/// inside std. For example, std::ranges contains reserved names to implement -/// the Niebloids. -/// -/// This test misses candidates which are not used in libc++ -/// * any identifier with two underscores not at the start -bool is_reserved_name(std::string_view name) { - if (name.starts_with("_")) { - // This is a public name declared in cstdlib. - if (name == "_Exit") - return false; - - return name.size() > 1 && (name[1] == '_' || std::isupper(name[1])); - } - - std::size_t pos = name.find("::_"); - if (pos == std::string::npos) - return false; - - if (pos + 3 > name.size()) - return false; - - // This is a public name declared in cstdlib. - if (name == "std::_Exit") - return false; - - return name[pos + 3] == '_' || std::isupper(name[pos + 3]); -} - /// Some declarations in the global namespace are exported from the std module. static bool is_global_name_exported_by_std_module(std::string_view name) { static const std::set valid{ @@ -297,9 +288,6 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde if (name.empty()) return; - if (is_reserved_name(name)) - return; - // For modules only take the declarations exported. if (is_module(file_type_)) if (decl->getModuleOwnershipKind() != clang::Decl::ModuleOwnershipKind::VisibleWhenImported) @@ -336,7 +324,7 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde return; std::string name = get_qualified_name(*decl); - if (is_reserved_name(name)) + if (name.empty()) return; if (global_decls_.contains(name)) diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py index deaac450381c..bd19fac314dd 100644 --- a/libcxx/utils/libcxx/test/modules.py +++ b/libcxx/utils/libcxx/test/modules.py @@ -141,14 +141,6 @@ class module_test_generator: f"# include <{header}>{nl}" f"#endif{nl}" ) - elif header == "chrono": - # When localization is disabled the header string is not included. - # When string is included chrono's operator""s is a named declaration - # using std::chrono_literals::operator""s; - # else it is a named declaration - # using std::operator""s; - # TODO MODULES investigate why - include = f"#include {nl}#include {nl}" else: include = f"#include <{header}>{nl}" -- GitLab From 97b25d91df3a553b92915a6f81db874dc8954b19 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Tue, 12 Dec 2023 13:19:42 -0800 Subject: [PATCH 007/281] [lld][WebAssembly] Don't set importUndefined when -shared is used. NFC (#75241) `importUndefined` is only used a couple of places and both of those already handle `isPIC` separately. --- lld/wasm/Driver.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp index f8b0fc357bd9..c68fe33a14e2 100644 --- a/lld/wasm/Driver.cpp +++ b/lld/wasm/Driver.cpp @@ -606,7 +606,6 @@ static void setConfigs() { config->memoryImport = std::pair(defaultModule, memoryName); } - config->importUndefined = true; } // If neither export-memory nor import-memory is specified, default to -- GitLab From cd9a641613eddf25d4b25eaa96b2c393d401d42c Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 12 Dec 2023 13:24:38 -0800 Subject: [PATCH 008/281] [bazel] Port 4b3446771f745bb5169354ad9027c0a1c9fca394 --- .../bazel/llvm-project-overlay/mlir/BUILD.bazel | 4 ++++ .../llvm-project-overlay/mlir/test/BUILD.bazel | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index d26299094bc8..2e3bb8ad40b7 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3267,11 +3267,14 @@ cc_library( hdrs = glob(["include/mlir/Dialect/Mesh/Transforms/*.h"]), includes = ["include"], deps = [ + ":ArithDialect", ":FuncDialect", + ":IR", ":MeshDialect", ":MeshShardingInterface", ":MeshTransformsPassIncGen", ":Pass", + ":TransformUtils", "//llvm:Support", ], ) @@ -8968,6 +8971,7 @@ cc_binary( "//mlir/test:TestLoopLikeInterface", "//mlir/test:TestMath", "//mlir/test:TestMemRef", + "//mlir/test:TestMesh", "//mlir/test:TestNVGPU", "//mlir/test:TestOneToNTypeConversion", "//mlir/test:TestPDLL", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index 5d5c4b450a0b..29a7ce7168fc 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -780,6 +780,22 @@ cc_library( ], ) +cc_library( + name = "TestMesh", + srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]), + hdrs = glob(["lib/Dialect/Mesh/**/*.h"]), + includes = ["lib/Dialect/Test"], + deps = [ + ":TestDialect", + "//mlir:ArithDialect", + "//mlir:FuncDialect", + "//mlir:MeshDialect", + "//mlir:MeshTransforms", + "//mlir:Pass", + "//mlir:Transforms", + ], +) + cc_library( name = "TestNVGPU", srcs = glob(["lib/Dialect/NVGPU/*.cpp"]), -- GitLab From 8180ea8694cbc58f105f27f2044dabc3febbcf6a Mon Sep 17 00:00:00 2001 From: michaelrj-google <71531609+michaelrj-google@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:36:11 -0800 Subject: [PATCH 009/281] [libc] Add bind function (#74014) This patch adds the bind function to go with the socket function. It also cleans up a lot of socket related data structures. --- libc/config/linux/api.td | 7 ++- libc/config/linux/x86_64/entrypoints.txt | 7 ++- libc/include/CMakeLists.txt | 4 +- libc/include/llvm-libc-types/CMakeLists.txt | 2 + libc/include/llvm-libc-types/socklen_t.h | 18 ++++++ .../include/llvm-libc-types/struct_sockaddr.h | 10 ++-- .../llvm-libc-types/struct_sockaddr_un.h | 22 ++++++++ libc/spec/posix.td | 17 +++++- libc/src/sys/socket/CMakeLists.txt | 6 ++ libc/src/sys/socket/bind.h | 20 +++++++ libc/src/sys/socket/linux/CMakeLists.txt | 13 +++++ libc/src/sys/socket/linux/bind.cpp | 43 +++++++++++++++ libc/src/sys/socket/linux/socket.cpp | 4 +- libc/test/src/sys/socket/linux/CMakeLists.txt | 16 ++++++ libc/test/src/sys/socket/linux/bind_test.cpp | 55 +++++++++++++++++++ .../test/src/sys/socket/linux/socket_test.cpp | 4 +- 16 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 libc/include/llvm-libc-types/socklen_t.h create mode 100644 libc/include/llvm-libc-types/struct_sockaddr_un.h create mode 100644 libc/src/sys/socket/bind.h create mode 100644 libc/src/sys/socket/linux/bind.cpp create mode 100644 libc/test/src/sys/socket/linux/bind_test.cpp diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td index 726e58f376ea..85f6b59264eb 100644 --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -205,7 +205,12 @@ def SysSelectAPI : PublicAPI<"sys/select.h"> { } def SysSocketAPI : PublicAPI<"sys/socket.h"> { - let Types = ["struct sockaddr", "sa_family_t"]; + let Types = [ + "sa_family_t", + "socklen_t", + "struct sockaddr", + "struct sockaddr_un", + ]; } def SysResourceAPI : PublicAPI<"sys/resource.h"> { diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 13b81d3b7ca7..1c93063e25e9 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -153,9 +153,6 @@ set(TARGET_LIBC_ENTRYPOINTS # sys/sendfile entrypoints libc.src.sys.sendfile.sendfile - # sys/socket.h entrypoints - libc.src.sys.socket.socket - # sys/stat.h entrypoints libc.src.sys.stat.chmod libc.src.sys.stat.fchmod @@ -557,6 +554,10 @@ if(LLVM_LIBC_FULL_BUILD) # sys/select.h entrypoints libc.src.sys.select.select + + # sys/socket.h entrypoints + libc.src.sys.socket.socket + libc.src.sys.socket.bind ) endif() diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt index 429c0f1f1286..59c6c4a9bb42 100644 --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -417,8 +417,10 @@ add_gen_header( DEPENDS .llvm_libc_common_h .llvm-libc-macros.sys_socket_macros - .llvm-libc-types.struct_sockaddr .llvm-libc-types.sa_family_t + .llvm-libc-types.socklen_t + .llvm-libc-types.struct_sockaddr + .llvm-libc-types.struct_sockaddr_un ) add_gen_header( diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt index 225ad780c4d0..500900ffa0bb 100644 --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -89,6 +89,8 @@ add_header(__getoptargv_t HDR __getoptargv_t.h) add_header(wchar_t HDR wchar_t.h) add_header(wint_t HDR wint_t.h) add_header(sa_family_t HDR sa_family_t.h) +add_header(socklen_t HDR socklen_t.h) +add_header(struct_sockaddr_un HDR struct_sockaddr_un.h) add_header(struct_sockaddr HDR struct_sockaddr.h) add_header(rpc_opcodes_t HDR rpc_opcodes_t.h) add_header(ACTION HDR ACTION.h) diff --git a/libc/include/llvm-libc-types/socklen_t.h b/libc/include/llvm-libc-types/socklen_t.h new file mode 100644 index 000000000000..3134a53390e7 --- /dev/null +++ b/libc/include/llvm-libc-types/socklen_t.h @@ -0,0 +1,18 @@ +//===-- Definition of socklen_t type ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_LIBC_TYPES_SOCKLEN_T_H__ +#define __LLVM_LIBC_TYPES_SOCKLEN_T_H__ + +// The posix standard only says of socklen_t that it must be an integer type of +// width of at least 32 bits. The long type is defined as being at least 32 +// bits, so an unsigned long should be fine. + +typedef unsigned long socklen_t; + +#endif // __LLVM_LIBC_TYPES_SOCKLEN_T_H__ diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h index 1ef907904ca3..9a6214c7d3e6 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr.h +++ b/libc/include/llvm-libc-types/struct_sockaddr.h @@ -1,4 +1,4 @@ -//===-- Definition of struct stat -----------------------------------------===// +//===-- Definition of struct sockaddr -------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef __LLVM_LIBC_TYPES_STRUCT_STAT_H__ -#define __LLVM_LIBC_TYPES_STRUCT_STAT_H__ +#ifndef __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__ +#define __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__ #include @@ -15,7 +15,7 @@ struct sockaddr { sa_family_t sa_family; // sa_data is a variable length array. It is provided with a length of one // here as a placeholder. - char sa_data[1]; + char sa_data[]; }; -#endif // __LLVM_LIBC_TYPES_STRUCT_STAT_H__ +#endif // __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__ diff --git a/libc/include/llvm-libc-types/struct_sockaddr_un.h b/libc/include/llvm-libc-types/struct_sockaddr_un.h new file mode 100644 index 000000000000..9c3efea27925 --- /dev/null +++ b/libc/include/llvm-libc-types/struct_sockaddr_un.h @@ -0,0 +1,22 @@ +//===-- Definition of struct sockaddr_un ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__ +#define __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__ + +#include + +// This is the sockaddr specialization for AF_UNIX or AF_LOCAL sockets, as +// defined by posix. + +struct sockaddr_un { + sa_family_t sun_family; /* AF_UNIX */ + char sun_path[108]; /* Pathname */ +}; + +#endif // __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__ diff --git a/libc/spec/posix.td b/libc/spec/posix.td index c7acf6d25a2d..7e1cf892135a 100644 --- a/libc/spec/posix.td +++ b/libc/spec/posix.td @@ -81,9 +81,14 @@ def RestrictedFdSetPtr : RestrictedPtrType; def GetoptArgvT : NamedType<"__getoptargv_t">; +def SAFamilyType : NamedType<"sa_family_t">; +def SocklenType : NamedType<"socklen_t">; + def StructSockAddr : NamedType<"struct sockaddr">; def StructSockAddrPtr : PtrType; -def SAFamilyType : NamedType<"sa_family_t">; +def ConstStructSockAddrPtr : ConstType; + +def StructSockAddrUn : NamedType<"struct sockaddr_un">; def POSIX : StandardSpec<"POSIX"> { PtrType CharPtr = PtrType; @@ -1400,7 +1405,10 @@ def POSIX : StandardSpec<"POSIX"> { Macro<"SOCK_PACKET">, ], // Macros [ - StructSockAddr, SAFamilyType, + SAFamilyType, + StructSockAddr, + StructSockAddrUn, + SocklenType, ], // Types [], // Enumerations [ @@ -1409,6 +1417,11 @@ def POSIX : StandardSpec<"POSIX"> { RetValSpec, [ArgSpec, ArgSpec, ArgSpec] >, + FunctionSpec< + "bind", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, ] // Functions >; diff --git a/libc/src/sys/socket/CMakeLists.txt b/libc/src/sys/socket/CMakeLists.txt index 7079d6e4466c..e0bc48735a03 100644 --- a/libc/src/sys/socket/CMakeLists.txt +++ b/libc/src/sys/socket/CMakeLists.txt @@ -9,3 +9,9 @@ add_entrypoint_object( .${LIBC_TARGET_OS}.socket ) +add_entrypoint_object( + bind + ALIAS + DEPENDS + .${LIBC_TARGET_OS}.bind +) diff --git a/libc/src/sys/socket/bind.h b/libc/src/sys/socket/bind.h new file mode 100644 index 000000000000..62e6221bf1b2 --- /dev/null +++ b/libc/src/sys/socket/bind.h @@ -0,0 +1,20 @@ +//===-- Implementation header for bind --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_SYS_SOCKET_BIND_H +#define LLVM_LIBC_SRC_SYS_SOCKET_BIND_H + +#include + +namespace LIBC_NAMESPACE { + +int bind(int domain, const struct sockaddr *address, socklen_t address_len); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_SYS_SOCKET_BIND_H diff --git a/libc/src/sys/socket/linux/CMakeLists.txt b/libc/src/sys/socket/linux/CMakeLists.txt index 41bcc9c9055f..fc9febdec2cc 100644 --- a/libc/src/sys/socket/linux/CMakeLists.txt +++ b/libc/src/sys/socket/linux/CMakeLists.txt @@ -10,3 +10,16 @@ add_entrypoint_object( libc.src.__support.OSUtil.osutil libc.src.errno.errno ) + +add_entrypoint_object( + bind + SRCS + bind.cpp + HDRS + ../bind.h + DEPENDS + libc.include.sys_syscall + libc.include.sys_socket + libc.src.__support.OSUtil.osutil + libc.src.errno.errno +) diff --git a/libc/src/sys/socket/linux/bind.cpp b/libc/src/sys/socket/linux/bind.cpp new file mode 100644 index 000000000000..36afc646d29f --- /dev/null +++ b/libc/src/sys/socket/linux/bind.cpp @@ -0,0 +1,43 @@ +//===-- Linux implementation of bind --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/sys/socket/bind.h" + +#include "src/__support/OSUtil/syscall.h" // For internal syscall function. +#include "src/__support/common.h" + +#include "src/errno/libc_errno.h" + +#include // For SYS_SOCKET socketcall number. +#include // For syscall numbers. + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, bind, + (int domain, const struct sockaddr *address, + socklen_t address_len)) { +#ifdef SYS_socket + int ret = + LIBC_NAMESPACE::syscall_impl(SYS_bind, domain, address, address_len); +#elif defined(SYS_socketcall) + unsigned long sockcall_args[3] = {static_cast(domain), + reinterpret_cast(address), + static_cast(address_len)}; + int ret = LIBC_NAMESPACE::syscall_impl(SYS_socketcall, SYS_BIND, + sockcall_args); +#else +#error "socket and socketcall syscalls unavailable for this platform." +#endif + if (ret < 0) { + libc_errno = -ret; + return -1; + } + return ret; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/sys/socket/linux/socket.cpp b/libc/src/sys/socket/linux/socket.cpp index 6429fd12013e..90a7dc632e26 100644 --- a/libc/src/sys/socket/linux/socket.cpp +++ b/libc/src/sys/socket/linux/socket.cpp @@ -23,7 +23,9 @@ LLVM_LIBC_FUNCTION(int, socket, (int domain, int type, int protocol)) { int ret = LIBC_NAMESPACE::syscall_impl(SYS_socket, domain, type, protocol); #elif defined(SYS_socketcall) - unsigned long sockcall_args[3] = {domain, type, protocol}; + unsigned long sockcall_args[3] = {static_cast(domain), + static_cast(type), + static_cast(protocol)}; int ret = LIBC_NAMESPACE::syscall_impl(SYS_socketcall, SYS_SOCKET, sockcall_args); #else diff --git a/libc/test/src/sys/socket/linux/CMakeLists.txt b/libc/test/src/sys/socket/linux/CMakeLists.txt index 4380597e5515..666dc28c7e4e 100644 --- a/libc/test/src/sys/socket/linux/CMakeLists.txt +++ b/libc/test/src/sys/socket/linux/CMakeLists.txt @@ -12,3 +12,19 @@ add_libc_unittest( libc.src.sys.socket.socket libc.src.unistd.close ) + + +add_libc_unittest( + bind_test + SUITE + libc_sys_socket_unittests + SRCS + bind_test.cpp + DEPENDS + libc.include.sys_socket + libc.src.errno.errno + libc.src.sys.socket.socket + libc.src.sys.socket.bind + libc.src.stdio.remove + libc.src.unistd.close +) diff --git a/libc/test/src/sys/socket/linux/bind_test.cpp b/libc/test/src/sys/socket/linux/bind_test.cpp new file mode 100644 index 000000000000..5a3a1c227c9b --- /dev/null +++ b/libc/test/src/sys/socket/linux/bind_test.cpp @@ -0,0 +1,55 @@ +//===-- Unittests for bind ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/sys/socket/bind.h" +#include "src/sys/socket/socket.h" + +#include "src/stdio/remove.h" +#include "src/unistd/close.h" + +#include "src/errno/libc_errno.h" +#include "test/UnitTest/LibcTest.h" +#include "test/UnitTest/Test.h" + +#include // For AF_UNIX and SOCK_DGRAM + +TEST(LlvmLibcSocketTest, BindLocalSocket) { + + const char *FILENAME = "bind_file.test"; + auto SOCK_PATH = libc_make_test_file_path(FILENAME); + + int sock = LIBC_NAMESPACE::socket(AF_UNIX, SOCK_DGRAM, 0); + ASSERT_GE(sock, 0); + ASSERT_EQ(libc_errno, 0); + + struct sockaddr_un my_addr; + + my_addr.sun_family = AF_UNIX; + unsigned int i = 0; + for (; + SOCK_PATH[i] != '\0' && (i < sizeof(sockaddr_un) - sizeof(sa_family_t)); + ++i) + my_addr.sun_path[i] = SOCK_PATH[i]; + my_addr.sun_path[i] = '\0'; + + // It's important that the path fits in the struct, if it doesn't then we + // can't try to bind to the file. + ASSERT_LT( + i, static_cast(sizeof(sockaddr_un) - sizeof(sa_family_t))); + + int result = + LIBC_NAMESPACE::bind(sock, reinterpret_cast(&my_addr), + sizeof(struct sockaddr_un)); + + ASSERT_EQ(result, 0); + ASSERT_EQ(libc_errno, 0); + + LIBC_NAMESPACE::close(sock); + + LIBC_NAMESPACE::remove(SOCK_PATH); +} diff --git a/libc/test/src/sys/socket/linux/socket_test.cpp b/libc/test/src/sys/socket/linux/socket_test.cpp index 9037888441a3..9d5bfacde0a4 100644 --- a/libc/test/src/sys/socket/linux/socket_test.cpp +++ b/libc/test/src/sys/socket/linux/socket_test.cpp @@ -13,10 +13,10 @@ #include "src/errno/libc_errno.h" #include "test/UnitTest/Test.h" -#include // For AF_LOCAL and SOCK_DGRAM +#include // For AF_UNIX and SOCK_DGRAM TEST(LlvmLibcSocketTest, LocalSocket) { - int sock = LIBC_NAMESPACE::socket(AF_LOCAL, SOCK_DGRAM, 0); + int sock = LIBC_NAMESPACE::socket(AF_UNIX, SOCK_DGRAM, 0); ASSERT_GE(sock, 0); ASSERT_EQ(libc_errno, 0); -- GitLab From 8227072f5aa87842295893175451213c88265a60 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 12 Dec 2023 13:33:53 -0800 Subject: [PATCH 010/281] [RISCV] Add missing break to last case in switch. NFC --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index f2ec422b54a9..0b682b0cbb33 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -16038,6 +16038,7 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode( assert(Subtarget.hasStdExtA()); return 33; } + break; } } -- GitLab From 9567b33fa15e5349cb522a3b3e39abc300c7644c Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Dec 2023 13:57:24 -0800 Subject: [PATCH 011/281] [asan] Switch initialization to "double-checked locking" This allows to remove `asan_init_is_running` which likely had a data race. Simplifies https://github.com/llvm/llvm-project/pull/74086 and reduces a difference between platforms. Reviewers: zacklj89, eugenis, dvyukov Reviewed By: zacklj89, dvyukov Pull Request: https://github.com/llvm/llvm-project/pull/74387 --- compiler-rt/lib/asan/asan_rtl.cpp | 49 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp index 04ecd20821fa..b28f9f181239 100644 --- a/compiler-rt/lib/asan/asan_rtl.cpp +++ b/compiler-rt/lib/asan/asan_rtl.cpp @@ -71,16 +71,16 @@ static void CheckUnwind() { } // -------------------------- Globals --------------------- {{{1 -static int asan_inited = 0; -static int asan_init_is_running = 0; +static StaticSpinMutex asan_inited_mutex; +static atomic_uint8_t asan_inited = {0}; -static void SetAsanInited() { asan_inited = 1; } - -static void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; } - -bool AsanInited() { return asan_inited == 1; } +static void SetAsanInited() { + atomic_store(&asan_inited, 1, memory_order_release); +} -static bool AsanInitIsRunning() { return asan_init_is_running == 1; } +bool AsanInited() { + return atomic_load(&asan_inited, memory_order_acquire) == 1; +} bool replace_intrin_cached; @@ -390,12 +390,10 @@ void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } -static void AsanInitInternal() { +static bool AsanInitInternal() { if (LIKELY(AsanInited())) - return; + return true; SanitizerToolName = "AddressSanitizer"; - CHECK(!AsanInitIsRunning() && "ASan init calls itself!"); - SetAsanInitIsRunning(1); CacheBinaryName(); @@ -408,9 +406,8 @@ static void AsanInitInternal() { // Stop performing init at this point if we are being loaded via // dlopen() and the platform supports it. if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { - SetAsanInitIsRunning(0); VReport(1, "AddressSanitizer init is being performed for dlopen().\n"); - return; + return false; } AsanCheckIncompatibleRT(); @@ -471,7 +468,6 @@ static void AsanInitInternal() { // should be set to 1 prior to initializing the threads. replace_intrin_cached = flags()->replace_intrin; SetAsanInited(); - SetAsanInitIsRunning(0); if (flags()->atexit) Atexit(asan_atexit); @@ -515,22 +511,27 @@ static void AsanInitInternal() { VReport(1, "AddressSanitizer Init done\n"); WaitForDebugger(flags()->sleep_after_init, "after init"); + + return true; } // Initialize as requested from some part of ASan runtime library (interceptors, // allocator, etc). void AsanInitFromRtl() { - CHECK(!AsanInitIsRunning()); - if (UNLIKELY(!AsanInited())) - AsanInitInternal(); + if (LIKELY(AsanInited())) + return; + SpinMutexLock lock(&asan_inited_mutex); + AsanInitInternal(); } bool TryAsanInitFromRtl() { - if (UNLIKELY(AsanInitIsRunning())) + if (LIKELY(AsanInited())) + return true; + if (!asan_inited_mutex.TryLock()) return false; - if (UNLIKELY(!AsanInited())) - AsanInitInternal(); - return true; + bool result = AsanInitInternal(); + asan_inited_mutex.Unlock(); + return result; } #if ASAN_DYNAMIC @@ -603,7 +604,7 @@ static void UnpoisonFakeStack() { using namespace __asan; void NOINLINE __asan_handle_no_return() { - if (AsanInitIsRunning()) + if (UNLIKELY(!AsanInited())) return; if (!PlatformUnpoisonStacks()) @@ -633,7 +634,7 @@ void NOINLINE __asan_set_death_callback(void (*callback)(void)) { // We use this call as a trigger to wake up ASan from deactivated state. void __asan_init() { AsanActivate(); - AsanInitInternal(); + AsanInitFromRtl(); } void __asan_version_mismatch_check() { -- GitLab From 8eff5704829ba5edd28754fd9ec7665b34fde22a Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Tue, 12 Dec 2023 14:10:06 -0800 Subject: [PATCH 012/281] Add missing dep on MLIRToLLVMIRTranslationRegistration to mlir-opt. (#75111) I was not able to fully triage why this just started failing on one of our bots as it seems that the use was added 4 months ago. I would assume that it was accidentally coming in transitively in some way as the dep was definitely missing. For context, this started failing in [our byo_llvm](https://github.com/openxla/iree/blob/main/build_tools/llvm/byo_llvm.sh) build on a stock build of MLIR on top of an existing LLVM. We were getting: ``` ld.lld: error: undefined symbol: mlir::registerSPIRVDialectTranslation(mlir::DialectRegistry&) >>> referenced by mlir-opt.cpp >>> tools/mlir-opt/CMakeFiles/mlir-opt.dir/mlir-opt.cpp.o:(main) ``` --- mlir/tools/mlir-opt/CMakeLists.txt | 4 ++++ mlir/tools/mlir-opt/mlir-opt.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt index bc8eed182155..b6ada66d3218 100644 --- a/mlir/tools/mlir-opt/CMakeLists.txt +++ b/mlir/tools/mlir-opt/CMakeLists.txt @@ -67,6 +67,10 @@ set(LIBS MLIRTransformUtils MLIRSupport MLIRIR + + # TODO: Remove when registerAllGPUToLLVMIRTranslations is no longer + # registered directly in mlir-opt.cpp. + MLIRToLLVMIRTranslationRegistration ) # Exclude from libMLIR.so because this has static options intended for diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp index c7cf1e55a556..b7c69eabbcd8 100644 --- a/mlir/tools/mlir-opt/mlir-opt.cpp +++ b/mlir/tools/mlir-opt/mlir-opt.cpp @@ -276,6 +276,10 @@ int main(int argc, char **argv) { DialectRegistry registry; registerAllDialects(registry); registerAllExtensions(registry); + + // TODO: Remove this and the corresponding MLIRToLLVMIRTranslationRegistration + // cmake dependency when a safe dialect interface registration mechanism is + // implemented, see D157703 (and corresponding note on the declaration). registerAllGPUToLLVMIRTranslations(registry); #ifdef MLIR_INCLUDE_TESTS -- GitLab From 365777ecbe18777431681fb54d068885044c6ef1 Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:34:31 -0800 Subject: [PATCH 013/281] [mlir][sparse] refactor utilities into transform/utils dir (#75250) Separates actual transformation files from supporting utility files in the transforms directory. Includes a bazel overlay fix for the build (as well as a bit of cleanup of that file to be less verbose and more flexible). --- .../SparseTensor/Transforms/CMakeLists.txt | 12 +- .../Transforms/SparseBufferRewriting.cpp | 2 +- .../Transforms/SparseGPUCodegen.cpp | 4 +- .../Transforms/SparseReinterpretMap.cpp | 4 +- .../SparseStorageSpecifierToLLVM.cpp | 2 +- .../Transforms/SparseTensorCodegen.cpp | 4 +- .../Transforms/SparseTensorConversion.cpp | 2 +- .../Transforms/SparseTensorRewriting.cpp | 4 +- .../Transforms/SparseVectorization.cpp | 4 +- .../Transforms/Sparsification.cpp | 6 +- .../Transforms/{ => Utils}/CodegenEnv.cpp | 0 .../Transforms/{ => Utils}/CodegenEnv.h | 0 .../Transforms/{ => Utils}/CodegenUtils.cpp | 0 .../Transforms/{ => Utils}/CodegenUtils.h | 0 .../{ => Utils}/IterationGraphSorter.cpp | 0 .../{ => Utils}/IterationGraphSorter.h | 0 .../Transforms/{ => Utils}/LoopEmitter.cpp | 0 .../Transforms/{ => Utils}/LoopEmitter.h | 0 .../{ => Utils}/SparseTensorDescriptor.cpp | 0 .../{ => Utils}/SparseTensorDescriptor.h | 0 .../llvm-project-overlay/mlir/BUILD.bazel | 117 ++++++++---------- 21 files changed, 76 insertions(+), 85 deletions(-) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/CodegenEnv.cpp (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/CodegenEnv.h (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/CodegenUtils.cpp (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/CodegenUtils.h (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/IterationGraphSorter.cpp (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/IterationGraphSorter.h (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/LoopEmitter.cpp (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/LoopEmitter.h (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/SparseTensorDescriptor.cpp (100%) rename mlir/lib/Dialect/SparseTensor/Transforms/{ => Utils}/SparseTensorDescriptor.h (100%) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt index 8459e46e5814..ad8b0d02eca3 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt @@ -1,9 +1,6 @@ add_mlir_dialect_library(MLIRSparseTensorTransforms + # Rewriting. BufferizableOpInterfaceImpl.cpp - CodegenEnv.cpp - CodegenUtils.cpp - IterationGraphSorter.cpp - LoopEmitter.cpp SparseBufferRewriting.cpp SparseGPUCodegen.cpp SparseReinterpretMap.cpp @@ -12,11 +9,16 @@ add_mlir_dialect_library(MLIRSparseTensorTransforms SparseTensorConversion.cpp SparseTensorPasses.cpp SparseTensorRewriting.cpp - SparseTensorDescriptor.cpp SparseVectorization.cpp Sparsification.cpp SparsificationAndBufferizationPass.cpp StageSparseOperations.cpp + # Utilities. + Utils/CodegenEnv.cpp + Utils/CodegenUtils.cpp + Utils/IterationGraphSorter.cpp + Utils/LoopEmitter.cpp + Utils/SparseTensorDescriptor.cpp ADDITIONAL_HEADER_DIRS ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SparseTensor diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp index cdbf4f048a00..248e9413b6e0 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" +#include "Utils/CodegenUtils.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Func/IR/FuncOps.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp index 5155cab772d4..477ff2e1c923 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" -#include "LoopEmitter.h" +#include "Utils/CodegenUtils.h" +#include "Utils/LoopEmitter.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp index 488079cfe4e3..f2e1b0bc58f1 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" -#include "IterationGraphSorter.h" +#include "Utils/CodegenUtils.h" +#include "Utils/IterationGraphSorter.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp index a6f4dd3c2f71..91dad050b288 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" +#include "Utils/CodegenUtils.h" #include "mlir/Conversion/LLVMCommon/StructBuilder.h" #include "mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp index 18b2bb0819e2..491501a3381b 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp @@ -15,8 +15,8 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" -#include "SparseTensorDescriptor.h" +#include "Utils/CodegenUtils.h" +#include "Utils/SparseTensorDescriptor.h" #include "mlir/Dialect/Arith/Utils/Utils.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp index e6052f2ca894..b0447b243661 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp @@ -16,7 +16,7 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" +#include "Utils/CodegenUtils.h" #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index 4fc692f2fe9d..3b9685b8ae1e 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" -#include "LoopEmitter.h" +#include "Utils/CodegenUtils.h" +#include "Utils/LoopEmitter.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp index 561c4e251146..7710a44a7ca0 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp @@ -16,8 +16,8 @@ // //===----------------------------------------------------------------------===// -#include "CodegenUtils.h" -#include "LoopEmitter.h" +#include "Utils/CodegenUtils.h" +#include "Utils/LoopEmitter.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 2367d3b5f37a..934e1e559f44 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CodegenEnv.h" -#include "CodegenUtils.h" -#include "LoopEmitter.h" +#include "Utils/CodegenEnv.h" +#include "Utils/CodegenUtils.h" +#include "Utils/LoopEmitter.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.cpp similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.cpp diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.h similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.h diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.cpp rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.h similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.h rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.h diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.cpp rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h similarity index 100% rename from mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.h rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 2e3bb8ad40b7..200998a3bf20 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -2773,13 +2773,9 @@ cc_library( td_library( name = "SparseTensorTdFiles", - srcs = [ - "include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td", - "include/mlir/Dialect/SparseTensor/IR/SparseTensorBase.td", - "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td", - "include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td", - "include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td", - ], + srcs = glob([ + "include/mlir/Dialect/SparseTensor/IR/*.td", + ]), includes = ["include"], deps = [ ":InferTypeOpInterfaceTdFiles", @@ -2788,15 +2784,6 @@ td_library( ], ) -td_library( - name = "SparseTensorInterfacesTdFiles", - srcs = [ - "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td", - ], - includes = ["include"], - deps = [":OpBaseTdFiles"], -) - gentbl_cc_library( name = "SparseTensorAttrDefsIncGen", tbl_outs = [ @@ -2918,7 +2905,37 @@ gentbl_cc_library( ], tblgen = ":mlir-tblgen", td_file = "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td", - deps = [":SparseTensorInterfacesTdFiles"], + deps = [":SparseTensorTdFiles"], +) + +td_library( + name = "SparseTensorTransformOpsTdFiles", + srcs = glob([ + "include/mlir/Dialect/SparseTensor/TransformOps/*.td", + ]), + includes = ["include"], + deps = [ + ":TransformDialectTdFiles", + ], +) + +gentbl_cc_library( + name = "SparseTensorTransformOpsIncGen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h.inc", + ), + ( + ["-gen-op-defs"], + "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.td", + deps = [ + ":SparseTensorTransformOpsTdFiles", + ], ) # This library is shared by both SparseTensorDialect and @@ -2932,19 +2949,11 @@ cc_library( cc_library( name = "SparseTensorDialect", - srcs = [ - "lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp", - "lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.h", - "lib/Dialect/SparseTensor/IR/Detail/DimLvlMapParser.cpp", - "lib/Dialect/SparseTensor/IR/Detail/DimLvlMapParser.h", - "lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp", - "lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.h", - "lib/Dialect/SparseTensor/IR/Detail/TemplateExtras.h", - "lib/Dialect/SparseTensor/IR/Detail/Var.cpp", - "lib/Dialect/SparseTensor/IR/Detail/Var.h", - "lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp", - "lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp", - ], + srcs = glob([ + "lib/Dialect/SparseTensor/IR/*.cpp", + "lib/Dialect/SparseTensor/IR/Detail/*.cpp", + "lib/Dialect/SparseTensor/IR/Detail/*.h", + ]), hdrs = [ "include/mlir/Dialect/SparseTensor/IR/SparseTensor.h", "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h", @@ -2987,40 +2996,14 @@ cc_library( ], ) -td_library( - name = "SparseTensorTransformOpsTdFiles", - srcs = glob([ - "include/mlir/Dialect/SparseTensor/TransformOps/*.td", - ]), - includes = ["include"], - deps = [ - ":TransformDialectTdFiles", - ], -) - -gentbl_cc_library( - name = "SparseTensorTransformOpsIncGen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h.inc", - ), - ( - ["-gen-op-defs"], - "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp.inc", - ), - ], - tblgen = ":mlir-tblgen", - td_file = "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.td", - deps = [ - ":SparseTensorTransformOpsTdFiles", - ], -) - cc_library( name = "SparseTensorUtils", - srcs = glob(["lib/Dialect/SparseTensor/Utils/*.cpp"]), - hdrs = glob(["include/mlir/Dialect/SparseTensor/Utils/*.h"]), + srcs = glob([ + "lib/Dialect/SparseTensor/Utils/*.cpp", + ]), + hdrs = glob([ + "include/mlir/Dialect/SparseTensor/Utils/*.h", + ]), includes = ["include"], deps = [ ":ArithDialect", @@ -3039,6 +3022,8 @@ cc_library( srcs = glob([ "lib/Dialect/SparseTensor/Transforms/*.cpp", "lib/Dialect/SparseTensor/Transforms/*.h", + "lib/Dialect/SparseTensor/Transforms/Utils/*.cpp", + "lib/Dialect/SparseTensor/Transforms/Utils/*.h", ]), hdrs = [ "include/mlir/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.h", @@ -3081,8 +3066,12 @@ cc_library( cc_library( name = "SparseTensorPipelines", - srcs = glob(["lib/Dialect/SparseTensor/Pipelines/*.cpp"]), - hdrs = ["include/mlir/Dialect/SparseTensor/Pipelines/Passes.h"], + srcs = glob([ + "lib/Dialect/SparseTensor/Pipelines/*.cpp", + ]), + hdrs = [ + "include/mlir/Dialect/SparseTensor/Pipelines/Passes.h", + ], includes = ["include"], local_defines = if_cuda_available(["MLIR_GPU_TO_CUBIN_PASS_ENABLE"]), deps = [ -- GitLab From 19fff858931bf575b63a0078cc553f8f93cced20 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Tue, 12 Dec 2023 16:27:12 -0800 Subject: [PATCH 014/281] Revert "[X86] Set SHF_X86_64_LARGE for globals with explicit well-known large section name (#74381)" This reverts commit 323451ab88866c42c87971cbc670771bd0d48692. Code with these section names in the wild doesn't compile because support for large globals in the small code model is not complete yet. --- llvm/lib/Target/TargetMachine.cpp | 4 ++-- llvm/test/CodeGen/X86/code-model-elf-sections.ll | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp index 246f6b4b492c..5428e14eca5f 100644 --- a/llvm/lib/Target/TargetMachine.cpp +++ b/llvm/lib/Target/TargetMachine.cpp @@ -55,6 +55,8 @@ bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const { // We should properly mark well-known section name prefixes as small/large, // because otherwise the output section may have the wrong section flags and // the linker will lay it out in an unexpected way. + // TODO: bring back lbss/ldata/lrodata checks after fixing accesses to large + // globals in the small code model. StringRef Name = GV->getSection(); if (!Name.empty()) { auto IsPrefix = [&](StringRef Prefix) { @@ -63,8 +65,6 @@ bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const { }; if (IsPrefix(".bss") || IsPrefix(".data") || IsPrefix(".rodata")) return false; - if (IsPrefix(".lbss") || IsPrefix(".ldata") || IsPrefix(".lrodata")) - return true; } // For x86-64, we treat an explicit GlobalVariable small code model to mean diff --git a/llvm/test/CodeGen/X86/code-model-elf-sections.ll b/llvm/test/CodeGen/X86/code-model-elf-sections.ll index 749d5b6bf904..cb19f0d34f59 100644 --- a/llvm/test/CodeGen/X86/code-model-elf-sections.ll +++ b/llvm/test/CodeGen/X86/code-model-elf-sections.ll @@ -21,16 +21,16 @@ ; SMALL: .data {{.*}} WA {{.*}} ; SMALL: .data.x {{.*}} WA {{.*}} ; SMALL: .data0 {{.*}} WA {{.*}} -; SMALL: .ldata {{.*}} WAl {{.*}} -; SMALL: .ldata.x {{.*}} WAl {{.*}} +; SMALL: .ldata {{.*}} WA {{.*}} +; SMALL: .ldata.x {{.*}} WA {{.*}} ; SMALL: .ldata0 {{.*}} WA {{.*}} ; SMALL: force_small {{.*}} WA {{.*}} ; SMALL: force_large {{.*}} WAl {{.*}} ; SMALL: foo {{.*}} WA {{.*}} ; SMALL: .bss {{.*}} WA {{.*}} -; SMALL: .lbss {{.*}} WAl {{.*}} +; SMALL: .lbss {{.*}} WA {{.*}} ; SMALL: .rodata {{.*}} A {{.*}} -; SMALL: .lrodata {{.*}} Al {{.*}} +; SMALL: .lrodata {{.*}} A {{.*}} ; SMALL: .data.rel.ro {{.*}} WA {{.*}} ; SMALL: .tbss {{.*}} WAT {{.*}} ; SMALL: .tdata {{.*}} WAT {{.*}} @@ -38,16 +38,16 @@ ; SMALL-DS: .data {{.*}} WA {{.*}} ; SMALL-DS: .data.x {{.*}} WA {{.*}} ; SMALL-DS: .data0 {{.*}} WA {{.*}} -; SMALL-DS: .ldata {{.*}} WAl {{.*}} -; SMALL-DS: .ldata.x {{.*}} WAl {{.*}} +; SMALL-DS: .ldata {{.*}} WA {{.*}} +; SMALL-DS: .ldata.x {{.*}} WA {{.*}} ; SMALL-DS: .ldata0 {{.*}} WA {{.*}} ; SMALL-DS: .data.data {{.*}} WA {{.*}} ; SMALL-DS: force_small {{.*}} WA {{.*}} ; SMALL-DS: force_large {{.*}} WAl {{.*}} ; SMALL-DS: foo {{.*}} WA {{.*}} -; SMALL-DS: .lbss {{.*}} WAl {{.*}} +; SMALL-DS: .lbss {{.*}} WA {{.*}} ; SMALL-DS: .bss.bss {{.*}} WA {{.*}} -; SMALL-DS: .lrodata {{.*}} Al {{.*}} +; SMALL-DS: .lrodata {{.*}} A {{.*}} ; SMALL-DS: .rodata.rodata {{.*}} A {{.*}} ; SMALL-DS: .data.rel.ro.relro {{.*}} WA {{.*}} ; SMALL-DS: .tbss.tbss {{.*}} WAT {{.*}} -- GitLab From 27259f17e9d273147c648331e92000a48677f489 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Wed, 13 Dec 2023 08:50:22 +0800 Subject: [PATCH 015/281] [CodeGen] Port `CFGuard` to new pass manager (#75146) Port `CFGuard` to new pass manager, add a pass parameter to choose guard mechanism. --- .../include/llvm/CodeGen/CodeGenPassBuilder.h | 1 + .../llvm/CodeGen/MachinePassRegistry.def | 3 +- llvm/include/llvm/Transforms/CFGuard.h | 13 +++ llvm/lib/Passes/CMakeLists.txt | 1 + llvm/lib/Passes/PassBuilder.cpp | 21 +++++ llvm/lib/Passes/PassRegistry.def | 4 + llvm/lib/Transforms/CFGuard/CFGuard.cpp | 85 +++++++++++-------- 7 files changed, 89 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 92bfef2b0148..fe604818886e 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -48,6 +48,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/CFGuard.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index 9ebf33b2b9a5..283fb14fee31 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -38,6 +38,7 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, #define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) #endif FUNCTION_PASS("callbrprepare", CallBrPreparePass, ()) +FUNCTION_PASS("cfguard", CFGuardPass, ()) FUNCTION_PASS("consthoist", ConstantHoistingPass, ()) FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false)) @@ -124,8 +125,6 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) #endif DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) -DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) -DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ()) DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) diff --git a/llvm/include/llvm/Transforms/CFGuard.h b/llvm/include/llvm/Transforms/CFGuard.h index 86fcbc3c13e8..caf822a2ec9f 100644 --- a/llvm/include/llvm/Transforms/CFGuard.h +++ b/llvm/include/llvm/Transforms/CFGuard.h @@ -11,10 +11,23 @@ #ifndef LLVM_TRANSFORMS_CFGUARD_H #define LLVM_TRANSFORMS_CFGUARD_H +#include "llvm/IR/PassManager.h" + namespace llvm { class FunctionPass; +class CFGuardPass : public PassInfoMixin { +public: + enum class Mechanism { Check, Dispatch }; + + CFGuardPass(Mechanism M = Mechanism::Check) : GuardMechanism(M) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + +private: + Mechanism GuardMechanism; +}; + /// Insert Control FLow Guard checks on indirect function calls. FunctionPass *createCFGuardCheckPass(); diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt index e42edfe94969..98d2de76c0e1 100644 --- a/llvm/lib/Passes/CMakeLists.txt +++ b/llvm/lib/Passes/CMakeLists.txt @@ -16,6 +16,7 @@ add_llvm_component_library(LLVMPasses LINK_COMPONENTS AggressiveInstCombine Analysis + CFGuard CodeGen Core Coroutines diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index c48e591fc600..f0417d6aa839 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -100,6 +100,7 @@ #include "llvm/Support/Regex.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" +#include "llvm/Transforms/CFGuard.h" #include "llvm/Transforms/Coroutines/CoroCleanup.h" #include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h" #include "llvm/Transforms/Coroutines/CoroEarly.h" @@ -738,6 +739,26 @@ Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { "PostOrderFunctionAttrs"); } +Expected parseCFGuardPassOptions(StringRef Params) { + if (Params.empty()) + return CFGuardPass::Mechanism::Check; + + auto [Param, RHS] = Params.split(';'); + if (!RHS.empty()) + return make_error( + formatv("too many CFGuardPass parameters '{0}' ", Params).str(), + inconvertibleErrorCode()); + + if (Param == "check") + return CFGuardPass::Mechanism::Check; + if (Param == "dispatch") + return CFGuardPass::Mechanism::Dispatch; + + return make_error( + formatv("invalid CFGuardPass mechanism: '{0}' ", Param).str(), + inconvertibleErrorCode()); +} + Expected parseEarlyCSEPassOptions(StringRef Params) { return parseSinglePassOption(Params, "memssa", "EarlyCSE"); } diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 6afc8b4898fe..1a9a34859332 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -433,6 +433,10 @@ FUNCTION_PASS("wasm-eh-prepare", WasmEHPreparePass()) #ifndef FUNCTION_PASS_WITH_PARAMS #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) #endif +FUNCTION_PASS_WITH_PARAMS( + "cfguard", "CFGuardPass", + [](CFGuardPass::Mechanism M) { return CFGuardPass(M); }, + parseCFGuardPassOptions, "check;dispatch") FUNCTION_PASS_WITH_PARAMS( "early-cse", "EarlyCSEPass", [](bool UseMemorySSA) { return EarlyCSEPass(UseMemorySSA); }, diff --git a/llvm/lib/Transforms/CFGuard/CFGuard.cpp b/llvm/lib/Transforms/CFGuard/CFGuard.cpp index 387734358775..4d4306576017 100644 --- a/llvm/lib/Transforms/CFGuard/CFGuard.cpp +++ b/llvm/lib/Transforms/CFGuard/CFGuard.cpp @@ -34,25 +34,22 @@ namespace { /// Adds Control Flow Guard (CFG) checks on indirect function calls/invokes. /// These checks ensure that the target address corresponds to the start of an -/// address-taken function. X86_64 targets use the CF_Dispatch mechanism. X86, -/// ARM, and AArch64 targets use the CF_Check machanism. -class CFGuard : public FunctionPass { +/// address-taken function. X86_64 targets use the Mechanism::Dispatch +/// mechanism. X86, ARM, and AArch64 targets use the Mechanism::Check machanism. +class CFGuardImpl { public: - static char ID; - - enum Mechanism { CF_Check, CF_Dispatch }; - - // Default constructor required for the INITIALIZE_PASS macro. - CFGuard() : FunctionPass(ID) { - initializeCFGuardPass(*PassRegistry::getPassRegistry()); - // By default, use the guard check mechanism. - GuardMechanism = CF_Check; - } - - // Recommended constructor used to specify the type of guard mechanism. - CFGuard(Mechanism Var) : FunctionPass(ID) { - initializeCFGuardPass(*PassRegistry::getPassRegistry()); - GuardMechanism = Var; + using Mechanism = CFGuardPass::Mechanism; + + CFGuardImpl(Mechanism M) : GuardMechanism(M) { + // Get or insert the guard check or dispatch global symbols. + switch (GuardMechanism) { + case Mechanism::Check: + GuardFnName = "__guard_check_icall_fptr"; + break; + case Mechanism::Dispatch: + GuardFnName = "__guard_dispatch_icall_fptr"; + break; + } } /// Inserts a Control Flow Guard (CFG) check on an indirect call using the CFG @@ -141,21 +138,37 @@ public: /// \param CB indirect call to instrument. void insertCFGuardDispatch(CallBase *CB); - bool doInitialization(Module &M) override; - bool runOnFunction(Function &F) override; + bool doInitialization(Module &M); + bool runOnFunction(Function &F); private: // Only add checks if the module has the cfguard=2 flag. int cfguard_module_flag = 0; - Mechanism GuardMechanism = CF_Check; + StringRef GuardFnName; + Mechanism GuardMechanism = Mechanism::Check; FunctionType *GuardFnType = nullptr; PointerType *GuardFnPtrType = nullptr; Constant *GuardFnGlobal = nullptr; }; +class CFGuard : public FunctionPass { + CFGuardImpl Impl; + +public: + static char ID; + + // Default constructor required for the INITIALIZE_PASS macro. + CFGuard(CFGuardImpl::Mechanism M) : FunctionPass(ID), Impl(M) { + initializeCFGuardPass(*PassRegistry::getPassRegistry()); + } + + bool doInitialization(Module &M) override { return Impl.doInitialization(M); } + bool runOnFunction(Function &F) override { return Impl.runOnFunction(F); } +}; + } // end anonymous namespace -void CFGuard::insertCFGuardCheck(CallBase *CB) { +void CFGuardImpl::insertCFGuardCheck(CallBase *CB) { assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() && "Only applicable for Windows targets"); @@ -184,7 +197,7 @@ void CFGuard::insertCFGuardCheck(CallBase *CB) { GuardCheck->setCallingConv(CallingConv::CFGuard_Check); } -void CFGuard::insertCFGuardDispatch(CallBase *CB) { +void CFGuardImpl::insertCFGuardDispatch(CallBase *CB) { assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() && "Only applicable for Windows targets"); @@ -218,7 +231,7 @@ void CFGuard::insertCFGuardDispatch(CallBase *CB) { CB->eraseFromParent(); } -bool CFGuard::doInitialization(Module &M) { +bool CFGuardImpl::doInitialization(Module &M) { // Check if this module has the cfguard flag and read its value. if (auto *MD = @@ -235,15 +248,6 @@ bool CFGuard::doInitialization(Module &M) { {PointerType::getUnqual(M.getContext())}, false); GuardFnPtrType = PointerType::get(GuardFnType, 0); - // Get or insert the guard check or dispatch global symbols. - llvm::StringRef GuardFnName; - if (GuardMechanism == CF_Check) { - GuardFnName = "__guard_check_icall_fptr"; - } else if (GuardMechanism == CF_Dispatch) { - GuardFnName = "__guard_dispatch_icall_fptr"; - } else { - assert(false && "Invalid CFGuard mechanism"); - } GuardFnGlobal = M.getOrInsertGlobal(GuardFnName, GuardFnPtrType, [&] { auto *Var = new GlobalVariable(M, GuardFnPtrType, false, GlobalVariable::ExternalLinkage, nullptr, @@ -255,7 +259,7 @@ bool CFGuard::doInitialization(Module &M) { return true; } -bool CFGuard::runOnFunction(Function &F) { +bool CFGuardImpl::runOnFunction(Function &F) { // Skip modules for which CFGuard checks have been disabled. if (cfguard_module_flag != 2) @@ -283,7 +287,7 @@ bool CFGuard::runOnFunction(Function &F) { } // For each indirect call/invoke, add the appropriate dispatch or check. - if (GuardMechanism == CF_Dispatch) { + if (GuardMechanism == Mechanism::Dispatch) { for (CallBase *CB : IndirectCalls) { insertCFGuardDispatch(CB); } @@ -296,13 +300,20 @@ bool CFGuard::runOnFunction(Function &F) { return true; } +PreservedAnalyses CFGuardPass::run(Function &F, FunctionAnalysisManager &FAM) { + CFGuardImpl Impl(GuardMechanism); + bool Changed = Impl.doInitialization(*F.getParent()); + Changed |= Impl.runOnFunction(F); + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + char CFGuard::ID = 0; INITIALIZE_PASS(CFGuard, "CFGuard", "CFGuard", false, false) FunctionPass *llvm::createCFGuardCheckPass() { - return new CFGuard(CFGuard::CF_Check); + return new CFGuard(CFGuardPass::Mechanism::Check); } FunctionPass *llvm::createCFGuardDispatchPass() { - return new CFGuard(CFGuard::CF_Dispatch); + return new CFGuard(CFGuardPass::Mechanism::Dispatch); } -- GitLab From dd9587795811ba21e6ca6ad52b4531e17e6babd6 Mon Sep 17 00:00:00 2001 From: Greg Clayton Date: Tue, 12 Dec 2023 16:51:49 -0800 Subject: [PATCH 016/281] [lldb] Make only one function that needs to be implemented when searching for types (#74786) This patch revives the effort to get this Phabricator patch into upstream: https://reviews.llvm.org/D137900 This patch was accepted before in Phabricator but I found some -gsimple-template-names issues that are fixed in this patch. A fixed up version of the description from the original patch starts now. This patch started off trying to fix Module::FindFirstType() as it sometimes didn't work. The issue was the SymbolFile plug-ins didn't do any filtering of the matching types they produced, and they only looked up types using the type basename. This means if you have two types with the same basename, your type lookup can fail when only looking up a single type. We would ask the Module::FindFirstType to lookup "Foo::Bar" and it would ask the symbol file to find only 1 type matching the basename "Bar", and then we would filter out any matches that didn't match "Foo::Bar". So if the SymbolFile found "Foo::Bar" first, then it would work, but if it found "Baz::Bar" first, it would return only that type and it would be filtered out. Discovering this issue lead me to think of the patch Alex Langford did a few months ago that was done for finding functions, where he allowed SymbolFile objects to make sure something fully matched before parsing the debug information into an AST type and other LLDB types. So this patch aimed to allow type lookups to also be much more efficient. As LLDB has been developed over the years, we added more ways to to type lookups. These functions have lots of arguments. This patch aims to make one API that needs to be implemented that serves all previous lookups: - Find a single type - Find all types - Find types in a namespace This patch introduces a `TypeQuery` class that contains all of the state needed to perform the lookup which is powerful enough to perform all of the type searches that used to be in our API. It contain a vector of CompilerContext objects that can fully or partially specify the lookup that needs to take place. If you just want to lookup all types with a matching basename, regardless of the containing context, you can specify just a single CompilerContext entry that has a name and a CompilerContextKind mask of CompilerContextKind::AnyType. Or you can fully specify the exact context to use when doing lookups like: CompilerContextKind::Namespace "std" CompilerContextKind::Class "foo" CompilerContextKind::Typedef "size_type" This change expands on the clang modules code that already used a vector items, but it modifies it to work with expression type lookups which have contexts, or user lookups where users query for types. The clang modules type lookup is still an option that can be enabled on the `TypeQuery` objects. This mirrors the most recent addition of type lookups that took a vector that allowed lookups to happen for the expression parser in certain places. Prior to this we had the following APIs in Module: ``` void Module::FindTypes(ConstString type_name, bool exact_match, size_t max_matches, llvm::DenseSet &searched_symbol_files, TypeList &types); void Module::FindTypes(llvm::ArrayRef pattern, LanguageSet languages, llvm::DenseSet &searched_symbol_files, TypeMap &types); void Module::FindTypesInNamespace(ConstString type_name, const CompilerDeclContext &parent_decl_ctx, size_t max_matches, TypeList &type_list); ``` The new Module API is much simpler. It gets rid of all three above functions and replaces them with: ``` void FindTypes(const TypeQuery &query, TypeResults &results); ``` The `TypeQuery` class contains all of the needed settings: - The vector that allow efficient lookups in the symbol file classes since they can look at basename matches only realize fully matching types. Before this any basename that matched was fully realized only to be removed later by code outside of the SymbolFile layer which could cause many types to be realized when they didn't need to. - If the lookup is exact or not. If not exact, then the compiler context must match the bottom most items that match the compiler context, otherwise it must match exactly - If the compiler context match is for clang modules or not. Clang modules matches include a Module compiler context kind that allows types to be matched only from certain modules and these matches are not needed when d oing user type lookups. - An optional list of languages to use to limit the search to only certain languages The `TypeResults` object contains all state required to do the lookup and store the results: - The max number of matches - The set of SymbolFile objects that have already been searched - The matching type list for any matches that are found The benefits of this approach are: - Simpler API, and only one API to implement in SymbolFile classes - Replaces the FindTypesInNamespace that used a CompilerDeclContext as a way to limit the search, but this only worked if the TypeSystem matched the current symbol file's type system, so you couldn't use it to lookup a type in another module - Fixes a serious bug in our FindFirstType functions where if we were searching for "foo::bar", and we found a "baz::bar" first, the basename would match and we would only fetch 1 type using the basename, only to drop it from the matching list and returning no results --- lldb/include/lldb/Core/Module.h | 77 +---- lldb/include/lldb/Core/ModuleList.h | 24 +- lldb/include/lldb/Symbol/CompilerDecl.h | 7 + .../include/lldb/Symbol/CompilerDeclContext.h | 8 + lldb/include/lldb/Symbol/SymbolFile.h | 29 +- lldb/include/lldb/Symbol/SymbolFileOnDemand.h | 13 +- lldb/include/lldb/Symbol/Type.h | 305 ++++++++++++++++++ lldb/include/lldb/Symbol/TypeMap.h | 6 +- lldb/include/lldb/Symbol/TypeSystem.h | 24 +- lldb/include/lldb/lldb-forward.h | 2 + lldb/include/lldb/lldb-private-enumerations.h | 5 +- lldb/source/API/SBModule.cpp | 54 ++-- lldb/source/API/SBTarget.cpp | 38 +-- lldb/source/Commands/CommandObjectMemory.cpp | 23 +- lldb/source/Commands/CommandObjectTarget.cpp | 51 +-- lldb/source/Core/Module.cpp | 94 +----- lldb/source/Core/ModuleList.cpp | 37 +-- lldb/source/DataFormatters/TypeFormat.cpp | 11 +- .../ExpressionParser/Clang/ClangASTSource.cpp | 97 +++--- .../ItaniumABI/ItaniumABILanguageRuntime.cpp | 30 +- .../ObjC/ObjCLanguageRuntime.cpp | 15 +- .../Breakpad/SymbolFileBreakpad.cpp | 9 - .../SymbolFile/Breakpad/SymbolFileBreakpad.h | 9 - .../Plugins/SymbolFile/CTF/SymbolFileCTF.cpp | 23 +- .../Plugins/SymbolFile/CTF/SymbolFileCTF.h | 8 +- .../SymbolFile/DWARF/DWARFASTParserClang.cpp | 32 +- .../Plugins/SymbolFile/DWARF/DWARFDIE.cpp | 48 +++ .../Plugins/SymbolFile/DWARF/DWARFDIE.h | 15 +- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 270 +++++++--------- .../SymbolFile/DWARF/SymbolFileDWARF.h | 10 +- .../DWARF/SymbolFileDWARFDebugMap.cpp | 23 +- .../DWARF/SymbolFileDWARFDebugMap.h | 9 +- .../NativePDB/SymbolFileNativePDB.cpp | 37 ++- .../NativePDB/SymbolFileNativePDB.h | 10 +- .../Plugins/SymbolFile/PDB/SymbolFilePDB.cpp | 59 ++-- .../Plugins/SymbolFile/PDB/SymbolFilePDB.h | 15 +- .../TypeSystem/Clang/TypeSystemClang.cpp | 151 ++++----- .../TypeSystem/Clang/TypeSystemClang.h | 8 +- lldb/source/Symbol/CompilerDecl.cpp | 5 + lldb/source/Symbol/CompilerDeclContext.cpp | 7 + lldb/source/Symbol/SymbolFile.cpp | 11 - lldb/source/Symbol/SymbolFileOnDemand.cpp | 23 +- lldb/source/Symbol/Type.cpp | 146 ++++++++- lldb/source/Symbol/TypeMap.cpp | 14 +- lldb/source/Symbol/TypeSystem.cpp | 10 + lldb/source/Target/Language.cpp | 10 +- .../functionalities/type_find_first/Makefile | 2 + .../type_find_first/TestFindFirstType.py | 38 +++ .../functionalities/type_find_first/main.cpp | 17 + .../cpp/unique-types4/TestUniqueTypes4.py | 21 +- lldb/test/API/lang/cpp/unique-types4/main.cpp | 4 + lldb/tools/lldb-test/lldb-test.cpp | 48 +-- 52 files changed, 1135 insertions(+), 907 deletions(-) create mode 100644 lldb/test/API/functionalities/type_find_first/Makefile create mode 100644 lldb/test/API/functionalities/type_find_first/TestFindFirstType.py create mode 100644 lldb/test/API/functionalities/type_find_first/main.cpp diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h index 2973ee0e7ec4..f4973cdda1ef 100644 --- a/lldb/include/lldb/Core/Module.h +++ b/lldb/include/lldb/Core/Module.h @@ -415,70 +415,19 @@ public: void FindGlobalVariables(const RegularExpression ®ex, size_t max_matches, VariableList &variable_list); - /// Find types by name. - /// - /// Type lookups in modules go through the SymbolFile. The SymbolFile needs to - /// be able to lookup types by basename and not the fully qualified typename. - /// This allows the type accelerator tables to stay small, even with heavily - /// templatized C++. The type search will then narrow down the search - /// results. If "exact_match" is true, then the type search will only match - /// exact type name matches. If "exact_match" is false, the type will match - /// as long as the base typename matches and as long as any immediate - /// containing namespaces/class scopes that are specified match. So to - /// search for a type "d" in "b::c", the name "b::c::d" can be specified and - /// it will match any class/namespace "b" which contains a class/namespace - /// "c" which contains type "d". We do this to allow users to not always - /// have to specify complete scoping on all expressions, but it also allows - /// for exact matching when required. - /// - /// \param[in] type_name - /// The name of the type we are looking for that is a fully - /// or partially qualified type name. - /// - /// \param[in] exact_match - /// If \b true, \a type_name is fully qualified and must match - /// exactly. If \b false, \a type_name is a partially qualified - /// name where the leading namespaces or classes can be - /// omitted to make finding types that a user may type - /// easier. - /// - /// \param[out] types - /// A type list gets populated with any matches. + /// Find types using a type-matching object that contains all search + /// parameters. /// - void - FindTypes(ConstString type_name, bool exact_match, size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeList &types); - - /// Find types by name. - /// - /// This behaves like the other FindTypes method but allows to - /// specify a DeclContext and a language for the type being searched - /// for. - /// - /// \param searched_symbol_files - /// Prevents one file from being visited multiple times. - void - FindTypes(llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types); - - lldb::TypeSP FindFirstType(const SymbolContext &sc, ConstString type_name, - bool exact_match); - - /// Find types by name that are in a namespace. This function is used by the - /// expression parser when searches need to happen in an exact namespace - /// scope. + /// \see lldb_private::TypeQuery /// - /// \param[in] type_name - /// The name of a type within a namespace that should not include - /// any qualifying namespaces (just a type basename). + /// \param[in] query + /// A type matching object that contains all of the details of the type + /// search. /// - /// \param[out] type_list - /// A type list gets populated with any matches. - void FindTypesInNamespace(ConstString type_name, - const CompilerDeclContext &parent_decl_ctx, - size_t max_matches, TypeList &type_list); + /// \param[in] results + /// Any matching types will be populated into the \a results object using + /// TypeMap::InsertUnique(...). + void FindTypes(const TypeQuery &query, TypeResults &results); /// Get const accessor for the module architecture. /// @@ -1122,12 +1071,6 @@ protected: private: Module(); // Only used internally by CreateJITModule () - void FindTypes_Impl( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types); - Module(const Module &) = delete; const Module &operator=(const Module &) = delete; diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h index 9826dd09e91d..d78f7c5ef3f7 100644 --- a/lldb/include/lldb/Core/ModuleList.h +++ b/lldb/include/lldb/Core/ModuleList.h @@ -340,26 +340,22 @@ public: lldb::SymbolType symbol_type, SymbolContextList &sc_list) const; - /// Find types by name. + /// Find types using a type-matching object that contains all search + /// parameters. /// /// \param[in] search_first /// If non-null, this module will be searched before any other /// modules. /// - /// \param[in] name - /// The name of the type we are looking for. - /// - /// \param[in] max_matches - /// Allow the number of matches to be limited to \a - /// max_matches. Specify UINT32_MAX to get all possible matches. - /// - /// \param[out] types - /// A type list gets populated with any matches. + /// \param[in] query + /// A type matching object that contains all of the details of the type + /// search. /// - void FindTypes(Module *search_first, ConstString name, - bool name_is_fully_qualified, size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeList &types) const; + /// \param[in] results + /// Any matching types will be populated into the \a results object using + /// TypeMap::InsertUnique(...). + void FindTypes(Module *search_first, const TypeQuery &query, + lldb_private::TypeResults &results) const; bool FindSourceFile(const FileSpec &orig_spec, FileSpec &new_spec) const; diff --git a/lldb/include/lldb/Symbol/CompilerDecl.h b/lldb/include/lldb/Symbol/CompilerDecl.h index 67290b9be066..825a4f15836f 100644 --- a/lldb/include/lldb/Symbol/CompilerDecl.h +++ b/lldb/include/lldb/Symbol/CompilerDecl.h @@ -84,6 +84,13 @@ public: // based argument index CompilerType GetFunctionArgumentType(size_t arg_idx) const; + /// Populate a valid compiler context from the current declaration. + /// + /// \returns A valid vector of CompilerContext entries that describes + /// this declaration. The first entry in the vector is the parent of + /// the subsequent entry, so the topmost entry is the global namespace. + std::vector GetCompilerContext() const; + private: TypeSystem *m_type_system = nullptr; void *m_opaque_decl = nullptr; diff --git a/lldb/include/lldb/Symbol/CompilerDeclContext.h b/lldb/include/lldb/Symbol/CompilerDeclContext.h index 61a9c9c341bf..89b4a9787688 100644 --- a/lldb/include/lldb/Symbol/CompilerDeclContext.h +++ b/lldb/include/lldb/Symbol/CompilerDeclContext.h @@ -11,6 +11,7 @@ #include +#include "lldb/Symbol/Type.h" #include "lldb/Utility/ConstString.h" #include "lldb/lldb-private.h" @@ -56,6 +57,13 @@ public: return m_type_system != nullptr && m_opaque_decl_ctx != nullptr; } + /// Populate a valid compiler context from the current decl context. + /// + /// \returns A valid vector of CompilerContext entries that describes + /// this declaration context. The first entry in the vector is the parent of + /// the subsequent entry, so the topmost entry is the global namespace. + std::vector GetCompilerContext() const; + std::vector FindDeclByName(ConstString name, const bool ignore_using_decls); diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h index a546b05bfd31..c9a2a647a039 100644 --- a/lldb/include/lldb/Symbol/SymbolFile.h +++ b/lldb/include/lldb/Symbol/SymbolFile.h @@ -301,21 +301,20 @@ public: bool include_inlines, SymbolContextList &sc_list); virtual void FindFunctions(const RegularExpression ®ex, bool include_inlines, SymbolContextList &sc_list); - virtual void - FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types); - - /// Find types specified by a CompilerContextPattern. - /// \param languages - /// Only return results in these languages. - /// \param searched_symbol_files - /// Prevents one file from being visited multiple times. - virtual void - FindTypes(llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types); + + /// Find types using a type-matching object that contains all search + /// parameters. + /// + /// \see lldb_private::TypeQuery + /// + /// \param[in] query + /// A type matching object that contains all of the details of the type + /// search. + /// + /// \param[in] results + /// Any matching types will be populated into the \a results object using + /// TypeMap::InsertUnique(...). + virtual void FindTypes(const TypeQuery &query, TypeResults &results) {} virtual void GetMangledNamesForFunction(const std::string &scope_qualified_name, diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h index 9cbcef2a111d..cde9f3c3b8ce 100644 --- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h +++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h @@ -152,17 +152,8 @@ public: const std::string &scope_qualified_name, std::vector &mangled_names) override; - void - FindTypes(lldb_private::ConstString name, - const lldb_private::CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) override; - - void FindTypes(llvm::ArrayRef pattern, - lldb_private::LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) override; + void FindTypes(const lldb_private::TypeQuery &query, + lldb_private::TypeResults &results) override; void GetTypes(lldb_private::SymbolContextScope *sc_scope, lldb::TypeClass type_mask, diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h index 15edbea3cc7a..307be6c55e01 100644 --- a/lldb/include/lldb/Symbol/Type.h +++ b/lldb/include/lldb/Symbol/Type.h @@ -12,11 +12,15 @@ #include "lldb/Core/Declaration.h" #include "lldb/Symbol/CompilerDecl.h" #include "lldb/Symbol/CompilerType.h" +#include "lldb/Symbol/TypeList.h" +#include "lldb/Symbol/TypeMap.h" +#include "lldb/Symbol/TypeSystem.h" #include "lldb/Utility/ConstString.h" #include "lldb/Utility/UserID.h" #include "lldb/lldb-private.h" #include "llvm/ADT/APSInt.h" +#include "llvm/ADT/DenseSet.h" #include #include @@ -24,6 +28,23 @@ namespace lldb_private { class SymbolFileCommon; +/// A SmallBitVector that represents a set of source languages (\p +/// lldb::LanguageType). Each lldb::LanguageType is represented by +/// the bit with the position of its enumerator. The largest +/// LanguageType is < 64, so this is space-efficient and on 64-bit +/// architectures a LanguageSet can be completely stack-allocated. +struct LanguageSet { + llvm::SmallBitVector bitvector; + LanguageSet(); + + /// If the set contains a single language only, return it. + std::optional GetSingularLanguage(); + void Insert(lldb::LanguageType language); + bool Empty() const; + size_t Size() const; + bool operator[](unsigned i) const; +}; + /// CompilerContext allows an array of these items to be passed to perform /// detailed lookups in SymbolVendor and SymbolFile functions. struct CompilerContext { @@ -45,6 +66,290 @@ struct CompilerContext { bool contextMatches(llvm::ArrayRef context_chain, llvm::ArrayRef pattern); +FLAGS_ENUM(TypeQueryOptions){ + e_none = 0u, + /// If set, TypeQuery::m_context contains an exact context that must match + /// the full context. If not set, TypeQuery::m_context can contain a partial + /// type match where the full context isn't fully specified. + e_exact_match = (1u << 0), + /// If set, TypeQuery::m_context is a clang module compiler context. If not + /// set TypeQuery::m_context is normal type lookup context. + e_module_search = (1u << 1), + /// When true, the find types call should stop the query as soon as a single + /// matching type is found. When false, the type query should find all + /// matching types. + e_find_one = (1u << 2), +}; +LLDB_MARK_AS_BITMASK_ENUM(TypeQueryOptions) + +/// A class that contains all state required for type lookups. +/// +/// Using a TypeQuery class for matching types simplifies the internal APIs we +/// need to implement type lookups in LLDB. Type lookups can fully specify the +/// exact typename by filling out a complete or partial CompilerContext array. +/// This technique allows for powerful searches and also allows the SymbolFile +/// classes to use the m_context array to lookup types by basename, then +/// eliminate potential matches without having to resolve types into each +/// TypeSystem. This makes type lookups vastly more efficient and allows the +/// SymbolFile objects to stop looking up types when the type matching is +/// complete, like if we are looking for only a single type in our search. +class TypeQuery { +public: + TypeQuery() = delete; + + /// Construct a type match object using a fully- or partially-qualified name. + /// + /// The specified \a type_name will be chopped up and the m_context will be + /// populated by separating the string by looking for "::". We do this because + /// symbol files have indexes that contain only the type's basename. This also + /// allows symbol files to efficiently not realize types that don't match the + /// specified context. Example of \a type_name values that can be specified + /// include: + /// "foo": Look for any type whose basename matches "foo". + /// If \a exact_match is true, then the type can't be contained in any + /// declaration context like a namespace, class, or other containing + /// scope. + /// If \a exact match is false, then we will find all matches including + /// ones that are contained in other declaration contexts, including top + /// level types. + /// "foo::bar": Look for any type whose basename matches "bar" but make sure + /// its parent declaration context is any named declaration context + /// (namespace, class, struct, etc) whose name matches "foo". + /// If \a exact_match is true, then the "foo" declaration context must + /// appear at the source file level or inside of a function. + /// If \a exact match is false, then the "foo" declaration context can + /// be contained in any other declaration contexts. + /// "class foo": Only match types that are classes whose basename matches + /// "foo". + /// "struct foo": Only match types that are structures whose basename + /// matches "foo". + /// "class foo::bar": Only match types that are classes whose basename + /// matches "bar" and that are contained in any named declaration context + /// named "foo". + /// + /// \param[in] type_name + /// A fully- or partially-qualified type name. This name will be parsed and + /// broken up and the m_context will be populated with the various parts of + /// the name. This typename can be prefixed with "struct ", "class ", + /// "union", "enum " or "typedef " before the actual type name to limit the + /// results of the types that match. The declaration context can be + /// specified with the "::" string. For example, "a::b::my_type". + /// + /// \param[in] options A set of boolean enumeration flags from the + /// TypeQueryOptions enumerations. \see TypeQueryOptions. + TypeQuery(llvm::StringRef name, TypeQueryOptions options = e_none); + + /// Construct a type-match object that matches a type basename that exists + /// in the specified declaration context. + /// + /// This allows the m_context to be first populated using a declaration + /// context to exactly identify the containing declaration context of a type. + /// This can be used when you have a forward declaration to a type and you + /// need to search for its complete type. + /// + /// \param[in] decl_ctx + /// A declaration context object that comes from a TypeSystem plug-in. This + /// object will be asked to populate the array of CompilerContext objects + /// by adding the top most declaration context first into the array and then + /// adding any containing declaration contexts. + /// + /// \param[in] type_basename + /// The basename of the type to lookup in the specified declaration context. + /// + /// \param[in] options A set of boolean enumeration flags from the + /// TypeQueryOptions enumerations. \see TypeQueryOptions. + TypeQuery(const CompilerDeclContext &decl_ctx, ConstString type_basename, + TypeQueryOptions options = e_none); + /// Construct a type-match object using a compiler declaration that specifies + /// a typename and a declaration context to use when doing exact type lookups. + /// + /// This allows the m_context to be first populated using a type declaration. + /// The type declaration might have a declaration context and each TypeSystem + /// plug-in can populate the declaration context needed to perform an exact + /// lookup for a type. + /// This can be used when you have a forward declaration to a type and you + /// need to search for its complete type. + /// + /// \param[in] decl + /// A type declaration context object that comes from a TypeSystem plug-in. + /// This object will be asked to full the array of CompilerContext objects + /// by adding the top most declaration context first into the array and then + /// adding any containing declaration contexts, and ending with the exact + /// typename and the kind of type it is (class, struct, union, enum, etc). + /// + /// \param[in] options A set of boolean enumeration flags from the + /// TypeQueryOptions enumerations. \see TypeQueryOptions. + TypeQuery(const CompilerDecl &decl, TypeQueryOptions options = e_none); + + /// Construct a type-match object using a CompilerContext array. + /// + /// Clients can manually create compiler contexts and use these to find + /// matches when searching for types. There are two types of contexts that + /// are supported when doing type searchs: type contexts and clang module + /// contexts. Type contexts have contexts that specify the type and its + /// containing declaration context like namespaces and classes. Clang module + /// contexts specify contexts more completely to find exact matches within + /// clang module debug information. They will include the modules that the + /// type is included in and any functions that the type might be defined in. + /// This allows very fine-grained type resolution. + /// + /// \param[in] context The compiler context to use when doing the search. + /// + /// \param[in] options A set of boolean enumeration flags from the + /// TypeQueryOptions enumerations. \see TypeQueryOptions. + TypeQuery(const llvm::ArrayRef &context, + TypeQueryOptions options = e_none); + + /// Construct a type-match object that duplicates all matching criterea, + /// but not any searched symbol files or the type map for matches. This allows + /// the m_context to be modified prior to performing another search. + TypeQuery(const TypeQuery &rhs) = default; + /// Assign a type-match object that duplicates all matching criterea, + /// but not any searched symbol files or the type map for matches. This allows + /// the m_context to be modified prior to performing another search. + TypeQuery &operator=(const TypeQuery &rhs) = default; + + /// Check of a CompilerContext array from matching type from a symbol file + /// matches the \a m_context. + /// + /// \param[in] context + /// A fully qualified CompilerContext array for a potential match that is + /// created by the symbol file prior to trying to actually resolve a type. + /// + /// \returns + /// True if the context matches, false if it doesn't. If e_exact_match + /// is set in m_options, then \a context must exactly match \a m_context. If + /// e_exact_match is not set, then the bottom m_context.size() objects in + /// \a context must match. This allows SymbolFile objects the fill in a + /// potential type basename match from the index into \a context, and see if + /// it matches prior to having to resolve a lldb_private::Type object for + /// the type from the index. This allows type parsing to be as efficient as + /// possible and only realize the types that match the query. + bool + ContextMatches(llvm::ArrayRef context) const; + + /// Get the type basename to use when searching the type indexes in each + /// SymbolFile object. + /// + /// Debug information indexes often contain indexes that track the basename + /// of types only, not a fully qualified path. This allows the indexes to be + /// smaller and allows for efficient lookups. + /// + /// \returns + /// The type basename to use when doing lookups as a constant string. + ConstString GetTypeBasename() const; + + /// Returns true if any matching languages have been specified in this type + /// matching object. + bool HasLanguage() const { return m_languages.has_value(); } + + /// Add a language family to the list of languages that should produce a + /// match. + void AddLanguage(lldb::LanguageType language); + + /// Check if the language matches any languages that have been added to this + /// match object. + /// + /// \returns + /// True if no language have been specified, or if some language have been + /// added using AddLanguage(...) and they match. False otherwise. + bool LanguageMatches(lldb::LanguageType language) const; + + bool GetExactMatch() const { return (m_options & e_exact_match) != 0; } + /// The \a m_context can be used in two ways: normal types searching with + /// the context containing a stanadard declaration context for a type, or + /// with the context being more complete for exact matches in clang modules. + /// Set this to true if you wish to search for a type in clang module. + bool GetModuleSearch() const { return (m_options & e_module_search) != 0; } + + /// Returns true if the type query is supposed to find only a single matching + /// type. Returns false if the type query should find all matches. + bool GetFindOne() const { return (m_options & e_find_one) != 0; } + void SetFindOne(bool b) { + if (b) + m_options |= e_find_one; + else + m_options &= (e_exact_match | e_find_one); + } + + /// Access the internal compiler context array. + /// + /// Clients can use this to populate the context manually. + std::vector &GetContextRef() { + return m_context; + } + +protected: + /// A full or partial compiler context array where the parent declaration + /// contexts appear at the top of the array starting at index zero and the + /// last entry contains the type and name of the type we are looking for. + std::vector m_context; + /// An options bitmask that contains enabled options for the type query. + /// \see TypeQueryOptions. + TypeQueryOptions m_options; + /// If this variable has a value, then the language family must match at least + /// one of the specified languages. If this variable has no value, then the + /// language of the type doesn't need to match any types that are searched. + std::optional m_languages; +}; + +/// This class tracks the state and results of a \ref TypeQuery. +/// +/// Any mutable state required for type lookups and the results are tracked in +/// this object. +class TypeResults { +public: + /// Construct a type results object + TypeResults() = default; + + /// When types that match a TypeQuery are found, this API is used to insert + /// the matching types. + /// + /// \return + /// True if the type was added, false if the \a type_sp was already in the + /// results. + bool InsertUnique(const lldb::TypeSP &type_sp); + + /// Check if the type matching has found all of the matches that it needs. + bool Done(const TypeQuery &query) const; + + /// Check if a SymbolFile object has already been searched by this type match + /// object. + /// + /// This function will add \a sym_file to the set of SymbolFile objects if it + /// isn't already in the set and return \a false. Returns true if \a sym_file + /// was already in the set and doesn't need to be searched. + /// + /// Any clients that search for types should first check that the symbol file + /// has not already been searched. If this function returns true, the type + /// search function should early return to avoid duplicating type searchihng + /// efforts. + /// + /// \param[in] sym_file + /// A SymbolFile pointer that will be used to track which symbol files have + /// already been searched. + /// + /// \returns + /// True if the symbol file has been search already, false otherwise. + bool AlreadySearched(lldb_private::SymbolFile *sym_file); + + /// Access the set of searched symbol files. + llvm::DenseSet &GetSearchedSymbolFiles() { + return m_searched_symbol_files; + } + + lldb::TypeSP GetFirstType() const { return m_type_map.FirstType(); } + TypeMap &GetTypeMap() { return m_type_map; } + const TypeMap &GetTypeMap() const { return m_type_map; } + +private: + /// Matching types get added to this map as type search continues. + TypeMap m_type_map; + /// This set is used to track and make sure we only perform lookups in a + /// symbol file one time. + llvm::DenseSet m_searched_symbol_files; +}; + class SymbolFileType : public std::enable_shared_from_this, public UserID { public: diff --git a/lldb/include/lldb/Symbol/TypeMap.h b/lldb/include/lldb/Symbol/TypeMap.h index c200ccb9844f..433711875e55 100644 --- a/lldb/include/lldb/Symbol/TypeMap.h +++ b/lldb/include/lldb/Symbol/TypeMap.h @@ -27,7 +27,7 @@ public: void Clear(); void Dump(Stream *s, bool show_context, - lldb::DescriptionLevel level = lldb::eDescriptionLevelFull); + lldb::DescriptionLevel level = lldb::eDescriptionLevelFull) const; TypeMap FindTypes(ConstString name); @@ -41,10 +41,12 @@ public: lldb::TypeSP GetTypeAtIndex(uint32_t idx); + lldb::TypeSP FirstType() const; + typedef std::multimap collection; typedef AdaptedIterable TypeIterable; - TypeIterable Types() { return TypeIterable(m_types); } + TypeIterable Types() const { return TypeIterable(m_types); } void ForEach( std::function const &callback) const; diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h index cd5004a3f34d..63829131556e 100644 --- a/lldb/include/lldb/Symbol/TypeSystem.h +++ b/lldb/include/lldb/Symbol/TypeSystem.h @@ -26,6 +26,7 @@ #include "lldb/Expression/Expression.h" #include "lldb/Symbol/CompilerDecl.h" #include "lldb/Symbol/CompilerDeclContext.h" +#include "lldb/Symbol/Type.h" #include "lldb/lldb-private.h" class PDBASTParser; @@ -43,23 +44,6 @@ namespace npdb { class PdbAstBuilder; } // namespace npdb -/// A SmallBitVector that represents a set of source languages (\p -/// lldb::LanguageType). Each lldb::LanguageType is represented by -/// the bit with the position of its enumerator. The largest -/// LanguageType is < 64, so this is space-efficient and on 64-bit -/// architectures a LanguageSet can be completely stack-allocated. -struct LanguageSet { - llvm::SmallBitVector bitvector; - LanguageSet(); - - /// If the set contains a single language only, return it. - std::optional GetSingularLanguage(); - void Insert(lldb::LanguageType language); - bool Empty() const; - size_t Size() const; - bool operator[](unsigned i) const; -}; - /// Interface for representing a type system. /// /// Implemented by language plugins to define the type system for a given @@ -122,6 +106,9 @@ public: virtual CompilerType DeclGetFunctionArgumentType(void *opaque_decl, size_t arg_idx); + virtual std::vector + DeclGetCompilerContext(void *opaque_decl); + virtual CompilerType GetTypeForDecl(void *opaque_decl) = 0; // CompilerDeclContext functions @@ -146,6 +133,9 @@ public: virtual CompilerDeclContext GetCompilerDeclContextForType(const CompilerType &type); + virtual std::vector + DeclContextGetCompilerContext(void *opaque_decl_ctx); + // Tests #ifndef NDEBUG /// Verify the integrity of the type to catch CompilerTypes that mix diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h index 068fce4976ed..4e0c62fa26ca 100644 --- a/lldb/include/lldb/lldb-forward.h +++ b/lldb/include/lldb/lldb-forward.h @@ -258,9 +258,11 @@ class TypeImpl; class TypeList; class TypeListImpl; class TypeMap; +class TypeQuery; class TypeMemberFunctionImpl; class TypeMemberImpl; class TypeNameSpecifierImpl; +class TypeResults; class TypeSummaryImpl; class TypeSummaryOptions; class TypeSystem; diff --git a/lldb/include/lldb/lldb-private-enumerations.h b/lldb/include/lldb/lldb-private-enumerations.h index 7f98220f9f16..5f1597200a83 100644 --- a/lldb/include/lldb/lldb-private-enumerations.h +++ b/lldb/include/lldb/lldb-private-enumerations.h @@ -198,12 +198,15 @@ enum class CompilerContextKind : uint16_t { Variable = 1 << 7, Enum = 1 << 8, Typedef = 1 << 9, + Builtin = 1 << 10, Any = 1 << 15, /// Match 0..n nested modules. AnyModule = Any | Module, /// Match any type. - AnyType = Any | Class | Struct | Union | Enum | Typedef + AnyType = Any | Class | Struct | Union | Enum | Typedef | Builtin, + /// Math any declaration context. + AnyDeclContext = Any | Namespace | Class | Struct | Union | Enum | Function }; // Enumerations that can be used to specify the kind of metric we're looking at diff --git a/lldb/source/API/SBModule.cpp b/lldb/source/API/SBModule.cpp index b865502228e0..262e26c6bf43 100644 --- a/lldb/source/API/SBModule.cpp +++ b/lldb/source/API/SBModule.cpp @@ -437,26 +437,25 @@ lldb::SBType SBModule::FindFirstType(const char *name_cstr) { LLDB_INSTRUMENT_VA(this, name_cstr); ModuleSP module_sp(GetSP()); - if (!name_cstr || !module_sp) - return {}; - SymbolContext sc; - const bool exact_match = false; - ConstString name(name_cstr); + if (name_cstr && module_sp) { + ConstString name(name_cstr); + TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_find_one); + TypeResults results; + module_sp->FindTypes(query, results); + TypeSP type_sp = results.GetFirstType(); + if (type_sp) + return SBType(type_sp); - SBType sb_type = SBType(module_sp->FindFirstType(sc, name, exact_match)); - - if (sb_type.IsValid()) - return sb_type; + auto type_system_or_err = + module_sp->GetTypeSystemForLanguage(eLanguageTypeC); + if (auto err = type_system_or_err.takeError()) { + llvm::consumeError(std::move(err)); + return {}; + } - auto type_system_or_err = module_sp->GetTypeSystemForLanguage(eLanguageTypeC); - if (auto err = type_system_or_err.takeError()) { - llvm::consumeError(std::move(err)); - return {}; + if (auto ts = *type_system_or_err) + return SBType(ts->GetBuiltinTypeByName(name)); } - - if (auto ts = *type_system_or_err) - return SBType(ts->GetBuiltinTypeByName(name)); - return {}; } @@ -471,7 +470,7 @@ lldb::SBType SBModule::GetBasicType(lldb::BasicType type) { llvm::consumeError(std::move(err)); } else { if (auto ts = *type_system_or_err) - return SBType(ts->GetBasicTypeFromAST(type)); + return SBType(ts->GetBasicTypeFromAST(type)); } } return SBType(); @@ -485,13 +484,11 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) { ModuleSP module_sp(GetSP()); if (type && module_sp) { TypeList type_list; - const bool exact_match = false; - ConstString name(type); - llvm::DenseSet searched_symbol_files; - module_sp->FindTypes(name, exact_match, UINT32_MAX, searched_symbol_files, - type_list); - - if (type_list.Empty()) { + TypeQuery query(type); + TypeResults results; + module_sp->FindTypes(query, results); + if (results.GetTypeMap().Empty()) { + ConstString name(type); auto type_system_or_err = module_sp->GetTypeSystemForLanguage(eLanguageTypeC); if (auto err = type_system_or_err.takeError()) { @@ -502,11 +499,8 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) { retval.Append(SBType(compiler_type)); } } else { - for (size_t idx = 0; idx < type_list.GetSize(); idx++) { - TypeSP type_sp(type_list.GetTypeAtIndex(idx)); - if (type_sp) - retval.Append(SBType(type_sp)); - } + for (const TypeSP &type_sp : results.GetTypeMap().Types()) + retval.Append(SBType(type_sp)); } } return retval; diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp index 2d029554492a..8e616afbcb4e 100644 --- a/lldb/source/API/SBTarget.cpp +++ b/lldb/source/API/SBTarget.cpp @@ -1804,21 +1804,13 @@ lldb::SBType SBTarget::FindFirstType(const char *typename_cstr) { TargetSP target_sp(GetSP()); if (typename_cstr && typename_cstr[0] && target_sp) { ConstString const_typename(typename_cstr); - SymbolContext sc; - const bool exact_match = false; - - const ModuleList &module_list = target_sp->GetImages(); - size_t count = module_list.GetSize(); - for (size_t idx = 0; idx < count; idx++) { - ModuleSP module_sp(module_list.GetModuleAtIndex(idx)); - if (module_sp) { - TypeSP type_sp( - module_sp->FindFirstType(sc, const_typename, exact_match)); - if (type_sp) - return SBType(type_sp); - } - } - + TypeQuery query(const_typename.GetStringRef(), + TypeQueryOptions::e_find_one); + TypeResults results; + target_sp->GetImages().FindTypes(/*search_first=*/nullptr, query, results); + TypeSP type_sp = results.GetFirstType(); + if (type_sp) + return SBType(type_sp); // Didn't find the type in the symbols; Try the loaded language runtimes. if (auto process_sp = target_sp->GetProcessSP()) { for (auto *runtime : process_sp->GetLanguageRuntimes()) { @@ -1859,17 +1851,11 @@ lldb::SBTypeList SBTarget::FindTypes(const char *typename_cstr) { if (typename_cstr && typename_cstr[0] && target_sp) { ModuleList &images = target_sp->GetImages(); ConstString const_typename(typename_cstr); - bool exact_match = false; - TypeList type_list; - llvm::DenseSet searched_symbol_files; - images.FindTypes(nullptr, const_typename, exact_match, UINT32_MAX, - searched_symbol_files, type_list); - - for (size_t idx = 0; idx < type_list.GetSize(); idx++) { - TypeSP type_sp(type_list.GetTypeAtIndex(idx)); - if (type_sp) - sb_type_list.Append(SBType(type_sp)); - } + TypeQuery query(typename_cstr); + TypeResults results; + images.FindTypes(nullptr, query, results); + for (const TypeSP &type_sp : results.GetTypeMap().Types()) + sb_type_list.Append(SBType(type_sp)); // Try the loaded language runtimes if (auto process_sp = target_sp->GetProcessSP()) { diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp index b02b7dee5619..4ecac732d0dc 100644 --- a/lldb/source/Commands/CommandObjectMemory.cpp +++ b/lldb/source/Commands/CommandObjectMemory.cpp @@ -372,8 +372,6 @@ protected: if (view_as_type_cstr && view_as_type_cstr[0]) { // We are viewing memory as a type - const bool exact_match = false; - TypeList type_list; uint32_t reference_count = 0; uint32_t pointer_count = 0; size_t idx; @@ -452,18 +450,18 @@ protected: } } - llvm::DenseSet searched_symbol_files; ConstString lookup_type_name(type_str.c_str()); StackFrame *frame = m_exe_ctx.GetFramePtr(); ModuleSP search_first; - if (frame) { + if (frame) search_first = frame->GetSymbolContext(eSymbolContextModule).module_sp; - } - target->GetImages().FindTypes(search_first.get(), lookup_type_name, - exact_match, 1, searched_symbol_files, - type_list); + TypeQuery query(lookup_type_name.GetStringRef(), + TypeQueryOptions::e_find_one); + TypeResults results; + target->GetImages().FindTypes(search_first.get(), query, results); + TypeSP type_sp = results.GetFirstType(); - if (type_list.GetSize() == 0 && lookup_type_name.GetCString()) { + if (!type_sp && lookup_type_name.GetCString()) { LanguageType language_for_type = m_memory_options.m_language_for_type.GetCurrentValue(); std::set languages_to_check; @@ -499,15 +497,14 @@ protected: } if (!compiler_type.IsValid()) { - if (type_list.GetSize() == 0) { + if (type_sp) { + compiler_type = type_sp->GetFullCompilerType(); + } else { result.AppendErrorWithFormat("unable to find any types that match " "the raw type '%s' for full type '%s'\n", lookup_type_name.GetCString(), view_as_type_cstr); return; - } else { - TypeSP type_sp(type_list.GetTypeAtIndex(0)); - compiler_type = type_sp->GetFullCompilerType(); } } diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp index 63232c221ad1..bc8bc51356c8 100644 --- a/lldb/source/Commands/CommandObjectTarget.cpp +++ b/lldb/source/Commands/CommandObjectTarget.cpp @@ -1706,16 +1706,18 @@ static size_t LookupTypeInModule(Target *target, CommandInterpreter &interpreter, Stream &strm, Module *module, const char *name_cstr, bool name_is_regex) { - TypeList type_list; if (module && name_cstr && name_cstr[0]) { - const uint32_t max_num_matches = UINT32_MAX; - bool name_is_fully_qualified = false; - - ConstString name(name_cstr); - llvm::DenseSet searched_symbol_files; - module->FindTypes(name, name_is_fully_qualified, max_num_matches, - searched_symbol_files, type_list); + TypeQuery query(name_cstr); + TypeResults results; + module->FindTypes(query, results); + TypeList type_list; + SymbolContext sc; + if (module) + sc.module_sp = module->shared_from_this(); + // Sort the type results and put the results that matched in \a module + // first if \a module was specified. + sc.SortTypeList(results.GetTypeMap(), type_list); if (type_list.Empty()) return 0; @@ -1748,22 +1750,21 @@ static size_t LookupTypeInModule(Target *target, } strm.EOL(); } + return type_list.GetSize(); } - return type_list.GetSize(); + return 0; } static size_t LookupTypeHere(Target *target, CommandInterpreter &interpreter, Stream &strm, Module &module, const char *name_cstr, bool name_is_regex) { + TypeQuery query(name_cstr); + TypeResults results; + module.FindTypes(query, results); TypeList type_list; - const uint32_t max_num_matches = UINT32_MAX; - bool name_is_fully_qualified = false; - - ConstString name(name_cstr); - llvm::DenseSet searched_symbol_files; - module.FindTypes(name, name_is_fully_qualified, max_num_matches, - searched_symbol_files, type_list); - + SymbolContext sc; + sc.module_sp = module.shared_from_this(); + sc.SortTypeList(results.GetTypeMap(), type_list); if (type_list.Empty()) return 0; @@ -2082,7 +2083,7 @@ protected: result.GetOutputStream().EOL(); result.GetOutputStream().EOL(); } - if (INTERRUPT_REQUESTED(GetDebugger(), + if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted in dump all symtabs with {0} " "of {1} dumped.", num_dumped, num_modules)) break; @@ -2112,8 +2113,8 @@ protected: result.GetOutputStream().EOL(); result.GetOutputStream().EOL(); } - if (INTERRUPT_REQUESTED(GetDebugger(), - "Interrupted in dump symtab list with {0} of {1} dumped.", + if (INTERRUPT_REQUESTED(GetDebugger(), + "Interrupted in dump symtab list with {0} of {1} dumped.", num_dumped, num_matches)) break; @@ -2175,7 +2176,7 @@ protected: result.GetOutputStream().Format("Dumping sections for {0} modules.\n", num_modules); for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) { - if (INTERRUPT_REQUESTED(GetDebugger(), + if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted in dump all sections with {0} of {1} dumped", image_idx, num_modules)) break; @@ -2196,7 +2197,7 @@ protected: FindModulesByName(target, arg_cstr, module_list, true); if (num_matches > 0) { for (size_t i = 0; i < num_matches; ++i) { - if (INTERRUPT_REQUESTED(GetDebugger(), + if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted in dump section list with {0} of {1} dumped.", i, num_matches)) break; @@ -2338,7 +2339,7 @@ protected: } for (size_t i = 0; i < num_matches; ++i) { - if (INTERRUPT_REQUESTED(GetDebugger(), + if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted in dump clang ast list with {0} of {1} dumped.", i, num_matches)) break; @@ -2477,9 +2478,9 @@ protected: if (num_modules > 0) { uint32_t num_dumped = 0; for (ModuleSP module_sp : target_modules.ModulesNoLocking()) { - if (INTERRUPT_REQUESTED(GetDebugger(), + if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted in dump all line tables with " - "{0} of {1} dumped", num_dumped, + "{0} of {1} dumped", num_dumped, num_modules)) break; diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp index e6279a0feda8..65a65c455efa 100644 --- a/lldb/source/Core/Module.cpp +++ b/lldb/source/Core/Module.cpp @@ -949,99 +949,9 @@ void Module::FindAddressesForLine(const lldb::TargetSP target_sp, } } -void Module::FindTypes_Impl( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { +void Module::FindTypes(const TypeQuery &query, TypeResults &results) { if (SymbolFile *symbols = GetSymbolFile()) - symbols->FindTypes(name, parent_decl_ctx, max_matches, - searched_symbol_files, types); -} - -void Module::FindTypesInNamespace(ConstString type_name, - const CompilerDeclContext &parent_decl_ctx, - size_t max_matches, TypeList &type_list) { - TypeMap types_map; - llvm::DenseSet searched_symbol_files; - FindTypes_Impl(type_name, parent_decl_ctx, max_matches, searched_symbol_files, - types_map); - if (types_map.GetSize()) { - SymbolContext sc; - sc.module_sp = shared_from_this(); - sc.SortTypeList(types_map, type_list); - } -} - -lldb::TypeSP Module::FindFirstType(const SymbolContext &sc, ConstString name, - bool exact_match) { - TypeList type_list; - llvm::DenseSet searched_symbol_files; - FindTypes(name, exact_match, 1, searched_symbol_files, type_list); - if (type_list.GetSize()) - return type_list.GetTypeAtIndex(0); - return TypeSP(); -} - -void Module::FindTypes( - ConstString name, bool exact_match, size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeList &types) { - const char *type_name_cstr = name.GetCString(); - llvm::StringRef type_scope; - llvm::StringRef type_basename; - TypeClass type_class = eTypeClassAny; - TypeMap typesmap; - - if (Type::GetTypeScopeAndBasename(type_name_cstr, type_scope, type_basename, - type_class)) { - // Check if "name" starts with "::" which means the qualified type starts - // from the root namespace and implies and exact match. The typenames we - // get back from clang do not start with "::" so we need to strip this off - // in order to get the qualified names to match - exact_match = type_scope.consume_front("::"); - - ConstString type_basename_const_str(type_basename); - FindTypes_Impl(type_basename_const_str, CompilerDeclContext(), max_matches, - searched_symbol_files, typesmap); - if (typesmap.GetSize()) - typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class, - exact_match); - } else { - // The type is not in a namespace/class scope, just search for it by - // basename - if (type_class != eTypeClassAny && !type_basename.empty()) { - // The "type_name_cstr" will have been modified if we have a valid type - // class prefix (like "struct", "class", "union", "typedef" etc). - FindTypes_Impl(ConstString(type_basename), CompilerDeclContext(), - UINT_MAX, searched_symbol_files, typesmap); - typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class, - exact_match); - } else { - FindTypes_Impl(name, CompilerDeclContext(), UINT_MAX, - searched_symbol_files, typesmap); - if (exact_match) { - typesmap.RemoveMismatchedTypes(type_scope, name, type_class, - exact_match); - } - } - } - if (typesmap.GetSize()) { - SymbolContext sc; - sc.module_sp = shared_from_this(); - sc.SortTypeList(typesmap, types); - } -} - -void Module::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { - // If a scoped timer is needed, place it in a SymbolFile::FindTypes override. - // A timer here is too high volume for some cases, for example when calling - // FindTypes on each object file. - if (SymbolFile *symbols = GetSymbolFile()) - symbols->FindTypes(pattern, languages, searched_symbol_files, types); + symbols->FindTypes(query, results); } static Debugger::DebuggerList diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index 04a9df7dd63b..aa89c93c8d05 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -557,36 +557,21 @@ ModuleSP ModuleList::FindModule(const UUID &uuid) const { return module_sp; } -void ModuleList::FindTypes(Module *search_first, ConstString name, - bool name_is_fully_qualified, size_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeList &types) const { +void ModuleList::FindTypes(Module *search_first, const TypeQuery &query, + TypeResults &results) const { std::lock_guard guard(m_modules_mutex); - - collection::const_iterator pos, end = m_modules.end(); if (search_first) { - for (pos = m_modules.begin(); pos != end; ++pos) { - if (search_first == pos->get()) { - search_first->FindTypes(name, name_is_fully_qualified, max_matches, - searched_symbol_files, types); - - if (types.GetSize() >= max_matches) - return; - } - } - } - - for (pos = m_modules.begin(); pos != end; ++pos) { - // Search the module if the module is not equal to the one in the symbol - // context "sc". If "sc" contains a empty module shared pointer, then the - // comparison will always be true (valid_module_ptr != nullptr). - if (search_first != pos->get()) - (*pos)->FindTypes(name, name_is_fully_qualified, max_matches, - searched_symbol_files, types); - - if (types.GetSize() >= max_matches) + search_first->FindTypes(query, results); + if (results.Done(query)) return; } + for (const auto &module_sp : m_modules) { + if (search_first != module_sp.get()) { + module_sp->FindTypes(query, results); + if (results.Done(query)) + return; + } + } } bool ModuleList::FindSourceFile(const FileSpec &orig_spec, diff --git a/lldb/source/DataFormatters/TypeFormat.cpp b/lldb/source/DataFormatters/TypeFormat.cpp index 126240aeca65..409c452110bd 100644 --- a/lldb/source/DataFormatters/TypeFormat.cpp +++ b/lldb/source/DataFormatters/TypeFormat.cpp @@ -161,13 +161,12 @@ bool TypeFormatImpl_EnumType::FormatObject(ValueObject *valobj, if (!target_sp) return false; const ModuleList &images(target_sp->GetImages()); - TypeList types; - llvm::DenseSet searched_symbol_files; - images.FindTypes(nullptr, m_enum_type, false, UINT32_MAX, - searched_symbol_files, types); - if (types.Empty()) + TypeQuery query(m_enum_type.GetStringRef()); + TypeResults results; + images.FindTypes(nullptr, query, results); + if (results.GetTypeMap().Empty()) return false; - for (lldb::TypeSP type_sp : types.Types()) { + for (lldb::TypeSP type_sp : results.GetTypeMap().Types()) { if (!type_sp) continue; if ((type_sp->GetForwardCompilerType().GetTypeInfo() & diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp index 5d7e1252038d..00ab6a04bd32 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp @@ -201,19 +201,17 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) { LLDB_LOG(log, " CTD Searching namespace {0} in module {1}", item.second.GetName(), item.first->GetFileSpec().GetFilename()); - TypeList types; - ConstString name(decl->getName()); - item.first->FindTypesInNamespace(name, item.second, UINT32_MAX, types); - - for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) { - lldb::TypeSP type = types.GetTypeAtIndex(ti); - - if (!type) - continue; + // Create a type matcher using the CompilerDeclContext for the namespace + // as the context (item.second) and search for the name inside of this + // context. + TypeQuery query(item.second, name); + TypeResults results; + item.first->FindTypes(query, results); - CompilerType clang_type(type->GetFullCompilerType()); + for (const lldb::TypeSP &type_sp : results.GetTypeMap().Types()) { + CompilerType clang_type(type_sp->GetFullCompilerType()); if (!ClangUtil::IsClangType(clang_type)) continue; @@ -233,24 +231,15 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) { } } } else { - TypeList types; - - ConstString name(decl->getName()); - const ModuleList &module_list = m_target->GetImages(); + // Create a type matcher using a CompilerDecl. Each TypeSystem class knows + // how to fill out a CompilerContext array using a CompilerDecl. + TypeQuery query(CompilerDecl(m_clang_ast_context, (void *)decl)); + TypeResults results; + module_list.FindTypes(nullptr, query, results); + for (const lldb::TypeSP &type_sp : results.GetTypeMap().Types()) { - bool exact_match = false; - llvm::DenseSet searched_symbol_files; - module_list.FindTypes(nullptr, name, exact_match, UINT32_MAX, - searched_symbol_files, types); - - for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) { - lldb::TypeSP type = types.GetTypeAtIndex(ti); - - if (!type) - continue; - - CompilerType clang_type(type->GetFullCompilerType()); + CompilerType clang_type(type_sp->GetFullCompilerType()); if (!ClangUtil::IsClangType(clang_type)) continue; @@ -263,13 +252,6 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) { TagDecl *candidate_tag_decl = const_cast(tag_type->getDecl()); - // We have found a type by basename and we need to make sure the decl - // contexts are the same before we can try to complete this type with - // another - if (!TypeSystemClang::DeclsAreEquivalent(const_cast(decl), - candidate_tag_decl)) - continue; - if (TypeSystemClang::GetCompleteDecl(&candidate_tag_decl->getASTContext(), candidate_tag_decl)) return candidate_tag_decl; @@ -614,41 +596,40 @@ void ClangASTSource::FindExternalVisibleDecls( if (context.m_found_type) return; - TypeList types; - const bool exact_match = true; - llvm::DenseSet searched_symbol_files; - if (module_sp && namespace_decl) - module_sp->FindTypesInNamespace(name, namespace_decl, 1, types); - else { - m_target->GetImages().FindTypes(module_sp.get(), name, exact_match, 1, - searched_symbol_files, types); + lldb::TypeSP type_sp; + TypeResults results; + if (module_sp && namespace_decl) { + // Match the name in the specified decl context. + TypeQuery query(namespace_decl, name, TypeQueryOptions::e_find_one); + module_sp->FindTypes(query, results); + type_sp = results.GetFirstType(); + } else { + // Match the exact name of the type at the root level. + TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_exact_match | + TypeQueryOptions::e_find_one); + m_target->GetImages().FindTypes(nullptr, query, results); + type_sp = results.GetFirstType(); } - if (size_t num_types = types.GetSize()) { - for (size_t ti = 0; ti < num_types; ++ti) { - lldb::TypeSP type_sp = types.GetTypeAtIndex(ti); - - if (log) { - const char *name_string = type_sp->GetName().GetCString(); - - LLDB_LOG(log, " CAS::FEVD Matching type found for \"{0}\": {1}", name, - (name_string ? name_string : "")); - } + if (type_sp) { + if (log) { + const char *name_string = type_sp->GetName().GetCString(); - CompilerType full_type = type_sp->GetFullCompilerType(); + LLDB_LOG(log, " CAS::FEVD Matching type found for \"{0}\": {1}", name, + (name_string ? name_string : "")); + } - CompilerType copied_clang_type(GuardedCopyType(full_type)); + CompilerType full_type = type_sp->GetFullCompilerType(); - if (!copied_clang_type) { - LLDB_LOG(log, " CAS::FEVD - Couldn't export a type"); + CompilerType copied_clang_type(GuardedCopyType(full_type)); - continue; - } + if (!copied_clang_type) { + LLDB_LOG(log, " CAS::FEVD - Couldn't export a type"); + } else { context.AddTypeDecl(copied_clang_type); context.m_found_type = true; - break; } } diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp index a5c9ead55f4c..0ea9201901ab 100644 --- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp @@ -82,24 +82,30 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo( lookup_name.append(class_name.data(), class_name.size()); type_info.SetName(class_name); - const bool exact_match = true; + ConstString const_lookup_name(lookup_name); TypeList class_types; - + ModuleSP module_sp = vtable_info.symbol->CalculateSymbolContextModule(); // First look in the module that the vtable symbol came from and // look for a single exact match. - llvm::DenseSet searched_symbol_files; - ModuleSP module_sp = vtable_info.symbol->CalculateSymbolContextModule(); - if (module_sp) - module_sp->FindTypes(ConstString(lookup_name), exact_match, 1, - searched_symbol_files, class_types); + TypeResults results; + TypeQuery query(const_lookup_name.GetStringRef(), + TypeQueryOptions::e_exact_match | + TypeQueryOptions::e_find_one); + if (module_sp) { + module_sp->FindTypes(query, results); + TypeSP type_sp = results.GetFirstType(); + if (type_sp) + class_types.Insert(type_sp); + } // If we didn't find a symbol, then move on to the entire module // list in the target and get as many unique matches as possible - Target &target = m_process->GetTarget(); - if (class_types.Empty()) - target.GetImages().FindTypes(nullptr, ConstString(lookup_name), - exact_match, UINT32_MAX, - searched_symbol_files, class_types); + if (class_types.Empty()) { + query.SetFindOne(false); + m_process->GetTarget().GetImages().FindTypes(nullptr, query, results); + for (const auto &type_sp : results.GetTypeMap().Types()) + class_types.Insert(type_sp); + } lldb::TypeSP type_sp; if (class_types.Empty()) { diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp index 289288a86245..ba52444f0c2f 100644 --- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp @@ -139,17 +139,10 @@ ObjCLanguageRuntime::LookupInCompleteClassCache(ConstString &name) { if (!module_sp) return TypeSP(); - const bool exact_match = true; - const uint32_t max_matches = UINT32_MAX; - TypeList types; - - llvm::DenseSet searched_symbol_files; - module_sp->FindTypes(name, exact_match, max_matches, searched_symbol_files, - types); - - for (uint32_t i = 0; i < types.GetSize(); ++i) { - TypeSP type_sp(types.GetTypeAtIndex(i)); - + TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_exact_match); + TypeResults results; + module_sp->FindTypes(query, results); + for (const TypeSP &type_sp : results.GetTypeMap().Types()) { if (TypeSystemClang::IsObjCObjectOrInterfaceType( type_sp->GetForwardCompilerType())) { if (TypePayloadClang(type_sp->GetPayload()).IsCompleteObjCClass()) { diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp index cd52233cc8cc..729d6af02402 100644 --- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp +++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp @@ -448,15 +448,6 @@ void SymbolFileBreakpad::FindFunctions(const RegularExpression ®ex, // TODO } -void SymbolFileBreakpad::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, llvm::DenseSet &searched_symbol_files, - TypeMap &types) {} - -void SymbolFileBreakpad::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, TypeMap &types) {} - void SymbolFileBreakpad::AddSymbols(Symtab &symtab) { Log *log = GetLog(LLDBLog::Symbols); Module &module = *m_objfile_sp->GetModule(); diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h index 4a01a64202ee..214fbdd3ff3a 100644 --- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h +++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h @@ -118,15 +118,6 @@ public: void FindFunctions(const RegularExpression ®ex, bool include_inlines, SymbolContextList &sc_list) override; - void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; - - void FindTypes(llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; - llvm::Expected GetTypeSystemForLanguage(lldb::LanguageType language) override { return llvm::make_error( diff --git a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp index 7a2b4c00eedf..d192944bb9d0 100644 --- a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp +++ b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp @@ -1020,23 +1020,18 @@ lldb_private::Type *SymbolFileCTF::ResolveTypeUID(lldb::user_id_t type_uid) { return type_sp.get(); } -void SymbolFileCTF::FindTypes( - lldb_private::ConstString name, - const lldb_private::CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) { - - searched_symbol_files.clear(); - searched_symbol_files.insert(this); +void SymbolFileCTF::FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) { + // Make sure we haven't already searched this SymbolFile before. + if (results.AlreadySearched(this)) + return; - size_t matches = 0; + ConstString name = match.GetTypeBasename(); for (TypeSP type_sp : GetTypeList().Types()) { - if (matches == max_matches) - break; if (type_sp && type_sp->GetName() == name) { - types.Insert(type_sp); - matches++; + results.InsertUnique(type_sp); + if (results.Done(match)) + return; } } } diff --git a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h index 787dc1892bb3..f111937dbd6e 100644 --- a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h +++ b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h @@ -105,12 +105,8 @@ public: lldb::TypeClass type_mask, lldb_private::TypeList &type_list) override {} - void - FindTypes(lldb_private::ConstString name, - const lldb_private::CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) override; + void FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) override; void FindTypesByRegex(const lldb_private::RegularExpression ®ex, uint32_t max_matches, lldb_private::TypeMap &types); diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp index e3c64640c791..334876620249 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp @@ -152,16 +152,16 @@ TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc, // If this type comes from a Clang module, recursively look in the // DWARF section of the .pcm file in the module cache. Clang // generates DWO skeleton units as breadcrumbs to find them. - std::vector decl_context = die.GetDeclContext(); - TypeMap pcm_types; + std::vector die_context = die.GetDeclContext(); + TypeQuery query(die_context, TypeQueryOptions::e_module_search | + TypeQueryOptions::e_find_one); + TypeResults results; // The type in the Clang module must have the same language as the current CU. - LanguageSet languages; - languages.Insert(SymbolFileDWARF::GetLanguageFamily(*die.GetCU())); - llvm::DenseSet searched_symbol_files; - clang_module_sp->GetSymbolFile()->FindTypes(decl_context, languages, - searched_symbol_files, pcm_types); - if (pcm_types.Empty()) { + query.AddLanguage(SymbolFileDWARF::GetLanguageFamily(*die.GetCU())); + clang_module_sp->FindTypes(query, results); + TypeSP pcm_type_sp = results.GetTypeMap().FirstType(); + if (!pcm_type_sp) { // Since this type is defined in one of the Clang modules imported // by this symbol file, search all of them. Instead of calling // sym_file->FindTypes(), which would return this again, go straight @@ -170,24 +170,20 @@ TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc, // Well-formed clang modules never form cycles; guard against corrupted // ones by inserting the current file. - searched_symbol_files.insert(&sym_file); + results.AlreadySearched(&sym_file); sym_file.ForEachExternalModule( - *sc.comp_unit, searched_symbol_files, [&](Module &module) { - module.GetSymbolFile()->FindTypes(decl_context, languages, - searched_symbol_files, pcm_types); - return pcm_types.GetSize(); + *sc.comp_unit, results.GetSearchedSymbolFiles(), [&](Module &module) { + module.FindTypes(query, results); + pcm_type_sp = results.GetTypeMap().FirstType(); + return !pcm_type_sp; }); } - if (!pcm_types.GetSize()) + if (!pcm_type_sp) return TypeSP(); // We found a real definition for this type in the Clang module, so lets use // it and cache the fact that we found a complete type for this die. - TypeSP pcm_type_sp = pcm_types.GetTypeAtIndex(0); - if (!pcm_type_sp) - return TypeSP(); - lldb_private::CompilerType pcm_type = pcm_type_sp->GetForwardCompilerType(); lldb_private::CompilerType type = GetClangASTImporter().CopyType(m_ast, pcm_type); diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp index 1f9524f8add9..bed68f45426f 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp @@ -418,6 +418,54 @@ std::vector DWARFDIE::GetDeclContext() const { return context; } +std::vector +DWARFDIE::GetTypeLookupContext() const { + std::vector context; + // If there is no name, then there is no need to look anything up for this + // DIE. + const char *name = GetName(); + if (!name || !name[0]) + return context; + const dw_tag_t tag = Tag(); + if (tag == DW_TAG_compile_unit || tag == DW_TAG_partial_unit) + return context; + DWARFDIE parent = GetParent(); + if (parent) + context = parent.GetTypeLookupContext(); + auto push_ctx = [&](CompilerContextKind kind, llvm::StringRef name) { + context.push_back({kind, ConstString(name)}); + }; + switch (tag) { + case DW_TAG_namespace: + push_ctx(CompilerContextKind::Namespace, name); + break; + case DW_TAG_structure_type: + push_ctx(CompilerContextKind::Struct, name); + break; + case DW_TAG_union_type: + push_ctx(CompilerContextKind::Union, name); + break; + case DW_TAG_class_type: + push_ctx(CompilerContextKind::Class, name); + break; + case DW_TAG_enumeration_type: + push_ctx(CompilerContextKind::Enum, name); + break; + case DW_TAG_variable: + push_ctx(CompilerContextKind::Variable, GetPubname()); + break; + case DW_TAG_typedef: + push_ctx(CompilerContextKind::Typedef, name); + break; + case DW_TAG_base_type: + push_ctx(CompilerContextKind::Builtin, name); + break; + default: + break; + } + return context; +} + DWARFDIE DWARFDIE::GetParentDeclContextDIE() const { if (IsValid()) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h index a68af62c8b3e..511ca62d0197 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h @@ -73,9 +73,22 @@ public: std::vector GetDeclContextDIEs() const; /// Return this DIE's decl context as it is needed to look up types - /// in Clang's -gmodules debug info format. + /// in Clang modules. This context will include any modules or functions that + /// the type is declared in so an exact module match can be efficiently made. std::vector GetDeclContext() const; + /// Get a context to a type so it can be looked up. + /// + /// This function uses the current DIE to fill in a CompilerContext array + /// that is suitable for type lookup for comparison to a TypeQuery's compiler + /// context (TypeQuery::GetContextRef()). If this DIE represents a named type, + /// it should fill out the compiler context with the type itself as the last + /// entry. The declaration context should be above the type and stop at an + /// appropriate time, like either the translation unit or at a function + /// context. This is designed to allow users to efficiently look for types + /// using a full or partial CompilerContext array. + std::vector GetTypeLookupContext() const; + // Getting attribute values from the DIE. // // GetAttributeValueAsXXX() functions should only be used if you are diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index d4c573ecd468..7eddc5074eff 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -2596,177 +2596,157 @@ void SymbolFileDWARF::GetMangledNamesForFunction( } } -void SymbolFileDWARF::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { - std::lock_guard guard(GetModuleMutex()); - // Make sure we haven't already searched this SymbolFile before. - if (!searched_symbol_files.insert(this).second) - return; - - Log *log = GetLog(DWARFLog::Lookups); +/// Split a name up into a basename and template parameters. +static bool SplitTemplateParams(llvm::StringRef fullname, + llvm::StringRef &basename, + llvm::StringRef &template_params) { + auto it = fullname.find('<'); + if (it == llvm::StringRef::npos) { + basename = fullname; + template_params = llvm::StringRef(); + return false; + } + basename = fullname.slice(0, it); + template_params = fullname.slice(it, fullname.size()); + return true; +} - if (log) { - if (parent_decl_ctx) - GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx = " - "{1:p} (\"{2}\"), max_matches={3}, type_list)", - name.GetCString(), static_cast(&parent_decl_ctx), - parent_decl_ctx.GetName().AsCString(""), max_matches); - else - GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx = " - "NULL, max_matches={1}, type_list)", - name.GetCString(), max_matches); +static bool UpdateCompilerContextForSimpleTemplateNames(TypeQuery &match) { + // We need to find any names in the context that have template parameters + // and strip them so the context can be matched when -gsimple-template-names + // is being used. Returns true if any of the context items were updated. + bool any_context_updated = false; + for (auto &context : match.GetContextRef()) { + llvm::StringRef basename, params; + if (SplitTemplateParams(context.name.GetStringRef(), basename, params)) { + context.name = ConstString(basename); + any_context_updated = true; + } } + return any_context_updated; +} +void SymbolFileDWARF::FindTypes(const TypeQuery &query, TypeResults &results) { - if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx)) + // Make sure we haven't already searched this SymbolFile before. + if (results.AlreadySearched(this)) return; - // Unlike FindFunctions(), FindTypes() following cannot produce false - // positives. - - const llvm::StringRef name_ref = name.GetStringRef(); - auto name_bracket_index = name_ref.find('<'); - m_index->GetTypes(name, [&](DWARFDIE die) { - if (!DIEInDeclContext(parent_decl_ctx, die)) - return true; // The containing decl contexts don't match + std::lock_guard guard(GetModuleMutex()); - Type *matching_type = ResolveType(die, true, true); - if (!matching_type) - return true; + bool have_index_match = false; + m_index->GetTypes(query.GetTypeBasename(), [&](DWARFDIE die) { + // Check the language, but only if we have a language filter. + if (query.HasLanguage()) { + if (!query.LanguageMatches(GetLanguageFamily(*die.GetCU()))) + return true; // Keep iterating over index types, language mismatch. + } - // With -gsimple-template-names, a templated type's DW_AT_name will not - // contain the template parameters. Make sure that if the original query - // didn't contain a '<', we filter out entries with template parameters. - if (name_bracket_index == llvm::StringRef::npos && - matching_type->IsTemplateType()) - return true; + // Check the context matches + std::vector die_context; + if (query.GetModuleSearch()) + die_context = die.GetDeclContext(); + else + die_context = die.GetTypeLookupContext(); + assert(!die_context.empty()); + if (!query.ContextMatches(die_context)) + return true; // Keep iterating over index types, context mismatch. - // We found a type pointer, now find the shared pointer form our type - // list - types.InsertUnique(matching_type->shared_from_this()); - return types.GetSize() < max_matches; + // Try to resolve the type. + if (Type *matching_type = ResolveType(die, true, true)) { + if (matching_type->IsTemplateType()) { + // We have to watch out for case where we lookup a type by basename and + // it matches a template with simple template names. Like looking up + // "Foo" and if we have simple template names then we will match + // "Foo" and "Foo" because all the DWARF has is "Foo" in + // the accelerator tables. The main case we see this in is when the + // expression parser is trying to parse "Foo" and it will first do + // a lookup on just "Foo". We verify the type basename matches before + // inserting the type in the results. + auto CompilerTypeBasename = + matching_type->GetForwardCompilerType().GetTypeName(true); + if (CompilerTypeBasename != query.GetTypeBasename()) + return true; // Keep iterating over index types, basename mismatch. + } + have_index_match = true; + results.InsertUnique(matching_type->shared_from_this()); + } + return !results.Done(query); // Keep iterating if we aren't done. }); + if (results.Done(query)) + return; + // With -gsimple-template-names, a templated type's DW_AT_name will not // contain the template parameters. Try again stripping '<' and anything // after, filtering out entries with template parameters that don't match. - if (types.GetSize() < max_matches) { - if (name_bracket_index != llvm::StringRef::npos) { - const llvm::StringRef name_no_template_params = - name_ref.slice(0, name_bracket_index); - const llvm::StringRef template_params = - name_ref.slice(name_bracket_index, name_ref.size()); - m_index->GetTypes(ConstString(name_no_template_params), [&](DWARFDIE die) { - if (!DIEInDeclContext(parent_decl_ctx, die)) - return true; // The containing decl contexts don't match - - const llvm::StringRef base_name = GetTypeForDIE(die)->GetBaseName().AsCString(); - auto it = base_name.find('<'); - // If the candidate qualified name doesn't have '<', it doesn't have - // template params to compare. - if (it == llvm::StringRef::npos) - return true; - - // Filter out non-matching instantiations by comparing template params. - const llvm::StringRef base_name_template_params = - base_name.slice(it, base_name.size()); - - if (template_params != base_name_template_params) - return true; - - Type *matching_type = ResolveType(die, true, true); - if (!matching_type) - return true; + if (!have_index_match) { + // Create a type matcher with a compiler context that is tuned for + // -gsimple-template-names. We will use this for the index lookup and the + // context matching, but will use the original "match" to insert matches + // into if things match. The "match_simple" has a compiler context with + // all template parameters removed to allow the names and context to match. + // The UpdateCompilerContextForSimpleTemplateNames(...) will return true if + // it trims any context items down by removing template parameter names. + TypeQuery query_simple(query); + if (UpdateCompilerContextForSimpleTemplateNames(query_simple)) { + + // Copy our match's context and update the basename we are looking for + // so we can use this only to compare the context correctly. + m_index->GetTypes(query_simple.GetTypeBasename(), [&](DWARFDIE die) { + // Check the language, but only if we have a language filter. + if (query.HasLanguage()) { + if (!query.LanguageMatches(GetLanguageFamily(*die.GetCU()))) + return true; // Keep iterating over index types, language mismatch. + } - // We found a type pointer, now find the shared pointer form our type - // list. - types.InsertUnique(matching_type->shared_from_this()); - return types.GetSize() < max_matches; + // Check the context matches + std::vector die_context; + if (query.GetModuleSearch()) + die_context = die.GetDeclContext(); + else + die_context = die.GetTypeLookupContext(); + assert(!die_context.empty()); + if (!query_simple.ContextMatches(die_context)) + return true; // Keep iterating over index types, context mismatch. + + // Try to resolve the type. + if (Type *matching_type = ResolveType(die, true, true)) { + ConstString name = matching_type->GetQualifiedName(); + // We have found a type that still might not match due to template + // parameters. If we create a new TypeQuery that uses the new type's + // fully qualified name, we can find out if this type matches at all + // context levels. We can't use just the "match_simple" context + // because all template parameters were stripped off. The fully + // qualified name of the type will have the template parameters and + // will allow us to make sure it matches correctly. + TypeQuery die_query(name.GetStringRef(), + TypeQueryOptions::e_exact_match); + if (!query.ContextMatches(die_query.GetContextRef())) + return true; // Keep iterating over index types, context mismatch. + + results.InsertUnique(matching_type->shared_from_this()); + } + return !results.Done(query); // Keep iterating if we aren't done. }); + if (results.Done(query)) + return; } } // Next search through the reachable Clang modules. This only applies for // DWARF objects compiled with -gmodules that haven't been processed by // dsymutil. - if (types.GetSize() < max_matches) { - UpdateExternalModuleListIfNeeded(); - - for (const auto &pair : m_external_type_modules) - if (ModuleSP external_module_sp = pair.second) - if (SymbolFile *sym_file = external_module_sp->GetSymbolFile()) - sym_file->FindTypes(name, parent_decl_ctx, max_matches, - searched_symbol_files, types); - } + UpdateExternalModuleListIfNeeded(); - if (log && types.GetSize()) { - if (parent_decl_ctx) { - GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx " - "= {1:p} (\"{2}\"), max_matches={3}, type_list) => {4}", - name.GetCString(), static_cast(&parent_decl_ctx), - parent_decl_ctx.GetName().AsCString(""), max_matches, - types.GetSize()); - } else { - GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx " - "= NULL, max_matches={1}, type_list) => {2}", - name.GetCString(), max_matches, types.GetSize()); + for (const auto &pair : m_external_type_modules) { + if (ModuleSP external_module_sp = pair.second) { + external_module_sp->FindTypes(query, results); + if (results.Done(query)) + return; } } } -void SymbolFileDWARF::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, TypeMap &types) { - // Make sure we haven't already searched this SymbolFile before. - if (!searched_symbol_files.insert(this).second) - return; - - std::lock_guard guard(GetModuleMutex()); - if (pattern.empty()) - return; - - ConstString name = pattern.back().name; - - if (!name) - return; - - m_index->GetTypes(name, [&](DWARFDIE die) { - if (!languages[GetLanguageFamily(*die.GetCU())]) - return true; - - std::vector die_context = die.GetDeclContext(); - if (!contextMatches(die_context, pattern)) - return true; - - if (Type *matching_type = ResolveType(die, true, true)) { - // We found a type pointer, now find the shared pointer form our type - // list. - types.InsertUnique(matching_type->shared_from_this()); - } - return true; - }); - - // Next search through the reachable Clang modules. This only applies for - // DWARF objects compiled with -gmodules that haven't been processed by - // dsymutil. - UpdateExternalModuleListIfNeeded(); - - for (const auto &pair : m_external_type_modules) - if (ModuleSP external_module_sp = pair.second) - external_module_sp->FindTypes(pattern, languages, searched_symbol_files, - types); -} - CompilerDeclContext SymbolFileDWARF::FindNamespace(ConstString name, const CompilerDeclContext &parent_decl_ctx, diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h index e6efbba7e249..78819edd0062 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h @@ -186,14 +186,8 @@ public: GetMangledNamesForFunction(const std::string &scope_qualified_name, std::vector &mangled_names) override; - void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; - - void FindTypes(llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; + void FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) override; void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask, TypeList &type_list) override; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp index 263ada9cbb87..e5b59460cb85 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp @@ -1227,27 +1227,12 @@ TypeSP SymbolFileDWARFDebugMap::FindCompleteObjCDefinitionTypeForDIE( return TypeSP(); } -void SymbolFileDWARFDebugMap::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { +void SymbolFileDWARFDebugMap::FindTypes(const TypeQuery &query, + TypeResults &results) { std::lock_guard guard(GetModuleMutex()); ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool { - oso_dwarf->FindTypes(name, parent_decl_ctx, max_matches, - searched_symbol_files, types); - return types.GetSize() >= max_matches; - }); -} - -void SymbolFileDWARFDebugMap::FindTypes( - llvm::ArrayRef context, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { - LLDB_SCOPED_TIMER(); - ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool { - oso_dwarf->FindTypes(context, languages, searched_symbol_files, types); - return false; + oso_dwarf->FindTypes(query, results); + return !results.Done(query); // Keep iterating if we aren't done. }); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h index 13f94f6d93e9..cd0a4bb6e41c 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h @@ -118,13 +118,8 @@ public: bool include_inlines, SymbolContextList &sc_list) override; void FindFunctions(const RegularExpression ®ex, bool include_inlines, SymbolContextList &sc_list) override; - void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; - void FindTypes(llvm::ArrayRef context, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; + void FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) override; CompilerDeclContext FindNamespace(ConstString name, const CompilerDeclContext &parent_decl_ctx, bool only_root_namespaces) override; diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp index eaca4761a485..35c2575028d8 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp @@ -1717,24 +1717,33 @@ void SymbolFileNativePDB::FindFunctions(const RegularExpression ®ex, bool include_inlines, SymbolContextList &sc_list) {} -void SymbolFileNativePDB::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, llvm::DenseSet &searched_symbol_files, - TypeMap &types) { - std::lock_guard guard(GetModuleMutex()); - if (!name) +void SymbolFileNativePDB::FindTypes(const lldb_private::TypeQuery &query, + lldb_private::TypeResults &results) { + + // Make sure we haven't already searched this SymbolFile before. + if (results.AlreadySearched(this)) return; - searched_symbol_files.clear(); - searched_symbol_files.insert(this); + std::lock_guard guard(GetModuleMutex()); - // There is an assumption 'name' is not a regex - FindTypesByName(name.GetStringRef(), max_matches, types); -} + std::vector matches = + m_index->tpi().findRecordsByName(query.GetTypeBasename().GetStringRef()); -void SymbolFileNativePDB::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, TypeMap &types) {} + for (TypeIndex type_idx : matches) { + TypeSP type_sp = GetOrCreateType(type_idx); + if (!type_sp) + continue; + + // We resolved a type. Get the fully qualified name to ensure it matches. + ConstString name = type_sp->GetQualifiedName(); + TypeQuery type_match(name.GetStringRef(), TypeQueryOptions::e_exact_match); + if (query.ContextMatches(type_match.GetContextRef())) { + results.InsertUnique(type_sp); + if (results.Done(query)) + return; + } + } +} void SymbolFileNativePDB::FindTypesByName(llvm::StringRef name, uint32_t max_matches, diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h index bf64cd330c1f..9d0458cf7ebf 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h @@ -140,14 +140,8 @@ public: std::optional FindSymbolScope(PdbCompilandSymId id); - void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; - - void FindTypes(llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) override; + void FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) override; llvm::Expected GetTypeSystemForLanguage(lldb::LanguageType language) override; diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp index 78eabc35ebf9..96036de5671d 100644 --- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp @@ -1446,24 +1446,6 @@ void SymbolFilePDB::AddSymbols(lldb_private::Symtab &symtab) { symtab.Finalize(); } -void SymbolFilePDB::FindTypes( - lldb_private::ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) { - std::lock_guard guard(GetModuleMutex()); - if (!name) - return; - if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx)) - return; - - searched_symbol_files.clear(); - searched_symbol_files.insert(this); - - // There is an assumption 'name' is not a regex - FindTypesByName(name.GetStringRef(), parent_decl_ctx, max_matches, types); -} - void SymbolFilePDB::DumpClangAST(Stream &s) { auto type_system_or_err = GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus); @@ -1536,25 +1518,27 @@ void SymbolFilePDB::FindTypesByRegex( } } -void SymbolFilePDB::FindTypesByName( - llvm::StringRef name, - const lldb_private::CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, lldb_private::TypeMap &types) { +void SymbolFilePDB::FindTypes(const lldb_private::TypeQuery &query, + lldb_private::TypeResults &type_results) { + + // Make sure we haven't already searched this SymbolFile before. + if (type_results.AlreadySearched(this)) + return; + + std::lock_guard guard(GetModuleMutex()); + std::unique_ptr results; - if (name.empty()) + llvm::StringRef basename = query.GetTypeBasename().GetStringRef(); + if (basename.empty()) return; results = m_global_scope_up->findAllChildren(PDB_SymType::None); if (!results) return; - uint32_t matches = 0; - while (auto result = results->getNext()) { - if (max_matches > 0 && matches >= max_matches) - break; if (MSVCUndecoratedNameParser::DropScope( - result->getRawSymbol().getName()) != name) + result->getRawSymbol().getName()) != basename) continue; switch (result->getSymTag()) { @@ -1573,23 +1557,20 @@ void SymbolFilePDB::FindTypesByName( if (!ResolveTypeUID(result->getSymIndexId())) continue; - if (parent_decl_ctx.IsValid() && - GetDeclContextContainingUID(result->getSymIndexId()) != parent_decl_ctx) - continue; - auto iter = m_types.find(result->getSymIndexId()); if (iter == m_types.end()) continue; - types.Insert(iter->second); - ++matches; + // We resolved a type. Get the fully qualified name to ensure it matches. + ConstString name = iter->second->GetQualifiedName(); + TypeQuery type_match(name.GetStringRef(), TypeQueryOptions::e_exact_match); + if (query.ContextMatches(type_match.GetContextRef())) { + type_results.InsertUnique(iter->second); + if (type_results.Done(query)) + return; + } } } -void SymbolFilePDB::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) {} - void SymbolFilePDB::GetTypesForPDBSymbol(const llvm::pdb::PDBSymbol &pdb_symbol, uint32_t type_mask, TypeCollection &type_collection) { diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h index 5b98c6e8b486..01851f1418f3 100644 --- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h +++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h @@ -134,19 +134,8 @@ public: std::vector &mangled_names) override; void AddSymbols(lldb_private::Symtab &symtab) override; - - void - FindTypes(lldb_private::ConstString name, - const lldb_private::CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) override; - - void FindTypes(llvm::ArrayRef pattern, - lldb_private::LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - lldb_private::TypeMap &types) override; - + void FindTypes(const lldb_private::TypeQuery &match, + lldb_private::TypeResults &results) override; void FindTypesByRegex(const lldb_private::RegularExpression ®ex, uint32_t max_matches, lldb_private::TypeMap &types); diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index 7c28935f5741..47024cd03536 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -2461,89 +2461,6 @@ void TypeSystemClang::DumpDeclHiearchy(clang::Decl *decl) { } } -bool TypeSystemClang::DeclsAreEquivalent(clang::Decl *lhs_decl, - clang::Decl *rhs_decl) { - if (lhs_decl && rhs_decl) { - // Make sure the decl kinds match first - const clang::Decl::Kind lhs_decl_kind = lhs_decl->getKind(); - const clang::Decl::Kind rhs_decl_kind = rhs_decl->getKind(); - - if (lhs_decl_kind == rhs_decl_kind) { - // Now check that the decl contexts kinds are all equivalent before we - // have to check any names of the decl contexts... - clang::DeclContext *lhs_decl_ctx = lhs_decl->getDeclContext(); - clang::DeclContext *rhs_decl_ctx = rhs_decl->getDeclContext(); - if (lhs_decl_ctx && rhs_decl_ctx) { - while (true) { - if (lhs_decl_ctx && rhs_decl_ctx) { - const clang::Decl::Kind lhs_decl_ctx_kind = - lhs_decl_ctx->getDeclKind(); - const clang::Decl::Kind rhs_decl_ctx_kind = - rhs_decl_ctx->getDeclKind(); - if (lhs_decl_ctx_kind == rhs_decl_ctx_kind) { - lhs_decl_ctx = lhs_decl_ctx->getParent(); - rhs_decl_ctx = rhs_decl_ctx->getParent(); - - if (lhs_decl_ctx == nullptr && rhs_decl_ctx == nullptr) - break; - } else - return false; - } else - return false; - } - - // Now make sure the name of the decls match - clang::NamedDecl *lhs_named_decl = - llvm::dyn_cast(lhs_decl); - clang::NamedDecl *rhs_named_decl = - llvm::dyn_cast(rhs_decl); - if (lhs_named_decl && rhs_named_decl) { - clang::DeclarationName lhs_decl_name = lhs_named_decl->getDeclName(); - clang::DeclarationName rhs_decl_name = rhs_named_decl->getDeclName(); - if (lhs_decl_name.getNameKind() == rhs_decl_name.getNameKind()) { - if (lhs_decl_name.getAsString() != rhs_decl_name.getAsString()) - return false; - } else - return false; - } else - return false; - - // We know that the decl context kinds all match, so now we need to - // make sure the names match as well - lhs_decl_ctx = lhs_decl->getDeclContext(); - rhs_decl_ctx = rhs_decl->getDeclContext(); - while (true) { - switch (lhs_decl_ctx->getDeclKind()) { - case clang::Decl::TranslationUnit: - // We don't care about the translation unit names - return true; - default: { - clang::NamedDecl *lhs_named_decl = - llvm::dyn_cast(lhs_decl_ctx); - clang::NamedDecl *rhs_named_decl = - llvm::dyn_cast(rhs_decl_ctx); - if (lhs_named_decl && rhs_named_decl) { - clang::DeclarationName lhs_decl_name = - lhs_named_decl->getDeclName(); - clang::DeclarationName rhs_decl_name = - rhs_named_decl->getDeclName(); - if (lhs_decl_name.getNameKind() == rhs_decl_name.getNameKind()) { - if (lhs_decl_name.getAsString() != rhs_decl_name.getAsString()) - return false; - } else - return false; - } else - return false; - } break; - } - lhs_decl_ctx = lhs_decl_ctx->getParent(); - rhs_decl_ctx = rhs_decl_ctx->getParent(); - } - } - } - } - return false; -} bool TypeSystemClang::GetCompleteDecl(clang::ASTContext *ast, clang::Decl *decl) { if (!decl) @@ -9070,6 +8987,66 @@ size_t TypeSystemClang::DeclGetFunctionNumArguments(void *opaque_decl) { return 0; } +static CompilerContextKind GetCompilerKind(clang::Decl::Kind clang_kind, + clang::DeclContext const *decl_ctx) { + switch (clang_kind) { + case Decl::TranslationUnit: + return CompilerContextKind::TranslationUnit; + case Decl::Namespace: + return CompilerContextKind::Namespace; + case Decl::Var: + return CompilerContextKind::Variable; + case Decl::Enum: + return CompilerContextKind::Enum; + case Decl::Typedef: + return CompilerContextKind::Typedef; + default: + // Many other kinds have multiple values + if (decl_ctx) { + if (decl_ctx->isFunctionOrMethod()) + return CompilerContextKind::Function; + else if (decl_ctx->isRecord()) + return (CompilerContextKind)((uint16_t)CompilerContextKind::Class | + (uint16_t)CompilerContextKind::Struct | + (uint16_t)CompilerContextKind::Union); + } + break; + } + return CompilerContextKind::Any; +} + +static void +InsertCompilerContext(TypeSystemClang *ts, clang::DeclContext *decl_ctx, + std::vector &context) { + if (decl_ctx == nullptr) + return; + InsertCompilerContext(ts, decl_ctx->getParent(), context); + clang::Decl::Kind clang_kind = decl_ctx->getDeclKind(); + if (clang_kind == Decl::TranslationUnit) + return; // Stop at the translation unit. + const CompilerContextKind compiler_kind = + GetCompilerKind(clang_kind, decl_ctx); + ConstString decl_ctx_name = ts->DeclContextGetName(decl_ctx); + context.push_back({compiler_kind, decl_ctx_name}); +} + +std::vector +TypeSystemClang::DeclGetCompilerContext(void *opaque_decl) { + std::vector context; + ConstString decl_name = DeclGetName(opaque_decl); + if (decl_name) { + clang::Decl *decl = (clang::Decl *)opaque_decl; + // Add the entire decl context first + clang::DeclContext *decl_ctx = decl->getDeclContext(); + InsertCompilerContext(this, decl_ctx, context); + // Now add the decl information + auto compiler_kind = + GetCompilerKind(decl->getKind(), dyn_cast(decl)); + context.push_back({compiler_kind, decl_name}); + } + return context; +} + CompilerType TypeSystemClang::DeclGetFunctionArgumentType(void *opaque_decl, size_t idx) { if (clang::FunctionDecl *func_decl = @@ -9308,6 +9285,14 @@ bool TypeSystemClang::DeclContextIsClassMethod(void *opaque_decl_ctx) { return false; } +std::vector +TypeSystemClang::DeclContextGetCompilerContext(void *opaque_decl_ctx) { + auto *decl_ctx = (clang::DeclContext *)opaque_decl_ctx; + std::vector context; + InsertCompilerContext(this, decl_ctx, context); + return context; +} + bool TypeSystemClang::DeclContextIsContainedInLookup( void *opaque_decl_ctx, void *other_opaque_decl_ctx) { auto *decl_ctx = (clang::DeclContext *)opaque_decl_ctx; diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h index 19f267396e0f..a73164895baa 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h @@ -183,8 +183,6 @@ public: static void DumpDeclContextHiearchy(clang::DeclContext *decl_ctx); - static bool DeclsAreEquivalent(clang::Decl *lhs_decl, clang::Decl *rhs_decl); - static bool GetCompleteDecl(clang::ASTContext *ast, clang::Decl *decl); void SetMetadataAsUserID(const clang::Decl *decl, lldb::user_id_t user_id); @@ -558,6 +556,9 @@ public: CompilerType DeclGetFunctionArgumentType(void *opaque_decl, size_t arg_idx) override; + std::vector + DeclGetCompilerContext(void *opaque_decl) override; + CompilerType GetTypeForDecl(void *opaque_decl) override; // CompilerDeclContext override functions @@ -587,6 +588,9 @@ public: lldb::LanguageType DeclContextGetLanguage(void *opaque_decl_ctx) override; + std::vector + DeclContextGetCompilerContext(void *opaque_decl_ctx) override; + // Clang specific clang::DeclContext functions static clang::DeclContext * diff --git a/lldb/source/Symbol/CompilerDecl.cpp b/lldb/source/Symbol/CompilerDecl.cpp index 3cafa9535a72..0eb630e5b9e1 100644 --- a/lldb/source/Symbol/CompilerDecl.cpp +++ b/lldb/source/Symbol/CompilerDecl.cpp @@ -47,3 +47,8 @@ bool lldb_private::operator!=(const lldb_private::CompilerDecl &lhs, return lhs.GetTypeSystem() != rhs.GetTypeSystem() || lhs.GetOpaqueDecl() != rhs.GetOpaqueDecl(); } + +std::vector +CompilerDecl::GetCompilerContext() const { + return m_type_system->DeclGetCompilerContext(m_opaque_decl); +} diff --git a/lldb/source/Symbol/CompilerDeclContext.cpp b/lldb/source/Symbol/CompilerDeclContext.cpp index a188e60251f7..b40a08e9b195 100644 --- a/lldb/source/Symbol/CompilerDeclContext.cpp +++ b/lldb/source/Symbol/CompilerDeclContext.cpp @@ -59,6 +59,13 @@ bool CompilerDeclContext::IsContainedInLookup(CompilerDeclContext other) const { other.m_opaque_decl_ctx); } +std::vector +CompilerDeclContext::GetCompilerContext() const { + if (IsValid()) + return m_type_system->DeclContextGetCompilerContext(m_opaque_decl_ctx); + return {}; +} + bool lldb_private::operator==(const lldb_private::CompilerDeclContext &lhs, const lldb_private::CompilerDeclContext &rhs) { return lhs.GetTypeSystem() == rhs.GetTypeSystem() && diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp index 4b9c3863e461..e318e2beb654 100644 --- a/lldb/source/Symbol/SymbolFile.cpp +++ b/lldb/source/Symbol/SymbolFile.cpp @@ -134,17 +134,6 @@ void SymbolFile::GetMangledNamesForFunction( const std::string &scope_qualified_name, std::vector &mangled_names) {} -void SymbolFile::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) {} - -void SymbolFile::FindTypes(llvm::ArrayRef pattern, - LanguageSet languages, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) {} - void SymbolFile::AssertModuleLock() { // The code below is too expensive to leave enabled in release builds. It's // enabled in debug builds or when the correct macro is set. diff --git a/lldb/source/Symbol/SymbolFileOnDemand.cpp b/lldb/source/Symbol/SymbolFileOnDemand.cpp index 19b519c63a8a..33995252bfe2 100644 --- a/lldb/source/Symbol/SymbolFileOnDemand.cpp +++ b/lldb/source/Symbol/SymbolFileOnDemand.cpp @@ -431,31 +431,14 @@ void SymbolFileOnDemand::GetMangledNamesForFunction( mangled_names); } -void SymbolFileOnDemand::FindTypes( - ConstString name, const CompilerDeclContext &parent_decl_ctx, - uint32_t max_matches, - llvm::DenseSet &searched_symbol_files, - TypeMap &types) { - if (!m_debug_info_enabled) { - Log *log = GetLog(); - LLDB_LOG(log, "[{0}] {1}({2}) is skipped", GetSymbolFileName(), - __FUNCTION__, name); - return; - } - return m_sym_file_impl->FindTypes(name, parent_decl_ctx, max_matches, - searched_symbol_files, types); -} - -void SymbolFileOnDemand::FindTypes( - llvm::ArrayRef pattern, LanguageSet languages, - llvm::DenseSet &searched_symbol_files, TypeMap &types) { +void SymbolFileOnDemand::FindTypes(const TypeQuery &match, + TypeResults &results) { if (!m_debug_info_enabled) { LLDB_LOG(GetLog(), "[{0}] {1} is skipped", GetSymbolFileName(), __FUNCTION__); return; } - return m_sym_file_impl->FindTypes(pattern, languages, searched_symbol_files, - types); + return m_sym_file_impl->FindTypes(match, results); } void SymbolFileOnDemand::GetTypes(SymbolContextScope *sc_scope, diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp index 54eeace93b96..293fe1b78f4a 100644 --- a/lldb/source/Symbol/Type.cpp +++ b/lldb/source/Symbol/Type.cpp @@ -64,6 +64,127 @@ bool lldb_private::contextMatches(llvm::ArrayRef context_chain, return true; } +static CompilerContextKind ConvertTypeClass(lldb::TypeClass type_class) { + if (type_class == eTypeClassAny) + return CompilerContextKind::AnyType; + uint16_t result = 0; + if (type_class & lldb::eTypeClassClass) + result |= (uint16_t)CompilerContextKind::Class; + if (type_class & lldb::eTypeClassStruct) + result |= (uint16_t)CompilerContextKind::Struct; + if (type_class & lldb::eTypeClassUnion) + result |= (uint16_t)CompilerContextKind::Union; + if (type_class & lldb::eTypeClassEnumeration) + result |= (uint16_t)CompilerContextKind::Enum; + if (type_class & lldb::eTypeClassFunction) + result |= (uint16_t)CompilerContextKind::Function; + if (type_class & lldb::eTypeClassTypedef) + result |= (uint16_t)CompilerContextKind::Typedef; + return (CompilerContextKind)result; +} + +TypeQuery::TypeQuery(llvm::StringRef name, TypeQueryOptions options) + : m_options(options) { + llvm::StringRef scope, basename; + lldb::TypeClass type_class = lldb::eTypeClassAny; + if (Type::GetTypeScopeAndBasename(name, scope, basename, type_class)) { + if (scope.consume_front("::")) + m_options |= e_exact_match; + if (!scope.empty()) { + std::pair scope_pair = + scope.split("::"); + while (!scope_pair.second.empty()) { + m_context.push_back({CompilerContextKind::AnyDeclContext, + ConstString(scope_pair.first.str())}); + scope_pair = scope_pair.second.split("::"); + } + m_context.push_back({CompilerContextKind::AnyDeclContext, + ConstString(scope_pair.first.str())}); + } + m_context.push_back( + {ConvertTypeClass(type_class), ConstString(basename.str())}); + } else { + m_context.push_back( + {CompilerContextKind::AnyType, ConstString(name.str())}); + } +} + +TypeQuery::TypeQuery(const CompilerDeclContext &decl_ctx, + ConstString type_basename, TypeQueryOptions options) + : m_options(options) { + // Always use an exact match if we are looking for a type in compiler context. + m_options |= e_exact_match; + m_context = decl_ctx.GetCompilerContext(); + m_context.push_back({CompilerContextKind::AnyType, type_basename}); +} + +TypeQuery::TypeQuery( + const llvm::ArrayRef &context, + TypeQueryOptions options) + : m_context(context), m_options(options) { + // Always use an exact match if we are looking for a type in compiler context. + m_options |= e_exact_match; +} + +TypeQuery::TypeQuery(const CompilerDecl &decl, TypeQueryOptions options) + : m_options(options) { + // Always for an exact match if we are looking for a type using a declaration. + m_options |= e_exact_match; + m_context = decl.GetCompilerContext(); +} + +ConstString TypeQuery::GetTypeBasename() const { + if (m_context.empty()) + return ConstString(); + return m_context.back().name; +} + +void TypeQuery::AddLanguage(LanguageType language) { + if (!m_languages) + m_languages = LanguageSet(); + m_languages->Insert(language); +} + +bool TypeQuery::ContextMatches( + llvm::ArrayRef context_chain) const { + if (GetExactMatch() || context_chain.size() == m_context.size()) + return ::contextMatches(context_chain, m_context); + + // We don't have an exact match, we need to bottom m_context.size() items to + // match for a successful lookup. + if (context_chain.size() < m_context.size()) + return false; // Not enough items in context_chain to allow for a match. + + size_t compare_count = context_chain.size() - m_context.size(); + return ::contextMatches( + llvm::ArrayRef(context_chain.data() + compare_count, + m_context.size()), + m_context); +} + +bool TypeQuery::LanguageMatches(lldb::LanguageType language) const { + // If we have no language filterm language always matches. + if (!m_languages.has_value()) + return true; + return (*m_languages)[language]; +} + +bool TypeResults::AlreadySearched(lldb_private::SymbolFile *sym_file) { + return !m_searched_symbol_files.insert(sym_file).second; +} + +bool TypeResults::InsertUnique(const lldb::TypeSP &type_sp) { + if (type_sp) + return m_type_map.InsertUnique(type_sp); + return false; +} + +bool TypeResults::Done(const TypeQuery &query) const { + if (query.GetFindOne()) + return !m_type_map.Empty(); + return false; +} + void CompilerContext::Dump(Stream &s) const { switch (kind) { default: @@ -641,6 +762,8 @@ bool Type::GetTypeScopeAndBasename(llvm::StringRef name, if (name.empty()) return false; + // Clear the scope in case we have just a type class and a basename. + scope = llvm::StringRef(); basename = name; if (basename.consume_front("struct ")) type_class = eTypeClassStruct; @@ -654,8 +777,10 @@ bool Type::GetTypeScopeAndBasename(llvm::StringRef name, type_class = eTypeClassTypedef; size_t namespace_separator = basename.find("::"); - if (namespace_separator == llvm::StringRef::npos) - return false; + if (namespace_separator == llvm::StringRef::npos) { + // If "name" started a type class we need to return true with no scope. + return type_class != eTypeClassAny; + } size_t template_begin = basename.find('<'); while (namespace_separator != llvm::StringRef::npos) { @@ -1049,16 +1174,19 @@ CompilerType TypeImpl::FindDirectNestedType(llvm::StringRef name) { return CompilerType(); auto type_system = GetTypeSystem(/*prefer_dynamic*/ false); auto *symbol_file = type_system->GetSymbolFile(); + if (!symbol_file) + return CompilerType(); auto decl_context = type_system->GetCompilerDeclContextForType(m_static_type); if (!decl_context.IsValid()) return CompilerType(); - llvm::DenseSet searched_symbol_files; - TypeMap search_result; - symbol_file->FindTypes(ConstString(name), decl_context, /*max_matches*/ 1, - searched_symbol_files, search_result); - if (search_result.Empty()) - return CompilerType(); - return search_result.GetTypeAtIndex(0)->GetFullCompilerType(); + TypeQuery query(decl_context, ConstString(name), + TypeQueryOptions::e_find_one); + TypeResults results; + symbol_file->FindTypes(query, results); + TypeSP type_sp = results.GetFirstType(); + if (type_sp) + return type_sp->GetFullCompilerType(); + return CompilerType(); } bool TypeMemberFunctionImpl::IsValid() { diff --git a/lldb/source/Symbol/TypeMap.cpp b/lldb/source/Symbol/TypeMap.cpp index 0d5f6d53e5a0..8933de53749c 100644 --- a/lldb/source/Symbol/TypeMap.cpp +++ b/lldb/source/Symbol/TypeMap.cpp @@ -91,6 +91,12 @@ TypeSP TypeMap::GetTypeAtIndex(uint32_t idx) { return TypeSP(); } +lldb::TypeSP TypeMap::FirstType() const { + if (m_types.empty()) + return TypeSP(); + return m_types.begin()->second; +} + void TypeMap::ForEach( std::function const &callback) const { for (auto pos = m_types.begin(), end = m_types.end(); pos != end; ++pos) { @@ -121,10 +127,10 @@ bool TypeMap::Remove(const lldb::TypeSP &type_sp) { return false; } -void TypeMap::Dump(Stream *s, bool show_context, lldb::DescriptionLevel level) { - for (iterator pos = m_types.begin(), end = m_types.end(); pos != end; ++pos) { - pos->second->Dump(s, show_context, level); - } +void TypeMap::Dump(Stream *s, bool show_context, + lldb::DescriptionLevel level) const { + for (const auto &pair : m_types) + pair.second->Dump(s, show_context, level); } void TypeMap::RemoveMismatchedTypes(llvm::StringRef type_scope, diff --git a/lldb/source/Symbol/TypeSystem.cpp b/lldb/source/Symbol/TypeSystem.cpp index 874f12573eca..59b1b39e635a 100644 --- a/lldb/source/Symbol/TypeSystem.cpp +++ b/lldb/source/Symbol/TypeSystem.cpp @@ -171,6 +171,16 @@ CompilerType TypeSystem::DeclGetFunctionArgumentType(void *opaque_decl, return CompilerType(); } +std::vector +TypeSystem::DeclGetCompilerContext(void *opaque_decl) { + return {}; +} + +std::vector +TypeSystem::DeclContextGetCompilerContext(void *opaque_decl_ctx) { + return {}; +} + std::vector TypeSystem::DeclContextFindDeclByName(void *opaque_decl_ctx, ConstString name, bool ignore_imported_decls) { diff --git a/lldb/source/Target/Language.cpp b/lldb/source/Target/Language.cpp index 42c3350806d9..caf3e6636c1d 100644 --- a/lldb/source/Target/Language.cpp +++ b/lldb/source/Target/Language.cpp @@ -434,12 +434,10 @@ bool Language::ImageListTypeScavenger::Find_Impl( Target *target = exe_scope->CalculateTarget().get(); if (target) { const auto &images(target->GetImages()); - ConstString cs_key(key); - llvm::DenseSet searched_sym_files; - TypeList matches; - images.FindTypes(nullptr, cs_key, false, UINT32_MAX, searched_sym_files, - matches); - for (const auto &match : matches.Types()) { + TypeQuery query(key); + TypeResults type_results; + images.FindTypes(nullptr, query, type_results); + for (const auto &match : type_results.GetTypeMap().Types()) { if (match) { CompilerType compiler_type(match->GetFullCompilerType()); compiler_type = AdjustForInclusion(compiler_type); diff --git a/lldb/test/API/functionalities/type_find_first/Makefile b/lldb/test/API/functionalities/type_find_first/Makefile new file mode 100644 index 000000000000..3d0b98f13f3d --- /dev/null +++ b/lldb/test/API/functionalities/type_find_first/Makefile @@ -0,0 +1,2 @@ +CXX_SOURCES := main.cpp +include Makefile.rules diff --git a/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py new file mode 100644 index 000000000000..432708d144f2 --- /dev/null +++ b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py @@ -0,0 +1,38 @@ +""" +Test the SBModule and SBTarget type lookup APIs. +""" + +import lldb +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TypeFindFirstTestCase(TestBase): + + NO_DEBUG_INFO_TESTCASE = True + + def test_find_first_type(self): + """ + Test SBTarget::FindFirstType() and SBModule::FindFirstType() APIs. + + This function had regressed after some past modification of the type + lookup internal code where if we had multiple types with the same + basename, FindFirstType() could end up failing depending on which + type was found first in the debug info indexes. This test will + ensure this doesn't regress in the future. + """ + self.build() + target = self.createTestTarget() + # Test the SBTarget APIs for FindFirstType + integer_type = target.FindFirstType("Integer::Point") + self.assertTrue(integer_type.IsValid()) + float_type = target.FindFirstType("Float::Point") + self.assertTrue(float_type.IsValid()) + + # Test the SBModule APIs for FindFirstType + exe_module = target.GetModuleAtIndex(0) + self.assertTrue(exe_module.IsValid()) + integer_type = exe_module.FindFirstType("Integer::Point") + self.assertTrue(integer_type.IsValid()) + float_type = exe_module.FindFirstType("Float::Point") + self.assertTrue(float_type.IsValid()) diff --git a/lldb/test/API/functionalities/type_find_first/main.cpp b/lldb/test/API/functionalities/type_find_first/main.cpp new file mode 100644 index 000000000000..f4e467286004 --- /dev/null +++ b/lldb/test/API/functionalities/type_find_first/main.cpp @@ -0,0 +1,17 @@ +namespace Integer { +struct Point { + int x, y; +}; +} // namespace Integer + +namespace Float { +struct Point { + float x, y; +}; +} // namespace Float + +int main(int argc, char const *argv[]) { + Integer::Point ip = {2, 3}; + Float::Point fp = {2.0, 3.0}; + return 0; +} diff --git a/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py b/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py index d9ac07fd00da..3fa694fa159e 100644 --- a/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py +++ b/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py @@ -17,27 +17,20 @@ class UniqueTypesTestCase4(TestBase): ) # FIXME: these should successfully print the values self.expect( - "expression ns::Foo::value", substrs=["no member named"], error=True + "expression ns::Foo::value", substrs=["'Foo' in namespace 'ns'"], error=True ) self.expect( - "expression ns::Foo::value", substrs=["no member named"], error=True + "expression ns::Foo::value", substrs=["'Foo' in namespace 'ns'"], error=True ) self.expect( - "expression ns::Bar::value", substrs=["no member named"], error=True + "expression ns::Bar::value", substrs=["'Bar' in namespace 'ns'"], error=True ) self.expect( - "expression ns::Bar::value", substrs=["no member named"], error=True - ) - self.expect( - "expression ns::FooDouble::value", - substrs=["Couldn't look up symbols"], - error=True, - ) - self.expect( - "expression ns::FooInt::value", - substrs=["Couldn't look up symbols"], - error=True, + "expression ns::Bar::value", substrs=["'Bar' in namespace 'ns'"], error=True ) + self.expect_expr("ns::FooDouble::value", result_type="double", result_value="0") + self.expect_expr("ns::FooInt::value", result_type="int", result_value="0") + @skipIf(compiler=no_match("clang")) @skipIf(compiler_version=["<", "15.0"]) diff --git a/lldb/test/API/lang/cpp/unique-types4/main.cpp b/lldb/test/API/lang/cpp/unique-types4/main.cpp index 830635202a75..815b4f92c7db 100644 --- a/lldb/test/API/lang/cpp/unique-types4/main.cpp +++ b/lldb/test/API/lang/cpp/unique-types4/main.cpp @@ -4,6 +4,8 @@ template struct Foo { static T value; }; +template T Foo::value = 0; + template using Bar = Foo; using FooInt = Foo; @@ -20,4 +22,6 @@ ns::FooDouble f; int main() { // Set breakpoint here + return (int)a.value + b.value + (int)c.value + d.value + e.value + + (int)f.value; } diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp index 45911b9065c2..e326a84c1dbd 100644 --- a/lldb/tools/lldb-test/lldb-test.cpp +++ b/lldb/tools/lldb-test/lldb-test.cpp @@ -290,8 +290,8 @@ int lldb_assert(Debugger &Dbg); } // namespace assert } // namespace opts -std::vector parseCompilerContext() { - std::vector result; +llvm::SmallVector parseCompilerContext() { + llvm::SmallVector result; if (opts::symbols::CompilerContext.empty()) return result; @@ -577,29 +577,33 @@ Error opts::symbols::findTypes(lldb_private::Module &Module) { Expected ContextOr = getDeclContext(Symfile); if (!ContextOr) return ContextOr.takeError(); - const CompilerDeclContext &ContextPtr = - ContextOr->IsValid() ? *ContextOr : CompilerDeclContext(); - - LanguageSet languages; - if (!Language.empty()) - languages.Insert(Language::GetLanguageTypeFromString(Language)); - - DenseSet SearchedFiles; - TypeMap Map; - if (!Name.empty()) - Symfile.FindTypes(ConstString(Name), ContextPtr, UINT32_MAX, SearchedFiles, - Map); - else - Module.FindTypes(parseCompilerContext(), languages, SearchedFiles, Map); - outs() << formatv("Found {0} types:\n", Map.GetSize()); + TypeResults results; + if (!Name.empty()) { + if (ContextOr->IsValid()) { + TypeQuery query(*ContextOr, ConstString(Name), + TypeQueryOptions::e_module_search); + if (!Language.empty()) + query.AddLanguage(Language::GetLanguageTypeFromString(Language)); + Symfile.FindTypes(query, results); + } else { + TypeQuery query(Name); + if (!Language.empty()) + query.AddLanguage(Language::GetLanguageTypeFromString(Language)); + Symfile.FindTypes(query, results); + } + } else { + TypeQuery query(parseCompilerContext(), TypeQueryOptions::e_module_search); + if (!Language.empty()) + query.AddLanguage(Language::GetLanguageTypeFromString(Language)); + Symfile.FindTypes(query, results); + } + outs() << formatv("Found {0} types:\n", results.GetTypeMap().GetSize()); StreamString Stream; // Resolve types to force-materialize typedef types. - Map.ForEach([&](TypeSP &type) { - type->GetFullCompilerType(); - return false; - }); - Map.Dump(&Stream, false, GetDescriptionLevel()); + for (const auto &type_sp : results.GetTypeMap().Types()) + type_sp->GetFullCompilerType(); + results.GetTypeMap().Dump(&Stream, false, GetDescriptionLevel()); outs() << Stream.GetData() << "\n"; return Error::success(); } -- GitLab From 0523bf1eca4475edafc8aab830c2ce48b01900c3 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 12 Dec 2023 17:40:54 -0800 Subject: [PATCH 017/281] [RISCV] Reduce the size of the index used for RVV intrinsics. NFC (#74906) Rather than using size_t, use uint32_t. We don't have more than 4 billion intrinsics. --- clang/lib/Sema/SemaRISCVVectorLookup.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp index 0d411fca0f9c..e4642e4da016 100644 --- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -43,7 +43,7 @@ struct RVVIntrinsicDef { struct RVVOverloadIntrinsicDef { // Indexes of RISCVIntrinsicManagerImpl::IntrinsicList. - SmallVector Indexes; + SmallVector Indexes; }; } // namespace @@ -162,7 +162,7 @@ private: // List of all RVV intrinsic. std::vector IntrinsicList; // Mapping function name to index of IntrinsicList. - StringMap Intrinsics; + StringMap Intrinsics; // Mapping function name to RVVOverloadIntrinsicDef. StringMap OverloadIntrinsics; @@ -174,7 +174,7 @@ private: // Create FunctionDecl for a vector intrinsic. void CreateRVVIntrinsicDecl(LookupResult &LR, IdentifierInfo *II, - Preprocessor &PP, unsigned Index, + Preprocessor &PP, uint32_t Index, bool IsOverload); void ConstructRVVIntrinsics(ArrayRef Recs, @@ -386,7 +386,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic( Record.HasFRMRoundModeOp); // Put into IntrinsicList. - size_t Index = IntrinsicList.size(); + uint32_t Index = IntrinsicList.size(); IntrinsicList.push_back({BuiltinName, Signature}); // Creating mapping to Intrinsics. @@ -403,7 +403,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic( void RISCVIntrinsicManagerImpl::CreateRVVIntrinsicDecl(LookupResult &LR, IdentifierInfo *II, Preprocessor &PP, - unsigned Index, + uint32_t Index, bool IsOverload) { ASTContext &Context = S.Context; RVVIntrinsicDef &IDef = IntrinsicList[Index]; -- GitLab From 53ecd3a2a5eb87975c85bfb5ccd3720b45b87a21 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Tue, 12 Dec 2023 22:27:56 -0500 Subject: [PATCH 018/281] [gn] port 27259f17e9d2 --- llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn index d98420100df0..5cd96737886c 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn @@ -8,6 +8,7 @@ static_library("Passes") { "//llvm/lib/Support", "//llvm/lib/Target", "//llvm/lib/Transforms/AggressiveInstCombine", + "//llvm/lib/Transforms/CFGuard", "//llvm/lib/Transforms/Coroutines", "//llvm/lib/Transforms/HipStdPar", "//llvm/lib/Transforms/IPO", -- GitLab From 634feddc84bfd402fc916d331627528c41346d8c Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Tue, 12 Dec 2023 19:50:24 -0800 Subject: [PATCH 019/281] [TextAPI] Add DylibReader (#75006) Add support for reading binary Mach-o dynamic libraries. It uses libObject APIs for extracting information relavant to TAPI and tbd files. This includes but is not limited to load commands encode data like install names, current/compat versions and symbols. --- llvm/include/llvm/TextAPI/DylibReader.h | 43 +++ llvm/include/llvm/TextAPI/Record.h | 4 + llvm/include/llvm/TextAPI/RecordsSlice.h | 2 + llvm/include/llvm/TextAPI/TextAPIError.h | 3 +- llvm/lib/TextAPI/CMakeLists.txt | 1 + llvm/lib/TextAPI/DylibReader.cpp | 410 +++++++++++++++++++++++ 6 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/TextAPI/DylibReader.h create mode 100644 llvm/lib/TextAPI/DylibReader.cpp diff --git a/llvm/include/llvm/TextAPI/DylibReader.h b/llvm/include/llvm/TextAPI/DylibReader.h new file mode 100644 index 000000000000..d99f22c59cf8 --- /dev/null +++ b/llvm/include/llvm/TextAPI/DylibReader.h @@ -0,0 +1,43 @@ +//===- TextAPI/DylibReader.h - TAPI MachO Dylib Reader ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Defines the MachO Dynamic Library Reader. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_DYLIBREADER_H +#define LLVM_TEXTAPI_DYLIBREADER_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TextAPI/ArchitectureSet.h" +#include "llvm/TextAPI/RecordsSlice.h" + +namespace llvm::MachO::DylibReader { + +struct ParseOption { + /// Determines arch slice to parse. + ArchitectureSet Archs = ArchitectureSet::All(); + /// Capture Mach-O header from binary, primarily load commands. + bool MachOHeader = true; + /// Capture defined symbols out of export trie and n-list. + bool SymbolTable = true; + /// Capture undefined symbols too. + bool Undefineds = true; +}; + +/// Parse Mach-O dynamic libraries to extract TAPI attributes. +/// +/// \param Buffer Data that points to dylib. +/// \param Options Determines which attributes to extract. +/// \return List of record slices. +Expected readFile(MemoryBufferRef Buffer, const ParseOption &Opt); + +} // namespace llvm::MachO::DylibReader + +#endif // LLVM_TEXTAPI_DYLIBREADER_H diff --git a/llvm/include/llvm/TextAPI/Record.h b/llvm/include/llvm/TextAPI/Record.h index 3b62af49902b..13d0bf6e6573 100644 --- a/llvm/include/llvm/TextAPI/Record.h +++ b/llvm/include/llvm/TextAPI/Record.h @@ -103,6 +103,10 @@ public: bool isFunction() const { return GV == Kind::Function; } bool isVariable() const { return GV == Kind::Variable; } + void setKind(const Kind &V) { + if (GV == Kind::Unknown) + GV = V; + } private: Kind GV; diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 8d733fd797ec..461a6d2dcc57 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -181,6 +181,8 @@ private: std::unique_ptr BA{nullptr}; }; +using Records = llvm::SmallVector, 4>; + } // namespace MachO } // namespace llvm #endif // LLVM_TEXTAPI_RECORDSLICE_H diff --git a/llvm/include/llvm/TextAPI/TextAPIError.h b/llvm/include/llvm/TextAPI/TextAPIError.h index de19f7894d35..f0578654697b 100644 --- a/llvm/include/llvm/TextAPI/TextAPIError.h +++ b/llvm/include/llvm/TextAPI/TextAPIError.h @@ -21,7 +21,8 @@ enum class TextAPIErrorCode { NoSuchArchitecture, EmptyResults, GenericFrontendError, - InvalidInputFormat + InvalidInputFormat, + UnsupportedTarget }; class TextAPIError : public llvm::ErrorInfo { diff --git a/llvm/lib/TextAPI/CMakeLists.txt b/llvm/lib/TextAPI/CMakeLists.txt index 5622ae7c6d72..357e563064c7 100644 --- a/llvm/lib/TextAPI/CMakeLists.txt +++ b/llvm/lib/TextAPI/CMakeLists.txt @@ -1,6 +1,7 @@ add_llvm_component_library(LLVMTextAPI Architecture.cpp ArchitectureSet.cpp + DylibReader.cpp InterfaceFile.cpp TextStubV5.cpp PackedVersion.cpp diff --git a/llvm/lib/TextAPI/DylibReader.cpp b/llvm/lib/TextAPI/DylibReader.cpp new file mode 100644 index 000000000000..aa13b27cc9ce --- /dev/null +++ b/llvm/lib/TextAPI/DylibReader.cpp @@ -0,0 +1,410 @@ +//===- DylibReader.cpp -------------- TAPI MachO Dylib Reader --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Implements the TAPI Reader for Mach-O dynamic libraries. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/TextAPI/DylibReader.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/MachOUniversal.h" +#include "llvm/Support/Endian.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/TextAPI/RecordsSlice.h" +#include "llvm/TextAPI/TextAPIError.h" +#include +#include +#include +#include + +using namespace llvm; +using namespace llvm::object; +using namespace llvm::MachO; +using namespace llvm::MachO::DylibReader; + +auto TripleCmp = [](const Triple &LHS, const Triple &RHS) { + return LHS.getTriple() < RHS.getTriple(); +}; +using TripleSet = std::set; + +static TripleSet constructTriples(MachOObjectFile *Obj, + const Architecture ArchT) { + auto getOSVersionStr = [](uint32_t V) { + PackedVersion OSVersion(V); + std::string Vers; + raw_string_ostream VStream(Vers); + VStream << OSVersion; + return VStream.str(); + }; + auto getOSVersion = [&](const MachOObjectFile::LoadCommandInfo &cmd) { + auto Vers = Obj->getVersionMinLoadCommand(cmd); + return getOSVersionStr(Vers.version); + }; + + // FIXME: Can remove TripleCmp arg when building in c++20. + TripleSet Triples(TripleCmp); + bool IsIntel = ArchitectureSet(ArchT).hasX86(); + auto Arch = getArchitectureName(ArchT); + + for (const auto &cmd : Obj->load_commands()) { + std::string OSVersion; + switch (cmd.C.cmd) { + case MachO::LC_VERSION_MIN_MACOSX: + OSVersion = getOSVersion(cmd); + Triples.emplace(Arch, "apple", "macos" + OSVersion); + break; + case MachO::LC_VERSION_MIN_IPHONEOS: + OSVersion = getOSVersion(cmd); + if (IsIntel) + Triples.emplace(Arch, "apple", "ios" + OSVersion, "simulator"); + else + Triples.emplace(Arch, "apple", "ios" + OSVersion); + break; + case MachO::LC_VERSION_MIN_TVOS: + OSVersion = getOSVersion(cmd); + if (IsIntel) + Triples.emplace(Arch, "apple", "tvos" + OSVersion, "simulator"); + else + Triples.emplace(Arch, "apple", "tvos" + OSVersion); + break; + case MachO::LC_VERSION_MIN_WATCHOS: + OSVersion = getOSVersion(cmd); + if (IsIntel) + Triples.emplace(Arch, "apple", "watchos" + OSVersion, "simulator"); + else + Triples.emplace(Arch, "apple", "watchos" + OSVersion); + break; + case MachO::LC_BUILD_VERSION: { + OSVersion = getOSVersionStr(Obj->getBuildVersionLoadCommand(cmd).minos); + switch (Obj->getBuildVersionLoadCommand(cmd).platform) { + case MachO::PLATFORM_MACOS: + Triples.emplace(Arch, "apple", "macos" + OSVersion); + break; + case MachO::PLATFORM_IOS: + Triples.emplace(Arch, "apple", "ios" + OSVersion); + break; + case MachO::PLATFORM_TVOS: + Triples.emplace(Arch, "apple", "tvos" + OSVersion); + break; + case MachO::PLATFORM_WATCHOS: + Triples.emplace(Arch, "apple", "watchos" + OSVersion); + break; + case MachO::PLATFORM_BRIDGEOS: + Triples.emplace(Arch, "apple", "bridgeos" + OSVersion); + break; + case MachO::PLATFORM_MACCATALYST: + Triples.emplace(Arch, "apple", "ios" + OSVersion, "macabi"); + break; + case MachO::PLATFORM_IOSSIMULATOR: + Triples.emplace(Arch, "apple", "ios" + OSVersion, "simulator"); + break; + case MachO::PLATFORM_TVOSSIMULATOR: + Triples.emplace(Arch, "apple", "tvos" + OSVersion, "simulator"); + break; + case MachO::PLATFORM_WATCHOSSIMULATOR: + Triples.emplace(Arch, "apple", "watchos" + OSVersion, "simulator"); + break; + case MachO::PLATFORM_DRIVERKIT: + Triples.emplace(Arch, "apple", "driverkit" + OSVersion); + break; + default: + break; // Skip any others. + } + break; + } + default: + break; + } + } + + // Record unknown platform for older binaries that don't enforce platform + // load commands. + if (Triples.empty()) + Triples.emplace(Arch, "apple", "unknown"); + + return Triples; +} + +static Error readMachOHeader(MachOObjectFile *Obj, RecordsSlice &Slice) { + auto H = Obj->getHeader(); + auto &BA = Slice.getBinaryAttrs(); + + switch (H.filetype) { + default: + llvm_unreachable("unsupported binary type"); + case MachO::MH_DYLIB: + BA.File = FileType::MachO_DynamicLibrary; + break; + case MachO::MH_DYLIB_STUB: + BA.File = FileType::MachO_DynamicLibrary_Stub; + break; + case MachO::MH_BUNDLE: + BA.File = FileType::MachO_Bundle; + break; + } + + if (H.flags & MachO::MH_TWOLEVEL) + BA.TwoLevelNamespace = true; + if (H.flags & MachO::MH_APP_EXTENSION_SAFE) + BA.AppExtensionSafe = true; + + for (const auto &LCI : Obj->load_commands()) { + switch (LCI.C.cmd) { + case MachO::LC_ID_DYLIB: { + auto DLLC = Obj->getDylibIDLoadCommand(LCI); + BA.InstallName = Slice.copyString(LCI.Ptr + DLLC.dylib.name); + BA.CurrentVersion = DLLC.dylib.current_version; + BA.CompatVersion = DLLC.dylib.compatibility_version; + break; + } + case MachO::LC_REEXPORT_DYLIB: { + auto DLLC = Obj->getDylibIDLoadCommand(LCI); + BA.RexportedLibraries.emplace_back( + Slice.copyString(LCI.Ptr + DLLC.dylib.name)); + break; + } + case MachO::LC_SUB_FRAMEWORK: { + auto SFC = Obj->getSubFrameworkCommand(LCI); + BA.ParentUmbrella = Slice.copyString(LCI.Ptr + SFC.umbrella); + break; + } + case MachO::LC_SUB_CLIENT: { + auto SCLC = Obj->getSubClientCommand(LCI); + BA.AllowableClients.emplace_back(Slice.copyString(LCI.Ptr + SCLC.client)); + break; + } + case MachO::LC_UUID: { + auto UUIDLC = Obj->getUuidCommand(LCI); + std::stringstream Stream; + for (unsigned I = 0; I < 16; ++I) { + if (I == 4 || I == 6 || I == 8 || I == 10) + Stream << '-'; + Stream << std::setfill('0') << std::setw(2) << std::uppercase + << std::hex << static_cast(UUIDLC.uuid[I]); + } + BA.UUID = Slice.copyString(Stream.str()); + break; + } + case MachO::LC_RPATH: { + auto RPLC = Obj->getRpathCommand(LCI); + BA.RPaths.emplace_back(Slice.copyString(LCI.Ptr + RPLC.path)); + break; + } + case MachO::LC_SEGMENT_SPLIT_INFO: { + auto SSILC = Obj->getLinkeditDataLoadCommand(LCI); + if (SSILC.datasize == 0) + BA.OSLibNotForSharedCache = true; + break; + } + default: + break; + } + } + + for (auto &Sect : Obj->sections()) { + auto SectName = Sect.getName(); + if (!SectName) + return SectName.takeError(); + if (*SectName != "__objc_imageinfo" && *SectName != "__image_info") + continue; + + auto Content = Sect.getContents(); + if (!Content) + return Content.takeError(); + + if ((Content->size() >= 8) && (Content->front() == 0)) { + uint32_t Flags; + if (Obj->isLittleEndian()) { + auto *p = + reinterpret_cast(Content->data() + 4); + Flags = *p; + } else { + auto *p = + reinterpret_cast(Content->data() + 4); + Flags = *p; + } + BA.SwiftABI = (Flags >> 8) & 0xFF; + } + } + return Error::success(); +} + +static Error readSymbols(MachOObjectFile *Obj, RecordsSlice &Slice, + const ParseOption &Opt) { + + auto parseExport = [](const auto ExportFlags, + auto Addr) -> std::tuple { + SymbolFlags Flags = SymbolFlags::None; + switch (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_KIND_MASK) { + case MachO::EXPORT_SYMBOL_FLAGS_KIND_REGULAR: + if (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION) + Flags |= SymbolFlags::WeakDefined; + break; + case MachO::EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: + Flags |= SymbolFlags::ThreadLocalValue; + break; + } + + RecordLinkage Linkage = (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_REEXPORT) + ? RecordLinkage::Rexported + : RecordLinkage::Exported; + return {Flags, Linkage}; + }; + + Error Err = Error::success(); + + StringMap> Exports; + // Collect symbols from export trie first. Sometimes, there are more exports + // in the trie than in n-list due to stripping. This is common for swift + // mangled symbols. + for (auto &Sym : Obj->exports(Err)) { + auto [Flags, Linkage] = parseExport(Sym.flags(), Sym.address()); + Slice.addRecord(Sym.name(), Flags, GlobalRecord::Kind::Unknown, Linkage); + Exports[Sym.name()] = {Flags, Linkage}; + } + + for (const auto &Sym : Obj->symbols()) { + auto FlagsOrErr = Sym.getFlags(); + if (!FlagsOrErr) + return FlagsOrErr.takeError(); + auto Flags = *FlagsOrErr; + + auto NameOrErr = Sym.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + auto Name = *NameOrErr; + + RecordLinkage Linkage = RecordLinkage::Unknown; + SymbolFlags RecordFlags = SymbolFlags::None; + + if (Opt.Undefineds && (Flags & SymbolRef::SF_Undefined)) { + Linkage = RecordLinkage::Undefined; + if (Flags & SymbolRef::SF_Weak) + RecordFlags |= SymbolFlags::WeakReferenced; + } else if (Flags & SymbolRef::SF_Exported) { + auto Exp = Exports.find(Name); + // This should never be possible when binaries are produced with Apple + // linkers. However it is possible to craft dylibs where the export trie + // is either malformed or has conflicting symbols compared to n_list. + if (Exp != Exports.end()) + std::tie(RecordFlags, Linkage) = Exp->second; + else + Linkage = RecordLinkage::Exported; + } else if (Flags & SymbolRef::SF_Hidden) { + Linkage = RecordLinkage::Internal; + } else + continue; + + auto TypeOrErr = Sym.getType(); + if (!TypeOrErr) + return TypeOrErr.takeError(); + auto Type = *TypeOrErr; + + GlobalRecord::Kind GV = (Type & SymbolRef::ST_Function) + ? GlobalRecord::Kind::Function + : GlobalRecord::Kind::Variable; + + if (GV == GlobalRecord::Kind::Function) + RecordFlags |= SymbolFlags::Text; + else + RecordFlags |= SymbolFlags::Data; + + Slice.addRecord(Name, RecordFlags, GV, Linkage); + } + return Err; +} + +static Error load(MachOObjectFile *Obj, RecordsSlice &Slice, + const ParseOption &Opt, const Architecture Arch) { + if (Arch == AK_unknown) + return make_error(TextAPIErrorCode::UnsupportedTarget); + + if (Opt.MachOHeader) + if (auto Err = readMachOHeader(Obj, Slice)) + return Err; + + if (Opt.SymbolTable) + if (auto Err = readSymbols(Obj, Slice, Opt)) + return Err; + + return Error::success(); +} + +Expected DylibReader::readFile(MemoryBufferRef Buffer, + const ParseOption &Opt) { + Records Results; + + auto BinOrErr = createBinary(Buffer); + if (!BinOrErr) + return BinOrErr.takeError(); + + Binary &Bin = *BinOrErr.get(); + if (auto *Obj = dyn_cast(&Bin)) { + const auto Arch = getArchitectureFromCpuType(Obj->getHeader().cputype, + Obj->getHeader().cpusubtype); + if (!Opt.Archs.has(Arch)) + return make_error(TextAPIErrorCode::NoSuchArchitecture); + + auto Triples = constructTriples(Obj, Arch); + for (const auto &T : Triples) { + if (mapToPlatformType(T) == PLATFORM_UNKNOWN) + return make_error(TextAPIErrorCode::UnsupportedTarget); + Results.emplace_back(std::make_shared(RecordsSlice({T}))); + if (auto Err = load(Obj, *Results.back(), Opt, Arch)) + return std::move(Err); + Results.back()->getBinaryAttrs().Path = Buffer.getBufferIdentifier(); + } + return Results; + } + + // Only expect MachO universal binaries at this point. + assert(isa(&Bin) && + "Expected a MachO universal binary."); + auto *UB = cast(&Bin); + + for (auto OI = UB->begin_objects(), OE = UB->end_objects(); OI != OE; ++OI) { + // Skip architecture if not requested. + auto Arch = + getArchitectureFromCpuType(OI->getCPUType(), OI->getCPUSubType()); + if (!Opt.Archs.has(Arch)) + continue; + + // Skip unknown architectures. + if (Arch == AK_unknown) + continue; + + // This can fail if the object is an archive. + auto ObjOrErr = OI->getAsObjectFile(); + + // Skip the archive and consume the error. + if (!ObjOrErr) { + consumeError(ObjOrErr.takeError()); + continue; + } + + auto &Obj = *ObjOrErr.get(); + switch (Obj.getHeader().filetype) { + default: + break; + case MachO::MH_BUNDLE: + case MachO::MH_DYLIB: + case MachO::MH_DYLIB_STUB: + for (const auto &T : constructTriples(&Obj, Arch)) { + Results.emplace_back(std::make_shared(RecordsSlice({T}))); + if (auto Err = load(&Obj, *Results.back(), Opt, Arch)) + return std::move(Err); + } + break; + } + } + + if (Results.empty()) + return make_error(TextAPIErrorCode::EmptyResults); + return Results; +} -- GitLab From aa217eb04459039b5ff18b6b134020ed0248a241 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Tue, 12 Dec 2023 19:55:18 -0800 Subject: [PATCH 020/281] [TextAPI] Add missing link to libObject * Reported in buildbot failures --- llvm/lib/TextAPI/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/TextAPI/CMakeLists.txt b/llvm/lib/TextAPI/CMakeLists.txt index 357e563064c7..d0067a6bcef8 100644 --- a/llvm/lib/TextAPI/CMakeLists.txt +++ b/llvm/lib/TextAPI/CMakeLists.txt @@ -21,4 +21,5 @@ add_llvm_component_library(LLVMTextAPI Support BinaryFormat TargetParser + Object ) -- GitLab From 6d8fe3dc9a4f6225c4c84de578469efc50d7684d Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Dec 2023 20:00:04 -0800 Subject: [PATCH 021/281] [sanitizer] Pre-commit disabled test for fork (#75257) --- .../TestCases/Posix/fork_threaded.cpp | 90 +++++++++++++++++++ .../sanitizer_common/sanitizer_specific.h | 18 +++- 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp new file mode 100644 index 000000000000..667f81d29853 --- /dev/null +++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp @@ -0,0 +1,90 @@ +// RUN: %clangxx -O0 %s -o %t && %env_tool_opts=die_after_fork=0 %run %t + +// UNSUPPORTED: asan, hwasan, lsan, msan, tsan, ubsan + +// Forking in multithread environment is unsupported. However we already have +// some workarounds, and will add more, so this is the test. +// The test try to check two things: +// 1. Internal mutexes used by `inparent` thread do not deadlock `inchild` +// thread. +// 2. Stack poisoned by `inparent` is not poisoned in `inchild` thread. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sanitizer_common/sanitizer_specific.h" + +static const size_t kBufferSize = 1 << 20; + +pthread_barrier_t bar; + +// Without appropriate workarounds this code can cause the forked process to +// start with locked internal mutexes. +void ShouldNotDeadlock() { + // Don't bother with leaks, we try to trigger allocator or lsan deadlock. + __lsan::ScopedDisabler disable; + char *volatile p = new char[10]; + __lsan_do_recoverable_leak_check(); + delete[] p; +} + +// Prevent stack buffer cleanup by instrumentation. +#define NOSAN __attribute__((no_sanitize("address", "hwaddress", "memory"))) + +NOSAN static void *inparent(void *arg) { + fprintf(stderr, "inparent %d\n", gettid()); + + char t[kBufferSize]; + make_mem_bad(t, sizeof(t)); + + pthread_barrier_wait(&bar); + + for (;;) + ShouldNotDeadlock(); + + return 0; +} + +NOSAN static void *inchild(void *arg) { + char t[kBufferSize]; + check_mem_is_good(t, sizeof(t)); + ShouldNotDeadlock(); + return 0; +} + +int main(void) { + pid_t pid; + + pthread_barrier_init(&bar, nullptr, 2); + pthread_t thread_id; + while (pthread_create(&thread_id, 0, &inparent, 0) != 0) { + } + pthread_barrier_wait(&bar); + + pid = fork(); + switch (pid) { + case -1: + perror("fork"); + return -1; + case 0: + while (pthread_create(&thread_id, 0, &inchild, 0) != 0) { + } + break; + default: { + fprintf(stderr, "fork %d\n", pid); + int status; + while (waitpid(-1, &status, __WALL) != pid) { + } + assert(WIFEXITED(status) && WEXITSTATUS(status) == 0); + break; + } + } + + return 0; +} diff --git a/compiler-rt/test/sanitizer_common/sanitizer_specific.h b/compiler-rt/test/sanitizer_common/sanitizer_specific.h index 1a802020cfd6..898899f00e37 100644 --- a/compiler-rt/test/sanitizer_common/sanitizer_specific.h +++ b/compiler-rt/test/sanitizer_common/sanitizer_specific.h @@ -1,6 +1,12 @@ #ifndef __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__ #define __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__ +#include + +__attribute__((weak)) int __lsan_do_recoverable_leak_check() { return 0; } +__attribute__((weak)) void __lsan_disable(void) {} +__attribute__((weak)) void __lsan_enable(void) {} + #ifndef __has_feature # define __has_feature(x) 0 #endif @@ -10,6 +16,8 @@ static void check_mem_is_good(void *p, size_t s) { __msan_check_mem_is_initialized(p, s); } +static void make_mem_good(void *p, size_t s) { __msan_unpoison(p, s); } +static void make_mem_bad(void *p, size_t s) { __msan_poison(p, s); } #elif __has_feature(address_sanitizer) # include # include @@ -17,8 +25,16 @@ static void check_mem_is_good(void *p, size_t s) { if (__asan_region_is_poisoned(p, s)) abort(); } +static void make_mem_good(void *p, size_t s) { + __asan_unpoison_memory_region(p, s); +} +static void make_mem_bad(void *p, size_t s) { + __asan_poison_memory_region(p, s); +} #else static void check_mem_is_good(void *p, size_t s) {} +static void make_mem_good(void *p, size_t s) {} +static void make_mem_bad(void *p, size_t s) {} #endif -#endif // __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__ \ No newline at end of file +#endif // __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__ -- GitLab From 1fef0fac328f047b0476e8f1edce88f1ccd30ea2 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Tue, 12 Dec 2023 20:00:15 -0800 Subject: [PATCH 022/281] Revert "[TextAPI] Add missing link to libObject" and "[TextAPI] Add DylibReader (#75006)" This reverts commit aa217eb04459039b5ff18b6b134020ed0248a241. This reverts commit 634feddc84bfd402fc916d331627528c41346d8c. This breaks buildbots by introducing cycle dependency between libObject and TextAPI and breaks gcc compiles on buildbots. --- llvm/include/llvm/TextAPI/DylibReader.h | 43 --- llvm/include/llvm/TextAPI/Record.h | 4 - llvm/include/llvm/TextAPI/RecordsSlice.h | 2 - llvm/include/llvm/TextAPI/TextAPIError.h | 3 +- llvm/lib/TextAPI/CMakeLists.txt | 2 - llvm/lib/TextAPI/DylibReader.cpp | 410 ----------------------- 6 files changed, 1 insertion(+), 463 deletions(-) delete mode 100644 llvm/include/llvm/TextAPI/DylibReader.h delete mode 100644 llvm/lib/TextAPI/DylibReader.cpp diff --git a/llvm/include/llvm/TextAPI/DylibReader.h b/llvm/include/llvm/TextAPI/DylibReader.h deleted file mode 100644 index d99f22c59cf8..000000000000 --- a/llvm/include/llvm/TextAPI/DylibReader.h +++ /dev/null @@ -1,43 +0,0 @@ -//===- TextAPI/DylibReader.h - TAPI MachO Dylib Reader ----------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Defines the MachO Dynamic Library Reader. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_TEXTAPI_DYLIBREADER_H -#define LLVM_TEXTAPI_DYLIBREADER_H - -#include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/TextAPI/ArchitectureSet.h" -#include "llvm/TextAPI/RecordsSlice.h" - -namespace llvm::MachO::DylibReader { - -struct ParseOption { - /// Determines arch slice to parse. - ArchitectureSet Archs = ArchitectureSet::All(); - /// Capture Mach-O header from binary, primarily load commands. - bool MachOHeader = true; - /// Capture defined symbols out of export trie and n-list. - bool SymbolTable = true; - /// Capture undefined symbols too. - bool Undefineds = true; -}; - -/// Parse Mach-O dynamic libraries to extract TAPI attributes. -/// -/// \param Buffer Data that points to dylib. -/// \param Options Determines which attributes to extract. -/// \return List of record slices. -Expected readFile(MemoryBufferRef Buffer, const ParseOption &Opt); - -} // namespace llvm::MachO::DylibReader - -#endif // LLVM_TEXTAPI_DYLIBREADER_H diff --git a/llvm/include/llvm/TextAPI/Record.h b/llvm/include/llvm/TextAPI/Record.h index 13d0bf6e6573..3b62af49902b 100644 --- a/llvm/include/llvm/TextAPI/Record.h +++ b/llvm/include/llvm/TextAPI/Record.h @@ -103,10 +103,6 @@ public: bool isFunction() const { return GV == Kind::Function; } bool isVariable() const { return GV == Kind::Variable; } - void setKind(const Kind &V) { - if (GV == Kind::Unknown) - GV = V; - } private: Kind GV; diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 461a6d2dcc57..8d733fd797ec 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -181,8 +181,6 @@ private: std::unique_ptr BA{nullptr}; }; -using Records = llvm::SmallVector, 4>; - } // namespace MachO } // namespace llvm #endif // LLVM_TEXTAPI_RECORDSLICE_H diff --git a/llvm/include/llvm/TextAPI/TextAPIError.h b/llvm/include/llvm/TextAPI/TextAPIError.h index f0578654697b..de19f7894d35 100644 --- a/llvm/include/llvm/TextAPI/TextAPIError.h +++ b/llvm/include/llvm/TextAPI/TextAPIError.h @@ -21,8 +21,7 @@ enum class TextAPIErrorCode { NoSuchArchitecture, EmptyResults, GenericFrontendError, - InvalidInputFormat, - UnsupportedTarget + InvalidInputFormat }; class TextAPIError : public llvm::ErrorInfo { diff --git a/llvm/lib/TextAPI/CMakeLists.txt b/llvm/lib/TextAPI/CMakeLists.txt index d0067a6bcef8..5622ae7c6d72 100644 --- a/llvm/lib/TextAPI/CMakeLists.txt +++ b/llvm/lib/TextAPI/CMakeLists.txt @@ -1,7 +1,6 @@ add_llvm_component_library(LLVMTextAPI Architecture.cpp ArchitectureSet.cpp - DylibReader.cpp InterfaceFile.cpp TextStubV5.cpp PackedVersion.cpp @@ -21,5 +20,4 @@ add_llvm_component_library(LLVMTextAPI Support BinaryFormat TargetParser - Object ) diff --git a/llvm/lib/TextAPI/DylibReader.cpp b/llvm/lib/TextAPI/DylibReader.cpp deleted file mode 100644 index aa13b27cc9ce..000000000000 --- a/llvm/lib/TextAPI/DylibReader.cpp +++ /dev/null @@ -1,410 +0,0 @@ -//===- DylibReader.cpp -------------- TAPI MachO Dylib Reader --*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Implements the TAPI Reader for Mach-O dynamic libraries. -/// -//===----------------------------------------------------------------------===// - -#include "llvm/TextAPI/DylibReader.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/MachOUniversal.h" -#include "llvm/Support/Endian.h" -#include "llvm/TargetParser/Triple.h" -#include "llvm/TextAPI/RecordsSlice.h" -#include "llvm/TextAPI/TextAPIError.h" -#include -#include -#include -#include - -using namespace llvm; -using namespace llvm::object; -using namespace llvm::MachO; -using namespace llvm::MachO::DylibReader; - -auto TripleCmp = [](const Triple &LHS, const Triple &RHS) { - return LHS.getTriple() < RHS.getTriple(); -}; -using TripleSet = std::set; - -static TripleSet constructTriples(MachOObjectFile *Obj, - const Architecture ArchT) { - auto getOSVersionStr = [](uint32_t V) { - PackedVersion OSVersion(V); - std::string Vers; - raw_string_ostream VStream(Vers); - VStream << OSVersion; - return VStream.str(); - }; - auto getOSVersion = [&](const MachOObjectFile::LoadCommandInfo &cmd) { - auto Vers = Obj->getVersionMinLoadCommand(cmd); - return getOSVersionStr(Vers.version); - }; - - // FIXME: Can remove TripleCmp arg when building in c++20. - TripleSet Triples(TripleCmp); - bool IsIntel = ArchitectureSet(ArchT).hasX86(); - auto Arch = getArchitectureName(ArchT); - - for (const auto &cmd : Obj->load_commands()) { - std::string OSVersion; - switch (cmd.C.cmd) { - case MachO::LC_VERSION_MIN_MACOSX: - OSVersion = getOSVersion(cmd); - Triples.emplace(Arch, "apple", "macos" + OSVersion); - break; - case MachO::LC_VERSION_MIN_IPHONEOS: - OSVersion = getOSVersion(cmd); - if (IsIntel) - Triples.emplace(Arch, "apple", "ios" + OSVersion, "simulator"); - else - Triples.emplace(Arch, "apple", "ios" + OSVersion); - break; - case MachO::LC_VERSION_MIN_TVOS: - OSVersion = getOSVersion(cmd); - if (IsIntel) - Triples.emplace(Arch, "apple", "tvos" + OSVersion, "simulator"); - else - Triples.emplace(Arch, "apple", "tvos" + OSVersion); - break; - case MachO::LC_VERSION_MIN_WATCHOS: - OSVersion = getOSVersion(cmd); - if (IsIntel) - Triples.emplace(Arch, "apple", "watchos" + OSVersion, "simulator"); - else - Triples.emplace(Arch, "apple", "watchos" + OSVersion); - break; - case MachO::LC_BUILD_VERSION: { - OSVersion = getOSVersionStr(Obj->getBuildVersionLoadCommand(cmd).minos); - switch (Obj->getBuildVersionLoadCommand(cmd).platform) { - case MachO::PLATFORM_MACOS: - Triples.emplace(Arch, "apple", "macos" + OSVersion); - break; - case MachO::PLATFORM_IOS: - Triples.emplace(Arch, "apple", "ios" + OSVersion); - break; - case MachO::PLATFORM_TVOS: - Triples.emplace(Arch, "apple", "tvos" + OSVersion); - break; - case MachO::PLATFORM_WATCHOS: - Triples.emplace(Arch, "apple", "watchos" + OSVersion); - break; - case MachO::PLATFORM_BRIDGEOS: - Triples.emplace(Arch, "apple", "bridgeos" + OSVersion); - break; - case MachO::PLATFORM_MACCATALYST: - Triples.emplace(Arch, "apple", "ios" + OSVersion, "macabi"); - break; - case MachO::PLATFORM_IOSSIMULATOR: - Triples.emplace(Arch, "apple", "ios" + OSVersion, "simulator"); - break; - case MachO::PLATFORM_TVOSSIMULATOR: - Triples.emplace(Arch, "apple", "tvos" + OSVersion, "simulator"); - break; - case MachO::PLATFORM_WATCHOSSIMULATOR: - Triples.emplace(Arch, "apple", "watchos" + OSVersion, "simulator"); - break; - case MachO::PLATFORM_DRIVERKIT: - Triples.emplace(Arch, "apple", "driverkit" + OSVersion); - break; - default: - break; // Skip any others. - } - break; - } - default: - break; - } - } - - // Record unknown platform for older binaries that don't enforce platform - // load commands. - if (Triples.empty()) - Triples.emplace(Arch, "apple", "unknown"); - - return Triples; -} - -static Error readMachOHeader(MachOObjectFile *Obj, RecordsSlice &Slice) { - auto H = Obj->getHeader(); - auto &BA = Slice.getBinaryAttrs(); - - switch (H.filetype) { - default: - llvm_unreachable("unsupported binary type"); - case MachO::MH_DYLIB: - BA.File = FileType::MachO_DynamicLibrary; - break; - case MachO::MH_DYLIB_STUB: - BA.File = FileType::MachO_DynamicLibrary_Stub; - break; - case MachO::MH_BUNDLE: - BA.File = FileType::MachO_Bundle; - break; - } - - if (H.flags & MachO::MH_TWOLEVEL) - BA.TwoLevelNamespace = true; - if (H.flags & MachO::MH_APP_EXTENSION_SAFE) - BA.AppExtensionSafe = true; - - for (const auto &LCI : Obj->load_commands()) { - switch (LCI.C.cmd) { - case MachO::LC_ID_DYLIB: { - auto DLLC = Obj->getDylibIDLoadCommand(LCI); - BA.InstallName = Slice.copyString(LCI.Ptr + DLLC.dylib.name); - BA.CurrentVersion = DLLC.dylib.current_version; - BA.CompatVersion = DLLC.dylib.compatibility_version; - break; - } - case MachO::LC_REEXPORT_DYLIB: { - auto DLLC = Obj->getDylibIDLoadCommand(LCI); - BA.RexportedLibraries.emplace_back( - Slice.copyString(LCI.Ptr + DLLC.dylib.name)); - break; - } - case MachO::LC_SUB_FRAMEWORK: { - auto SFC = Obj->getSubFrameworkCommand(LCI); - BA.ParentUmbrella = Slice.copyString(LCI.Ptr + SFC.umbrella); - break; - } - case MachO::LC_SUB_CLIENT: { - auto SCLC = Obj->getSubClientCommand(LCI); - BA.AllowableClients.emplace_back(Slice.copyString(LCI.Ptr + SCLC.client)); - break; - } - case MachO::LC_UUID: { - auto UUIDLC = Obj->getUuidCommand(LCI); - std::stringstream Stream; - for (unsigned I = 0; I < 16; ++I) { - if (I == 4 || I == 6 || I == 8 || I == 10) - Stream << '-'; - Stream << std::setfill('0') << std::setw(2) << std::uppercase - << std::hex << static_cast(UUIDLC.uuid[I]); - } - BA.UUID = Slice.copyString(Stream.str()); - break; - } - case MachO::LC_RPATH: { - auto RPLC = Obj->getRpathCommand(LCI); - BA.RPaths.emplace_back(Slice.copyString(LCI.Ptr + RPLC.path)); - break; - } - case MachO::LC_SEGMENT_SPLIT_INFO: { - auto SSILC = Obj->getLinkeditDataLoadCommand(LCI); - if (SSILC.datasize == 0) - BA.OSLibNotForSharedCache = true; - break; - } - default: - break; - } - } - - for (auto &Sect : Obj->sections()) { - auto SectName = Sect.getName(); - if (!SectName) - return SectName.takeError(); - if (*SectName != "__objc_imageinfo" && *SectName != "__image_info") - continue; - - auto Content = Sect.getContents(); - if (!Content) - return Content.takeError(); - - if ((Content->size() >= 8) && (Content->front() == 0)) { - uint32_t Flags; - if (Obj->isLittleEndian()) { - auto *p = - reinterpret_cast(Content->data() + 4); - Flags = *p; - } else { - auto *p = - reinterpret_cast(Content->data() + 4); - Flags = *p; - } - BA.SwiftABI = (Flags >> 8) & 0xFF; - } - } - return Error::success(); -} - -static Error readSymbols(MachOObjectFile *Obj, RecordsSlice &Slice, - const ParseOption &Opt) { - - auto parseExport = [](const auto ExportFlags, - auto Addr) -> std::tuple { - SymbolFlags Flags = SymbolFlags::None; - switch (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_KIND_MASK) { - case MachO::EXPORT_SYMBOL_FLAGS_KIND_REGULAR: - if (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION) - Flags |= SymbolFlags::WeakDefined; - break; - case MachO::EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: - Flags |= SymbolFlags::ThreadLocalValue; - break; - } - - RecordLinkage Linkage = (ExportFlags & MachO::EXPORT_SYMBOL_FLAGS_REEXPORT) - ? RecordLinkage::Rexported - : RecordLinkage::Exported; - return {Flags, Linkage}; - }; - - Error Err = Error::success(); - - StringMap> Exports; - // Collect symbols from export trie first. Sometimes, there are more exports - // in the trie than in n-list due to stripping. This is common for swift - // mangled symbols. - for (auto &Sym : Obj->exports(Err)) { - auto [Flags, Linkage] = parseExport(Sym.flags(), Sym.address()); - Slice.addRecord(Sym.name(), Flags, GlobalRecord::Kind::Unknown, Linkage); - Exports[Sym.name()] = {Flags, Linkage}; - } - - for (const auto &Sym : Obj->symbols()) { - auto FlagsOrErr = Sym.getFlags(); - if (!FlagsOrErr) - return FlagsOrErr.takeError(); - auto Flags = *FlagsOrErr; - - auto NameOrErr = Sym.getName(); - if (!NameOrErr) - return NameOrErr.takeError(); - auto Name = *NameOrErr; - - RecordLinkage Linkage = RecordLinkage::Unknown; - SymbolFlags RecordFlags = SymbolFlags::None; - - if (Opt.Undefineds && (Flags & SymbolRef::SF_Undefined)) { - Linkage = RecordLinkage::Undefined; - if (Flags & SymbolRef::SF_Weak) - RecordFlags |= SymbolFlags::WeakReferenced; - } else if (Flags & SymbolRef::SF_Exported) { - auto Exp = Exports.find(Name); - // This should never be possible when binaries are produced with Apple - // linkers. However it is possible to craft dylibs where the export trie - // is either malformed or has conflicting symbols compared to n_list. - if (Exp != Exports.end()) - std::tie(RecordFlags, Linkage) = Exp->second; - else - Linkage = RecordLinkage::Exported; - } else if (Flags & SymbolRef::SF_Hidden) { - Linkage = RecordLinkage::Internal; - } else - continue; - - auto TypeOrErr = Sym.getType(); - if (!TypeOrErr) - return TypeOrErr.takeError(); - auto Type = *TypeOrErr; - - GlobalRecord::Kind GV = (Type & SymbolRef::ST_Function) - ? GlobalRecord::Kind::Function - : GlobalRecord::Kind::Variable; - - if (GV == GlobalRecord::Kind::Function) - RecordFlags |= SymbolFlags::Text; - else - RecordFlags |= SymbolFlags::Data; - - Slice.addRecord(Name, RecordFlags, GV, Linkage); - } - return Err; -} - -static Error load(MachOObjectFile *Obj, RecordsSlice &Slice, - const ParseOption &Opt, const Architecture Arch) { - if (Arch == AK_unknown) - return make_error(TextAPIErrorCode::UnsupportedTarget); - - if (Opt.MachOHeader) - if (auto Err = readMachOHeader(Obj, Slice)) - return Err; - - if (Opt.SymbolTable) - if (auto Err = readSymbols(Obj, Slice, Opt)) - return Err; - - return Error::success(); -} - -Expected DylibReader::readFile(MemoryBufferRef Buffer, - const ParseOption &Opt) { - Records Results; - - auto BinOrErr = createBinary(Buffer); - if (!BinOrErr) - return BinOrErr.takeError(); - - Binary &Bin = *BinOrErr.get(); - if (auto *Obj = dyn_cast(&Bin)) { - const auto Arch = getArchitectureFromCpuType(Obj->getHeader().cputype, - Obj->getHeader().cpusubtype); - if (!Opt.Archs.has(Arch)) - return make_error(TextAPIErrorCode::NoSuchArchitecture); - - auto Triples = constructTriples(Obj, Arch); - for (const auto &T : Triples) { - if (mapToPlatformType(T) == PLATFORM_UNKNOWN) - return make_error(TextAPIErrorCode::UnsupportedTarget); - Results.emplace_back(std::make_shared(RecordsSlice({T}))); - if (auto Err = load(Obj, *Results.back(), Opt, Arch)) - return std::move(Err); - Results.back()->getBinaryAttrs().Path = Buffer.getBufferIdentifier(); - } - return Results; - } - - // Only expect MachO universal binaries at this point. - assert(isa(&Bin) && - "Expected a MachO universal binary."); - auto *UB = cast(&Bin); - - for (auto OI = UB->begin_objects(), OE = UB->end_objects(); OI != OE; ++OI) { - // Skip architecture if not requested. - auto Arch = - getArchitectureFromCpuType(OI->getCPUType(), OI->getCPUSubType()); - if (!Opt.Archs.has(Arch)) - continue; - - // Skip unknown architectures. - if (Arch == AK_unknown) - continue; - - // This can fail if the object is an archive. - auto ObjOrErr = OI->getAsObjectFile(); - - // Skip the archive and consume the error. - if (!ObjOrErr) { - consumeError(ObjOrErr.takeError()); - continue; - } - - auto &Obj = *ObjOrErr.get(); - switch (Obj.getHeader().filetype) { - default: - break; - case MachO::MH_BUNDLE: - case MachO::MH_DYLIB: - case MachO::MH_DYLIB_STUB: - for (const auto &T : constructTriples(&Obj, Arch)) { - Results.emplace_back(std::make_shared(RecordsSlice({T}))); - if (auto Err = load(&Obj, *Results.back(), Opt, Arch)) - return std::move(Err); - } - break; - } - } - - if (Results.empty()) - return make_error(TextAPIErrorCode::EmptyResults); - return Results; -} -- GitLab From 82d750239e554f0c73f8db3581761fffab0292c6 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 13 Dec 2023 10:55:58 +0700 Subject: [PATCH 023/281] llvm-reduce: Handle nneg flag --- llvm/test/tools/llvm-reduce/reduce-flags.ll | 16 ++++++++++++++++ .../deltas/ReduceInstructionFlags.cpp | 3 +++ 2 files changed, 19 insertions(+) diff --git a/llvm/test/tools/llvm-reduce/reduce-flags.ll b/llvm/test/tools/llvm-reduce/reduce-flags.ll index 4745f98db46b..429086effce8 100644 --- a/llvm/test/tools/llvm-reduce/reduce-flags.ll +++ b/llvm/test/tools/llvm-reduce/reduce-flags.ll @@ -200,3 +200,19 @@ define float @fadd_nnan_ninf_keep_nnan(float %a, float %b) { %op = fadd nnan ninf float %a, %b ret float %op } + +; CHECK-LABEL: @zext_nneg_drop( +; INTERESTING: = zext +; RESULT: zext i32 +define i64 @zext_nneg_drop(i32 %a) { + %op = zext nneg i32 %a to i64 + ret i64 %op +} + +; CHECK-LABEL: @zext_nneg_keep( +; INTERESTING: = zext nneg +; RESULT: zext nneg i32 +define i64 @zext_nneg_keep(i32 %a) { + %op = zext nneg i32 %a to i64 + ret i64 %op +} diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp index c73e74e4c25c..96d3de01af7d 100644 --- a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp +++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp @@ -30,6 +30,9 @@ static void reduceFlagsInModule(Oracle &O, ReducerWorkItem &WorkItem) { } else if (auto *PE = dyn_cast(&I)) { if (PE->isExact() && !O.shouldKeep()) I.setIsExact(false); + } else if (auto *NNI = dyn_cast(&I)) { + if (NNI->hasNonNeg() && !O.shouldKeep()) + NNI->setNonNeg(false); } else if (auto *GEP = dyn_cast(&I)) { if (GEP->isInBounds() && !O.shouldKeep()) GEP->setIsInBounds(false); -- GitLab From b2c7cac3ad0a642bcb5c0805c79d4a6e933112ad Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 13 Dec 2023 10:58:15 +0700 Subject: [PATCH 024/281] llvm-reduce: Handle disjoint flag --- llvm/test/tools/llvm-reduce/reduce-flags.ll | 16 ++++++++++++++++ .../deltas/ReduceInstructionFlags.cpp | 3 +++ 2 files changed, 19 insertions(+) diff --git a/llvm/test/tools/llvm-reduce/reduce-flags.ll b/llvm/test/tools/llvm-reduce/reduce-flags.ll index 429086effce8..036bfdc84ac4 100644 --- a/llvm/test/tools/llvm-reduce/reduce-flags.ll +++ b/llvm/test/tools/llvm-reduce/reduce-flags.ll @@ -216,3 +216,19 @@ define i64 @zext_nneg_keep(i32 %a) { %op = zext nneg i32 %a to i64 ret i64 %op } + +; CHECK-LABEL: @or_disjoint_drop( +; INTERESTING: = or +; RESULT: or i32 +define i32 @or_disjoint_drop(i32 %a, i32 %b) { + %op = or disjoint i32 %a, %b + ret i32 %op +} + +; CHECK-LABEL: @or_disjoint_keep( +; INTERESTING: = or disjoint +; RESULT: or disjoint i32 +define i32 @or_disjoint_keep(i32 %a, i32 %b) { + %op = or disjoint i32 %a, %b + ret i32 %op +} diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp index 96d3de01af7d..7b6fe7e5f917 100644 --- a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp +++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp @@ -33,6 +33,9 @@ static void reduceFlagsInModule(Oracle &O, ReducerWorkItem &WorkItem) { } else if (auto *NNI = dyn_cast(&I)) { if (NNI->hasNonNeg() && !O.shouldKeep()) NNI->setNonNeg(false); + } else if (auto *PDI = dyn_cast(&I)) { + if (PDI->isDisjoint() && !O.shouldKeep()) + PDI->setIsDisjoint(false); } else if (auto *GEP = dyn_cast(&I)) { if (GEP->isInBounds() && !O.shouldKeep()) GEP->setIsInBounds(false); -- GitLab From 18b41576ca067dc6eecf03ee49b098c1de3d2264 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Dec 2023 20:24:36 -0800 Subject: [PATCH 025/281] [test][sanitizer] Allow fork_threaded test on Msan, Tsan, Ubsan (#75260) They already include workarounds. --- .../test/sanitizer_common/TestCases/Posix/fork_threaded.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp index 667f81d29853..e2d67341846c 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp +++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.cpp @@ -1,6 +1,6 @@ // RUN: %clangxx -O0 %s -o %t && %env_tool_opts=die_after_fork=0 %run %t -// UNSUPPORTED: asan, hwasan, lsan, msan, tsan, ubsan +// UNSUPPORTED: asan, lsan, hwasan // Forking in multithread environment is unsupported. However we already have // some workarounds, and will add more, so this is the test. -- GitLab From a930fec033a80bc92f5a11cc334ff4fc44cbe0ca Mon Sep 17 00:00:00 2001 From: paperchalice Date: Wed, 13 Dec 2023 12:46:22 +0800 Subject: [PATCH 026/281] [CodeGen] Port `InterleavedLoadCombine` to new pass manager (#75164) --- .../include/llvm/CodeGen/CodeGenPassBuilder.h | 1 + .../llvm/CodeGen/InterleavedLoadCombine.h | 29 +++++++++++++++++++ .../llvm/CodeGen/MachinePassRegistry.def | 1 + .../CodeGen/InterleavedLoadCombinePass.cpp | 12 +++++++- llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + .../AArch64/aarch64-interleaved-ld-combine.ll | 1 + .../new-load-requires-renaming-in-mssa.ll | 1 + 8 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/CodeGen/InterleavedLoadCombine.h diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index fe604818886e..2a8aa7b158ed 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -27,6 +27,7 @@ #include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandReductions.h" #include "llvm/CodeGen/InterleavedAccess.h" +#include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/JMCInstrumenter.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" diff --git a/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h new file mode 100644 index 000000000000..fa99aa316c2a --- /dev/null +++ b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h @@ -0,0 +1,29 @@ +//===- llvm/CodeGen/InterleavedLoadCombine.h --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_INTERLEAVEDLOADCOMBINE_H +#define LLVM_CODEGEN_INTERLEAVEDLOADCOMBINE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class TargetMachine; + +class InterleavedLoadCombinePass + : public PassInfoMixin { + const TargetMachine *TM; + +public: + explicit InterleavedLoadCombinePass(const TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm + +#endif // InterleavedLoadCombine diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index 283fb14fee31..4e2bee49a71c 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -47,6 +47,7 @@ FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ()) FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ()) FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ()) FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM)) +FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass, (TM)) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ()) FUNCTION_PASS("lowerinvoke", LowerInvokePass, ()) FUNCTION_PASS("mergeicmps", MergeICmpsPass, ()) diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp index 3b1d26cfed79..f2d5c3c867c2 100644 --- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp +++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp @@ -23,6 +23,7 @@ #include "llvm/Analysis/MemorySSAUpdater.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" @@ -63,7 +64,7 @@ struct VectorInfo; struct InterleavedLoadCombineImpl { public: InterleavedLoadCombineImpl(Function &F, DominatorTree &DT, MemorySSA &MSSA, - TargetMachine &TM) + const TargetMachine &TM) : F(F), DT(DT), MSSA(MSSA), TLI(*TM.getSubtargetImpl(F)->getTargetLowering()), TTI(TM.getTargetTransformInfo(F)) {} @@ -1339,6 +1340,15 @@ private: }; } // anonymous namespace +PreservedAnalyses +InterleavedLoadCombinePass::run(Function &F, FunctionAnalysisManager &FAM) { + + auto &DT = FAM.getResult(F); + auto &MemSSA = FAM.getResult(F).getMSSA(); + bool Changed = InterleavedLoadCombineImpl(F, DT, MemSSA, *TM).run(); + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + char InterleavedLoadCombine::ID = 0; INITIALIZE_PASS_BEGIN( diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index f0417d6aa839..302fac68782c 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -78,6 +78,7 @@ #include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/HardwareLoops.h" #include "llvm/CodeGen/InterleavedAccess.h" +#include "llvm/CodeGen/InterleavedLoadCombine.h" #include "llvm/CodeGen/JMCInstrumenter.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/SelectOptimize.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 1a9a34859332..746446d61325 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -320,6 +320,7 @@ FUNCTION_PASS("instcount", InstCountPass()) FUNCTION_PASS("instnamer", InstructionNamerPass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM)) +FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass(TM)) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("irce", IRCEPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll index a5d94c13ef01..35a140cb4bd4 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll @@ -1,5 +1,6 @@ ; RUN: llc < %s | FileCheck --check-prefix AS %s ; RUN: opt -S -interleaved-load-combine < %s | FileCheck %s +; RUN: opt -S -passes=interleaved-load-combine < %s | FileCheck %s ; ModuleID = 'aarch64_interleaved-ld-combine.bc' target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll b/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll index 6ba29a664be1..d45c06a6811c 100644 --- a/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll +++ b/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -interleaved-load-combine -S -verify-memoryssa %s | FileCheck %s +; RUN: opt -passes=interleaved-load-combine -S -verify-memoryssa %s | FileCheck %s target triple = "arm64-apple-darwin" -- GitLab From 04580edd8a394dc2ccee7363c8a41ee05b1a6b98 Mon Sep 17 00:00:00 2001 From: Matheus Izvekov Date: Wed, 13 Dec 2023 01:50:05 -0300 Subject: [PATCH 027/281] [clangd] Add test for GH75115 (#75116) Add test for https://github.com/llvm/llvm-project/issues/75115 --- clang-tools-extra/clangd/test/GH75115.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 clang-tools-extra/clangd/test/GH75115.test diff --git a/clang-tools-extra/clangd/test/GH75115.test b/clang-tools-extra/clangd/test/GH75115.test new file mode 100644 index 000000000000..030392f1d69b --- /dev/null +++ b/clang-tools-extra/clangd/test/GH75115.test @@ -0,0 +1,12 @@ +// RUN: rm -rf %t.dir && mkdir -p %t.dir +// RUN: echo '[{"directory": "%/t.dir", "command": "clang --target=x86_64-pc-windows-msvc -x c GH75115.test", "file": "GH75115.test"}]' > %t.dir/compile_commands.json +// RUN: not --crash clangd -enable-config=0 --compile-commands-dir=%t.dir -check=%s 2>&1 | FileCheck -strict-whitespace %s + +// FIXME: Crashes + +// CHECK: Building preamble... +// CHECK-NEXT: Built preamble +// CHECK-NEXT: Indexing headers... +// CHECK-NEXT: !KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && "Empty/Tombstone value shouldn't be inserted into map!" + +#define assert -- GitLab From e1e5f3540950f8fc645093410b0de9cad126deb2 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Dec 2023 20:50:01 -0800 Subject: [PATCH 028/281] [NFC][lsan] clang-format includes --- compiler-rt/lib/lsan/lsan_posix.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler-rt/lib/lsan/lsan_posix.cpp b/compiler-rt/lib/lsan/lsan_posix.cpp index d99e1cc0105e..732b8af20967 100644 --- a/compiler-rt/lib/lsan/lsan_posix.cpp +++ b/compiler-rt/lib/lsan/lsan_posix.cpp @@ -14,11 +14,11 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_POSIX -#include "lsan.h" -#include "lsan_allocator.h" -#include "lsan_thread.h" -#include "sanitizer_common/sanitizer_stacktrace.h" -#include "sanitizer_common/sanitizer_tls_get_addr.h" +# include "lsan.h" +# include "lsan_allocator.h" +# include "lsan_thread.h" +# include "sanitizer_common/sanitizer_stacktrace.h" +# include "sanitizer_common/sanitizer_tls_get_addr.h" namespace __lsan { -- GitLab From 9ed20568e7de53dce85f1631d7d8c1415e7930ae Mon Sep 17 00:00:00 2001 From: Tacet Date: Wed, 13 Dec 2023 06:05:34 +0100 Subject: [PATCH 029/281] [ASan][libc++] std::basic_string annotations (#72677) This commit introduces basic annotations for `std::basic_string`, mirroring the approach used in `std::vector` and `std::deque`. Initially, only long strings with the default allocator will be annotated. Short strings (_SSO - short string optimization_) and strings with non-default allocators will be annotated in the near future, with separate commits dedicated to enabling them. The process will be similar to the workflow employed for enabling annotations in `std::deque`. **Please note**: these annotations function effectively only when libc++ and libc++abi dylibs are instrumented (with ASan). This aligns with the prevailing behavior of Memory Sanitizer. To avoid breaking everything, this commit also appends `_LIBCPP_INSTRUMENTED_WITH_ASAN` to `__config_site` whenever libc++ is compiled with ASan. If this macro is not defined, string annotations are not enabled. However, linking a binary that does **not** annotate strings with a dynamic library that annotates strings, is not permitted. Originally proposed here: https://reviews.llvm.org/D132769 Related patches on Phabricator: - Turning on annotations for short strings: https://reviews.llvm.org/D147680 - Turning on annotations for all allocators: https://reviews.llvm.org/D146214 This PR is a part of a series of patches extending AddressSanitizer C++ container overflow detection capabilities by adding annotations, similar to those existing in `std::vector` and `std::deque` collections. These enhancements empower ASan to effectively detect instances where the instrumented program attempts to access memory within a collection's internal allocation that remains unused. This includes cases where access occurs before or after the stored elements in `std::deque`, or between the `std::basic_string`'s size (including the null terminator) and capacity bounds. The introduction of these annotations was spurred by a real-world software bug discovered by Trail of Bits, involving an out-of-bounds memory access during the comparison of two strings using the `std::equals` function. This function was taking iterators (`iter1_begin`, `iter1_end`, `iter2_begin`) to perform the comparison, using a custom comparison function. When the `iter1` object exceeded the length of `iter2`, an out-of-bounds read could occur on the `iter2` object. Container sanitization, upon enabling these annotations, would effectively identify and flag this potential vulnerability. This Pull Request introduces basic annotations for `std::basic_string`. Long strings exhibit structural similarities to `std::vector` and will be annotated accordingly. Short strings are already implemented, but will be turned on separately in a forthcoming commit. Look at [a comment](https://github.com/llvm/llvm-project/pull/72677#issuecomment-1850554465) below to read about SSO issues at current moment. Due to the functionality introduced in [D132522](https://github.com/llvm/llvm-project/commit/dd1b7b797a116eed588fd752fbe61d34deeb24e4), the `__sanitizer_annotate_contiguous_container` function now offers compatibility with all allocators. However, enabling this support will be done in a subsequent commit. For the time being, only strings with the default allocator will be annotated. If you have any questions, please email: - advenam.tacet@trailofbits.com - disconnect3d@trailofbits.com --- libcxx/CMakeLists.txt | 13 + libcxx/include/__config_site.in | 1 + libcxx/include/string | 282 ++++++++++++++---- .../string.capacity/capacity.pass.cpp | 9 + .../string.capacity/clear.pass.cpp | 8 + .../string.capacity/reserve.pass.cpp | 3 + .../reserve_size.asan.pass.cpp | 63 ++++ .../string.capacity/reserve_size.pass.cpp | 3 + .../resize_and_overwrite.pass.cpp | 6 + .../string.capacity/resize_size.pass.cpp | 3 + .../string.capacity/resize_size_char.pass.cpp | 13 + .../string.capacity/shrink_to_fit.pass.cpp | 9 + .../string.cons/T_size_size.pass.cpp | 3 + .../basic.string/string.cons/alloc.pass.cpp | 6 + .../string.cons/brace_assignment.pass.cpp | 27 ++ .../string.cons/char_assignment.pass.cpp | 3 + .../basic.string/string.cons/copy.pass.cpp | 4 + .../string.cons/copy_alloc.pass.cpp | 4 + .../string.cons/copy_assignment.pass.cpp | 4 + .../basic.string/string.cons/default.pass.cpp | 2 + .../string.cons/from_range.pass.cpp | 3 + .../string.cons/from_range_deduction.pass.cpp | 3 + .../string.cons/initializer_list.pass.cpp | 14 + .../initializer_list_assignment.pass.cpp | 23 ++ .../string.cons/iter_alloc.pass.cpp | 3 + .../string.cons/iter_alloc_deduction.pass.cpp | 6 + .../basic.string/string.cons/move.pass.cpp | 4 + .../string.cons/move_alloc.pass.cpp | 4 + .../string.cons/move_assignment.pass.cpp | 5 + .../string.cons/pointer_alloc.pass.cpp | 4 + .../string.cons/pointer_assignment.pass.cpp | 3 + .../string.cons/pointer_size_alloc.pass.cpp | 4 + .../string.cons/size_char_alloc.pass.cpp | 4 + .../string.cons/string_view.pass.cpp | 6 + .../string_view_assignment.pass.cpp | 3 + .../basic.string/string.cons/substr.pass.cpp | 5 + .../string.cons/substr_rvalue.pass.cpp | 9 + .../string_append/append_range.pass.cpp | 5 + .../string_append/initializer_list.pass.cpp | 3 + .../string_append/iterator.pass.cpp | 3 + .../string_append/pointer.pass.cpp | 3 + .../string_append/pointer_size.pass.cpp | 3 + .../string_append/push_back.pass.cpp | 3 + .../string_append/size_char.pass.cpp | 3 + .../string_append/string.pass.cpp | 3 + .../string_append/string_size_size.pass.cpp | 3 + .../string_assign/T_size_size.pass.cpp | 3 + .../string_assign/assign_range.pass.cpp | 5 + .../string_assign/initializer_list.pass.cpp | 13 + .../string_assign/iterator.pass.cpp | 3 + .../string_assign/pointer.pass.cpp | 3 + .../string_assign/pointer_size.pass.cpp | 3 + .../string_assign/size_char.pass.cpp | 3 + .../string_assign/string.pass.cpp | 3 + .../string_assign/string_size_size.pass.cpp | 3 + .../string_copy/copy.pass.cpp | 5 + .../string_erase/iter.pass.cpp | 3 + .../string_erase/iter_iter.pass.cpp | 3 + .../string_erase/pop_back.pass.cpp | 5 + .../string_erase/size_size.pass.cpp | 3 + .../string_insert/iter_char.pass.cpp | 4 + .../iter_initializer_list.pass.cpp | 8 + ...iter_iter_iter.infinite_recursion.pass.cpp | 2 + .../string_insert/iter_iter_iter.pass.cpp | 3 + .../string_insert/iter_size_char.pass.cpp | 3 + .../string_insert/size_pointer.pass.cpp | 3 + .../string_insert/size_pointer_size.pass.cpp | 3 + .../string_insert/size_size_char.pass.cpp | 3 + .../string_insert/size_string.pass.cpp | 3 + .../size_string_size_size.pass.cpp | 3 + .../string_op_plus_equal/char.pass.cpp | 3 + .../initializer_list.pass.cpp | 17 ++ .../string_op_plus_equal/pointer.pass.cpp | 3 + .../string_op_plus_equal/string.pass.cpp | 4 + .../string_replace/iter_iter_string.pass.cpp | 3 + .../size_size_T_size_size.pass.cpp | 3 + .../string_replace/size_size_pointer.pass.cpp | 3 + .../size_size_pointer_size.pass.cpp | 3 + .../size_size_size_char.pass.cpp | 3 + .../string_replace/size_size_string.pass.cpp | 3 + .../size_size_string_size_size.pass.cpp | 3 + .../size_size_string_view.pass.cpp | 3 + .../string_swap/swap.asan.pass.cpp | 91 ++++++ .../string_swap/swap.pass.cpp | 9 + .../string.special/swap.pass.cpp | 4 + .../string_op+/char_string.pass.cpp | 3 + .../string_op+/string_char.pass.cpp | 3 + .../string_op+/string_pointer.pass.cpp | 3 + .../string_op+/string_string.pass.cpp | 3 + .../string.ops/string_substr/substr.pass.cpp | 4 + .../string_substr/substr_rvalue.pass.cpp | 3 + libcxx/test/support/asan_testing.h | 61 +++- 92 files changed, 862 insertions(+), 64 deletions(-) create mode 100644 libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp create mode 100644 libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.asan.pass.cpp diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt index 7751bf1efc59..5970322505dd 100644 --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -651,6 +651,19 @@ get_sanitizer_flags(SANITIZER_FLAGS "${LLVM_USE_SANITIZER}") add_library(cxx-sanitizer-flags INTERFACE) target_compile_options(cxx-sanitizer-flags INTERFACE ${SANITIZER_FLAGS}) +# _LIBCPP_INSTRUMENTED_WITH_ASAN informs that library was built with ASan. +# Defining _LIBCPP_INSTRUMENTED_WITH_ASAN while building the library with ASan is required. +# Normally, the _LIBCPP_INSTRUMENTED_WITH_ASAN flag is used to keep information whether +# dylibs are built with AddressSanitizer. However, when building libc++, +# this flag needs to be defined so that the resulting dylib has all ASan functionalities guarded by this flag. +# If the _LIBCPP_INSTRUMENTED_WITH_ASAN flag is not defined, then parts of the ASan instrumentation code in libc++ +# will not be compiled into it, resulting in false positives. +# For context, read: https://github.com/llvm/llvm-project/pull/72677#pullrequestreview-1765402800 +string(FIND "${LLVM_USE_SANITIZER}" "Address" building_with_asan) +if (NOT "${building_with_asan}" STREQUAL "-1") + config_define(ON _LIBCPP_INSTRUMENTED_WITH_ASAN) +endif() + # Link system libraries ======================================================= function(cxx_link_system_libraries target) if (NOT MSVC) diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in index 6cade6f10d8a..7c002c5bfcf8 100644 --- a/libcxx/include/__config_site.in +++ b/libcxx/include/__config_site.in @@ -29,6 +29,7 @@ #cmakedefine _LIBCPP_HAS_NO_WIDE_CHARACTERS #cmakedefine _LIBCPP_HAS_NO_STD_MODULES #cmakedefine _LIBCPP_HAS_NO_TIME_ZONE_DATABASE +#cmakedefine _LIBCPP_INSTRUMENTED_WITH_ASAN // PSTL backends #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL diff --git a/libcxx/include/string b/libcxx/include/string index 9c97abefcb8d..80ccf442ce69 100644 --- a/libcxx/include/string +++ b/libcxx/include/string @@ -649,6 +649,17 @@ basic_string operator""s( const char32_t *str, size_t len ); _LIBCPP_PUSH_MACROS #include <__undef_macros> +#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN) +# define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address"))) +// This macro disables AddressSanitizer (ASan) instrumentation for a specific function, +// allowing memory accesses that would normally trigger ASan errors to proceed without crashing. +// This is useful for accessing parts of objects memory, which should not be accessed, +// such as unused bytes in short strings, that should never be accessed +// by other parts of the program. +#else +# define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS +#endif +#define _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED false _LIBCPP_BEGIN_NAMESPACE_STD @@ -706,6 +717,9 @@ struct __init_with_sentinel_tag {}; template class basic_string { +private: + using __default_allocator_type = allocator<_CharT>; + public: typedef basic_string __self; typedef basic_string_view<_CharT, _Traits> __self_view; @@ -860,6 +874,7 @@ private: __set_long_pointer(__allocation); __set_long_size(__size); } + __annotate_new(__size); } template @@ -882,7 +897,9 @@ public: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string() _NOEXCEPT_(is_nothrow_default_constructible::value) - : __r_(__value_init_tag(), __default_init_tag()) {} + : __r_(__value_init_tag(), __default_init_tag()) { + __annotate_new(0); + } _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const allocator_type& __a) #if _LIBCPP_STD_VER <= 14 @@ -890,44 +907,65 @@ public: #else _NOEXCEPT #endif - : __r_(__value_init_tag(), __a) {} + : __r_(__value_init_tag(), __a) { + __annotate_new(0); + } - _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const basic_string& __str) + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str) : __r_(__default_init_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc())) { if (!__str.__is_long()) + { __r_.first() = __str.__r_.first(); + __annotate_new(__get_short_size()); + } else __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size()); } - _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const basic_string& __str, const allocator_type& __a) + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str, const allocator_type& __a) : __r_(__default_init_tag(), __a) { if (!__str.__is_long()) + { __r_.first() = __str.__r_.first(); + __annotate_new(__get_short_size()); + } else __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size()); } #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str) + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + basic_string(basic_string&& __str) # if _LIBCPP_STD_VER <= 14 _NOEXCEPT_(is_nothrow_move_constructible::value) # else _NOEXCEPT # endif - : __r_(std::move(__str.__r_)) { + // Turning off ASan instrumentation for variable initialization with _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS + // does not work consistently during initialization of __r_, so we instead unpoison __str's memory manually first. + // __str's memory needs to be unpoisoned only in the case where it's a short string. + : __r_( ( (__str.__is_long() ? 0 : (__str.__annotate_delete(), 0)), std::move(__str.__r_)) ) { __str.__r_.first() = __rep(); + __str.__annotate_new(0); + if(!__is_long()) + __annotate_new(size()); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str, const allocator_type& __a) + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + basic_string(basic_string&& __str, const allocator_type& __a) : __r_(__default_init_tag(), __a) { if (__str.__is_long() && __a != __str.__alloc()) // copy, not move __init(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size()); else { if (__libcpp_is_constant_evaluated()) __r_.first() = __rep(); + if (!__str.__is_long()) + __str.__annotate_delete(); __r_.first() = __str.__r_.first(); __str.__r_.first() = __rep(); + __str.__annotate_new(0); + if(!__is_long() && this != &__str) + __annotate_new(size()); } } #endif // _LIBCPP_CXX03_LANG @@ -1085,6 +1123,7 @@ public: #endif // _LIBCPP_CXX03_LANG inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() { + __annotate_delete(); if (__is_long()) __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap()); } @@ -1092,7 +1131,7 @@ public: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator __self_view() const _NOEXCEPT { return __self_view(data(), size()); } - _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const basic_string& __str); + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string& operator=(const basic_string& __str); template ::value && !__is_same_uncvref<_Tp, basic_string>::value, int> = 0> @@ -1102,8 +1141,8 @@ public: } #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(basic_string&& __str) - _NOEXCEPT_((__noexcept_move_assign_container<_Allocator, __alloc_traits>::value)) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& + operator=(basic_string&& __str) _NOEXCEPT_((__noexcept_move_assign_container<_Allocator, __alloc_traits>::value)) { __move_assign(__str, integral_constant()); return *this; } @@ -1116,7 +1155,7 @@ public: #if _LIBCPP_STD_VER >= 23 basic_string& operator=(nullptr_t) = delete; #endif - _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(value_type __c); + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string& operator=(value_type __c); _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT @@ -1339,12 +1378,22 @@ public: void __move_assign(basic_string&& __str, size_type __pos, size_type __len) { // Pilfer the allocation from __str. _LIBCPP_ASSERT_INTERNAL(__alloc() == __str.__alloc(), "__move_assign called with wrong allocator"); + size_type __old_sz = __str.size(); + if (!__str.__is_long()) + __str.__annotate_delete(); __r_.first() = __str.__r_.first(); __str.__r_.first() = __rep(); + __str.__annotate_new(0); _Traits::move(data(), data() + __pos, __len); __set_size(__len); _Traits::assign(data()[__len], value_type()); + + if (!__is_long()) { + __annotate_new(__len); + } else if(__old_sz > __len) { + __annotate_shrink(__old_sz); + } } #endif @@ -1742,7 +1791,7 @@ private: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __shrink_or_extend(size_type __target_capacity); - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS bool __is_long() const _NOEXCEPT { if (__libcpp_is_constant_evaluated() && __builtin_constant_p(__r_.first().__l.__is_long_)) { return __r_.first().__l.__is_long_; @@ -1782,6 +1831,7 @@ private: value_type* __p; if (__cap - __sz >= __n) { + __annotate_increase(__n); __p = std::__to_address(__get_pointer()); size_type __n_move = __sz - __ip; if (__n_move != 0) @@ -1808,7 +1858,7 @@ private: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 allocator_type& __alloc() _NOEXCEPT { return __r_.second(); } _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const allocator_type& __alloc() const _NOEXCEPT { return __r_.second(); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void __set_short_size(size_type __s) _NOEXCEPT { _LIBCPP_ASSERT_INTERNAL( __s < __min_cap, "__s should never be greater than or equal to the short string capacity"); @@ -1816,7 +1866,7 @@ private: __r_.first().__s.__is_long_ = false; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS size_type __get_short_size() const _NOEXCEPT { _LIBCPP_ASSERT_INTERNAL( !__r_.first().__s.__is_long_, "String has to be short when trying to get the short size"); @@ -1866,6 +1916,42 @@ private: const_pointer __get_pointer() const _NOEXCEPT {return __is_long() ? __get_long_pointer() : __get_short_pointer();} + // The following functions are no-ops outside of AddressSanitizer mode. + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_contiguous_container( + [[__maybe_unused__]] const void* __old_mid, [[__maybe_unused__]] const void* __new_mid) const { +#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN) + const void* __begin = data(); + const void* __end = data() + capacity() + 1; + if (!__libcpp_is_constant_evaluated() && __begin != nullptr && is_same::value) + __sanitizer_annotate_contiguous_container(__begin, __end, __old_mid, __new_mid); +#endif + } + + // ASan: short string is poisoned if and only if this function returns true. + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __asan_short_string_is_annotated() const _NOEXCEPT { + return _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED && !__libcpp_is_constant_evaluated(); + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT { + if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long())) + __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1); + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT { + if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long())) + __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1); + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT { + if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long())) + __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n); + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT { + if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long())) + __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1); + } + template static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __align_it(size_type __s) _NOEXCEPT @@ -1968,6 +2054,7 @@ private: } else { + __annotate_delete(); allocator_type __a = __str.__alloc(); auto __allocation = std::__allocate_at_least(__a, __str.__get_long_cap()); __begin_lifetime(__allocation.ptr, __allocation.count); @@ -1977,6 +2064,7 @@ private: __set_long_pointer(__allocation.ptr); __set_long_cap(__allocation.count); __set_long_size(__str.size()); + __annotate_new(__get_long_size()); } } } @@ -1989,7 +2077,7 @@ private: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign(basic_string& __str, false_type) _NOEXCEPT_(__alloc_traits::is_always_equal::value); - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void __move_assign(basic_string& __str, true_type) #if _LIBCPP_STD_VER >= 17 _NOEXCEPT; @@ -2024,18 +2112,28 @@ private: // Assigns the value in __s, guaranteed to be __n < __min_cap in length. inline _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& __assign_short(const value_type* __s, size_type __n) { + size_type __old_size = size(); + if (__n > __old_size) + __annotate_increase(__n - __old_size); pointer __p = __is_long() ? (__set_long_size(__n), __get_long_pointer()) : (__set_short_size(__n), __get_short_pointer()); traits_type::move(std::__to_address(__p), __s, __n); traits_type::assign(__p[__n], value_type()); + if (__old_size > __n) + __annotate_shrink(__old_size); return *this; } _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& __null_terminate_at(value_type* __p, size_type __newsz) { + size_type __old_size = size(); + if (__newsz > __old_size) + __annotate_increase(__newsz - __old_size); __set_size(__newsz); traits_type::assign(__p[__newsz], value_type()); + if (__old_size > __newsz) + __annotate_shrink(__old_size); return *this; } @@ -2142,6 +2240,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, } traits_type::copy(std::__to_address(__p), __s, __sz); traits_type::assign(__p[__sz], value_type()); + __annotate_new(__sz); } template @@ -2170,6 +2269,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_ty } traits_type::copy(std::__to_address(__p), __s, __sz); traits_type::assign(__p[__sz], value_type()); + __annotate_new(__sz); } template @@ -2194,6 +2294,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_copy_ctor_external( __set_long_size(__sz); } traits_type::copy(std::__to_address(__p), __s, __sz + 1); + __annotate_new(__sz); } template @@ -2223,6 +2324,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(size_type __n, value_type __c) } traits_type::assign(std::__to_address(__p), __n, __c); traits_type::assign(__p[__n], value_type()); + __annotate_new(__n); } template @@ -2238,6 +2340,7 @@ template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator __first, _Sentinel __last) { __r_.first() = __rep(); + __annotate_new(0); #ifndef _LIBCPP_HAS_NO_EXCEPTIONS try @@ -2249,6 +2352,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputItera } catch (...) { + __annotate_delete(); if (__is_long()) __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap()); throw; @@ -2309,6 +2413,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_with_size( throw; } #endif // _LIBCPP_HAS_NO_EXCEPTIONS + __annotate_new(__sz); } template @@ -2325,6 +2430,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_and_replace size_type __cap = __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1; + __annotate_delete(); auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1); pointer __p = __allocation.ptr; __begin_lifetime(__p, __allocation.count); @@ -2344,6 +2450,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_and_replace __old_sz = __n_copy + __n_add + __sec_cp_sz; __set_long_size(__old_sz); traits_type::assign(__p[__old_sz], value_type()); + __annotate_new(__old_cap + __delta_cap); } // __grow_by is deprecated because it does not set the size. It may not update the size when the size is changed, and it @@ -2366,6 +2473,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by(size_type __old_cap, size_t size_type __cap = __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1; + __annotate_delete(); auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1); pointer __p = __allocation.ptr; __begin_lifetime(__p, __allocation.count); @@ -2396,6 +2504,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_without_replace( __grow_by(__old_cap, __delta_cap, __old_sz, __n_copy, __n_del, __n_add); _LIBCPP_SUPPRESS_DEPRECATED_POP __set_long_size(__old_sz - __n_del + __n_add); + __annotate_new(__old_sz - __n_del + __n_add); } // assign @@ -2408,10 +2517,15 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias( const value_type* __s, size_type __n) { size_type __cap = __is_short ? static_cast(__min_cap) : __get_long_cap(); if (__n < __cap) { + size_type __old_size = __is_short ? __get_short_size() : __get_long_size(); + if (__n > __old_size) + __annotate_increase(__n - __old_size); pointer __p = __is_short ? __get_short_pointer() : __get_long_pointer(); __is_short ? __set_short_size(__n) : __set_long_size(__n); traits_type::copy(std::__to_address(__p), __s, __n); traits_type::assign(__p[__n], value_type()); + if (__old_size > __n) + __annotate_shrink(__old_size); } else { size_type __sz = __is_short ? __get_short_size() : __get_long_size(); __grow_by_and_replace(__cap - 1, __n - __cap + 1, __sz, 0, __sz, __n, __s); @@ -2426,6 +2540,9 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_external( const value_type* __s, size_type __n) { size_type __cap = capacity(); if (__cap >= __n) { + size_type __old_size = size(); + if (__n > __old_size) + __annotate_increase(__n - __old_size); value_type* __p = std::__to_address(__get_pointer()); traits_type::move(__p, __s, __n); return __null_terminate_at(__p, __n); @@ -2453,11 +2570,15 @@ basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c) { size_type __cap = capacity(); + size_type __old_size = size(); if (__cap < __n) { size_type __sz = size(); __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz); + __annotate_increase(__n); } + else if(__n > __old_size) + __annotate_increase(__n - __old_size); value_type* __p = std::__to_address(__get_pointer()); traits_type::assign(__p, __n, __c); return __null_terminate_at(__p, __n); @@ -2468,24 +2589,26 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::operator=(value_type __c) { - pointer __p; - if (__is_long()) - { - __p = __get_long_pointer(); - __set_long_size(1); - } - else - { - __p = __get_short_pointer(); - __set_short_size(1); - } - traits_type::assign(*__p, __c); - traits_type::assign(*++__p, value_type()); - return *this; + pointer __p; + size_type __old_size = size(); + if (__old_size == 0) + __annotate_increase(1); + if (__is_long()) { + __p = __get_long_pointer(); + __set_long_size(1); + } else { + __p = __get_short_pointer(); + __set_short_size(1); + } + traits_type::assign(*__p, __c); + traits_type::assign(*++__p, value_type()); + if (__old_size > 1) + __annotate_shrink(__old_size); + return *this; } template -_LIBCPP_CONSTEXPR_SINCE_CXX20 +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str) { @@ -2493,7 +2616,12 @@ basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str) __copy_assign_alloc(__str); if (!__is_long()) { if (!__str.__is_long()) { + size_type __old_size = __get_short_size(); + if (__get_short_size() < __str.__get_short_size()) + __annotate_increase(__str.__get_short_size() - __get_short_size()); __r_.first() = __str.__r_.first(); + if (__old_size > __get_short_size()) + __annotate_shrink(__old_size); } else { return __assign_no_alias(__str.data(), __str.size()); } @@ -2519,7 +2647,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, fa } template -inline _LIBCPP_CONSTEXPR_SINCE_CXX20 +inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, true_type) #if _LIBCPP_STD_VER >= 17 @@ -2528,6 +2656,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr _NOEXCEPT_(is_nothrow_move_assignable::value) #endif { + __annotate_delete(); if (__is_long()) { __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap()); @@ -2535,13 +2664,35 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr if (!is_nothrow_move_assignable::value) { __set_short_size(0); traits_type::assign(__get_short_pointer()[0], value_type()); + __annotate_new(0); } #endif } + size_type __str_old_size = __str.size(); + bool __str_was_short = !__str.__is_long(); + __move_assign_alloc(__str); __r_.first() = __str.__r_.first(); __str.__set_short_size(0); traits_type::assign(__str.__get_short_pointer()[0], value_type()); + + if (__str_was_short && this != &__str) + __str.__annotate_shrink(__str_old_size); + else + // ASan annotations: was long, so object memory is unpoisoned as new. + // Or is same as *this, and __annotate_delete() was called. + __str.__annotate_new(0); + + // ASan annotations: Guard against `std::string s; s = std::move(s);` + // You can find more here: https://en.cppreference.com/w/cpp/utility/move + // Quote: "Unless otherwise specified, all standard library objects that have been moved + // from are placed in a "valid but unspecified state", meaning the object's class + // invariants hold (so functions without preconditions, such as the assignment operator, + // can be safely used on the object after it was moved from):" + // Quote: "v = std::move(v); // the value of v is unspecified" + if (!__is_long() && &__str != this) + // If it is long string, delete was never called on original __str's buffer. + __annotate_new(__get_short_size()); } #endif @@ -2587,6 +2738,7 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _ _LIBCPP_ASSERT_INTERNAL( __string_is_trivial_iterator<_Iterator>::value, "The iterator type given to `__assign_trivial` must be trivial"); + size_type __old_size = size(); size_type __cap = capacity(); if (__cap < __n) { // Unlike `append` functions, if the input range points into the string itself, there is no case that the input @@ -2597,12 +2749,17 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _ // object itself stays valid even if reallocation happens. size_type __sz = size(); __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz); + __annotate_increase(__n); } + else if (__n > __old_size) + __annotate_increase(__n - __old_size); pointer __p = __get_pointer(); for (; __first != __last; ++__p, (void) ++__first) traits_type::assign(*__p, *__first); traits_type::assign(*__p, value_type()); __set_size(__n); + if (__n < __old_size) + __annotate_shrink(__old_size); } template @@ -2663,6 +2820,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_ty { if (__n) { + __annotate_increase(__n); value_type* __p = std::__to_address(__get_pointer()); traits_type::copy(__p + __sz, __s, __n); __sz += __n; @@ -2686,6 +2844,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c) size_type __sz = size(); if (__cap - __sz < __n) __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0); + __annotate_increase(__n); pointer __p = __get_pointer(); traits_type::assign(std::__to_address(__p) + __sz, __n, __c); __sz += __n; @@ -2705,6 +2864,7 @@ basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n) size_type __sz = size(); if (__cap - __sz < __n) __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0); + __annotate_increase(__n); pointer __p = __get_pointer(); __sz += __n; __set_size(__sz); @@ -2733,8 +2893,10 @@ basic_string<_CharT, _Traits, _Allocator>::push_back(value_type __c) if (__sz == __cap) { __grow_by_without_replace(__cap, 1, __sz, __sz, 0); + __annotate_increase(1); __is_short = false; // the string is always long after __grow_by - } + } else + __annotate_increase(1); pointer __p = __get_pointer(); if (__is_short) { @@ -2766,6 +2928,7 @@ basic_string<_CharT, _Traits, _Allocator>::append( { if (__cap - __sz < __n) __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0); + __annotate_increase(__n); pointer __p = __get_pointer() + __sz; for (; __first != __last; ++__p, (void) ++__first) traits_type::assign(*__p, *__first); @@ -2831,6 +2994,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_t { if (__n) { + __annotate_increase(__n); value_type* __p = std::__to_address(__get_pointer()); size_type __n_move = __sz - __pos; if (__n_move != 0) @@ -2864,6 +3028,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, size_type __n value_type* __p; if (__cap - __sz >= __n) { + __annotate_increase(__n); __p = std::__to_address(__get_pointer()); size_type __n_move = __sz - __pos; if (__n_move != 0) @@ -2972,6 +3137,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, value_ty } else { + __annotate_increase(1); __p = std::__to_address(__get_pointer()); size_type __n_move = __sz - __ip; if (__n_move != 0) @@ -3002,6 +3168,8 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __ value_type* __p = std::__to_address(__get_pointer()); if (__n1 != __n2) { + if (__n2 > __n1) + __annotate_increase(__n2 - __n1); size_type __n_move = __sz - __pos - __n1; if (__n_move != 0) { @@ -3046,20 +3214,18 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __ __n1 = std::min(__n1, __sz - __pos); size_type __cap = capacity(); value_type* __p; - if (__cap - __sz + __n1 >= __n2) - { - __p = std::__to_address(__get_pointer()); - if (__n1 != __n2) - { - size_type __n_move = __sz - __pos - __n1; - if (__n_move != 0) - traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move); - } - } - else - { - __grow_by_without_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2); - __p = std::__to_address(__get_long_pointer()); + if (__cap - __sz + __n1 >= __n2) { + __p = std::__to_address(__get_pointer()); + if (__n1 != __n2) { + if (__n2 > __n1) + __annotate_increase(__n2 - __n1); + size_type __n_move = __sz - __pos - __n1; + if (__n_move != 0) + traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move); + } + } else { + __grow_by_without_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2); + __p = std::__to_address(__get_long_pointer()); } traits_type::assign(__p + __pos, __n2, __c); return __null_terminate_at(__p, __sz - (__n1 - __n2)); @@ -3187,6 +3353,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT { + size_type __old_size = size(); if (__is_long()) { traits_type::assign(*__get_long_pointer(), value_type()); @@ -3197,6 +3364,7 @@ basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT traits_type::assign(*__get_short_pointer(), value_type()); __set_short_size(0); } + __annotate_shrink(__old_size); } template @@ -3259,6 +3427,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target_capacity) { + __annotate_delete(); size_type __cap = capacity(); size_type __sz = size(); @@ -3315,6 +3484,7 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target } else __set_short_size(__sz); + __annotate_new(__sz); } template @@ -3365,8 +3535,16 @@ basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str) __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value || __alloc() == __str.__alloc(), "swapping non-equal allocators"); + if (!__is_long()) + __annotate_delete(); + if (this != &__str && !__str.__is_long()) + __str.__annotate_delete(); std::swap(__r_.first(), __str.__r_.first()); std::__swap_allocator(__alloc(), __str.__alloc()); + if (!__is_long()) + __annotate_new(__get_short_size()); + if (this != &__str && !__str.__is_long()) + __str.__annotate_new(__str.__get_short_size()); } // find @@ -3854,12 +4032,12 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__clear_and_shrink() _NOEXCEPT { - clear(); - if(__is_long()) - { - __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1); - __r_.first() = __rep(); - } + clear(); + if (__is_long()) { + __annotate_delete(); + __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1); + __r_.first() = __rep(); + } } // operator== diff --git a/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp index e1d20662e41d..61867cfb087b 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp @@ -15,6 +15,7 @@ #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" #include "test_macros.h" @@ -28,6 +29,7 @@ TEST_CONSTEXPR_CXX20 void test_invariant(S s, test_allocator_statistics& alloc_s while (s.size() < s.capacity()) s.push_back(typename S::value_type()); assert(s.size() == s.capacity()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } #ifndef TEST_HAS_NO_EXCEPTIONS catch (...) { @@ -43,10 +45,12 @@ TEST_CONSTEXPR_CXX20 void test_string(const Alloc& a) { { S const s((Alloc(a))); assert(s.capacity() >= 0); + LIBCPP_ASSERT(is_string_asan_correct(s)); } { S const s(3, 'x', Alloc(a)); assert(s.capacity() >= 3); + LIBCPP_ASSERT(is_string_asan_correct(s)); } #if TEST_STD_VER >= 11 // Check that we perform SSO @@ -54,6 +58,7 @@ TEST_CONSTEXPR_CXX20 void test_string(const Alloc& a) { S const s; assert(s.capacity() > 0); ASSERT_NOEXCEPT(s.capacity()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } #endif } @@ -63,18 +68,22 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(test_allocator()); test_string(test_allocator(3)); test_string(min_allocator()); + test_string(safe_allocator()); { test_allocator_statistics alloc_stats; typedef std::basic_string, test_allocator > S; S s((test_allocator(&alloc_stats))); test_invariant(s, alloc_stats); + LIBCPP_ASSERT(is_string_asan_correct(s)); s.assign(10, 'a'); s.erase(5); test_invariant(s, alloc_stats); + LIBCPP_ASSERT(is_string_asan_correct(s)); s.assign(100, 'a'); s.erase(50); test_invariant(s, alloc_stats); + LIBCPP_ASSERT(is_string_asan_correct(s)); } return true; diff --git a/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp index 3a308de9b756..643ea4a3bdad 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp @@ -15,31 +15,39 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s) { s.clear(); assert(s.size() == 0); + LIBCPP_ASSERT(is_string_asan_correct(s)); } template TEST_CONSTEXPR_CXX20 void test_string() { S s; test(s); + LIBCPP_ASSERT(is_string_asan_correct(s)); s.assign(10, 'a'); s.erase(5); + LIBCPP_ASSERT(is_string_asan_correct(s)); test(s); + LIBCPP_ASSERT(is_string_asan_correct(s)); s.assign(100, 'a'); s.erase(50); + LIBCPP_ASSERT(is_string_asan_correct(s)); test(s); + LIBCPP_ASSERT(is_string_asan_correct(s)); } TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp index b740901be1c4..43414da3794a 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp @@ -18,6 +18,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template void test(typename S::size_type min_cap, typename S::size_type erased_index) { @@ -33,6 +34,7 @@ void test(typename S::size_type min_cap, typename S::size_type erased_index) { assert(s == s0); assert(s.capacity() <= old_cap); assert(s.capacity() >= s.size()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } template @@ -47,6 +49,7 @@ bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp new file mode 100644 index 000000000000..d35a5bcefc46 --- /dev/null +++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// This test verifies that the ASan annotations for basic_string objects remain accurate +// after invoking basic_string::reserve(size_type __requested_capacity). +// Different types are used to confirm that ASan works correctly with types of different sizes. +#include +#include + +#include "test_macros.h" +#include "asan_testing.h" + +template +void test() { + S short_s1(3, 'a'), long_s1(100, 'c'); + short_s1.reserve(0x1337); + long_s1.reserve(0x1337); + + LIBCPP_ASSERT(is_string_asan_correct(short_s1)); + LIBCPP_ASSERT(is_string_asan_correct(long_s1)); + + short_s1.clear(); + long_s1.clear(); + + LIBCPP_ASSERT(is_string_asan_correct(short_s1)); + LIBCPP_ASSERT(is_string_asan_correct(long_s1)); + + short_s1.reserve(0x1); + long_s1.reserve(0x1); + + LIBCPP_ASSERT(is_string_asan_correct(short_s1)); + LIBCPP_ASSERT(is_string_asan_correct(long_s1)); + + S short_s2(3, 'a'), long_s2(100, 'c'); + short_s2.reserve(0x1); + long_s2.reserve(0x1); + + LIBCPP_ASSERT(is_string_asan_correct(short_s2)); + LIBCPP_ASSERT(is_string_asan_correct(long_s2)); +} + +int main(int, char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif +#if TEST_STD_VER >= 11 + test(); + test(); +#endif +#if TEST_STD_VER >= 20 + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp index dfb3b270f750..30c171680a23 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp @@ -20,6 +20,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void @@ -28,6 +29,7 @@ test(typename S::size_type min_cap, typename S::size_type erased_index, typename s.erase(erased_index); assert(s.size() == erased_index); assert(s.capacity() >= min_cap); // Check that we really have at least this capacity. + LIBCPP_ASSERT(is_string_asan_correct(s)); #if TEST_STD_VER > 17 typename S::size_type old_cap = s.capacity(); @@ -39,6 +41,7 @@ test(typename S::size_type min_cap, typename S::size_type erased_index, typename assert(s == s0); assert(s.capacity() >= res_arg); assert(s.capacity() >= s.size()); + LIBCPP_ASSERT(is_string_asan_correct(s)); #if TEST_STD_VER > 17 assert(s.capacity() >= old_cap); // reserve never shrinks as of P0966 (C++20) #endif diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp index bbe6551a0ff1..edc8b67808b8 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp @@ -19,6 +19,7 @@ #include "make_string.h" #include "test_macros.h" +#include "asan_testing.h" template constexpr void test_appending(std::size_t k, size_t N, size_t new_capacity) { @@ -37,6 +38,7 @@ constexpr void test_appending(std::size_t k, size_t N, size_t new_capacity) { const S expected = S(k, 'a') + S(N - k, 'b'); assert(s == expected); assert(s.c_str()[N] == '\0'); + LIBCPP_ASSERT(is_string_asan_correct(s)); } template @@ -55,6 +57,7 @@ constexpr void test_truncating(std::size_t o, size_t N) { const S expected = S(N - 1, 'a') + S(1, 'b'); assert(s == expected); assert(s.c_str()[N] == '\0'); + LIBCPP_ASSERT(is_string_asan_correct(s)); } template @@ -76,11 +79,14 @@ constexpr bool test() { void test_value_categories() { std::string s; s.resize_and_overwrite(10, [](char*&&, std::size_t&&) { return 0; }); + LIBCPP_ASSERT(is_string_asan_correct(s)); s.resize_and_overwrite(10, [](char* const&, const std::size_t&) { return 0; }); + LIBCPP_ASSERT(is_string_asan_correct(s)); struct RefQualified { int operator()(char*, std::size_t) && { return 0; } }; s.resize_and_overwrite(10, RefQualified{}); + LIBCPP_ASSERT(is_string_asan_correct(s)); } int main(int, char**) { diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp index 487b12d9df87..7cf4b7ca3b6e 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, S expected) { @@ -23,6 +24,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, S expected) { s.resize(n); LIBCPP_ASSERT(s.__invariants()); assert(s == expected); + LIBCPP_ASSERT(is_string_asan_correct(s)); } #ifndef TEST_HAS_NO_EXCEPTIONS else if (!TEST_IS_CONSTANT_EVALUATED) { @@ -61,6 +63,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp index 3b6adc0b0afe..e3b925ca8bcd 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_type c, S expected) { @@ -23,6 +24,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_t s.resize(n, c); LIBCPP_ASSERT(s.__invariants()); assert(s == expected); + LIBCPP_ASSERT(is_string_asan_correct(s)); } #ifndef TEST_HAS_NO_EXCEPTIONS else if (!TEST_IS_CONSTANT_EVALUATED) { @@ -57,12 +59,23 @@ TEST_CONSTEXPR_CXX20 void test_string() { 'a', S("12345678901234567890123456789012345678901234567890aaaaaaaaaa")); test(S(), S::npos, 'a', S("not going to happen")); + //ASan: + test(S(), 21, 'a', S("aaaaaaaaaaaaaaaaaaaaa")); + test(S(), 22, 'a', S("aaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 23, 'a', S("aaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 24, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 29, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 30, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 31, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 32, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + test(S(), 33, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); } TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp index 66eefdd383dc..057050cdcf7f 100644 --- a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp @@ -15,6 +15,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s) { @@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s) { assert(s == s0); assert(s.capacity() <= old_cap); assert(s.capacity() >= s.size()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } template @@ -43,12 +45,19 @@ TEST_CONSTEXPR_CXX20 void test_string() { s.assign(100, 'a'); s.erase(50); test(s); + + s.assign(100, 'a'); + for (int i = 0; i <= 9; ++i) { + s.erase(90 - 10 * i); + test(s); + } } TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp index a6b625b7b0e8..dcf697bed752 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp @@ -23,6 +23,7 @@ #include "test_macros.h" #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(SV sv, std::size_t pos, std::size_t n) { @@ -38,6 +39,7 @@ TEST_CONSTEXPR_CXX20 void test(SV sv, std::size_t pos, std::size_t n) { assert(T::compare(s2.data(), sv.data() + pos, rlen) == 0); assert(s2.get_allocator() == A()); assert(s2.capacity() >= s2.size()); + LIBCPP_ASSERT(is_string_asan_correct(s2)); } #ifndef TEST_HAS_NO_EXCEPTIONS else if (!TEST_IS_CONSTANT_EVALUATED) { @@ -113,6 +115,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(test_allocator(8)); #if TEST_STD_VER >= 11 test_string(min_allocator()); + test_string(safe_allocator()); #endif { diff --git a/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp index 97a0566ba031..91beac37764d 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test() { @@ -31,6 +32,7 @@ TEST_CONSTEXPR_CXX20 void test() { assert(s.size() == 0); assert(s.capacity() >= s.size()); assert(s.get_allocator() == typename S::allocator_type()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } { #if TEST_STD_VER > 14 @@ -46,6 +48,7 @@ TEST_CONSTEXPR_CXX20 void test() { assert(s.size() == 0); assert(s.capacity() >= s.size()); assert(s.get_allocator() == typename S::allocator_type(5)); + LIBCPP_ASSERT(is_string_asan_correct(s)); } } @@ -65,6 +68,7 @@ TEST_CONSTEXPR_CXX20 void test2() { assert(s.size() == 0); assert(s.capacity() >= s.size()); assert(s.get_allocator() == typename S::allocator_type()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } { # if TEST_STD_VER > 14 @@ -80,6 +84,7 @@ TEST_CONSTEXPR_CXX20 void test2() { assert(s.size() == 0); assert(s.capacity() >= s.size()); assert(s.get_allocator() == typename S::allocator_type()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } } @@ -89,6 +94,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test, test_allocator > >(); #if TEST_STD_VER >= 11 test2, min_allocator > >(); + test2, safe_allocator > >(); test2, explicit_allocator > >(); #endif diff --git a/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp index e7d18b4ca871..49a90872c56f 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp @@ -17,6 +17,7 @@ #include #include "test_macros.h" +#include "asan_testing.h" TEST_CONSTEXPR_CXX20 bool test() { // Test that assignment from {} and {ptr, len} are allowed and are not @@ -25,11 +26,37 @@ TEST_CONSTEXPR_CXX20 bool test() { std::string s = "hello world"; s = {}; assert(s.empty()); + LIBCPP_ASSERT(is_string_asan_correct(s)); } { std::string s = "hello world"; s = {"abc", 2}; assert(s == "ab"); + LIBCPP_ASSERT(is_string_asan_correct(s)); + } + { + std::string s = "hello world"; + s = {"It'sALongString!NoSSO!qwertyuiop", 30}; + assert(s == "It'sALongString!NoSSO!qwertyui"); + LIBCPP_ASSERT(is_string_asan_correct(s)); + } + { + std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!"; + s = {"It'sALongString!NoSSO!qwertyuiop", 30}; + assert(s == "It'sALongString!NoSSO!qwertyui"); + LIBCPP_ASSERT(is_string_asan_correct(s)); + } + { + std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!"; + s = {"abc", 2}; + assert(s == "ab"); + LIBCPP_ASSERT(is_string_asan_correct(s)); + } + { + std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!"; + s = {"abc", 0}; + assert(s == ""); + LIBCPP_ASSERT(is_string_asan_correct(s)); } return true; diff --git a/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp index 3cffc82e9483..1019dc8bca5d 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp @@ -15,6 +15,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s1, typename S::value_type s2) { @@ -24,6 +25,7 @@ TEST_CONSTEXPR_CXX20 void test(S s1, typename S::value_type s2) { assert(s1.size() == 1); assert(T::eq(s1[0], s2)); assert(s1.capacity() >= s1.size()); + LIBCPP_ASSERT(is_string_asan_correct(s1)); } template @@ -38,6 +40,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp index 3afe76e88316..f65f8e97c982 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s1) { @@ -24,6 +25,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1) { assert(s2 == s1); assert(s2.capacity() >= s2.size()); assert(s2.get_allocator() == s1.get_allocator()); + LIBCPP_ASSERT(is_string_asan_correct(s1)); + LIBCPP_ASSERT(is_string_asan_correct(s2)); } template @@ -40,6 +43,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(test_allocator(3)); #if TEST_STD_VER >= 11 test_string(min_allocator()); + test_string(safe_allocator()); #endif return true; diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp index 6b0040376a42..b0045cb4afbb 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" #ifndef TEST_HAS_NO_EXCEPTIONS struct alloc_imp { @@ -83,6 +84,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, const typename S::allocator_type& a) { assert(s2 == s1); assert(s2.capacity() >= s2.size()); assert(s2.get_allocator() == a); + LIBCPP_ASSERT(is_string_asan_correct(s1)); + LIBCPP_ASSERT(is_string_asan_correct(s2)); } template @@ -99,6 +102,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(test_allocator(3)); #if TEST_STD_VER >= 11 test_string(min_allocator()); + test_string(safe_allocator()); #endif #if TEST_STD_VER >= 11 diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp index eb522aafa243..2e98fccb5394 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp @@ -16,6 +16,7 @@ #include "test_macros.h" #include "min_allocator.h" +#include "asan_testing.h" template TEST_CONSTEXPR_CXX20 void test(S s1, const S& s2) { @@ -23,6 +24,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, const S& s2) { LIBCPP_ASSERT(s1.__invariants()); assert(s1 == s2); assert(s1.capacity() >= s1.size()); + LIBCPP_ASSERT(is_string_asan_correct(s1)); + LIBCPP_ASSERT(is_string_asan_correct(s2)); } template @@ -47,6 +50,7 @@ TEST_CONSTEXPR_CXX20 bool test() { test_string(); #if TEST_STD_VER >= 11 test_string, min_allocator>>(); + test_string, safe_allocator>>(); #endif #if TEST_STD_VER >= 11 diff --git a/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp index 3993a40dd5a1..fc263f9820cb 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp @@ -15,6 +15,7 @@ #include "test_macros.h" #include "test_allocator.h" +#include "asan_testing.h" #if TEST_STD_VER >= 11 // Test the noexcept specification, which is a conforming extension @@ -30,6 +31,7 @@ LIBCPP_STATIC_ASSERT(!std::is_nothrow_default_constructible< TEST_CONSTEXPR_CXX20 bool test() { std::string str; assert(str.empty()); + LIBCPP_ASSERT(is_string_asan_correct(str)); return true; } diff --git a/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp index 3ae5b74a3504..7f33237de463 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp @@ -19,6 +19,7 @@ #include "../../../containers/from_range_helpers.h" #include "../../../containers/sequences/from_range_sequence_containers.h" #include "test_macros.h" +#include "asan_testing.h" template concept StringHasFromRangeAllocCtr = @@ -70,6 +71,7 @@ constexpr void test_with_input(std::vector input) { LIBCPP_ASSERT(c.__invariants()); assert(c.size() == static_cast(std::distance(c.begin(), c.end()))); assert(std::ranges::equal(in, c)); + LIBCPP_ASSERT(is_string_asan_correct(c)); } { // (range, allocator) @@ -80,6 +82,7 @@ constexpr void test_with_input(std::vector input) { assert(c.get_allocator() == alloc); assert(c.size() == static_cast(std::distance(c.begin(), c.end()))); assert(std::ranges::equal(in, c)); + LIBCPP_ASSERT(is_string_asan_correct(c)); } } diff --git a/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp index b2dab03506f0..83c3dfdfa79d 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp @@ -26,6 +26,7 @@ #include "deduction_guides_sfinae_checks.h" #include "test_allocator.h" +#include "asan_testing.h" int main(int, char**) { using Char = char16_t; @@ -33,12 +34,14 @@ int main(int, char**) { { std::basic_string c(std::from_range, std::array()); static_assert(std::is_same_v>); + LIBCPP_ASSERT(is_string_asan_correct(c)); } { using Alloc = test_allocator; std::basic_string c(std::from_range, std::array(), Alloc()); static_assert(std::is_same_v, Alloc>>); + LIBCPP_ASSERT(is_string_asan_correct(c)); } // Note: defining `value_type` is a workaround because one of the deduction guides will end up instantiating diff --git a/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp index 5b7e8bde2e6e..ebdcc523f055 100644 --- a/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp +++ b/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp @@ -18,6 +18,7 @@ #include "test_macros.h" #include "test_allocator.h" #include "min_allocator.h" +#include "asan_testing.h" // clang-format off template