From 7f416cc426384ad1f891addb61d93e7ca1ffa0f2 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 8 Sep 2015 08:05:57 +0000 Subject: Compute and preserve alignment more faithfully in IR-generation. Introduce an Address type to bundle a pointer value with an alignment. Introduce APIs on CGBuilderTy to work with Address values. Change core APIs on CGF/CGM to traffic in Address where appropriate. Require alignments to be non-zero. Update a ton of code to compute and propagate alignment information. As part of this, I've promoted CGBuiltin's EmitPointerWithAlignment helper function to CGF and made use of it in a number of places in the expression emitter. The end result is that we should now be significantly more correct when performing operations on objects that are locally known to be under-aligned. Since alignment is not reliably tracked in the type system, there are inherent limits to this, but at least we are no longer confused by standard operations like derived-to-base conversions and array-to-pointer decay. I've also fixed a large number of bugs where we were applying the complete-object alignment to a pointer instead of the non-virtual alignment, although most of these were hidden by the very conservative approach we took with member alignment. Also, because IRGen now reliably asserts on zero alignments, we should no longer be subject to an absurd but frustrating recurring bug where an incomplete type would report a zero alignment and then we'd naively do a alignmentAtOffset on it and emit code using an alignment equal to the largest power-of-two factor of the offset. We should also now be emitting much more aggressive alignment attributes in the presence of over-alignment. In particular, field access now uses alignmentAtOffset instead of min. Several times in this patch, I had to change the existing code-generation pattern in order to more effectively use the Address APIs. For the most part, this seems to be a strict improvement, like doing pointer arithmetic with GEPs instead of ptrtoint. That said, I've tried very hard to not change semantics, but it is likely that I've failed in a few places, for which I apologize. ABIArgInfo now always carries the assumed alignment of indirect and indirect byval arguments. In order to cut down on what was already a dauntingly large patch, I changed the code to never set align attributes in the IR on non-byval indirect arguments. That is, we still generate code which assumes that indirect arguments have the given alignment, but we don't express this information to the backend except where it's semantically required (i.e. on byvals). This is likely a minor regression for those targets that did provide this information, but it'll be trivial to add it back in a later patch. I partially punted on applying this work to CGBuiltin. Please do not add more uses of the CreateDefaultAligned{Load,Store} APIs; they will be going away eventually. llvm-svn: 246985 --- clang/lib/CodeGen/CodeGenFunction.cpp | 173 ++++++++++++++++++++++------------ 1 file changed, 112 insertions(+), 61 deletions(-) (limited to 'clang/lib/CodeGen/CodeGenFunction.cpp') diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index ea387d3..39bd2bc 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGBlocks.h" #include "CGCleanup.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" @@ -37,12 +38,14 @@ using namespace CodeGen; CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()), - Builder(cgm.getModule().getContext(), llvm::ConstantFolder(), + Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(), CGBuilderInserterTy(this)), - CurFn(nullptr), CapturedStmtInfo(nullptr), + CurFn(nullptr), ReturnValue(Address::invalid()), + CapturedStmtInfo(nullptr), SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false), CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false), - IsOutlinedSEHHelper(false), BlockInfo(nullptr), BlockPointer(nullptr), + IsOutlinedSEHHelper(false), + BlockInfo(nullptr), BlockPointer(nullptr), LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr), NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr), EHSelectorSlot(nullptr), @@ -52,7 +55,7 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0), NumSimpleReturnExprs(0), CXXABIThisDecl(nullptr), CXXABIThisValue(nullptr), CXXThisValue(nullptr), - CXXDefaultInitExprThis(nullptr), CXXStructorImplicitParamDecl(nullptr), + CXXStructorImplicitParamDecl(nullptr), CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr), CurLexicalScope(nullptr), TerminateLandingPad(nullptr), TerminateHandler(nullptr), TrapBB(nullptr) { @@ -92,18 +95,69 @@ CodeGenFunction::~CodeGenFunction() { } } -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { +CharUnits CodeGenFunction::getNaturalPointeeTypeAlignment(QualType T, + AlignmentSource *Source) { + return getNaturalTypeAlignment(T->getPointeeType(), Source, + /*forPointee*/ true); +} + +CharUnits CodeGenFunction::getNaturalTypeAlignment(QualType T, + AlignmentSource *Source, + bool forPointeeType) { + // Honor alignment typedef attributes even on incomplete types. + // We also honor them straight for C++ class types, even as pointees; + // there's an expressivity gap here. + if (auto TT = T->getAs()) { + if (auto Align = TT->getDecl()->getMaxAlignment()) { + if (Source) *Source = AlignmentSource::AttributedType; + return getContext().toCharUnitsFromBits(Align); + } + } + + if (Source) *Source = AlignmentSource::Type; + CharUnits Alignment; - if (CGM.getCXXABI().isTypeInfoCalculable(T)) { - Alignment = getContext().getTypeAlignInChars(T); - unsigned MaxAlign = getContext().getLangOpts().MaxTypeAlign; - if (MaxAlign && Alignment.getQuantity() > MaxAlign && - !getContext().isAlignmentRequired(T)) - Alignment = CharUnits::fromQuantity(MaxAlign); + if (!CGM.getCXXABI().isTypeInfoCalculable(T)) { + Alignment = CharUnits::One(); // Shouldn't be used, but pessimistic is best. + } else { + // For C++ class pointees, we don't know whether we're pointing at a + // base or a complete object, so we generally need to use the + // non-virtual alignment. + const CXXRecordDecl *RD; + if (forPointeeType && (RD = T->getAsCXXRecordDecl())) { + Alignment = CGM.getClassPointerAlignment(RD); + } else { + Alignment = getContext().getTypeAlignInChars(T); + } + + // Cap to the global maximum type alignment unless the alignment + // was somehow explicit on the type. + if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) { + if (Alignment.getQuantity() > MaxAlign && + !getContext().isAlignmentRequired(T)) + Alignment = CharUnits::fromQuantity(MaxAlign); + } } - return LValue::MakeAddr(V, T, Alignment, getContext(), CGM.getTBAAInfo(T)); + return Alignment; +} + +LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { + AlignmentSource AlignSource; + CharUnits Alignment = getNaturalTypeAlignment(T, &AlignSource); + return LValue::MakeAddr(Address(V, Alignment), T, getContext(), AlignSource, + CGM.getTBAAInfo(T)); +} + +/// Given a value of type T* that may not be to a complete object, +/// construct an l-value with the natural pointee alignment of T. +LValue +CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { + AlignmentSource AlignSource; + CharUnits Align = getNaturalTypeAlignment(T, &AlignSource, /*pointee*/ true); + return MakeAddrLValue(Address(V, Align), T, AlignSource); } + llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); } @@ -296,7 +350,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { EscapeArgs[Pair.second] = Pair.first; llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration( &CGM.getModule(), llvm::Intrinsic::localescape); - CGBuilderTy(AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); + CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); } // Remove the AllocaInsertPt instruction, which is just a convenience for us. @@ -697,7 +751,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (RetTy->isVoidType()) { // Void type; nothing to return. - ReturnValue = nullptr; + ReturnValue = Address::invalid(); // Count the implicit return. if (!endsWithReturn(D)) @@ -709,7 +763,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = AI; + ReturnValue = Address(AI, CurFnInfo->getReturnInfo().getIndirectAlign()); } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { // Load the sret pointer from the argument struct and return into that. @@ -717,7 +771,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; llvm::Value *Addr = Builder.CreateStructGEP(nullptr, EI, Idx); - ReturnValue = Builder.CreateLoad(Addr, "agg.result"); + Addr = Builder.CreateAlignedLoad(Addr, getPointerAlign(), "agg.result"); + ReturnValue = Address(Addr, getNaturalTypeAlignment(RetTy)); } else { ReturnValue = CreateIRTemp(RetTy, "retval"); @@ -1249,20 +1304,18 @@ void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) { /// base element of the array /// \param sizeInChars - the total size of the VLA, in chars static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, - llvm::Value *dest, llvm::Value *src, + Address dest, Address src, llvm::Value *sizeInChars) { - std::pair baseSizeAndAlign - = CGF.getContext().getTypeInfoInChars(baseType); - CGBuilderTy &Builder = CGF.Builder; + CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType); llvm::Value *baseSizeInChars - = llvm::ConstantInt::get(CGF.IntPtrTy, baseSizeAndAlign.first.getQuantity()); + = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); - llvm::Type *i8p = Builder.getInt8PtrTy(); - - llvm::Value *begin = Builder.CreateBitCast(dest, i8p, "vla.begin"); - llvm::Value *end = Builder.CreateInBoundsGEP(dest, sizeInChars, "vla.end"); + Address begin = + Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin"); + llvm::Value *end = + Builder.CreateInBoundsGEP(begin.getPointer(), sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -1272,17 +1325,19 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, // count must be nonzero. CGF.EmitBlock(loopBB); - llvm::PHINode *cur = Builder.CreatePHI(i8p, 2, "vla.cur"); - cur->addIncoming(begin, originBB); + llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); + cur->addIncoming(begin.getPointer(), originBB); + + CharUnits curAlign = + dest.getAlignment().alignmentOfArrayElement(baseSize); // memcpy the individual element bit-pattern. - Builder.CreateMemCpy(cur, src, baseSizeInChars, - baseSizeAndAlign.second.getQuantity(), + Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars, /*volatile*/ false); // Go to the next element. - llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), - cur, 1, "vla.next"); + llvm::Value *next = + Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next"); // Leave if that's the end of the VLA. llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone"); @@ -1293,7 +1348,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, } void -CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { +CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Ignore empty classes in C++. if (getLangOpts().CPlusPlus) { if (const RecordType *RT = Ty->getAs()) { @@ -1303,23 +1358,17 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { } // Cast the dest ptr to the appropriate i8 pointer type. - unsigned DestAS = - cast(DestPtr->getType())->getAddressSpace(); - llvm::Type *BP = Builder.getInt8PtrTy(DestAS); - if (DestPtr->getType() != BP) - DestPtr = Builder.CreateBitCast(DestPtr, BP); + if (DestPtr.getElementType() != Int8Ty) + DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); // Get size and alignment info for this aggregate. - std::pair TypeInfo = - getContext().getTypeInfoInChars(Ty); - CharUnits Size = TypeInfo.first; - CharUnits Align = TypeInfo.second; + CharUnits size = getContext().getTypeSizeInChars(Ty); llvm::Value *SizeVal; const VariableArrayType *vla; // Don't bother emitting a zero-byte memset. - if (Size.isZero()) { + if (size.isZero()) { // But note that getTypeInfo returns 0 for a VLA. if (const VariableArrayType *vlaType = dyn_cast_or_null( @@ -1337,7 +1386,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { return; } } else { - SizeVal = CGM.getSize(Size); + SizeVal = CGM.getSize(size); vla = nullptr; } @@ -1356,21 +1405,22 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, NullConstant, Twine()); - llvm::Value *SrcPtr = - Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()); + CharUnits NullAlign = DestPtr.getAlignment(); + NullVariable->setAlignment(NullAlign.getQuantity()); + Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()), + NullAlign); if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal); // Get and call the appropriate llvm.memcpy overload. - Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity(), false); + Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false); return; } // Otherwise, just memset the whole thing to zero. This is legal // because in LLVM, all default initializers (other than the ones we just // handled above) are guaranteed to have a bit pattern of all zeros. - Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, - Align.getQuantity(), false); + Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); } llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) { @@ -1389,7 +1439,7 @@ llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() { // If we already made the indirect branch for indirect goto, return its block. if (IndirectBranch) return IndirectBranch->getParent(); - CGBuilderTy TmpBuilder(createBasicBlock("indirectgoto")); + CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto")); // Create the PHI node that indirect gotos will add entries to. llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0, @@ -1404,7 +1454,7 @@ llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() { /// element type and a properly-typed first element pointer. llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, QualType &baseType, - llvm::Value *&addr) { + Address &addr) { const ArrayType *arrayType = origArrayType; // If it's a VLA, we have to load the stored size. Note that @@ -1443,8 +1493,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, QualType eltType; llvm::ArrayType *llvmArrayType = - dyn_cast( - cast(addr->getType())->getElementType()); + dyn_cast(addr.getElementType()); while (llvmArrayType) { assert(isa(arrayType)); assert(cast(arrayType)->getSize().getZExtValue() @@ -1472,12 +1521,13 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, arrayType = getContext().getAsArrayType(eltType); } - unsigned AddressSpace = addr->getType()->getPointerAddressSpace(); - llvm::Type *BaseType = ConvertType(eltType)->getPointerTo(AddressSpace); - addr = Builder.CreateBitCast(addr, BaseType, "array.begin"); + llvm::Type *baseType = ConvertType(eltType); + addr = Builder.CreateElementBitCast(addr, baseType, "array.begin"); } else { // Create the actual GEP. - addr = Builder.CreateInBoundsGEP(addr, gepIndices, "array.begin"); + addr = Address(Builder.CreateInBoundsGEP(addr.getPointer(), + gepIndices, "array.begin"), + addr.getAlignment()); } baseType = eltType; @@ -1662,9 +1712,9 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { } while (type->isVariablyModifiedType()); } -llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) { +Address CodeGenFunction::EmitVAListRef(const Expr* E) { if (getContext().getBuiltinVaListType()->isArrayType()) - return EmitScalarExpr(E); + return EmitPointerWithAlignment(E); return EmitLValue(E).getAddress(); } @@ -1726,9 +1776,10 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { I->getAnnotation(), D->getLocation()); } -llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, - llvm::Value *V) { +Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, + Address Addr) { assert(D->hasAttr() && "no annotate attribute"); + llvm::Value *V = Addr.getPointer(); llvm::Type *VTy = V->getType(); llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation, CGM.Int8PtrTy); @@ -1743,7 +1794,7 @@ llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, V = Builder.CreateBitCast(V, VTy); } - return V; + return Address(V, Addr.getAlignment()); } CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { } -- cgit v1.1