From 072e83500e1f4291692753fa19840b17c694244e Mon Sep 17 00:00:00 2001 From: Evgeniy Stepanov Date: Fri, 11 Sep 2015 20:29:07 +0000 Subject: Always_inline codegen rewrite. Current implementation may end up emitting an undefined reference for an "inline __attribute__((always_inline))" function by generating an "available_externally alwaysinline" IR function for it and then failing to inline all the calls. This happens when a call to such function is in dead code. As the inliner is an SCC pass, it does not process dead code. Libc++ relies on the compiler never emitting such undefined reference. With this patch, we emit a pair of 1. internal alwaysinline definition (called F.alwaysinline) 2a. A stub F() { musttail call F.alwaysinline } -- or, depending on the linkage -- 2b. A declaration of F. The frontend ensures that F.inlinefunction is only used for direct calls, and the stub is used for everything else (taking the address of the function, really). Declaration (2b) is emitted in the case when "inline" is meant for inlining only (like __gnu_inline__ and some other cases). This approach, among other nice properties, ensures that alwaysinline functions are always internal, making it impossible for a direct call to such function to produce an undefined symbol reference. This patch is based on ideas by Chandler Carruth and Richard Smith. llvm-svn: 247465 --- clang/lib/CodeGen/CodeGenModule.cpp | 99 ++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) (limited to 'clang/lib/CodeGen/CodeGenModule.cpp') diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index bbc01cb..ef1dcec 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -448,6 +448,103 @@ void CodeGenModule::Release() { EmitVersionIdentMetadata(); EmitTargetMetadata(); + + RewriteAlwaysInlineFunctions(); +} + +void CodeGenModule::AddAlwaysInlineFunction(llvm::Function *Fn) { + AlwaysInlineFunctions.push_back(Fn); +} + +/// Find all uses of GV that are not direct calls or invokes. +static void FindNonDirectCallUses(llvm::GlobalValue *GV, + llvm::SmallVectorImpl *Uses) { + llvm::GlobalValue::use_iterator UI = GV->use_begin(), E = GV->use_end(); + for (; UI != E;) { + llvm::Use &U = *UI; + ++UI; + + llvm::CallSite CS(U.getUser()); + bool isDirectCall = (CS.isCall() || CS.isInvoke()) && CS.isCallee(&U); + if (!isDirectCall) + Uses->push_back(&U); + } +} + +/// Replace a list of uses. +static void ReplaceUsesWith(const llvm::SmallVectorImpl &Uses, + llvm::GlobalValue *V, + llvm::GlobalValue *Replacement) { + for (llvm::Use *U : Uses) { + auto *C = dyn_cast(U->getUser()); + if (C && !isa(C)) + C->handleOperandChange(V, Replacement, U); + else + U->set(Replacement); + } +} + +void CodeGenModule::RewriteAlwaysInlineFunction(llvm::Function *Fn) { + std::string Name = Fn->getName(); + std::string InlineName = Name + ".alwaysinline"; + Fn->setName(InlineName); + + llvm::SmallVector NonDirectCallUses; + Fn->removeDeadConstantUsers(); + FindNonDirectCallUses(Fn, &NonDirectCallUses); + // Do not create the wrapper if there are no non-direct call uses, and we are + // not required to emit an external definition. + if (NonDirectCallUses.empty() && Fn->isDiscardableIfUnused()) + return; + + llvm::FunctionType *FT = Fn->getFunctionType(); + llvm::LLVMContext &Ctx = getModule().getContext(); + llvm::Function *StubFn = + llvm::Function::Create(FT, Fn->getLinkage(), Name, &getModule()); + assert(StubFn->getName() == Name && "name was uniqued!"); + + // Insert the stub immediately after the original function. Helps with the + // fragile tests, among other things. + StubFn->removeFromParent(); + TheModule.getFunctionList().insertAfter(Fn, StubFn); + + StubFn->copyAttributesFrom(Fn); + StubFn->setPersonalityFn(nullptr); + + // AvailableExternally functions are replaced with a declaration. + // Everyone else gets a wrapper that musttail-calls the original function. + if (Fn->hasAvailableExternallyLinkage()) { + StubFn->setLinkage(llvm::GlobalValue::ExternalLinkage); + } else { + llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", StubFn); + std::vector Args; + for (llvm::Function::arg_iterator ai = StubFn->arg_begin(); + ai != StubFn->arg_end(); ++ai) + Args.push_back(&*ai); + llvm::CallInst *CI = llvm::CallInst::Create(Fn, Args, "", BB); + CI->setCallingConv(Fn->getCallingConv()); + CI->setTailCallKind(llvm::CallInst::TCK_MustTail); + CI->setAttributes(Fn->getAttributes()); + if (FT->getReturnType()->isVoidTy()) + llvm::ReturnInst::Create(Ctx, BB); + else + llvm::ReturnInst::Create(Ctx, CI, BB); + } + + if (Fn->hasComdat()) + StubFn->setComdat(Fn->getComdat()); + + ReplaceUsesWith(NonDirectCallUses, Fn, StubFn); +} + +void CodeGenModule::RewriteAlwaysInlineFunctions() { + for (llvm::Function *Fn : AlwaysInlineFunctions) { + RewriteAlwaysInlineFunction(Fn); + Fn->setLinkage(llvm::GlobalValue::InternalLinkage); + Fn->addFnAttr(llvm::Attribute::AlwaysInline); + Fn->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); + Fn->setVisibility(llvm::GlobalValue::DefaultVisibility); + } } void CodeGenModule::UpdateCompletedType(const TagDecl *TD) { @@ -772,7 +869,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, !F->getAttributes().hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoInline)) { // (noinline wins over always_inline, and we can't specify both in IR) - B.addAttribute(llvm::Attribute::AlwaysInline); + AddAlwaysInlineFunction(F); } if (D->hasAttr()) { -- cgit v1.1