Reapply 7d77bbef4ad92, adding new debug-info classes

This reverts commit 957efa4ce4f0391147cec62746e997226ee2b836.

Original commit message below -- in this follow up, I've shifted
un-necessary inclusions of DebugProgramInstruction.h into being forward
declarations (fixes clang-compile time I hope), and a memory leak in the
DebugInfoTest.cpp IR unittests.

I also tracked a compile-time regression in D154080, more explanation
there, but the result of which is hiding some of the changes behind the
EXPERIMENTAL_DEBUGINFO_ITERATORS compile-time flag. This is tested by the
"new-debug-iterators" buildbot.

[DebugInfo][RemoveDIs] Add prototype storage classes for "new" debug-info

This patch adds a variety of classes needed to record variable location
debug-info without using the existing intrinsic approach, see the rationale
at [0].

The two added files and corresponding unit tests are the majority of the
plumbing required for this, but at this point isn't accessible from the
rest of LLVM as we need to stage it into the repo gently. An overview is
that classes are added for recording variable information attached to Real
(TM) instructions, in the form of DPValues and DPMarker objects. The
metadata-uses of DPValues is plumbed into the metadata hierachy, and a
field added to class Instruction, which are all stimulated in the unit
tests. The next few patches in this series add utilities to convert to/from
this new debug-info format and add instruction/block utilities to have
debug-info automatically updated in the background when various operations
occur.

This patch was reviewed in Phab in D153990 and D154080, I've squashed them
together into this commit as there are dependencies between the two
patches, and there's little profit in landing them separately.

[0] https://discourse.llvm.org/t/rfc-instruction-api-changes-needed-to-eliminate-debug-intrinsics-from-ir/68939

1 files changed, 309 insertions, 0 deletions

diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 84ed733..034a923 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -11,6 +11,7 @@
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DebugProgramInstruction.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
@@ -23,6 +24,8 @@
 
 using namespace llvm;
 
+extern cl::opt<bool> UseNewDbgInfoFormat;
+
 static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
   SMDiagnostic Err;
   std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
@@ -743,4 +746,310 @@ TEST(AssignmentTrackingTest, InstrMethods) {
   }
 }
 
+// Test some very straight-forward operations on DPValues -- these are
+// dbg.values that have been converted to a non-instruction format.
+TEST(MetadataTest, ConvertDbgToDPValue) {
+  LLVMContext C;
+  std::unique_ptr<Module> M = parseIR(C, R"(
+    define i16 @f(i16 %a) !dbg !6 {
+      call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
+      %b = add i16 %a, 1, !dbg !11
+      call void @llvm.dbg.value(metadata i16 %b, metadata !9, metadata !DIExpression()), !dbg !11
+      ret i16 0, !dbg !11
+
+    exit:
+      %c = add i16 %b, 1, !dbg !11
+      ret i16 0, !dbg !11
+    }
+    declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+    attributes #0 = { nounwind readnone speculatable willreturn }
+
+    !llvm.dbg.cu = !{!0}
+    !llvm.module.flags = !{!5}
+
+    !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+    !1 = !DIFile(filename: "t.ll", directory: "/")
+    !2 = !{}
+    !5 = !{i32 2, !"Debug Info Version", i32 3}
+    !6 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+    !7 = !DISubroutineType(types: !2)
+    !8 = !{!9}
+    !9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+    !10 = !DIBasicType(name: "ty16", size: 16, encoding: DW_ATE_unsigned)
+    !11 = !DILocation(line: 1, column: 1, scope: !6)
+)");
+
+  // Find the first dbg.value,
+  Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHI();
+  const DILocalVariable *Var = nullptr;
+  const DIExpression *Expr = nullptr;
+  const DILocation *Loc = nullptr;
+  const Metadata *MLoc = nullptr;
+  DPValue *DPV1 = nullptr;
+  {
+    DbgValueInst *DPI = dyn_cast<DbgValueInst>(&I);
+    ASSERT_TRUE(DPI);
+    Var = DPI->getVariable();
+    Expr = DPI->getExpression();
+    Loc = DPI->getDebugLoc().get();
+    MLoc = DPI->getRawLocation();
+
+    // Test the creation of a DPValue and it's conversion back to a dbg.value.
+    DPV1 = new DPValue(DPI);
+    EXPECT_EQ(DPV1->getVariable(), Var);
+    EXPECT_EQ(DPV1->getExpression(), Expr);
+    EXPECT_EQ(DPV1->getDebugLoc().get(), Loc);
+    EXPECT_EQ(DPV1->getRawLocation(), MLoc);
+
+    // Erase dbg.value,
+    DPI->eraseFromParent();
+    // Re-create from DPV1, inserting at front.
+    DPV1->createDebugIntrinsic(&*M,
+                               &M->getFunction("f")->getEntryBlock().front());
+
+    Instruction *NewDPI = &M->getFunction("f")->getEntryBlock().front();
+    DbgValueInst *DPI2 = dyn_cast<DbgValueInst>(NewDPI);
+    ASSERT_TRUE(DPI2);
+    EXPECT_EQ(DPI2->getVariable(), Var);
+    EXPECT_EQ(DPI2->getExpression(), Expr);
+    EXPECT_EQ(DPI2->getDebugLoc().get(), Loc);
+    EXPECT_EQ(DPI2->getRawLocation(), MLoc);
+  }
+
+  // Fetch the second dbg.value, convert it to a DPValue,
+  BasicBlock::iterator It = M->getFunction("f")->getEntryBlock().begin();
+  It = std::next(std::next(It));
+  DbgValueInst *DPI3 = dyn_cast<DbgValueInst>(It);
+  ASSERT_TRUE(DPI3);
+  DPValue *DPV2 = new DPValue(DPI3);
+
+  // These dbg.values are supposed to refer to different values.
+  EXPECT_NE(DPV1->getRawLocation(), DPV2->getRawLocation());
+
+  // Try manipulating DPValues and markers in the exit block.
+  BasicBlock *ExitBlock = &*std::next(M->getFunction("f")->getEntryBlock().getIterator());
+  Instruction *FirstInst = &ExitBlock->front();
+  Instruction *RetInst = &*std::next(FirstInst->getIterator());
+
+  // Set-up DPMarkers in this block.
+  ExitBlock->IsNewDbgInfoFormat = true;
+  ExitBlock->createMarker(FirstInst);
+  ExitBlock->createMarker(RetInst);
+
+  // Insert DPValues into markers, order should come out DPV2, DPV1.
+  FirstInst->DbgMarker->insertDPValue(DPV1, false);
+  FirstInst->DbgMarker->insertDPValue(DPV2, true);
+  unsigned int ItCount = 0;
+  for (DPValue &Item : FirstInst->DbgMarker->getDbgValueRange()) {
+    EXPECT_TRUE((&Item == DPV2 && ItCount == 0) ||
+              (&Item == DPV1 && ItCount == 1));
+    EXPECT_EQ(Item.getMarker(), FirstInst->DbgMarker);
+    ++ItCount;
+  }
+
+  // Clone them onto the second marker -- should allocate new DPVs.
+  RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, std::nullopt, false);
+  EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 2u);
+  ItCount = 0;
+  // Check these things store the same information; but that they're not the same
+  // objects.
+  for (DPValue &Item : RetInst->DbgMarker->getDbgValueRange()) {
+    EXPECT_TRUE((Item.getRawLocation() == DPV2->getRawLocation() && ItCount == 0) ||
+                (Item.getRawLocation() == DPV1->getRawLocation() && ItCount == 1));
+    
+    EXPECT_EQ(Item.getMarker(), RetInst->DbgMarker);
+    EXPECT_NE(&Item, DPV1);
+    EXPECT_NE(&Item, DPV2);
+    ++ItCount;
+  }
+
+  RetInst->DbgMarker->dropDPValues();
+  EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 0u);
+
+  // Try cloning one single DPValue.
+  auto DIIt = std::next(FirstInst->DbgMarker->getDbgValueRange().begin());
+  RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, DIIt, false);
+  EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 1u);
+  // The second DPValue should have been cloned; it should have the same values
+  // as DPV1.
+  EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.begin()->getRawLocation(),
+            DPV1->getRawLocation());
+  // We should be able to drop individual DPValues.
+  RetInst->DbgMarker->dropOneDPValue(&*RetInst->DbgMarker->StoredDPValues.begin());
+
+  // "Aborb" a DPMarker: this means pretend that the instruction it's attached
+  // to is disappearing so it needs to be transferred into "this" marker.
+  RetInst->DbgMarker->absorbDebugValues(*FirstInst->DbgMarker, true);
+  EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 2u);
+  // Should be the DPV1 and DPV2 objects.
+  ItCount = 0;
+  for (DPValue &Item : RetInst->DbgMarker->getDbgValueRange()) {
+    EXPECT_TRUE((&Item == DPV2 && ItCount == 0) ||
+              (&Item == DPV1 && ItCount == 1));
+    EXPECT_EQ(Item.getMarker(), RetInst->DbgMarker);
+    ++ItCount;
+  }
+
+  // Finally -- there are two DPValues left over. If we remove evrything in the
+  // basic block, then they should sink down into the "TrailingDPValues"
+  // container for dangling debug-info. Future facilities will restore them
+  // back when a terminator is inserted.
+  FirstInst->DbgMarker->removeMarker();
+  FirstInst->eraseFromParent();
+  RetInst->DbgMarker->removeMarker();
+  RetInst->eraseFromParent();
+
+  DPMarker *EndMarker = ExitBlock->getTrailingDPValues();
+  ASSERT_NE(EndMarker, nullptr);
+  EXPECT_EQ(EndMarker->StoredDPValues.size(), 2u);
+  // Test again that it's those two DPValues, DPV1 and DPV2.
+  ItCount = 0;
+  for (DPValue &Item : EndMarker->getDbgValueRange()) {
+    EXPECT_TRUE((&Item == DPV2 && ItCount == 0) ||
+              (&Item == DPV1 && ItCount == 1));
+    EXPECT_EQ(Item.getMarker(), EndMarker);
+    ++ItCount;
+  }
+
+  // Cleanup the trailing DPValue records and marker.
+  EndMarker->eraseFromParent();
+
+  // The record of those trailing DPValues would dangle and cause an assertion
+  // failure if it lived until the end of the LLVMContext.
+  ExitBlock->deleteTrailingDPValues();
+}
+
+TEST(MetadataTest, DPValueConversionRoutines) {
+  LLVMContext C;
+
+  // For the purpose of this test, set and un-set the command line option
+  // corresponding to UseNewDbgInfoFormat.
+  UseNewDbgInfoFormat = true;
+
+  std::unique_ptr<Module> M = parseIR(C, R"(
+    define i16 @f(i16 %a) !dbg !6 {
+      call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
+      %b = add i16 %a, 1, !dbg !11
+      call void @llvm.dbg.value(metadata i16 %b, metadata !9, metadata !DIExpression()), !dbg !11
+      ret i16 0, !dbg !11
+
+    exit:
+      %c = add i16 %b, 1, !dbg !11
+      ret i16 0, !dbg !11
+    }
+    declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+    attributes #0 = { nounwind readnone speculatable willreturn }
+
+    !llvm.dbg.cu = !{!0}
+    !llvm.module.flags = !{!5}
+
+    !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+    !1 = !DIFile(filename: "t.ll", directory: "/")
+    !2 = !{}
+    !5 = !{i32 2, !"Debug Info Version", i32 3}
+    !6 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+    !7 = !DISubroutineType(types: !2)
+    !8 = !{!9}
+    !9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+    !10 = !DIBasicType(name: "ty16", size: 16, encoding: DW_ATE_unsigned)
+    !11 = !DILocation(line: 1, column: 1, scope: !6)
+)");
+
+  // Check that the conversion routines and utilities between dbg.value
+  // debug-info format and DPValues works.
+  Function *F = M->getFunction("f");
+  BasicBlock *BB1 = &F->getEntryBlock();
+  // First instruction should be a dbg.value.
+  EXPECT_TRUE(isa<DbgValueInst>(BB1->front()));
+  EXPECT_FALSE(BB1->IsNewDbgInfoFormat);
+  // Validating the block for DPValues / DPMarkers shouldn't fail -- there's
+  // no data stored right now.
+  EXPECT_FALSE(BB1->validateDbgValues(false, false));
+
+  // Function and module should be marked as not having the new format too.
+  EXPECT_FALSE(F->IsNewDbgInfoFormat);
+  EXPECT_FALSE(M->IsNewDbgInfoFormat);
+
+  // Now convert.
+  M->convertToNewDbgValues();
+  EXPECT_TRUE(M->IsNewDbgInfoFormat);
+  EXPECT_TRUE(F->IsNewDbgInfoFormat);
+  EXPECT_TRUE(BB1->IsNewDbgInfoFormat);
+
+  // There should now be no dbg.value instructions!
+  // Ensure the first instruction exists, the test all of them.
+  EXPECT_FALSE(isa<DbgValueInst>(BB1->front()));
+  for (auto &BB : *F)
+    for (auto &I : BB)
+      EXPECT_FALSE(isa<DbgValueInst>(I));
+
+  // There should be a DPMarker on each of the two instructions in the entry
+  // block, each containing one DPValue.
+  EXPECT_EQ(BB1->size(), 2u);
+  Instruction *FirstInst = &BB1->front();
+  Instruction *SecondInst = FirstInst->getNextNode();
+  ASSERT_TRUE(FirstInst->DbgMarker);
+  ASSERT_TRUE(SecondInst->DbgMarker);
+  EXPECT_NE(FirstInst->DbgMarker, SecondInst->DbgMarker);
+  EXPECT_EQ(FirstInst, FirstInst->DbgMarker->MarkedInstr);
+  EXPECT_EQ(SecondInst, SecondInst->DbgMarker->MarkedInstr);
+
+  EXPECT_EQ(FirstInst->DbgMarker->StoredDPValues.size(), 1u);
+  DPValue *DPV1 = &*FirstInst->DbgMarker->getDbgValueRange().begin();
+  EXPECT_EQ(DPV1->getMarker(), FirstInst->DbgMarker);
+  // Should point at %a, an argument.
+  EXPECT_TRUE(isa<Argument>(DPV1->getVariableLocationOp(0)));
+
+  EXPECT_EQ(SecondInst->DbgMarker->StoredDPValues.size(), 1u);
+  DPValue *DPV2 = &*SecondInst->DbgMarker->getDbgValueRange().begin();
+  EXPECT_EQ(DPV2->getMarker(), SecondInst->DbgMarker);
+  // Should point at FirstInst.
+  EXPECT_EQ(DPV2->getVariableLocationOp(0), FirstInst);
+
+  // There should be no DPValues / DPMarkers in the second block, but it should
+  // be marked as being in the new format.
+  BasicBlock *BB2 = BB1->getNextNode();
+  EXPECT_TRUE(BB2->IsNewDbgInfoFormat);
+  for (auto &Inst : *BB2)
+    // Either there should be no marker, or it should be empty.
+    EXPECT_TRUE(!Inst.DbgMarker || Inst.DbgMarker->StoredDPValues.empty());
+
+  // Validating the first block should continue to not be a problem,
+  EXPECT_FALSE(BB1->validateDbgValues(false, false));
+  // But if we were to break something, it should be able to fire. Don't attempt
+  // to comprehensively test the validator, it's a smoke-test rather than a
+  // "proper" verification pass.
+  DPV1->setMarker(nullptr);
+  // A marker pointing the wrong way should be an error.
+  EXPECT_TRUE(BB1->validateDbgValues(false, false));
+  DPV1->setMarker(FirstInst->DbgMarker);
+
+  DILocalVariable *DLV1 = DPV1->getVariable();
+  DIExpression *Expr1 = DPV1->getExpression();
+  DILocalVariable *DLV2 = DPV2->getVariable();
+  DIExpression *Expr2 = DPV2->getExpression();
+
+  // Convert everything back to the "old" format and ensure it's right.
+  M->convertFromNewDbgValues();
+  EXPECT_FALSE(M->IsNewDbgInfoFormat);
+  EXPECT_FALSE(F->IsNewDbgInfoFormat);
+  EXPECT_FALSE(BB1->IsNewDbgInfoFormat);
+
+  EXPECT_EQ(BB1->size(), 4u);
+  ASSERT_TRUE(isa<DbgValueInst>(BB1->front()));
+  DbgValueInst *DVI1 = cast<DbgValueInst>(&BB1->front());
+  // These dbg.values should still point at the same places.
+  EXPECT_TRUE(isa<Argument>(DVI1->getVariableLocationOp(0)));
+  DbgValueInst *DVI2 = cast<DbgValueInst>(DVI1->getNextNode()->getNextNode());
+  EXPECT_EQ(DVI2->getVariableLocationOp(0), FirstInst);
+
+  // Check a few fields too,
+  EXPECT_EQ(DVI1->getVariable(), DLV1);
+  EXPECT_EQ(DVI1->getExpression(), Expr1);
+  EXPECT_EQ(DVI2->getVariable(), DLV2);
+  EXPECT_EQ(DVI2->getExpression(), Expr2);
+
+  UseNewDbgInfoFormat = false;
+}
+
 } // end namespace