//===----------------------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "CASTestConfig.h" #include "OnDiskCommonUtils.h" #include "llvm/Testing/Support/Error.h" #include "llvm/Testing/Support/SupportHelpers.h" #include "gtest/gtest.h" using namespace llvm; using namespace llvm::cas; using namespace llvm::cas::ondisk; using namespace llvm::unittest::cas; TEST_F(OnDiskCASTest, OnDiskGraphDBTest) { unittest::TempDir Temp("ondiskcas", /*Unique=*/true); std::unique_ptr DB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB), Succeeded()); auto digest = [&DB](StringRef Data, ArrayRef Refs) -> ObjectID { return ::digest(*DB, Data, Refs); }; auto store = [&](StringRef Data, ArrayRef Refs) -> Expected { return ::store(*DB, Data, Refs); }; std::optional ID1; ASSERT_THAT_ERROR(store("hello", {}).moveInto(ID1), Succeeded()); std::optional Obj1; ASSERT_THAT_ERROR(DB->load(*ID1).moveInto(Obj1), Succeeded()); ASSERT_TRUE(Obj1.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj1)), "hello"); ArrayRef Digest1 = DB->getDigest(*ID1); std::optional ID2; ASSERT_THAT_ERROR(DB->getReference(Digest1).moveInto(ID2), Succeeded()); EXPECT_EQ(ID1, ID2); ObjectID ID3 = digest("world", {}); EXPECT_FALSE(DB->containsObject(ID3)); std::optional Obj2; ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj2), Succeeded()); EXPECT_FALSE(Obj2.has_value()); ASSERT_THAT_ERROR(DB->store(ID3, {}, arrayRefFromStringRef("world")), Succeeded()); EXPECT_TRUE(DB->containsObject(ID3)); ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj2), Succeeded()); ASSERT_TRUE(Obj2.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj2)), "world"); size_t LargeDataSize = 256LL * 1024LL; // 256K. // The precise size number is not important, we mainly check that the large // object will be properly accounted for. EXPECT_TRUE(DB->getStorageSize() > 10 && DB->getStorageSize() < LargeDataSize); SmallString<16> Buffer; Buffer.resize(LargeDataSize); ASSERT_THAT_ERROR(store(Buffer, {}).moveInto(ID1), Succeeded()); size_t StorageSize = DB->getStorageSize(); EXPECT_TRUE(StorageSize > LargeDataSize); // Close & re-open the DB and check that it reports the same storage size. DB.reset(); ASSERT_THAT_ERROR( OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB), Succeeded()); EXPECT_EQ(DB->getStorageSize(), StorageSize); } TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInSingleNode) { unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true); std::unique_ptr UpstreamDB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType)) .moveInto(UpstreamDB), Succeeded()); { std::optional ID1; ASSERT_THAT_ERROR(store(*UpstreamDB, "hello", {}).moveInto(ID1), Succeeded()); std::optional ID2; ASSERT_THAT_ERROR(store(*UpstreamDB, "another", {}).moveInto(ID2), Succeeded()); std::optional ID3; ASSERT_THAT_ERROR(store(*UpstreamDB, "world", {*ID1, *ID2}).moveInto(ID3), Succeeded()); } unittest::TempDir Temp("ondiskcas", /*Unique=*/true); std::unique_ptr DB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), std::move(UpstreamDB), OnDiskGraphDB::FaultInPolicy::SingleNode) .moveInto(DB), Succeeded()); ObjectID ID1 = digest(*DB, "hello", {}); ObjectID ID2 = digest(*DB, "another", {}); ObjectID ID3 = digest(*DB, "world", {ID1, ID2}); ObjectID ID4 = digest(*DB, "world", {}); EXPECT_TRUE(DB->containsObject(ID1)); EXPECT_TRUE(DB->containsObject(ID2)); EXPECT_TRUE(DB->containsObject(ID3)); EXPECT_FALSE(DB->containsObject(ID4)); EXPECT_TRUE(DB->getExistingReference(digest("hello", {})).has_value()); EXPECT_TRUE(DB->getExistingReference(DB->getDigest(ID3)).has_value()); EXPECT_FALSE(DB->getExistingReference(digest("world", {})).has_value()); { std::optional Obj; ASSERT_THAT_ERROR(DB->load(ID1).moveInto(Obj), Succeeded()); ASSERT_TRUE(Obj.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "hello"); auto Refs = DB->getObjectRefs(*Obj); EXPECT_TRUE(Refs.empty()); } { std::optional Obj; ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj), Succeeded()); ASSERT_TRUE(Obj.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "world"); auto Refs = DB->getObjectRefs(*Obj); ASSERT_EQ(std::distance(Refs.begin(), Refs.end()), 2); EXPECT_EQ(Refs.begin()[0], ID1); EXPECT_EQ(Refs.begin()[1], ID2); } { std::optional Obj; ASSERT_THAT_ERROR(DB->load(ID4).moveInto(Obj), Succeeded()); EXPECT_FALSE(Obj.has_value()); } // Re-open the primary without chaining, to verify the data were copied from // the upstream. ASSERT_THAT_ERROR( OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), /*UpstreamDB=*/nullptr, OnDiskGraphDB::FaultInPolicy::SingleNode) .moveInto(DB), Succeeded()); ID1 = digest(*DB, "hello", {}); ID2 = digest(*DB, "another", {}); ID3 = digest(*DB, "world", {ID1, ID2}); EXPECT_TRUE(DB->containsObject(ID1)); EXPECT_FALSE(DB->containsObject(ID2)); EXPECT_TRUE(DB->containsObject(ID3)); { std::optional Obj; ASSERT_THAT_ERROR(DB->load(ID1).moveInto(Obj), Succeeded()); ASSERT_TRUE(Obj.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "hello"); auto Refs = DB->getObjectRefs(*Obj); EXPECT_TRUE(Refs.empty()); } } TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInFullTree) { unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true); std::unique_ptr UpstreamDB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType)) .moveInto(UpstreamDB), Succeeded()); HashType RootHash; { std::optional ID11; ASSERT_THAT_ERROR(store(*UpstreamDB, "11", {}).moveInto(ID11), Succeeded()); std::optional ID121; ASSERT_THAT_ERROR(store(*UpstreamDB, "121", {}).moveInto(ID121), Succeeded()); std::optional ID12; ASSERT_THAT_ERROR(store(*UpstreamDB, "12", {*ID121}).moveInto(ID12), Succeeded()); std::optional ID1; ASSERT_THAT_ERROR(store(*UpstreamDB, "1", {*ID11, *ID12}).moveInto(ID1), Succeeded()); std::optional ID21; ASSERT_THAT_ERROR(store(*UpstreamDB, "21", {}).moveInto(ID21), Succeeded()); std::optional ID22; ASSERT_THAT_ERROR(store(*UpstreamDB, "22", {}).moveInto(ID22), Succeeded()); std::optional ID2; ASSERT_THAT_ERROR( store(*UpstreamDB, "2", {*ID12, *ID21, *ID22}).moveInto(ID2), Succeeded()); std::optional IDRoot; ASSERT_THAT_ERROR(store(*UpstreamDB, "root", {*ID1, *ID2}).moveInto(IDRoot), Succeeded()); ArrayRef Digest = UpstreamDB->getDigest(*IDRoot); ASSERT_EQ(Digest.size(), RootHash.size()); llvm::copy(Digest, RootHash.data()); } unittest::TempDir Temp("ondiskcas", /*Unique=*/true); std::unique_ptr DB; ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), std::move(UpstreamDB), OnDiskGraphDB::FaultInPolicy::FullTree) .moveInto(DB), Succeeded()); { std::optional IDRoot; ASSERT_THAT_ERROR(DB->getReference(RootHash).moveInto(IDRoot), Succeeded()); std::optional Obj; ASSERT_THAT_ERROR(DB->load(*IDRoot).moveInto(Obj), Succeeded()); ASSERT_TRUE(Obj.has_value()); EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "root"); auto Refs = DB->getObjectRefs(*Obj); ASSERT_EQ(std::distance(Refs.begin(), Refs.end()), 2); } // Re-open the primary without chaining, to verify the data were copied from // the upstream. ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), /*UpstreamDB=*/nullptr, OnDiskGraphDB::FaultInPolicy::FullTree) .moveInto(DB), Succeeded()); std::optional IDRoot; ASSERT_THAT_ERROR(DB->getReference(RootHash).moveInto(IDRoot), Succeeded()); std::string PrintedTree; raw_string_ostream OS(PrintedTree); ASSERT_THAT_ERROR(printTree(*DB, *IDRoot, OS), Succeeded()); StringRef Expected = R"(root 1 11 12 121 2 12 121 21 22 )"; EXPECT_EQ(PrintedTree, Expected); } TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInPolicyConflict) { auto tryFaultInPolicyConflict = [](OnDiskGraphDB::FaultInPolicy Policy1, OnDiskGraphDB::FaultInPolicy Policy2) { unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true); std::unique_ptr UpstreamDB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType)) .moveInto(UpstreamDB), Succeeded()); unittest::TempDir Temp("ondiskcas", /*Unique=*/true); std::unique_ptr DB; ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), std::move(UpstreamDB), Policy1) .moveInto(DB), Succeeded()); DB.reset(); ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType), std::move(UpstreamDB), Policy2) .moveInto(DB), Failed()); }; // Open as 'single', then as 'full'. tryFaultInPolicyConflict(OnDiskGraphDB::FaultInPolicy::SingleNode, OnDiskGraphDB::FaultInPolicy::FullTree); // Open as 'full', then as 'single'. tryFaultInPolicyConflict(OnDiskGraphDB::FaultInPolicy::FullTree, OnDiskGraphDB::FaultInPolicy::SingleNode); } #if defined(EXPENSIVE_CHECKS) TEST_F(OnDiskCASTest, OnDiskGraphDBSpaceLimit) { setMaxOnDiskCASMappingSize(); unittest::TempDir Temp("ondiskcas", /*Unique=*/true); std::unique_ptr DB; ASSERT_THAT_ERROR( OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB), Succeeded()); std::optional ID; std::string Data(500, '0'); auto storeSmallObject = [&]() { SmallVector Refs; if (ID) Refs.push_back(*ID); ASSERT_THAT_ERROR(store(*DB, Data, Refs).moveInto(ID), Succeeded()); }; // Insert enough small elements to overflow the data pool. for (unsigned I = 0; I < 1024 * 256; ++I) storeSmallObject(); EXPECT_GE(DB->getHardStorageLimitUtilization(), 99U); } #endif