diff options
Diffstat (limited to 'llvm/unittests/CAS/UnifiedOnDiskCacheTest.cpp')
| -rw-r--r-- | llvm/unittests/CAS/UnifiedOnDiskCacheTest.cpp | 198 |
1 files changed, 198 insertions, 0 deletions
diff --git a/llvm/unittests/CAS/UnifiedOnDiskCacheTest.cpp b/llvm/unittests/CAS/UnifiedOnDiskCacheTest.cpp new file mode 100644 index 0000000..09aebc2 --- /dev/null +++ b/llvm/unittests/CAS/UnifiedOnDiskCacheTest.cpp @@ -0,0 +1,198 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CAS/UnifiedOnDiskCache.h" +#include "CASTestConfig.h" +#include "OnDiskCommonUtils.h" +#include "llvm/Testing/Support/Error.h" +#include "llvm/Testing/Support/SupportHelpers.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::cas; +using namespace llvm::cas::ondisk; +using namespace llvm::unittest::cas; + +/// Visits all the files of a directory recursively and returns the sum of their +/// sizes. +static Expected<size_t> countFileSizes(StringRef Path) { + size_t TotalSize = 0; + std::error_code EC; + for (sys::fs::directory_iterator DirI(Path, EC), DirE; !EC && DirI != DirE; + DirI.increment(EC)) { + if (DirI->type() == sys::fs::file_type::directory_file) { + Expected<size_t> Subsize = countFileSizes(DirI->path()); + if (!Subsize) + return Subsize.takeError(); + TotalSize += *Subsize; + continue; + } + ErrorOr<sys::fs::basic_file_status> Stat = DirI->status(); + if (!Stat) + return createFileError(DirI->path(), Stat.getError()); + TotalSize += Stat->getSize(); + } + if (EC) + return createFileError(Path, EC); + return TotalSize; +} + +TEST_F(OnDiskCASTest, UnifiedOnDiskCacheTest) { + unittest::TempDir Temp("ondisk-unified", /*Unique=*/true); + std::unique_ptr<UnifiedOnDiskCache> UniDB; + + const uint64_t SizeLimit = 1024ull * 64; + auto reopenDB = [&]() { + UniDB.reset(); + ASSERT_THAT_ERROR(UnifiedOnDiskCache::open(Temp.path(), SizeLimit, "blake3", + sizeof(HashType)) + .moveInto(UniDB), + Succeeded()); + }; + + reopenDB(); + + HashType RootHash; + HashType OtherHash; + HashType Key1Hash; + HashType Key2Hash; + { + OnDiskGraphDB &DB = UniDB->getGraphDB(); + std::optional<ObjectID> ID1; + ASSERT_THAT_ERROR(store(DB, "1", {}).moveInto(ID1), Succeeded()); + std::optional<ObjectID> ID2; + ASSERT_THAT_ERROR(store(DB, "2", {}).moveInto(ID2), Succeeded()); + std::optional<ObjectID> IDRoot; + ASSERT_THAT_ERROR(store(DB, "root", {*ID1, *ID2}).moveInto(IDRoot), + Succeeded()); + ArrayRef<uint8_t> Digest = DB.getDigest(*IDRoot); + ASSERT_EQ(Digest.size(), RootHash.size()); + llvm::copy(Digest, RootHash.data()); + + std::optional<ObjectID> IDOther; + ASSERT_THAT_ERROR(store(DB, "other", {}).moveInto(IDOther), Succeeded()); + Digest = DB.getDigest(*IDOther); + ASSERT_EQ(Digest.size(), OtherHash.size()); + llvm::copy(Digest, OtherHash.data()); + + Key1Hash = digest("key1"); + std::optional<ObjectID> Val; + ASSERT_THAT_ERROR( + cachePut(UniDB->getKeyValueDB(), Key1Hash, *IDRoot).moveInto(Val), + Succeeded()); + EXPECT_EQ(IDRoot, Val); + + Key2Hash = digest("key2"); + std::optional<ObjectID> KeyID; + ASSERT_THAT_ERROR(DB.getReference(Key2Hash).moveInto(KeyID), Succeeded()); + ASSERT_THAT_ERROR(cachePut(UniDB->getKeyValueDB(), + UniDB->getGraphDB().getDigest(*KeyID), *ID1) + .moveInto(Val), + Succeeded()); + } + + auto checkTree = [&](const HashType &Digest, StringRef ExpectedTree) { + OnDiskGraphDB &DB = UniDB->getGraphDB(); + std::optional<ObjectID> ID; + ASSERT_THAT_ERROR(DB.getReference(Digest).moveInto(ID), Succeeded()); + std::string PrintedTree; + raw_string_ostream OS(PrintedTree); + ASSERT_THAT_ERROR(printTree(DB, *ID, OS), Succeeded()); + EXPECT_EQ(PrintedTree, ExpectedTree); + }; + auto checkRootTree = [&]() { + return checkTree(RootHash, "root\n 1\n 2\n"); + }; + + auto checkKey = [&](const HashType &Key, StringRef ExpectedData) { + OnDiskGraphDB &DB = UniDB->getGraphDB(); + std::optional<ObjectID> Val; + ASSERT_THAT_ERROR(cacheGet(UniDB->getKeyValueDB(), Key).moveInto(Val), + Succeeded()); + + ASSERT_TRUE(Val.has_value()); + std::optional<ondisk::ObjectHandle> Obj; + ASSERT_THAT_ERROR(DB.load(*Val).moveInto(Obj), Succeeded()); + EXPECT_EQ(toStringRef(DB.getObjectData(*Obj)), ExpectedData); + }; + + checkRootTree(); + checkTree(OtherHash, "other\n"); + checkKey(Key1Hash, "root"); + checkKey(Key2Hash, "1"); + + auto storeBigObject = [&](unsigned Index) { + SmallString<1000> Buf; + Buf.append(970, 'a'); + raw_svector_ostream(Buf) << Index; + std::optional<ObjectID> ID; + ASSERT_THAT_ERROR(store(UniDB->getGraphDB(), Buf, {}).moveInto(ID), + Succeeded()); + }; + + uint64_t PrevStoreSize = UniDB->getStorageSize(); + unsigned Index = 0; + while (!UniDB->hasExceededSizeLimit()) { + storeBigObject(Index++); + } + EXPECT_GT(UniDB->getStorageSize(), PrevStoreSize); + UniDB->setSizeLimit(SizeLimit * 2); + EXPECT_FALSE(UniDB->hasExceededSizeLimit()); + UniDB->setSizeLimit(SizeLimit); + EXPECT_TRUE(UniDB->hasExceededSizeLimit()); + + reopenDB(); + + EXPECT_FALSE(UniDB->hasExceededSizeLimit()); + EXPECT_FALSE(UniDB->needsGarbageCollection()); + + checkRootTree(); + checkKey(Key1Hash, "root"); + + while (!UniDB->hasExceededSizeLimit()) { + storeBigObject(Index++); + } + PrevStoreSize = UniDB->getStorageSize(); + ASSERT_THAT_ERROR(UniDB->close(), Succeeded()); + EXPECT_TRUE(UniDB->needsGarbageCollection()); + + reopenDB(); + EXPECT_TRUE(UniDB->needsGarbageCollection()); + + std::optional<size_t> DirSizeBefore; + ASSERT_THAT_ERROR(countFileSizes(Temp.path()).moveInto(DirSizeBefore), + Succeeded()); + + ASSERT_THAT_ERROR(UnifiedOnDiskCache::collectGarbage(Temp.path()), + Succeeded()); + + std::optional<size_t> DirSizeAfter; + ASSERT_THAT_ERROR(countFileSizes(Temp.path()).moveInto(DirSizeAfter), + Succeeded()); + EXPECT_LT(*DirSizeAfter, *DirSizeBefore); + + reopenDB(); + EXPECT_FALSE(UniDB->needsGarbageCollection()); + + checkRootTree(); + checkKey(Key1Hash, "root"); + + EXPECT_LT(UniDB->getStorageSize(), PrevStoreSize); + + // 'Other' tree and 'Key2' got garbage-collected. + { + OnDiskGraphDB &DB = UniDB->getGraphDB(); + std::optional<ObjectID> ID; + ASSERT_THAT_ERROR(DB.getReference(OtherHash).moveInto(ID), Succeeded()); + EXPECT_FALSE(DB.containsObject(*ID)); + std::optional<ObjectID> Val; + ASSERT_THAT_ERROR(cacheGet(UniDB->getKeyValueDB(), Key2Hash).moveInto(Val), + Succeeded()); + EXPECT_FALSE(Val.has_value()); + } +} |