From fc89001cec9833b0058d57d17045b61aeb958b38 Mon Sep 17 00:00:00 2001 From: Kirill Bobyrev Date: Fri, 24 Aug 2018 09:12:54 +0000 Subject: [PATCH] [clangd] Log memory usage of DexIndex and MemIndex This patch prints information about built index size estimation to verbose logs. This is useful for optimizing memory usage of DexIndex and comparisons with MemIndex. Reviewed by: sammccall Differential Revision: https://reviews.llvm.org/D51154 llvm-svn: 340601 --- clang-tools-extra/clangd/index/FileIndex.cpp | 4 ++++ clang-tools-extra/clangd/index/FileIndex.h | 3 +++ clang-tools-extra/clangd/index/Index.h | 6 ++++++ clang-tools-extra/clangd/index/MemIndex.cpp | 8 ++++++++ clang-tools-extra/clangd/index/MemIndex.h | 3 +++ clang-tools-extra/clangd/index/Merge.cpp | 4 ++++ clang-tools-extra/clangd/index/dex/DexIndex.cpp | 17 +++++++++++++++++ clang-tools-extra/clangd/index/dex/DexIndex.h | 3 +++ .../unittests/clangd/CodeCompleteTests.cpp | 4 ++++ 9 files changed, 52 insertions(+) diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp index 3d8e17bfb2f1..48d05959dae2 100644 --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -118,5 +118,9 @@ void FileIndex::findOccurrences( log("findOccurrences is not implemented."); } +size_t FileIndex::estimateMemoryUsage() const { + return Index.estimateMemoryUsage(); +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h index 6f24f9d2c23e..58fad2f62e2f 100644 --- a/clang-tools-extra/clangd/index/FileIndex.h +++ b/clang-tools-extra/clangd/index/FileIndex.h @@ -81,6 +81,9 @@ public: void findOccurrences(const OccurrencesRequest &Req, llvm::function_ref Callback) const override; + + size_t estimateMemoryUsage() const override; + private: FileSymbols FSymbols; MemIndex Index; diff --git a/clang-tools-extra/clangd/index/Index.h b/clang-tools-extra/clangd/index/Index.h index eac4c5e7d856..95b081acec77 100644 --- a/clang-tools-extra/clangd/index/Index.h +++ b/clang-tools-extra/clangd/index/Index.h @@ -385,6 +385,12 @@ public: virtual void findOccurrences( const OccurrencesRequest &Req, llvm::function_ref Callback) const = 0; + + /// Returns estimated size of index (in bytes). + // FIXME(kbobyrev): Currently, this only returns the size of index itself + // excluding the size of actual symbol slab index refers to. We should include + // both. + virtual size_t estimateMemoryUsage() const = 0; }; } // namespace clangd diff --git a/clang-tools-extra/clangd/index/MemIndex.cpp b/clang-tools-extra/clangd/index/MemIndex.cpp index db11244d8acb..19a64ad95259 100644 --- a/clang-tools-extra/clangd/index/MemIndex.cpp +++ b/clang-tools-extra/clangd/index/MemIndex.cpp @@ -26,6 +26,9 @@ void MemIndex::build(std::shared_ptr> Syms) { Index = std::move(TempIndex); Symbols = std::move(Syms); // Relase old symbols. } + + vlog("Built MemIndex with estimated memory usage {0} bytes.", + estimateMemoryUsage()); } std::unique_ptr MemIndex::build(SymbolSlab Slab) { @@ -98,5 +101,10 @@ getSymbolsFromSlab(SymbolSlab Slab) { &Snap->Pointers); } +size_t MemIndex::estimateMemoryUsage() const { + std::lock_guard Lock(Mutex); + return Index.getMemorySize(); +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/MemIndex.h b/clang-tools-extra/clangd/index/MemIndex.h index 8b12f8c7921e..feca87af5e43 100644 --- a/clang-tools-extra/clangd/index/MemIndex.h +++ b/clang-tools-extra/clangd/index/MemIndex.h @@ -39,7 +39,10 @@ public: llvm::function_ref Callback) const override; + size_t estimateMemoryUsage() const override; + private: + std::shared_ptr> Symbols; // Index is a set of symbols that are deduplicated by symbol IDs. // FIXME: build smarter index structure. diff --git a/clang-tools-extra/clangd/index/Merge.cpp b/clang-tools-extra/clangd/index/Merge.cpp index 4834bfba69ff..84928ca1a5e5 100644 --- a/clang-tools-extra/clangd/index/Merge.cpp +++ b/clang-tools-extra/clangd/index/Merge.cpp @@ -84,6 +84,10 @@ class MergedIndex : public SymbolIndex { log("findOccurrences is not implemented."); } + size_t estimateMemoryUsage() const override { + return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage(); + } + private: const SymbolIndex *Dynamic, *Static; }; diff --git a/clang-tools-extra/clangd/index/dex/DexIndex.cpp b/clang-tools-extra/clangd/index/dex/DexIndex.cpp index 534f51e0c3de..9280cc8fdfc8 100644 --- a/clang-tools-extra/clangd/index/dex/DexIndex.cpp +++ b/clang-tools-extra/clangd/index/dex/DexIndex.cpp @@ -67,6 +67,9 @@ void DexIndex::build(std::shared_ptr> Syms) { InvertedIndex = std::move(TempInvertedIndex); SymbolQuality = std::move(TempSymbolQuality); } + + vlog("Built DexIndex with estimated memory usage {0} bytes.", + estimateMemoryUsage()); } std::unique_ptr DexIndex::build(SymbolSlab Slab) { @@ -171,6 +174,20 @@ void DexIndex::findOccurrences( log("findOccurrences is not implemented."); } +size_t DexIndex::estimateMemoryUsage() const { + std::lock_guard Lock(Mutex); + + size_t Bytes = + LookupTable.size() * sizeof(std::pair); + Bytes += SymbolQuality.size() * sizeof(std::pair); + Bytes += InvertedIndex.size() * sizeof(Token); + + for (const auto &P : InvertedIndex) { + Bytes += P.second.size() * sizeof(DocID); + } + return Bytes; +} + } // namespace dex } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/dex/DexIndex.h b/clang-tools-extra/clangd/index/dex/DexIndex.h index 4485150b1328..d9b2cd143898 100644 --- a/clang-tools-extra/clangd/index/dex/DexIndex.h +++ b/clang-tools-extra/clangd/index/dex/DexIndex.h @@ -57,7 +57,10 @@ public: llvm::function_ref Callback) const override; + size_t estimateMemoryUsage() const override; + private: + mutable std::mutex Mutex; std::shared_ptr> Symbols /*GUARDED_BY(Mutex)*/; diff --git a/clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp b/clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp index 8f7f85377b69..3f855844aed9 100644 --- a/clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp +++ b/clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp @@ -923,6 +923,10 @@ public: llvm::function_ref Callback) const override {} + // This is incorrect, but IndexRequestCollector is not an actual index and it + // isn't used in production code. + size_t estimateMemoryUsage() const override { return 0; } + const std::vector allRequests() const { return Requests; } private: -- GitLab