feat: supplement RocksDB Metrics.

include/pika_admin.h

@@ -203,6 +203,7 @@ class InfoCmd : public Cmd {
     kInfoKeyspace,
     kInfoLog,
     kInfoData,
+    kInfoRocksDB,
     kInfo,
     kInfoAll,
     kInfoDebug
@@ -230,6 +231,7 @@ class InfoCmd : public Cmd {
   const static std::string kReplicationSection;
   const static std::string kKeyspaceSection;
   const static std::string kDataSection;
+  const static std::string kRocksDBSection;
   const static std::string kDebugSection;
 
   void DoInitial() override;
@@ -248,6 +250,7 @@ class InfoCmd : public Cmd {
   void InfoReplication(std::string& info);
   void InfoKeyspace(std::string& info);
   void InfoData(std::string& info);
+  void InfoRocksDB(std::string& info);
   void InfoDebug(std::string& info);
 };
 

src/pika_admin.cc

@@ -632,6 +632,7 @@ const std::string InfoCmd::kCPUSection = "cpu";
 const std::string InfoCmd::kReplicationSection = "replication";
 const std::string InfoCmd::kKeyspaceSection = "keyspace";
 const std::string InfoCmd::kDataSection = "data";
+const std::string InfoCmd::kRocksDBSection = "rocksdb";
 const std::string InfoCmd::kDebugSection = "debug";
 
 void InfoCmd::DoInitial() {
@@ -697,6 +698,8 @@ void InfoCmd::DoInitial() {
     return;
   } else if (strcasecmp(argv_[1].data(), kDataSection.data()) == 0) {
     info_section_ = kInfoData;
+  } else if (strcasecmp(argv_[1].data(), kRocksDBSection.data()) == 0) {
+    info_section_ = kInfoRocksDB;
   } else if (strcasecmp(argv_[1].data(), kDebugSection.data()) == 0) {
     info_section_ = kInfoDebug;
   } else {
@@ -724,6 +727,8 @@ void InfoCmd::Do(std::shared_ptr<Partition> partition) {
       InfoReplication(info);
       info.append("\r\n");
       InfoKeyspace(info);
+      info.append("\r\n");
+      InfoRocksDB(info);
       break;
     case kInfoAll:
       InfoServer(info);
@@ -741,6 +746,8 @@ void InfoCmd::Do(std::shared_ptr<Partition> partition) {
       InfoReplication(info);
       info.append("\r\n");
       InfoKeyspace(info);
+      info.append("\r\n");
+      InfoRocksDB(info);
       break;
     case kInfoServer:
       InfoServer(info);
@@ -766,6 +773,9 @@ void InfoCmd::Do(std::shared_ptr<Partition> partition) {
     case kInfoData:
       InfoData(info);
       break;
+    case kInfoRocksDB:
+      InfoRocksDB(info);
+      break;
     case kInfoDebug:
       InfoDebug(info);
       break;
@@ -1053,7 +1063,7 @@ void InfoCmd::InfoKeyspace(std::string& info) {
     tmp_stream << "# Start async statistics"
                << "\r\n";
   } else {  // command => `info keyspace` or `info`
-    tmp_stream << "# Use `info keyspace 1` do async statistics"
+    tmp_stream << "# Use \"info keyspace 1\" do async statistics"
                << "\r\n";
   }
 
@@ -1114,29 +1124,32 @@ void InfoCmd::InfoData(std::string& info) {
   tmp_stream << "compression:" << g_pika_conf->compression() << "\r\n";
 
   // rocksdb related memory usage
-  std::map<std::string, uint64_t> type_result;
+  std::map<std::string, uint64_t> background_errors;
   uint64_t total_background_errors = 0;
   uint64_t total_memtable_usage = 0;
   uint64_t memtable_usage = 0;
   uint64_t total_table_reader_usage = 0;
   uint64_t table_reader_usage = 0;
   std::shared_lock table_rwl(g_pika_server->tables_rw_);
   for (const auto& table_item : g_pika_server->tables_) {
+    if (!table_item.second) {
+        continue;
+    }
     std::shared_lock partition_rwl(table_item.second->partitions_rw_);
-    for (const auto& patition_item : table_item.second->partitions_) {
-      type_result.clear();
+    for (const auto& partition_item : table_item.second->partitions_) {
+      background_errors.clear();
       memtable_usage = table_reader_usage = 0;
-      patition_item.second->DbRWLockReader();
-      patition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_MEMTABLE, &memtable_usage);
-      patition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_TABLE_READER, &table_reader_usage);
-      patition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &type_result);
-      patition_item.second->DbRWUnLock();
+      partition_item.second->DbRWLockReader();
+      partition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES, &memtable_usage);
+      partition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM, &table_reader_usage);
+      partition_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors);
+      partition_item.second->DbRWUnLock();
       total_memtable_usage += memtable_usage;
       total_table_reader_usage += table_reader_usage;
-      for (const auto& item : type_result) {
+      for (const auto& item : background_errors) {
         if (item.second != 0) {
           db_fatal_msg_stream << (total_background_errors != 0 ? "," : "");
-          db_fatal_msg_stream << patition_item.second->GetPartitionName() << "/" << item.first;
+          db_fatal_msg_stream << partition_item.second->GetPartitionName() << "/" << item.first;
           total_background_errors += item.second;
         }
       }
@@ -1153,6 +1166,30 @@ void InfoCmd::InfoData(std::string& info) {
   info.append(tmp_stream.str());
 }
 
+void InfoCmd::InfoRocksDB(std::string &info) {
+    std::stringstream tmp_stream;
+
+    tmp_stream << "# RocksDB"
+               << "\r\n";
+
+    std::shared_lock table_rwl(g_pika_server->tables_rw_);
+    for (const auto& table_item : g_pika_server->tables_) {
+        if (!table_item.second) {
+            continue;
+        }
+        std::shared_lock partition_rwl(table_item.second->partitions_rw_);
+        for (const auto& partition_item : table_item.second->partitions_) {
+            std::string rocksdb_info;
+            partition_item.second->DbRWLockReader();
+            partition_item.second->db()->GetRocksDBInfo(rocksdb_info);
+            partition_item.second->DbRWUnLock();
+            tmp_stream << rocksdb_info;
+        }
+    }
+
+    info.append(tmp_stream.str());
+}
+
 void InfoCmd::InfoDebug(std::string& info) {
   std::stringstream tmp_stream;
   tmp_stream << "# Synchronization Status"

src/storage/include/storage/storage.h

@@ -29,8 +29,8 @@ namespace storage {
 inline constexpr double ZSET_SCORE_MAX = std::numeric_limits<double>::max();
 inline constexpr double ZSET_SCORE_MIN = std::numeric_limits<double>::lowest();
 
-inline const std::string PROPERTY_TYPE_ROCKSDB_MEMTABLE = "rocksdb.cur-size-all-mem-tables";
-inline const std::string PROPERTY_TYPE_ROCKSDB_TABLE_READER = "rocksdb.estimate-table-readers-mem";
+inline const std::string PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES = "rocksdb.cur-size-all-mem-tables";
+inline const std::string PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM = "rocksdb.estimate-table-readers-mem";
 inline const std::string PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS = "rocksdb.background-errors";
 
 inline const std::string ALL_DB = "all";
@@ -1019,6 +1019,7 @@ class Storage {
 
   Status SetOptions(const OptionType& option_type, const std::string& db_type,
                     const std::unordered_map<std::string, std::string>& options);
+  void GetRocksDBInfo(std::string &info);
 
  private:
   std::unique_ptr<RedisStrings> strings_db_;

src/storage/src/redis.cc

@@ -4,6 +4,7 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "src/redis.h"
+#include <sstream>
 
 namespace storage {
 
@@ -87,4 +88,67 @@ Status Redis::SetOptions(const OptionType& option_type, const std::unordered_map
   return s;
 }
 
+void Redis::GetRocksDBInfo(std::string &info, const char *prefix) {
+    std::ostringstream string_stream;
+    string_stream << "#" << prefix << " RocksDB"
+                  << "\r\n";
+
+    auto write_stream_key_value=[&](const Slice& property, const char *metric) {
+        uint64_t value;
+        db_->GetAggregatedIntProperty(property, &value);
+        string_stream << prefix << metric << ':' << value << "\r\n";
+    };
+
+    // memtables num
+    write_stream_key_value(rocksdb::DB::Properties::kNumImmutableMemTable, "num_immutable_mem_table");
+    write_stream_key_value(rocksdb::DB::Properties::kNumImmutableMemTableFlushed, "num_immutable_mem_table_flushed");
+    write_stream_key_value(rocksdb::DB::Properties::kMemTableFlushPending, "mem_table_flush_pending");
+    write_stream_key_value(rocksdb::DB::Properties::kNumRunningFlushes, "num_running_flushes");
+
+    // compaction
+    write_stream_key_value(rocksdb::DB::Properties::kCompactionPending, "compaction_pending");
+    write_stream_key_value(rocksdb::DB::Properties::kNumRunningCompactions, "num_running_compactions");
+
+    // background errors
+    write_stream_key_value(rocksdb::DB::Properties::kBackgroundErrors, "background_errors");
+
+    // memtables size
+    write_stream_key_value(rocksdb::DB::Properties::kCurSizeActiveMemTable, "cur_size_active_mem_table");
+    write_stream_key_value(rocksdb::DB::Properties::kCurSizeAllMemTables, "cur_size_all_mem_tables");
+    write_stream_key_value(rocksdb::DB::Properties::kSizeAllMemTables, "size_all_mem_tables");
+
+    // keys
+    write_stream_key_value(rocksdb::DB::Properties::kEstimateNumKeys, "estimate_num_keys");
+
+    // table readers mem
+    write_stream_key_value(rocksdb::DB::Properties::kEstimateTableReadersMem, "estimate_table_readers_mem");
+
+    // snapshot
+    write_stream_key_value(rocksdb::DB::Properties::kNumSnapshots, "num_snapshots");
+
+    // version
+    write_stream_key_value(rocksdb::DB::Properties::kNumLiveVersions, "num_live_versions");
+    write_stream_key_value(rocksdb::DB::Properties::kCurrentSuperVersionNumber, "current_super_version_number");
+
+    // live data size
+    write_stream_key_value(rocksdb::DB::Properties::kEstimateLiveDataSize, "estimate_live_data_size");
+
+    // sst files
+    write_stream_key_value(rocksdb::DB::Properties::kTotalSstFilesSize, "total_sst_files_size");
+    write_stream_key_value(rocksdb::DB::Properties::kLiveSstFilesSize, "live_sst_files_size");
+
+    // block cache
+    write_stream_key_value(rocksdb::DB::Properties::kBlockCacheCapacity, "block_cache_capacity");
+    write_stream_key_value(rocksdb::DB::Properties::kBlockCacheUsage, "block_cache_usage");
+    write_stream_key_value(rocksdb::DB::Properties::kBlockCachePinnedUsage, "block_cache_pinned_usage");
+
+    // blob files
+    write_stream_key_value(rocksdb::DB::Properties::kNumBlobFiles, "num_blob_files");
+    write_stream_key_value(rocksdb::DB::Properties::kBlobStats, "blob_stats");
+    write_stream_key_value(rocksdb::DB::Properties::kTotalBlobFileSize, "total_blob_file_size");
+    write_stream_key_value(rocksdb::DB::Properties::kLiveBlobFileSize, "live_blob_file_size");
+
+    info.append(string_stream.str());
+}
+
 }  // namespace storage

src/storage/src/redis.h

@@ -54,6 +54,7 @@ class Redis {
 
   Status SetMaxCacheStatisticKeys(size_t max_cache_statistic_keys);
   Status SetSmallCompactionThreshold(size_t small_compaction_threshold);
+  void GetRocksDBInfo(std::string &info, const char *prefix);
 
  protected:
   Storage* const storage_;

src/storage/src/storage.cc

@@ -1749,4 +1749,12 @@ Status Storage::SetOptions(const OptionType& option_type, const std::string& db_
   return s;
 }
 
+void Storage::GetRocksDBInfo(std::string &info) {
+    strings_db_->GetRocksDBInfo(info, "strings_");
+    hashes_db_->GetRocksDBInfo(info, "hashes_");
+    lists_db_->GetRocksDBInfo(info, "lists_");
+    sets_db_->GetRocksDBInfo(info, "sets_");
+    zsets_db_->GetRocksDBInfo(info, "zsets_");
+}
+
 }  //  namespace storage

tools/pika_exporter/Makefile

@@ -4,7 +4,8 @@
 
 # export PATH := $(PATH):$(GOPATH)/bin
 
-# for mac 
+UNAME := $(shell uname)
+# for mac
 BRANCH			:= $(shell git branch | sed  's/* \(.*\)/\1/p')
 # for Linux
 # BRANCH			:= $(shell git branch | sed --quiet 's/* \(.*\)/\1/p')
@@ -62,7 +63,11 @@ export TEST_COVER
 all: build 
 
 build: deps
+ifeq ($(UNAME), Linux)
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/$(PROJNAME)
+else ifeq ($(UNAME), Darwin)
+	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/$(PROJNAME)
+endif
 
 deps: generateVer
 	@mkdir -p bin

tools/pika_exporter/README.md

@@ -9,30 +9,36 @@ Pika-Exporter is based on [Redis-Exporter](https://github.com/oliver006/redis_ex
 **Build and run locally:**
 
 To start using `pika_exporter`, install `Go` and run go get
-```
+```shell
 $ go get github.com/OpenAtomFoundation/pika/tools/pika_exporter
 $ cd $GOPATH/src/github.com/OpenAtomFoundation/pika/tools/pika_exporter
 $ make
 $ ./bin/pika_exporter <flags>
 ```
 
+For example:
+```shell
+$ nohup ./bin/pika_exporter -pika.addr 127.0.0.1:9221 &
+```
+
 
 **Prometheus Configuration:**
 
 Add a block to the scrape_configs of your prometheus.yml config file:
-```
+```yml
 scrape_configs:
+  - job_name: pika
+    scrape_interval: 15s
+    static_configs:
+      - targets: ['127.0.0.1:9121']
+        labels:
+          group: 'test'
+```
 
-...
-
-- job_name: pika
-  scrape_interval: 15s
-  static_configs:
-  - targets: ['XXXXXX:9121']
-    labels:
-      group: 'test'
+Run prometheus:
 
-...
+```shell
+prometheus --config.file=./grafana/prometheus.yml
 ```
 
 ## Flags ##
@@ -136,8 +142,36 @@ scrape_configs:
 
 ### Rocksdb Metrics
 
+| Serial Number | Metric | Meaning |
+| ------------- | ------ | ------- |
+| 11 | rocksdb.num-immutable-mem-table | Number of immutable memtables not yet flushed. |
+| 12 | rocksdb.num-immutable-mem-table-flushed | Number of immutable memtables that have been flushed. |
+| 13 | rocksdb.mem-table-flush-pending | Returns 1 if there is a pending memtable flush; otherwise returns 0. |
+| 14 | rocksdb.num-running-flushes | Number of currently running flush operations. |
+| 15 | rocksdb.compaction-pending | Returns 1 if at least one compaction operation is pending; otherwise returns 0. |
+| 16 | rocksdb.num-running-compactions | Number of running compactions. |
+| 17 | rocksdb.background-errors | Total number of background errors. |
+| 18 | rocksdb.cur-size-active-mem-table | Approximate size, in bytes, of the active memtable. |
+| 19 | rocksdb.cur-size-all-mem-tables | Total size in bytes of memtables not yet flushed, including the current active memtable and the unflushed immutable memtables. |
+| 20 | rocksdb.size-all-mem-tables | Total size in bytes of all memtables, including the active memtable, unflushed immutable memtables, and pinned immutable memtables. |
+| 25 | rocksdb.estimate-num-keys | Estimated number of keys in active memtable, unflushed immutable memtables, and flushed SST files. |
+| 26 | rocksdb.estimate-table-readers-mem | Estimated memory size used for reading SST files, excluding block cache (such as filter and index blocks). |
+| 28 | rocksdb.num-snapshots | Number of unreleased snapshots in the database. |
+| 31 | rocksdb.num-live-versions | Number of current versions. More current versions usually indicate more SST files being used by iterators or incomplete compactions. |
+| 32 | rocksdb.current-super-version-number | Current number of the LSM version. It is a uint64_t integer that increments after any changes in the LSM tree. This number is not preserved after restarting the database and starts from 0 after a database restart. |
+| 33 | rocksdb.estimate-live-data-size | Estimated size of the activity data in bytes.For BlobDB, it also includes the actual live bytes in the version's blob file. |
+| 36 | rocksdb.total-sst-files-size | Total size (in bytes) of all SST files.Note: If there are too many files, it may slow down the online query. |
+| 37 | rocksdb.live-sst-files-size | Total size in bytes of all SST files belonging to the latest LSM tree. |
+| 47 | rocksdb.block-cache-capacity | The capacity of the block cache. |
+| 48 | rocksdb.block-cache-usage | Memory size occupied by entries in the block cache. |
+| 49 | rocksdb.block-cache-pinned-usage | Memory size occupied by pinned entries in the block cache. |
+| 51 | rocksdb.num-blob-files | The number of blob files in the current version. |
+| 52 | rocksdb.blob-stats | The total and size of all blob files, and the total amount of garbage (in bytes) in blob files in the current version. |
+| 53 | rocksdb.total-blob-file-size | The total size of all blob files across all versions. |
+| 54 | rocksdb.live-blob-file-size | The total size of all blob files in the current version. |
+
 ## Grafana Dashboard ##
-See [here](./contrib/grafana_prometheus_pika_dashboard.json)
+See [here](./grafana/grafana_prometheus_pika_dashboard.json)
 
 Screenshots:
 ![Overview](./contrib/overview.png)
@@ -150,3 +184,5 @@ Screenshots:
 ![TimeConsumingOperation](./contrib/time_consuming_operation.png)
 
 ![KeysMetrics](./contrib/keys_metrics.png)
+
+![RocksDB](./contrib/rocksdb.png)