Skip to content

Commit

Permalink
tools: make tools to be compatible with both original data and Pegasu…
Browse files Browse the repository at this point in the history 
…s data (#20)
  • Loading branch information
@acelyc111
acelyc111 authored Apr 22, 2019
1 parent 95b5d46 commit 4f6ccda
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 24 deletions.
9 changes: 6 additions & 3 deletions db/version_set.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2361,11 +2361,11 @@ std::string Version::DebugString(bool hex, bool print_stats) const {
for (int level = 0; level < storage_info_.num_levels_; level++) {
// E.g.,
// --- level 1 ---
// 17:123['a' .. 'd']
// 20:43['e' .. 'g']
// 17:123[1 .. 124]['a' .. 'd']
// 20:43[125 .. 128]['e' .. 'g']
//
// if print_stats=true:
// 17:123['a' .. 'd'](4096)
// 17:123[1 .. 124]['a' .. 'd'](4096)
r.append("--- level ");
AppendNumberTo(&r, level);
r.append(" --- version# ");
Expand All @@ -2377,12 +2377,15 @@ std::string Version::DebugString(bool hex, bool print_stats) const {
AppendNumberTo(&r, files[i]->fd.GetNumber());
r.push_back(':');
AppendNumberTo(&r, files[i]->fd.GetFileSize());
// TODO(laiyingchun): Pegasus added code to dump seqnos, maybe we should
// commit these code to facebook/rocksdb later.
r.append("[");
AppendNumberTo(&r, files[i]->smallest_seqno);
r.append(" .. ");
AppendNumberTo(&r, files[i]->largest_seqno);
r.append("]");
r.append("[");
// TODO(laiyingchun): Pegasus data DebugString() not work correctly now.
r.append(files[i]->smallest.DebugString(hex));
r.append(" .. ");
r.append(files[i]->largest.DebugString(hex));
Expand Down
22 changes: 15 additions & 7 deletions tools/ldb_cmd.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ namespace {
void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
LDBCommandExecuteResult* exec_state);

void DumpSstFile(std::string filename, bool output_hex, bool show_properties);
void DumpSstFile(std::string filename, bool output_hex, bool show_properties, bool pegasus_data);
};

LDBCommand* LDBCommand::InitFromCmdLineArgs(
Expand Down Expand Up @@ -1283,6 +1283,7 @@ const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
const std::string DBDumperCommand::ARG_STATS = "stats";
const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
const std::string DBDumperCommand::ARG_PEGASUS_DATA = "pegasus_data";

DBDumperCommand::DBDumperCommand(
const std::vector<std::string>& params,
Expand All @@ -1293,13 +1294,14 @@ DBDumperCommand::DBDumperCommand(
{ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
ARG_TIMESTAMP, ARG_PATH})),
ARG_TIMESTAMP, ARG_PATH, ARG_PEGASUS_DATA})),
null_from_(true),
null_to_(true),
max_keys_(-1),
count_only_(false),
count_delim_(false),
print_stats_(false) {
print_stats_(false),
pegasus_data_(false) {
std::map<std::string, std::string>::const_iterator itr =
options.find(ARG_FROM);
if (itr != options.end()) {
Expand Down Expand Up @@ -1340,6 +1342,7 @@ DBDumperCommand::DBDumperCommand(

print_stats_ = IsFlagPresent(flags, ARG_STATS);
count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
pegasus_data_ = IsFlagPresent(flags, ARG_PEGASUS_DATA);

if (is_key_hex_) {
if (!null_from_) {
Expand Down Expand Up @@ -1373,6 +1376,7 @@ void DBDumperCommand::Help(std::string& ret) {
ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
ret.append(" [--" + ARG_PEGASUS_DATA + "]");
ret.append("\n");
}

Expand Down Expand Up @@ -1408,7 +1412,7 @@ void DBDumperCommand::DoCommand() {
&exec_state_);
break;
case kTableFile:
DumpSstFile(path_, is_key_hex_, /* show_properties */ true);
DumpSstFile(path_, is_key_hex_, /* show_properties */ true, pegasus_data_);
break;
case kDescriptorFile:
DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_,
Expand Down Expand Up @@ -1542,6 +1546,7 @@ void DBDumperCommand::DoDumpCommand() {
if (is_db_ttl_ && timestamp_) {
fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
}
// TODO(laiyingchun): data decoding depends on whether it is Pegasus data.
std::string str =
PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
is_key_hex_, is_value_hex_);
Expand Down Expand Up @@ -2532,6 +2537,7 @@ void DBQuerierCommand::DoCommand() {

const std::string& cmd = tokens[0];

// TODO(laiyingchun): data encoding/decoding depends on whether it is Pegasus data.
if (cmd == HELP_CMD) {
fprintf(stdout,
"get <key>\n"
Expand Down Expand Up @@ -2796,7 +2802,7 @@ void RestoreCommand::DoCommand() {

namespace {

void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
void DumpSstFile(std::string filename, bool output_hex, bool show_properties, bool pegasus_data) {
std::string from_key;
std::string to_key;
if (filename.length() <= 4 ||
Expand All @@ -2805,7 +2811,9 @@ void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
return;
}
// no verification
rocksdb::SstFileReader reader(filename, false, output_hex);
Options options;
options.pegasus_data = pegasus_data;
rocksdb::SstFileReader reader(filename, false, output_hex, options);
Status st = reader.ReadSequential(true, std::numeric_limits<uint64_t>::max(), false, // has_from
from_key, false, // has_to
to_key);
Expand Down Expand Up @@ -2885,7 +2893,7 @@ void DBFileDumperCommand::DoCommand() {
std::string filename = fileMetadata.db_path + fileMetadata.name;
std::cout << filename << " level:" << fileMetadata.level << std::endl;
std::cout << "------------------------------" << std::endl;
DumpSstFile(filename, false, true);
DumpSstFile(filename, false, true, db_->GetDBOptions().pegasus_data);
std::cout << std::endl;
}
std::cout << std::endl;
Expand Down
2 changes: 2 additions & 0 deletions tools/ldb_cmd_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,14 @@ class DBDumperCommand : public LDBCommand {
bool count_only_;
bool count_delim_;
bool print_stats_;
bool pegasus_data_;
std::string path_;

static const std::string ARG_COUNT_ONLY;
static const std::string ARG_COUNT_DELIM;
static const std::string ARG_STATS;
static const std::string ARG_TTL_BUCKET;
static const std::string ARG_PEGASUS_DATA;
};

class InternalDumpCommand : public LDBCommand {
Expand Down
41 changes: 28 additions & 13 deletions tools/sst_dump_tool.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,10 @@ namespace rocksdb {

SstFileReader::SstFileReader(const std::string& file_path,
bool verify_checksum,
bool output_hex)
bool output_hex,
Options options)
:file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
ioptions_(options_),
output_hex_(output_hex), options_(std::move(options)), ioptions_(options_),
internal_comparator_(BytewiseComparator()) {
fprintf(stdout, "Process %s\n", file_path.c_str());
init_result_ = GetTableReader(file_name_);
Expand Down Expand Up @@ -405,16 +406,22 @@ Status SstFileReader::ReadSequential(bool print_kv, uint64_t read_num,
}

if (print_kv) {
if (ikey.user_key.size() >= 2) {
uint32_t expire_ts = 0;
std::string hash_key, sort_key, user_data;
pegasus_restore_key(ikey.user_key, hash_key, sort_key);
pegasus_restore_value(value, expire_ts, user_data);
std::ostringstream oss;
oss << "\"" << escape_string(hash_key) << "\" : \"" << escape_string(sort_key)
<< "\" @ " << ikey.sequence << " : " << ikey.type << " => "
<< expire_ts << " : \"" << escape_string(user_data) << "\"";
fprintf(stdout, "%s\n", oss.str().c_str());
if (!options_.pegasus_data) {
fprintf(stdout, "%s => %s\n",
ikey.DebugString(output_hex_).c_str(),
value.ToString(output_hex_).c_str());
} else {
if (ikey.user_key.size() >= 2) {
uint32_t expire_ts = 0;
std::string hash_key, sort_key, user_data;
pegasus_restore_key(ikey.user_key, hash_key, sort_key);
pegasus_restore_value(value, expire_ts, user_data);
std::ostringstream oss;
oss << "\"" << escape_string(hash_key) << "\" : \"" << escape_string(sort_key)
<< "\" @ " << ikey.sequence << " : " << ikey.type << " => "
<< expire_ts << " : \"" << escape_string(user_data) << "\"";
fprintf(stdout, "%s\n", oss.str().c_str());
}
}
}
}
Expand Down Expand Up @@ -455,6 +462,9 @@ void print_help() {
--output_hex
Can be combined with scan command to print the keys and values in Hex
--pegasus_data
Whether to read Pegasus data.
--from=<user_key>
Key to start reading from when executing check|scan
Expand Down Expand Up @@ -504,6 +514,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
uint64_t n;
bool verify_checksum = false;
bool output_hex = false;
bool pegasus_data = false;
bool input_key_hex = false;
bool has_from = false;
bool has_to = false;
Expand All @@ -526,6 +537,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
dir_or_file = argv[i] + 7;
} else if (strcmp(argv[i], "--output_hex") == 0) {
output_hex = true;
} else if (strcmp(argv[i], "--pegasus_data") == 0) {
pegasus_data = true;
} else if (strcmp(argv[i], "--input_key_hex") == 0) {
input_key_hex = true;
} else if (sscanf(argv[i],
Expand Down Expand Up @@ -648,8 +661,10 @@ int SSTDumpTool::Run(int argc, char** argv) {
filename = std::string(dir_or_file) + "/" + filename;
}

Options options;
options.pegasus_data = pegasus_data;
rocksdb::SstFileReader reader(filename, verify_checksum,
output_hex);
output_hex, options);
if (!reader.getStatus().ok()) {
fprintf(stderr, "%s: %s\n", filename.c_str(),
reader.getStatus().ToString().c_str());
Expand Down
3 changes: 2 additions & 1 deletion tools/sst_dump_tool_imp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace rocksdb {
class SstFileReader {
public:
explicit SstFileReader(const std::string& file_name, bool verify_checksum,
bool output_hex);
bool output_hex, Options options);

Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
const std::string& from_key, bool has_to,
Expand Down Expand Up @@ -62,6 +62,7 @@ class SstFileReader {
std::string file_name_;
uint64_t read_num_;
bool verify_checksum_;
bool output_hex_;
EnvOptions soptions_;

// options_ and internal_comparator_ will also be used in
Expand Down

0 comments on commit 4f6ccda

Please sign in to comment.