diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 822de1ea8b8..66a26a59d8e 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -102,7 +102,7 @@ namespace eosio { namespace chain { uint64_t previous(); uint32_t version() const { return _version; } uint32_t first_block_num() const { return _first_block_num; } - constexpr static uint32_t _buf_len = 1U << 24; + static uint32_t _buf_len; private: void update_buffer(); @@ -759,6 +759,8 @@ namespace eosio { namespace chain { , _buffer_ptr(std::make_unique(_buf_len)) { } + uint32_t detail::reverse_iterator::_buf_len = 1U << 24; + uint32_t detail::reverse_iterator::open(const fc::path& block_file_name) { _block_file_name = block_file_name.generic_string(); _file.reset( FC_FOPEN(_block_file_name.c_str(), "r")); @@ -956,6 +958,20 @@ namespace eosio { namespace chain { return std::clamp(version, min_supported_version, max_supported_version) == version; } + namespace { + template + T read_buffer(const char* buf) { + T result; + memcpy(&result, buf, sizeof(T)); + return result; + } + + template + void write_buffer(char* des, const T* src) { + memcpy(des, src, sizeof(T)); + } + } + bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block) { using namespace std; EOS_ASSERT( block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories" ); @@ -1024,7 +1040,8 @@ namespace eosio { namespace chain { detail::index_writer index(new_index_filename, num_blocks); uint64_t read_size = 0; - for(uint64_t to_write_remaining = to_write; to_write_remaining > 0; to_write_remaining -= read_size) { + uint64_t write_size = 0; + for(uint64_t to_write_remaining = to_write; to_write_remaining > 0; to_write_remaining -= write_size) { read_size = to_write_remaining; if (read_size > detail::reverse_iterator::_buf_len) { read_size = detail::reverse_iterator::_buf_len; @@ -1038,17 +1055,26 @@ namespace eosio { namespace chain { // walk this memory section to adjust block position to match the adjusted location // of the block start and store in the new index file + write_size = read_size; while(original_pos >= start_of_blk_buffer_pos) { const auto buffer_index = original_pos - start_of_blk_buffer_pos; - uint64_t& pos_content = *(uint64_t*)(buf + buffer_index); + uint64_t pos_content = read_buffer(buf + buffer_index); + + if ( (pos_content - start_of_blk_buffer_pos) > 0 && (pos_content - start_of_blk_buffer_pos) < pos_size ) { + // avoid the whole 8 bytes that contains a blk pos being split by the buffer + write_size = read_size - (pos_content - start_of_blk_buffer_pos); + } const auto start_of_this_block = pos_content; pos_content = start_of_this_block - pos_delta; + write_buffer(buf + buffer_index, &pos_content); index.write(pos_content); original_pos = start_of_this_block - pos_size; } - new_block_file.seek(new_block_file_first_block_pos + to_write_remaining - read_size); - new_block_file.write(buf, read_size); + new_block_file.seek(new_block_file_first_block_pos + to_write_remaining - write_size); + uint64_t offset = read_size - write_size; + new_block_file.write(buf+offset, write_size); } + index.complete(); fclose(original_block_log.blk_in); original_block_log.blk_in = nullptr; @@ -1185,3 +1211,8 @@ namespace eosio { namespace chain { } } } /// eosio::chain + +// used only for unit test to adjust the buffer length +void block_log_set_buff_len(uint64_t len){ + eosio::chain::detail::reverse_iterator::_buf_len = len; +} \ No newline at end of file diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 06958d25532..a33d57ce8a5 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -22,6 +22,8 @@ void remove_existing_blocks(controller::config& config) { remove(block_index_path); } +void block_log_set_buff_len(uint64_t len); + BOOST_AUTO_TEST_SUITE(restart_chain_tests) BOOST_AUTO_TEST_CASE(test_existing_state_without_block_log) @@ -78,4 +80,73 @@ BOOST_AUTO_TEST_CASE(test_restart_with_different_chain_id) BOOST_REQUIRE_EXCEPTION(other.open(chain_id), chain_id_type_exception, fc_exception_message_starts_with("chain ID in state ")); } +namespace{ + struct scoped_temp_path { + boost::filesystem::path path; + scoped_temp_path() { + path = boost::filesystem::unique_path(); + if (boost::unit_test::framework::master_test_suite().argc >= 2) { + path += boost::unit_test::framework::master_test_suite().argv[1]; + } + } + ~scoped_temp_path() { + boost::filesystem::remove_all(path); + } + }; +} + +enum class buf_len_type { small, medium, large }; + +void trim_blocklog_front(uint32_t truncate_at_block, buf_len_type len_type) { + tester chain; + chain.produce_blocks(30); + chain.close(); + + namespace bfs = boost::filesystem; + + auto blocks_dir = chain.get_config().blocks_dir; + auto old_index_size = fc::file_size(blocks_dir / "blocks.index"); + + scoped_temp_path temp1, temp2; + boost::filesystem::create_directory(temp1.path); + bfs::copy(blocks_dir / "blocks.log", temp1.path / "blocks.log"); + bfs::copy(blocks_dir / "blocks.index", temp1.path / "blocks.index"); + + trim_data old_log(temp1.path); + uint64_t blk_size = old_log.block_pos(30) - old_log.block_pos(29); + uint64_t log_size = old_log.block_pos(30) + blk_size; + + switch (len_type){ + case buf_len_type::small: + block_log_set_buff_len( blk_size + (sizeof(uint64_t) - 1)); + break; + case buf_len_type::medium: + block_log_set_buff_len( log_size / 3); + break; + case buf_len_type::large: + block_log_set_buff_len(log_size); + break; + default: + return; + } + + BOOST_CHECK( block_log::trim_blocklog_front(temp1.path, temp2.path, truncate_at_block) == true); + trim_data new_log(temp1.path); + BOOST_CHECK(new_log.first_block == truncate_at_block); + BOOST_CHECK(new_log.last_block == old_log.last_block); + BOOST_CHECK(old_log.version == new_log.version); + + int num_blocks_trimmed = truncate_at_block - 1; + BOOST_CHECK(fc::file_size(temp1.path / "blocks.index") == old_index_size - sizeof(uint64_t) * num_blocks_trimmed); +} + +BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { + trim_blocklog_front(5, buf_len_type::small); + trim_blocklog_front(6, buf_len_type::small); + trim_blocklog_front(10, buf_len_type::medium); + trim_blocklog_front(11, buf_len_type::medium); + trim_blocklog_front(15, buf_len_type::large); + trim_blocklog_front(16, buf_len_type::large); +} + BOOST_AUTO_TEST_SUITE_END()