Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

eth: add debug_accountRange API #19645

Merged
merged 7 commits into from
Mar 31, 2020
Merged

eth: add debug_accountRange API #19645

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
dd31661
core, eth: add debug_iteratorAccountAt rpc call
jsvisa Sep 19, 2019
649e01f
eth,core: debug_iteratorAccountAt use rpc.BlockNumerOrHash
jsvisa Oct 16, 2019
c9557ed
eth: merge debug_iteratorAccountAt into debug_accountRangeAt
jsvisa Nov 17, 2019
c4c3765
state/dump, eth/tests: fix dump tests + general formatting
holiman Feb 18, 2020
650b5e8
eth tests: linter nitpicks
holiman Feb 20, 2020
3db0872
eth: rename AccountRangeAt to AccountRange
fjl Mar 31, 2020
78a43f3
eth: remove context parameter in storageRangeAt
fjl Mar 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 41 additions & 7 deletions core/state/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"github.com/ethereum/go-ethereum/trie"
)

// DumpAccount represents an account in the state
// DumpAccount represents an account in the state.
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Expand All @@ -40,17 +40,24 @@ type DumpAccount struct {

}

// Dump represents the full dump in a collected format, as one large map
// Dump represents the full dump in a collected format, as one large map.
type Dump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}

// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
type iterativeDump struct {
*json.Encoder
}

// IteratorDump is an implementation for iterating over data.
type IteratorDump struct {
Root string `json:"root"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Root string `json:"root"`
Root common.Hash `json:"root"`

Copy link
Contributor Author

@jsvisa jsvisa Oct 16, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The other Dump method all returns as string, let me know if I've missed something. BTW, consistency is more important.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are correct. This is how the current dump looks:

{"balance":"1000000000000000000000","nonce":0,"root":"56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","codeHash":"c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","address":"0xc518799a5925576213e21896e0539abb85b05ae3"}

No 0x-prefixed codeHash, nor code, nor root.

Accounts map[common.Address]DumpAccount `json:"accounts"`
Next []byte `json:"next,omitempty"` // nil if no more accounts
}

// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
Expand All @@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d *IteratorDump) onRoot(root common.Hash) {
d.Root = fmt.Sprintf("%x", root)
}

func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}

func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
Expand All @@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
}{root})
}

func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(s.trie.Hash())
it := trie.NewIterator(s.trie.NodeIterator(nil))

var count int
it := trie.NewIterator(s.trie.NodeIterator(start))
for it.Next() {
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
Expand Down Expand Up @@ -130,18 +146,27 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
}
}
c.onAccount(addr, account)
count++
if maxResults > 0 && count >= maxResults {
if it.Next() {
nextKey = it.Key
}
break
}
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}

return nextKey
}

// RawDump returns the entire state an a single large object
func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
return *dump
}

Expand All @@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool

// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
}

// IteratorDump dumps out a batch of accounts starts with the given start key
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
iterator := &IteratorDump{
Accounts: make(map[common.Address]DumpAccount),
}
iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
return *iterator
}
88 changes: 34 additions & 54 deletions eth/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,70 +351,50 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs,
return results, nil
}

// AccountRangeResult returns a mapping from the hash of an account addresses
// to its preimage. It will return the JSON null if no preimage is found.
// Since a query can return a limited amount of results, a "next" field is
// also present for paging.
type AccountRangeResult struct {
Accounts map[common.Hash]*common.Address `json:"accounts"`
Next common.Hash `json:"next"`
}

func accountRange(st state.Trie, start *common.Hash, maxResults int) (AccountRangeResult, error) {
if start == nil {
start = &common.Hash{0}
}
it := trie.NewIterator(st.NodeIterator(start.Bytes()))
result := AccountRangeResult{Accounts: make(map[common.Hash]*common.Address), Next: common.Hash{}}

if maxResults > AccountRangeMaxResults {
maxResults = AccountRangeMaxResults
}

for i := 0; i < maxResults && it.Next(); i++ {
if preimage := st.GetKey(it.Key); preimage != nil {
addr := &common.Address{}
addr.SetBytes(preimage)
result.Accounts[common.BytesToHash(it.Key)] = addr
} else {
result.Accounts[common.BytesToHash(it.Key)] = nil
}
}

if it.Next() {
result.Next = common.BytesToHash(it.Key)
}

return result, nil
}

// AccountRangeMaxResults is the maximum number of results to be returned per call
const AccountRangeMaxResults = 256

// AccountRange enumerates all accounts in the latest state
func (api *PrivateDebugAPI) AccountRange(ctx context.Context, start *common.Hash, maxResults int) (AccountRangeResult, error) {
var statedb *state.StateDB
// AccountRangeAt enumerates all accounts in the given block and start point in paging request
func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) {
var stateDb *state.StateDB
var err error
block := api.eth.blockchain.CurrentBlock()

if len(block.Transactions()) == 0 {
statedb, err = api.computeStateDB(block, defaultTraceReexec)
if err != nil {
return AccountRangeResult{}, err
if number, ok := blockNrOrHash.Number(); ok {
if number == rpc.PendingBlockNumber {
// If we're dumping the pending state, we need to request
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb = api.eth.miner.Pending()
} else {
var block *types.Block
if number == rpc.LatestBlockNumber {
block = api.eth.blockchain.CurrentBlock()
} else {
block = api.eth.blockchain.GetBlockByNumber(uint64(number))
}
if block == nil {
return state.IteratorDump{}, fmt.Errorf("block #%d not found", number)
}
stateDb, err = api.eth.BlockChain().StateAt(block.Root())
if err != nil {
return state.IteratorDump{}, err
}
}
} else {
_, _, statedb, err = api.computeTxEnv(block.Hash(), len(block.Transactions())-1, 0)
} else if hash, ok := blockNrOrHash.Hash(); ok {
block := api.eth.blockchain.GetBlockByHash(hash)
if block == nil {
return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex())
}
stateDb, err = api.eth.BlockChain().StateAt(block.Root())
if err != nil {
return AccountRangeResult{}, err
return state.IteratorDump{}, err
}
}

trie, err := statedb.Database().OpenTrie(block.Header().Root)
if err != nil {
return AccountRangeResult{}, err
if maxResults > AccountRangeMaxResults || maxResults <= 0 {
maxResults = AccountRangeMaxResults
}

return accountRange(trie, start, maxResults)
return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil
}

// StorageRangeResult is the result of a debug_storageRangeAt API call.
Expand All @@ -431,7 +411,7 @@ type storageEntry struct {
}

// StorageRangeAt returns the storage at the given block height and transaction index.
func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
_, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0)
if err != nil {
return StorageRangeResult{}, err
Expand Down
98 changes: 27 additions & 71 deletions eth/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,29 +33,24 @@ import (

var dumper = spew.ConfigState{Indent: " "}

func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start *common.Hash, requestedNum int, expectedNum int) AccountRangeResult {
result, err := accountRange(*trie, start, requestedNum)
if err != nil {
t.Fatal(err)
}
func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump {
result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum)

if len(result.Accounts) != expectedNum {
t.Fatalf("expected %d results. Got %d", expectedNum, len(result.Accounts))
t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts))
}

for _, address := range result.Accounts {
if address == nil {
t.Fatalf("null address returned")
for address := range result.Accounts {
if address == (common.Address{}) {
t.Fatalf("empty address returned")
}
if !statedb.Exist(*address) {
if !statedb.Exist(address) {
t.Fatalf("account not found in state %s", address.Hex())
}
}

return result
}

type resultHash []*common.Hash
type resultHash []common.Hash

func (h resultHash) Len() int { return len(h) }
func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
Expand All @@ -80,76 +75,47 @@ func TestAccountRange(t *testing.T) {
m[addr] = true
}
}

state.Commit(true)
root := state.IntermediateRoot(true)

trie, err := statedb.OpenTrie(root)
if err != nil {
t.Fatal(err)
}

t.Logf("test getting number of results less than max")
accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)

t.Logf("test getting number of results greater than max %d", AccountRangeMaxResults)
accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults*2, AccountRangeMaxResults)

t.Logf("test with empty 'start' hash")
accountRangeTest(t, &trie, state, nil, AccountRangeMaxResults, AccountRangeMaxResults)

t.Logf("test pagination")

accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
// test pagination
firstResult := accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults, AccountRangeMaxResults)

t.Logf("test pagination 2")
secondResult := accountRangeTest(t, &trie, state, &firstResult.Next, AccountRangeMaxResults, AccountRangeMaxResults)
firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults)
secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults)

hList := make(resultHash, 0)
for h1, addr1 := range firstResult.Accounts {
h := &common.Hash{}
h.SetBytes(h1.Bytes())
hList = append(hList, h)
for h2, addr2 := range secondResult.Accounts {
// Make sure that the hashes aren't the same
if bytes.Equal(h1.Bytes(), h2.Bytes()) {
t.Fatalf("pagination test failed: results should not overlap")
}

// If either address is nil, then it makes no sense to compare
// them as they might be two different accounts.
if addr1 == nil || addr2 == nil {
continue
}

// Since the two hashes are different, they should not have
// the same preimage, but let's check anyway in case there
// is a bug in the (hash, addr) map generation code.
if bytes.Equal(addr1.Bytes(), addr2.Bytes()) {
t.Fatalf("pagination test failed: addresses should not repeat")
}
for addr1 := range firstResult.Accounts {
// If address is empty, then it makes no sense to compare
// them as they might be two different accounts.
if addr1 == (common.Address{}) {
continue
}
if _, duplicate := secondResult.Accounts[addr1]; duplicate {
t.Fatalf("pagination test failed: results should not overlap")
}
hList = append(hList, crypto.Keccak256Hash(addr1.Bytes()))
}

// Test to see if it's possible to recover from the middle of the previous
// set and get an even split between the first and second sets.
t.Logf("test random access pagination")
sort.Sort(hList)
middleH := hList[AccountRangeMaxResults/2]
middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults)
innone, infirst, insecond := 0, 0, 0
missing, infirst, insecond := 0, 0, 0
for h := range middleResult.Accounts {
if _, ok := firstResult.Accounts[h]; ok {
infirst++
} else if _, ok := secondResult.Accounts[h]; ok {
insecond++
} else {
innone++
missing++
}
}
if innone != 0 {
t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", innone)
if missing != 0 {
t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing)
}
if infirst != AccountRangeMaxResults/2 {
t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2)
Expand All @@ -164,20 +130,10 @@ func TestEmptyAccountRange(t *testing.T) {
statedb = state.NewDatabase(rawdb.NewMemoryDatabase())
state, _ = state.New(common.Hash{}, statedb)
)

state.Commit(true)
root := state.IntermediateRoot(true)

trie, err := statedb.OpenTrie(root)
if err != nil {
t.Fatal(err)
}

results, err := accountRange(trie, &common.Hash{0x0}, AccountRangeMaxResults)
if err != nil {
t.Fatalf("Empty results should not trigger an error: %v", err)
}
if results.Next != common.HexToHash("0") {
state.IntermediateRoot(true)
results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults)
if bytes.Equal(results.Next, (common.Hash{}).Bytes()) {
t.Fatalf("Empty results should not return a second page")
}
if len(results.Accounts) != 0 {
Expand Down