cmd/era: rewrite era tool

cmd/era/main.go

@@ -1,43 +1,364 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/internal/e2store"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/era"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/internal/flags"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/urfave/cli/v2"
+)
+
+var app = flags.NewApp("go-ethereum era tool")
+
+var (
+	dirFlag = &cli.StringFlag{
+		Name:  "dir",
+		Usage: "directory storing all relevant Era files",
+		Value: "eras",
+	}
+	networkFlag = &cli.StringFlag{
+		Name:  "network",
+		Usage: "network name associated with Era files",
+		Value: "mainnet",
+	}
+	txsFlag = &cli.BoolFlag{
+		Name:  "txs",
+		Usage: "print full transaction values",
+	}
 )
 
+var (
+	blockCommand = &cli.Command{
+		Name:      "block",
+		Usage:     "get block data",
+		ArgsUsage: "<number>",
+		Action:    block,
+		Flags: []cli.Flag{
+			txsFlag,
+		},
+	}
+	infoCommand = &cli.Command{
+		Name:      "info",
+		ArgsUsage: "<epoch>",
+		Usage:     "get epoch information",
+		Action:    info,
+	}
+	verifyCommand = &cli.Command{
+		Name:   "verify",
+		Usage:  "verifies each epoch",
+		Action: verify,
+	}
+	convertCommand = &cli.Command{
+		Name:   "convert",
+		Usage:  "convert to new format",
+		Action: recompute,
+	}
+)
+
+func init() {
+	app.Commands = []*cli.Command{
+		blockCommand,
+		infoCommand,
+		verifyCommand,
+		convertCommand,
+	}
+	app.Flags = []cli.Flag{
+		dirFlag,
+		networkFlag,
+	}
+}
+
 func main() {
-	if len(os.Args) != 2 {
-		exit(errors.New("usage: era <FILE>"))
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
 	}
+}
 
-	f, err := os.Open(os.Args[1])
+// block prints the specified block from an Era store.
+func block(ctx *cli.Context) error {
+	num, err := strconv.ParseUint(ctx.Args().First(), 10, 64)
 	if err != nil {
-		exit(err)
+		return fmt.Errorf("invalid block number: %w", err)
 	}
-	defer f.Close()
 
-	r := e2store.NewReader(f)
-	for {
-		entry, err := r.Read()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			exit(err)
+	r, err := openEra(ctx, num/uint64(era.MaxEraBatchSize))
+	defer r.Close()
+
+	// Read block with number.
+	block, err := r.ReadBlock(num)
+	if err != nil {
+		return fmt.Errorf("error reading era: %w", err)
+	}
+
+	// Convert block to JSON and print.
+	val, err := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig)
+	if err != nil {
+		return fmt.Errorf("error marshaling json: %w", err)
+	}
+	b, err := json.MarshalIndent(val, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling json: %w", err)
+	}
+	fmt.Println(string(b))
+	return nil
+}
+
+// info prints some high-level information about the Era file.
+func info(ctx *cli.Context) error {
+	epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64)
+	if err != nil {
+		return fmt.Errorf("invalid block number: %w", err)
+	}
+	era, err := openEra(ctx, epoch)
+	if err != nil {
+		return err
+	}
+	defer era.Close()
+
+	acc, err := era.Accumulator()
+	if err != nil {
+		return fmt.Errorf("error reading accumulator: %w", err)
+	}
+	td, err := era.TotalDifficulty()
+	if err != nil {
+		return fmt.Errorf("error reading total difficulty: %w", err)
+	}
+	start, err := era.Start()
+	if err != nil {
+		return fmt.Errorf("error reading start block: %w", err)
+	}
+	count, err := era.Count()
+	if err != nil {
+		return fmt.Errorf("error reading count: %w", err)
+	}
+	info := struct {
+		Accumulator     common.Hash `json:"accumulator"`
+		TotalDifficulty *big.Int    `json:"totalDifficulty"`
+		StartBlock      uint64      `json:"startBlock"`
+		Count           uint64      `json:"count"`
+	}{
+		acc, td, start, count,
+	}
+	b, _ := json.MarshalIndent(info, "", "  ")
+	fmt.Println(string(b))
+	return nil
+}
+
+// openEra opens an Era file at a certain epoch.
+func openEra(ctx *cli.Context, epoch uint64) (*era.Reader, error) {
+	var (
+		dir     = ctx.String(dirFlag.Name)
+		network = ctx.String(networkFlag.Name)
+	)
+	filename := path.Join(dir, era.Filename(int(epoch), network))
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error opening era dir: %w", err)
+	}
+	return era.NewReader(f), nil
+}
+
+// verify checks each Era file in a directory to ensure it is well-formed and
+// that the accumulator matches the expected value.
+func verify(ctx *cli.Context) error {
+	if ctx.Args().Len() != 1 {
+		return fmt.Errorf("missing accumulators file")
+	}
+
+	roots, err := readHashes(ctx.Args().First())
+	if err != nil {
+		return fmt.Errorf("unable to read expected roots file: %w", err)
+	}
+
+	var (
+		dir     = ctx.String(dirFlag.Name)
+		network = ctx.String(networkFlag.Name)
+
+		start    = time.Now()
+		reported = time.Now()
+	)
+
+	// Verify each epoch matches the expected root.
+	for i, want := range roots {
+		name := path.Join(dir, era.Filename(i, network))
+		f, err := os.Open(name)
+		if err != nil {
+			return fmt.Errorf("error opening era file %s: %w", name, err)
+		}
+		defer f.Close()
+		r := era.NewReader(f)
+
+		var (
+			got    common.Hash
+			td     *big.Int
+			hashes = make([]common.Hash, 0)
+			tds    = make([]*big.Int, 0)
+		)
+		if got, err = r.Accumulator(); err != nil {
+			return fmt.Errorf("error reading accumulator: %w", err)
+		}
+		if td, err = r.TotalDifficulty(); err != nil {
+			return fmt.Errorf("error reading total difficulty: %w", err)
+		}
+		// Verify that the embedded root matches expected
+		if got != want {
+			return fmt.Errorf("listed accumulator root in epoch %d does not match expected", i)
+		}
+
+		// Starting at epoch 0, iterate through all available Era files
+		// and check the following:
+		//   * the block index is constructed correctly
+		//   * the starting total difficulty value is correct
+		//   * the accumulator is correct by recomputing it locally,
+		//     which verfies the blocks are all correct (via hash)
+		//   * the receipts root matches the value in the block
+		for j := 0; ; j++ {
+			// read() walks the block index, so we're able to
+			// implicitly verify it.
+			block, receipts, err := r.Read()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				return fmt.Errorf("error reading block %d: %w", i*era.MaxEraBatchSize+j, err)
+			}
+			// Calculate receipt root from receipt list and check
+			// value against block.
+			rr := types.DeriveSha(receipts, trie.NewStackTrie(nil))
+			if rr != block.ReceiptHash() {
+				return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr)
+			}
+			hashes = append(hashes, block.Hash())
+			td.Add(td, block.Difficulty())
+			tds = append(tds, new(big.Int).Set(td))
+
+			// Give the user some feedback that something is happening.
+			if time.Since(reported) >= 8*time.Second {
+				fmt.Printf("Verifying Era files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start)))
+				reported = time.Now()
+			}
+		}
+		got, err = era.ComputeAccumulator(hashes, tds)
+		if err != nil {
+			return fmt.Errorf("error computing accumulator for epoch %d: %w", i, err)
 		}
-		// fmt.Printf("%x: %d\n", entry.Type, len(entry.Value))
-		if entry.Type == era.TypeAccumulator {
-			fmt.Printf("root: %s\n", common.Bytes2Hex(entry.Value))
+		if got != want {
+			return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want)
 		}
 	}
+	return nil
 }
 
-func exit(err error) {
-	fmt.Fprintf(os.Stderr, "%v", err)
-	os.Exit(1)
+// readHashes reads a file of newline-delimited hashes.
+func readHashes(f string) ([]common.Hash, error) {
+	b, err := os.ReadFile(f)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open accumulators file")
+	}
+	s := strings.Split(string(b), "\n")
+	// Remove empty last element, if present.
+	if s[len(s)-1] == "" {
+		s = s[:len(s)-1]
+	}
+	// Convert to hashes.
+	r := make([]common.Hash, len(s))
+	for i := range s {
+		r[i] = common.HexToHash(s[i])
+	}
+	return r, nil
+}
+
+// recompute will read each Era file and rewrite it with updated rules (if any).
+func recompute(ctx *cli.Context) error {
+	var (
+		dir     = ctx.String(dirFlag.Name)
+		network = ctx.String(networkFlag.Name)
+		td      = big.NewInt(0)
+	)
+	// iterate over all consecutively available epochs, starting at 0.
+	for i := 0; ; i++ {
+		fmt.Println("epoch", i)
+		fn := func() error {
+			name := path.Join(dir, era.Filename(i, network))
+			f, err := os.OpenFile(name, os.O_RDWR, os.ModePerm)
+			if errors.Is(os.ErrNotExist, err) {
+				return fmt.Errorf("not found")
+			} else if err != nil {
+				return fmt.Errorf("error opening era file %s: %w", name, err)
+			}
+			defer f.Close()
+			var (
+				r        = era.NewReader(f)
+				tds      = make([]*big.Int, 0)
+				blocks   = make([]*types.Block, 0)
+				receipts = make([]types.Receipts, 0)
+			)
+			// Read each block and store in memory.
+			for j := 0; ; j++ {
+				b, r, err := r.Read()
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					return fmt.Errorf("error reading block %d: %w", i*era.MaxEraBatchSize+j, err)
+				}
+				td.Add(td, b.Difficulty())
+				blocks = append(blocks, b)
+				receipts = append(receipts, *r)
+				tds = append(tds, new(big.Int).Set(td))
+			}
+
+			// Delete open Era.
+			f.Truncate(0)
+			f.Seek(0, io.SeekStart)
+
+			// Rewrite Era.
+			b := era.NewBuilder(f)
+			for i := range blocks {
+				err := b.Add(blocks[i], receipts[i], tds[i])
+				if err != nil {
+					return err
+				}
+			}
+			if err := b.Finalize(); err != nil {
+				return err
+			}
+			return nil
+		}
+		if err := fn(); err != nil && err.Error() == "not found" {
+			fmt.Printf("finished at epoch %d\n", i-1)
+			return nil
+		} else if err != nil {
+			return err
+		}
+	}
 }