make leveldb configurable

cmd/XDC/main.go

@@ -89,10 +89,11 @@ var (
 		//utils.LightServFlag,
 		//utils.LightPeersFlag,
 		//utils.LightKDFFlag,
-		//utils.CacheFlag,
-		//utils.CacheDatabaseFlag,
+		utils.CacheFlag,
+		utils.CacheDatabaseFlag,
 		//utils.CacheGCFlag,
 		//utils.TrieCacheGenFlag,
+		utils.FDLimitFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,

cmd/XDC/usage.go

@@ -123,15 +123,16 @@ var AppHelpFlagGroups = []flagGroup{
 	//		utils.TxPoolLifetimeFlag,
 	//	},
 	//},
-	//{
-	//	Name: "PERFORMANCE TUNING",
-	//	Flags: []cli.Flag{
-	//		utils.CacheFlag,
-	//		utils.CacheDatabaseFlag,
-	//		utils.CacheGCFlag,
-	//		utils.TrieCacheGenFlag,
-	//	},
-	//},
+	{
+		Name: "PERFORMANCE TUNING",
+		Flags: []cli.Flag{
+			utils.CacheFlag,
+			utils.CacheDatabaseFlag,
+			// utils.CacheGCFlag,
+			// utils.TrieCacheGenFlag,
+			utils.FDLimitFlag,
+		},
+	},
 	{
 		Name: "ACCOUNT",
 		Flags: []cli.Flag{

cmd/gc/main.go

@@ -52,7 +52,7 @@ type ResultProcessNode struct {
 
 func main() {
 	flag.Parse()
-	db, _ := leveldb.New(*dir, ethconfig.Defaults.DatabaseCache, utils.MakeDatabaseHandles(), "")
+	db, _ := leveldb.New(*dir, ethconfig.Defaults.DatabaseCache, utils.MakeDatabaseHandles(0), "")
 	lddb := rawdb.NewDatabase(db)
 	head := core.GetHeadBlockHash(lddb)
 	currentHeader := core.GetHeader(lddb, head, core.GetBlockNumber(lddb, head))

cmd/utils/flags.go

@@ -20,10 +20,12 @@ package utils
 import (
 	"crypto/ecdsa"
 	"fmt"
+	"math"
 	"math/big"
 	"os"
 	"path/filepath"
 	"runtime"
+	godebug "runtime/debug"
 	"strconv"
 	"strings"
 
@@ -53,6 +55,7 @@ import (
 	"github.com/XinFinOrg/XDPoSChain/p2p/netutil"
 	"github.com/XinFinOrg/XDPoSChain/params"
 	whisper "github.com/XinFinOrg/XDPoSChain/whisper/whisperv6"
+	gopsutil "github.com/shirou/gopsutil/mem"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -313,6 +316,10 @@ var (
 		Usage: "Percentage of cache memory allowance to use for trie pruning",
 		Value: 25,
 	}
+	FDLimitFlag = cli.IntFlag{
+		Name:  "fdlimit",
+		Usage: "Raise the open file descriptor resource limit (default = system fd limit)",
+	}
 	// Miner settings
 	StakingEnabledFlag = cli.BoolFlag{
 		Name:  "mine",
@@ -816,20 +823,29 @@ func setPrefix(ctx *cli.Context, cfg *node.Config) {
 
 // MakeDatabaseHandles raises out the number of allowed file handles per process
 // for XDC and returns half of the allowance to assign to the database.
-func MakeDatabaseHandles() int {
-	limit, err := fdlimit.Current()
+func MakeDatabaseHandles(max int) int {
+	limit, err := fdlimit.Maximum()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if limit < 2048 {
-		if err := fdlimit.Raise(2048); err != nil {
-			Fatalf("Failed to raise file descriptor allowance: %v", err)
-		}
-	}
-	if limit > 2048 { // cap database file descriptors even if more is available
-		limit = 2048
+	switch {
+	case max == 0:
+		// User didn't specify a meaningful value, use system limits
+	case max < 128:
+		// User specified something unhealthy, just use system defaults
+		log.Error("File descriptor limit invalid (<128)", "had", max, "updated", limit)
+	case max > limit:
+		// User requested more than the OS allows, notify that we can't allocate it
+		log.Warn("Requested file descriptors denied by OS", "req", max, "limit", limit)
+	default:
+		// User limit is meaningful and within allowed range, use that
+		limit = max
+	}
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
+		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	return limit / 2 // Leave half for networking and other stuff
+	return int(raised / 2) // Leave half for networking and other stuff
 }
 
 // MakeAddress converts an account specified directly as a hex encoded string or
@@ -1161,6 +1177,26 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setTxPool(ctx, &cfg.TxPool)
 	setEthash(ctx, cfg)
 
+	// Cap the cache allowance and tune the garbage collector
+	mem, err := gopsutil.VirtualMemory()
+	if err == nil {
+		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
+			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
+			mem.Total = 2 * 1024 * 1024 * 1024
+		}
+		allowance := int(mem.Total / 1024 / 1024 / 3)
+		if cache := ctx.Int(CacheFlag.Name); cache > allowance {
+			log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
+			ctx.Set(CacheFlag.Name, strconv.Itoa(allowance))
+		}
+	}
+	// Ensure Go's GC ignores the database cache for trigger percentage
+	cache := ctx.Int(CacheFlag.Name)
+	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
+
+	log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
+	godebug.SetGCPercent(int(gogc))
+
 	switch {
 	case ctx.GlobalIsSet(SyncModeFlag.Name):
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
@@ -1182,7 +1218,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
 		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 	}
-	cfg.DatabaseHandles = MakeDatabaseHandles()
+	cfg.DatabaseHandles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name))
 
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@@ -1271,7 +1307,7 @@ func SetupNetwork(ctx *cli.Context) {
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	var (
 		cache   = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
-		handles = MakeDatabaseHandles()
+		handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name))
 	)
 	name := "chaindata"
 	if ctx.GlobalBool(LightModeFlag.Name) {

common/fdlimit/fdlimit_freebsd.go → common/fdlimit/fdlimit_bsd.go

@@ -14,33 +14,37 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build freebsd
+//go:build freebsd || dragonfly
+// +build freebsd dragonfly
 
 package fdlimit
 
 import "syscall"
 
 // This file is largely identical to fdlimit_unix.go,
-// but Rlimit fields have type int64 on FreeBSD so it needs
+// but Rlimit fields have type int64 on *BSD so it needs
 // an extra conversion.
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
 	if limit.Cur > int64(max) {
 		limit.Cur = int64(max)
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
+	}
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
 	}
-	return nil
+	return uint64(limit.Cur), nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_darwin.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "syscall"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 10240
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	// Retrieve the maximum allowed by dynamic OS limits
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Cap it to OPEN_MAX (10240) because macos is a special snowflake
+	if limit.Max > hardlimit {
+		limit.Max = hardlimit
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_test.go

@@ -17,7 +17,6 @@
 package fdlimit
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -30,13 +29,13 @@ func TestFileDescriptorLimits(t *testing.T) {
 		t.Fatal(err)
 	}
 	if hardlimit < target {
-		t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target))
+		t.Skipf("system limit is less than desired test target: %d < %d", hardlimit, target)
 	}
 
 	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := Raise(uint64(target)); err != nil {
+	if _, err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
 	if limit, err := Current(); err != nil || limit < target {

common/fdlimit/fdlimit_unix.go

@@ -14,29 +14,35 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin netbsd openbsd solaris
+//go:build linux || netbsd || openbsd || solaris
+// +build linux netbsd openbsd solaris
 
 package fdlimit
 
 import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
 	if limit.Cur > max {
 		limit.Cur = max
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
 	}
-	return nil
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_windows.go

@@ -16,28 +16,31 @@
 
 package fdlimit
 
-import "errors"
+import "fmt"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
 	//    changeable from within a running process
 	// This way we can always "request" raising the limits, which will either have
 	// or not have effect based on the platform we're running on.
-	if max > 16384 {
-		return errors.New("file descriptor limit (16384) reached")
+	if max > hardlimit {
+		return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
 	}
-	return nil
+	return max, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this
 // process.
 func Current() (int, error) {
 	// Please see Raise for the reason why we use hard coded 16K as the limit
-	return 16384, nil
+	return hardlimit, nil
 }
 
 // Maximum retrieves the maximum number of file descriptors this process is