From 0b5688eb7648efce2312b53869d7b7633c2f4ceb Mon Sep 17 00:00:00 2001 From: Yuecai Liu <38887641+luky116@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:19:36 +0800 Subject: [PATCH] pika_for_cluster (#1632) * add integrate test --------- Co-authored-by: liuyuecai Co-authored-by: chejinge <945997690@qq.com> Co-authored-by: Xin.Zh --- .clang-tidy | 28 +- .github/workflows/pika.yml | 8 - CMakeLists.txt | 6 +- codis/cmd/admin/dashboard.go | 45 - codis/cmd/fe/assets/index.html | 101 - codis/config/dashboard.toml | 4 + codis/config/proxy.toml | 2 +- .../codis/etc/10.4.10.200_19000/proxy.toml | 2 +- codis/pkg/models/client.go | 6 +- codis/pkg/models/group.go | 26 + codis/pkg/proxy/proxy.go | 104 - codis/pkg/proxy/proxy_api.go | 32 - codis/pkg/topom/config.go | 29 +- codis/pkg/topom/context.go | 20 +- codis/pkg/topom/topom.go | 34 +- codis/pkg/topom/topom_api.go | 102 - codis/pkg/topom/topom_cache.go | 13 + codis/pkg/topom/topom_group.go | 110 +- codis/pkg/topom/topom_proxy.go | 4 - codis/pkg/topom/topom_sentinel.go | 259 +-- codis/pkg/topom/topom_stats.go | 73 +- codis/pkg/topom/topom_test.go | 2 +- codis/pkg/utils/redis/client.go | 95 +- codis/pkg/utils/redis/client_test.go | 63 + codis/pkg/utils/redis/codis_sentinel.go | 144 ++ codis/pkg/utils/redis/sentinel.go | 791 +------- conf/pika.conf | 15 +- include/pika_command.h | 27 +- include/pika_conf.h | 24 + include/pika_consensus.h | 1 + include/pika_db.h | 2 +- include/pika_define.h | 15 +- include/pika_migrate_thread.h | 112 ++ include/pika_rm.h | 1 + include/pika_server.h | 151 +- include/pika_slot.h | 7 +- include/pika_slot_command.h | 280 +++ integrate_test.sh | 42 + src/net/include/net_cli.h | 2 + src/net/include/net_thread.h | 10 +- src/net/src/net_thread.cc | 6 + src/pika.cc | 3 +- src/pika_admin.cc | 14 +- src/pika_bit.cc | 2 + src/pika_command.cc | 64 +- src/pika_conf.cc | 28 +- src/pika_consensus.cc | 4 + src/pika_db.cc | 2 +- src/pika_hash.cc | 6 + src/pika_kv.cc | 32 +- src/pika_list.cc | 10 + src/pika_migrate_thread.cc | 1012 ++++++++++ src/pika_rm.cc | 4 + src/pika_server.cc | 180 +- src/pika_set.cc | 8 +- src/pika_slot.cc | 5 +- src/pika_slot_command.cc | 1697 +++++++++++++++++ src/pika_zset.cc | 5 + .../integration/rpoplpush_replication_test.py | 7 +- 59 files changed, 4416 insertions(+), 1465 deletions(-) create mode 100644 codis/pkg/utils/redis/client_test.go create mode 100644 codis/pkg/utils/redis/codis_sentinel.go create mode 100644 include/pika_migrate_thread.h create mode 100644 include/pika_slot_command.h create mode 100644 integrate_test.sh create mode 100644 src/pika_migrate_thread.cc create mode 100644 src/pika_slot_command.cc diff --git a/.clang-tidy b/.clang-tidy index 50cda6f249..dd9e39d382 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -45,23 +45,21 @@ Checks: ' -clang-diagnostic-ignored-optimization-argument, -readability-implicit-bool-conversion, ' -# naming check -CheckOptions: - - { key: readability-identifier-naming.ClassCase, value: CamelCase } - - { key: readability-identifier-naming.EnumCase, value: CamelCase } - - { key: readability-identifier-naming.FunctionCase, value: CamelCase } - - { key: readability-identifier-naming.GlobalConstantCase, value: UPPER_CASE } - - { key: readability-identifier-naming.MemberCase, value: lower_case } - - { key: readability-identifier-naming.MemberSuffix, value: _ } - - { key: readability-identifier-naming.NamespaceCase, value: lower_case } - - { key: readability-identifier-naming.StructCase, value: CamelCase } - - { key: readability-identifier-naming.UnionCase, value: CamelCase } - - { key: readability-identifier-naming.VariableCase, value: lower_case } - - { key: readability-identifier-naming.GlobalVariableCase, value: lower_case } - - { key: readability-identifier-naming.GlobalVariablePrefix,value: g_pika } +# CheckOptions: +# - { key: readability-identifier-naming.ClassCase, value: CamelCase } +# - { key: readability-identifier-naming.EnumCase, value: CamelCase } +# - { key: readability-identifier-naming.FunctionCase, value: CamelCase } +# - { key: readability-identifier-naming.GlobalConstantCase, value: UPPER_CASE } +# - { key: readability-identifier-naming.MemberCase, value: lower_case } +# - { key: readability-identifier-naming.MemberSuffix, value: _ } +# - { key: readability-identifier-naming.NamespaceCase, value: lower_case } +# - { key: readability-identifier-naming.StructCase, value: CamelCase } +# - { key: readability-identifier-naming.UnionCase, value: CamelCase } +# - { key: readability-identifier-naming.VariableCase, value: lower_case } WarningsAsErrors: '*' -HeaderFilterRegex: '(src/net/..|src/storage/include|src/pstd/..|src/pstd/include|src/net/..)' +# HeaderFilterRegex: '(|/src|/src/net|/src/pstd|/src/storage)/include' +# HeaderFilterRegex: '/src/(net|storage|pstd)/include' AnalyzeTemporaryDtors: true #### Disabled checks and why: ##### diff --git a/.github/workflows/pika.yml b/.github/workflows/pika.yml index dbeb33a467..ef0f0bcf09 100644 --- a/.github/workflows/pika.yml +++ b/.github/workflows/pika.yml @@ -47,10 +47,6 @@ jobs: # Build your program with the given configuration run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Clang-tidy Check - working-directory: ${{github.workspace}}/build - run: make clang-tidy - - name: Test working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. @@ -125,10 +121,6 @@ jobs: source /opt/rh/devtoolset-10/enable cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Clang-tidy Check - working-directory: ${{github.workspace}}/build - run: make clang-tidy - - name: Test # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eb35fcee7..6dc2becbf5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,8 +97,8 @@ if (${AUTOCONF} MATCHES AUTOCONF-NOTFOUND) message(FATAL_ERROR "not find autoconf on localhost") endif() -set(CLANG_SEARCH_PATH "/usr/local/bin" "/usr/bin" "/usr/local/opt/llvm/bin" - "/usr/local/opt/llvm@12/bin") +#set(CLANG_SEARCH_PATH "/usr/local/bin" "/usr/bin" "/usr/local/opt/llvm/bin" +# "/usr/local/opt/llvm@12/bin") find_program(CLANG_TIDY_BIN NAMES clang-tidy clang-tidy-12 HINTS ${CLANG_SEARCH_PATH}) @@ -784,6 +784,7 @@ add_custom_target( clang-tidy COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/run_clang_tidy.py -clang-tidy-binary ${CLANG_TIDY_BIN} + -header-filter='${PROJECT_SOURCE_DIR}(/include|/tools|/src)/.*' -p ${CMAKE_BINARY_DIR} -quiet ) @@ -792,6 +793,7 @@ add_custom_target(clang-tidy-fix ${CMAKE_CURRENT_SOURCE_DIR}/run_clang_tidy.py -clang-tidy-binary ${CLANG_TIDY_BIN} -p ${CMAKE_BINARY_DIR} + -header-filter='${PROJECT_SOURCE_DIR}(/include|/tools|/src)/.*' -clang-apply-replacements-binary ${CLANG_APPLY_REPLACEMENTS_BIN} -fix ) \ No newline at end of file diff --git a/codis/cmd/admin/dashboard.go b/codis/cmd/admin/dashboard.go index ed0c3a40b3..86cea7ad26 100644 --- a/codis/cmd/admin/dashboard.go +++ b/codis/cmd/admin/dashboard.go @@ -68,13 +68,6 @@ func (t *cmdDashboard) Main(d map[string]interface{}) { case d["--promote-server"].(bool): t.handleGroupCommand(d) - case d["--sentinel-add"].(bool): - fallthrough - case d["--sentinel-del"].(bool): - fallthrough - case d["--sentinel-resync"].(bool): - t.handleSentinelCommand(d) - case d["--sync-action"].(bool): t.handleSyncActionCommand(d) @@ -632,44 +625,6 @@ func (t *cmdDashboard) handleGroupCommand(d map[string]interface{}) { } } -func (t *cmdDashboard) handleSentinelCommand(d map[string]interface{}) { - c := t.newTopomClient() - - switch { - - case d["--sentinel-add"].(bool): - - addr := utils.ArgumentMust(d, "--addr") - - log.Debugf("call rpc add-sentinel to dashboard %s", t.addr) - if err := c.AddSentinel(addr); err != nil { - log.PanicErrorf(err, "call rpc add-sentinel to dashboard %s failed", t.addr) - } - log.Debugf("call rpc add-sentinel OK") - - case d["--sentinel-del"].(bool): - - addr := utils.ArgumentMust(d, "--addr") - - force := d["--force"].(bool) - - log.Debugf("call rpc del-sentinel to dashboard %s", t.addr) - if err := c.DelSentinel(addr, force); err != nil { - log.PanicErrorf(err, "call rpc del-sentinel to dashboard %s failed", t.addr) - } - log.Debugf("call rpc del-sentinel OK") - - case d["--sentinel-resync"].(bool): - - log.Debugf("call rpc resync-sentinels to dashboard %s", t.addr) - if err := c.ResyncSentinels(); err != nil { - log.PanicErrorf(err, "call rpc resync-sentinels to dashboard %s failed", t.addr) - } - log.Debugf("call rpc resync-sentinels OK") - - } -} - func (t *cmdDashboard) handleSyncActionCommand(d map[string]interface{}) { c := t.newTopomClient() diff --git a/codis/cmd/fe/assets/index.html b/codis/cmd/fe/assets/index.html index f2426cf917..8da3fbacf5 100644 --- a/codis/cmd/fe/assets/index.html +++ b/codis/cmd/fe/assets/index.html @@ -654,107 +654,6 @@

Group

- -
-
-
-

Sentinels

-
-
-
-
- - - - -
-
-
- - - - - - - - - - - - - - - - - -
- - - - - - - - - - Sentinels - - (OUT OF SYNC) - - - - Status - - -
- WATCHED - - S - - [[sentinel.server]] - - - - - [[sentinel.status]] - [[sentinel.status]] - [[sentinel.status]] - [[sentinel.status_text]] - - [[sentinel.runid_error]] - - - -
-
- -
diff --git a/codis/config/dashboard.toml b/codis/config/dashboard.toml index f0de868c62..7a806fe4c1 100644 --- a/codis/config/dashboard.toml +++ b/codis/config/dashboard.toml @@ -30,6 +30,10 @@ migration_async_numkeys = 500 migration_timeout = "30s" # Set configs for redis sentinel. +sentinel_check_server_state_interval = "5s" +sentinel_check_master_failover_interval = "1s" +sentinel_master_dead_check_times = 5 + sentinel_client_timeout = "10s" sentinel_quorum = 2 sentinel_parallel_syncs = 1 diff --git a/codis/config/proxy.toml b/codis/config/proxy.toml index ddbc4efe89..ee3d19f199 100644 --- a/codis/config/proxy.toml +++ b/codis/config/proxy.toml @@ -75,7 +75,7 @@ backend_replica_parallel = 1 backend_keepalive_period = "75s" # Set number of databases of backend. -backend_number_databases = 16 +backend_number_databases = 6 # If there is no request from client for a long time, the connection will be closed. (0 to disable) # Set session recv buffer size & timeout. diff --git a/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml b/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml index 4219d525f3..853136c2ee 100644 --- a/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml +++ b/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml @@ -58,7 +58,7 @@ backend_replica_parallel = 1 backend_keepalive_period = "75s" # Set number of databases of backend. -backend_number_databases = 16 +backend_number_databases = 6 # If there is no request from client for a long time, the connection will be closed. (0 to disable) # Set session recv buffer size & timeout. diff --git a/codis/pkg/models/client.go b/codis/pkg/models/client.go index 4bcde5bfca..44ef2318a8 100644 --- a/codis/pkg/models/client.go +++ b/codis/pkg/models/client.go @@ -6,9 +6,9 @@ package models import ( "time" - "pika/codis/v2/pkg/models/etcd" - "pika/codis/v2/pkg/models/fs" - "pika/codis/v2/pkg/models/zk" + etcdclient "pika/codis/v2/pkg/models/etcd" + fsclient "pika/codis/v2/pkg/models/fs" + zkclient "pika/codis/v2/pkg/models/zk" "pika/codis/v2/pkg/utils/errors" ) diff --git a/codis/pkg/models/group.go b/codis/pkg/models/group.go index 0d105a9fb8..11d6e7bf52 100644 --- a/codis/pkg/models/group.go +++ b/codis/pkg/models/group.go @@ -17,6 +17,22 @@ type Group struct { OutOfSync bool `json:"out_of_sync"` } +func (g *Group) GetServersMap() map[string]*GroupServer { + results := make(map[string]*GroupServer) + for _, server := range g.Servers { + results[server.Addr] = server + } + return results +} + +type GroupServerState int8 + +const ( + GroupServerStateNormal GroupServerState = iota + GroupServerStateSubjectiveOffline + GroupServerStateOffline +) + type GroupServer struct { Addr string `json:"server"` DataCenter string `json:"datacenter"` @@ -26,6 +42,16 @@ type GroupServer struct { State string `json:"state,omitempty"` } `json:"action"` + // master or slave + Role string `json:"role"` + // If it is a master node, take the master_repl_offset field, otherwise take the slave_repl_offset field + ReplyOffset int `json:"reply_offset"` + // Monitoring status, 0 normal, 1 subjective offline, 2 actual offline + // If marked as 2 , no service is provided + State GroupServerState `json:"state"` + + ReCallTimes int8 `json:"recall_times"` + ReplicaGroup bool `json:"replica_group"` } diff --git a/codis/pkg/proxy/proxy.go b/codis/pkg/proxy/proxy.go index a7bb1ccd61..688b4a197c 100644 --- a/codis/pkg/proxy/proxy.go +++ b/codis/pkg/proxy/proxy.go @@ -11,7 +11,6 @@ import ( "os/exec" "path/filepath" "runtime" - "strconv" "strings" "sync" "time" @@ -21,7 +20,6 @@ import ( "pika/codis/v2/pkg/utils/errors" "pika/codis/v2/pkg/utils/log" "pika/codis/v2/pkg/utils/math2" - "pika/codis/v2/pkg/utils/redis" "pika/codis/v2/pkg/utils/rpc" "pika/codis/v2/pkg/utils/unsafe2" ) @@ -46,7 +44,6 @@ type Proxy struct { ladmin net.Listener ha struct { - monitor *redis.Sentinel masters map[int]string servers []string } @@ -195,9 +192,6 @@ func (s *Proxy) Close() error { if s.router != nil { s.router.Close() } - if s.ha.monitor != nil { - s.ha.monitor.Cancel() - } return nil } @@ -283,92 +277,6 @@ func (s *Proxy) GetSentinels() ([]string, map[int]string) { return s.ha.servers, s.ha.masters } -func (s *Proxy) SetSentinels(servers []string) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return ErrClosedProxy - } - s.ha.servers = servers - log.Warnf("[%p] set sentinels = %v", s, s.ha.servers) - - s.rewatchSentinels(s.ha.servers) - return nil -} - -func (s *Proxy) RewatchSentinels() error { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return ErrClosedProxy - } - log.Warnf("[%p] rewatch sentinels = %v", s, s.ha.servers) - - s.rewatchSentinels(s.ha.servers) - return nil -} - -func (s *Proxy) rewatchSentinels(servers []string) { - if s.ha.monitor != nil { - s.ha.monitor.Cancel() - s.ha.monitor = nil - s.ha.masters = nil - } - if len(servers) != 0 { - s.ha.monitor = redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - s.ha.monitor.LogFunc = log.Warnf - s.ha.monitor.ErrFunc = log.WarnErrorf - go func(p *redis.Sentinel) { - var trigger = make(chan struct{}, 1) - delayUntil := func(deadline time.Time) { - for !p.IsCanceled() { - var d = deadline.Sub(time.Now()) - if d <= 0 { - return - } - time.Sleep(math2.MinDuration(d, time.Second)) - } - } - go func() { - defer close(trigger) - callback := func() { - select { - case trigger <- struct{}{}: - default: - } - } - for !p.IsCanceled() { - timeout := time.Minute * 15 - retryAt := time.Now().Add(time.Second * 10) - if !p.Subscribe(servers, timeout, callback) { - delayUntil(retryAt) - } else { - callback() - } - } - }() - go func() { - for range trigger { - var success int - for i := 0; i != 10 && !p.IsCanceled() && success != 2; i++ { - timeout := time.Second * 5 - masters, err := p.Masters(servers, timeout) - if err != nil { - log.WarnErrorf(err, "[%p] fetch group masters failed", s) - } else { - if !p.IsCanceled() { - s.SwitchMasters(masters) - } - success += 1 - } - delayUntil(time.Now().Add(time.Second * 5)) - } - } - }() - }(s.ha.monitor) - } -} - func (s *Proxy) serveAdmin() { if s.IsClosed() { return @@ -568,18 +476,6 @@ func (s *Proxy) Stats(flags StatsFlags) *Stats { stats.Online = s.IsOnline() stats.Closed = s.IsClosed() - servers, masters := s.GetSentinels() - if servers != nil { - stats.Sentinels.Servers = servers - } - if masters != nil { - stats.Sentinels.Masters = make(map[string]string) - for gid, addr := range masters { - stats.Sentinels.Masters[strconv.Itoa(gid)] = addr - } - } - stats.Sentinels.Switched = s.HasSwitched() - stats.Ops.Total = OpTotal() stats.Ops.Fails = OpFails() stats.Ops.Redis.Errors = OpRedisErrors() diff --git a/codis/pkg/proxy/proxy_api.go b/codis/pkg/proxy/proxy_api.go index a052aa1c3f..f365b1811b 100644 --- a/codis/pkg/proxy/proxy_api.go +++ b/codis/pkg/proxy/proxy_api.go @@ -78,8 +78,6 @@ func newApiServer(p *Proxy) http.Handler { r.Put("/shutdown/:xauth", api.Shutdown) r.Put("/loglevel/:xauth/:value", api.LogLevel) r.Put("/fillslots/:xauth", binding.Json([]*models.Slot{}), api.FillSlots) - r.Put("/sentinels/:xauth", binding.Json(models.Sentinel{}), api.SetSentinels) - r.Put("/sentinels/:xauth/rewatch", api.RewatchSentinels) }) m.MapTo(r, (*martini.Routes)(nil)) @@ -215,26 +213,6 @@ func (s *apiServer) FillSlots(slots []*models.Slot, params martini.Params) (int, return rpc.ApiResponseJson("OK") } -func (s *apiServer) SetSentinels(sentinel models.Sentinel, params martini.Params) (int, string) { - if err := s.verifyXAuth(params); err != nil { - return rpc.ApiResponseError(err) - } - if err := s.proxy.SetSentinels(sentinel.Servers); err != nil { - return rpc.ApiResponseError(err) - } - return rpc.ApiResponseJson("OK") -} - -func (s *apiServer) RewatchSentinels(params martini.Params) (int, string) { - if err := s.verifyXAuth(params); err != nil { - return rpc.ApiResponseError(err) - } - if err := s.proxy.RewatchSentinels(); err != nil { - return rpc.ApiResponseError(err) - } - return rpc.ApiResponseJson("OK") -} - type ApiClient struct { addr string xauth string @@ -331,13 +309,3 @@ func (c *ApiClient) FillSlots(slots ...*models.Slot) error { url := c.encodeURL("/api/proxy/fillslots/%s", c.xauth) return rpc.ApiPutJson(url, slots, nil) } - -func (c *ApiClient) SetSentinels(sentinel *models.Sentinel) error { - url := c.encodeURL("/api/proxy/sentinels/%s", c.xauth) - return rpc.ApiPutJson(url, sentinel, nil) -} - -func (c *ApiClient) RewatchSentinels() error { - url := c.encodeURL("/api/proxy/sentinels/%s/rewatch", c.xauth) - return rpc.ApiPutJson(url, nil, nil) -} diff --git a/codis/pkg/topom/config.go b/codis/pkg/topom/config.go index e9790ad1e7..b6c36f21be 100644 --- a/codis/pkg/topom/config.go +++ b/codis/pkg/topom/config.go @@ -47,6 +47,9 @@ migration_async_numkeys = 500 migration_timeout = "30s" # Set configs for redis sentinel. +sentinel_check_server_state_interval = "5s" +sentinel_check_master_failover_interval = "1s" +sentinel_master_dead_check_times = 5 sentinel_client_timeout = "10s" sentinel_quorum = 2 sentinel_parallel_syncs = 1 @@ -75,13 +78,16 @@ type Config struct { MigrationAsyncNumKeys int `toml:"migration_async_numkeys" json:"migration_async_numkeys"` MigrationTimeout timesize.Duration `toml:"migration_timeout" json:"migration_timeout"` - SentinelClientTimeout timesize.Duration `toml:"sentinel_client_timeout" json:"sentinel_client_timeout"` - SentinelQuorum int `toml:"sentinel_quorum" json:"sentinel_quorum"` - SentinelParallelSyncs int `toml:"sentinel_parallel_syncs" json:"sentinel_parallel_syncs"` - SentinelDownAfter timesize.Duration `toml:"sentinel_down_after" json:"sentinel_down_after"` - SentinelFailoverTimeout timesize.Duration `toml:"sentinel_failover_timeout" json:"sentinel_failover_timeout"` - SentinelNotificationScript string `toml:"sentinel_notification_script" json:"sentinel_notification_script"` - SentinelClientReconfigScript string `toml:"sentinel_client_reconfig_script" json:"sentinel_client_reconfig_script"` + SentinelCheckServerStateInterval timesize.Duration `toml:"sentinel_check_server_state_interval" json:"sentinel_client_timeout"` + SentinelCheckMasterFailoverInterval timesize.Duration `toml:"sentinel_check_master_failover_interval" json:"sentinel_check_master_failover_interval"` + SentinelMasterDeadCheckTimes int8 `toml:"sentinel_master_dead_check_times" json:"sentinel_master_dead_check_times"` + SentinelClientTimeout timesize.Duration `toml:"sentinel_client_timeout" json:"sentinel_client_timeout"` + SentinelQuorum int `toml:"sentinel_quorum" json:"sentinel_quorum"` + SentinelParallelSyncs int `toml:"sentinel_parallel_syncs" json:"sentinel_parallel_syncs"` + SentinelDownAfter timesize.Duration `toml:"sentinel_down_after" json:"sentinel_down_after"` + SentinelFailoverTimeout timesize.Duration `toml:"sentinel_failover_timeout" json:"sentinel_failover_timeout"` + SentinelNotificationScript string `toml:"sentinel_notification_script" json:"sentinel_notification_script"` + SentinelClientReconfigScript string `toml:"sentinel_client_reconfig_script" json:"sentinel_client_reconfig_script"` } func NewDefaultConfig() *Config { @@ -145,6 +151,15 @@ func (c *Config) Validate() error { if c.SentinelClientTimeout <= 0 { return errors.New("invalid sentinel_client_timeout") } + if c.SentinelCheckServerStateInterval <= 0 { + return errors.New("invalid sentinel_check_server_state_interval") + } + if c.SentinelCheckMasterFailoverInterval <= 0 { + return errors.New("invalid sentinel_check_master_failover_interval") + } + if c.SentinelMasterDeadCheckTimes <= 0 { + return errors.New("invalid sentinel_master_dead_check_times") + } if c.SentinelQuorum <= 0 { return errors.New("invalid sentinel_quorum") } diff --git a/codis/pkg/topom/context.go b/codis/pkg/topom/context.go index d31aa1d778..0426100bbd 100644 --- a/codis/pkg/topom/context.go +++ b/codis/pkg/topom/context.go @@ -153,7 +153,7 @@ func (ctx *context) toReplicaGroups(gid int, p *models.Proxy) [][]string { } var groups [3][]string for _, s := range g.Servers { - if s.ReplicaGroup { + if s.ReplicaGroup && s.State == models.GroupServerStateNormal { p := getPriority(s) groups[p] = append(groups[p], s.Addr) } @@ -182,6 +182,24 @@ func (ctx *context) getGroup(gid int) (*models.Group, error) { return nil, errors.Errorf("group-[%d] doesn't exist", gid) } +func (ctx *context) getGroups() []*models.Group { + gs := make([]*models.Group, 0, len(ctx.group)) + for _, g := range ctx.group { + gs = append(gs, g) + } + return gs +} + +func (ctx *context) getGroupServers() map[int][]*models.GroupServer { + groupServers := make(map[int][]*models.GroupServer) + + for gid, group := range ctx.group { + groupServers[gid] = group.Servers + } + + return groupServers +} + func (ctx *context) getGroupIndex(g *models.Group, addr string) (int, error) { for i, x := range g.Servers { if x.Addr == addr { diff --git a/codis/pkg/topom/topom.go b/codis/pkg/topom/topom.go index 699040b298..582452aa1d 100644 --- a/codis/pkg/topom/topom.go +++ b/codis/pkg/topom/topom.go @@ -72,7 +72,7 @@ type Topom struct { ha struct { redisp *redis.Pool - monitor *redis.Sentinel + monitor *redis.CodisSentinel masters map[int]string } } @@ -195,11 +195,33 @@ func (s *Topom) Start(routines bool) error { if !routines { return nil } - ctx, err := s.newContext() - if err != nil { - return err - } - s.rewatchSentinels(ctx.sentinel.Servers) + + // Check the status of all masters and slaves every 5 seconds + go func() { + for !s.IsClosed() { + if s.IsOnline() { + w, _ := s.CheckMastersAndSlavesState(10 * time.Second) + if w != nil { + w.Wait() + } + } + time.Sleep(s.Config().SentinelCheckServerStateInterval.Duration()) + } + }() + + // Check the status of the pre-offline master every 1 second + // to determine whether to automatically switch master and slave + go func() { + for !s.IsClosed() { + if s.IsOnline() { + w, _ := s.CheckPreOffineMastersState(5 * time.Second) + if w != nil { + w.Wait() + } + } + time.Sleep(s.Config().SentinelCheckMasterFailoverInterval.Duration()) + } + }() go func() { for !s.IsClosed() { diff --git a/codis/pkg/topom/topom_api.go b/codis/pkg/topom/topom_api.go index aae33f4731..6b8d9cc0e3 100644 --- a/codis/pkg/topom/topom_api.go +++ b/codis/pkg/topom/topom_api.go @@ -113,13 +113,6 @@ func newApiServer(t *Topom) http.Handler { r.Put("/assign/:xauth/offline", binding.Json([]*models.SlotMapping{}), api.SlotsAssignOffline) r.Put("/rebalance/:xauth/:confirm", api.SlotsRebalance) }) - r.Group("/sentinels", func(r martini.Router) { - r.Put("/add/:xauth/:addr", api.AddSentinel) - r.Put("/del/:xauth/:addr/:force", api.DelSentinel) - r.Put("/resync-all/:xauth", api.ResyncSentinels) - r.Get("/info/:addr", api.InfoSentinel) - r.Get("/info/:addr/monitored", api.InfoSentinelMonitored) - }) }) m.MapTo(r, (*martini.Routes)(nil)) @@ -459,51 +452,6 @@ func (s *apiServer) EnableReplicaGroupsAll(params martini.Params) (int, string) } } -func (s *apiServer) AddSentinel(params martini.Params) (int, string) { - if err := s.verifyXAuth(params); err != nil { - return rpc.ApiResponseError(err) - } - addr, err := s.parseAddr(params) - if err != nil { - return rpc.ApiResponseError(err) - } - if err := s.topom.AddSentinel(addr); err != nil { - return rpc.ApiResponseError(err) - } else { - return rpc.ApiResponseJson("OK") - } -} - -func (s *apiServer) DelSentinel(params martini.Params) (int, string) { - if err := s.verifyXAuth(params); err != nil { - return rpc.ApiResponseError(err) - } - addr, err := s.parseAddr(params) - if err != nil { - return rpc.ApiResponseError(err) - } - force, err := s.parseInteger(params, "force") - if err != nil { - return rpc.ApiResponseError(err) - } - if err := s.topom.DelSentinel(addr, force != 0); err != nil { - return rpc.ApiResponseError(err) - } else { - return rpc.ApiResponseJson("OK") - } -} - -func (s *apiServer) ResyncSentinels(params martini.Params) (int, string) { - if err := s.verifyXAuth(params); err != nil { - return rpc.ApiResponseError(err) - } - if err := s.topom.ResyncSentinels(); err != nil { - return rpc.ApiResponseError(err) - } else { - return rpc.ApiResponseJson("OK") - } -} - func (s *apiServer) InfoServer(params martini.Params) (int, string) { addr, err := s.parseAddr(params) if err != nil { @@ -522,37 +470,6 @@ func (s *apiServer) InfoServer(params martini.Params) (int, string) { } } -func (s *apiServer) InfoSentinel(params martini.Params) (int, string) { - addr, err := s.parseAddr(params) - if err != nil { - return rpc.ApiResponseError(err) - } - c, err := redis.NewClientNoAuth(addr, time.Second) - if err != nil { - log.WarnErrorf(err, "create redis client to %s failed", addr) - return rpc.ApiResponseError(err) - } - defer c.Close() - if info, err := c.Info(); err != nil { - return rpc.ApiResponseError(err) - } else { - return rpc.ApiResponseJson(info) - } -} - -func (s *apiServer) InfoSentinelMonitored(params martini.Params) (int, string) { - addr, err := s.parseAddr(params) - if err != nil { - return rpc.ApiResponseError(err) - } - sentinel := redis.NewSentinel(s.topom.Config().ProductName, s.topom.Config().ProductAuth) - if info, err := sentinel.MastersAndSlaves(addr, s.topom.Config().SentinelClientTimeout.Duration()); err != nil { - return rpc.ApiResponseError(err) - } else { - return rpc.ApiResponseJson(info) - } -} - func (s *apiServer) SyncCreateAction(params martini.Params) (int, string) { if err := s.verifyXAuth(params); err != nil { return rpc.ApiResponseError(err) @@ -910,25 +827,6 @@ func (c *ApiClient) EnableReplicaGroupsAll(value bool) error { return rpc.ApiPutJson(url, nil, nil) } -func (c *ApiClient) AddSentinel(addr string) error { - url := c.encodeURL("/api/topom/sentinels/add/%s/%s", c.xauth, addr) - return rpc.ApiPutJson(url, nil, nil) -} - -func (c *ApiClient) DelSentinel(addr string, force bool) error { - var value int - if force { - value = 1 - } - url := c.encodeURL("/api/topom/sentinels/del/%s/%s/%d", c.xauth, addr, value) - return rpc.ApiPutJson(url, nil, nil) -} - -func (c *ApiClient) ResyncSentinels() error { - url := c.encodeURL("/api/topom/sentinels/resync-all/%s", c.xauth) - return rpc.ApiPutJson(url, nil, nil) -} - func (c *ApiClient) SyncCreateAction(addr string) error { url := c.encodeURL("/api/topom/group/action/create/%s/%s", c.xauth, addr) return rpc.ApiPutJson(url, nil, nil) diff --git a/codis/pkg/topom/topom_cache.go b/codis/pkg/topom/topom_cache.go index db28bcfbd8..3a00bd5630 100644 --- a/codis/pkg/topom/topom_cache.go +++ b/codis/pkg/topom/topom_cache.go @@ -25,6 +25,12 @@ func (s *Topom) dirtyGroupCache(gid int) { }) } +func (s *Topom) dirtyGroupsCache() { + s.cache.hooks.PushBack(func() { + s.cache.group = nil + }) +} + func (s *Topom) dirtyProxyCache(token string) { s.cache.hooks.PushBack(func() { if s.cache.proxy != nil { @@ -184,6 +190,13 @@ func (s *Topom) storeUpdateGroup(g *models.Group) error { return nil } +func (s *Topom) storeUpdateGroups(gs []*models.Group) error { + for _, g := range gs { + s.storeUpdateGroup(g) + } + return nil +} + func (s *Topom) storeRemoveGroup(g *models.Group) error { log.Warnf("remove group-[%d]:\n%s", g.Id, g.Encode()) if err := s.store.DeleteGroup(g.Id); err != nil { diff --git a/codis/pkg/topom/topom_group.go b/codis/pkg/topom/topom_group.go index 92d412ce78..46a53c417f 100644 --- a/codis/pkg/topom/topom_group.go +++ b/codis/pkg/topom/topom_group.go @@ -4,6 +4,7 @@ package topom import ( + "encoding/json" "time" "pika/codis/v2/pkg/models" @@ -273,11 +274,6 @@ func (s *Topom) GroupPromoteServer(gid int, addr string) error { if err := s.storeUpdateSentinel(p); err != nil { return err } - groupIds := map[int]bool{g.Id: true} - sentinel := redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - if err := sentinel.RemoveGroups(p.Servers, s.config.SentinelClientTimeout.Duration(), groupIds); err != nil { - log.WarnErrorf(err, "group-[%d] remove sentinels failed", g.Id) - } if s.ha.masters != nil { delete(s.ha.masters, gid) } @@ -307,16 +303,17 @@ func (s *Topom) GroupPromoteServer(gid int, addr string) error { return err } - var master = slice[0].Addr - if c, err := redis.NewClient(master, s.config.ProductAuth, time.Second); err != nil { + var ( + master = slice[0].Addr + client *redis.Client + ) + if client, err = redis.NewClient(master, s.config.ProductAuth, time.Second); err != nil { log.WarnErrorf(err, "create redis client to %s failed", master) - } else { - defer c.Close() - if err := c.SetMaster("NO:ONE"); err != nil { - log.WarnErrorf(err, "redis %s set master to NO:ONE failed", master) - } } - + defer client.Close() + if err = client.SetMaster("NO:ONE"); err != nil { + log.WarnErrorf(err, "redis %s set master to NO:ONE failed", master) + } fallthrough case models.ActionFinished: @@ -344,7 +341,61 @@ func (s *Topom) GroupPromoteServer(gid int, addr string) error { } } -func (s *Topom) trySwitchGroupMaster(gid int, master string, cache *redis.InfoCache) error { +func (s *Topom) trySwitchGroupMaster(gid int, cache *redis.InfoCache) error { + ctx, err := s.newContext() + if err != nil { + return err + } + g, err := ctx.getGroup(gid) + if err != nil { + return err + } + + master := s.selectNextMaster(g.Servers) + + if master == "" { + servers, _ := json.Marshal(g) + log.Errorf("group %d donn't has any slaves to switch master, %s", gid, servers) + return errors.Errorf("cann't switch slave to master") + } + + return s.doSwitchGroupMaster(gid, master, cache) +} + +// Choose to change to the next master node in the group +func (s *Topom) selectNextMaster(servers []*models.GroupServer) string { + if len(servers) == 0 { + return "" + } + + var masterServer *models.GroupServer + + for _, server := range servers { + if server.State != models.GroupServerStateNormal { + continue + } + + // If there is already a master node in the group working normally, return directly + if server.Role == "master" { + return server.Addr + } + + if masterServer == nil { + masterServer = server + } else if server.ReplyOffset > masterServer.ReplyOffset { + // Select the slave node with the latest offset as the master node + masterServer = server + } + } + + if masterServer == nil { + return "" + } + + return masterServer.Addr +} + +func (s *Topom) doSwitchGroupMaster(gid int, master string, cache *redis.InfoCache) error { ctx, err := s.newContext() if err != nil { return err @@ -379,7 +430,38 @@ func (s *Topom) trySwitchGroupMaster(gid int, master string, cache *redis.InfoCa log.Warnf("group-[%d] will switch master to server[%d] = %s", g.Id, index, g.Servers[index].Addr) + // Set the slave node as the new master node + var client *redis.Client + if client, err = redis.NewClient(master, s.config.ProductAuth, 100*time.Millisecond); err != nil { + log.WarnErrorf(err, "create redis client to %s failed", master) + return err + } + + defer client.Close() + if err = client.SetMaster("NO:ONE"); err != nil { + log.WarnErrorf(err, "redis %s set master to NO:ONE failed", master) + return err + } + + // Set other nodes in the group as slave nodes of the new master node + for _, server := range g.Servers { + if server.State != models.GroupServerStateNormal || server.Addr == master { + continue + } + var client2 *redis.Client + if client2, err = redis.NewClient(server.Addr, s.config.ProductAuth, 100*time.Millisecond); err != nil { + log.WarnErrorf(err, "create redis client to %s failed", master) + return err + } + defer client2.Close() + if err = client2.SetMaster(master); err != nil { + log.WarnErrorf(err, "redis %s set master to %s failed", server.Addr, master) + return err + } + } + g.Servers[0], g.Servers[index] = g.Servers[index], g.Servers[0] + g.Servers[0].Role = "master" g.OutOfSync = true return s.storeUpdateGroup(g) } diff --git a/codis/pkg/topom/topom_proxy.go b/codis/pkg/topom/topom_proxy.go index 4af5db1849..bf0f3f4b3a 100644 --- a/codis/pkg/topom/topom_proxy.go +++ b/codis/pkg/topom/topom_proxy.go @@ -133,10 +133,6 @@ func (s *Topom) reinitProxy(ctx *context, p *models.Proxy, c *proxy.ApiClient) e log.ErrorErrorf(err, "proxy-[%s] start failed", p.Token) return errors.Errorf("proxy-[%s] start failed", p.Token) } - if err := c.SetSentinels(ctx.sentinel); err != nil { - log.ErrorErrorf(err, "proxy-[%s] set sentinels failed", p.Token) - return errors.Errorf("proxy-[%s] set sentinels failed", p.Token) - } return nil } diff --git a/codis/pkg/topom/topom_sentinel.go b/codis/pkg/topom/topom_sentinel.go index 5b70b30d43..3b861e5a13 100644 --- a/codis/pkg/topom/topom_sentinel.go +++ b/codis/pkg/topom/topom_sentinel.go @@ -7,14 +7,11 @@ import ( "time" "pika/codis/v2/pkg/models" - "pika/codis/v2/pkg/utils/errors" "pika/codis/v2/pkg/utils/log" - "pika/codis/v2/pkg/utils/math2" "pika/codis/v2/pkg/utils/redis" - "pika/codis/v2/pkg/utils/sync2" ) -func (s *Topom) AddSentinel(addr string) error { +func (s *Topom) CheckAndSwitchSlavesAndMasters(filter func(index int, g *models.GroupServer) bool) error { s.mu.Lock() defer s.mu.Unlock() ctx, err := s.newContext() @@ -22,207 +19,109 @@ func (s *Topom) AddSentinel(addr string) error { return err } - if addr == "" { - return errors.Errorf("invalid sentinel address") + config := &redis.MonitorConfig{ + Quorum: s.config.SentinelQuorum, + ParallelSyncs: s.config.SentinelParallelSyncs, + DownAfter: s.config.SentinelDownAfter.Duration(), + FailoverTimeout: s.config.SentinelFailoverTimeout.Duration(), + NotificationScript: s.config.SentinelNotificationScript, + ClientReconfigScript: s.config.SentinelClientReconfigScript, } - p := ctx.sentinel - for _, x := range p.Servers { - if x == addr { - return errors.Errorf("sentinel-[%s] already exists", addr) + sentinel := redis.NewCodisSentinel(s.config.ProductName, s.config.ProductAuth) + gs := make(map[int][]*models.GroupServer) + for gid, servers := range ctx.getGroupServers() { + for i, server := range servers { + if filter(i, server) { + if val, ok := gs[gid]; ok { + gs[gid] = append(val, server) + } else { + gs[gid] = []*models.GroupServer{server} + } + } } } - - sentinel := redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - if err := sentinel.FlushConfig(addr, s.config.SentinelClientTimeout.Duration()); err != nil { - return err + if len(gs) == 0 { + return nil } - defer s.dirtySentinelCache() - - p.Servers = append(p.Servers, addr) - p.OutOfSync = true - return s.storeUpdateSentinel(p) -} -func (s *Topom) DelSentinel(addr string, force bool) error { - s.mu.Lock() - defer s.mu.Unlock() - ctx, err := s.newContext() - if err != nil { - return err - } + states := sentinel.RefreshMastersAndSlavesClient(config.ParallelSyncs, gs) - if addr == "" { - return errors.Errorf("invalid sentinel address") - } - p := ctx.sentinel + var pending []*models.Group - var slice []string - for _, x := range p.Servers { - if x != addr { - slice = append(slice, x) + for _, state := range states { + var g *models.Group + if g, err = ctx.getGroup(state.GroupID); err != nil { + return err } - } - if len(slice) == len(p.Servers) { - return errors.Errorf("sentinel-[%s] not found", addr) - } - defer s.dirtySentinelCache() - p.OutOfSync = true - if err := s.storeUpdateSentinel(p); err != nil { - return err - } - - sentinel := redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - if err := sentinel.RemoveGroupsAll([]string{addr}, s.config.SentinelClientTimeout.Duration()); err != nil { - log.WarnErrorf(err, "remove sentinel %s failed", addr) - if !force { - return errors.Errorf("remove sentinel %s failed", addr) + serversMap := g.GetServersMap() + if len(serversMap) == 0 { + continue } - } - p.Servers = slice - return s.storeUpdateSentinel(p) -} - -func (s *Topom) SwitchMasters(masters map[int]string) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return ErrClosedTopom - } - s.ha.masters = masters - - if len(masters) != 0 { - cache := &redis.InfoCache{ - Auth: s.config.ProductAuth, Timeout: time.Millisecond * 100, - } - for gid, master := range masters { - if err := s.trySwitchGroupMaster(gid, master, cache); err != nil { - log.WarnErrorf(err, "sentinel switch group master failed") + // It was the master node before, the master node hangs up, and it is currently the master node + if state.Index == 0 && state.Err != nil && g.Servers[0].Addr == state.Addr { + if g.Servers[0].State == models.GroupServerStateNormal { + g.Servers[0].State = models.GroupServerStateSubjectiveOffline + } else { + // update retries + g.Servers[0].ReCallTimes++ + + // Retry more than config times, start election + if g.Servers[0].ReCallTimes >= s.Config().SentinelMasterDeadCheckTimes { + // Mark enters objective offline state + g.Servers[0].State = models.GroupServerStateOffline + } + // Start the election master node + if g.Servers[0].State == models.GroupServerStateOffline { + pending = append(pending, g) + } } } - } - return nil -} -func (s *Topom) rewatchSentinels(servers []string) { - if s.ha.monitor != nil { - s.ha.monitor.Cancel() - s.ha.monitor = nil - } - if len(servers) == 0 { - s.ha.masters = nil - } else { - s.ha.monitor = redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - s.ha.monitor.LogFunc = log.Warnf - s.ha.monitor.ErrFunc = log.WarnErrorf - go func(p *redis.Sentinel) { - var trigger = make(chan struct{}, 1) - delayUntil := func(deadline time.Time) { - for !p.IsCanceled() { - var d = deadline.Sub(time.Now()) - if d <= 0 { - return - } - time.Sleep(math2.MinDuration(d, time.Second)) + // Update the offset information of the state and role nodes + if val, ok := serversMap[state.Addr]; ok { + if state.Err != nil { + if val.State == models.GroupServerStateNormal { + val.State = models.GroupServerStateSubjectiveOffline } + continue } - go func() { - defer close(trigger) - callback := func() { - select { - case trigger <- struct{}{}: - default: - } - } - for !p.IsCanceled() { - timeout := time.Minute * 15 - retryAt := time.Now().Add(time.Second * 10) - if !p.Subscribe(servers, timeout, callback) { - delayUntil(retryAt) - } else { - callback() - } - } - }() - go func() { - for range trigger { - var success int - for i := 0; i != 10 && !p.IsCanceled() && success != 2; i++ { - timeout := time.Second * 5 - masters, err := p.Masters(servers, timeout) - if err != nil { - log.WarnErrorf(err, "fetch group masters failed") - } else { - if !p.IsCanceled() { - s.SwitchMasters(masters) - } - success += 1 - } - delayUntil(time.Now().Add(time.Second * 5)) - } - } - }() - }(s.ha.monitor) - } - log.Warnf("rewatch sentinels = %v", servers) -} -func (s *Topom) ResyncSentinels() error { - s.mu.Lock() - defer s.mu.Unlock() - ctx, err := s.newContext() - if err != nil { - return err + val.State = models.GroupServerStateNormal + val.ReCallTimes = 0 + val.Role = state.Replication.Role + if val.Role == "master" { + val.ReplyOffset = state.Replication.MasterReplOffset + } else { + val.ReplyOffset = state.Replication.SlaveReplOffset + } + } } - defer s.dirtySentinelCache() - p := ctx.sentinel - p.OutOfSync = true - if err := s.storeUpdateSentinel(p); err != nil { - return err + if len(pending) == 0 { + return nil } - config := &redis.MonitorConfig{ - Quorum: s.config.SentinelQuorum, - ParallelSyncs: s.config.SentinelParallelSyncs, - DownAfter: s.config.SentinelDownAfter.Duration(), - FailoverTimeout: s.config.SentinelFailoverTimeout.Duration(), - NotificationScript: s.config.SentinelNotificationScript, - ClientReconfigScript: s.config.SentinelClientReconfigScript, + cache := &redis.InfoCache{ + Auth: s.config.ProductAuth, Timeout: time.Millisecond * 100, } + // Try to switch master slave + for _, g := range pending { + if err = s.trySwitchGroupMaster(g.Id, cache); err != nil { + log.Errorf("gid-[%d] switch master failed, %v", g.Id, err) + continue + } - sentinel := redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - if err := sentinel.RemoveGroupsAll(p.Servers, s.config.SentinelClientTimeout.Duration()); err != nil { - log.WarnErrorf(err, "remove sentinels failed") - } - if err := sentinel.MonitorGroups(p.Servers, s.config.SentinelClientTimeout.Duration(), config, ctx.getGroupMasters()); err != nil { - log.WarnErrorf(err, "resync sentinels failed") - return err - } - s.rewatchSentinels(p.Servers) - - var fut sync2.Future - for _, p := range ctx.proxy { - fut.Add() - go func(p *models.Proxy) { - err := s.newProxyClient(p).SetSentinels(ctx.sentinel) - if err != nil { - log.ErrorErrorf(err, "proxy-[%s] resync sentinel failed", p.Token) - } - fut.Done(p.Token, err) - }(p) - } - for t, v := range fut.Wait() { - switch err := v.(type) { - case error: - if err != nil { - return errors.Errorf("proxy-[%s] sentinel failed", t) - } + slots := ctx.getSlotMappingsByGroupId(g.Id) + // Notify all servers to update slot information + if err = s.resyncSlotMappings(ctx, slots...); err != nil { + log.Warnf("group-[%d] resync-rollback to preparing", g.Id) + continue } + s.dirtyGroupCache(g.Id) } - p.OutOfSync = false - return s.storeUpdateSentinel(p) + return nil } diff --git a/codis/pkg/topom/topom_stats.go b/codis/pkg/topom/topom_stats.go index 82ff72451a..0d6acb726a 100644 --- a/codis/pkg/topom/topom_stats.go +++ b/codis/pkg/topom/topom_stats.go @@ -4,6 +4,7 @@ package topom import ( + "sync" "time" "pika/codis/v2/pkg/models" @@ -73,25 +74,6 @@ func (s *Topom) RefreshRedisStats(timeout time.Duration) (*sync2.Future, error) }) } } - for _, server := range ctx.sentinel.Servers { - goStats(server, func(addr string) (*RedisStats, error) { - c, err := s.ha.redisp.GetClient(addr) - if err != nil { - return nil, err - } - defer s.ha.redisp.PutClient(c) - m, err := c.Info() - if err != nil { - return nil, err - } - sentinel := redis.NewSentinel(s.config.ProductName, s.config.ProductAuth) - p, err := sentinel.MastersAndSlavesClient(c) - if err != nil { - return nil, err - } - return &RedisStats{Stats: m, Sentinel: p}, nil - }) - } go func() { stats := make(map[string]*RedisStats) for k, v := range fut.Wait() { @@ -170,3 +152,56 @@ func (s *Topom) RefreshProxyStats(timeout time.Duration) (*sync2.Future, error) }() return &fut, nil } + +type MastersAndSlavesStats struct { + Error error `json:"error,omitempty"` + + UnixTime int64 `json:"unixtime"` + Timeout bool `json:"timeout,omitempty"` +} + +func (s *Topom) newMastersAndSlavesStats(timeout time.Duration, filter func(index int, g *models.GroupServer) bool, wg *sync.WaitGroup) *MastersAndSlavesStats { + var ch = make(chan struct{}) + stats := &MastersAndSlavesStats{} + defer wg.Done() + + go func() { + defer close(ch) + err := s.CheckAndSwitchSlavesAndMasters(filter) + if err != nil { + log.Errorf("refresh masters and slaves failed, %v", err) + stats.Error = err + } + }() + + select { + case <-ch: + return stats + case <-time.After(timeout): + return &MastersAndSlavesStats{Timeout: true} + } +} + +func (s *Topom) CheckMastersAndSlavesState(timeout time.Duration) (*sync.WaitGroup, error) { + s.mu.Lock() + defer s.mu.Unlock() + + wg := &sync.WaitGroup{} + wg.Add(1) + go s.newMastersAndSlavesStats(timeout, func(index int, g *models.GroupServer) bool { + return index != 0 || g.State == models.GroupServerStateNormal + }, wg) + return wg, nil +} + +func (s *Topom) CheckPreOffineMastersState(timeout time.Duration) (*sync.WaitGroup, error) { + s.mu.Lock() + defer s.mu.Unlock() + + wg := &sync.WaitGroup{} + wg.Add(1) + go s.newMastersAndSlavesStats(timeout, func(index int, g *models.GroupServer) bool { + return index == 0 && g.State != models.GroupServerStateNormal + }, wg) + return wg, nil +} diff --git a/codis/pkg/topom/topom_test.go b/codis/pkg/topom/topom_test.go index aebfa9568b..9d503600e6 100644 --- a/codis/pkg/topom/topom_test.go +++ b/codis/pkg/topom/topom_test.go @@ -9,7 +9,7 @@ import ( "testing" "pika/codis/v2/pkg/models" - "pika/codis/v2/pkg/models/fs" + fsclient "pika/codis/v2/pkg/models/fs" "pika/codis/v2/pkg/proxy" "pika/codis/v2/pkg/utils/assert" "pika/codis/v2/pkg/utils/log" diff --git a/codis/pkg/utils/redis/client.go b/codis/pkg/utils/redis/client.go index 9427d47fb9..f0ef98e98e 100644 --- a/codis/pkg/utils/redis/client.go +++ b/codis/pkg/utils/redis/client.go @@ -5,16 +5,19 @@ package redis import ( "container/list" + "encoding/json" "net" + "regexp" "strconv" "strings" "sync" "time" + redigo "github.com/garyburd/redigo/redis" + "pika/codis/v2/pkg/utils/errors" + "pika/codis/v2/pkg/utils/log" "pika/codis/v2/pkg/utils/math2" - - redigo "github.com/garyburd/redigo/redis" ) type Client struct { @@ -176,6 +179,63 @@ func (c *Client) InfoKeySpace() (map[int]string, error) { return info, nil } +func (c *Client) InfoReplication() (*InfoReplication, error) { + text, err := redigo.String(c.Do("INFO", "replication")) + if err != nil { + return nil, errors.Trace(err) + } + + var ( + info = make(map[string]string) + slaveMap = make([]map[string]string, 0) + infoReplication InfoReplication + slaves []InfoSlave + ) + + for _, line := range strings.Split(text, "\n") { + kv := strings.SplitN(line, ":", 2) + if len(kv) != 2 { + continue + } + + if key := strings.TrimSpace(kv[0]); key != "" { + if ok, _ := regexp.Match("slave[0-9]+", []byte(key)); ok { + slaveKvs := strings.Split(kv[1], ",") + + slave := make(map[string]string) + for _, slaveKvStr := range slaveKvs { + slaveKv := strings.Split(slaveKvStr, "=") + if len(slaveKv) != 2 { + log.Warnf("invalid replication info, slaveKvs = %s, slaveKv = %s", slaveKvs, slaveKv) + continue + } + slave[slaveKv[0]] = slaveKv[1] + } + + slaveMap = append(slaveMap, slave) + } else { + info[key] = strings.TrimSpace(kv[1]) + } + } + } + + if len(slaveMap) > 0 { + slavesStr, _ := json.Marshal(slaveMap) + if err = json.Unmarshal(slavesStr, &slaves); err != nil { + log.Errorf("unmarshal to slaves failed, %v", err) + } + } + + str, _ := json.Marshal(info) + if err = json.Unmarshal(str, &infoReplication); err != nil { + log.Errorf("unmarshal to infoReplication failed, %v", err) + } else { + infoReplication.Slaves = slaves + } + + return &infoReplication, nil +} + func (c *Client) InfoFull() (map[string]string, error) { if info, err := c.Info(); err != nil { return nil, errors.Trace(err) @@ -203,23 +263,24 @@ func (c *Client) InfoFull() (map[string]string, error) { } func (c *Client) SetMaster(master string) error { - host, port, err := net.SplitHostPort(master) - if err != nil { - return errors.Trace(err) - } - c.Send("MULTI") - c.Send("CONFIG", "SET", "masterauth", c.Auth) - c.Send("SLAVEOF", host, port) - c.Send("CONFIG", "REWRITE") - c.Send("CLIENT", "KILL", "TYPE", "normal") - values, err := redigo.Values(c.Do("EXEC")) - if err != nil { - return errors.Trace(err) - } - for _, r := range values { - if err, ok := r.(redigo.Error); ok { + if master == "" || strings.ToUpper(master) == "NO:ONE" { + if _, err := c.Do("SLAVEOF", "NO", "ONE"); err != nil { + return err + } + } else { + host, port, err := net.SplitHostPort(master) + if err != nil { return errors.Trace(err) } + if _, err := c.Do("CONFIG", "set", "masterauth", c.Auth); err != nil { + return err + } + if _, err := c.Do("SLAVEOF", host, port); err != nil { + return err + } + } + if _, err := c.Do("CONFIG", "REWRITE"); err != nil { + return err } return nil } diff --git a/codis/pkg/utils/redis/client_test.go b/codis/pkg/utils/redis/client_test.go new file mode 100644 index 0000000000..db726f2b4c --- /dev/null +++ b/codis/pkg/utils/redis/client_test.go @@ -0,0 +1,63 @@ +package redis + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" +) + +func TestKk(t *testing.T) { + ok, err := regexp.Match("slave[0-9]+", []byte("slave_01")) + + fmt.Sprintln(ok, err) +} + +func TestParseInfo(t *testing.T) { + text := "# Replication\nrole:master\nconnected_slaves:1\nslave0:ip=10.174.22.228,port=9225,state=online,offset=2175592,lag=0\nmaster_repl_offset:2175592\nrepl_backlog_active:1\nrepl_backlog_size:1048576\nrepl_backlog_first_byte_offset:1127017\nrepl_backlog_histlen:1048576\n" + info := make(map[string]string) + slaveMap := make([]map[string]string, 0) + var slaves []InfoSlave + var infoReplication InfoReplication + + for _, line := range strings.Split(text, "\n") { + kv := strings.SplitN(line, ":", 2) + if len(kv) != 2 { + continue + } + + if key := strings.TrimSpace(kv[0]); key != "" { + if ok, _ := regexp.Match("slave[0-9]+", []byte(key)); ok { + slaveKvs := strings.Split(kv[1], ",") + + slave := make(map[string]string) + for _, slaveKvStr := range slaveKvs { + slaveKv := strings.Split(slaveKvStr, "=") + if len(slaveKv) != 2 { + continue + } + slave[slaveKv[0]] = slaveKv[1] + } + + slaveMap = append(slaveMap, slave) + } else { + info[key] = strings.TrimSpace(kv[1]) + } + } + } + if len(slaveMap) > 0 { + slavesStr, _ := json.Marshal(slaveMap) + err := json.Unmarshal(slavesStr, &slaves) + + _ = err + info["slaveMap"] = string(slavesStr) + } + + str, _ := json.Marshal(info) + err := json.Unmarshal(str, &infoReplication) + infoReplication.Slaves = slaves + + _ = err + fmt.Println(err) +} diff --git a/codis/pkg/utils/redis/codis_sentinel.go b/codis/pkg/utils/redis/codis_sentinel.go new file mode 100644 index 0000000000..0b8b150ebd --- /dev/null +++ b/codis/pkg/utils/redis/codis_sentinel.go @@ -0,0 +1,144 @@ +package redis + +import ( + "context" + "fmt" + "time" + + "pika/codis/v2/pkg/models" + "pika/codis/v2/pkg/utils/errors" + "pika/codis/v2/pkg/utils/log" + "pika/codis/v2/pkg/utils/math2" + "pika/codis/v2/pkg/utils/sync2" +) + +type CodisSentinel struct { + context.Context + Cancel context.CancelFunc + + Product, Auth string + + LogFunc func(format string, args ...interface{}) + ErrFunc func(err error, format string, args ...interface{}) +} + +func NewCodisSentinel(product, auth string) *CodisSentinel { + s := &CodisSentinel{Product: product, Auth: auth} + s.Context, s.Cancel = context.WithCancel(context.Background()) + return s +} + +func (s *CodisSentinel) IsCanceled() bool { + select { + case <-s.Context.Done(): + return true + default: + return false + } +} + +func (s *CodisSentinel) printf(format string, args ...interface{}) { + if s.LogFunc != nil { + s.LogFunc(format, args...) + } +} + +func (s *CodisSentinel) errorf(err error, format string, args ...interface{}) { + if s.ErrFunc != nil { + s.ErrFunc(err, format, args...) + } +} + +func (s *CodisSentinel) do(sentinel string, timeout time.Duration, + fn func(client *Client) error) error { + c, err := NewClientNoAuth(sentinel, timeout) + if err != nil { + return err + } + defer c.Close() + return fn(c) +} + +func (s *CodisSentinel) dispatch(ctx context.Context, sentinel string, timeout time.Duration, + fn func(client *Client) error) error { + c, err := NewClientNoAuth(sentinel, timeout) + if err != nil { + return err + } + defer c.Close() + + var exit = make(chan error, 1) + + go func() { + exit <- fn(c) + }() + + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + case err := <-exit: + return err + } +} + +func (s *CodisSentinel) RefreshMastersAndSlavesClient(parallel int, groupServers map[int][]*models.GroupServer) []*ReplicationState { + if len(groupServers) == 0 { + s.printf("there's no groups") + return nil + } + + parallel = math2.MaxInt(10, parallel) + limit := make(chan struct{}, parallel) + defer close(limit) + + var fut sync2.Future + + for gid, servers := range groupServers { + for index, server := range servers { + limit <- struct{}{} + fut.Add() + + go func(gid, index int, server *models.GroupServer) { + defer func() { + <-limit + }() + + info, err := s.infoReplicationDispatch(server.Addr) + state := &ReplicationState{ + Index: index, + GroupID: gid, + Addr: server.Addr, + Replication: info, + Err: err, + } + fut.Done(fmt.Sprintf("%d_%d", gid, index), state) + }(gid, index, server) + } + } + + results := make([]*ReplicationState, 0) + + for _, v := range fut.Wait() { + switch val := v.(type) { + case *ReplicationState: + if val != nil { + results = append(results, val) + } + } + } + + return results +} + +func (s *CodisSentinel) infoReplicationDispatch(addr string) (*InfoReplication, error) { + var ( + client *Client + err error + ) + if client, err = NewClient(addr, s.Auth, time.Second); err != nil { + log.WarnErrorf(err, "create redis client to %s failed", addr) + return nil, err + } + defer client.Close() + return client.InfoReplication() +} diff --git a/codis/pkg/utils/redis/sentinel.go b/codis/pkg/utils/redis/sentinel.go index 5d949bfc7a..e71155c065 100644 --- a/codis/pkg/utils/redis/sentinel.go +++ b/codis/pkg/utils/redis/sentinel.go @@ -4,426 +4,17 @@ package redis import ( - "fmt" - "net" + "encoding/json" "strconv" - "strings" "time" - - "golang.org/x/net/context" - - "pika/codis/v2/pkg/utils/errors" - "pika/codis/v2/pkg/utils/sync2/atomic2" - - redigo "github.com/garyburd/redigo/redis" ) -type Sentinel struct { - context.Context - Cancel context.CancelFunc - - Product, Auth string - - LogFunc func(format string, args ...interface{}) - ErrFunc func(err error, format string, args ...interface{}) -} - -func NewSentinel(product, auth string) *Sentinel { - s := &Sentinel{Product: product, Auth: auth} - s.Context, s.Cancel = context.WithCancel(context.Background()) - return s -} - -func (s *Sentinel) IsCanceled() bool { - select { - case <-s.Context.Done(): - return true - default: - return false - } -} - -func (s *Sentinel) NodeName(gid int) string { - return fmt.Sprintf("%s-%d", s.Product, gid) -} - -func (s *Sentinel) isSameProduct(name string) (gid int, _ bool) { - if !strings.HasPrefix(name, s.Product) { - return 0, false - } - var suffix = name[len(s.Product):] - if len(suffix) <= 1 || suffix[0] != '-' { - return 0, false - } - n, err := strconv.Atoi(suffix[1:]) - if err != nil { - return 0, false - } - return n, true -} - -func (s *Sentinel) printf(format string, args ...interface{}) { - if s.LogFunc != nil { - s.LogFunc(format, args...) - } -} - -func (s *Sentinel) errorf(err error, format string, args ...interface{}) { - if s.ErrFunc != nil { - s.ErrFunc(err, format, args...) - } -} - -func (s *Sentinel) do(sentinel string, timeout time.Duration, - fn func(client *Client) error) error { - c, err := NewClientNoAuth(sentinel, timeout) - if err != nil { - return err - } - defer c.Close() - return fn(c) -} - -func (s *Sentinel) dispatch(ctx context.Context, sentinel string, timeout time.Duration, - fn func(client *Client) error) error { - c, err := NewClientNoAuth(sentinel, timeout) - if err != nil { - return err - } - defer c.Close() - - var exit = make(chan error, 1) - - go func() { - exit <- fn(c) - }() - - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - case err := <-exit: - return err - } -} - -func (s *Sentinel) subscribeCommand(client *Client, sentinel string, - onSubscribed func()) error { - defer func() { - client.Close() - }() - var channels = []interface{}{"+switch-master"} - go func() { - client.Send("SUBSCRIBE", channels...) - client.Flush() - }() - for _, sub := range channels { - values, err := redigo.Values(client.Receive()) - if err != nil { - return errors.Trace(err) - } else if len(values) != 3 { - return errors.Errorf("invalid response = %v", values) - } - s, err := redigo.Strings(values[:2], nil) - if err != nil || s[0] != "subscribe" || s[1] != sub.(string) { - return errors.Errorf("invalid response = %v", values) - } - } - onSubscribed() - for { - values, err := redigo.Values(client.Receive()) - if err != nil { - return errors.Trace(err) - } else if len(values) < 2 { - return errors.Errorf("invalid response = %v", values) - } - message, err := redigo.Strings(values, nil) - if err != nil || message[0] != "message" { - return errors.Errorf("invalid response = %v", values) - } - s.printf("sentinel-[%s] subscribe event %v", sentinel, message) - - switch message[1] { - case "+switch-master": - if len(message) != 3 { - return errors.Errorf("invalid response = %v", values) - } - var params = strings.SplitN(message[2], " ", 2) - if len(params) != 2 { - return errors.Errorf("invalid response = %v", values) - } - _, yes := s.isSameProduct(params[0]) - if yes { - return nil - } - } - } -} - -func (s *Sentinel) subscribeDispatch(ctx context.Context, sentinel string, timeout time.Duration, - onSubscribed func()) (bool, error) { - var err = s.dispatch(ctx, sentinel, timeout, func(c *Client) error { - return s.subscribeCommand(c, sentinel, onSubscribed) - }) - if err != nil { - switch errors.Cause(err) { - case context.Canceled, context.DeadlineExceeded: - return false, nil - default: - return false, err - } - } - return true, nil -} - -func (s *Sentinel) Subscribe(sentinels []string, timeout time.Duration, onMajoritySubscribed func()) bool { - cntx, cancel := context.WithTimeout(s.Context, timeout) - defer cancel() - - timeout += time.Second * 5 - results := make(chan bool, len(sentinels)) - - var majority = 1 + len(sentinels)/2 - - var subscribed atomic2.Int64 - for i := range sentinels { - go func(sentinel string) { - notified, err := s.subscribeDispatch(cntx, sentinel, timeout, func() { - if subscribed.Incr() == int64(majority) { - onMajoritySubscribed() - } - }) - if err != nil { - s.errorf(err, "sentinel-[%s] subscribe failed", sentinel) - } - results <- notified - }(sentinels[i]) - } - - for alive := len(sentinels); ; alive-- { - if alive < majority { - if cntx.Err() == nil { - s.printf("sentinel subscribe lost majority (%d/%d)", alive, len(sentinels)) - } - return false - } - select { - case <-cntx.Done(): - if cntx.Err() != context.DeadlineExceeded { - s.printf("sentinel subscribe canceled (%v)", cntx.Err()) - } - return false - case notified := <-results: - if notified { - s.printf("sentinel subscribe notified +switch-master") - return true - } - } - } -} - -func (s *Sentinel) existsCommand(client *Client, names []string) (map[string]bool, error) { - defer func() { - if !client.isRecyclable() { - client.Close() - } - }() - go func() { - for _, name := range names { - client.Send("SENTINEL", "get-master-addr-by-name", name) - } - if len(names) != 0 { - client.Flush() - } - }() - exists := make(map[string]bool, len(names)) - for _, name := range names { - r, err := client.Receive() - if err != nil { - return nil, errors.Trace(err) - } - exists[name] = (r != nil) - } - return exists, nil -} - -func (s *Sentinel) slavesCommand(client *Client, names []string) (map[string][]map[string]string, error) { - defer func() { - if !client.isRecyclable() { - client.Close() - } - }() - exists, err := s.existsCommand(client, names) - if err != nil { - return nil, err - } - go func() { - var pending int - for _, name := range names { - if !exists[name] { - continue - } - pending++ - client.Send("SENTINEL", "slaves", name) - } - if pending != 0 { - client.Flush() - } - }() - results := make(map[string][]map[string]string, len(names)) - for _, name := range names { - if !exists[name] { - continue - } - values, err := redigo.Values(client.Receive()) - if err != nil { - return nil, errors.Trace(err) - } - var slaves []map[string]string - for i := range values { - m, err := redigo.StringMap(values[i], nil) - if err != nil { - return nil, errors.Trace(err) - } - slaves = append(slaves, m) - } - results[name] = slaves - } - return results, nil -} - -func (s *Sentinel) mastersCommand(client *Client) (map[int]map[string]string, error) { - defer func() { - if !client.isRecyclable() { - client.Close() - } - }() - values, err := redigo.Values(client.Do("SENTINEL", "masters")) - if err != nil { - return nil, errors.Trace(err) - } - var masters = make(map[int]map[string]string) - for i := range values { - p, err := redigo.StringMap(values[i], nil) - if err != nil { - return nil, errors.Trace(err) - } - gid, yes := s.isSameProduct(p["name"]) - if yes { - masters[gid] = p - } - } - return masters, nil -} - -func (s *Sentinel) mastersDispatch(ctx context.Context, sentinel string, timeout time.Duration) (map[int]*SentinelMaster, error) { - var masters = make(map[int]*SentinelMaster) - var err = s.dispatch(ctx, sentinel, timeout, func(c *Client) error { - p, err := s.mastersCommand(c) - if err != nil { - return err - } - for gid, master := range p { - epoch, err := strconv.ParseInt(master["config-epoch"], 10, 64) - if err != nil { - s.printf("sentinel-[%s] masters parse %s failed, config-epoch = '%s', %s", - sentinel, master["name"], master["config-epoch"], err) - continue - } - var ip, port = master["ip"], master["port"] - if ip == "" || port == "" { - s.printf("sentinel-[%s] masters parse %s failed, ip:port = '%s:%s'", - sentinel, master["name"], ip, port) - continue - } - masters[gid] = &SentinelMaster{ - Addr: net.JoinHostPort(ip, port), - Info: master, Epoch: epoch, - } - } - return nil - }) - if err != nil { - switch errors.Cause(err) { - case context.Canceled: - return nil, nil - default: - return nil, err - } - } - return masters, nil -} - type SentinelMaster struct { Addr string Info map[string]string Epoch int64 } -func (s *Sentinel) Masters(sentinels []string, timeout time.Duration) (map[int]string, error) { - cntx, cancel := context.WithTimeout(s.Context, timeout) - defer cancel() - - timeout += time.Second * 5 - results := make(chan map[int]*SentinelMaster, len(sentinels)) - - var majority = 1 + len(sentinels)/2 - - for i := range sentinels { - go func(sentinel string) { - masters, err := s.mastersDispatch(cntx, sentinel, timeout) - if err != nil { - s.errorf(err, "sentinel-[%s] masters failed", sentinel) - } - results <- masters - }(sentinels[i]) - } - - masters := make(map[int]string) - current := make(map[int]*SentinelMaster) - - var voted int - for alive := len(sentinels); ; alive-- { - if alive == 0 { - switch { - case cntx.Err() != context.DeadlineExceeded && cntx.Err() != nil: - s.printf("sentinel masters canceled (%v)", cntx.Err()) - return nil, errors.Trace(cntx.Err()) - case voted != len(sentinels): - s.printf("sentinel masters voted = (%d/%d) masters = %d (%v)", voted, len(sentinels), len(masters), cntx.Err()) - } - if voted < majority { - return nil, errors.Errorf("lost majority (%d/%d)", voted, len(sentinels)) - } - return masters, nil - } - select { - case <-cntx.Done(): - switch { - case cntx.Err() != context.DeadlineExceeded: - s.printf("sentinel masters canceled (%v)", cntx.Err()) - return nil, errors.Trace(cntx.Err()) - default: - s.printf("sentinel masters voted = (%d/%d) masters = %d (%v)", voted, len(sentinels), len(masters), cntx.Err()) - } - if voted < majority { - return nil, errors.Errorf("lost majority (%d/%d)", voted, len(sentinels)) - } - return masters, nil - case m := <-results: - if m == nil { - continue - } - for gid, master := range m { - if current[gid] == nil || current[gid].Epoch < master.Epoch { - current[gid] = master - masters[gid] = master.Addr - } - } - voted += 1 - } - } -} - type MonitorConfig struct { Quorum int ParallelSyncs int @@ -434,357 +25,83 @@ type MonitorConfig struct { ClientReconfigScript string } -func (s *Sentinel) monitorGroupsCommand(client *Client, sentniel string, config *MonitorConfig, groups map[int]*net.TCPAddr) error { - defer func() { - if !client.isRecyclable() { - client.Close() - } - }() - var names []string - for gid := range groups { - names = append(names, s.NodeName(gid)) - } - if err := s.removeCommand(client, names); err != nil { - return err - } - go func() { - for gid, tcpAddr := range groups { - var ip, port = tcpAddr.IP.String(), tcpAddr.Port - client.Send("SENTINEL", "monitor", s.NodeName(gid), ip, port, config.Quorum) - } - if len(groups) != 0 { - client.Flush() - } - }() - for range groups { - _, err := client.Receive() - if err != nil { - return errors.Trace(err) - } - } - go func() { - for gid := range groups { - var args = []interface{}{"set", s.NodeName(gid)} - if config.ParallelSyncs != 0 { - args = append(args, "parallel-syncs", config.ParallelSyncs) - } - if config.DownAfter != 0 { - args = append(args, "down-after-milliseconds", int(config.DownAfter/time.Millisecond)) - } - if config.FailoverTimeout != 0 { - args = append(args, "failover-timeout", int(config.FailoverTimeout/time.Millisecond)) - } - if s.Auth != "" { - args = append(args, "auth-pass", s.Auth) - } - if config.NotificationScript != "" { - args = append(args, "notification-script", config.NotificationScript) - } - if config.ClientReconfigScript != "" { - args = append(args, "client-reconfig-script", config.ClientReconfigScript) - } - client.Send("SENTINEL", args...) - } - if len(groups) != 0 { - client.Flush() - } - }() - for range groups { - _, err := client.Receive() - if err != nil { - return errors.Trace(err) - } - } - return nil +type SentinelGroup struct { + Master map[string]string `json:"master"` + Slaves []map[string]string `json:"slaves,omitempty"` } -func (s *Sentinel) monitorGroupsDispatch(ctx context.Context, sentinel string, timeout time.Duration, - config *MonitorConfig, groups map[int]*net.TCPAddr) error { - var err = s.dispatch(ctx, sentinel, timeout, func(c *Client) error { - return s.monitorGroupsCommand(c, sentinel, config, groups) - }) - if err != nil { - switch errors.Cause(err) { - case context.Canceled: - return nil - default: - return err - } - } - return nil +type InfoSlave struct { + IP string `json:"ip"` + Port string `json:"port"` + State string `json:"state"` + Offset int `json:"offset"` + Lag int `json:"lag"` } -func (s *Sentinel) MonitorGroups(sentinels []string, timeout time.Duration, config *MonitorConfig, groups map[int]string) error { - cntx, cancel := context.WithTimeout(s.Context, timeout) - defer cancel() - - resolve := make(map[int]*net.TCPAddr) - - var exit = make(chan error, 1) - - go func() (err error) { - defer func() { - exit <- err - }() - for gid, addr := range groups { - if err := cntx.Err(); err != nil { - return errors.Trace(err) - } - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) - if err != nil { - s.printf("sentinel monitor resolve tcp address of %s failed, %s", addr, err) - return errors.Trace(err) - } - resolve[gid] = tcpAddr - } - return nil - }() - - select { - case <-cntx.Done(): - if cntx.Err() != context.DeadlineExceeded { - s.printf("sentinel monitor canceled (%v)", cntx.Err()) - } else { - s.printf("sentinel montior resolve tcp address (%v)", cntx.Err()) - } - return errors.Trace(cntx.Err()) - case err := <-exit: - if err != nil { - return err - } - } - - timeout += time.Second * 5 - results := make(chan error, len(sentinels)) - - for i := range sentinels { - go func(sentinel string) { - err := s.monitorGroupsDispatch(cntx, sentinel, timeout, config, resolve) - if err != nil { - s.errorf(err, "sentinel-[%s] monitor failed", sentinel) - } - results <- err - }(sentinels[i]) +func (i *InfoSlave) UnmarshalJSON(b []byte) error { + var kvmap map[string]string + if err := json.Unmarshal(b, &kvmap); err != nil { + return err } - var last error - for range sentinels { - select { - case <-cntx.Done(): - if last != nil { - return last - } - return errors.Trace(cntx.Err()) - case err := <-results: - if err != nil { - last = err - } - } - } - return last -} + i.IP = kvmap["ip"] + i.Port = kvmap["port"] + i.State = kvmap["state"] -func (s *Sentinel) removeCommand(client *Client, names []string) error { - defer func() { - if !client.isRecyclable() { - client.Close() - } - }() - exists, err := s.existsCommand(client, names) - if err != nil { - return err - } - go func() { - var pending int - for _, name := range names { - if !exists[name] { - continue - } - pending++ - client.Send("SENTINEL", "remove", name) + if val, ok := kvmap["offset"]; ok { + if intval, err := strconv.Atoi(val); err == nil { + i.Offset = intval } - if pending != 0 { - client.Flush() - } - }() - for _, name := range names { - if !exists[name] { - continue - } - _, err := client.Receive() - if err != nil { - return errors.Trace(err) - } - } - return nil -} - -func (s *Sentinel) removeGroupsDispatch(ctx context.Context, sentinel string, timeout time.Duration, - groups map[int]bool) error { - var names []string - for gid := range groups { - names = append(names, s.NodeName(gid)) } - var err = s.dispatch(ctx, sentinel, timeout, func(c *Client) error { - return s.removeCommand(c, names) - }) - if err != nil { - switch errors.Cause(err) { - case context.Canceled: - return nil - default: - return err + if val, ok := kvmap["lag"]; ok { + if intval, err := strconv.Atoi(val); err == nil { + i.Lag = intval } } return nil } -func (s *Sentinel) RemoveGroups(sentinels []string, timeout time.Duration, groups map[int]bool) error { - cntx, cancel := context.WithTimeout(s.Context, timeout) - defer cancel() - - timeout += time.Second * 5 - results := make(chan error, len(sentinels)) - - for i := range sentinels { - go func(sentinel string) { - err := s.removeGroupsDispatch(cntx, sentinel, timeout, groups) - if err != nil { - s.errorf(err, "sentinel-[%s] remove failed", sentinel) - } - results <- err - }(sentinels[i]) - } - - var last error - for range sentinels { - select { - case <-cntx.Done(): - if last != nil { - return last - } - return errors.Trace(cntx.Err()) - case err := <-results: - if err != nil { - last = err - } - } - } - return last +type InfoReplication struct { + Role string `json:"role"` + ConnectedSlaves int `json:"connected_slaves"` + MasterHost string `json:"master_host"` + MasterPort string `json:"master_port"` + SlaveReplOffset int `json:"slave_repl_offset"` + MasterReplOffset int `json:"master_repl_offset"` + Slaves []InfoSlave `json:"-"` } -func (s *Sentinel) removeGroupsAllDispatch(ctx context.Context, sentinel string, timeout time.Duration) error { - var err = s.dispatch(ctx, sentinel, timeout, func(c *Client) error { - masters, err := s.mastersCommand(c) - if err != nil { - return err - } - var names []string - for gid := range masters { - names = append(names, s.NodeName(gid)) - } - return s.removeCommand(c, names) - }) - if err != nil { - switch errors.Cause(err) { - case context.Canceled: - return nil - default: - return err - } - } - return nil +type ReplicationState struct { + GroupID int + Index int + Addr string + Replication *InfoReplication + Err error } -func (s *Sentinel) RemoveGroupsAll(sentinels []string, timeout time.Duration) error { - cntx, cancel := context.WithTimeout(s.Context, timeout) - defer cancel() - - timeout += time.Second * 5 - results := make(chan error, len(sentinels)) - - for i := range sentinels { - go func(sentinel string) { - err := s.removeGroupsAllDispatch(cntx, sentinel, timeout) - if err != nil { - s.errorf(err, "sentinel-[%s] remove failed", sentinel) - } - results <- err - }(sentinels[i]) - } - - var last error - for range sentinels { - select { - case <-cntx.Done(): - if last != nil { - return last - } - return errors.Trace(cntx.Err()) - case err := <-results: - if err != nil { - last = err - } - } +func (i *InfoReplication) UnmarshalJSON(b []byte) error { + var kvmap map[string]string + if err := json.Unmarshal(b, &kvmap); err != nil { + return err } - return last -} - -type SentinelGroup struct { - Master map[string]string `json:"master"` - Slaves []map[string]string `json:"slaves,omitempty"` -} -func (s *Sentinel) MastersAndSlavesClient(client *Client) (map[string]*SentinelGroup, error) { - defer func() { - if !client.isRecyclable() { - client.Close() + if val, ok := kvmap["connected_slaves"]; ok { + if intval, err := strconv.Atoi(val); err == nil { + i.ConnectedSlaves = intval } - }() - masters, err := s.mastersCommand(client) - if err != nil { - return nil, err } - var names []string - for gid := range masters { - names = append(names, s.NodeName(gid)) - } - slaves, err := s.slavesCommand(client, names) - if err != nil { - return nil, err - } - results := make(map[string]*SentinelGroup, len(masters)) - for gid, master := range masters { - var name = s.NodeName(gid) - results[name] = &SentinelGroup{ - Master: master, Slaves: slaves[name], + if val, ok := kvmap["slave_repl_offset"]; ok { + if intval, err := strconv.Atoi(val); err == nil { + i.SlaveReplOffset = intval } } - return results, nil -} - -func (s *Sentinel) MastersAndSlaves(sentinel string, timeout time.Duration) (map[string]*SentinelGroup, error) { - var results map[string]*SentinelGroup - var err = s.do(sentinel, timeout, func(c *Client) error { - m, err := s.MastersAndSlavesClient(c) - if err != nil { - return err + if val, ok := kvmap["master_repl_offset"]; ok { + if intval, err := strconv.Atoi(val); err == nil { + i.MasterReplOffset = intval } - results = m - return nil - }) - if err != nil { - return nil, err } - return results, nil -} - -func (s *Sentinel) FlushConfig(sentinel string, timeout time.Duration) error { - return s.do(sentinel, timeout, func(c *Client) error { - _, err := c.Do("SENTINEL", "flushconfig") - if err != nil { - return err - } - return nil - }) + i.Role = kvmap["role"] + i.MasterPort = kvmap["master_host"] + i.MasterHost = kvmap["master_port"] + return nil } diff --git a/conf/pika.conf b/conf/pika.conf index bd8993c8d2..65847e323d 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -79,24 +79,17 @@ userpass : # By default, this list is empty. userblacklist : -# Running Mode of Pika. +# Running Mode of Pika, The current version only supports running in "classic mode". # If set to 'classic', Pika will create multiple DBs whose number is the value of configure item "databases". # Meanwhile the following configure item "databases" is valid and the "default-slot-num" is disabled. -# If set to 'sharding', Pika will create multiple Tables whose number is the value of configure item "default-slot-num". -# Meanwhile the following configure item "default-slot-num" is valid and the "databases" is disabled. -# Pika supports two instance modes: [classic | sharding] instance-mode : classic - # The number of databases when Pika runs in classic mode. # The default database id is DB 0. You can select a different one on # a per-connection by using SELECT. The db id range is [0, 'databases' value -1]. # The value range of this parameter is [1, 8]. databases : 1 -# The default slot number of each table when Pika runs in sharding mode. -default-slot-num : 1024 - # The number of followers of a master. Only [0, 1, 2, 3, 4] is valid at present. # By default, this num is set to 0, which means this feature is [not enabled] # and the Pika runs in standalone mode. @@ -307,6 +300,11 @@ max-cache-files : 5000 # Its default value is 10(x). You can also change it to 5(x). max-bytes-for-level-multiplier : 10 +# slotmigrate is mainly used to migrate slots, usually we will set it to no. +# When you migrate slots, you need to set it to yes, and reload slotskeys before. +# slotmigrate [yes | no] +slotmigrate : no + # BlockBasedTable block_size, default 4k # block-size: 4096 @@ -391,4 +389,3 @@ max-bytes-for-level-multiplier : 10 # blob-num-shard-bits default -1, the number of bits from cache keys to be use as shard id. # The cache will be sharded into 2^blob-num-shard-bits shards. # blob-num-shard-bits : -1 - diff --git a/include/pika_command.h b/include/pika_command.h index 3abfbea801..cd4c6aa61b 100644 --- a/include/pika_command.h +++ b/include/pika_command.h @@ -48,6 +48,24 @@ const std::string kCmdDummy = "dummy"; const std::string kCmdNameQuit = "quit"; const std::string kCmdNameHello = "hello"; +//Migrate slot +const std::string kCmdNameSlotsMgrtSlot = "slotsmgrtslot"; +const std::string kCmdNameSlotsMgrtTagSlot = "slotsmgrttagslot"; +const std::string kCmdNameSlotsMgrtOne = "slotsmgrtone"; +const std::string kCmdNameSlotsMgrtTagOne = "slotsmgrttagone"; +const std::string kCmdNameSlotsInfo = "slotsinfo"; +const std::string kCmdNameSlotsHashKey = "slotshashkey"; +const std::string kCmdNameSlotsReload = "slotsreload"; +const std::string kCmdNameSlotsReloadOff = "slotsreloadoff"; +const std::string kCmdNameSlotsDel = "slotsdel"; +const std::string kCmdNameSlotsScan = "slotsscan"; +const std::string kCmdNameSlotsCleanup = "slotscleanup"; +const std::string kCmdNameSlotsCleanupOff = "slotscleanupoff"; +const std::string kCmdNameSlotsMgrtTagSlotAsync = "slotsmgrttagslot-async"; +const std::string kCmdNameSlotsMgrtSlotAsync = "slotsmgrtslot-async"; +const std::string kCmdNameSlotsMgrtExecWrapper = "slotsmgrt-exec-wrapper"; +const std::string kCmdNameSlotsMgrtAsyncStatus = "slotsmgrt-async-status"; +const std::string kCmdNameSlotsMgrtAsyncCancel = "slotsmgrt-async-cancel"; // Kv const std::string kCmdNameSet = "set"; const std::string kCmdNameGet = "get"; @@ -203,7 +221,10 @@ enum CmdFlagsMask { kCmdFlagsMaskSuspend = 64, kCmdFlagsMaskPrior = 128, kCmdFlagsMaskAdminRequire = 256, - kCmdFlagsMaskSlot = 1536 + kCmdFlagsMaskPreDo = 512, + kCmdFlagsMaskCacheDo = 1024, + kCmdFlagsMaskPostDo = 2048, + kCmdFlagsMaskSlot = 1536, }; enum CmdFlags { @@ -229,7 +250,8 @@ enum CmdFlags { kCmdFlagsAdminRequire = 256, kCmdFlagsDoNotSpecifyPartition = 0, // default do not specify partition kCmdFlagsSingleSlot = 512, - kCmdFlagsMultiSlot = 1024 + kCmdFlagsMultiSlot = 1024, + kCmdFlagsPreDo = 2048, }; void inline RedisAppendContent(std::string& str, const std::string& value); @@ -414,6 +436,7 @@ class Cmd : public std::enable_shared_from_this { void Initial(const PikaCmdArgsType& argv, const std::string& db_name); + bool is_read() const; bool is_write() const; bool is_local() const; bool is_suspend() const; diff --git a/include/pika_conf.h b/include/pika_conf.h index 95946b4047..8da01e5767 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -90,6 +90,14 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return arena_block_size_; } + int64_t slotmigrate_thread_num() { + std::shared_lock l(rwlock_); + return slotmigrate_thread_num_; + } + int64_t thread_migrate_keys_num() { + std::shared_lock l(rwlock_); + return thread_migrate_keys_num_; + } int64_t max_write_buffer_size() { std::shared_lock l(rwlock_); return max_write_buffer_size_; @@ -106,6 +114,14 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return timeout_; } + int binlog_writer_num() { + std::shared_lock l(rwlock_); + return binlog_writer_num_; + } + bool slotmigrate() { + std::shared_lock l(rwlock_); + return slotmigrate_; + } std::string server_id() { std::shared_lock l(rwlock_); return server_id_; @@ -388,6 +404,10 @@ class PikaConf : public pstd::BaseConf { pstd::StringToLower(item); } } + void SetSlotMigrate(const std::string &value) { + std::lock_guard l(rwlock_); + slotmigrate_ = (value == "yes") ? true : false; + } void SetExpireLogsNums(const int value) { std::lock_guard l(rwlock_); TryPushDiffCommands("expire-logs-nums", std::to_string(value)); @@ -502,6 +522,8 @@ class PikaConf : public pstd::BaseConf { std::string compact_interval_; int64_t write_buffer_size_ = 0; int64_t arena_block_size_ = 0; + int64_t slotmigrate_thread_num_ = 0; + int64_t thread_migrate_keys_num_ = 0; int64_t max_write_buffer_size_ = 0; int max_write_buffer_num_ = 0; int64_t max_client_response_size_ = 0; @@ -528,6 +550,8 @@ class PikaConf : public pstd::BaseConf { int root_connection_num_ = 0; std::atomic slowlog_write_errorlog_; std::atomic slowlog_log_slower_than_; + std::atomic slotmigrate_; + std::atomic binlog_writer_num_; int slowlog_max_len_ = 0; int expire_logs_days_ = 0; int expire_logs_nums_ = 0; diff --git a/include/pika_consensus.h b/include/pika_consensus.h index d37cf597af..a09733200a 100644 --- a/include/pika_consensus.h +++ b/include/pika_consensus.h @@ -118,6 +118,7 @@ class ConsensusCoordinator { pstd::Status ProposeLog(const std::shared_ptr& cmd_ptr, std::shared_ptr conn_ptr, std::shared_ptr resp_ptr); + pstd::Status ProposeLog(const std::shared_ptr& cmd_ptr); pstd::Status UpdateSlave(const std::string& ip, int port, const LogOffset& start, const LogOffset& end); pstd::Status AddSlaveNode(const std::string& ip, int port, int session_id); pstd::Status RemoveSlaveNode(const std::string& ip, int port); diff --git a/include/pika_db.h b/include/pika_db.h index 5388db835f..9d0cd680a0 100644 --- a/include/pika_db.h +++ b/include/pika_db.h @@ -50,7 +50,7 @@ class DB : public std::enable_shared_from_this, public pstd::noncopyable { void Compact(const storage::DataType& type); void LeaveAllSlot(); - std::set GetSlotIds(); + std::set GetSlotIDs(); std::shared_ptr GetSlotById(uint32_t slot_id); std::shared_ptr GetSlotByKey(const std::string& key); bool DBIsEmpty(); diff --git a/include/pika_define.h b/include/pika_define.h index 7259da744a..bafe0cede4 100644 --- a/include/pika_define.h +++ b/include/pika_define.h @@ -12,6 +12,13 @@ #include "net/include/redis_cli.h" +/* + * TTL type + */ +#define PIKA_TTL_ZERO 0 +#define PIKA_TTL_NONE -1 +#define PIKA_TTL_STALE -2 + #define PIKA_SYNC_BUFFER_SIZE 1000 #define PIKA_MAX_WORKER_THREAD_NUM 24 #define PIKA_REPL_SERVER_TP_SIZE 3 @@ -21,6 +28,8 @@ #define PIKA_MAX_CONN_RBUF_LB (1 << 26) // 64MB #define PIKA_MAX_CONN_RBUF_HB (1 << 29) // 512MB #define PIKA_SERVER_ID_MAX 65535 +#define HASH_SLOTS_MASK 0x000003ff +#define HASH_SLOTS_SIZE (HASH_SLOTS_MASK + 1) class PikaServer; @@ -33,7 +42,7 @@ const std::string kPikaSecretFile = "rsync.secret"; const std::string kDefaultRsyncAuth = "default"; struct DBStruct { - DBStruct(std::string tn, const uint32_t pn, std::set pi) + DBStruct(std::string tn, const uint32_t pn, std::set pi) : db_name(std::move(tn)), slot_num(pn), slot_ids(std::move(pi)) {} bool operator==(const DBStruct& db_struct) const { @@ -156,7 +165,7 @@ struct DBSyncArg { int port; std::string db_name; uint32_t slot_id; - DBSyncArg(PikaServer* const _p, std::string _ip, int _port, std::string _db_name, + DBSyncArg(PikaServer* const _p, std::string _ip, int _port, std::string _db_name, uint32_t _slot_id) : p(_p), ip(std::move(_ip)), port(_port), db_name(std::move(_db_name)), slot_id(_slot_id) {} }; @@ -191,7 +200,7 @@ struct BinlogChip { }; struct SlotInfo { - SlotInfo(std::string db_name, uint32_t slot_id) + SlotInfo(std::string db_name, uint32_t slot_id) : db_name_(std::move(db_name)), slot_id_(slot_id) {} SlotInfo() = default; diff --git a/include/pika_migrate_thread.h b/include/pika_migrate_thread.h new file mode 100644 index 0000000000..e6d16e8178 --- /dev/null +++ b/include/pika_migrate_thread.h @@ -0,0 +1,112 @@ +#ifndef PIKA_MIGRATE_THREAD_H_ +#define PIKA_MIGRATE_THREAD_H_ + +#include "include/pika_client_conn.h" +#include "include/pika_command.h" +#include "include/pika_slot.h" +#include "net/include/net_cli.h" +#include "net/include/net_thread.h" +#include "pika_client_conn.h" +#include "storage/storage.h" +#include "strings.h" + +void WriteDelKeyToBinlog(const std::string &key, const std::shared_ptr& slot); +static int DoMigrate(net::NetCli *cli, std::string send_str); + +class PikaMigrateThread; +class PikaParseSendThread : public net::Thread { + public: + PikaParseSendThread(PikaMigrateThread *migrate_thread, const std::shared_ptr& slot_); + ~PikaParseSendThread(); + + bool Init(const std::string &ip, int64_t port, int64_t timeout_ms, int64_t mgrtkeys_num); + void ExitThread(void); + + private: + int MigrateOneKey(net::NetCli *cli, const std::string key, const char key_type, bool async); + void DelKeysAndWriteBinlog(std::deque> &send_keys, const std::shared_ptr& slot); + bool CheckMigrateRecv(int64_t need_receive_num); + virtual void *ThreadMain(); + + private: + std::string dest_ip_; + int64_t dest_port_ = 0; + int64_t timeout_ms_ = 60; + int32_t mgrtkeys_num_ = 0; + std::atomic should_exit_; + PikaMigrateThread *migrate_thread_ = nullptr; + net::NetCli *cli_ = nullptr; + pstd::Mutex working_mutex_; + std::shared_ptr slot_; +}; + +class PikaMigrateThread : public net::Thread { + public: + PikaMigrateThread(); + virtual ~PikaMigrateThread(); + bool ReqMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_num, int64_t keys_num, + const std::shared_ptr& slot); + int ReqMigrateOne(const std::string &key, const std::shared_ptr& slot); + void GetMigrateStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, + int64_t *remained); + void CancelMigrate(void); + void IncWorkingThreadNum(void); + void DecWorkingThreadNum(void); + void OnTaskFailed(void); + void AddResponseNum(int32_t response_num); + + private: + void ResetThread(void); + void DestroyThread(bool is_self_exit); + void NotifyRequestMigrate(void); + bool IsMigrating(std::pair &kpair); + void ReadSlotKeys(const std::string &slotKey, int64_t need_read_num, int64_t &real_read_num, int32_t *finish); + bool CreateParseSendThreads(int32_t dispatch_num); + void DestroyParseSendThreads(void); + virtual void *ThreadMain(); + + private: + std::string dest_ip_; + int64_t dest_port_ = 0; + int64_t timeout_ms_ = 60; + int64_t slot_id_ = 0; + int64_t keys_num_ = 0; + std::shared_ptr slot_; + std::atomic is_migrating_; + std::atomic should_exit_; + std::atomic is_task_success_; + std::atomic send_num_; + std::atomic response_num_; + std::atomic moved_num_; + std::shared_ptr slot; + + bool request_migrate_ = false; + pstd::CondVar request_migrate_cond_; + std::mutex request_migrate_mutex_; + + int32_t workers_num_ = 0; + std::vector workers_; + + std::atomic working_thread_num_; + pstd::CondVar workers_cond_; + std::mutex workers_mutex_; + + std::deque> mgrtone_queue_; + std::mutex mgrtone_queue_mutex_; + + int64_t cursor_ = 0; + std::deque> mgrtkeys_queue_; + pstd::CondVar mgrtkeys_cond_; + std::mutex mgrtkeys_queue_mutex_; + + std::map, std::string> mgrtkeys_map_; + std::mutex mgrtkeys_map_mutex_; + + std::mutex migrator_mutex_; + + friend class PikaParseSendThread; +}; + +#endif + +/* EOF */ \ No newline at end of file diff --git a/include/pika_rm.h b/include/pika_rm.h index 9de068d674..64dd1ce76d 100644 --- a/include/pika_rm.h +++ b/include/pika_rm.h @@ -87,6 +87,7 @@ class SyncMasterSlot : public SyncSlot { pstd::Status ConsensusUpdateSlave(const std::string& ip, int port, const LogOffset& start, const LogOffset& end); pstd::Status ConsensusProposeLog(const std::shared_ptr& cmd_ptr, std::shared_ptr conn_ptr, std::shared_ptr resp_ptr); + Status ConsensusProposeLog(const std::shared_ptr& cmd_ptr); pstd::Status ConsensusSanityCheck(); pstd::Status ConsensusProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute); pstd::Status ConsensusProcessLocalUpdate(const LogOffset& leader_commit); diff --git a/include/pika_server.h b/include/pika_server.h index de19db6c4c..e7e5a6af26 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -36,6 +36,10 @@ #include "include/pika_repl_server.h" #include "include/pika_rsync_service.h" #include "include/pika_statistic.h" +#include "include/pika_slot_command.h" +#include "include/pika_migrate_thread.h" + + /* static std::set MultiKvCommands {kCmdNameDel, @@ -124,6 +128,9 @@ enum TaskType { kBgSave, }; +void DoBgslotscleanup(void* arg); +void DoBgslotsreload(void* arg); + class PikaServer : public pstd::noncopyable { public: PikaServer(); @@ -195,6 +202,11 @@ class PikaServer : public pstd::noncopyable { pstd::Mutex slave_mutex_; // protect slaves_; std::vector slaves_; + /** + * Sotsmgrt use + */ + std::unique_ptr pika_migrate_; + /* * Slave use */ @@ -316,6 +328,127 @@ class PikaServer : public pstd::noncopyable { // info debug use void ServerStatus(std::string* info); + /* + * * Async migrate used + */ + int SlotsMigrateOne(const std::string &key, std::shared_ptrslot); + bool SlotsMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slots, int64_t keys_num, std::shared_ptrslot); + void GetSlotsMgrtSenderStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained); + bool SlotsMigrateAsyncCancel(); + + std::shared_mutex bgsave_protector_; + BgSaveInfo bgsave_info_; + + /* + * BGSlotsReload used + */ + struct BGSlotsReload { + bool reloading = false; + time_t start_time = 0; + std::string s_start_time; + int64_t cursor = 0; + std::string pattern = "*"; + int64_t count = 0; + std::shared_ptr slot; + BGSlotsReload() : reloading(false), cursor(0), pattern("*"), count(100) {} + void Clear() { + reloading = false; + pattern = "*"; + count = 100; + cursor = 0; + } + }; + + BGSlotsReload bgslots_reload_; + + BGSlotsReload bgslots_reload() { + std::lock_guard ml(bgsave_protector_); + return bgslots_reload_; + } + bool GetSlotsreloading() { + std::lock_guard ml(bgsave_protector_); + return bgslots_reload_.reloading; + } + void SetSlotsreloading(bool reloading) { + std::lock_guard ml(bgsave_protector_); + bgslots_reload_.reloading = reloading; + } + void SetSlotsreloadingCursor(int64_t cursor) { + std::lock_guard ml(bgsave_protector_); + bgslots_reload_.cursor = cursor; + } + int64_t GetSlotsreloadingCursor() { + std::lock_guard ml(bgsave_protector_); + return bgslots_reload_.cursor; + } + void Bgslotsreload(std::shared_ptr slot); + + /* + * BGSlotsCleanup used + */ + struct BGSlotsCleanup { + bool cleaningup = false; + time_t start_time = 0; + std::string s_start_time; + int64_t cursor = 0; + std::string pattern = "*"; + int64_t count = 0; + std::shared_ptr slot; + storage::DataType type_; + std::vector cleanup_slots; + BGSlotsCleanup() : cleaningup(false), cursor(0), pattern("*"), count(100){} + void Clear() { + cleaningup = false; + pattern = "*"; + count = 100; + cursor = 0; + } + }; + + /* + * BGSlotsCleanup use + */ + BGSlotsCleanup bgslots_cleanup_; + net::BGThread bgslots_cleanup_thread_; + + + BGSlotsCleanup bgslots_cleanup() { + std::lock_guard ml(bgsave_protector_); + return bgslots_cleanup_; + } + bool GetSlotscleaningup() { + std::lock_guard ml(bgsave_protector_); + return bgslots_cleanup_.cleaningup; + } + void SetSlotscleaningup(bool cleaningup) { + std::lock_guard ml(bgsave_protector_); + bgslots_cleanup_.cleaningup = cleaningup; + } + void SetSlotscleaningupCursor(int64_t cursor) { + std::lock_guard ml(bgsave_protector_); + bgslots_cleanup_.cursor = cursor; + } + int64_t GetSlotscleaningupCursor() { + std::lock_guard ml(bgsave_protector_); + return bgslots_cleanup_.cursor; + } + void SetCleanupSlots(std::vector cleanup_slots) { + std::lock_guard ml(bgsave_protector_); + bgslots_cleanup_.cleanup_slots.swap(cleanup_slots); + } + std::vector GetCleanupSlots() { + std::lock_guard ml(bgsave_protector_); + return bgslots_cleanup_.cleanup_slots; + } + void Bgslotscleanup(std::vector cleanup_slots, std::shared_ptr slot); + void StopBgslotscleanup() { + std::lock_guard ml(bgsave_protector_); + bgslots_cleanup_.cleaningup = false; + std::vector cleanup_slots; + bgslots_cleanup_.cleanup_slots.swap(cleanup_slots); + } + + /* * StorageOptions used */ @@ -324,10 +457,6 @@ class PikaServer : public pstd::noncopyable { friend class Cmd; friend class InfoCmd; - friend class PkClusterAddSlotsCmd; - friend class PkClusterDelSlotsCmd; - friend class PkClusterAddDBCmd; - friend class PkClusterDelDBCmd; friend class PikaReplClientConn; friend class PkClusterInfoCmd; @@ -376,15 +505,15 @@ class PikaServer : public pstd::noncopyable { * Slave used */ std::string master_ip_; - int master_port_ = 0; - int repl_state_ = PIKA_REPL_NO_CONNECT; + int master_port_ = 0; + int repl_state_ = PIKA_REPL_NO_CONNECT; int role_ = PIKA_ROLE_SINGLE; int last_meta_sync_timestamp_ = 0; bool first_meta_sync_ = false; bool loop_slot_state_machine_ = false; bool force_full_sync_ = false; - bool leader_protected_mode_ = false; // reject request after master slave sync done - std::shared_mutex state_protector_; // protect below, use for master-slave mode + bool leader_protected_mode_ = false; // reject request after master slave sync done + std::shared_mutex state_protector_; // protect below, use for master-slave mode /* * Bgsave used @@ -428,6 +557,11 @@ class PikaServer : public pstd::noncopyable { */ std::unique_ptr pika_auxiliary_thread_; + /* + * Async slotsMgrt use + */ + std::unique_ptr pika_migrate_thread_; + /* * Slowlog used */ @@ -439,7 +573,6 @@ class PikaServer : public pstd::noncopyable { * Statistic used */ Statistic statistic_; - }; #endif diff --git a/include/pika_slot.h b/include/pika_slot.h index aecd506e45..0aa2a49cf0 100644 --- a/include/pika_slot.h +++ b/include/pika_slot.h @@ -52,7 +52,7 @@ class Slot : public std::enable_shared_from_this,public pstd::noncopyable virtual ~Slot(); std::string GetDBName() const; - uint32_t GetSlotId() const; + uint32_t GetSlotID() const; std::string GetSlotName() const; std::shared_ptr db() const; @@ -85,6 +85,11 @@ class Slot : public std::enable_shared_from_this,public pstd::noncopyable pstd::Status GetKeyNum(std::vector* key_info); KeyScanInfo GetKeyScanInfo(); + /* + * SlotsMgrt used + */ + void GetSlotsMgrtSenderStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained); + private: std::string db_name_; uint32_t slot_id_ = 0; diff --git a/include/pika_slot_command.h b/include/pika_slot_command.h new file mode 100644 index 0000000000..1ed14b4b97 --- /dev/null +++ b/include/pika_slot_command.h @@ -0,0 +1,280 @@ +#ifndef PIKA_SLOT_COMMAND_H_ +#define PIKA_SLOT_COMMAND_H_ + +#include "include/pika_client_conn.h" +#include "include/pika_command.h" +#include "include/pika_slot.h" +#include "net/include/net_cli.h" +#include "net/include/net_thread.h" +#include "storage/storage.h" +#include "strings.h" + +const std::string SlotKeyPrefix = "_internal:slotkey:4migrate:"; +const std::string SlotTagPrefix = "_internal:slottag:4migrate:"; +const size_t MaxKeySendSize = 10 * 1024; + +// crc32 +#define HASH_SLOTS_MASK 0x000003ff +#define HASH_SLOTS_SIZE (HASH_SLOTS_MASK + 1) + +extern uint32_t crc32tab[256]; + +void CRC32TableInit(uint32_t poly); + +extern void InitCRC32Table(); + +extern uint32_t CRC32Update(uint32_t crc, const char *buf, int len); +extern uint32_t CRC32CheckSum(const char *buf, int len); + +int GetSlotID(const std::string &str); +int GetKeyType(const std::string key, std::string &key_type, const std::shared_ptr& slot); +void AddSlotKey(const std::string type, const std::string key, const std::shared_ptr& slot); +void RemKeyNotExists(const std::string type, const std::string key, const std::shared_ptr& slot); +void RemSlotKey(const std::string key, const std::shared_ptr& slot); +int DeleteKey(const std::string key, const char key_type, const std::shared_ptr& slot); +std::string GetSlotKey(int slot); +std::string GetSlotsTagKey(uint32_t crc); +int GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag); +void RemSlotKeyByType(const std::string &type, const std::string &key, const std::shared_ptr& slot); +void WriteSAddToBinlog(const std::string &key, const std::string &value, const std::shared_ptr& slot); + +class PikaMigrate { + public: + PikaMigrate(); + virtual ~PikaMigrate(); + + int MigrateKey(const std::string &host, const int port, int timeout, const std::string &key, const char type, + std::string &detail, std::shared_ptr slot); + void CleanMigrateClient(); + + void Lock() { + mutex_.lock(); + } + int Trylock() { + return mutex_.try_lock(); + } + void Unlock() { + mutex_.unlock(); + } + net::NetCli *GetMigrateClient(const std::string &host, const int port, int timeout); + + private: + std::map migrate_clients_; + pstd::Mutex mutex_; + + void KillMigrateClient(net::NetCli *migrate_cli); + void KillAllMigrateClient(); + + int MigrateSend(net::NetCli *migrate_cli, const std::string &key, const char type, std::string &detail, + const std::shared_ptr& slot); + bool MigrateRecv(net::NetCli *migrate_cli, int need_receive, std::string &detail); + + int ParseKey(const std::string &key, const char type, std::string &wbuf_str, const std::shared_ptr& slot); + int64_t TTLByType(const char key_type, const std::string &key, const std::shared_ptr& slot); + int ParseKKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); + int ParseZKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); + int ParseSKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); + int ParseHKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); + int ParseLKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); + bool SetTTL(const std::string &key, std::string &wbuf_str, int64_t ttl); +}; + +class SlotsMgrtTagSlotCmd : public Cmd { + public: + SlotsMgrtTagSlotCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsMgrtTagSlotCmd(*this); } + private: + std::string dest_ip_; + int64_t dest_port_ = 0; + int64_t timeout_ms_ = 60; + int64_t slot_id_ = 0; + std::basic_string, std::allocator> key_; + + void DoInitial() override; +}; + +class SlotsMgrtTagSlotAsyncCmd : public Cmd { + public: + SlotsMgrtTagSlotAsyncCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag){} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge(){}; + Cmd* Clone() override { return new SlotsMgrtTagSlotAsyncCmd(*this); } + private: + std::string dest_ip_; + int64_t dest_port_ = 0; + int64_t timeout_ms_ = 60; + int64_t max_bulks_ = 0; + int64_t max_bytes_ = 0; + int64_t slot_id_ = 0; + int64_t keys_num_ = 0; + + void DoInitial() override; +}; + +class SlotsMgrtTagOneCmd : public Cmd { + public: + SlotsMgrtTagOneCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsMgrtTagOneCmd(*this); } + private: + std::string dest_ip_; + int64_t dest_port_ = 0; + int64_t timeout_ms_ = 60; + std::string key_; + int64_t slot_id_ = 0; + char key_type_ = '\0'; + void DoInitial() override; + int KeyTypeCheck(const std::shared_ptr& slot); +}; + +class SlotsMgrtAsyncStatusCmd : public Cmd { + public: + SlotsMgrtAsyncStatusCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot = nullptr) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsMgrtAsyncStatusCmd(*this); } + + private: + void DoInitial() override; +}; + +class SlotsInfoCmd : public Cmd { + public: + SlotsInfoCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsInfoCmd(*this); } + private: + void DoInitial() override; + + int64_t begin_ = 0; + int64_t end_ = 1024; +}; + +class SlotsMgrtAsyncCancelCmd : public Cmd { + public: + SlotsMgrtAsyncCancelCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsMgrtAsyncCancelCmd(*this); } + private: + void DoInitial() override; +}; + +class SlotsDelCmd : public Cmd { + public: + SlotsDelCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsDelCmd(*this); } + private: + std::vector slots_; + void DoInitial() override; +}; + +class SlotsHashKeyCmd : public Cmd { + public: + SlotsHashKeyCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsHashKeyCmd(*this); } + private: + std::vector keys_; + void DoInitial() override; +}; + +class SlotsScanCmd : public Cmd { + public: + SlotsScanCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsScanCmd(*this); } + private: + std::string key_; + std::string pattern_ = "*"; + int64_t cursor_ = 0; + int64_t count_ = 10; + void DoInitial() override; + void Clear() override { + pattern_ = "*"; + count_ = 10; + } +}; + +/* * +* SLOTSMGRT-EXEC-WRAPPER $hashkey $command [$arg1 ...] +* SLOTSMGRT-EXEC-WRAPPER $hashkey $command [$key1 $arg1 ...] +* SLOTSMGRT-EXEC-WRAPPER $hashkey $command [$key1 $arg1 ...] [$key2 $arg2 ...] +* */ +class SlotsMgrtExecWrapperCmd : public Cmd { + public: + SlotsMgrtExecWrapperCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsMgrtExecWrapperCmd(*this); } + private: + std::string key_; + std::vector args; + void DoInitial() override; +}; + + +class SlotsReloadCmd : public Cmd { + public: + SlotsReloadCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsReloadCmd(*this); } + private: + void DoInitial() override; +}; + +class SlotsReloadOffCmd : public Cmd { + public: + SlotsReloadOffCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptrslot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsReloadOffCmd(*this); } + private: + void DoInitial() override; +}; + +class SlotsCleanupCmd : public Cmd { + public: + SlotsCleanupCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsCleanupCmd(*this); } + std::vector cleanup_slots_; + private: + void DoInitial() override; +}; + +class SlotsCleanupOffCmd : public Cmd { + public: + SlotsCleanupOffCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} + void Do(std::shared_ptr slot) override; + void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Merge() override {}; + Cmd* Clone() override { return new SlotsCleanupOffCmd(*this); } + private: + void DoInitial() override; +}; + +#endif diff --git a/integrate_test.sh b/integrate_test.sh new file mode 100644 index 0000000000..cc555b3fd0 --- /dev/null +++ b/integrate_test.sh @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -e +set -x + +echo 'start integrate-test' + +# set root workspace +ROOT_DIR=$(pwd) +echo "integrate-test root work-space -> ${ROOT_DIR}" + +# show all github-env +echo "github current commit id -> $2" +echo "github pull request branch -> ${GITHUB_REF}" +echo "github pull request slug -> ${GITHUB_REPOSITORY}" +echo "github pull request repo slug -> ${GITHUB_REPOSITORY}" +echo "github pull request actor -> ${GITHUB_ACTOR}" +echo "github pull request repo param -> $1" +echo "github pull request base branch -> $3" +echo "github pull request head branch -> ${GITHUB_HEAD_REF}" + +git clone -b master https://github.com/luky116/pika-integration.git pika-integration && cd pika-integration + +chmod +x ./start_integrate_test.sh +# start integrate test +./start_integrate_test.sh diff --git a/src/net/include/net_cli.h b/src/net/include/net_cli.h index 8fff4b9c4b..dd5aab198c 100644 --- a/src/net/include/net_cli.h +++ b/src/net/include/net_cli.h @@ -36,6 +36,8 @@ class NetCli : public pstd::noncopyable { bool Available() const; + struct timeval last_interaction_; + // default connect timeout is 1000ms int set_send_timeout(int send_timeout); int set_recv_timeout(int recv_timeout); diff --git a/src/net/include/net_thread.h b/src/net/include/net_thread.h index 133a611dea..ac700819a5 100644 --- a/src/net/include/net_thread.h +++ b/src/net/include/net_thread.h @@ -28,7 +28,7 @@ class Thread : public pstd::noncopyable { void set_should_stop() { should_stop_.store(true); } - bool is_running() { return running_; } + bool is_running() { return running_.load(); } pthread_t thread_id() const { return thread_id_; } @@ -37,14 +37,18 @@ class Thread : public pstd::noncopyable { void set_thread_name(const std::string& name) { thread_name_ = name; } protected: - std::atomic should_stop_; + std::atomic_bool should_stop_; + void set_is_running(bool is_running) { + std::lock_guard l(running_mu_); + running_ = is_running; + } private: static void* RunThread(void* arg); virtual void* ThreadMain() = 0; pstd::Mutex running_mu_; - bool running_{false}; + std::atomic_bool running_ = false; pthread_t thread_id_{}; std::string thread_name_; }; diff --git a/src/net/src/net_thread.cc b/src/net/src/net_thread.cc index c6c669c5ee..a6a7b08994 100644 --- a/src/net/src/net_thread.cc +++ b/src/net/src/net_thread.cc @@ -24,6 +24,9 @@ void* Thread::RunThread(void* arg) { } int Thread::StartThread() { + if (!should_stop() && is_running()) { + return 0; + } std::lock_guard l(running_mu_); should_stop_ = false; if (!running_) { @@ -34,6 +37,9 @@ int Thread::StartThread() { } int Thread::StopThread() { + if (should_stop() && !is_running()) { + return 0; + } std::lock_guard l(running_mu_); should_stop_ = true; if (running_) { diff --git a/src/pika.cc b/src/pika.cc index b3355fd612..17bc22d674 100644 --- a/src/pika.cc +++ b/src/pika.cc @@ -15,7 +15,7 @@ #include "include/pika_rm.h" #include "include/pika_server.h" #include "include/pika_version.h" - +#include "include/pika_slot_command.h" #include "pstd/include/env.h" #include "pstd/include/pstd_defer.h" @@ -186,6 +186,7 @@ int main(int argc, char* argv[]) { PikaGlogInit(); PikaSignalSetup(); + InitCRC32Table(); LOG(INFO) << "Server at: " << path; g_pika_cmd_table_manager = std::make_unique(); diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 90a2531991..9222593fce 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -941,7 +941,7 @@ void InfoCmd::InfoReplication(std::string& info) { std::shared_lock slot_rwl(db_item.second->slots_rw_); for (const auto& slot_item : db_item.second->slots_) { std::shared_ptr slave_slot = g_pika_rm->GetSyncSlaveSlotByName( - SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotId())); + SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); if (!slave_slot) { out_of_sync << "(" << slot_item.second->GetSlotName() << ": InternalError)"; continue; @@ -1026,7 +1026,7 @@ void InfoCmd::InfoReplication(std::string& info) { std::shared_lock slot_rwl(t_item.second->slots_rw_); for (const auto& p_item : t_item.second->slots_) { std::string db_name = p_item.second->GetDBName(); - uint32_t slot_id = p_item.second->GetSlotId(); + uint32_t slot_id = p_item.second->GetSlotID(); master_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); if (!master_slot) { LOG(WARNING) << "Sync Master Slot: " << db_name << ":" << slot_id << ", NotFound"; @@ -1387,6 +1387,12 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeString(&config_body, g_pika_conf->daemonize() ? "yes" : "no"); } + if (pstd::stringmatch(pattern.data(), "slotmigrate", 1)) { + elements += 2; + EncodeString(&config_body, "slotmigrate"); + EncodeString(&config_body, g_pika_conf->slotmigrate() ? "yes" : "no"); + } + if (pstd::stringmatch(pattern.data(), "dump-path", 1) != 0) { elements += 2; EncodeString(&config_body, "dump-path"); @@ -1676,6 +1682,7 @@ void ConfigCmd::ConfigSet(std::string& ret) { EncodeString(&ret, "timeout"); EncodeString(&ret, "requirepass"); EncodeString(&ret, "masterauth"); + EncodeString(&ret, "slotmigrate"); EncodeString(&ret, "userpass"); EncodeString(&ret, "userblacklist"); EncodeString(&ret, "dump-prefix"); @@ -1724,6 +1731,9 @@ void ConfigCmd::ConfigSet(std::string& ret) { } else if (set_item == "userpass") { g_pika_conf->SetUserPass(value); ret = "+OK\r\n"; + } else if (set_item == "slotmigrate") { + g_pika_conf->SetSlotMigrate(value); + ret = "+OK\r\n"; } else if (set_item == "userblacklist") { g_pika_conf->SetUserBlackList(value); ret = "+OK\r\n"; diff --git a/src/pika_bit.cc b/src/pika_bit.cc index 2ccf4ac5c5..9784ac6295 100644 --- a/src/pika_bit.cc +++ b/src/pika_bit.cc @@ -8,6 +8,7 @@ #include "pstd/include/pstd_string.h" #include "include/pika_define.h" +#include "include/pika_slot_command.h" void BitSetCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -44,6 +45,7 @@ void BitSetCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->SetBit(key_, bit_offset_, on_, &bit_val); if (s.ok()) { res_.AppendInteger(static_cast(bit_val)); + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/pika_command.cc b/src/pika_command.cc index e00ce3c01e..eab81b250e 100644 --- a/src/pika_command.cc +++ b/src/pika_command.cc @@ -19,6 +19,7 @@ #include "include/pika_rm.h" #include "include/pika_server.h" #include "include/pika_set.h" +#include "include/pika_slot_command.h" #include "include/pika_zset.h" using pstd::Status; @@ -92,6 +93,50 @@ void InitCmdTable(CmdTable* cmd_table) { std::unique_ptr quitptr = std::make_unique(kCmdNameQuit, 1, kCmdFlagsRead); cmd_table->insert(std::pair>(kCmdNameQuit, std::move(quitptr))); + // Slots related + std::unique_ptr slotsinfoptr = std::make_unique(kCmdNameSlotsInfo, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsInfo, std::move(slotsinfoptr))); + std::unique_ptr slotmgrttagslotasyncptr = + std::make_unique(kCmdNameSlotsMgrtTagSlotAsync, 8, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtTagSlotAsync, std::move(slotmgrttagslotasyncptr))); + std::unique_ptr slotmgrtasyncstatus = + std::make_unique(kCmdNameSlotsMgrtAsyncStatus, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtAsyncStatus, std::move(slotmgrtasyncstatus))); + std::unique_ptr slotmgrtasynccancel = + std::make_unique(kCmdNameSlotsMgrtAsyncCancel, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtAsyncCancel, std::move(slotmgrtasynccancel))); + + std::unique_ptr slotmgrttagoneptr = std::make_unique(kCmdNameSlotsMgrtTagOne, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtTagOne, std::move(slotmgrttagoneptr))); + std::unique_ptr slotmgrtoneptr = std::make_unique(kCmdNameSlotsMgrtOne, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtOne, std::move(slotmgrtoneptr))); + + std::unique_ptr slotmgrttagslotptr = + std::make_unique(kCmdNameSlotsMgrtTagSlot, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtTagSlot, std::move(slotmgrttagslotptr))); + std::unique_ptr slotmgrttagslottagptr = + std::make_unique(kCmdNameSlotsMgrtSlot, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtSlot, std::move(slotmgrttagslottagptr))); + + std::unique_ptr slotsdelptr = std::make_unique(kCmdNameSlotsDel, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsDel, std::move(slotsdelptr))); + std::unique_ptr slotshashkeyptr = std::make_unique(kCmdNameSlotsHashKey, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsHashKey, std::move(slotshashkeyptr))); + std::unique_ptr slotsscanptr = std::make_unique(kCmdNameSlotsScan, -3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsScan, std::move(slotsscanptr))); + std::unique_ptr slotsmgrtexecwrapper = + std::make_unique(kCmdNameSlotsMgrtExecWrapper, -3, kCmdFlagsWrite | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsMgrtExecWrapper, std::move(slotsmgrtexecwrapper))); + std::unique_ptr slotsreloadptr = std::make_unique(kCmdNameSlotsReload, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsReload, std::move(slotsreloadptr))); + std::unique_ptr slotsreloadoffptr = std::make_unique(kCmdNameSlotsReloadOff, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsReloadOff, std::move(slotsreloadoffptr))); + std::unique_ptr slotscleanupptr = std::make_unique(kCmdNameSlotsCleanup, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsCleanup, std::move(slotscleanupptr))); + std::unique_ptr slotscleanupoffptr = std::make_unique(kCmdNameSlotsCleanupOff, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair>(kCmdNameSlotsCleanupOff, std::move(slotscleanupoffptr))); + + // Kv ////SetCmd std::unique_ptr setptr = @@ -327,8 +372,10 @@ void InitCmdTable(CmdTable* cmd_table) { std::unique_ptr lpushptr = std::make_unique(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); cmd_table->insert(std::pair>(kCmdNameLPush, std::move(lpushptr))); - std::unique_ptr lpushxptr = + std::unique_ptr lpushxptr = + std::make_unique(kCmdNameLPushx, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); + std::make_unique(kCmdNameLPushx, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); cmd_table->insert(std::pair>(kCmdNameLPushx, std::move(lpushxptr))); std::unique_ptr lrangeptr = std::make_unique(kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsList); @@ -351,9 +398,9 @@ void InitCmdTable(CmdTable* cmd_table) { std::unique_ptr rpushptr = std::make_unique(kCmdNameRPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); cmd_table->insert(std::pair>(kCmdNameRPush, std::move(rpushptr))); - std::unique_ptr rpushxptr = + std::unique_ptr rpushxptr = std::make_unique(kCmdNameRPushx, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); - + std::make_unique(kCmdNameRPushx, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); cmd_table->insert(std::pair>(kCmdNameRPushx, std::move(rpushxptr))); // Zset @@ -649,7 +696,7 @@ void Cmd::ProcessFlushDBCmd() { std::lock_guard s_prw(g_pika_rm->slots_rw_); for (const auto& slot_item : db->slots_) { std::shared_ptr slot = slot_item.second; - SlotInfo p_info(slot->GetDBName(), slot->GetSlotId()); + SlotInfo p_info(slot->GetDBName(), slot->GetSlotID()); if (g_pika_rm->sync_master_slots_.find(p_info) == g_pika_rm->sync_master_slots_.end()) { res_.SetRes(CmdRes::kErrOther, "Slot not found"); return; @@ -675,7 +722,7 @@ void Cmd::ProcessFlushAllCmd() { std::lock_guard s_prw(g_pika_rm->slots_rw_); for (const auto& slot_item : db_item.second->slots_) { std::shared_ptr slot = slot_item.second; - SlotInfo p_info(slot->GetDBName(), slot->GetSlotId()); + SlotInfo p_info(slot->GetDBName(), slot->GetSlotID()); if (g_pika_rm->sync_master_slots_.find(p_info) == g_pika_rm->sync_master_slots_.end()) { res_.SetRes(CmdRes::kErrOther, "Slot not found"); return; @@ -696,7 +743,7 @@ void Cmd::ProcessSingleSlotCmd() { } std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotId())); + g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); if (!sync_slot) { res_.SetRes(CmdRes::kErrOther, "Slot not found"); return; @@ -808,7 +855,7 @@ void Cmd::ProcessMultiSlotCmd() { return; } std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotId())); + g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); if (!sync_slot) { res_.SetRes(CmdRes::kErrOther, "Slot not found"); return; @@ -833,8 +880,9 @@ void Cmd::ProcessMultiSlotCmd() { } } -void Cmd::ProcessDoNotSpecifySlotCmd() { Do(); } +void Cmd::ProcessDoNotSpecifySlotCmd() {Do();} +bool Cmd::is_read() const {return ((flag_ & kCmdFlagsMaskRW) == kCmdFlagsRead);} bool Cmd::is_write() const { return ((flag_ & kCmdFlagsMaskRW) == kCmdFlagsWrite); } bool Cmd::is_local() const { return ((flag_ & kCmdFlagsMaskLocal) == kCmdFlagsLocal); } // Others need to be suspended when a suspend command run diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 1ce60ea3c1..7be6fe7e4c 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -170,7 +170,20 @@ int PikaConf::Load() { std::string swe; GetConfStr("slowlog-write-errorlog", &swe); - slowlog_write_errorlog_.store(swe == "yes"); + slowlog_write_errorlog_.store(swe == "yes" ? true : false); + + // slot migrate + std::string smgrt = "no"; + GetConfStr("slotmigrate", &smgrt); + slotmigrate_ = (smgrt == "yes") ? true : false; + + int binlog_writer_num = 1 ; + GetConfInt("binlog-writer-num", &binlog_writer_num); + if (binlog_writer_num <= 0 || binlog_writer_num > 24) { + binlog_writer_num_ = 1; + } else { + binlog_writer_num_ = binlog_writer_num; + } int tmp_slowlog_log_slower_than; GetConfInt("slowlog-log-slower-than", &tmp_slowlog_log_slower_than); @@ -346,6 +359,18 @@ int PikaConf::Load() { arena_block_size_ = write_buffer_size_ >> 3; // 1/8 of the write_buffer_size_ } + // arena_block_size + GetConfInt64Human("slotmigrate-thread-num_", &slotmigrate_thread_num_); + if (slotmigrate_thread_num_ < 1 || slotmigrate_thread_num_ > 24) { + slotmigrate_thread_num_ = 8; // 1/8 of the write_buffer_size_ + } + + // arena_block_size + GetConfInt64Human("thread-migrate-keys-num", &thread_migrate_keys_num_); + if (thread_migrate_keys_num_ < 64 || thread_migrate_keys_num_ > 128) { + thread_migrate_keys_num_ = 64; // 1/8 of the write_buffer_size_ + } + // max_write_buffer_size GetConfInt64Human("max-write-buffer-size", &max_write_buffer_size_); if (max_write_buffer_size_ <= 0) { @@ -590,6 +615,7 @@ int PikaConf::ConfigRewrite() { SetConfInt("max-write-buffer-num", max_write_buffer_num_); SetConfInt64("write-buffer-size", write_buffer_size_); SetConfInt64("arena-block-size", arena_block_size_); + SetConfInt64("slotmigrate", slotmigrate_); // slaveof config item is special SetConfStr("slaveof", slaveof_); diff --git a/src/pika_consensus.cc b/src/pika_consensus.cc index dcb80652e9..e43f055fc8 100644 --- a/src/pika_consensus.cc +++ b/src/pika_consensus.cc @@ -400,6 +400,10 @@ Status ConsensusCoordinator::ProposeLog(const std::shared_ptr& cmd_ptr, std return Status::OK(); } +Status ConsensusCoordinator::ProposeLog(const std::shared_ptr& cmd_ptr) { + return ProposeLog(cmd_ptr, nullptr, nullptr); +} + Status ConsensusCoordinator::InternalAppendLog(const BinlogItem& item, const std::shared_ptr& cmd_ptr, std::shared_ptr conn_ptr, std::shared_ptr resp_ptr) { diff --git a/src/pika_db.cc b/src/pika_db.cc index c8a2f699b4..5cfeddd6a5 100644 --- a/src/pika_db.cc +++ b/src/pika_db.cc @@ -233,7 +233,7 @@ void DB::LeaveAllSlot() { slots_.clear(); } -std::set DB::GetSlotIds() { +std::set DB::GetSlotIDs() { std::set ids; std::shared_lock l(slots_rw_); for (const auto& item : slots_) { diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 153604024a..9c14939203 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -8,6 +8,7 @@ #include "pstd/include/pstd_string.h" #include "include/pika_conf.h" +#include "include/pika_slot_command.h" extern std::unique_ptr g_pika_conf; @@ -48,6 +49,7 @@ void HSetCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->HSet(key_, field_, value_, &ret); if (s.ok()) { res_.AppendContent(":" + std::to_string(ret)); + AddSlotKey("h", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -161,6 +163,7 @@ void HIncrbyCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->HIncrby(key_, field_, by_, &new_value); if (s.ok() || s.IsNotFound()) { res_.AppendContent(":" + std::to_string(new_value)); + AddSlotKey("h", key_, slot); } else if (s.IsCorruption() && s.ToString() == "Corruption: hash value is not an integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s.IsInvalidArgument()) { @@ -186,6 +189,7 @@ void HIncrbyfloatCmd::Do(std::shared_ptr slot) { if (s.ok()) { res_.AppendStringLen(new_value.size()); res_.AppendContent(new_value); + AddSlotKey("h", key_, slot); } else if (s.IsCorruption() && s.ToString() == "Corruption: value is not a vaild float") { res_.SetRes(CmdRes::kInvalidFloat); } else if (s.IsInvalidArgument()) { @@ -286,6 +290,7 @@ void HMsetCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->HMSet(key_, fvs_); if (s.ok()) { res_.SetRes(CmdRes::kOk); + AddSlotKey("h", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -306,6 +311,7 @@ void HSetnxCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->HSetnx(key_, field_, value_, &ret); if (s.ok()) { res_.AppendContent(":" + std::to_string(ret)); + AddSlotKey("h", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 24d225882e..3d1f63c0c2 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -9,7 +9,7 @@ #include "include/pika_binlog_transverter.h" #include "include/pika_conf.h" -#include "include/pika_data_distribution.h" +#include "include/pika_slot_command.h" extern std::unique_ptr g_pika_conf; @@ -89,6 +89,7 @@ void SetCmd::Do(std::shared_ptr slot) { } else { if (res == 1) { res_.SetRes(CmdRes::kOk); + AddSlotKey("k", key_, slot); } else { res_.AppendStringLen(-1); } @@ -164,6 +165,10 @@ void DelCmd::Do(std::shared_ptr slot) { int64_t count = slot->db()->Del(keys_, &type_status); if (count >= 0) { res_.AppendInteger(count); + std::vector::const_iterator it; + for (it = keys_.begin(); it != keys_.end(); it++) { + RemSlotKey(*it, slot); + } } else { res_.SetRes(CmdRes::kErrOther, "delete error"); } @@ -195,6 +200,7 @@ void IncrCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Incrby(key_, 1, &new_value_); if (s.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); + AddSlotKey("k", key_, slot); } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s.IsInvalidArgument()) { @@ -220,6 +226,7 @@ void IncrbyCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Incrby(key_, by_, &new_value_); if (s.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); + AddSlotKey("k", key_, slot); } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s.IsInvalidArgument()) { @@ -247,6 +254,7 @@ void IncrbyfloatCmd::Do(std::shared_ptr slot) { if (s.ok()) { res_.AppendStringLen(new_value_.size()); res_.AppendContent(new_value_); + AddSlotKey("k", key_, slot); } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a vaild float") { res_.SetRes(CmdRes::kInvalidFloat); } else if (s.IsInvalidArgument()) { @@ -321,6 +329,7 @@ void GetsetCmd::Do(std::shared_ptr slot) { res_.AppendStringLen(old_value.size()); res_.AppendContent(old_value); } + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -340,6 +349,7 @@ void AppendCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Append(key_, value_, &new_len); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(new_len); + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -466,6 +476,7 @@ void SetnxCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Setnx(key_, value_, &success_); if (s.ok()) { res_.AppendInteger(success_); + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -510,6 +521,7 @@ void SetexCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Setex(key_, value_, sec_); if (s.ok()) { res_.SetRes(CmdRes::kOk); + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -623,12 +635,16 @@ void MsetCmd::DoInitial() { for (size_t index = 1; index != argc; index += 2) { kvs_.push_back({argv_[index], argv_[index + 1]}); } - } +} void MsetCmd::Do(std::shared_ptr slot) { storage::Status s = slot->db()->MSet(kvs_); if (s.ok()) { res_.SetRes(CmdRes::kOk); + std::vector::const_iterator it; + for (it = kvs_.begin(); it != kvs_.end(); it++) { + AddSlotKey("k", it->key, slot); + } } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -681,6 +697,10 @@ void MsetnxCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->MSetnx(kvs_, &success_); if (s.ok()) { res_.AppendInteger(success_); + std::vector::const_iterator it; + for (it = kvs_.begin(); it != kvs_.end(); it++) { + AddSlotKey("k", it->key, slot); + } } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -731,6 +751,7 @@ void SetrangeCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->Setrange(key_, offset_, value_, &new_len); if (s.ok()) { res_.AppendInteger(new_len); + AddSlotKey("k", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -1205,7 +1226,7 @@ void ScanxCmd::DoInitial() { } index++; } - } +} void ScanxCmd::Do(std::shared_ptr slot) { std::string next_key; @@ -1225,7 +1246,7 @@ void ScanxCmd::Do(std::shared_ptr slot) { } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } - } +} void PKSetexAtCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -1392,8 +1413,7 @@ void PKRScanRangeCmd::Do(std::shared_ptr slot) { std::string next_key; std::vector keys; std::vector kvs; - rocksdb::Status s = - slot->db()->PKRScanRange(type_, key_start_, key_end_, pattern_, limit_, &keys, &kvs, &next_key); + rocksdb::Status s = slot->db()->PKRScanRange(type_, key_start_, key_end_, pattern_, limit_, &keys, &kvs, &next_key); if (s.ok()) { res_.AppendArrayLen(2); diff --git a/src/pika_list.cc b/src/pika_list.cc index 79da6a7cf6..55b38c435a 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -6,6 +6,7 @@ #include "include/pika_list.h" #include "include/pika_data_distribution.h" #include "pstd/include/pstd_string.h" +#include "include/pika_slot_command.h" void LIndexCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -53,6 +54,7 @@ void LInsertCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LInsert(key_, dir_, pivot_, value_, &llen); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(llen); + AddSlotKey("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -91,6 +93,7 @@ void LPushCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LPush(key_, values_, &llen); if (s.ok()) { res_.AppendInteger(llen); + AddSlotKey("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -108,6 +111,7 @@ void LPopCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LPop(key_, &value); if (s.ok()) { res_.AppendString(value); + AddSlotKey("l", key_, slot); } else if (s.IsNotFound()) { res_.AppendStringLen(-1); } else { @@ -131,6 +135,7 @@ void LPushxCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LPushx(key_, values_, &llen); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(llen); + AddSlotKey("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -185,6 +190,7 @@ void LRemCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LRem(key_, count_, value_, &res); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(res); + RemKeyNotExists("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -207,6 +213,7 @@ void LSetCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->LSet(key_, index_, value_); if (s.ok()) { res_.SetRes(CmdRes::kOk); + AddSlotKey("l", key_, slot); } else if (s.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); } else if (s.IsCorruption() && s.ToString() == "Corruption: index out of range") { @@ -254,6 +261,7 @@ void RPopCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->RPop(key_, &value); if (s.ok()) { res_.AppendString(value); + RemKeyNotExists("l", key_, slot); } else if (s.IsNotFound()) { res_.AppendStringLen(-1); } else { @@ -328,6 +336,7 @@ void RPushCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->RPush(key_, values_, &llen); if (s.ok()) { res_.AppendInteger(llen); + RemKeyNotExists("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -349,6 +358,7 @@ void RPushxCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->RPushx(key_, values_, &llen); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(llen); + RemKeyNotExists("l", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/pika_migrate_thread.cc b/src/pika_migrate_thread.cc new file mode 100644 index 0000000000..2c99dde7e8 --- /dev/null +++ b/src/pika_migrate_thread.cc @@ -0,0 +1,1012 @@ +#include + +#include "include/pika_command.h" +#include "include/pika_conf.h" +#include "include/pika_define.h" +#include "include/pika_migrate_thread.h" +#include "include/pika_server.h" +#include "include/pika_slot_command.h" + +#include "include/pika_admin.h" +#include "include/pika_cmd_table_manager.h" +#include "include/pika_rm.h" + +#define min(a, b) (((a) > (b)) ? (b) : (a)) + +const int32_t MAX_MEMBERS_NUM = 512; +const std::string INVALID_STR = "NL"; + +extern std::unique_ptr g_pika_server; +extern std::unique_ptr g_pika_conf; +extern std::unique_ptr g_pika_rm; +extern std::unique_ptr g_pika_cmd_table_manager; + +// do migrate key to dest pika server +static int doMigrate(net::NetCli *cli, std::string send_str) { + pstd::Status s; + s = cli->Send(&send_str); + if (!s.ok()) { + LOG(WARNING) << "Slot Migrate Send error: " << s.ToString(); + return -1; + } + return 1; +} + +// do migrate cli auth +static int doAuth(net::NetCli *cli) { + net::RedisCmdArgsType argv; + std::string wbuf_str; + std::string requirepass = g_pika_conf->requirepass(); + if (requirepass != "") { + argv.push_back("auth"); + argv.push_back(requirepass); + } else { + argv.push_back("ping"); + } + net::SerializeRedisCommand(argv, &wbuf_str); + + pstd::Status s; + s = cli->Send(&wbuf_str); + if (!s.ok()) { + LOG(WARNING) << "Slot Migrate auth Send error: " << s.ToString(); + return -1; + } + // Recv + s = cli->Recv(&argv); + if (!s.ok()) { + LOG(WARNING) << "Slot Migrate auth Recv error: " << s.ToString(); + return -1; + } + pstd::StringToLower(argv[0]); + if (argv[0] != "ok" && argv[0] != "pong" && argv[0].find("no password") == std::string::npos) { + LOG(WARNING) << "Slot Migrate auth error: " << argv[0]; + return -1; + } + return 0; +} + +// get kv key value +static int kvGet(const std::string key, std::string &value, const std::shared_ptr& slot) { + rocksdb::Status s = slot->db()->Get(key, &value); + if (!s.ok()) { + if (s.IsNotFound()) { + value = ""; + LOG(WARNING) << "Get kv key: " << key << " not found "; + return 0; + } else { + value = ""; + LOG(WARNING) << "Get kv key: " << key << " error: " << s.ToString(); + return -1; + } + } + return 0; +} + +static int migrateKeyTTl(net::NetCli *cli, const std::string key, storage::DataType data_type, + const std::shared_ptr& slot) { + net::RedisCmdArgsType argv; + std::string send_str; + std::map type_timestamp; + std::map type_status; + type_timestamp = slot->db()->TTL(key, &type_status); + if (PIKA_TTL_ZERO == type_timestamp[data_type] || PIKA_TTL_STALE == type_timestamp[data_type]) { + argv.push_back("del"); + argv.push_back(key); + net::SerializeRedisCommand(argv, &send_str); + } else if (0 < type_timestamp[data_type]) { + argv.push_back("expire"); + argv.push_back(key); + argv.push_back(std::to_string(type_timestamp[data_type])); + net::SerializeRedisCommand(argv, &send_str); + } else { + // no expire + return 0; + } + + if (doMigrate(cli, send_str) < 0) { + return -1; + } + + return 1; +} + +// get set key all values +static int setGetall(const std::string key, std::vector *members, const std::shared_ptr& slot) { + rocksdb::Status s = slot->db()->SMembers(key, members); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(WARNING) << "Set get key: " << key << " value not found "; + return 0; + } else { + LOG(WARNING) << "Set get key: " << key << " value error: " << s.ToString(); + return -1; + } + } + return 1; +} + +static int MigrateKv(net::NetCli *cli, const std::string key, const std::shared_ptr& slot) { + std::string value; + rocksdb::Status s = slot->db()->Get(key, &value); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(WARNING) << "Get kv key: " << key << " not found "; + return 0; + } else { + LOG(WARNING) << "Get kv key: " << key << " error: " << strerror(errno); + return -1; + } + } + + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("SET"); + argv.push_back(key); + argv.push_back(value); + net::SerializeRedisCommand(argv, &send_str); + + int send_num = 0; + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + + int r; + if (0 > (r = migrateKeyTTl(cli, key, storage::kStrings, slot))) { + return -1; + } else { + send_num += r; + } + + return send_num; +} + +static int MigrateHash(net::NetCli *cli, const std::string key, const std::shared_ptr& slot) { + int send_num = 0; + int64_t cursor = 0; + std::vector field_values; + rocksdb::Status s; + + do { + s = slot->db()->HScan(key, cursor, "*", MAX_MEMBERS_NUM, &field_values, &cursor); + if (s.ok() && field_values.size() > 0) { + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("HMSET"); + argv.push_back(key); + for (const auto &field_value : field_values) { + argv.push_back(field_value.field); + argv.push_back(field_value.value); + } + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + } + } while (cursor != 0 && s.ok()); + + if (send_num > 0) { + int r; + if ((r = migrateKeyTTl(cli, key, storage::kHashes, slot)) < 0) { + return -1; + } else { + send_num += r; + } + } + + return send_num; +} + +static int MigrateList(net::NetCli *cli, const std::string key, const std::shared_ptr& slot) { + // del old key, before migrate list; prevent redo when failed + int send_num = 0; + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("DEL"); + argv.push_back(key); + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + + std::vector values; + rocksdb::Status s = slot->db()->LRange(key, 0, -1, &values); + if (s.ok()) { + auto iter = values.begin(); + while (iter != values.end()) { + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("RPUSH"); + argv.push_back(key); + + for (int i = 0; iter != values.end() && i < MAX_MEMBERS_NUM; ++iter, ++i) { + argv.push_back(*iter); + } + + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + } + } + + // has send del key command + if (send_num > 1) { + int r; + if (0 > (r = migrateKeyTTl(cli, key, storage::kLists, slot))) { + return -1; + } else { + send_num += r; + } + } + + return send_num; +} + +static int MigrateSet(net::NetCli *cli, const std::string key, const std::shared_ptr& slot) { + int send_num = 0; + int64_t cursor = 0; + std::vector members; + rocksdb::Status s; + + do { + s = slot->db()->SScan(key, cursor, "*", MAX_MEMBERS_NUM, &members, &cursor); + if (s.ok() && members.size() > 0) { + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("SADD"); + argv.push_back(key); + + for (const auto &member : members) { + argv.push_back(member); + } + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + } + } while (cursor != 0 && s.ok()); + + if (0 < send_num) { + int r; + if (0 > (r = migrateKeyTTl(cli, key, storage::kSets, slot))) { + return -1; + } else { + send_num += r; + } + } + + return send_num; +} + +static int MigrateZset(net::NetCli *cli, const std::string key, const std::shared_ptr& slot) { + int send_num = 0; + int64_t cursor = 0; + std::vector score_members; + rocksdb::Status s; + + do { + s = slot->db()->ZScan(key, cursor, "*", MAX_MEMBERS_NUM, &score_members, &cursor); + if (s.ok() && score_members.size() > 0) { + net::RedisCmdArgsType argv; + std::string send_str; + argv.push_back("ZADD"); + argv.push_back(key); + + for (const auto &score_member : score_members) { + argv.push_back(std::to_string(score_member.score)); + argv.push_back(score_member.member); + } + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ++send_num; + } + } + } while (cursor != 0 && s.ok()); + + if (send_num > 0) { + int r; + if ((r = migrateKeyTTl(cli, key, storage::kZSets, slot)) < 0) { + return -1; + } else { + send_num += r; + } + } + + return send_num; +} + +// get list key all values +static int listGetall(const std::string key, std::vector *values, const std::shared_ptr& slot) { + rocksdb::Status s = slot->db()->LRange(key, 0, -1, values); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(WARNING) << "List get key: " << key << " value not found "; + return 0; + } else { + LOG(WARNING) << "List get key: " << key << " value error: " << s.ToString(); + return -1; + } + } + return 1; +} + +// migrate one list key +static int migrateList(net::NetCli *cli, const std::string key, bool async, const std::shared_ptr& slot) { + int r, ret = 0; + std::vector values; + if (listGetall(key, &values, slot) < 0) { + return -1; + } + size_t keySize = values.size(); + if (keySize == 0) { + return 0; + } + + net::RedisCmdArgsType argv; + std::string send_str; + for (size_t i = 0; i <= keySize / MaxKeySendSize; ++i) { + if (i > 0) { + LOG(WARNING) << "Migrate big key: " << key << " size: " << keySize + << " migrated value: " << min((i + 1) * MaxKeySendSize, keySize); + } + argv.clear(); + send_str = ""; + argv.push_back("lpush"); + argv.push_back(key); + for (size_t j = i * MaxKeySendSize; j < (i + 1) * MaxKeySendSize && j < keySize; ++j) { + argv.push_back(values[j]); + } + net::SerializeRedisCommand(argv, &send_str); + if (doMigrate(cli, send_str) < 0) { + return -1; + } else { + ret++; + } + } + + if ((r = migrateKeyTTl(cli, key, storage::kLists, slot)) < 0) { + return -1; + } else { + ret += r; + } + + if (!async) { + DeleteKey(key, 'l', slot); // key already been migrated successfully, del error doesn't matter + } + return ret; +} + +PikaParseSendThread::PikaParseSendThread(PikaMigrateThread *migrate_thread, const std::shared_ptr& slot) + : dest_ip_("none"), + dest_port_(-1), + timeout_ms_(3000), + mgrtkeys_num_(64), + should_exit_(false), + migrate_thread_(migrate_thread), + slot_(slot), + cli_(NULL) {} + +PikaParseSendThread::~PikaParseSendThread() { + if (is_running()) { + should_exit_ = true; + StopThread(); + } + + if (cli_) { + delete cli_; + cli_ = NULL; + } +} + +bool PikaParseSendThread::Init(const std::string &ip, int64_t port, int64_t timeout_ms, int64_t mgrtkeys_num) { + dest_ip_ = ip; + dest_port_ = port; + timeout_ms_ = timeout_ms; + mgrtkeys_num_ = mgrtkeys_num; + + cli_ = net::NewRedisCli(); + cli_->set_connect_timeout(timeout_ms_); + cli_->set_send_timeout(timeout_ms_); + cli_->set_recv_timeout(timeout_ms_); + LOG(INFO) << "PikaParseSendThread init cli_, dest_ip_: " << dest_ip_ << " ,dest_port_: " << dest_port_; + pstd::Status result = cli_->Connect(dest_ip_, dest_port_, g_pika_server->host()); + if (!result.ok()) { + LOG(ERROR) << "PikaParseSendThread::Init failed. Connect server(" << dest_ip_ << ":" << dest_port_ << ") " + << result.ToString(); + return false; + } + + // do auth + if (doAuth(cli_) < 0) { + LOG(WARNING) << "PikaParseSendThread::Init do auth failed !!"; + cli_->Close(); + return false; + } + + return true; +} + +void PikaParseSendThread::ExitThread(void) { should_exit_ = true; } + +int PikaParseSendThread::MigrateOneKey(net::NetCli *cli, const std::string key, const char key_type, bool async) { + int send_num; + switch (key_type) { + case 'k': + if (0 > (send_num = MigrateKv(cli_, key, slot_))) { + return -1; + } + break; + case 'h': + if (0 > (send_num = MigrateHash(cli_, key, slot_))) { + return -1; + } + break; + case 'l': + if (0 > (send_num = MigrateList(cli_, key, slot_))) { + return -1; + } + break; + case 's': + if (0 > (send_num = MigrateSet(cli_, key, slot_))) { + return -1; + } + break; + case 'z': + if (0 > (send_num = MigrateZset(cli_, key, slot_))) { + return -1; + } + break; + default: + return -1; + break; + } + return send_num; +} + +void PikaParseSendThread::DelKeysAndWriteBinlog(std::deque> &send_keys, + const std::shared_ptr& slot) { + for (auto iter = send_keys.begin(); iter != send_keys.end(); ++iter) { + DeleteKey(iter->second, iter->first, slot); + WriteDelKeyToBinlog(iter->second, slot); + } +} + +// write del key to binlog for slave +void WriteDelKeyToBinlog(const std::string &key, const std::shared_ptr& slot) { + std::shared_ptr cmd_ptr = g_pika_cmd_table_manager->GetCmd("del"); + std::unique_ptr args = std::unique_ptr(new PikaCmdArgsType()); + args->push_back("DEL"); + args->push_back(key); + cmd_ptr->Initial(*args, slot->GetDBName()); + + std::shared_ptr sync_slot = + g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); + Status s = sync_slot->ConsensusProposeLog(cmd_ptr); + if (!s.ok()) { + LOG(ERROR) << "write delete key to binlog failed, key: " << key; + } +} + +bool PikaParseSendThread::CheckMigrateRecv(int64_t need_receive_num) { + net::RedisCmdArgsType argv; + for (int64_t i = 0; i < need_receive_num; ++i) { + pstd::Status s; + s = cli_->Recv(&argv); + if (!s.ok()) { + LOG(ERROR) << "PikaParseSendThread::CheckMigrateRecv Recv error: " << s.ToString(); + return false; + } + + // set return ok + // zadd return number + // hset return 0 or 1 + // hmset return ok + // sadd return number + // rpush return length + std::string reply = argv[0]; + int64_t ret; + if (1 == argv.size() && + (kInnerReplOk == pstd::StringToLower(reply) || pstd::string2int(reply.data(), reply.size(), &ret))) { + continue; + } else { + LOG(ERROR) << "PikaParseSendThread::CheckMigrateRecv reply error: " << reply; + return false; + } + } + return true; +} + +void *PikaParseSendThread::ThreadMain() { + while (!should_exit_) { + std::deque> send_keys; + { + std::unique_lock lq(migrate_thread_->mgrtkeys_queue_mutex_); + while (!should_exit_ && 0 >= migrate_thread_->mgrtkeys_queue_.size()) { + migrate_thread_->mgrtkeys_cond_.wait(lq); + } + + if (should_exit_) { + LOG(INFO) << "PikaParseSendThread::ThreadMain :" << pthread_self() << " exit !!!"; + return NULL; + } + + migrate_thread_->IncWorkingThreadNum(); + for (int32_t i = 0; i < mgrtkeys_num_; ++i) { + if (migrate_thread_->mgrtkeys_queue_.empty()) { + break; + } + send_keys.push_back(migrate_thread_->mgrtkeys_queue_.front()); + migrate_thread_->mgrtkeys_queue_.pop_front(); + } + } + + int64_t send_num = 0; + int64_t need_receive_num = 0; + int32_t migrate_keys_num = 0; + for (auto iter = send_keys.begin(); iter != send_keys.end(); ++iter) { + if (0 > (send_num = MigrateOneKey(cli_, iter->second, iter->first, false))) { + LOG(WARNING) << "PikaParseSendThread::ThreadMain MigrateOneKey: " << iter->second << " failed !!!"; + migrate_thread_->OnTaskFailed(); + migrate_thread_->DecWorkingThreadNum(); + return NULL; + } else { + need_receive_num += send_num; + ++migrate_keys_num; + } + } + + // check response + if (!CheckMigrateRecv(need_receive_num)) { + LOG(INFO) << "PikaMigrateThread::ThreadMain CheckMigrateRecv failed !!!"; + migrate_thread_->OnTaskFailed(); + migrate_thread_->DecWorkingThreadNum(); + return NULL; + } else { + DelKeysAndWriteBinlog(send_keys, slot_); + } + + migrate_thread_->AddResponseNum(migrate_keys_num); + migrate_thread_->DecWorkingThreadNum(); + } + + return NULL; +} + +PikaMigrateThread::PikaMigrateThread() + : net::Thread(), + dest_ip_("none"), + dest_port_(-1), + timeout_ms_(3000), + slot_id_(-1), + keys_num_(-1), + is_migrating_(false), + should_exit_(false), + is_task_success_(true), + send_num_(0), + response_num_(0), + moved_num_(0), + request_migrate_(false), + workers_num_(8), + working_thread_num_(0), + cursor_(0) {} + +PikaMigrateThread::~PikaMigrateThread() { + LOG(INFO) << "PikaMigrateThread::~PikaMigrateThread"; + + if (is_running()) { + should_exit_ = true; + NotifyRequestMigrate(); + workers_cond_.notify_all(); + StopThread(); + } +} + +bool PikaMigrateThread::ReqMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_id, + int64_t keys_num, const std::shared_ptr& slot) { + if (migrator_mutex_.try_lock()) { + if (is_migrating_) { + if (dest_ip_ != ip || dest_port_ != port || slot_id_ != slot_id) { + LOG(INFO) << "PikaMigrateThread::ReqMigrate current: " << dest_ip_ << ":" << dest_port_ << " slot[" << slot_id_ + << "]" + << "request: " << ip << ":" << port << " slot[" << slot << "]"; + migrator_mutex_.unlock(); + return false; + } + slot_ = slot; + timeout_ms_ = time_out; + keys_num_ = keys_num; + NotifyRequestMigrate(); + migrator_mutex_.unlock(); + return true; + } else { + dest_ip_ = ip; + dest_port_ = port; + timeout_ms_ = time_out; + slot_id_ = slot_id; + keys_num_ = keys_num; + should_exit_ = false; + slot_ = slot; + + ResetThread(); + int ret = StartThread(); + if (0 != ret) { + LOG(ERROR) << "PikaMigrateThread::ReqMigrateBatch StartThread failed. " + << " ret=" << ret; + is_migrating_ = false; + StopThread(); + } else { + LOG(INFO) << "PikaMigrateThread::ReqMigrateBatch slot: " << slot_id; + is_migrating_ = true; + NotifyRequestMigrate(); + } + migrator_mutex_.unlock(); + return true; + } + } + return false; +} + +int PikaMigrateThread::ReqMigrateOne(const std::string &key, const std::shared_ptr& slot) { + std::unique_lock lm(migrator_mutex_); + + int slot_id = GetSlotID(key); + std::string type_str; + char key_type; + rocksdb::Status s = slot->db()->Type(key, &type_str); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(INFO) << "PikaMigrateThread::ReqMigrateOne key: " << key << " not found"; + return 0; + } else { + LOG(WARNING) << "PikaMigrateThread::ReqMigrateOne key: " << key << " error: " << strerror(errno); + return -1; + } + } + + if (type_str == "string") { + key_type = 'k'; + } else if (type_str == "hash") { + key_type = 'h'; + } else if (type_str == "list") { + key_type = 'l'; + } else if (type_str == "set") { + key_type = 's'; + } else if (type_str == "zset") { + key_type = 'z'; + } else if (type_str == "none") { + return 0; + } else { + LOG(WARNING) << "PikaMigrateThread::ReqMigrateOne key: " << key << " type: " << type_str << " is illegal"; + return -1; + } + + if (slot_id != slot_id_) { + LOG(WARNING) << "PikaMigrateThread::ReqMigrateOne Slot : " << slot_id << " is not the migrating slot:" << slot_id_; + return -2; + } + + // if the migrate thread exit, start it + if (!is_migrating_) { + ResetThread(); + int ret = StartThread(); + if (0 != ret) { + LOG(ERROR) << "PikaMigrateThread::ReqMigrateOne StartThread failed. " + << " ret=" << ret; + is_migrating_ = false; + StopThread(); + } else { + LOG(INFO) << "PikaMigrateThread::ReqMigrateOne StartThread"; + is_migrating_ = true; + usleep(100); + } + } else { + // check the key is migrating + std::pair kpair = std::make_pair(key_type, key); + if (IsMigrating(kpair)) { + LOG(INFO) << "PikaMigrateThread::ReqMigrateOne key: " << key << " is migrating ! "; + return 1; + } else { + std::unique_lock lo(mgrtone_queue_mutex_); + mgrtone_queue_.push_back(kpair); + NotifyRequestMigrate(); + } + } + + return 1; +} + +void PikaMigrateThread::GetMigrateStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, + int64_t *remained) { + std::unique_lock lm(migrator_mutex_); + // todo for sure + if (!is_migrating_) { + *remained = -1; + return; + } + + *ip = dest_ip_; + *port = dest_port_; + *slot = slot_id_; + *migrating = is_migrating_; + *moved = moved_num_; + std::unique_lock lq(mgrtkeys_queue_mutex_); + int64_t migrating_keys_num = mgrtkeys_queue_.size(); + std::string slotKey = GetSlotKey(slot_id_); + int32_t slot_size = 0; + rocksdb::Status s = slot_->db()->SCard(slotKey, &slot_size); + if (s.ok()) { + *remained = slot_size + migrating_keys_num; + } else { + *remained = migrating_keys_num; + } +} + +void PikaMigrateThread::CancelMigrate(void) { + LOG(INFO) << "PikaMigrateThread::CancelMigrate"; + + if (is_running()) { + should_exit_ = true; + NotifyRequestMigrate(); + workers_cond_.notify_one(); + StopThread(); + } +} + +void PikaMigrateThread::IncWorkingThreadNum(void) { ++working_thread_num_; } + +void PikaMigrateThread::DecWorkingThreadNum(void) { + std::unique_lock lw(workers_mutex_); + --working_thread_num_; + workers_cond_.notify_one(); +} + +void PikaMigrateThread::OnTaskFailed() { + LOG(ERROR) << "PikaMigrateThread::OnTaskFailed !!!"; + is_task_success_ = false; +} + +void PikaMigrateThread::AddResponseNum(int32_t response_num) { response_num_ += response_num; } + +void PikaMigrateThread::ResetThread(void) { + if (0 != thread_id()) { + JoinThread(); + } +} + +void PikaMigrateThread::DestroyThread(bool is_self_exit) { + std::unique_lock lm(migrator_mutex_); + LOG(INFO) << "PikaMigrateThread::DestroyThread"; + + // Destroy work threads + DestroyParseSendThreads(); + + if (is_self_exit) { + set_is_running(false); + } + + { + std::unique_lock lq(mgrtkeys_queue_mutex_); + std::unique_lock lm(mgrtkeys_map_mutex_); + std::deque>().swap(mgrtkeys_queue_); + std::map, std::string>().swap(mgrtkeys_map_); + } + + cursor_ = 0; + is_migrating_ = false; + is_task_success_ = true; + moved_num_ = 0; +} + +void PikaMigrateThread::NotifyRequestMigrate(void) { + std::unique_lock lr(request_migrate_mutex_); + request_migrate_ = true; + request_migrate_cond_.notify_one(); +} + +bool PikaMigrateThread::IsMigrating(std::pair &kpair) { + std::unique_lock lo(mgrtone_queue_mutex_); + std::unique_lock lm(mgrtkeys_map_mutex_); + + for (auto iter = mgrtone_queue_.begin(); iter != mgrtone_queue_.end(); ++iter) { + if (iter->first == kpair.first && iter->second == kpair.second) { + return true; + } + } + + auto iter = mgrtkeys_map_.find(kpair); + if (iter != mgrtkeys_map_.end()) { + return true; + } + + return false; +} + +void PikaMigrateThread::ReadSlotKeys(const std::string &slotKey, int64_t need_read_num, int64_t &real_read_num, + int32_t *finish) { + real_read_num = 0; + std::string key; + char key_type; + int32_t is_member = 0; + std::vector members; + + rocksdb::Status s = slot_->db()->SScan(slotKey, cursor_, "*", need_read_num, &members, &cursor_); + if (s.ok() && 0 < members.size()) { + for (const auto &member : members) { + slot_->db()->SIsmember(slotKey, member, &is_member); + if (is_member) { + key = member; + key_type = key.at(0); + key.erase(key.begin()); + std::pair kpair = std::make_pair(key_type, key); + if (mgrtkeys_map_.find(kpair) == mgrtkeys_map_.end()) { + mgrtkeys_queue_.push_back(kpair); + mgrtkeys_map_[kpair] = INVALID_STR; + ++real_read_num; + } + } else { + LOG(INFO) << "PikaMigrateThread::ReadSlotKeys key " << member << " not found in" << slotKey; + } + } + } + + *finish = (0 == cursor_) ? 1 : 0; +} + +bool PikaMigrateThread::CreateParseSendThreads(int32_t dispatch_num) { + workers_num_ = g_pika_conf->slotmigrate_thread_num(); + for (int32_t i = 0; i < workers_num_; ++i) { + PikaParseSendThread *worker = new PikaParseSendThread(this, slot_); + if (!worker->Init(dest_ip_, dest_port_, timeout_ms_, dispatch_num)) { + delete worker; + DestroyParseSendThreads(); + return false; + } else { + int ret = worker->StartThread(); + if (0 != ret) { + LOG(INFO) << "PikaMigrateThread::CreateParseSendThreads start work thread failed ret=" << ret; + delete worker; + DestroyParseSendThreads(); + return false; + } else { + workers_.push_back(worker); + } + } + } + return true; +} + +void PikaMigrateThread::DestroyParseSendThreads(void) { + if (!workers_.empty()) { + for (auto iter = workers_.begin(); iter != workers_.end(); ++iter) { + (*iter)->ExitThread(); + } + + { + std::unique_lock lm(mgrtkeys_queue_mutex_); + mgrtkeys_cond_.notify_all(); + } + + for (auto iter = workers_.begin(); iter != workers_.end(); ++iter) { + delete *iter; + } + workers_.clear(); + } +} + +void *PikaMigrateThread::ThreadMain() { + LOG(INFO) << "PikaMigrateThread::ThreadMain Start"; + + // Create parse_send_threads + int32_t dispatch_num = g_pika_conf->thread_migrate_keys_num(); + if (!CreateParseSendThreads(dispatch_num)) { + LOG(INFO) << "PikaMigrateThread::ThreadMain CreateParseSendThreads failed !!!"; + DestroyThread(true); + return NULL; + } + + std::string slotKey = GetSlotKey(slot_id_); + int32_t slot_size = 0; + slot_->db()->SCard(slotKey, &slot_size); + + while (!should_exit_) { + // Waiting migrate task + { + std::unique_lock lm(request_migrate_mutex_); + while (!request_migrate_) { + request_migrate_cond_.wait(lm); + } + request_migrate_ = false; + + if (should_exit_) { + LOG(INFO) << "PikaMigrateThread::ThreadMain :" << pthread_self() << " exit1 !!!"; + DestroyThread(false); + return NULL; + } + } + + // read keys form slot and push to mgrtkeys_queue_ + int64_t round_remained_keys = keys_num_; + int64_t real_read_num = 0; + int32_t is_finish = 0; + send_num_ = 0; + response_num_ = 0; + do { + std::unique_lock lq(mgrtkeys_queue_mutex_); + std::unique_lock lo(mgrtone_queue_mutex_); + std::unique_lock lm(mgrtkeys_map_mutex_); + + // first check whether need migrate one key + if (!mgrtone_queue_.empty()) { + while (!mgrtone_queue_.empty()) { + mgrtkeys_queue_.push_front(mgrtone_queue_.front()); + mgrtkeys_map_[mgrtone_queue_.front()] = INVALID_STR; + mgrtone_queue_.pop_front(); + ++send_num_; + } + } else { + int64_t need_read_num = (0 < round_remained_keys - dispatch_num) ? dispatch_num : round_remained_keys; + ReadSlotKeys(slotKey, need_read_num, real_read_num, &is_finish); + round_remained_keys -= need_read_num; + send_num_ += real_read_num; + } + mgrtkeys_cond_.notify_all(); + + } while (0 < round_remained_keys && !is_finish); + + LOG(INFO) << "PikaMigrateThread:: wait ParseSenderThread finish"; + // wait all ParseSenderThread finish + { + std::unique_lock lw(workers_mutex_); + while (!should_exit_ && is_task_success_ && send_num_ != response_num_) { + workers_cond_.wait(lw); + } + } + LOG(INFO) << "PikaMigrateThread::ThreadMain send_num:" << send_num_ << " response_num:" << response_num_; + + if (should_exit_) { + LOG(INFO) << "PikaMigrateThread::ThreadMain :" << pthread_self() << " exit2 !!!"; + DestroyThread(false); + return NULL; + } + + // check one round migrate task success + if (!is_task_success_) { + LOG(ERROR) << "PikaMigrateThread::ThreadMain one round migrate task failed !!!"; + DestroyThread(true); + return NULL; + } else { + moved_num_ += response_num_; + + std::unique_lock lm(mgrtkeys_map_mutex_); + std::map, std::string>().swap(mgrtkeys_map_); + } + + // check slot migrate finish + int32_t slot_remained_keys = 0; + slot_->db()->SCard(slotKey, &slot_remained_keys); + if (0 == slot_remained_keys) { + LOG(INFO) << "PikaMigrateThread::ThreadMain slot_size:" << slot_size << " moved_num:" << moved_num_; + if (slot_size != moved_num_) { + LOG(ERROR) << "PikaMigrateThread::ThreadMain moved_num != slot_size !!!"; + } + DestroyThread(true); + return NULL; + } + } + + return NULL; +} + +/* EOF */ diff --git a/src/pika_rm.cc b/src/pika_rm.cc index 7e299cd455..4dc6095590 100644 --- a/src/pika_rm.cc +++ b/src/pika_rm.cc @@ -478,6 +478,10 @@ Status SyncMasterSlot::ConsensusProposeLog(const std::shared_ptr& cmd_ptr, return coordinator_.ProposeLog(cmd_ptr, std::move(conn_ptr), std::move(resp_ptr)); } +Status SyncMasterSlot::ConsensusProposeLog(const std::shared_ptr& cmd_ptr) { + return coordinator_.ProposeLog(cmd_ptr); +} + Status SyncMasterSlot::ConsensusSanityCheck() { return coordinator_.CheckEnoughFollower(); } Status SyncMasterSlot::ConsensusProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute) { diff --git a/src/pika_server.cc b/src/pika_server.cc index 31c64b5fd0..521570e96d 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -74,6 +74,8 @@ PikaServer::PikaServer() pika_rsync_service_ = std::make_unique(g_pika_conf->db_sync_path(), g_pika_conf->port() + kPortShiftRSync); pika_pubsub_thread_ = std::make_unique(); pika_auxiliary_thread_ = std::make_unique(); + pika_migrate_ = std::make_unique(); + pika_migrate_thread_ = std::make_unique(); pika_client_processor_ = std::make_unique(g_pika_conf->thread_pool_size(), 100000); exit_mutex_.lock(); @@ -94,6 +96,7 @@ PikaServer::~PikaServer() { } bgsave_thread_.StopThread(); key_scan_thread_.StopThread(); + pika_migrate_thread_->StopThread(); dbs_.clear(); @@ -358,7 +361,7 @@ std::set PikaServer::GetDBSlotIds(const std::string& db_name) { std::set empty; std::shared_lock l(dbs_rw_); auto iter = dbs_.find(db_name); - return (iter == dbs_.end()) ? empty : iter->second->GetSlotIds(); + return (iter == dbs_.end()) ? empty : iter->second->GetSlotIDs(); } bool PikaServer::IsBgSaving() { @@ -478,7 +481,7 @@ void PikaServer::PrepareSlotTrySync() { for (const auto& slot_item : db_item.second->slots_) { Status s = g_pika_rm->ActivateSyncSlaveSlot( RmNode(g_pika_server->master_ip(), g_pika_server->master_port(), db_item.second->GetDBName(), - slot_item.second->GetSlotId()), + slot_item.second->GetSlotID()), state); if (!s.ok()) { LOG(WARNING) << s.ToString(); @@ -547,19 +550,19 @@ Status PikaServer::DoSameThingEverySlot(const TaskType& type) { switch (type) { case TaskType::kResetReplState: { slave_slot = g_pika_rm->GetSyncSlaveSlotByName( - SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotId())); + SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); if (!slave_slot) { LOG(WARNING) << "Slave Slot: " << db_item.second->GetDBName() << ":" - << slot_item.second->GetSlotId() << " Not Found"; + << slot_item.second->GetSlotID() << " Not Found"; } slave_slot->SetReplState(ReplState::kNoConnect); break; } case TaskType::kPurgeLog: { std::shared_ptr slot = g_pika_rm->GetSyncMasterSlotByName( - SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotId())); + SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); if (!slot) { - LOG(WARNING) << db_item.second->GetDBName() << slot_item.second->GetSlotId() + LOG(WARNING) << db_item.second->GetDBName() << slot_item.second->GetSlotID() << " Not Found."; break; } @@ -790,10 +793,10 @@ bool PikaServer::AllSlotConnectSuccess() { for (const auto& db_item : dbs_) { for (const auto& slot_item : db_item.second->slots_) { slave_slot = g_pika_rm->GetSyncSlaveSlotByName( - SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotId())); + SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); if (!slave_slot) { LOG(WARNING) << "Slave Slot: " << db_item.second->GetDBName() << ":" - << slot_item.second->GetSlotId() << ", NotFound"; + << slot_item.second->GetSlotID() << ", NotFound"; return false; } @@ -1461,7 +1464,7 @@ void PikaServer::InitStorageOptions() { storage_options_.options.compression_per_level.push_back(storage_options_.options.compression); } - // For rocksdb::BlockBasedTableOptions + // For rocksdb::BlockBasedDBOptions storage_options_.table_options.block_size = g_pika_conf->block_size(); storage_options_.table_options.cache_index_and_filter_blocks = g_pika_conf->cache_index_and_filter_blocks(); storage_options_.block_cache_size = g_pika_conf->block_cache(); @@ -1541,3 +1544,162 @@ void PikaServer::ServerStatus(std::string* info) { tmp_stream << "Client Processor thread-pool queue size: " << q_size << "\r\n"; info->append(tmp_stream.str()); } + +bool PikaServer:: SlotsMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_num,int64_t keys_num, std::shared_ptrslot) { + return pika_migrate_thread_->ReqMigrateBatch(ip, port, time_out, slot_num, keys_num, slot); +} + +void PikaServer:: GetSlotsMgrtSenderStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained){ + return pika_migrate_thread_->GetMigrateStatus(ip, port, slot, migrating, moved, remained); +} + +int PikaServer:: SlotsMigrateOne(const std::string &key, std::shared_ptrslot) { + return pika_migrate_thread_->ReqMigrateOne(key, slot); +} + +bool PikaServer::SlotsMigrateAsyncCancel() { + pika_migrate_thread_->CancelMigrate(); + return true; +} + +void PikaServer::Bgslotsreload(std::shared_ptrslot) { + // Only one thread can go through + { + std::lock_guard ml(bgsave_protector_); + if (bgslots_reload_.reloading || bgsave_info_.bgsaving) { + return; + } + bgslots_reload_.reloading = true; + } + + bgslots_reload_.start_time = time(nullptr); + char s_time[32]; + int len = strftime(s_time, sizeof(s_time), "%Y%m%d%H%M%S", localtime(&bgslots_reload_.start_time)); + bgslots_reload_.s_start_time.assign(s_time, len); + bgslots_reload_.cursor = 0; + bgslots_reload_.pattern = "*"; + bgslots_reload_.count = 100; + bgslots_reload_.slot = slot; + + LOG(INFO) << "Start slot reloading"; + + // Start new thread if needed + bgsave_thread_.StartThread(); + bgsave_thread_.Schedule(&DoBgslotsreload, static_cast(this)); +} + +void DoBgslotsreload(void* arg) { + PikaServer* p = static_cast(arg); + PikaServer::BGSlotsReload reload = p->bgslots_reload(); + + // Do slotsreload + rocksdb::Status s; + std::vector keys; + int64_t cursor_ret = -1; + while(cursor_ret != 0 && p->GetSlotsreloading()){ + cursor_ret = reload.slot->db()->Scan(storage::DataType::kAll, reload.cursor, reload.pattern, reload.count, &keys); + + std::vector::const_iterator iter; + for (iter = keys.begin(); iter != keys.end(); iter++){ + std::string key_type; + + int s = GetKeyType(*iter, key_type, reload.slot); + //if key is slotkey, can't add to SlotKey + if (s > 0){ + if (key_type == "s" && ((*iter).find(SlotKeyPrefix) != std::string::npos || (*iter).find(SlotTagPrefix) != std::string::npos)){ + continue; + } + + AddSlotKey(key_type, *iter, reload.slot); + } + } + + reload.cursor = cursor_ret; + p->SetSlotsreloadingCursor(cursor_ret); + keys.clear(); + } + p->SetSlotsreloading(false); + + if (cursor_ret == 0) { + LOG(INFO) << "Finish slot reloading"; + } else{ + LOG(INFO) << "Stop slot reloading"; + } +} + +void PikaServer::Bgslotscleanup(std::vector cleanupSlots, std::shared_ptr slot) { + // Only one thread can go through + { + std::lock_guard ml(bgsave_protector_); + if (bgslots_cleanup_.cleaningup || bgslots_reload_.reloading || bgsave_info_.bgsaving) { + return; + } + bgslots_cleanup_.cleaningup = true; + } + + bgslots_cleanup_.start_time = time(NULL); + char s_time[32]; + int len = strftime(s_time, sizeof(s_time), "%Y%m%d%H%M%S", localtime(&bgslots_cleanup_.start_time)); + bgslots_cleanup_.s_start_time.assign(s_time, len); + bgslots_cleanup_.cursor = 0; + bgslots_cleanup_.pattern = "*"; + bgslots_cleanup_.count = 100; + bgslots_cleanup_.slot = slot; + bgslots_cleanup_.cleanup_slots.swap(cleanupSlots); + + std::string slotsStr; + slotsStr.assign(cleanupSlots.begin(), cleanupSlots.end()); + LOG(INFO) << "Start slot cleanup, slots: " << slotsStr << std::endl; + + // Start new thread if needed + bgslots_cleanup_thread_.StartThread(); + bgslots_cleanup_thread_.Schedule(&DoBgslotscleanup, static_cast(this)); +} + +void DoBgslotscleanup(void* arg) { + PikaServer* p = static_cast(arg); + PikaServer::BGSlotsCleanup cleanup = p->bgslots_cleanup(); + + // Do slotscleanup + std::vector keys; + int64_t cursor_ret = -1; + std::vector cleanupSlots(cleanup.cleanup_slots); + while (cursor_ret != 0 && p->GetSlotscleaningup()){ + cursor_ret = g_pika_server->bgslots_cleanup_.slot->db()->Scan(storage::DataType::kAll, cleanup.cursor, cleanup.pattern, cleanup.count, &keys); + + std::string key_type; + std::vector::const_iterator iter; + for (iter = keys.begin(); iter != keys.end(); iter++){ + if ((*iter).find(SlotKeyPrefix) != std::string::npos || (*iter).find(SlotTagPrefix) != std::string::npos){ + continue; + } + if (std::find(cleanupSlots.begin(), cleanupSlots.end(), GetSlotID(*iter)) != cleanupSlots.end()){ + if (GetKeyType(*iter, key_type, g_pika_server->bgslots_cleanup_.slot) <= 0) { + LOG(WARNING) << "slots clean get key type for slot " << GetSlotID(*iter) << " key " << *iter << " error"; + continue ; + } + + if (DeleteKey(*iter, key_type[0], g_pika_server->bgslots_cleanup_.slot) <= 0){ + LOG(WARNING) << "slots clean del for slot " << GetSlotID(*iter) << " key "<< *iter << " error"; + } + } + } + + cleanup.cursor = cursor_ret; + p->SetSlotscleaningupCursor(cursor_ret); + keys.clear(); + } + + for (std::vector::const_iterator iter = cleanupSlots.begin(); iter != cleanupSlots.end(); iter++){ + WriteDelKeyToBinlog(GetSlotKey(*iter), g_pika_server->bgslots_cleanup_.slot); + WriteDelKeyToBinlog(GetSlotsTagKey(*iter), g_pika_server->bgslots_cleanup_.slot); + } + + p->SetSlotscleaningup(false); + std::vector empty; + p->SetCleanupSlots(empty); + + std::string slotsStr; + slotsStr.assign(cleanup.cleanup_slots.begin(), cleanup.cleanup_slots.end()); + LOG(INFO) << "Finish slots cleanup, slots " << slotsStr; +} diff --git a/src/pika_set.cc b/src/pika_set.cc index 1ee839ffdb..f8e4c507cc 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -5,6 +5,7 @@ #include "include/pika_set.h" +#include "include/pika_slot_command.h" #include "pstd/include/pstd_string.h" void SAddCmd::DoInitial() { @@ -24,20 +25,20 @@ void SAddCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->SAdd(key_, members_, &count); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); + AddSlotKey("s", key_, slot); return; } res_.AppendInteger(count); } void SPopCmd::DoInitial() { - size_t argc = argv_.size(); size_t index = 2; if (!CheckArg(argc)) { res_.SetRes(CmdRes::kWrongNum, kCmdNameSPop); return; } - + key_ = argv_[1]; count_ = 1; @@ -61,6 +62,7 @@ void SPopCmd::Do(std::shared_ptr slot) { for (const auto& member : members) { res_.AppendStringLen(member.size()); res_.AppendContent(member); + AddSlotKey("s", key_, slot); } } else if (s.IsNotFound()) { res_.AppendContent("$-1"); @@ -183,6 +185,7 @@ void SRemCmd::Do(std::shared_ptr slot) { int32_t count = 0; rocksdb::Status s = slot->db()->SRem(key_, members_, &count); res_.AppendInteger(count); + AddSlotKey("s", key_, slot); } void SUnionCmd::DoInitial() { @@ -339,6 +342,7 @@ void SMoveCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->SMove(src_key_, dest_key_, member_, &res); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(res); + AddSlotKey("s", src_key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/pika_slot.cc b/src/pika_slot.cc index c2880ff3c6..da55131918 100644 --- a/src/pika_slot.cc +++ b/src/pika_slot.cc @@ -82,8 +82,7 @@ void Slot::Close() { opened_ = false; } -// Before call this function, should -// close db and log first +// Before call this function, you should close db and log firstly void Slot::MoveToTrash() { if (opened_) { return; @@ -105,7 +104,7 @@ void Slot::MoveToTrash() { std::string Slot::GetDBName() const { return db_name_; } -uint32_t Slot::GetSlotId() const { return slot_id_; } +uint32_t Slot::GetSlotID() const { return slot_id_; } std::string Slot::GetSlotName() const { return slot_name_; } diff --git a/src/pika_slot_command.cc b/src/pika_slot_command.cc new file mode 100644 index 0000000000..5516af20ad --- /dev/null +++ b/src/pika_slot_command.cc @@ -0,0 +1,1697 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_slot_command.h" +#include +#include +#include +#include "include/pika_command.h" +#include "include/pika_conf.h" +#include "include/pika_data_distribution.h" +#include "include/pika_define.h" +#include "include/pika_migrate_thread.h" +#include "include/pika_server.h" +#include "pstd/include/pstd_status.h" +#include "pstd/include/pstd_string.h" +#include "storage/include/storage/storage.h" +#include "include/pika_command.h" +#include "include/pika_define.h" +#include "include/pika_migrate_thread.h" + +#include "include/pika_admin.h" +#include "include/pika_cmd_table_manager.h" +#include "include/pika_rm.h" + +#include "include/pika_admin.h" +#include "include/pika_cmd_table_manager.h" +#include "include/pika_rm.h" + +#define min(a, b) (((a) > (b)) ? (b) : (a)) +#define MAX_MEMBERS_NUM 512 + +extern std::unique_ptr g_pika_server; +extern std::unique_ptr g_pika_conf; +extern std::unique_ptr g_pika_rm; +extern std::unique_ptr g_pika_cmd_table_manager; + +uint32_t crc32tab[256]; +void CRC32TableInit(uint32_t poly) { + int i, j; + for (i = 0; i < 256; i++) { + uint32_t crc = i; + for (j = 0; j < 8; j++) { + if (crc & 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + crc32tab[i] = crc; + } +} + +void InitCRC32Table() { + CRC32TableInit(IEEE_POLY); +} + +uint32_t CRC32Update(uint32_t crc, const char *buf, int len) { + int i; + crc = ~crc; + for (i = 0; i < len; i++) { + crc = crc32tab[(uint8_t)((char)crc ^ buf[i])] ^ (crc >> 8); + } + return ~crc; +} + +PikaMigrate::PikaMigrate() { migrate_clients_.clear(); } + +PikaMigrate::~PikaMigrate() { + // close and release all clients + // get the mutex lock + std::lock_guard lm(mutex_); + KillAllMigrateClient(); +} + +net::NetCli *PikaMigrate::GetMigrateClient(const std::string &host, const int port, int timeout) { + std::string ip_port = host + ":" + std::to_string(port); + net::NetCli *migrate_cli; + pstd::Status s; + + std::map::iterator migrate_clients_iter = migrate_clients_.find(ip_port); + if (migrate_clients_iter == migrate_clients_.end()) { + migrate_cli = net::NewRedisCli(); + s = migrate_cli->Connect(host, port, g_pika_server->host()); + if (!s.ok()) { + LOG(ERROR) << "GetMigrateClient: new migrate_cli[" << ip_port.c_str() << "] failed"; + + delete migrate_cli; + return NULL; + } + + LOG(INFO) << "GetMigrateClient: new migrate_cli[" << ip_port.c_str() << "]"; + + std::string userpass = g_pika_conf->userpass(); + if (userpass != "") { + net::RedisCmdArgsType argv; + std::string wbuf_str; + argv.push_back("auth"); + argv.push_back(userpass); + net::SerializeRedisCommand(argv, &wbuf_str); + + s = migrate_cli->Send(&wbuf_str); + if (!s.ok()) { + LOG(ERROR) << "GetMigrateClient: new migrate_cli Send, error: " << s.ToString(); + delete migrate_cli; + return NULL; + } + + s = migrate_cli->Recv(&argv); + if (!s.ok()) { + LOG(ERROR) << "GetMigrateClient: new migrate_cli Recv, error: " << s.ToString(); + delete migrate_cli; + return NULL; + } + + if (strcasecmp(argv[0].data(), kInnerReplOk.data())) { + LOG(ERROR) << "GetMigrateClient: new migrate_cli auth error"; + delete migrate_cli; + return NULL; + } + } + + // add a new migrate client to the map + migrate_clients_[ip_port] = migrate_cli; + } else { + migrate_cli = static_cast(migrate_clients_iter->second); + } + + // set the client connect timeout + migrate_cli->set_send_timeout(timeout); + migrate_cli->set_recv_timeout(timeout); + + // modify the client last time + gettimeofday(&migrate_cli->last_interaction_, NULL); + + return migrate_cli; +} + +void PikaMigrate::KillMigrateClient(net::NetCli *migrate_cli) { + std::map::iterator migrate_clients_iter = migrate_clients_.begin(); + while (migrate_clients_iter != migrate_clients_.end()) { + if (migrate_cli == static_cast(migrate_clients_iter->second)) { + LOG(INFO) << "KillMigrateClient: kill migrate_cli[" << migrate_clients_iter->first.c_str() << "]"; + + migrate_cli->Close(); + delete migrate_cli; + migrate_cli = NULL; + + migrate_clients_.erase(migrate_clients_iter); + break; + } + + ++migrate_clients_iter; + } +} + +// clean and realse timeout client +void PikaMigrate::CleanMigrateClient() { + struct timeval now; + + // if the size of migrate_clients_ <= 0, don't need clean + if (migrate_clients_.size() <= 0) { + return; + } + + gettimeofday(&now, NULL); + std::map::iterator migrate_clients_iter = migrate_clients_.begin(); + while (migrate_clients_iter != migrate_clients_.end()) { + net::NetCli *migrate_cli = static_cast(migrate_clients_iter->second); + // pika_server do DoTimingTask every 10s, so we Try colse the migrate_cli before pika timeout, do it at least 20s in + // advance + int timeout = (g_pika_conf->timeout() > 0) ? g_pika_conf->timeout() : 60; + if (now.tv_sec - migrate_cli->last_interaction_.tv_sec > timeout - 20) { + LOG(INFO) << "CleanMigrateClient: clean migrate_cli[" << migrate_clients_iter->first.c_str() << "]"; + migrate_cli->Close(); + delete migrate_cli; + + migrate_clients_iter = migrate_clients_.erase(migrate_clients_iter); + } else { + ++migrate_clients_iter; + } + } +} + +// clean and realse all client +void PikaMigrate::KillAllMigrateClient() { + std::map::iterator migrate_clients_iter = migrate_clients_.begin(); + while (migrate_clients_iter != migrate_clients_.end()) { + net::NetCli *migrate_cli = static_cast(migrate_clients_iter->second); + + LOG(INFO) << "KillAllMigrateClient: kill migrate_cli[" << migrate_clients_iter->first.c_str() << "]"; + + migrate_cli->Close(); + delete migrate_cli; + + migrate_clients_iter = migrate_clients_.erase(migrate_clients_iter); + } +} + +/* * + * do migrate a key-value for slotsmgrt/slotsmgrtone commands + * return value: + * -1 - error happens + * >=0 - # of success migration (0 or 1) + * */ +int PikaMigrate::MigrateKey(const std::string &host, const int port, int timeout, const std::string &key, + const char type, std::string &detail, std::shared_ptr slot) { + int send_command_num = -1; + + net::NetCli *migrate_cli = GetMigrateClient(host, port, timeout); + if (!migrate_cli) { + detail = "IOERR error or timeout connecting to the client"; + LOG(INFO) << "GetMigrateClient failed, key: " << key; + return -1; + } + + send_command_num = MigrateSend(migrate_cli, key, type, detail, slot); + if (send_command_num <= 0) { + return send_command_num; + } + + if (MigrateRecv(migrate_cli, send_command_num, detail)) { + return send_command_num; + } + + return -1; +} + +int PikaMigrate::MigrateSend(net::NetCli *migrate_cli, const std::string &key, const char type, std::string &detail, + const std::shared_ptr& slot) { + std::string wbuf_str; + pstd::Status s; + int command_num = -1; + + // chech the client is alive + if (!migrate_cli) { + return -1; + } + + command_num = ParseKey(key, type, wbuf_str, slot); + if (command_num < 0) { + detail = "ParseKey failed"; + return command_num; + } + + // don't need seed data, key is not exists + if (command_num == 0 || wbuf_str.empty()) { + return 0; + } + + s = migrate_cli->Send(&wbuf_str); + if (!s.ok()) { + LOG(ERROR) << "Connect slots target, Send error: " << s.ToString(); + detail = "Connect slots target, Send error: " + s.ToString(); + KillMigrateClient(migrate_cli); + return -1; + } + + return command_num; +} + +bool PikaMigrate::MigrateRecv(net::NetCli *migrate_cli, int need_receive, std::string &detail) { + pstd::Status s; + std::string reply; + int64_t ret; + + if (NULL == migrate_cli || need_receive < 0) { + return false; + } + + net::RedisCmdArgsType argv; + while (need_receive) { + s = migrate_cli->Recv(&argv); + if (!s.ok()) { + LOG(ERROR) << "Connect slots target, Recv error: " << s.ToString(); + detail = "Connect slots target, Recv error: " + s.ToString(); + KillMigrateClient(migrate_cli); + return false; + } + + reply = argv[0]; + need_receive--; + + // set return ok + // zadd return number + // hset return 0 or 1 + // hmset return ok + // sadd return number + // rpush return length + if (argv.size() == 1 && + (kInnerReplOk == pstd::StringToLower(reply) || pstd::string2int(reply.data(), reply.size(), &ret))) { + // continue reiceve response + if (need_receive > 0) { + continue; + } + + // has got all responses + break; + } + + // failed + detail = "something wrong with slots migrate, reply: " + reply; + LOG(ERROR) << "something wrong with slots migrate, reply:" << reply; + return false; + } + + return true; +} + +// return -1 is error; 0 don't migrate; >0 the number of commond +int PikaMigrate::ParseKey(const std::string &key, const char type, std::string &wbuf_str, const std::shared_ptr& slot) { + int command_num = -1; + int64_t ttl = 0; + rocksdb::Status s; + + switch (type) { + case 'k': + command_num = ParseKKey(key, wbuf_str, slot); + break; + case 'h': + command_num = ParseHKey(key, wbuf_str, slot); + break; + case 'l': + command_num = ParseLKey(key, wbuf_str, slot); + break; + case 'z': + command_num = ParseZKey(key, wbuf_str, slot); + break; + case 's': + command_num = ParseSKey(key, wbuf_str, slot); + break; + default: + LOG(INFO) << "ParseKey key[" << key << "], the type[" << type << "] is not support."; + return -1; + break; + } + + // error or key is not existed + if (command_num <= 0) { + LOG(INFO) << "ParseKey key[" << key << "], parse return " << command_num + << ", the key maybe is not exist or expired."; + return command_num; + } + + // skip kv, because kv cmd: SET key value ttl + if (type == 'k') { + return command_num; + } + + ttl = TTLByType(type, key, slot); + + //-1 indicates the key is valid forever + if (ttl == -1) { + return command_num; + } + + // key is expired or not exist, don't migrate + if (ttl == 0 or ttl == -2) { + wbuf_str.clear(); + return 0; + } + + // no kv, because kv cmd: SET key value ttl + if (SetTTL(key, wbuf_str, ttl)) { + command_num += 1; + } + + return command_num; +} + +bool PikaMigrate::SetTTL(const std::string &key, std::string &wbuf_str, int64_t ttl) { + //-1 indicates the key is valid forever + if (ttl == -1) { + return false; + } + + // if ttl = -2 indicates, the key is not existed + if (ttl < 0) { + LOG(INFO) << "SetTTL key[" << key << "], ttl is " << ttl; + ttl = 0; + } + + net::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("EXPIRE"); + argv.push_back(key); + argv.push_back(std::to_string(ttl)); + + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + + return true; +} + +// return -1 is error; 0 don't migrate; >0 the number of commond +int PikaMigrate::ParseKKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { + net::RedisCmdArgsType argv; + std::string cmd; + std::string value; + int64_t ttl = 0; + rocksdb::Status s; + + s = slot->db()->Get(key, &value); + + // if key is not existed, don't migrate + if (s.IsNotFound()) { + return 0; + } + + if (!s.ok()) { + return -1; + } + + argv.push_back("SET"); + argv.push_back(key); + argv.push_back(value); + + ttl = TTLByType('k', key, slot); + + // ttl = -1 indicates the key is valid forever, dont process + // key is expired or not exist, dont migrate + // todo check ttl + if (ttl == 0 || ttl == -2) { + wbuf_str.clear(); + return 0; + } + + if (ttl > 0) { + argv.push_back("EX"); + argv.push_back(std::to_string(ttl)); + } + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + return 1; +} + +int64_t PikaMigrate::TTLByType(const char key_type, const std::string &key, const std::shared_ptr& slot) { + std::map type_timestamp; + std::map type_status; + type_timestamp = slot->db()->TTL(key, &type_status); + + switch (key_type) { + case 'k': { + return type_timestamp[storage::kStrings]; + } break; + case 'h': { + return type_timestamp[storage::kHashes]; + } break; + case 'z': { + return type_timestamp[storage::kZSets]; + } break; + case 's': { + return type_timestamp[storage::kSets]; + } break; + case 'l': { + return type_timestamp[storage::kLists]; + } break; + default: + return -3; + } +} + +int PikaMigrate::ParseZKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { + int command_num = 0; + + int64_t next_cursor = 0; + std::vector score_members; + do { + score_members.clear(); + rocksdb::Status s = slot->db()->ZScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &score_members, &next_cursor); + if (s.ok()) { + if (score_members.empty()) { + break; + } + + net::RedisCmdArgsType argv; + std::string cmd; + argv.push_back("ZADD"); + argv.push_back(key); + + for (const auto &score_member : score_members) { + argv.push_back(std::to_string(score_member.score)); + argv.push_back(score_member.member); + } + + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + command_num++; + } else if (s.IsNotFound()) { + wbuf_str.clear(); + return 0; + } else { + wbuf_str.clear(); + return -1; + } + } while (next_cursor > 0); + + return command_num; +} + +// return -1 is error; 0 don't migrate; >0 the number of commond +int PikaMigrate::ParseHKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { + int64_t next_cursor = 0; + int command_num = 0; + std::vector field_values; + do { + field_values.clear(); + rocksdb::Status s = slot->db()->HScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &field_values, &next_cursor); + if (s.ok()) { + if (field_values.empty()) { + break; + } + + net::RedisCmdArgsType argv; + std::string cmd; + argv.push_back("HMSET"); + argv.push_back(key); + + for (const auto &field_value : field_values) { + argv.push_back(field_value.field); + argv.push_back(field_value.value); + } + + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + command_num++; + } else if (s.IsNotFound()) { + wbuf_str.clear(); + return 0; + } else { + wbuf_str.clear(); + return -1; + } + } while (next_cursor > 0); + + return command_num; +} + +// return -1 is error; 0 don't migrate; >0 the number of commond +int PikaMigrate::ParseSKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { + int command_num = 0; + int64_t next_cursor = 0; + std::vector members; + + do { + members.clear(); + rocksdb::Status s = slot->db()->SScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &members, &next_cursor); + + if (s.ok()) { + if (members.empty()) { + break; + } + + net::RedisCmdArgsType argv; + std::string cmd; + argv.push_back("SADD"); + argv.push_back(key); + + for (const auto &member : members) { + argv.push_back(member); + } + + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + command_num++; + } else if (s.IsNotFound()) { + wbuf_str.clear(); + return 0; + } else { + wbuf_str.clear(); + return -1; + } + } while (next_cursor > 0); + + return command_num; +} + +// return -1 is error; 0 don't migrate; >0 the number of commond +int PikaMigrate::ParseLKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { + int64_t left = 0; + int command_num = 0; + std::vector values; + + net::RedisCmdArgsType argv; + std::string cmd; + + // del old key, before migrate list; prevent redo when failed + argv.push_back("DEL"); + argv.push_back(key); + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + command_num++; + + do { + values.clear(); + rocksdb::Status s = slot->db()->LRange(key, left, left + (MAX_MEMBERS_NUM - 1), &values); + if (s.ok()) { + if (values.empty()) { + break; + } + + net::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("RPUSH"); + argv.push_back(key); + + for (const auto &value : values) { + argv.push_back(value); + } + + net::SerializeRedisCommand(argv, &cmd); + wbuf_str.append(cmd); + command_num++; + + left += MAX_MEMBERS_NUM; + } else if (s.IsNotFound()) { + wbuf_str.clear(); + return 0; + } else { + wbuf_str.clear(); + return -1; + } + } while (!values.empty()); + + if (command_num == 1) { + wbuf_str.clear(); + command_num = 0; + } + + return command_num; +} + +/* * + * do migrate a key-value for slotsmgrt/slotsmgrtone commands + * return value: + * -1 - error happens + * >=0 - # of success migration (0 or 1) + * */ +static int SlotsMgrtOne(const std::string &host, const int port, int timeout, const std::string &key, const char type, + std::string &detail, std::shared_ptr slot) { + int send_command_num = 0; + rocksdb::Status s; + std::map type_status; + + send_command_num = g_pika_server->pika_migrate_->MigrateKey(host, port, timeout, key, type, detail, slot); + + // the key is migrated to target, delete key and slotsinfo + if (send_command_num >= 1) { + LOG(INFO) << "【send command success】Migrate key: " << key << " success, host: " << host << ", port: " << port; + std::vector keys; + keys.push_back(key); + int64_t count = slot->db()->Del(keys, &type_status); + if (count > 0) { + WriteDelKeyToBinlog(key, slot); + } + + // del slots info + RemSlotKeyByType(std::string(1, type), key, slot); + return 1; + } + + // key is not existed, only del slotsinfo + if (send_command_num == 0) { + // del slots info + RemSlotKeyByType(std::string(1, type), key, slot); + return 0; + } + return -1; +} + +void RemSlotKeyByType(const std::string &type, const std::string &key, const std::shared_ptr& slot) { + uint32_t crc; + int hastag; + int slotNum = GetSlotsID(key, &crc, &hastag); + + std::string slot_key = GetSlotKey(slotNum); + int32_t res = 0; + + std::vector members; + members.push_back(type + key); + rocksdb::Status s = slot->db()->SRem(slot_key, members, &res); + if (!s.ok()) { + LOG(ERROR) << "srem key[" << key << "] from slotKey[" << slot_key << "] failed, error: " << s.ToString(); + return; + } + + if (hastag) { + std::string tag_key = GetSlotsTagKey(crc); + s = slot->db()->SRem(tag_key, members, &res); + if (!s.ok()) { + LOG(ERROR) << "srem key[" << key << "] from tagKey[" << tag_key << "] failed, error: " << s.ToString(); + return; + } + } +} + +/* * + * do migrate mutli key-value(s) for {slotsmgrt/slotsmgrtone}with tag commands + * return value: + * -1 - error happens + * >=0 - # of success migration + * */ +static int SlotsMgrtTag(const std::string &host, const int port, int timeout, const std::string &key, const char type, + std::string &detail, std::shared_ptr slot) { + int count = 0; + uint32_t crc; + int hastag; + GetSlotsID(key, &crc, &hastag); + if (!hastag) { + if (type == 0) { + return 0; + } + int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, slot); + if (ret == 0) { + LOG(INFO) << "slots migrate without tag failed, key: " << key << ", detail: " << detail; + } + return ret; + } + + std::string tag_key = GetSlotsTagKey(crc); + std::vector members; + + // get all keys that have the same crc + rocksdb::Status s = slot->db()->SMembers(tag_key, &members); + if (!s.ok()) { + return -1; + } + + std::vector::const_iterator iter = members.begin(); + for (; iter != members.end(); iter++) { + std::string key = *iter; + char type = key.at(0); + key.erase(key.begin()); + int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, slot); + + // the key is migrated to target + if (ret == 1) { + count++; + continue; + } + + if (ret == 0) { + LOG(WARNING) << "slots migrate tag failed, key: " << key << ", detail: " << detail; + continue; + } + + return -1; + } + + return count; +} + +// get slot tag +static const char *GetSlotsTag(const std::string &str, int *plen) { + const char *s = str.data(); + int i, j, n = str.length(); + for (i = 0; i < n && s[i] != '{'; i++) { + } + if (i == n) { + return NULL; + } + i++; + for (j = i; j < n && s[j] != '}'; j++) { + } + if (j == n) { + return NULL; + } + if (plen != NULL) { + *plen = j - i; + } + return s + i; +} + +std::string GetSlotKey(int slot) { + return SlotKeyPrefix + std::to_string(slot); +} + +// get slot number of the key +int GetSlotID(const std::string &str) { return GetSlotsID(str, NULL, NULL); } + +// get the slot number by key +int GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag) { + const char *s = str.data(); + int taglen; + int hastag = 0; + const char *tag = GetSlotsTag(str, &taglen); + if (tag == NULL) { + tag = s, taglen = str.length(); + } else { + hastag = 1; + } + uint32_t crc = CRC32CheckSum(tag, taglen); + if (pcrc != NULL) { + *pcrc = crc; + } + if (phastag != NULL) { + *phastag = hastag; + } + return (int)(crc & HASH_SLOTS_MASK); +} + +uint32_t CRC32CheckSum(const char *buf, int len) { return CRC32Update(0, buf, len); } + +// add key to slotkey +void AddSlotKey(const std::string type, const std::string key, const std::shared_ptr& slot) { + if (g_pika_conf->slotmigrate() != true) { + return; + } + + rocksdb::Status s; + int32_t res = -1; + uint32_t crc; + int hastag; + int slotID = GetSlotsID(key, &crc, &hastag); + std::string slot_key = GetSlotKey(slotID); + std::vector members; + members.push_back(type + key); + s = slot->db()->SAdd(slot_key, members, &res); + if (!s.ok()) { + LOG(ERROR) << "sadd key[" << key << "] to slotKey[" << slot_key << "] failed, error: " << s.ToString(); + return; + } + WriteSAddToBinlog(slot_key, members.front(), slot); + + // if res == 0, indicate the key is existed; may return, + // prevent write slot_key success, but write tag_key failed, so always write tag_key + if (hastag) { + std::string tag_key = GetSlotsTagKey(crc); + s = slot->db()->SAdd(tag_key, members, &res); + if (!s.ok()) { + LOG(ERROR) << "sadd key[" << key << "] to tagKey[" << tag_key << "] failed, error: " << s.ToString(); + return; + } + WriteSAddToBinlog(tag_key, members.front(), slot); + } +} + +// write sadd key to binlog for slave +void WriteSAddToBinlog(const std::string &key, const std::string &value, const std::shared_ptr& slot) { + std::shared_ptr cmd_ptr = g_pika_cmd_table_manager->GetCmd("sadd"); + std::unique_ptr args = std::unique_ptr(new PikaCmdArgsType()); + args->push_back("SADD"); + args->push_back(key); + args->push_back(value); + cmd_ptr->Initial(*args, slot->GetDBName()); + + std::shared_ptr sync_slot = + g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); + Status s = sync_slot->ConsensusProposeLog(cmd_ptr); + if (!s.ok()) { + LOG(ERROR) << "write sadd key to binlog failed, key: " << key; + } +} + +// check key exists +void RemKeyNotExists(const std::string type, const std::string key, const std::shared_ptr& slot) { + if (g_pika_conf->slotmigrate() != true) { + return; + } + std::vector vkeys; + vkeys.push_back(key); + std::map type_status; + int64_t res = slot->db()->Exists(vkeys, &type_status); + if (res == 0) { + std::string slotKey = GetSlotKey(GetSlotID(key)); + std::vector members(1, type + key); + int32_t count = 0; + rocksdb::Status s = slot->db()->SRem(slotKey, members, &count); + if (!s.ok()) { + LOG(WARNING) << "Zrem key: " << key << " from slotKey, error: " << s.ToString(); + return; + } + } + return; +} + +// del key from slotkey +void RemSlotKey(const std::string key, const std::shared_ptr& slot) { + if (g_pika_conf->slotmigrate() != true) { + return; + } + std::string type; + if (GetKeyType(key, type, slot) < 0) { + LOG(WARNING) << "SRem key: " << key << " from slotKey error"; + return; + } + std::string slotKey = GetSlotKey(GetSlotID(key)); + int32_t count = 0; + std::vector members(1, type + key); + rocksdb::Status s = slot->db()->SRem(slotKey, members, &count); + if (!s.ok()) { + LOG(WARNING) << "SRem key: " << key << " from slotKey, error: " << s.ToString(); + return; + } +} + +int GetKeyType(const std::string key, std::string &key_type, const std::shared_ptr& slot) { + std::string type_str; + rocksdb::Status s = slot->db()->Type(key, &type_str); + if (!s.ok()) { + LOG(WARNING) << "Get key type error: " << key << " " << s.ToString(); + key_type = ""; + return -1; + } + if (type_str == "string") { + key_type = "k"; + } else if (type_str == "hash") { + key_type = "h"; + } else if (type_str == "list") { + key_type = "l"; + } else if (type_str == "set") { + key_type = "s"; + } else if (type_str == "zset") { + key_type = "z"; + } else { + LOG(WARNING) << "Get key type error: " << key; + key_type = ""; + return -1; + } + return 1; +} + +// get slotstagkey by key +std::string GetSlotsTagKey(uint32_t crc) { + return SlotTagPrefix + std::to_string(crc); +} + +// delete key from db +int DeleteKey(const std::string key, const char key_type, const std::shared_ptr& slot) { + LOG(INFO) << "Del key Srem key " << key; + int32_t res = 0; + std::string slotKey = GetSlotKey(GetSlotID(key)); + LOG(INFO) << "Del key Srem key " << key; + + // delete key from slot + std::vector members; + members.push_back(key_type + key); + rocksdb::Status s = slot->db()->SRem(slotKey, members, &res); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(INFO) << "Del key Srem key " << key << " not found"; + return 0; + } else { + LOG(WARNING) << "Del key Srem key: " << key << " from slotKey, error: " << strerror(errno); + return -1; + } + } + + // delete key from db + members.clear(); + members.push_back(key); + std::map type_status; + int64_t del_nums = slot->db()->Del(members, &type_status); + if (0 > del_nums) { + LOG(WARNING) << "Del key: " << key << " at slot " << GetSlotID(key) << " error"; + return -1; + } + WriteDelKeyToBinlog(key, slot); + + return 1; +} + +// get list key all values +static int listGetall(const std::string key, std::vector *values, std::shared_ptr slot) { + rocksdb::Status s = slot->db()->LRange(key, 0, -1, values); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(WARNING) << "List get key: " << key << " value not found "; + return 0; + } else { + LOG(WARNING) << "List get key: " << key << " value error: " << s.ToString(); + return -1; + } + } + return 1; +} + +void SlotsMgrtTagSlotCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtTagSlot); + return; + } + // Remember the first args is the opt name + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; + dest_ip_ = *it++; + pstd::StringToLower(dest_ip_); + + std::string str_dest_port = *it++; + if (!pstd::string2int(str_dest_port.data(), str_dest_port.size(), &dest_port_)) { + std::string detail = "invalid port number " + std::to_string(dest_port_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + if (dest_port_ < 0 || dest_port_ > 65535) { + std::string detail = "invalid port number " + std::to_string(dest_port_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + + if ((dest_ip_ == "127.0.0.1" || dest_ip_ == g_pika_server->host()) && dest_port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "destination address error"); + return; + } + + std::string str_timeout_ms = *it++; + if (!pstd::string2int(str_timeout_ms.data(), str_timeout_ms.size(), &timeout_ms_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (timeout_ms_ < 0) { + std::string detail = "invalid timeout number " + std::to_string(timeout_ms_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + if (timeout_ms_ == 0) { + timeout_ms_ = 100; + } + + std::string str_slot_num = *it++; + if (!pstd::string2int(str_slot_num.data(), str_slot_num.size(), &slot_id_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (slot_id_ < 0 || slot_id_ >= HASH_SLOTS_SIZE) { + std::string detail = "invalid slot number " + std::to_string(slot_id_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } +} + +void SlotsMgrtTagSlotCmd::Do(std::shared_ptr slot) { + if (g_pika_conf->slotmigrate() != true) { + LOG(WARNING) << "Not in slotmigrate mode"; + res_.SetRes(CmdRes::kErrOther, "not set slotmigrate"); + return; + } + + int32_t len = 0; + int ret = 0; + std::string detail; + std::string slot_key = GetSlotKey(slot_id_); + + // first, get the count of slot_key, prevent to sscan key very slowly when the key is not found + rocksdb::Status s = slot->db()->SCard(slot_key, &len); + LOG(INFO) << "【SlotsMgrtTagSlotCmd::Do】Get count, slot_key: " << slot_key << ", len: " << len; + if (len < 0) { + detail = "Get the len of slot Error"; + } + // mutex between SlotsMgrtTagSlotCmd、SlotsMgrtTagOneCmd and migrator_thread + if (len > 0 && g_pika_server->pika_migrate_->Trylock()) { + g_pika_server->pika_migrate_->CleanMigrateClient(); + int64_t next_cursor = 0; + std::vector members; + rocksdb::Status s = slot->db()->SScan(slot_key, 0, "*", 1, &members, &next_cursor); + if (s.ok()) { + for (const auto &member : members) { + std::string key = member; + char type = key.at(0); + key.erase(key.begin()); + ret = SlotsMgrtTag(dest_ip_, dest_port_, timeout_ms_, key, type, detail, slot); + } + } + // unlock + g_pika_server->pika_migrate_->Unlock(); + } else { + LOG(WARNING) << "pika migrate is running, try again later, slot_id_: " << slot_id_; + } + if (ret == 0) { + LOG(WARNING) << "slots migrate without tag failed, slot_id_: " << slot_id_ << ", detail: " << detail; + } + if (len >= 0 && ret >= 0) { + res_.AppendArrayLen(2); + // the number of keys migrated + res_.AppendInteger(ret); + // the number of keys remained + res_.AppendInteger(len - ret); + } else { + res_.SetRes(CmdRes::kErrOther, detail); + } + + return; +} + +// check key type +int SlotsMgrtTagOneCmd::KeyTypeCheck(const std::shared_ptr& slot) { + std::string type_str; + rocksdb::Status s = slot->db()->Type(key_, &type_str); + if (!s.ok()) { + if (s.IsNotFound()) { + LOG(INFO) << "Migrate slot key " << key_ << " not found"; + res_.AppendInteger(0); + } else { + LOG(WARNING) << "Migrate slot key: " << key_ << " error: " << s.ToString(); + res_.SetRes(CmdRes::kErrOther, "migrate slot error"); + } + return -1; + } + if (type_str == "string") { + key_type_ = 'k'; + } else if (type_str == "hash") { + key_type_ = 'h'; + } else if (type_str == "list") { + key_type_ = 'l'; + } else if (type_str == "set") { + key_type_ = 's'; + } else if (type_str == "zset") { + key_type_ = 'z'; + } else { + LOG(WARNING) << "Migrate slot key: " << key_ << " not found"; + res_.AppendInteger(0); + return -1; + } + return 0; +} + +void SlotsMgrtTagOneCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtTagSlot); + return; + } + // Remember the first args is the opt name + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; + dest_ip_ = *it++; + pstd::StringToLower(dest_ip_); + + std::string str_dest_port = *it++; + if (!pstd::string2int(str_dest_port.data(), str_dest_port.size(), &dest_port_)) { + std::string detail = "invalid port number " + std::to_string(dest_port_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + if (dest_port_ < 0 || dest_port_ > 65535) { + std::string detail = "invalid port number " + std::to_string(dest_port_); + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + + if ((dest_ip_ == "127.0.0.1" || dest_ip_ == g_pika_server->host()) && dest_port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "destination address error"); + return; + } + + std::string str_timeout_ms = *it++; + if (!pstd::string2int(str_timeout_ms.data(), str_timeout_ms.size(), &timeout_ms_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (timeout_ms_ < 0) { + std::string detail = "invalid timeout number " + timeout_ms_; + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + if (timeout_ms_ == 0) { + timeout_ms_ = 100; + } + + key_ = *it++; +} + +void SlotsMgrtTagOneCmd::Do(std::shared_ptr slot) { + if (g_pika_conf->slotmigrate() != true) { + LOG(WARNING) << "Not in slotmigrate mode"; + res_.SetRes(CmdRes::kErrOther, "not set slotmigrate"); + return; + } + + int64_t ret = 0; + int32_t len = 0; + int hastag = 0; + uint32_t crc; + std::string detail; + rocksdb::Status s; + std::map type_status; + + // if you need migrates key, if the key is not existed, return + GetSlotsID(key_, &crc, &hastag); + if (!hastag) { + std::vector keys; + keys.push_back(key_); + + // check the key is not existed + ret = slot->db()->Exists(keys, &type_status); + + // when the key is not existed, ret = 0 + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "exists internal error"); + return; + } + + if (ret == 0) { + res_.AppendInteger(0); + return; + } + + // else need to migrate + } else { + // key is tag_key, check the number of the tag_key + std::string tag_key = GetSlotsTagKey(crc); + s = slot->db()->SCard(tag_key, &len); + if (s.IsNotFound()) { + res_.AppendInteger(0); + return; + } + if (!s.ok() || len == -1) { + res_.SetRes(CmdRes::kErrOther, "can't get the number of tag_key"); + return; + } + + if (len == 0) { + res_.AppendInteger(0); + return; + } + + // else need to migrate + } + + // lock batch migrate, dont do slotsmgrttagslot when do slotsmgrttagone + // pika_server thread exit(~PikaMigrate) and dispatch thread do CronHandle nead lock() + g_pika_server->pika_migrate_->Lock(); + + // if the key is not existed, return + if (!hastag) { + std::vector keys; + keys.push_back(key_); + // the key may be deleted by another thread + std::map type_status; + ret = slot->db()->Exists(keys, &type_status); + + // when the key is not existed, ret = 0 + if (ret == -1) { + detail = s.ToString(); + } else if (KeyTypeCheck(slot) != 0) { + detail = "cont get the key type."; + ret = -1; + } else { + ret = SlotsMgrtTag(dest_ip_, dest_port_, timeout_ms_, key_, key_type_, detail, slot); + } + } else { + // key maybe doesn't exist, the key is tag key, migrate the same tag key + ret = SlotsMgrtTag(dest_ip_, dest_port_, timeout_ms_, key_, 0, detail, slot); + } + + // unlock the record lock + g_pika_server->pika_migrate_->Unlock(); + + if (ret >= 0) { + res_.AppendInteger(ret); + } else { + if (detail.size() == 0) { + detail = "Unknown Error"; + } + res_.SetRes(CmdRes::kErrOther, detail); + } + + return; +} + +/* * + * slotsinfo [start] [count] + * */ +void SlotsInfoCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsInfo); + return; + } + + if (argv_.size() >= 2) { + if (!pstd::string2int(argv_[1].data(), argv_[1].size(), &begin_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + if (begin_ < 0 || begin_ >= end_) { + std::string detail = "invalid slot begin = " + argv_[1]; + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + } + + if (argv_.size() >= 3) { + int64_t count = 0; + if (!pstd::string2int(argv_[2].data(), argv_[2].size(), &count)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + if (count < 0) { + std::string detail = "invalid slot count = " + argv_[2]; + res_.SetRes(CmdRes::kErrOther, detail); + return; + } + + if (begin_ + count < end_) { + end_ = begin_ + count; + } + } + + if (argv_.size() >= 4) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsInfo); + return; + } +} + +void SlotsInfoCmd::Do(std::shared_ptr slot) { + int slots_slot[HASH_SLOTS_SIZE] = {0}; + int slots_size[HASH_SLOTS_SIZE] = {0}; + int n = 0; + int i = 0; + int32_t len = 0; + std::string slot_key; + + for (i = begin_; i < end_; i++) { + slot_key = GetSlotKey(i); + len = 0; + rocksdb::Status s = slot->db()->SCard(slot_key, &len); + if (!s.ok() || len == 0) { + continue; + } + + slots_slot[n] = i; + slots_size[n] = len; + n++; + } + + res_.AppendArrayLen(n); + for (i = 0; i < n; i++) { + res_.AppendArrayLen(2); + res_.AppendInteger(slots_slot[i]); + res_.AppendInteger(slots_size[i]); + } + + return; +} + +void SlotsMgrtTagSlotAsyncCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtTagSlotAsync); + } + // Remember the first args is the opt name + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; + dest_ip_ = *it++; + pstd::StringToLower(dest_ip_); + + std::string str_dest_port = *it++; + if (!pstd::string2int(str_dest_port.data(), str_dest_port.size(), &dest_port_) || dest_port_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + if ((dest_ip_ == "127.0.0.1" || dest_ip_ == g_pika_server->host()) && dest_port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "destination address error"); + return; + } + + std::string str_timeout_ms = *it++; + if (!pstd::string2int(str_timeout_ms.data(), str_timeout_ms.size(), &timeout_ms_) || timeout_ms_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + std::string str_max_bulks = *it++; + if (!pstd::string2int(str_max_bulks.data(), str_max_bulks.size(), &max_bulks_) || max_bulks_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + std::string str_max_bytes_ = *it++; + if (!pstd::string2int(str_max_bytes_.data(), str_max_bytes_.size(), &max_bytes_) || max_bytes_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + std::string str_slot_num = *it++; + if (!pstd::string2int(str_slot_num.data(), str_slot_num.size(), &slot_id_) || slot_id_ < 0 || + slot_id_ >= HASH_SLOTS_SIZE) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + std::string str_keys_num = *it++; + if (!pstd::string2int(str_keys_num.data(), str_keys_num.size(), &keys_num_) || keys_num_ < 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void SlotsMgrtTagSlotAsyncCmd::Do(std::shared_ptr slot) { + // check whether open slotmigrate + if (!g_pika_conf->slotmigrate()) { + res_.SetRes(CmdRes::kErrOther, "please open slotmigrate and reload slot"); + return; + } + + int32_t remained = 0; + std::string slotKey = GetSlotKey(slot_id_); + storage::Status status = slot->db()->SCard(slotKey, &remained); + if (status.IsNotFound()) { + LOG(INFO) << "find no record in slot " << slot_id_; + res_.AppendArrayLen(2); + res_.AppendInteger(0); + res_.AppendInteger(remained); + return; + } + if (!status.ok()) { + LOG(WARNING) << "Slot batch migrate keys get result error"; + res_.SetRes(CmdRes::kErrOther, "Slot batch migrating keys get result error"); + return; + } + + bool ret = g_pika_server->SlotsMigrateBatch(dest_ip_, dest_port_, timeout_ms_, slot_id_, keys_num_, slot); + if (!ret) { + LOG(WARNING) << "Slot batch migrate keys error"; + res_.SetRes(CmdRes::kErrOther, "Slot batch migrating keys error, may be currently migrating"); + return; + } + + res_.AppendArrayLen(2); + res_.AppendInteger(0); + res_.AppendInteger(remained); + return; +} + +void SlotsMgrtAsyncStatusCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtAsyncStatus); + } + return; +} + +void SlotsMgrtAsyncStatusCmd::Do(std::shared_ptr slot) { + std::string status; + std::string ip; + int64_t port = -1, slots = -1, moved = -1, remained = -1; + bool migrating; + g_pika_server->GetSlotsMgrtSenderStatus(&ip, &port, &slots, &migrating, &moved, &remained); + std::string mstatus = migrating ? "yes" : "no"; + res_.AppendArrayLen(5); + status = "dest server: " + ip + ":" + std::to_string(port); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "slot number: " + std::to_string(slots); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "migrating : " + mstatus; + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "moved keys : " + std::to_string(moved); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "remain keys: " + std::to_string(remained); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + + return; +} + +void SlotsMgrtAsyncCancelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtAsyncCancel); + } + return; +} + +void SlotsMgrtAsyncCancelCmd::Do(std::shared_ptr slot) { + bool ret = g_pika_server->SlotsMigrateAsyncCancel(); + if (!ret) { + res_.SetRes(CmdRes::kErrOther, "slotsmgrt-async-cancel error"); + } + res_.SetRes(CmdRes::kOk); + return; +} + +void SlotsDelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsDel); + } + slots_.assign(argv_.begin(), argv_.end()); + return; +} + +void SlotsDelCmd::Do(std::shared_ptr slot) { + std::vector keys; + std::vector::const_iterator iter; + for (iter = slots_.begin(); iter != slots_.end(); iter++) { + keys.push_back(SlotKeyPrefix + *iter); + } + std::map type_status; + int64_t count = slot->db()->Del(keys, &type_status); + if (count >= 0) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, "SlotsDel error"); + } + return; +} + +/* * + * slotshashkey [key1 key2...] + * */ +void SlotsHashKeyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsHashKey); + return; + } + + auto iter = argv_.begin(); + keys_.assign(++iter, argv_.end()); + return; +} + +void SlotsHashKeyCmd::Do(std::shared_ptr slot) { + std::vector::const_iterator keys_it; + + res_.AppendArrayLen(keys_.size()); + for (keys_it = keys_.begin(); keys_it != keys_.end(); ++keys_it) { + res_.AppendInteger(GetSlotsID(*keys_it, NULL, NULL)); + } + + return; +} + +void SlotsScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsScan); + return; + } + key_ = SlotKeyPrefix + argv_[1]; + if (std::stoll(argv_[1].data()) < 0 || std::stoll(argv_[1].data()) >= HASH_SLOTS_SIZE) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsScan); + return; + } + if (!pstd::string2int(argv_[2].data(), argv_[2].size(), &cursor_)) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsScan); + return; + } + size_t argc = argv_.size(), index = 3; + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!pstd::string2int(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + if (count_ < 0) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + return; +} + +void SlotsScanCmd::Do(std::shared_ptr slot) { + std::vector members; + rocksdb::Status s = slot->db()->SScan(key_, cursor_, pattern_, count_, &members, &cursor_); + + if (members.size() <= 0) { + cursor_ = 0; + } + res_.AppendContent("*2"); + + char buf[32]; + int64_t len = pstd::ll2string(buf, sizeof(buf), cursor_); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(members.size()); + std::vector::const_iterator iter_member = members.begin(); + for (; iter_member != members.end(); iter_member++) { + res_.AppendStringLen(iter_member->size()); + res_.AppendContent(*iter_member); + } + return; +} + +void SlotsMgrtExecWrapperCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtExecWrapper); + } + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; + key_ = *it++; + pstd::StringToLower(key_); + return; +} + +// return 0 means key doesn't exist, or key is not migrating +// return 1 means key is migrating +// return -1 means something wrong +void SlotsMgrtExecWrapperCmd::Do(std::shared_ptr slot) { + res_.AppendArrayLen(2); + int ret = g_pika_server->SlotsMigrateOne(key_, slot); + switch (ret) { + case 0: + case -2: + res_.AppendInteger(0); + res_.AppendInteger(0); + return; + case 1: + res_.AppendInteger(1); + res_.AppendInteger(1); + return; + default: + res_.AppendInteger(-1); + res_.AppendInteger(-1); + return; + } + return; +} + +void SlotsReloadCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsReload); + } + return; +} + +void SlotsReloadCmd::Do(std::shared_ptr slot) { + g_pika_server->Bgslotsreload(slot); + const PikaServer::BGSlotsReload &info = g_pika_server->bgslots_reload(); + char buf[256]; + snprintf(buf, sizeof(buf), "+%s : %lu", info.s_start_time.c_str(), g_pika_server->GetSlotsreloadingCursor()); + res_.AppendContent(buf); + return; +} + +void SlotsReloadOffCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsReloadOff); + } + return; +} + +void SlotsReloadOffCmd::Do(std::shared_ptr slot) { + g_pika_server->SetSlotsreloading(false); + res_.SetRes(CmdRes::kOk); + return; +} + +void SlotsCleanupCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsCleanup); + } + + PikaCmdArgsType::const_iterator iter = argv_.begin() + 1; + std::string slot; + long slotLong; + std::vector slots; + for (; iter != argv_.end(); iter++) { + slot = *iter; + if (!pstd::string2int(slot.data(), slot.size(), &slotLong) || slotLong < 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + slots.push_back(int(slotLong)); + } + cleanup_slots_.swap(slots); + return; +} + +void SlotsCleanupCmd::Do(std::shared_ptr slot) { + g_pika_server->Bgslotscleanup(cleanup_slots_, slot); + std::vector cleanup_slots(g_pika_server->GetCleanupSlots()); + res_.AppendArrayLen(cleanup_slots.size()); + std::vector::const_iterator iter = cleanup_slots.begin(); + for (; iter != cleanup_slots.end(); iter++) { + res_.AppendInteger(*iter); + } + return; +} + +void SlotsCleanupOffCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsCleanupOff); + } + return; +} + +void SlotsCleanupOffCmd::Do(std::shared_ptr slot) { + g_pika_server->StopBgslotscleanup(); + res_.SetRes(CmdRes::kOk); + return; +} diff --git a/src/pika_zset.cc b/src/pika_zset.cc index 207a23946b..95bad7636b 100644 --- a/src/pika_zset.cc +++ b/src/pika_zset.cc @@ -4,6 +4,7 @@ // of patent rights can be found in the PATENTS file in the same directory. #include "include/pika_zset.h" +#include "include/pika_slot_command.h" #include @@ -37,6 +38,7 @@ void ZAddCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->ZAdd(key_, score_members, &count); if (s.ok()) { res_.AppendInteger(count); + AddSlotKey("z", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -143,6 +145,7 @@ void ZIncrbyCmd::Do(std::shared_ptr slot) { int64_t len = pstd::d2string(buf, sizeof(buf), score); res_.AppendStringLen(len); res_.AppendContent(buf); + AddSlotKey("z", key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } @@ -445,6 +448,7 @@ void ZRemCmd::Do(std::shared_ptr slot) { int32_t count = 0; rocksdb::Status s = slot->db()->ZRem(key_, members_, &count); if (s.ok() || s.IsNotFound()) { + AddSlotKey("z", key_, slot); res_.AppendInteger(count); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -522,6 +526,7 @@ void ZUnionstoreCmd::Do(std::shared_ptr slot) { rocksdb::Status s = slot->db()->ZUnionstore(dest_key_, keys_, weights_, aggregate_, &count); if (s.ok()) { res_.AppendInteger(count); + AddSlotKey("z", dest_key_, slot); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/tests/integration/rpoplpush_replication_test.py b/tests/integration/rpoplpush_replication_test.py index 0de00223e3..f681d9c9b8 100644 --- a/tests/integration/rpoplpush_replication_test.py +++ b/tests/integration/rpoplpush_replication_test.py @@ -19,7 +19,7 @@ def test_master_slave_replication(): slave.slaveof(master_ip, master_port) master.delete('blist0', 'blist1', 'blist') - time.sleep(1) + time.sleep(10) m_keys = master.keys() s_keys = slave.keys() assert s_keys == m_keys, f'Expected: s_keys == m_keys, but got {s_keys == m_keys}' @@ -77,11 +77,10 @@ def thread1(): t9.join() t10.join() - time.sleep(3) + time.sleep(10) m_keys = master.keys() s_keys = slave.keys() - assert s_keys == m_keys, f'Expected: s_keys == m_keys, but got {s_keys == m_keys}' - + assert s_keys == m_keys, f'Expected: s_keys == m_keys, but got s_keys = {s_keys}, m_keys = {m_keys}' for i in range(0, master.llen('blist')): # print(master.lindex('blist', i)) # print(slave.lindex('blist', i))