pika_for_cluster (OpenAtomFoundation#1632)

* add integrate test --------- Co-authored-by: liuyuecai <[email protected]> Co-authored-by: chejinge <[email protected]> Co-authored-by: Xin.Zh <[email protected]>
cheniujh · Jun 26, 2023 · 0b5688e · 0b5688e
1 parent c159d59
commit 0b5688e
Show file tree

Hide file tree

Showing 59 changed files with 4,416 additions and 1,465 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -45,23 +45,21 @@ Checks:     '
             -clang-diagnostic-ignored-optimization-argument,
             -readability-implicit-bool-conversion,
             '
-# naming check
-CheckOptions:
-  - { key: readability-identifier-naming.ClassCase,           value: CamelCase  }
-  - { key: readability-identifier-naming.EnumCase,            value: CamelCase  }
-  - { key: readability-identifier-naming.FunctionCase,        value: CamelCase  }
-  - { key: readability-identifier-naming.GlobalConstantCase,  value: UPPER_CASE }
-  - { key: readability-identifier-naming.MemberCase,          value: lower_case }
-  - { key: readability-identifier-naming.MemberSuffix,        value: _          }
-  - { key: readability-identifier-naming.NamespaceCase,       value: lower_case }
-  - { key: readability-identifier-naming.StructCase,          value: CamelCase  }
-  - { key: readability-identifier-naming.UnionCase,           value: CamelCase  }
-  - { key: readability-identifier-naming.VariableCase,        value: lower_case }
-  - { key: readability-identifier-naming.GlobalVariableCase,  value: lower_case }
-  - { key: readability-identifier-naming.GlobalVariablePrefix,value: g_pika     }
+# CheckOptions:
+#   - { key: readability-identifier-naming.ClassCase,           value: CamelCase  }
+#   - { key: readability-identifier-naming.EnumCase,            value: CamelCase  }
+#   - { key: readability-identifier-naming.FunctionCase,        value: CamelCase  }
+#   - { key: readability-identifier-naming.GlobalConstantCase,  value: UPPER_CASE }
+#   - { key: readability-identifier-naming.MemberCase,          value: lower_case }
+#   - { key: readability-identifier-naming.MemberSuffix,        value: _          }
+#   - { key: readability-identifier-naming.NamespaceCase,       value: lower_case }
+#   - { key: readability-identifier-naming.StructCase,          value: CamelCase  }
+#   - { key: readability-identifier-naming.UnionCase,           value: CamelCase  }
+#   - { key: readability-identifier-naming.VariableCase,        value: lower_case }
 
 WarningsAsErrors: '*'
-HeaderFilterRegex: '(src/net/..|src/storage/include|src/pstd/..|src/pstd/include|src/net/..)'
+# HeaderFilterRegex: '(|/src|/src/net|/src/pstd|/src/storage)/include'
+# HeaderFilterRegex: '/src/(net|storage|pstd)/include'
 AnalyzeTemporaryDtors: true
 
 #### Disabled checks and why: #####

diff --git a/.github/workflows/pika.yml b/.github/workflows/pika.yml
@@ -47,10 +47,6 @@ jobs:
         # Build your program with the given configuration
         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
 
-      - name: Clang-tidy Check
-        working-directory: ${{github.workspace}}/build
-        run: make clang-tidy
-
       - name: Test
         working-directory: ${{github.workspace}}/build
         # Execute tests defined by the CMake configuration.
@@ -125,10 +121,6 @@ jobs:
           source /opt/rh/devtoolset-10/enable
           cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
 
-      - name: Clang-tidy Check
-        working-directory: ${{github.workspace}}/build
-        run: make clang-tidy
-
       - name: Test
         # Execute tests defined by the CMake configuration.
         # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -97,8 +97,8 @@ if (${AUTOCONF} MATCHES AUTOCONF-NOTFOUND)
   message(FATAL_ERROR "not find autoconf on localhost")
 endif()
 
-set(CLANG_SEARCH_PATH "/usr/local/bin" "/usr/bin" "/usr/local/opt/llvm/bin"
-                      "/usr/local/opt/llvm@12/bin")
+#set(CLANG_SEARCH_PATH "/usr/local/bin" "/usr/bin" "/usr/local/opt/llvm/bin"
+#                      "/usr/local/opt/llvm@12/bin")
 find_program(CLANG_TIDY_BIN
         NAMES clang-tidy clang-tidy-12
         HINTS ${CLANG_SEARCH_PATH})
@@ -784,6 +784,7 @@ add_custom_target(
         clang-tidy
         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/run_clang_tidy.py
         -clang-tidy-binary ${CLANG_TIDY_BIN}
+        -header-filter='${PROJECT_SOURCE_DIR}(/include|/tools|/src)/.*'
         -p ${CMAKE_BINARY_DIR}
         -quiet
     )
@@ -792,6 +793,7 @@ add_custom_target(clang-tidy-fix
     ${CMAKE_CURRENT_SOURCE_DIR}/run_clang_tidy.py
     -clang-tidy-binary ${CLANG_TIDY_BIN}
     -p ${CMAKE_BINARY_DIR}
+    -header-filter='${PROJECT_SOURCE_DIR}(/include|/tools|/src)/.*'
     -clang-apply-replacements-binary ${CLANG_APPLY_REPLACEMENTS_BIN}
     -fix
 )
diff --git a/codis/cmd/admin/dashboard.go b/codis/cmd/admin/dashboard.go
@@ -68,13 +68,6 @@ func (t *cmdDashboard) Main(d map[string]interface{}) {
 	case d["--promote-server"].(bool):
 		t.handleGroupCommand(d)
 
-	case d["--sentinel-add"].(bool):
-		fallthrough
-	case d["--sentinel-del"].(bool):
-		fallthrough
-	case d["--sentinel-resync"].(bool):
-		t.handleSentinelCommand(d)
-
 	case d["--sync-action"].(bool):
 		t.handleSyncActionCommand(d)
 
@@ -632,44 +625,6 @@ func (t *cmdDashboard) handleGroupCommand(d map[string]interface{}) {
 	}
 }
 
-func (t *cmdDashboard) handleSentinelCommand(d map[string]interface{}) {
-	c := t.newTopomClient()
-
-	switch {
-
-	case d["--sentinel-add"].(bool):
-
-		addr := utils.ArgumentMust(d, "--addr")
-
-		log.Debugf("call rpc add-sentinel to dashboard %s", t.addr)
-		if err := c.AddSentinel(addr); err != nil {
-			log.PanicErrorf(err, "call rpc add-sentinel to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc add-sentinel OK")
-
-	case d["--sentinel-del"].(bool):
-
-		addr := utils.ArgumentMust(d, "--addr")
-
-		force := d["--force"].(bool)
-
-		log.Debugf("call rpc del-sentinel to dashboard %s", t.addr)
-		if err := c.DelSentinel(addr, force); err != nil {
-			log.PanicErrorf(err, "call rpc del-sentinel to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc del-sentinel OK")
-
-	case d["--sentinel-resync"].(bool):
-
-		log.Debugf("call rpc resync-sentinels to dashboard %s", t.addr)
-		if err := c.ResyncSentinels(); err != nil {
-			log.PanicErrorf(err, "call rpc resync-sentinels to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc resync-sentinels OK")
-
-	}
-}
-
 func (t *cmdDashboard) handleSyncActionCommand(d map[string]interface{}) {
 	c := t.newTopomClient()
 

diff --git a/codis/cmd/fe/assets/index.html b/codis/cmd/fe/assets/index.html
@@ -654,107 +654,6 @@ <h4 style="padding-left:30px; padding-right:20px; display: inline;">Group</h4>
                 </table>
             </div>
         </div>
-
-        <div class="row" style="min-width: 1200px">
-            <div class="col-md-12"
-                 style="margin-bottom: 10px; margin-top: 30px; padding-bottom: 10px; border-bottom: solid 1px lightgray;">
-                <form class="form-inline">
-                    <h4 style="padding-left:30px; padding-right:20px; display: inline;">Sentinels</h4>
-                </form>
-            </div>
-            <div class="col-md-12"
-                 style="padding-bottom: 10px">
-                <form class="form-inline">
-                    <span ng-if="codis_addr != 'NA'">
-                        <button class="btn btn-warning btn-sm active" style="width: 120px; font-size: 14px; padding: 2px;"
-                                ng-click="addSentinel(new_sentinel_addr)"
-                                ng-disabled="!new_sentinel_addr">Add Sentinel
-                        </button>
-                        <input style="height: 28px; width: 250px; " type="text" class="form-control"
-                               placeholder="Redis Sentinel Address"
-                               ng-model="new_sentinel_addr">
-                    </span>
-                </form>
-            </div>
-            <div class="col-md-4"
-                 style="padding-bottom: 10px">
-                <table class="table table-bordered table-striped table-hover table-condensed" style="white-space: nowrap">
-                    <thead>
-                    <tr>
-                        <th style="width: 85px; text-align: center">
-                            <span ng-switch="sentinel_out_of_sync">
-                                <span ng-switch-when="true">
-                                    <button class="btn btn-danger btn-xs active"
-                                            data-toggle="tooltip" data-placement="right" title="RESYNC ALL SENTINELS & PROXIES"
-                                            style="width: 70px;"
-                                            ng-click="resyncSentinels()">
-                                        SYNC
-                                    </button>
-                                </span>
-                                <span ng-switch-default>
-                                    <button class="btn btn-primary btn-xs active"
-                                            data-toggle="tooltip" data-placement="right" title="RESYNC ALL SENTINELS & PROXIES"
-                                            style="width: 70px;"
-                                            ng-click="resyncSentinels()">
-                                        SYNC
-                                    </button>
-                                </span>
-                            </span>
-                        </th>
-                        <th style="min-width: 220px;">
-                            Sentinels
-                            <span ng-if="sentinel_out_of_sync" style="color: white; background-color: red">
-                                (OUT OF SYNC)
-                            </span>
-                        </th>
-                        <th style="min-width: 35px;"/>
-                        <th style="min-width: 350px;">
-                            Status
-                        </th>
-                        <th style="min-width: 35px;"/>
-                        <th style="min-width: 35px;"/>
-                    </tr>
-                    </thead>
-                    <tr ng-repeat="sentinel in sentinel_servers">
-                        <td style="text-align: center">
-                            <a ng-href="http://[[codis_addr]]/api/topom/sentinels/info/[[sentinel.server]]/monitored" style="width: 70px;" target="_blank" class="btn btn-default btn-xs active" role="button">WATCHED</a>
-                        </td>
-                        <td>
-                            <a ng-href="http://[[codis_addr]]/api/topom/sentinels/info/[[sentinel.server]]" target="_blank" class="btn btn-default btn-xs active" role="button">S</a>
-                            <span>
-                                [[sentinel.server]]
-                            </span>
-                        </td>
-                        <td class="button_tight_column" style="text-align: center">
-                            <button class="btn btn-danger btn-xs active" ng-if="sentinel.status == 'ERROR'"
-                                data-toggle="tooltip" data-placement="right" title="FORCE REMOVE SENTINEL [[sentinel.server]]"
-                                ng-click="delSentinel(sentinel, true)">
-                                <span class="glyphicon glyphicon-remove"></span>
-                            </button>
-                        </td>
-                        <td ng-switch="sentinel.status">
-                            <span ng-switch-when="ERROR" class="status_label_error">[[sentinel.status]]</span>
-                            <span ng-switch-when="TIMEOUT" class="status_label_warning">[[sentinel.status]]</span>
-                            <span ng-switch-when="PENDING" class="status_label_pending">[[sentinel.status]]</span>
-                            <span ng-switch-default>[[sentinel.status_text]]</span>
-                        </td>
-                        <td ng-switch="sentinel.runid_error != ''">
-                            <span ng-switch-when="true" style="color: red">[[sentinel.runid_error]]</span>
-                            <span ng-switch-default/>
-                        </td>
-                        <td class="button_tight_column">
-                            <button class="btn btn-danger btn-xs active"
-                                data-toggle="tooltip" data-placement="right" title="REMOVE SENTINEL [[sentinel.server]]"
-                                ng-click="delSentinel(sentinel, false)">
-                                <span class="glyphicon glyphicon-minus"></span>
-                            </button>
-                        </td>
-                    </tr>
-                    </tbody>
-                </table>
-            </div>
-
-        </div>
     </div>
 </div>
 

diff --git a/codis/config/dashboard.toml b/codis/config/dashboard.toml
@@ -30,6 +30,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
+sentinel_check_server_state_interval = "5s"
+sentinel_check_master_failover_interval = "1s"
+sentinel_master_dead_check_times = 5
+
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1

diff --git a/codis/config/proxy.toml b/codis/config/proxy.toml
@@ -75,7 +75,7 @@ backend_replica_parallel = 1
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 6
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

diff --git a/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml b/codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml
@@ -58,7 +58,7 @@ backend_replica_parallel = 1
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 6
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

diff --git a/codis/pkg/models/client.go b/codis/pkg/models/client.go
@@ -6,9 +6,9 @@ package models
 import (
 	"time"
 
-	"pika/codis/v2/pkg/models/etcd"
-	"pika/codis/v2/pkg/models/fs"
-	"pika/codis/v2/pkg/models/zk"
+	etcdclient "pika/codis/v2/pkg/models/etcd"
+	fsclient "pika/codis/v2/pkg/models/fs"
+	zkclient "pika/codis/v2/pkg/models/zk"
 	"pika/codis/v2/pkg/utils/errors"
 )
 

diff --git a/codis/pkg/models/group.go b/codis/pkg/models/group.go
@@ -17,6 +17,22 @@ type Group struct {
 	OutOfSync bool `json:"out_of_sync"`
 }
 
+func (g *Group) GetServersMap() map[string]*GroupServer {
+	results := make(map[string]*GroupServer)
+	for _, server := range g.Servers {
+		results[server.Addr] = server
+	}
+	return results
+}
+
+type GroupServerState int8
+
+const (
+	GroupServerStateNormal GroupServerState = iota
+	GroupServerStateSubjectiveOffline
+	GroupServerStateOffline
+)
+
 type GroupServer struct {
 	Addr       string `json:"server"`
 	DataCenter string `json:"datacenter"`
@@ -26,6 +42,16 @@ type GroupServer struct {
 		State string `json:"state,omitempty"`
 	} `json:"action"`
 
+	// master or slave
+	Role string `json:"role"`
+	// If it is a master node, take the master_repl_offset field, otherwise take the slave_repl_offset field
+	ReplyOffset int `json:"reply_offset"`
+	// Monitoring status, 0 normal, 1 subjective offline, 2 actual offline
+	// If marked as 2 , no service is provided
+	State GroupServerState `json:"state"`
+
+	ReCallTimes int8 `json:"recall_times"`
+
 	ReplicaGroup bool `json:"replica_group"`
 }