enhance node list

linkernetworks · Jul 5, 2018 · 1962d80 · 1962d80
1 parent d79afe4
commit 1962d80
Show file tree

Hide file tree

Showing 4 changed files with 162 additions and 51 deletions.
diff --git a/deploy/kubernetes/apps/monitoring/prometheus/prometheus-config.yaml b/deploy/kubernetes/apps/monitoring/prometheus/prometheus-config.yaml
@@ -73,4 +73,7 @@ data:
         target_label: kubernetes_namespace
       - source_labels: [__meta_kubernetes_pod_name]
         action: replace
-        target_label: kubernetes_pod_name
+        target_label: kubernetes_pod_name
+      - source_labels: [__meta_kubernetes_pod_node_name]
+        action: replace
+        target_label: node
diff --git a/src/entity/node_metrics.go b/src/entity/node_metrics.go
@@ -15,27 +15,51 @@ type NICMetrics struct {
 }
 
 type NodeResourceMetrics struct {
-	AllocatableCPU    float32 `json:"allocatableCPU"`
-	AllocatableMemory float32 `json:"allocatableMemory"`
-	CapacityCPU       float32 `json:"capacityCPU"`
-	CapacityMemory    float32 `json:"capacityMemory"`
-	CPURequests       float32 `json:"cPURequests"`
-	CPULimits         float32 `json:"cPULimits"`
-	MemoryRequests    float32 `json:"memoryRequests"`
-	MemoryLimits      float32 `json:"memoryLimits"`
+	AllocatableCPU              float32 `json:"allocatableCPU"`
+	AllocatableMemory           float32 `json:"allocatableMemory"`
+	AllocatablePods             float32 `json:"allocatablePods"`
+	AllocatableEphemeralStorage float32 `json:"allocatableEphemeralStorage"`
+	CapacityCPU                 float32 `json:"capacityCPU"`
+	CapacityMemory              float32 `json:"capacityMemory"`
+	CapacityPods                float32 `json:"capacityPods"`
+	CapacityEphemeralStorage    float32 `json:"capacityEphemeralStorage"`
+	CPURequests                 float32 `json:"cpuRequests"`
+	CPULimits                   float32 `json:"cpuLimits"`
+	MemoryRequests              float32 `json:"memoryRequests"`
+	MemoryLimits                float32 `json:"memoryLimits"`
 }
 
-type NodeInfoMetrics struct {
+type NodeResourceShortMetrics struct {
+	CPURequests    float32 `json:"cpuRequests"`
+	CPULimits      float32 `json:"cpuLimits"`
+	MemoryRequests float32 `json:"memoryRequests"`
+	MemoryLimits   float32 `json:"memoryLimits"`
+}
+
+type NodeDetailMetrics struct {
 	Hostname          string            `json:"hostname"`
-	KernelVersion     string            `json:"kernelVersion"`
 	CreatedAt         int               `json:"createAt"`
+	Status            string            `json:"status"`
 	OS                string            `json:"os"`
+	KernelVersion     string            `json:"kernelVersion"`
+	KubeproxyVersion  string            `json:"kubeproxyVersion"`
 	KubernetesVersion string            `json:"kubernetesVersion"`
 	Labels            map[string]string `json:"labels"`
 }
 
+type NodeInfoMetrics struct {
+	NodeName string                   `json:"nodeName"`
+	Labels   map[string]string        `json:"labels"`
+	Status   string                   `json:"status"`
+	Resource NodeResourceShortMetrics `json:"resource"`
+}
+
+type NodeListMetrics struct {
+	Node map[string]NodeInfoMetrics `json:"node"`
+}
+
 type NodeMetrics struct {
-	Info     NodeInfoMetrics       `json:"info"`
+	Detail   NodeDetailMetrics     `json:"detail"`
 	Resource NodeResourceMetrics   `json:"resource"`
-	NICs     map[string]NICMetrics `json:nics""`
+	NICs     map[string]NICMetrics `json:"nics"`
 }
diff --git a/src/server/handler_node_metrics.go b/src/server/handler_node_metrics.go
@@ -4,29 +4,85 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/linkernetworks/logger"
 	"github.com/linkernetworks/vortex/src/entity"
 	response "github.com/linkernetworks/vortex/src/net/http"
 	"github.com/linkernetworks/vortex/src/web"
-	"github.com/prometheus/common/model"
 )
 
 func listNodeMetricsHandler(ctx *web.Context) {
 	sp, req, resp := ctx.ServiceProvider, ctx.Request, ctx.Response
 
-	result, err := queryFromPrometheus(sp, "sum by (node)(kube_node_info)")
+	nodeList := entity.NodeListMetrics{}
+	nodeList.Node = map[string]entity.NodeInfoMetrics{}
 
-	if result == nil {
-		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v: %v", result, err))
+	// kube_node_info, kube_node_labels
+	results, err := queryFromPrometheus(sp, `{__name__=~"kube_node_info|kube_node_labels"}`)
+
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
+	}
+
+	for _, result := range results {
+		switch result.Metric["__name__"] {
+
+		case "kube_node_info":
+			nodeInfo := entity.NodeInfoMetrics{}
+			nodeInfo.NodeName = string(result.Metric["node"])
+			nodeList.Node[string(result.Metric["node"])] = nodeInfo
+
+		case "kube_node_labels":
+			nodeInfo := nodeList.Node[string(result.Metric["node"])]
+			nodeInfo.Labels = map[string]string{}
+			for key, value := range result.Metric {
+				if strings.HasPrefix(string(key), "label_") {
+					nodeInfo.Labels[strings.TrimPrefix(string(key), "label_")] = string(value)
+				}
+			}
+			nodeList.Node[string(result.Metric["node"])] = nodeInfo
+		}
+	}
+
+	// kube_node_status_condition
+	results, err = queryFromPrometheus(sp, `{__name__=~"kube_node_status_condition",status="true"}==1`)
+
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
+	}
+
+	for _, result := range results {
+		nodeInfo := nodeList.Node[string(result.Metric["node"])]
+		nodeInfo.Status = string(result.Metric["condition"])
+		nodeList.Node[string(result.Metric["node"])] = nodeInfo
 	}
 
-	nodeList := []model.LabelValue{}
+	// kube_pod_container_resource_limits, kube_pod_container_resource_requests
+	results, err = queryFromPrometheus(sp, `sum by(__name__, resource,node) ({__name__=~"kube_pod_container_resource_limits|kube_pod_container_resource_requests"})`)
 
-	for _, node := range result {
-		nodeList = append(nodeList, node.Metric["node"])
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
+	}
+
+	for _, result := range results {
+		nodeInfo := nodeList.Node[string(result.Metric["node"])]
+		switch result.Metric["__name__"] {
+		case "kube_pod_container_resource_requests":
+			switch result.Metric["resource"] {
+			case "cpu":
+				nodeInfo.Resource.CPURequests = float32(result.Value)
+			case "memory":
+				nodeInfo.Resource.MemoryRequests = float32(result.Value)
+			}
+		case "kube_pod_container_resource_limits":
+			switch result.Metric["resource"] {
+			case "cpu":
+				nodeInfo.Resource.CPULimits = float32(result.Value)
+			case "memory":
+				nodeInfo.Resource.MemoryLimits = float32(result.Value)
+			}
+		}
+		nodeList.Node[string(result.Metric["node"])] = nodeInfo
 	}
 
-	logger.Infof("fetching all nodes. found %d nodes", len(nodeList))
 	resp.WriteEntity(nodeList)
 }
 
@@ -35,50 +91,76 @@ func getNodeMetricsHandler(ctx *web.Context) {
 
 	id := req.PathParameter("id")
 	node := entity.NodeMetrics{}
-	node.Info.Labels = map[string]string{}
+	node.Detail.Labels = map[string]string{}
 	node.NICs = map[string]entity.NICMetrics{}
 
-	results, err := queryFromPrometheus(sp, `{__name__=~"kube_node_info|kube_node_created|node_network_interface|kube_node_labels|kube_node_status_allocatable_cpu_cores|kube_node_status_allocatable_memory_bytes|kube_node_status_capacity_cpu_cores|kube_node_status_capacity_memory_bytes|",node=~"`+id+`"}`)
-	if results == nil {
-		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v: %v", results, err))
+	// kube_node_info, kube_node_created, node_network_interface, kube_node_labels, kube_node_status_capacity, kube_node_status_allocatable
+	results, err := queryFromPrometheus(sp, `{__name__=~"kube_node_info|kube_node_created|node_network_interface|kube_node_labels|kube_node_status_capacity|kube_node_status_allocatable",node=~"`+id+`"}`)
+
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
 	}
 
 	for _, result := range results {
 		switch result.Metric["__name__"] {
 
 		case "kube_node_info":
-			node.Info.Hostname = id
-			node.Info.KernelVersion = string(result.Metric["kernel_version"])
-			node.Info.OS = string(result.Metric["os_image"])
-			node.Info.KubernetesVersion = string(result.Metric["kubelet_version"])
+			node.Detail.Hostname = id
+			node.Detail.KernelVersion = string(result.Metric["kernel_version"])
+			node.Detail.KubeproxyVersion = string(result.Metric["kubeproxy_version"])
+			node.Detail.OS = string(result.Metric["os_image"])
+			node.Detail.KubernetesVersion = string(result.Metric["kubelet_version"])
 
 		case "kube_node_created":
-			node.Info.CreatedAt = int(result.Value)
+			node.Detail.CreatedAt = int(result.Value)
 
 		case "kube_node_labels":
 			for key, value := range result.Metric {
 				if strings.HasPrefix(string(key), "label_") {
-					node.Info.Labels[strings.TrimPrefix(string(key), "label_")] = string(value)
+					node.Detail.Labels[strings.TrimPrefix(string(key), "label_")] = string(value)
 				}
 			}
 
-		case "kube_node_status_allocatable_cpu_cores":
-			node.Resource.AllocatableCPU = float32(result.Value)
+		case "kube_node_status_allocatable":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.AllocatableCPU = float32(result.Value)
+			case "memory":
+				node.Resource.AllocatableMemory = float32(result.Value)
+			case "pods":
+				node.Resource.AllocatablePods = float32(result.Value)
+			case "ephemeral_storage":
+				node.Resource.AllocatableEphemeralStorage = float32(result.Value)
+			}
 
-		case "kube_node_status_allocatable_memory_bytes":
-			node.Resource.AllocatableMemory = float32(result.Value)
+		case "kube_node_status_capacity":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.CapacityCPU = float32(result.Value)
+			case "memory":
+				node.Resource.CapacityMemory = float32(result.Value)
+			case "pods":
+				node.Resource.CapacityPods = float32(result.Value)
+			case "ephemeral_storage":
+				node.Resource.CapacityEphemeralStorage = float32(result.Value)
+			}
+		}
+	}
 
-		case "kube_node_status_capacity_cpu_cores":
-			node.Resource.CapacityCPU = float32(result.Value)
+	// kube_node_status_condition
+	results, err = queryFromPrometheus(sp, `	{__name__=~"kube_node_status_condition",node=~"`+id+`",status="true"}==1`)
 
-		case "kube_node_status_capacity_memory_bytes":
-			node.Resource.CapacityMemory = float32(result.Value)
-		}
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
 	}
 
+	node.Detail.Status = string(results[0].Metric["condition"])
+
+	// kube_pod_container_resource_limits, kube_pod_container_resource_requests
 	results, err = queryFromPrometheus(sp, `sum by(__name__, resource) ({__name__=~"kube_pod_container_resource_limits|kube_pod_container_resource_requests",node=~"`+id+`"})`)
-	if results == nil {
-		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v: %v", results, err))
+
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
 	}
 
 	for _, result := range results {
@@ -100,9 +182,11 @@ func getNodeMetricsHandler(ctx *web.Context) {
 		}
 	}
 
+	// node_network_interface, node_network_receive_bytes_total, node_network_transmit_bytes_total, node_network_receive_packets_total, node_network_transmit_packets_total
 	results, err = queryFromPrometheus(sp, `{__name__=~"node_network_interface|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_network_receive_packets_total|node_network_transmit_packets_total",node=~"`+id+`"}`)
-	if results == nil {
-		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v: %v", results, err))
+
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v", err))
 	}
 
 	for _, result := range results {
@@ -139,5 +223,4 @@ func getNodeMetricsHandler(ctx *web.Context) {
 	}
 
 	resp.WriteEntity(node)
-
 }
diff --git a/src/server/handler_prometheus.go b/src/server/handler_prometheus.go
@@ -22,11 +22,12 @@ func queryMetrics(ctx *web.Context) {
 	var expression string
 	if q, ok := query.Str("query"); ok {
 		expression = q
+	} else {
+		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("wrong expression: %v", q))
 	}
 
 	result, err := queryFromPrometheus(sp, expression)
-
-	if result == nil {
+	if err != nil {
 		response.BadRequest(req.Request, resp.ResponseWriter, fmt.Errorf("%v: %v", result, err))
 	}
 
@@ -52,8 +53,8 @@ func queryFromPrometheus(sp *serviceprovider.Container, expression string) (mode
 
 	switch {
 	case result.Type() == model.ValVector:
-		return result.(model.Vector), err
+		return result.(model.Vector), nil
+	default:
+		return nil, fmt.Errorf("the type of the return result can not be identify")
 	}
-
-	return nil, err
 }