DEV VX-142: Node metrics handler

config/k8s.json

@@ -20,7 +20,7 @@
         }
     },
     "prometheus": {
-        "url": "http://prometheus.monitoring.svc.cluster.local:30003"
+        "url": "http://prometheus.monitoring.svc.cluster.local:9090"
     },
     "logger": {
         "dir": "./logs",

deploy/kubernetes/apps/monitoring/node-exporter/node-exporter.yaml

@@ -9,13 +9,13 @@ spec:
       name:  node-exporter
       labels:
         name: node-exporter
+      annotations:
+        prometheus.io/scrape: 'true'
     spec:
       hostPID: true
       containers:
       - name:  node-exporter
         image: sdnvortex/node-exporter:latest
         securityContext:
           privileged: true
-        ports:
-        - containerPort: 9100
       hostNetwork: true

deploy/kubernetes/apps/monitoring/prometheus/prometheus-config.yaml

@@ -45,6 +45,9 @@ data:
       - source_labels: [__meta_kubernetes_service_name]
         action: replace
         target_label: kubernetes_name
+      - source_labels: [__meta_kubernetes_pod_node_name]
+        action: replace
+        target_label: node
 
 
     - job_name: 'kubernetes-pods'
@@ -70,4 +73,7 @@ data:
         target_label: kubernetes_namespace
       - source_labels: [__meta_kubernetes_pod_name]
         action: replace
-        target_label: kubernetes_pod_name
+        target_label: kubernetes_pod_name
+      - source_labels: [__meta_kubernetes_pod_node_name]
+        action: replace
+        target_label: node

deploy/kubernetes/apps/monitoring/prometheus/prometheus-svc-external.yaml

@@ -0,0 +1,15 @@
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    app: prometheus
+  name: prometheus-external
+  namespace: monitoring
+spec:
+  type: NodePort
+  ports:
+  - port: 9090
+    targetPort: 9090
+    nodePort: 30003
+  selector:
+    app: prometheus

deploy/kubernetes/apps/monitoring/prometheus/prometheus-svc.yaml

@@ -6,10 +6,8 @@ metadata:
   name: prometheus
   namespace: monitoring
 spec:
-  type: NodePort
   ports:
   - port: 9090
     targetPort: 9090
-    nodePort: 30003
   selector:
-    app: prometheus
+    app: prometheus

src/entity/node_metrics.go

@@ -0,0 +1,65 @@
+package entity
+
+type NICNetworkTrafficMetrics struct {
+	ReceiveBytesTotal    int `json:"receiveBytesTotal"`
+	TransmitBytesTotal   int `json:"transmitBytesTotal"`
+	ReceivePacketsTotal  int `json:"receivePacketsTotal"`
+	TransmitPacketsTotal int `json:"transmitPacketsTotal"`
+}
+
+type NICMetrics struct {
+	Default           string                   `json:"default"`
+	Type              string                   `json:"type"`
+	IP                string                   `json:"ip"`
+	NICNetworkTraffic NICNetworkTrafficMetrics `json:"nicNetworkTraffic"`
+}
+
+type NodeResourceMetrics struct {
+	AllocatableCPU              float32 `json:"allocatableCPU"`
+	AllocatableMemory           float32 `json:"allocatableMemory"`
+	AllocatablePods             float32 `json:"allocatablePods"`
+	AllocatableEphemeralStorage float32 `json:"allocatableEphemeralStorage"`
+	CapacityCPU                 float32 `json:"capacityCPU"`
+	CapacityMemory              float32 `json:"capacityMemory"`
+	CapacityPods                float32 `json:"capacityPods"`
+	CapacityEphemeralStorage    float32 `json:"capacityEphemeralStorage"`
+	CPURequests                 float32 `json:"cpuRequests"`
+	CPULimits                   float32 `json:"cpuLimits"`
+	MemoryRequests              float32 `json:"memoryRequests"`
+	MemoryLimits                float32 `json:"memoryLimits"`
+}
+
+type NodeResourceShortMetrics struct {
+	CPURequests    float32 `json:"cpuRequests"`
+	CPULimits      float32 `json:"cpuLimits"`
+	MemoryRequests float32 `json:"memoryRequests"`
+	MemoryLimits   float32 `json:"memoryLimits"`
+}
+
+type NodeDetailMetrics struct {
+	Hostname          string            `json:"hostname"`
+	CreatedAt         int               `json:"createAt"`
+	Status            string            `json:"status"`
+	OS                string            `json:"os"`
+	KernelVersion     string            `json:"kernelVersion"`
+	KubeproxyVersion  string            `json:"kubeproxyVersion"`
+	KubernetesVersion string            `json:"kubernetesVersion"`
+	Labels            map[string]string `json:"labels"`
+}
+
+type NodeInfoMetrics struct {
+	NodeName string                   `json:"nodeName"`
+	Labels   map[string]string        `json:"labels"`
+	Status   string                   `json:"status"`
+	Resource NodeResourceShortMetrics `json:"resource"`
+}
+
+type NodeListMetrics struct {
+	Node map[string]NodeInfoMetrics `json:"node"`
+}
+
+type NodeMetrics struct {
+	Detail   NodeDetailMetrics     `json:"detail"`
+	Resource NodeResourceMetrics   `json:"resource"`
+	NICs     map[string]NICMetrics `json:"nics"`
+}

src/server/handler_node_metrics.go

@@ -0,0 +1,225 @@
+package server
+
+import (
+	"strings"
+
+	"github.com/linkernetworks/vortex/src/entity"
+	response "github.com/linkernetworks/vortex/src/net/http"
+	"github.com/linkernetworks/vortex/src/web"
+)
+
+func listNodeMetricsHandler(ctx *web.Context) {
+	sp, req, resp := ctx.ServiceProvider, ctx.Request, ctx.Response
+
+	nodeList := entity.NodeListMetrics{}
+	nodeList.Node = map[string]entity.NodeInfoMetrics{}
+
+	// kube_node_info, kube_node_labels
+	results, err := queryFromPrometheus(sp, `{__name__=~"kube_node_info|kube_node_labels"}`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		switch result.Metric["__name__"] {
+
+		case "kube_node_info":
+			nodeInfo := entity.NodeInfoMetrics{}
+			nodeInfo.NodeName = string(result.Metric["node"])
+			nodeList.Node[string(result.Metric["node"])] = nodeInfo
+
+		case "kube_node_labels":
+			nodeInfo := nodeList.Node[string(result.Metric["node"])]
+			nodeInfo.Labels = map[string]string{}
+			for key, value := range result.Metric {
+				if strings.HasPrefix(string(key), "label_") {
+					nodeInfo.Labels[strings.TrimPrefix(string(key), "label_")] = string(value)
+				}
+			}
+			nodeList.Node[string(result.Metric["node"])] = nodeInfo
+		}
+	}
+
+	// kube_node_status_condition
+	results, err = queryFromPrometheus(sp, `{__name__=~"kube_node_status_condition",status="true"}==1`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		nodeInfo := nodeList.Node[string(result.Metric["node"])]
+		nodeInfo.Status = string(result.Metric["condition"])
+		nodeList.Node[string(result.Metric["node"])] = nodeInfo
+	}
+
+	// kube_pod_container_resource_limits, kube_pod_container_resource_requests
+	results, err = queryFromPrometheus(sp, `sum by(__name__, resource,node) ({__name__=~"kube_pod_container_resource_limits|kube_pod_container_resource_requests"})`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		nodeInfo := nodeList.Node[string(result.Metric["node"])]
+		switch result.Metric["__name__"] {
+		case "kube_pod_container_resource_requests":
+			switch result.Metric["resource"] {
+			case "cpu":
+				nodeInfo.Resource.CPURequests = float32(result.Value)
+			case "memory":
+				nodeInfo.Resource.MemoryRequests = float32(result.Value)
+			}
+		case "kube_pod_container_resource_limits":
+			switch result.Metric["resource"] {
+			case "cpu":
+				nodeInfo.Resource.CPULimits = float32(result.Value)
+			case "memory":
+				nodeInfo.Resource.MemoryLimits = float32(result.Value)
+			}
+		}
+		nodeList.Node[string(result.Metric["node"])] = nodeInfo
+	}
+
+	resp.WriteEntity(nodeList)
+}
+
+func getNodeMetricsHandler(ctx *web.Context) {
+	sp, req, resp := ctx.ServiceProvider, ctx.Request, ctx.Response
+
+	id := req.PathParameter("id")
+	node := entity.NodeMetrics{}
+	node.Detail.Labels = map[string]string{}
+	node.NICs = map[string]entity.NICMetrics{}
+
+	// kube_node_info, kube_node_created, node_network_interface, kube_node_labels, kube_node_status_capacity, kube_node_status_allocatable
+	results, err := queryFromPrometheus(sp, `{__name__=~"kube_node_info|kube_node_created|node_network_interface|kube_node_labels|kube_node_status_capacity|kube_node_status_allocatable",node=~"`+id+`"}`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		switch result.Metric["__name__"] {
+
+		case "kube_node_info":
+			node.Detail.Hostname = id
+			node.Detail.KernelVersion = string(result.Metric["kernel_version"])
+			node.Detail.KubeproxyVersion = string(result.Metric["kubeproxy_version"])
+			node.Detail.OS = string(result.Metric["os_image"])
+			node.Detail.KubernetesVersion = string(result.Metric["kubelet_version"])
+
+		case "kube_node_created":
+			node.Detail.CreatedAt = int(result.Value)
+
+		case "kube_node_labels":
+			for key, value := range result.Metric {
+				if strings.HasPrefix(string(key), "label_") {
+					node.Detail.Labels[strings.TrimPrefix(string(key), "label_")] = string(value)
+				}
+			}
+
+		case "kube_node_status_allocatable":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.AllocatableCPU = float32(result.Value)
+			case "memory":
+				node.Resource.AllocatableMemory = float32(result.Value)
+			case "pods":
+				node.Resource.AllocatablePods = float32(result.Value)
+			case "ephemeral_storage":
+				node.Resource.AllocatableEphemeralStorage = float32(result.Value)
+			}
+
+		case "kube_node_status_capacity":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.CapacityCPU = float32(result.Value)
+			case "memory":
+				node.Resource.CapacityMemory = float32(result.Value)
+			case "pods":
+				node.Resource.CapacityPods = float32(result.Value)
+			case "ephemeral_storage":
+				node.Resource.CapacityEphemeralStorage = float32(result.Value)
+			}
+		}
+	}
+
+	// kube_node_status_condition
+	results, err = queryFromPrometheus(sp, `	{__name__=~"kube_node_status_condition",node=~"`+id+`",status="true"}==1`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	node.Detail.Status = string(results[0].Metric["condition"])
+
+	// kube_pod_container_resource_limits, kube_pod_container_resource_requests
+	results, err = queryFromPrometheus(sp, `sum by(__name__, resource) ({__name__=~"kube_pod_container_resource_limits|kube_pod_container_resource_requests",node=~"`+id+`"})`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		switch result.Metric["__name__"] {
+		case "kube_pod_container_resource_requests":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.CPURequests = float32(result.Value)
+			case "memory":
+				node.Resource.MemoryRequests = float32(result.Value)
+			}
+		case "kube_pod_container_resource_limits":
+			switch result.Metric["resource"] {
+			case "cpu":
+				node.Resource.CPULimits = float32(result.Value)
+			case "memory":
+				node.Resource.MemoryLimits = float32(result.Value)
+			}
+		}
+	}
+
+	// node_network_interface, node_network_receive_bytes_total, node_network_transmit_bytes_total, node_network_receive_packets_total, node_network_transmit_packets_total
+	results, err = queryFromPrometheus(sp, `{__name__=~"node_network_interface|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_network_receive_packets_total|node_network_transmit_packets_total",node=~"`+id+`"}`)
+	if err != nil {
+		response.BadRequest(req.Request, resp.ResponseWriter, err)
+		return
+	}
+
+	for _, result := range results {
+		switch result.Metric["__name__"] {
+
+		case "node_network_interface":
+			nic := entity.NICMetrics{}
+			nic.Default = string(result.Metric["default"])
+			nic.Type = string(result.Metric["type"])
+			nic.IP = string(result.Metric["ip_address"])
+			nic.NICNetworkTraffic = entity.NICNetworkTrafficMetrics{}
+			node.NICs[string(result.Metric["device"])] = nic
+
+		case "node_network_receive_bytes_total":
+			nic := node.NICs[string(result.Metric["device"])]
+			nic.NICNetworkTraffic.ReceiveBytesTotal = int(result.Value)
+			node.NICs[string(result.Metric["device"])] = nic
+
+		case "node_network_transmit_bytes_total":
+			nic := node.NICs[string(result.Metric["device"])]
+			nic.NICNetworkTraffic.TransmitBytesTotal = int(result.Value)
+			node.NICs[string(result.Metric["device"])] = nic
+
+		case "node_network_receive_packets_total":
+			nic := node.NICs[string(result.Metric["device"])]
+			nic.NICNetworkTraffic.ReceivePacketsTotal = int(result.Value)
+			node.NICs[string(result.Metric["device"])] = nic
+
+		case "node_network_transmit_packets_total":
+			nic := node.NICs[string(result.Metric["device"])]
+			nic.NICNetworkTraffic.TransmitPacketsTotal = int(result.Value)
+			node.NICs[string(result.Metric["device"])] = nic
+		}
+	}
+
+	resp.WriteEntity(node)
+}