diff --git a/clusters/kind-cluster/base/flux-system/gotk-sync.yaml b/clusters/kind-cluster/base/flux-system/gotk-sync.yaml index 48609d96..d9471eee 100644 --- a/clusters/kind-cluster/base/flux-system/gotk-sync.yaml +++ b/clusters/kind-cluster/base/flux-system/gotk-sync.yaml @@ -7,7 +7,7 @@ metadata: spec: interval: 1m0s ref: - branch: main + branch: test/rename-podmonitors url: https://github.com/digicatapult/sqnc-flux-infra.git --- diff --git a/clusters/kind-cluster/monitoring/configs/dashboards/cluster.json b/clusters/kind-cluster/monitoring/dashboards/cluster.json similarity index 100% rename from clusters/kind-cluster/monitoring/configs/dashboards/cluster.json rename to clusters/kind-cluster/monitoring/dashboards/cluster.json diff --git a/clusters/kind-cluster/monitoring/configs/dashboards/control-plane.json b/clusters/kind-cluster/monitoring/dashboards/control-plane.json similarity index 100% rename from clusters/kind-cluster/monitoring/configs/dashboards/control-plane.json rename to clusters/kind-cluster/monitoring/dashboards/control-plane.json diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/kube-state-metrics-config.yaml b/clusters/kind-cluster/monitoring/kube-state-metrics-config.yaml similarity index 100% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/kube-state-metrics-config.yaml rename to clusters/kind-cluster/monitoring/kube-state-metrics-config.yaml diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomization.yaml b/clusters/kind-cluster/monitoring/kustomization.yaml similarity index 67% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomization.yaml rename to clusters/kind-cluster/monitoring/kustomization.yaml index 65989127..db2d743e 100644 --- a/clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomization.yaml +++ b/clusters/kind-cluster/monitoring/kustomization.yaml @@ -16,6 +16,15 @@ configMapGenerator: labels: app.kubernetes.io/part-of: flux app.kubernetes.io/component: monitoring + - name: flux-grafana-dashboards + files: + - dashboards/control-plane.json + - dashboards/cluster.json + options: + labels: + grafana_dashboard: "1" + app.kubernetes.io/part-of: flux + app.kubernetes.io/component: monitoring configurations: - kustomize-config.yaml generatorOptions: diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomize-config.yaml b/clusters/kind-cluster/monitoring/kustomize-config.yaml similarity index 100% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomize-config.yaml rename to clusters/kind-cluster/monitoring/kustomize-config.yaml diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/monitoring-podmonitor-sync.yaml b/clusters/kind-cluster/monitoring/monitoring-podmonitor-sync.yaml similarity index 86% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/monitoring-podmonitor-sync.yaml rename to clusters/kind-cluster/monitoring/monitoring-podmonitor-sync.yaml index 87c0da06..c47c59d5 100644 --- a/clusters/kind-cluster/monitoring/kube-prometheus-stack/monitoring-podmonitor-sync.yaml +++ b/clusters/kind-cluster/monitoring/monitoring-podmonitor-sync.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: interval: 1m - path: ./clusters/kind-cluster/monitoring-configs + path: ./clusters/kind-cluster/podmonitors prune: true dependsOn: - name: monitoring-sync diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/release.yaml b/clusters/kind-cluster/monitoring/release.yaml similarity index 100% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/release.yaml rename to clusters/kind-cluster/monitoring/release.yaml diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/source.yaml b/clusters/kind-cluster/monitoring/source.yaml similarity index 100% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/source.yaml rename to clusters/kind-cluster/monitoring/source.yaml diff --git a/clusters/kind-cluster/monitoring/kube-prometheus-stack/values.yaml b/clusters/kind-cluster/monitoring/values.yaml similarity index 100% rename from clusters/kind-cluster/monitoring/kube-prometheus-stack/values.yaml rename to clusters/kind-cluster/monitoring/values.yaml diff --git a/clusters/kind-cluster/monitoring-configs/kustomization.yaml b/clusters/kind-cluster/podmonitors/kustomization.yaml similarity index 100% rename from clusters/kind-cluster/monitoring-configs/kustomization.yaml rename to clusters/kind-cluster/podmonitors/kustomization.yaml diff --git a/clusters/kind-cluster/monitoring-configs/podmonitor.yaml b/clusters/kind-cluster/podmonitors/podmonitor.yaml similarity index 100% rename from clusters/kind-cluster/monitoring-configs/podmonitor.yaml rename to clusters/kind-cluster/podmonitors/podmonitor.yaml diff --git a/clusters/l3-sqnc/base/app-sync.yaml b/clusters/l3-sqnc/base/app-sync.yaml index 1cb43eb9..36c5ef20 100644 --- a/clusters/l3-sqnc/base/app-sync.yaml +++ b/clusters/l3-sqnc/base/app-sync.yaml @@ -1,6 +1,44 @@ --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization +metadata: + name: monitoring-sync + namespace: monitoring +spec: + interval: 1m + path: ./clusters/l3-sqnc/monitoring + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + healthChecks: + - apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: podmonitors.monitoring.coreos.com + - apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: servicemonitors.monitoring.coreos.com +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nginx-sync + namespace: infra +spec: + interval: 1m + path: ./clusters/l3-sqnc/nginx + prune: true + dependsOn: + - name: monitoring-sync + namespace: monitoring + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization metadata: name: capacity-sync namespace: capacity @@ -8,6 +46,9 @@ spec: interval: 1m path: ./clusters/l3-sqnc/capacity prune: true + dependsOn: + - name: monitoring-sync + namespace: monitoring sourceRef: kind: GitRepository name: flux-system @@ -22,6 +63,9 @@ spec: interval: 1m path: ./clusters/l3-sqnc/optimiser prune: true + dependsOn: + - name: monitoring-sync + namespace: monitoring sourceRef: kind: GitRepository name: flux-system @@ -36,6 +80,9 @@ spec: interval: 1m path: ./clusters/l3-sqnc/order prune: true + dependsOn: + - name: monitoring-sync + namespace: monitoring sourceRef: kind: GitRepository name: flux-system diff --git a/clusters/l3-sqnc/base/namespaces.yaml b/clusters/l3-sqnc/base/namespaces.yaml index 69710260..0f4283ba 100644 --- a/clusters/l3-sqnc/base/namespaces.yaml +++ b/clusters/l3-sqnc/base/namespaces.yaml @@ -1,6 +1,22 @@ --- apiVersion: v1 kind: Namespace +metadata: + labels: + app.kubernetes.io/instance: sqnc + app.kubernetes.io/version: latest + name: monitoring +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/instance: sqnc + app.kubernetes.io/version: latest + name: infra +--- +apiVersion: v1 +kind: Namespace metadata: labels: app.kubernetes.io/instance: sqnc diff --git a/clusters/l3-sqnc/capacity/identity-service/values.yaml b/clusters/l3-sqnc/capacity/identity-service/values.yaml index f0af7a92..d958bc13 100644 --- a/clusters/l3-sqnc/capacity/identity-service/values.yaml +++ b/clusters/l3-sqnc/capacity/identity-service/values.yaml @@ -16,3 +16,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 5Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: capacity + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace diff --git a/clusters/l3-sqnc/capacity/magenta-node/values.yaml b/clusters/l3-sqnc/capacity/magenta-node/values.yaml index 0991b20c..06f37f0d 100644 --- a/clusters/l3-sqnc/capacity/magenta-node/values.yaml +++ b/clusters/l3-sqnc/capacity/magenta-node/values.yaml @@ -17,4 +17,7 @@ node: - "--unsafe-rpc-external" - "--prometheus-external" - "--state-pruning=100000" + serviceMonitor: + enabled: true + namespace: capacity storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/capacity/matchmaker-api/values.yaml b/clusters/l3-sqnc/capacity/matchmaker-api/values.yaml index 04d3eaa6..b04550d3 100644 --- a/clusters/l3-sqnc/capacity/matchmaker-api/values.yaml +++ b/clusters/l3-sqnc/capacity/matchmaker-api/values.yaml @@ -21,3 +21,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 15Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: capacity + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace \ No newline at end of file diff --git a/clusters/l3-sqnc/capacity/nginx/values.yaml b/clusters/l3-sqnc/capacity/nginx/values.yaml index b0b010df..375b2865 100644 --- a/clusters/l3-sqnc/capacity/nginx/values.yaml +++ b/clusters/l3-sqnc/capacity/nginx/values.yaml @@ -21,3 +21,8 @@ tolerations: containerPorts: http: 80 https: 443 +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: capacity diff --git a/clusters/l3-sqnc/capacity/red-node/values.yaml b/clusters/l3-sqnc/capacity/red-node/values.yaml index c81acc2f..40c81105 100644 --- a/clusters/l3-sqnc/capacity/red-node/values.yaml +++ b/clusters/l3-sqnc/capacity/red-node/values.yaml @@ -17,4 +17,7 @@ node: - "--unsafe-rpc-external" - "--prometheus-external" - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'" + serviceMonitor: + enabled: true + namespace: capacity storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/monitoring/dashboards/cluster.json b/clusters/l3-sqnc/monitoring/dashboards/cluster.json new file mode 100644 index 00000000..0fbb5d00 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/dashboards/cluster.json @@ -0,0 +1,1383 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "red", + "name": "flux events", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "flux" + ], + "type": "tags" + } + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"})", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Reconcilers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\", ready=\"False\"})", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Failing Reconcilers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"})", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Kubernetes Manifests Sources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\", ready=\"False\"})", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Failing Sources", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 61 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)", + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "title": "Reconciler ops avg. duration", + "type": "bargauge" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 61 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 31, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)", + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "title": "Source ops avg. duration", + "type": "bargauge" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 15, + "panels": [], + "title": "Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "Not Ready" + }, + "True": { + "color": "blue", + "index": 0, + "text": "Ready" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "string" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 22, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Status" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cluster reconciliation readiness ", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false, + "Value": false, + "__name__": false, + "app": false, + "chart_name": false, + "chart_source_name": false, + "container": false, + "customresource_group": false, + "customresource_kind": false, + "customresource_version": false, + "endpoint": false, + "exported_namespace": false, + "gotk_type": false, + "instance": false, + "job": false, + "kubernetes_namespace": false, + "kubernetes_pod_name": false, + "namespace": false, + "pod": false, + "pod_template_hash": false, + "revision": false, + "service": false, + "source_name": false, + "status": false, + "suspended": false, + "type": false + }, + "indexByName": { + "Time": 0, + "Value": 15, + "__name__": 1, + "container": 2, + "customresource_group": 4, + "customresource_kind": 5, + "customresource_version": 6, + "endpoint": 7, + "exported_namespace": 3, + "instance": 8, + "job": 9, + "name": 10, + "namespace": 11, + "pod": 12, + "ready": 13, + "service": 14 + }, + "renameByName": { + "Value": "", + "customresource_kind": "Kind", + "exported_namespace": "Namespace", + "kind": "Kind", + "name": "Name", + "namespace": "Operator Namespace", + "ready": "Status" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "(Namespace|Kind|Name|Status)" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "Not Ready" + }, + "True": { + "color": "blue", + "index": 0, + "text": "Ready" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "string" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "noValue", + "value": "Ready" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 34, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Status" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Source acquisition readiness ", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false, + "Value": false, + "__name__": false, + "app": false, + "bucket_name": false, + "container": false, + "customresource_group": false, + "customresource_kind": false, + "customresource_version": false, + "endpoint": false, + "exported_namespace": false, + "gotk_type": false, + "instance": false, + "job": false, + "kubernetes_namespace": false, + "kubernetes_pod_name": false, + "namespace": false, + "pod": false, + "pod_template_hash": false, + "ready": false, + "revision": false, + "service": false, + "status": false, + "suspended": false, + "type": false, + "url": false + }, + "indexByName": { + "Time": 0, + "Value": 15, + "__name__": 1, + "container": 2, + "customresource_group": 5, + "customresource_kind": 6, + "customresource_version": 7, + "endpoint": 8, + "exported_namespace": 4, + "instance": 9, + "job": 10, + "name": 11, + "namespace": 3, + "pod": 12, + "ready": 13, + "service": 14 + }, + "renameByName": { + "Value": "", + "customresource_kind": "Kind", + "exported_namespace": "Namespace", + "kind": "Kind", + "name": "Name", + "namespace": "Operator Namespace", + "ready": "Status" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "(Namespace|Kind|Name|Status)" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "string" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 36, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Namespace" + } + ] + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", suspended=\"true\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Suspended Objects", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false, + "Value": false, + "__name__": false, + "app": false, + "bucket_name": false, + "container": false, + "customresource_group": false, + "customresource_kind": false, + "customresource_version": false, + "endpoint": false, + "exported_namespace": false, + "gotk_type": false, + "instance": false, + "job": false, + "kubernetes_namespace": false, + "kubernetes_pod_name": false, + "namespace": false, + "pod": false, + "pod_template_hash": false, + "ready": false, + "revision": false, + "service": false, + "source_name": false, + "status": false, + "suspended": false, + "type": false, + "url": false + }, + "indexByName": { + "Time": 0, + "Value": 15, + "__name__": 1, + "container": 2, + "customresource_group": 5, + "customresource_kind": 6, + "customresource_version": 7, + "endpoint": 8, + "exported_namespace": 4, + "instance": 9, + "job": 10, + "name": 11, + "namespace": 3, + "pod": 12, + "ready": 13, + "service": 14 + }, + "renameByName": { + "customresource_kind": "Kind", + "exported_namespace": "Namespace", + "name": "Name" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "(Namespace|Name|Kind)" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 17, + "panels": [], + "title": "Timing", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}/{{name}}", + "refId": "B" + } + ], + "title": "Cluster reconciliation duration", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}/{{name}}", + "refId": "B" + } + ], + "title": "Source acquisition duration", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "light", + "tags": [ + "flux" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "definition": "label_values(gotk_reconcile_condition, namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "operator_namespace", + "options": [], + "query": { + "query": "label_values(gotk_reconcile_condition, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "definition": "label_values(gotk_resource_info,exported_namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(gotk_resource_info,exported_namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Flux Cluster Stats", + "uid": "flux-cluster", + "version": 4, + "weekStart": "" +} diff --git a/clusters/l3-sqnc/monitoring/dashboards/control-plane.json b/clusters/l3-sqnc/monitoring/dashboards/control-plane.json new file mode 100644 index 00000000..d47d391c --- /dev/null +++ b/clusters/l3-sqnc/monitoring/dashboards/control-plane.json @@ -0,0 +1,1730 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "red", + "name": "flux events", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "flux" + ], + "type": "tags" + } + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(go_info{namespace=\"$namespace\",pod=~\".*-controller-.*\"})", + "interval": "", + "legendFormat": "pods", + "refId": "A" + } + ], + "title": "Controllers", + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})", + "hide": false, + "interval": "", + "legendFormat": "seconds", + "refId": "B" + } + ], + "title": "Max Work Queue", + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "#EAB839", + "value": 500000000 + }, + { + "color": "red", + "value": 900000000 + } + ] + }, + "unit": "decbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 25, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))", + "interval": "", + "legendFormat": "requests", + "refId": "A" + } + ], + "title": "API Requests", + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "total", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "errors", + "refId": "B" + } + ], + "title": "Kubernetes API Requests", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 15, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Resource Usage", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 17, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Reconciliation Stats", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}", + "hide": false, + "interval": "", + "legendFormat": "kustomizations", + "refId": "B" + } + ], + "title": "Cluster Reconciliation Duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "successful reconciliations ", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "failed reconciliations ", + "refId": "B" + } + ], + "title": "Cluster Reconciliations ops/min", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 29, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Sources Stats", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "successful git pulls", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "failed git pulls", + "refId": "B" + } + ], + "title": "Git Repos ops/min", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "successful oci pulls", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "failed oci pulls", + "refId": "B" + } + ], + "title": "OCI Repos ops/min", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "successful helm pulls", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "failed helm pulls", + "refId": "B" + } + ], + "title": "Helm Repos ops/min", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "successful bucket pulls", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))", + "format": "time_series", + "interval": "", + "legendFormat": "failed bucket pulls", + "refId": "B" + } + ], + "title": "Buckets ops/min", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Helm Stats", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))", + "hide": true, + "interval": "", + "legendFormat": "P50", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))", + "hide": true, + "interval": "", + "legendFormat": "P90", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "P99", + "refId": "C" + } + ], + "title": "Helm Release Duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "successful reconciliations ", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "failed reconciliations ", + "refId": "B" + } + ], + "title": "Helm Releases ops/min", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "successful chart pulls", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)", + "format": "time_series", + "interval": "", + "legendFormat": "failed chart pulls", + "refId": "B" + } + ], + "title": "Helm Charts ops/min", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "light", + "tags": [ + "flux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "flux-system", + "value": "flux-system" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "workqueue_work_duration_seconds_count", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "workqueue_work_duration_seconds_count", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Flux Control Plane", + "uid": "flux-control-plane", + "version": 2, + "weekStart": "" +} diff --git a/clusters/l3-sqnc/monitoring/kube-state-metrics-config.yaml b/clusters/l3-sqnc/monitoring/kube-state-metrics-config.yaml new file mode 100644 index 00000000..547a37a3 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/kube-state-metrics-config.yaml @@ -0,0 +1,270 @@ +# pulled from https://github.com/fluxcd/flux2-monitoring-example/blob/main/monitoring/controllers/kube-prometheus-stack/kube-state-metrics-config.yaml +# configuration docs: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics#configuration +kube-state-metrics: + collectors: [ ] + extraArgs: + - --custom-resource-state-only=true + rbac: + extraRules: + - apiGroups: + - source.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - helm.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + resources: + - gitrepositories + - buckets + - helmrepositories + - helmcharts + - ocirepositories + - kustomizations + - helmreleases + - alerts + - providers + - receivers + verbs: [ "list", "watch" ] + customResourceState: + enabled: true + config: + spec: + resources: + - groupVersionKind: + group: kustomize.toolkit.fluxcd.io + version: v1 + kind: Kustomization + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + source_name: [ spec, sourceRef, name ] + - groupVersionKind: + group: helm.toolkit.fluxcd.io + version: v2beta2 + kind: HelmRelease + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + chart_name: [ spec, chart, spec, chart ] + chart_source_name: [ spec, chart, spec, sourceRef, name ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1 + kind: GitRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: Bucket + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + endpoint: [ spec, endpoint ] + bucket_name: [ spec, bucketName ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmChart + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + chart_name: [ spec, chart ] + chart_version: [ spec, version ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: OCIRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Alert + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Provider + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1 + kind: Receiver + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + webhook_path: [ status, webhookPath ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImageRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + image: [ spec, image ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImagePolicy + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, imageRepositoryRef, name ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta1 + kind: ImageUpdateAutomation + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, sourceRef, name ] \ No newline at end of file diff --git a/clusters/kind-cluster/monitoring/configs/kustomization.yaml b/clusters/l3-sqnc/monitoring/kustomization.yaml similarity index 51% rename from clusters/kind-cluster/monitoring/configs/kustomization.yaml rename to clusters/l3-sqnc/monitoring/kustomization.yaml index 385f0ee3..db2d743e 100644 --- a/clusters/kind-cluster/monitoring/configs/kustomization.yaml +++ b/clusters/l3-sqnc/monitoring/kustomization.yaml @@ -1,7 +1,21 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: monitoring +resources: + - source.yaml + - release.yaml + - monitoring-podmonitor-sync.yaml configMapGenerator: + - name: monitoring-values + files: + - values.yaml + - name: flux-kube-state-metrics-config + files: + - kube-state-metrics-config.yaml + options: + labels: + app.kubernetes.io/part-of: flux + app.kubernetes.io/component: monitoring - name: flux-grafana-dashboards files: - dashboards/control-plane.json @@ -11,5 +25,7 @@ configMapGenerator: grafana_dashboard: "1" app.kubernetes.io/part-of: flux app.kubernetes.io/component: monitoring +configurations: + - kustomize-config.yaml generatorOptions: disableNameSuffixHash: true diff --git a/clusters/l3-sqnc/monitoring/kustomize-config.yaml b/clusters/l3-sqnc/monitoring/kustomize-config.yaml new file mode 100644 index 00000000..7c31b227 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/kustomize-config.yaml @@ -0,0 +1,6 @@ +nameReference: +- kind: ConfigMap + version: v1 + fieldSpecs: + - path: spec/valuesFrom/name + kind: HelmRelease diff --git a/clusters/l3-sqnc/monitoring/monitoring-pod-monitor-sync.yaml b/clusters/l3-sqnc/monitoring/monitoring-pod-monitor-sync.yaml new file mode 100644 index 00000000..10b976d9 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/monitoring-pod-monitor-sync.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: monitoring-podmonitoring-sync + namespace: monitoring +spec: + interval: 1m + path: ./clusters/l3-sqnc/podmonitors + prune: true + dependsOn: + - name: monitoring-sync + namespace: monitoring + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system diff --git a/clusters/l3-sqnc/monitoring/release.yaml b/clusters/l3-sqnc/monitoring/release.yaml new file mode 100644 index 00000000..db81f7ea --- /dev/null +++ b/clusters/l3-sqnc/monitoring/release.yaml @@ -0,0 +1,36 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: kube-prometheus-stack +spec: + interval: 1h + chart: + spec: + version: "57.x" + chart: kube-prometheus-stack + sourceRef: + kind: HelmRepository + name: prometheus-community + interval: 1h + install: + crds: Create + upgrade: + crds: CreateReplace + driftDetection: + mode: enabled + ignore: + # Ignore "validated" annotation which is not inserted during install + - paths: [ "/metadata/annotations/prometheus-operator-validated" ] + target: + kind: PrometheusRule + valuesFrom: + - kind: ConfigMap + name: flux-kube-state-metrics-config + valuesKey: kube-state-metrics-config.yaml + - kind: ConfigMap + name: monitoring-values + valuesKey: values.yaml + - kind: Secret + name: monitoring-secrets + valuesKey: grafana_password + targetPath: grafana.adminPassword diff --git a/clusters/l3-sqnc/monitoring/source.yaml b/clusters/l3-sqnc/monitoring/source.yaml new file mode 100644 index 00000000..f14b8844 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/source.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: prometheus-community +spec: + interval: 12h + type: oci + url: oci://ghcr.io/prometheus-community/charts diff --git a/clusters/l3-sqnc/monitoring/values.yaml b/clusters/l3-sqnc/monitoring/values.yaml new file mode 100644 index 00000000..163159c6 --- /dev/null +++ b/clusters/l3-sqnc/monitoring/values.yaml @@ -0,0 +1,109 @@ +alertmanager: + enabled: true + serviceMonitor: + selfMonitor: true + namespace: monitoring + config: + receivers: + - name: "null" + # - + # - name: slack + # webhook_configs: + # - send_resolved: true + # url: + alertmanagerSpec: + replicas: 2 + podAntiAffinity: soft +prometheus: + serviceMonitor: + selfMonitor: true + namespace: monitoring + service: + sessionAffinity: ClientIP # Ensures grafana always pulls from the same replica + prometheusSpec: + replicas: 2 + podAntiAffinity: soft + retention: 12h + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: managed-csi-premium + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 2000m + memory: 2Gi + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorNamespaceSelector: { } + podMonitorSelectorNilUsesHelmValues: false + podMonitorNamespaceSelector: { } + podMonitorSelector: + matchLabels: + app.kubernetes.io/component: monitoring +grafana: + defaultDashboardsEnabled: false + serviceMonitor: + enabled: true + namespace: monitoring + ingress: + enabled: true + ingressClassName: nginx + hosts: + - grafana.dsch-l3.com + persistence: + enabled: true + type: pvc + storageClassName: managed-csi-premium + accessModes: + - ReadWriteOnce + size: 10Gi + finalizers: + - kubernetes.io/pvc-protection + plugins: + - camptocamp-prometheus-alertmanager-datasource + additionalDataSources: + - name: "Prometheus AlertManager" + type: camptocamp-prometheus-alertmanager-datasource + uid: prometheus-alertmanager + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + dashboards: + default: + postgresql: + gnetId: 9628 + revision: 7 + datasource: Prometheus + nginx: + gnetId: 9614 + revision: 1 + datasource: Prometheus + flux2: + gnetId: 16714 + revision: 1 + datasource: Prometheus + alertmanager: + gnetId: 9578 + revision: 4 + datasource: Prometheus + polkadot: + gnetId: 13840 + revision: 4 + datasource: Prometheus diff --git a/clusters/l3-sqnc/nginx/kustomization.yaml b/clusters/l3-sqnc/nginx/kustomization.yaml new file mode 100644 index 00000000..3b6bce4e --- /dev/null +++ b/clusters/l3-sqnc/nginx/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - release.yaml + - source.yaml diff --git a/clusters/l3-sqnc/nginx/release.yaml b/clusters/l3-sqnc/nginx/release.yaml new file mode 100644 index 00000000..cbeeaf22 --- /dev/null +++ b/clusters/l3-sqnc/nginx/release.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx + namespace: infra +spec: + install: + remediation: + retries: -1 + releaseName: nginx-ingress-controller + chart: + spec: + chart: nginx-ingress-controller + sourceRef: + kind: HelmRepository + name: bitnami + interval: 10m + # Default values + # https://github.com/bitnami/charts/blob/master/bitnami/nginx-ingress-controller/values.yaml + configMapGenerator: + - name: nginx-values + files: + - values.yaml diff --git a/clusters/l3-sqnc/nginx/source.yaml b/clusters/l3-sqnc/nginx/source.yaml new file mode 100644 index 00000000..7746de68 --- /dev/null +++ b/clusters/l3-sqnc/nginx/source.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: bitnami + namespace: infra +spec: + interval: 10m + url: https://charts.bitnami.com/bitnami diff --git a/clusters/l3-sqnc/nginx/values.yaml b/clusters/l3-sqnc/nginx/values.yaml new file mode 100644 index 00000000..ae2d63b6 --- /dev/null +++ b/clusters/l3-sqnc/nginx/values.yaml @@ -0,0 +1,28 @@ +watchIngressWithoutClass: true +scope: + enabled: true +ingressClassResource: + name: nginx + default: true +service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path: /healthz + external-dns.alpha.kubernetes.io/hostname: sqnc.dsch-l3.com +nodeSelector: + kubernetes.io/os: linux +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal +containerPorts: + http: 80 + https: 443 +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: monitoring diff --git a/clusters/l3-sqnc/optimiser/green-node/values.yaml b/clusters/l3-sqnc/optimiser/green-node/values.yaml index c81acc2f..fe76407e 100644 --- a/clusters/l3-sqnc/optimiser/green-node/values.yaml +++ b/clusters/l3-sqnc/optimiser/green-node/values.yaml @@ -17,4 +17,7 @@ node: - "--unsafe-rpc-external" - "--prometheus-external" - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'" + serviceMonitor: + enabled: true + namespace: optimiser storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/optimiser/identity-service/values.yaml b/clusters/l3-sqnc/optimiser/identity-service/values.yaml index 132607bf..6f38ec29 100644 --- a/clusters/l3-sqnc/optimiser/identity-service/values.yaml +++ b/clusters/l3-sqnc/optimiser/identity-service/values.yaml @@ -16,3 +16,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 5Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: optimiser + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace diff --git a/clusters/l3-sqnc/optimiser/matchmaker-api/values.yaml b/clusters/l3-sqnc/optimiser/matchmaker-api/values.yaml index fd8f3f1b..64bf6a64 100644 --- a/clusters/l3-sqnc/optimiser/matchmaker-api/values.yaml +++ b/clusters/l3-sqnc/optimiser/matchmaker-api/values.yaml @@ -21,3 +21,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 15Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: optimiser + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace diff --git a/clusters/l3-sqnc/optimiser/nginx/values.yaml b/clusters/l3-sqnc/optimiser/nginx/values.yaml index 3d8126f4..f9411880 100644 --- a/clusters/l3-sqnc/optimiser/nginx/values.yaml +++ b/clusters/l3-sqnc/optimiser/nginx/values.yaml @@ -21,3 +21,8 @@ tolerations: containerPorts: http: 80 https: 443 +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: optimiser diff --git a/clusters/l3-sqnc/optimiser/yellow-node/values.yaml b/clusters/l3-sqnc/optimiser/yellow-node/values.yaml index e1799b5c..d07033cd 100644 --- a/clusters/l3-sqnc/optimiser/yellow-node/values.yaml +++ b/clusters/l3-sqnc/optimiser/yellow-node/values.yaml @@ -18,4 +18,7 @@ node: - "--prometheus-external" - "--state-pruning=100000" - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'" + serviceMonitor: + enabled: true + namespace: optimiser storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/order/blue-node/values.yaml b/clusters/l3-sqnc/order/blue-node/values.yaml index a293ee48..63b147ce 100644 --- a/clusters/l3-sqnc/order/blue-node/values.yaml +++ b/clusters/l3-sqnc/order/blue-node/values.yaml @@ -19,4 +19,7 @@ node: - "--prometheus-external" - "--state-pruning=100000" - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'" + serviceMonitor: + enabled: true + namespace: order storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/order/cyan-node/values.yaml b/clusters/l3-sqnc/order/cyan-node/values.yaml index a0829fb8..907e6f62 100644 --- a/clusters/l3-sqnc/order/cyan-node/values.yaml +++ b/clusters/l3-sqnc/order/cyan-node/values.yaml @@ -18,4 +18,7 @@ node: - "--unsafe-rpc-external" - "--prometheus-external" - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'" + serviceMonitor: + enabled: true + namespace: order storageClass: managed-csi-premium diff --git a/clusters/l3-sqnc/order/identity-service/values.yaml b/clusters/l3-sqnc/order/identity-service/values.yaml index 2b1686fb..57a9390e 100644 --- a/clusters/l3-sqnc/order/identity-service/values.yaml +++ b/clusters/l3-sqnc/order/identity-service/values.yaml @@ -16,3 +16,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 5Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: order + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace diff --git a/clusters/l3-sqnc/order/matchmaker-api/values.yaml b/clusters/l3-sqnc/order/matchmaker-api/values.yaml index 7cca2535..b24e2194 100644 --- a/clusters/l3-sqnc/order/matchmaker-api/values.yaml +++ b/clusters/l3-sqnc/order/matchmaker-api/values.yaml @@ -21,3 +21,12 @@ postgresql: persistence: storageClass: managed-csi-premium size: 15Gi + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: order + relabelings: + - action: replace + sourceLabels: [namespace] + targetLabel: kubernetes_namespace diff --git a/clusters/l3-sqnc/order/nginx/values.yaml b/clusters/l3-sqnc/order/nginx/values.yaml index 5a8beb04..2ae6bc6d 100644 --- a/clusters/l3-sqnc/order/nginx/values.yaml +++ b/clusters/l3-sqnc/order/nginx/values.yaml @@ -21,3 +21,8 @@ tolerations: containerPorts: http: 80 https: 443 +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: order diff --git a/clusters/l3-sqnc/podmonitors/kustomization.yaml b/clusters/l3-sqnc/podmonitors/kustomization.yaml new file mode 100644 index 00000000..dd2d95f8 --- /dev/null +++ b/clusters/l3-sqnc/podmonitors/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - podmonitor.yaml diff --git a/clusters/l3-sqnc/podmonitors/podmonitor.yaml b/clusters/l3-sqnc/podmonitors/podmonitor.yaml new file mode 100644 index 00000000..d338f1f3 --- /dev/null +++ b/clusters/l3-sqnc/podmonitors/podmonitor.yaml @@ -0,0 +1,29 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: flux-system + labels: + app.kubernetes.io/part-of: flux + app.kubernetes.io/component: monitoring +spec: + namespaceSelector: + matchNames: + - flux-system + selector: + matchExpressions: + - key: app + operator: In + values: + - helm-controller + - source-controller + - kustomize-controller + - notification-controller + - image-automation-controller + - image-reflector-controller + podMetricsEndpoints: + - port: http-prom + relabelings: + # https://github.com/prometheus-operator/prometheus-operator/issues/4816 + - sourceLabels: [__meta_kubernetes_pod_phase] + action: keep + regex: Running diff --git a/clusters/l3-sqnc/secrets/monitoring-secrets.yaml b/clusters/l3-sqnc/secrets/monitoring-secrets.yaml new file mode 100644 index 00000000..b65b7547 --- /dev/null +++ b/clusters/l3-sqnc/secrets/monitoring-secrets.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +data: + grafana_password: ENC[AES256_GCM,data:tvyuwyN5Pk9Udvn3ZylqJQ==,iv:CTdsAgJz404JwPLQlNyGcb7Ewm44ID4gXBeJBp/LrDA=,tag:kGk54DyeieGQMr8U8ti/jA==,type:str] +kind: Secret +metadata: + creationTimestamp: null + name: monitoring-secrets + namespace: monitoring +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-04-23T17:44:37Z" + mac: ENC[AES256_GCM,data:XrqGHxB89Ndhxr4WhyjDfR2qsMunIyFnG9NBUnEHvxYJlIbgHn6j+FNMIK29vduws0urIGUs3ZLlPvBPuN5SpqwRghmAHUEAXgR1m2S660/hZM9tUwL7pH6CoMnjeIN9k7aYXgm6HyE+UriPpQsy9uPTVL7bWNgEd93Zza9ylE4=,iv:LLWqZyKH7wiGoLFyy0LQ8zRe8lx43NSc0/I96oPT56I=,tag:LNC1ItqrqRcR6XYHXQgRkg==,type:str] + pgp: + - created_at: "2024-04-23T17:44:37Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DWK9rONmXZEsSAQdAKfIUZPH6B2yosul91VJaK4cUaMFcHkesBOFRXX4nclkw + G2CcCEomJvHe1E18hv28P5OkoY+/kqC9Xesnw7Nr/DzWdUJiMUX735VzgHrDt+mJ + 1GYBCQIQpz43qrPszR+iayhkWdqIjL+ZonsamstJHSda+hkLD2y3+T1T07MwwT+F + D9QALwTic2IqbCzX6Re2qFggvKRNs0F7zkcXu1CAW0Vwl75ykV8Rx5LE/hqAVcEV + DUkl6r1oeBA= + =tvjl + -----END PGP MESSAGE----- + fp: BC3A767578FF432C7414DE6A73A07B99E207F7AF + encrypted_regex: ^(data|stringData)$ + version: 3.8.1