L3 231/add production monitoring

clusters/kind-cluster/base/flux-system/gotk-sync.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   interval: 1m0s
   ref:
-    branch: main
+    branch: test/rename-podmonitors
   url: https://github.com/digicatapult/sqnc-flux-infra.git
 
 ---

clusters/kind-cluster/monitoring/kube-prometheus-stack/kustomization.yaml → clusters/kind-cluster/monitoring/kustomization.yaml

@@ -16,6 +16,15 @@ configMapGenerator:
       labels:
         app.kubernetes.io/part-of: flux
         app.kubernetes.io/component: monitoring
+  - name: flux-grafana-dashboards
+    files:
+      - dashboards/control-plane.json
+      - dashboards/cluster.json
+    options:
+      labels:
+        grafana_dashboard: "1"
+        app.kubernetes.io/part-of: flux
+        app.kubernetes.io/component: monitoring
 configurations:
   - kustomize-config.yaml
 generatorOptions:

clusters/kind-cluster/monitoring/kube-prometheus-stack/monitoring-podmonitor-sync.yaml → clusters/kind-cluster/monitoring/monitoring-podmonitor-sync.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: monitoring
 spec:
   interval: 1m
-  path: ./clusters/kind-cluster/monitoring-configs
+  path: ./clusters/kind-cluster/podmonitors
   prune: true
   dependsOn:
     - name: monitoring-sync

clusters/l3-sqnc/base/app-sync.yaml

@@ -1,13 +1,54 @@
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
+metadata:
+  name: monitoring-sync
+  namespace: monitoring
+spec:
+  interval: 1m
+  path: ./clusters/l3-sqnc/monitoring
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  healthChecks:
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: podmonitors.monitoring.coreos.com
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: servicemonitors.monitoring.coreos.com
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nginx-sync
+  namespace: infra
+spec:
+  interval: 1m
+  path: ./clusters/l3-sqnc/nginx
+  prune: true
+  dependsOn:
+    - name: monitoring-sync
+      namespace: monitoring
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
 metadata:
   name: capacity-sync
   namespace: capacity
 spec:
   interval: 1m
   path: ./clusters/l3-sqnc/capacity
   prune: true
+  dependsOn:
+    - name: monitoring-sync
+      namespace: monitoring
   sourceRef:
     kind: GitRepository
     name: flux-system
@@ -22,6 +63,9 @@ spec:
   interval: 1m
   path: ./clusters/l3-sqnc/optimiser
   prune: true
+  dependsOn:
+    - name: monitoring-sync
+      namespace: monitoring
   sourceRef:
     kind: GitRepository
     name: flux-system
@@ -36,6 +80,9 @@ spec:
   interval: 1m
   path: ./clusters/l3-sqnc/order
   prune: true
+  dependsOn:
+    - name: monitoring-sync
+      namespace: monitoring
   sourceRef:
     kind: GitRepository
     name: flux-system

clusters/l3-sqnc/base/namespaces.yaml

@@ -1,6 +1,22 @@
 ---
 apiVersion: v1
 kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/instance: sqnc
+    app.kubernetes.io/version: latest
+  name: monitoring
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/instance: sqnc
+    app.kubernetes.io/version: latest
+  name: infra
+---
+apiVersion: v1
+kind: Namespace
 metadata:
   labels:
     app.kubernetes.io/instance: sqnc

clusters/l3-sqnc/capacity/identity-service/values.yaml

@@ -16,3 +16,12 @@ postgresql:
     persistence:
       storageClass: managed-csi-premium
       size: 5Gi
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      namespace: capacity
+      relabelings:
+        - action: replace
+          sourceLabels: [namespace]
+          targetLabel: kubernetes_namespace

clusters/l3-sqnc/capacity/magenta-node/values.yaml

@@ -17,4 +17,7 @@ node:
     - "--unsafe-rpc-external"
     - "--prometheus-external"
     - "--state-pruning=100000"
+  serviceMonitor:
+    enabled: true
+    namespace: capacity
 storageClass: managed-csi-premium

clusters/l3-sqnc/capacity/matchmaker-api/values.yaml

@@ -21,3 +21,12 @@ postgresql:
     persistence:
       storageClass: managed-csi-premium
       size: 15Gi
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      namespace: capacity
+      relabelings:
+        - action: replace
+          sourceLabels: [namespace]
+          targetLabel: kubernetes_namespace

clusters/l3-sqnc/capacity/nginx/values.yaml

@@ -21,3 +21,8 @@ tolerations:
 containerPorts:
   http: 80
   https: 443
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    namespace: capacity

clusters/l3-sqnc/capacity/red-node/values.yaml

@@ -17,4 +17,7 @@ node:
     - "--unsafe-rpc-external"
     - "--prometheus-external"
     - "--bootnodes '/dns4/magenta-sqnc-node-0-rc-p2p.capacity.svc.cluster.local/tcp/30333/p2p/12D3KooWDvUB5uBh6msz4D1Te3vkfiod9FyMwPvHM56W46c1osKk'"
+  serviceMonitor:
+    enabled: true
+    namespace: capacity
 storageClass: managed-csi-premium