ceph-test

kubernetes/apps/storage/longhorn-system/longhorn-system/base/release.yaml

@@ -31,7 +31,7 @@ spec:
       taintToleration: "node-role.kubernetes.io/master=true:NoSchedule"
     global:
       nodeSelector:
-        storage: "longhorn"
+        storage/longhorn: "true"
       tolerations:
         - key: node-role.kubernetes.io/master
           operator: "Equal"

kubernetes/apps/storage/longhorn-system/longhorn-system/ks.yaml

@@ -6,8 +6,8 @@ metadata:
   name: longhorn-system-base
   namespace: flux-system
 spec:
-  # dependsOn:
-  #   - name: snapshot-controller
+  dependsOn:
+    - name: snapshot-controller
   path: ./kubernetes/apps/storage/longhorn-system/longhorn-system/base
   prune: true
   wait: true

kubernetes/apps/storage/rook-ceph/kustomization.yaml

@@ -2,5 +2,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
-  # - ./rook-ceph/ks.yaml
+  # - namespace.yaml
+  - ./rook-ceph/ks.yaml

kubernetes/apps/storage/rook-ceph/rook-ceph/app/dashboard-secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: rook-ceph-dashboard-password
+type: Opaque
+data:
+    password: "asd"  # asd

kubernetes/apps/storage/rook-ceph/rook-ceph/app/http-route.yaml

@@ -0,0 +1,124 @@
+# kind: Deployment
+# apiVersion: apps/v1
+# metadata:
+#   name: whoami2
+#   namespace: whoami
+# spec:
+#   replicas: 2
+#   selector:
+#     matchLabels:
+#       app: whoami2
+#   template:
+#     metadata:
+#       labels:
+#         app: whoami2
+#     spec:
+#       containers:
+#         - name: whoami2
+#           image: traefik/whoami
+# ---
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   name: whoami2
+#   namespace: whoami
+#   labels:
+#     app: whoami2
+#     service-type: ingress
+# spec:
+#   type: LoadBalancer
+#   ports:
+#     - port: 80
+#       name: whoami2
+#   selector:
+#     app: whoami2
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: rook-ceph-dashboard-httproute
+  namespace: whoami
+spec:
+  parentRefs:
+    - name: traefik-gateway
+      namespace: network
+  hostnames:
+    - rook-ceph.icylair.com
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: rook-ceph-mgr-dashboard  
+          namespace: storage
+          port: 7000
+---
+# Allow HTTPRoute in whoami namespace to reference Gateway and Service in network namespace
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: allow-rook-routes
+  namespace: network  # ReferenceGrant must be in the namespace being referenced
+spec:
+  from:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      namespace: storage  # Source namespace containing the HTTPRoute
+  to:
+    # Allow references to Gateway resources
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: traefik-gateway  # Specific Gateway name that can be referenced
+    # Allow references to Service resources
+    - group: ""  # Core API group
+      kind: Service
+      name: rook-ceph-mgr-dashboard # Specific Service name that can be referenced
+# ---
+# apiVersion: gateway.networking.k8s.io/v1
+# kind: HTTPRoute
+# metadata:
+#   name: whoami-tls-httproute
+#   namespace: network
+# spec:
+#   parentRefs:
+#     - name: wildcard-tls-gateway
+#       namespace: network
+#   hostnames:
+#     - whoami.icylair.com
+#   rules:
+#     - matches:
+#         - path:
+#             type: PathPrefix
+#             value: /
+#       backendRefs:
+#         - name: whoami
+#           namespace: network
+#           port: 80
+# ---
+# apiVersion: gateway.networking.k8s.io/v1
+# kind: Gateway
+# metadata:
+#   name: wildcard-tls-gateway
+#   namespace: network
+# spec:
+#   gatewayClassName: traefik
+#   listeners:
+#   # - name: foo-https
+#   #   protocol: HTTPS
+#   #   port: 8443
+#   #   hostname: test.icylair.com
+#   #   tls:
+#   #     certificateRefs:
+#   #     - kind: Secret
+#   #       group: ""
+#   #       name: icylair-com-all-prod
+#   - name: wildcard-https
+#     protocol: HTTPS
+#     port: 8443
+#     hostname: "*.icylair.com"
+#     tls:
+#       certificateRefs:
+#       - kind: Secret
+#         # group: ""
+#         name: icylair-com-all-prod 

kubernetes/apps/storage/rook-ceph/rook-ceph/app/kustomization.yaml

@@ -6,3 +6,5 @@ namespace: storage
 resources:
   - ./release-operator.yaml
   - ./release-cluster.yaml
+  # - ./http-route.yaml
+  # - ./dashboard-secret.yaml

kubernetes/apps/storage/rook-ceph/rook-ceph/app/release-cluster.yaml

@@ -10,7 +10,7 @@ spec:
   chart:
     spec:
       chart: rook-ceph-cluster
-      version: v1.16.1
+      version: v1.16.2
       sourceRef:
         kind: HelmRepository
         name: rook-ceph
@@ -26,21 +26,25 @@ spec:
   uninstall:
     keepHistory: false
   values:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 16Mi
     operatorNamespace: &namespace storage
     monitoring:
       enabled: false
       createPrometheusRules: false
     ingress:
       dashboard:
-        # annotations:
-        #   traefik.ingress.kubernetes.io/router.middlewares: network-forward-auth@kubernetescrd
+        annotations:
+          traefik.ingress.kubernetes.io/router.middlewares: network-forward-auth@kubernetescrd
         ingressClassName: traefik-external
         host:
-          name: rook.cloud.icylair.com
+          name: &host rook-ceph.cloud.icylair.com
           path: /
         tls:
         - hosts:
-            - rook.cloud.icylair.com
+            - *host
           secretName: icylair-com-all-prod
     # toolbox:
     #   enabled: true
@@ -60,6 +64,60 @@ spec:
             enabled: true
       mon:
         count: 1
+
+      resources:
+        mgr:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: 2Gi
+        mon:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: 1Gi
+        osd:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: 6Gi
+        mgr-sidecar:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: 256Mi
+        prepareosd:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+        crashcollector:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: "60Mi"
+        logcollector:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: "1Gi"
+        cleanup:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: "1Gi"
+        exporter:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            memory: "128Mi"
       dashboard:
         enabled: true
         urlPrefix: /
@@ -75,7 +133,18 @@ spec:
     #     deviceFilter: nvme0n1
     #     config:
     #       osdsPerDevice: "1"
-    #   placement:
+    cleanupPolicy:
+      confirmation: "yes-really-destroy-data"
+    placement:
+      # all:
+      #   nodeAffinity:
+      #     requiredDuringSchedulingIgnoredDuringExecution:
+      #       nodeSelectorTerms:
+      #         - matchExpressions:
+      #           - key: storage/ceph
+      #             operator: In
+      #             values:
+      #             - "true"
     #     mgr: &placement
     #       nodeAffinity:
     #         requiredDuringSchedulingIgnoredDuringExecution:
@@ -84,31 +153,7 @@ spec:
     #                 - key: node-role.kubernetes.io/control-plane
     #                   operator: Exists
     #     mon: *placement
-      resources:
-        mgr:
-          requests:
-            cpu: 50m
-            memory: 32Mi
-          limits:
-            memory: 2Gi
-        mon:
-          requests:
-            cpu: 50m
-            memory: 32Mi
-          limits:
-            memory: 1Gi
-        osd:
-          requests:
-            cpu: 50m
-            memory: 32Mi
-          limits:
-            memory: 6Gi
-        mgr-sidecar:
-          requests:
-            cpu: 50m
-            memory: 32Mi
-          limits:
-            memory: 256Mi
+      removeOSDsIfOutAndSafeToRemove: true
     cephBlockPools:
       - name: ceph-blockpool
         spec:

kubernetes/apps/storage/rook-ceph/rook-ceph/app/release-operator.yaml

@@ -10,7 +10,7 @@ spec:
   chart:
     spec:
       chart: rook-ceph
-      version: v1.16.1
+      version: v1.16.2
       sourceRef:
         kind: HelmRepository
         name: rook-ceph
@@ -28,41 +28,40 @@ spec:
   values:
     csi:
       provisionerReplicas: 1
-    #   cephFSKernelMountOptions: ms_mode=prefer-crc
+      cephFSKernelMountOptions: ms_mode=prefer-crc
       enableLiveness: true
       serviceMonitor:
         enabled: true
       nfs:
-        enabled: false
+        enabled: true
       csiRBDPluginVolume:
         - name: lib-modules
           hostPath:
             path: /run/booted-system/kernel-modules/lib/modules/
         - name: host-nix
           hostPath:
             path: /nix
-      # -- The volume mounts of the CephCSI RBD plugin DaemonSet
       csiRBDPluginVolumeMount:
         - name: host-nix
           mountPath: /nix
           readOnly: true
-      # -- The volume of the CephCSI CephFS plugin DaemonSet
       csiCephFSPluginVolume:
         - name: lib-modules
           hostPath:
             path: /run/booted-system/kernel-modules/lib/modules/
         - name: host-nix
           hostPath:
             path: /nix
-      # -- The volume mounts of the CephCSI CephFS plugin DaemonSet
       csiCephFSPluginVolumeMount:
         - name: host-nix
           mountPath: /nix
           readOnly: true
     monitoring:
       enabled: false
+    nodeSelector: 
+      storage/ceph: "true"
     resources:
       requests:
-        memory: 128Mi # unchangable
-        cpu: 1m # unchangable
+        memory: 128Mi
+        cpu: 1m
       limits: {}