feat(app): experimental azure blob storage support (#1374)

SwissDataScienceCenter · Jan 10, 2023 · 4cbcb14 · 4cbcb14
1 parent 57b70eb
commit 4cbcb14
Show file tree

Hide file tree

Showing 25 changed files with 834 additions and 373 deletions.
diff --git a/helm-chart/renku-notebooks/templates/statefulset.yaml b/helm-chart/renku-notebooks/templates/statefulset.yaml
@@ -154,8 +154,14 @@ spec:
             {{- include "certificates.env.python" . | nindent 12 }}
             - name: NB_SESSIONS__ENFORCE_CPU_LIMITS
               value: {{ .Values.enforceCPULimits | quote }}
-            - name: NB_S3_MOUNTS_ENABLED
+            - name: NB_CLOUD_STORAGE__S3__ENABLED
               value: {{ .Values.cloudstorage.s3.enabled | quote }}
+            - name: NB_CLOUD_STORAGE__S3__READ_ONLY
+              value: {{ .Values.cloudstorage.s3.readOnly | quote }}
+            - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED
+              value: {{ .Values.cloudstorage.azureBlob.enabled | quote }}
+            - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY
+              value: {{ .Values.cloudstorage.azureBlob.readOnly | quote }}
             - name: NB_SESSIONS__TERMINATION_GRACE_PERIOD_SECONDS
               value: {{ .Values.sessionAutosave.terminationGracePeriodSeconds | quote }}
             - name: NB_SESSIONS__AUTOSAVE_MINIMUM_LFS_FILE_SIZE_BYTES

diff --git a/helm-chart/renku-notebooks/templates/test.yaml b/helm-chart/renku-notebooks/templates/test.yaml
@@ -83,8 +83,14 @@ spec:
         {{- end }}
         - name: NB_SESSIONS__ENFORCE_CPU_LIMITS
           value: {{ $.Values.enforceCPULimits | quote }}
-        - name: NB_S3_MOUNTS_ENABLED
+        - name: NB_CLOUD_STORAGE__S3__ENABLED
           value: {{ $.Values.cloudstorage.s3.enabled | quote }}
+        - name: NB_CLOUD_STORAGE__S3__READ_ONLY
+          value: {{ $.Values.cloudstorage.s3.readOnly | quote }}
+        - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED
+          value: {{ $.Values.cloudstorage.azureBlob.enabled | quote }}
+        - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY
+          value: {{ $.Values.cloudstorage.azureBlob.readOnly | quote }}
         - name: NB_SESSIONS__TERMINATION_GRACE_PERIOD_SECONDS
           value: {{ $.Values.sessionAutosave.terminationGracePeriodSeconds | quote }}
         - name: NB_SESSIONS__AUTOSAVE_MINIMUM_LFS_FILE_SIZE_BYTES

diff --git a/helm-chart/renku-notebooks/values.yaml b/helm-chart/renku-notebooks/values.yaml
@@ -57,6 +57,13 @@ cloudstorage:
   s3:
     enabled: false
     installDatashim: false
+    readOnly: true
+  ## Azure blob storage mounting support is experimental. The CSI
+  ## storage driver for mounting azure blob storage has to be already installed
+  ## in the cluster if enabled.
+  azureBlob:
+    enabled: false
+    readOnly: true
 
 # configuration for user session persistent volumes
 userSessionPersistentVolumes:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -26,6 +26,7 @@ marshmallow = "*"
 apispec = {extras = ["marshmallow"], version = "*"}
 importlib-metadata = "*"
 dataconf = "^2.0.0"
+azure-storage-blob = "^12.14.1"
 
 [tool.poetry.dev-dependencies]
 pylint = "*"

diff --git a/renku_notebooks/api/amalthea_patches/cloudstorage.py b/renku_notebooks/api/amalthea_patches/cloudstorage.py
@@ -1,16 +1,18 @@
-from typing import TYPE_CHECKING
+from typing import Any, Dict, List, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from renku_notebooks.api.classes.server import UserServer
+    from renku_notebooks.api.classes.cloud_storage import ICloudStorageRequest
 
 
-def main(server: "UserServer"):
-    s3mount_patches = []
-    for i, s3mount in enumerate(server.cloudstorage):
+def main(server: "UserServer") -> List[Dict[str, Any]]:
+    cloud_storage_patches: List[Dict[str, Any]] = []
+    cloud_storage_request: "ICloudStorageRequest"
+    for i, cloud_storage_request in enumerate(server.cloudstorage):
         s3mount_name = f"{server.server_name}-ds-{i}"
-        s3mount_patches.append(
-            s3mount.get_manifest_patches(
+        cloud_storage_patches.append(
+            cloud_storage_request.get_manifest_patch(
                 s3mount_name, server._k8s_client.preferred_namespace
             )
         )
-    return s3mount_patches
+    return cloud_storage_patches
diff --git a/renku_notebooks/api/amalthea_patches/init_containers.py b/renku_notebooks/api/amalthea_patches/init_containers.py
@@ -72,8 +72,8 @@ def git_clone(server: "UserServer"):
         },
         {
             "name": "GIT_CLONE_S3_MOUNT",
-            "value": server.cloudstorage[0].mount_folder
-            if config.s3_mounts_enabled and server.cloudstorage
+            "value": config.cloud_storage.mount_folder
+            if config.cloud_storage.any_enabled and server.cloudstorage
             else "",
         },
     ]

diff --git a/renku_notebooks/api/classes/cloud_storage/__init__.py b/renku_notebooks/api/classes/cloud_storage/__init__.py
@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod, abstractproperty
+from typing import Any, Dict
+
+
+class ICloudStorageRequest(ABC):
+    @abstractproperty
+    def exists(self) -> bool:
+        pass
+
+    @abstractproperty
+    def mount_folder(self) -> str:
+        pass
+
+    @abstractproperty
+    def bucket(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_manifest_patch(
+        self,
+        base_name: str,
+        namespace: str,
+        labels: Dict[str, str] = {},
+        annotations: Dict[str, str] = {},
+    ) -> Dict[str, Any]:
+        pass