From d65b1f2750f5e94caf4bf338dc335d95b4b378c4 Mon Sep 17 00:00:00 2001 From: raaizik <132667934+raaizik@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:57:48 +0300 Subject: [PATCH] Sends V*ReplicationClass to client Sends two V*RCs (one for image flattening) for RDR Signed-off-by: raaizik <132667934+raaizik@users.noreply.github.com> --- services/provider/server/server.go | 89 ++++++++++++++++++++- services/provider/server/server_test.go | 100 +++++++++++++++++++++++- 2 files changed, 183 insertions(+), 6 deletions(-) diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 9f481f0ce5..12cc2732b7 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -656,9 +656,24 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S return nil, status.Error(codes.Internal, msg) } } + + // fetch storage cluster peer to indicate whether replication is enabled + scp := &ocsv1.StorageClusterPeerList{} + if err = s.client.List(ctx, scp, client.InNamespace(s.namespace)); err != nil { + return nil, status.Errorf(codes.Internal, "failed to get StorageClusterPeerList. %v", err) + } + replicationEnabled := len(scp.Items) > 1 + var replicationID string + if replicationEnabled { + replicationID = util.CalculateMD5Hash(req.StorageClaimName) + } + var extR []*pb.ExternalResource storageRequestHash := getStorageRequestHash(req.StorageConsumerUUID, req.StorageClaimName) + // SID for RamenDR + storageID := storageRequestHash + for _, cephRes := range storageRequest.Status.CephResources { switch cephRes.Kind { case "CephClient": @@ -690,13 +705,12 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S }) case "CephBlockPoolRadosNamespace": - rns := &rookCephv1.CephBlockPoolRadosNamespace{} err = s.client.Get(ctx, types.NamespacedName{Name: cephRes.Name, Namespace: s.namespace}, rns) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get %s CephBlockPoolRadosNamespace. %v", cephRes.Name, err) } - + mirroringEnabled := len(rns.Spec.Mirroring.Mode) > 0 provisionerSecretName := storageClaimCephCsiSecretName("provisioner", storageRequestHash) nodeSecretName := storageClaimCephCsiSecretName("node", storageRequestHash) rbdStorageClassData := map[string]string{ @@ -733,6 +747,50 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": provisionerSecretName, })}, + &pb.ExternalResource{ + Name: "ceph-rbd", + Kind: "VolumeReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: getExternalResourceLabels("VolumeReplicationClass", mirroringEnabled, false, replicationID, storageID), + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + &pb.ExternalResource{ + Name: "ceph-rbd-flatten", + Kind: "VolumeReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: getExternalResourceLabels("VolumeReplicationClass", mirroringEnabled, true, replicationID, storageID), + Annotations: map[string]string{}, + }, + &pb.ExternalResource{ + Name: "ceph-rbd", + Kind: "VolumeGroupReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: getExternalResourceLabels("VolumeGroupReplicationClass", mirroringEnabled, false, replicationID, storageID), + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + &pb.ExternalResource{ + Name: "ceph-rbd-flatten", + Kind: "VolumeGroupReplicationClass", + Data: mustMarshal(map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + }), + Labels: getExternalResourceLabels("VolumeGroupReplicationClass", mirroringEnabled, true, replicationID, storageID), + Annotations: map[string]string{}, + }, &pb.ExternalResource{ Kind: "ClientProfile", Name: "ceph-rbd", @@ -817,6 +875,33 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S } +func getExternalResourceLabels(kind string, isMirroringReplicationEnabled bool, isFlattenMode bool, + replicationID string, storageID string) map[string]string { + labels := make(map[string]string) + switch kind { + case "VolumeReplicationClass", "VolumeGroupReplicationClass": + if isMirroringReplicationEnabled { + labels["ramendr.openshift.io/replicationid"] = replicationID + if isFlattenMode { + labels["replication.storage.openshift.io/flatten-mode"] = "force" + } + labels["ramendr.openshift.io/storageID"] = storageID + } + case "VolumeSnapshotClass", "VolumeGroupSnapshotClass": + if isMirroringReplicationEnabled { + labels["ramendr.openshift.io/storageID"] = storageID + } + case "StorageClass": + if isMirroringReplicationEnabled { + labels["ramendr.openshift.io/storageID"] = storageID + } + + default: + panic(fmt.Sprintf("unknown storage class kind %q", kind)) + } + return labels +} + // ReportStatus rpc call to check if a consumer can reach to the provider. func (s *OCSProviderServer) ReportStatus(ctx context.Context, req *pb.ReportStatusRequest) (*pb.ReportStatusResponse, error) { // Update the status in storageConsumer CR diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index 36d0628ec7..97c3e0c8fa 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -29,9 +29,11 @@ import ( ) type externalResource struct { - Kind string `json:"kind"` - Data any `json:"data"` - Name string `json:"name"` + Kind string `json:"kind"` + Data any `json:"data"` + Name string `json:"name"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` } var serverNamespace = "openshift-storage" @@ -310,7 +312,10 @@ func TestGetExternalResources(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + } + assert.Equal(t, extResource.GetLabels(), mockResoruce.Labels) + assert.Equal(t, extResource.GetAnnotations(), mockResoruce.Annotations) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -348,7 +353,8 @@ func TestGetExternalResources(t *testing.T) { assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) } - + assert.Equal(t, extResource.GetLabels(), mockResoruce.Labels) + assert.Equal(t, extResource.GetAnnotations(), mockResoruce.Annotations) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -641,6 +647,65 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", }, }, + "ceph-rbd-volumereplicationclass": { + Name: "ceph-rbd", + Kind: "VolumeReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + + "ramendr.openshift.io/replicationid": "", + "ramendr.openshift.io/storageID": "8d40b6be71600457b5dec219d2ce2d4c", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + "ceph-rbd-flatten-volumereplicationclass": { + Name: "ceph-rbd-flatten", + Kind: "VolumeReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + "ramendr.openshift.io/replicationid": "", + "ramendr.openshift.io/storageID": "8d40b6be71600457b5dec219d2ce2d4c", + }, + Annotations: map[string]string{}, + }, + "ceph-rbd-volumegroupreplicationclass": { + Name: "ceph-rbd", + Kind: "VolumeGroupReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + "ramendr.openshift.io/replicationid": "", + "ramendr.openshift.io/storageID": "8d40b6be71600457b5dec219d2ce2d4c", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + }, + "ceph-rbd-flatten-volumegroupreplicationclass": { + Name: "ceph-rbd-flatten", + Kind: "VolumeGroupReplicationClass", + Data: map[string]string{ + "replication.storage.openshift.io/group-replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", + "mirroringMode": "snapshot", + }, + Labels: map[string]string{ + "replication.storage.openshift.io/flatten-mode": "force", + "ramendr.openshift.io/replicationid": "", + "ramendr.openshift.io/storageID": "8d40b6be71600457b5dec219d2ce2d4c", + }, + Annotations: map[string]string{}, + }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", @@ -1003,10 +1068,25 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { }, Spec: rookCephv1.CephBlockPoolRadosNamespaceSpec{ BlockPoolName: "cephblockpool", + Mirroring: &rookCephv1.RadosNamespaceMirroring{ + Mode: "pool", + }, }, } assert.NoError(t, client.Create(ctx, radosNamespace)) + cephBlockPool := &rookCephv1.CephBlockPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cephblockpool", + Namespace: server.namespace, + }, + Spec: rookCephv1.NamedBlockPoolSpec{ + PoolSpec: rookCephv1.PoolSpec{ + Mirroring: rookCephv1.MirroringSpec{Enabled: false}, + }}, + } + assert.NoError(t, client.Create(ctx, cephBlockPool)) + // get the storage class request config for block pool req := pb.StorageClaimConfigRequest{ StorageConsumerUUID: string(consumerResource.UID), @@ -1025,6 +1105,10 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { name = fmt.Sprintf("%s-storageclass", name) } else if extResource.Kind == "VolumeGroupSnapshotClass" { name = fmt.Sprintf("%s-volumegroupsnapshotclass", name) + } else if extResource.Kind == "VolumeReplicationClass" { + name = fmt.Sprintf("%s-volumereplicationclass", name) + } else if extResource.Kind == "VolumeGroupReplicationClass" { + name = fmt.Sprintf("%s-volumegroupreplicationclass", name) } else if extResource.Kind == "ClientProfile" { name = fmt.Sprintf("%s-clientprofile", name) } @@ -1034,6 +1118,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.GetLabels(), mockResoruce.Labels) + assert.Equal(t, extResource.GetAnnotations(), mockResoruce.Annotations) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -1056,6 +1142,10 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { name = fmt.Sprintf("%s-storageclass", name) } else if extResource.Kind == "VolumeGroupSnapshotClass" { name = fmt.Sprintf("%s-volumegroupsnapshotclass", name) + } else if extResource.Kind == "VolumeReplicationClass" { + name = fmt.Sprintf("%s-volumereplicationclass", name) + } else if extResource.Kind == "VolumeGroupReplicationClass" { + name = fmt.Sprintf("%s-volumegroupreplicationclass", name) } else if extResource.Kind == "ClientProfile" { name = fmt.Sprintf("%s-clientprofile", name) } @@ -1064,6 +1154,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.GetLabels(), mockResoruce.Labels) + assert.Equal(t, extResource.GetAnnotations(), mockResoruce.Annotations) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) }