init

Signed-off-by: yaroslavborbat <[email protected]>

images/cdi-artifact/patches/012-add-caps-for-deckhouse-provisioners.patch

@@ -0,0 +1,26 @@
+diff --git a/pkg/storagecapabilities/storagecapabilities.go b/pkg/storagecapabilities/storagecapabilities.go
+index 24bbac64b..fccdd0c08 100644
+--- a/pkg/storagecapabilities/storagecapabilities.go
++++ b/pkg/storagecapabilities/storagecapabilities.go
+@@ -8,9 +8,10 @@ import (
+
+ 	v1 "k8s.io/api/core/v1"
+ 	storagev1 "k8s.io/api/storage/v1"
+-	"kubevirt.io/containerized-data-importer/pkg/util"
+ 	"sigs.k8s.io/controller-runtime/pkg/client"
+
++	"kubevirt.io/containerized-data-importer/pkg/util"
++
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
+ )
+
+@@ -102,6 +103,9 @@ var CapabilitiesByProvisionerKey = map[string][]StorageCapabilities{
+ 	"manila.csi.openstack.org": {{rwx, file}},
+ 	// ovirt csi
+ 	"csi.ovirt.org": createRWOBlockAndFilesystemCapabilities(),
++	// Deckhouse
++	"replicated.csi.storage.deckhouse.io": createLinstorCapabilities(),
++	"local.csi.storage.deckhouse.io":      createTopoLVMCapabilities(),
+ }
+
+ // SourceFormatsByProvisionerKey defines the advised data import cron source format

images/cdi-artifact/patches/013-support-converting-images-to-different-formats.patch

@@ -0,0 +1,217 @@
+diff --git a/pkg/common/common.go b/pkg/common/common.go
+index 7edc88040..81bd80690 100644
+--- a/pkg/common/common.go
++++ b/pkg/common/common.go
+@@ -287,6 +287,8 @@ const (
+
+ 	// CDIControllerLeaderElectionHelperName is the name of the configmap that is used as a helper for controller leader election
+ 	CDIControllerLeaderElectionHelperName = "cdi-controller-leader-election-helper"
++
++	ConvertImageFormatEnv = "CONVERT_IMAGE_FORMAT"
+ )
+
+ // ProxyPaths are all supported paths
+diff --git a/pkg/controller/common/util.go b/pkg/controller/common/util.go
+index 8cab9ab08..2b2c637dd 100644
+--- a/pkg/controller/common/util.go
++++ b/pkg/controller/common/util.go
+@@ -55,15 +55,16 @@ import (
+
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
+ 	cdiv1utils "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1/utils"
++	sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api"
++	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
++	"sigs.k8s.io/controller-runtime/pkg/client"
++	"sigs.k8s.io/controller-runtime/pkg/client/fake"
++
+ 	"kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned/scheme"
+ 	"kubevirt.io/containerized-data-importer/pkg/common"
+ 	featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates"
+ 	"kubevirt.io/containerized-data-importer/pkg/token"
+ 	"kubevirt.io/containerized-data-importer/pkg/util"
+-	sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api"
+-	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
+-	"sigs.k8s.io/controller-runtime/pkg/client"
+-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+ )
+
+ const (
+@@ -74,7 +75,8 @@ const (
+ 	ScratchVolName = "cdi-scratch-vol"
+
+ 	// AnnAPIGroup is the APIGroup for CDI
+-	AnnAPIGroup = "cdi.kubevirt.io"
++	AnnAPIGroup          = "cdi.kubevirt.io"
++	AnnDeckhouseApiGroup = "virt.deckhouse.io"
+ 	// AnnCreatedBy is a pod annotation indicating if the pod was created by the PVC
+ 	AnnCreatedBy = AnnAPIGroup + "/storage.createdByController"
+ 	// AnnPodPhase is a PVC annotation indicating the related pod progress (phase)
+@@ -241,6 +243,7 @@ const (
+ 	// be dynamically provisioned. Its value is the name of the selected node.
+ 	AnnSelectedNode = "volume.kubernetes.io/selected-node"
+
++	AnnUnpackFormat = AnnDeckhouseApiGroup + "/unpack.format"
+ 	// CloneUniqueID is used as a special label to be used when we search for the pod
+ 	CloneUniqueID = "cdi.kubevirt.io/storage.clone.cloneUniqeId"
+
+diff --git a/pkg/controller/datavolume/controller-base.go b/pkg/controller/datavolume/controller-base.go
+index a24523e80..9fc7a945b 100644
+--- a/pkg/controller/datavolume/controller-base.go
++++ b/pkg/controller/datavolume/controller-base.go
+@@ -48,6 +48,7 @@ import (
+ 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
++
+ 	"kubevirt.io/containerized-data-importer/pkg/common"
+ 	cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
+ 	featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates"
+@@ -709,6 +710,9 @@ func (r *ReconcilerBase) createPvcForDatavolume(datavolume *cdiv1.DataVolume, pv
+ 		return nil, err
+ 	}
+ 	util.SetRecommendedLabels(newPvc, r.installerLabels, "cdi-controller")
++	if ann := datavolume.GetAnnotations()[cc.AnnUnpackFormat]; ann != "" {
++		newPvc.Annotations[cc.AnnUnpackFormat] = ann
++	}
+ 	if err := r.client.Create(context.TODO(), newPvc); err != nil {
+ 		return nil, err
+ 	}
+diff --git a/pkg/controller/import-controller.go b/pkg/controller/import-controller.go
+index 011be11f6..8c6368206 100644
+--- a/pkg/controller/import-controller.go
++++ b/pkg/controller/import-controller.go
+@@ -29,12 +29,13 @@ import (
+ 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
++	sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api"
++
+ 	"kubevirt.io/containerized-data-importer/pkg/common"
+ 	cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
+ 	featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates"
+ 	"kubevirt.io/containerized-data-importer/pkg/util"
+ 	"kubevirt.io/containerized-data-importer/pkg/util/naming"
+-	sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api"
+ )
+
+ const (
+@@ -99,6 +100,7 @@ type importPodEnvVar struct {
+ 	certConfigMapProxy string
+ 	extraHeaders       []string
+ 	secretExtraHeaders []string
++	convertImageFormat string
+ }
+
+ type importerPodArgs struct {
+@@ -542,6 +544,7 @@ func (r *ImportReconciler) createImportEnvVar(pvc *corev1.PersistentVolumeClaim)
+ 	podEnvVar := &importPodEnvVar{}
+ 	podEnvVar.source = cc.GetSource(pvc)
+ 	podEnvVar.contentType = string(cc.GetPVCContentType(pvc))
++	podEnvVar.convertImageFormat = getValueFromAnnotation(pvc, cc.AnnUnpackFormat)
+
+ 	var err error
+ 	if podEnvVar.source != cc.SourceNone {
+@@ -1296,6 +1299,10 @@ func makeImportEnv(podEnvVar *importPodEnvVar, uid types.UID) []corev1.EnvVar {
+ 			Name:  common.Preallocation,
+ 			Value: strconv.FormatBool(podEnvVar.preallocation),
+ 		},
++		{
++			Name:  common.ConvertImageFormatEnv,
++			Value: podEnvVar.convertImageFormat,
++		},
+ 	}
+ 	if podEnvVar.secretName != "" && podEnvVar.source != cc.SourceGCS {
+ 		env = append(env, corev1.EnvVar{
+diff --git a/pkg/image/qemu.go b/pkg/image/qemu.go
+index 651fb5fc8..85bd85672 100644
+--- a/pkg/image/qemu.go
++++ b/pkg/image/qemu.go
+@@ -61,6 +61,7 @@ type ImgInfo struct {
+ // QEMUOperations defines the interface for executing qemu subprocesses
+ type QEMUOperations interface {
+ 	ConvertToRawStream(*url.URL, string, bool) error
++	ConvertToFormatStream(url *url.URL, format, dest string, preallocate bool) error
+ 	Resize(string, resource.Quantity, bool) error
+ 	Info(url *url.URL) (*ImgInfo, error)
+ 	Validate(*url.URL, int64) error
+@@ -114,6 +115,36 @@ func NewQEMUOperations() QEMUOperations {
+ 	return &qemuOperations{}
+ }
+
++func convertTo(format, src, dest string, preallocate bool) error {
++	switch format {
++	case "qcow2", "vmdk", "vdi", "iso", "raw":
++		// Do nothing.
++	default:
++		format = "raw"
++	}
++	args := []string{"convert", "-t", "writeback", "-p", "-O", format, src, dest}
++	var err error
++
++	if preallocate {
++		err = addPreallocation(args, convertPreallocationMethods, func(args []string) ([]byte, error) {
++			return qemuExecFunction(nil, reportProgress, "qemu-img", args...)
++		})
++	} else {
++		klog.V(1).Infof("Running qemu-img with args: %v", args)
++		_, err = qemuExecFunction(nil, reportProgress, "qemu-img", args...)
++	}
++	if err != nil {
++		os.Remove(dest)
++		errorMsg := fmt.Sprintf("could not convert image to %s", format)
++		if nbdkitLog, err := os.ReadFile(common.NbdkitLogPath); err == nil {
++			errorMsg += " " + string(nbdkitLog)
++		}
++		return errors.Wrap(err, errorMsg)
++	}
++
++	return nil
++}
++
+ func convertToRaw(src, dest string, preallocate bool) error {
+ 	args := []string{"convert", "-t", "writeback", "-p", "-O", "raw", src, dest}
+ 	var err error
+@@ -145,6 +176,13 @@ func (o *qemuOperations) ConvertToRawStream(url *url.URL, dest string, prealloca
+ 	return convertToRaw(url.String(), dest, preallocate)
+ }
+
++func (o *qemuOperations) ConvertToFormatStream(url *url.URL, format, dest string, preallocate bool) error {
++	if len(url.Scheme) > 0 && url.Scheme != "nbd+unix" {
++		return fmt.Errorf("not valid schema %s", url.Scheme)
++	}
++	return convertTo(format, url.String(), dest, preallocate)
++}
++
+ // convertQuantityToQemuSize translates a quantity string into a Qemu compatible string.
+ func convertQuantityToQemuSize(size resource.Quantity) string {
+ 	int64Size, asInt := size.AsInt64()
+@@ -246,6 +284,10 @@ func ConvertToRawStream(url *url.URL, dest string, preallocate bool) error {
+ 	return qemuIterface.ConvertToRawStream(url, dest, preallocate)
+ }
+
++func ConvertToFormatStream(url *url.URL, format, dest string, preallocate bool) error {
++	return qemuIterface.ConvertToFormatStream(url, format, dest, preallocate)
++}
++
+ // Validate does basic validation of a qemu image
+ func Validate(url *url.URL, availableSize int64) error {
+ 	return qemuIterface.Validate(url, availableSize)
+diff --git a/pkg/importer/data-processor.go b/pkg/importer/data-processor.go
+index ca7b2e853..dd983cd62 100644
+--- a/pkg/importer/data-processor.go
++++ b/pkg/importer/data-processor.go
+@@ -276,8 +276,12 @@ func (dp *DataProcessor) convert(url *url.URL) (ProcessingPhase, error) {
+ 	if err != nil {
+ 		return ProcessingPhaseError, err
+ 	}
+-	klog.V(3).Infoln("Converting to Raw")
+-	err = qemuOperations.ConvertToRawStream(url, dp.dataFile, dp.preallocation)
++	format := "raw"
++	if f := os.Getenv(common.ConvertImageFormatEnv); f != "" {
++		format = f
++	}
++	klog.V(3).Infoln("Converting to", "format", format)
++	err = qemuOperations.ConvertToFormatStream(url, format, dp.dataFile, dp.preallocation)
+ 	if err != nil {
+ 		return ProcessingPhaseError, errors.Wrap(err, "Conversion to Raw failed")
+ 	}

images/cdi-artifact/patches/README.md

@@ -34,3 +34,11 @@ Do not manage DataVolume CRD with cdi-operator. Module will install this CRD usi
 #### `011-change-storage-class-for-scratch-pvc.patch`
 
 Set the storage class name for the scratch pvc from the original pvc that will own the scratch pvc, or set it to an empty value if not available.
+
+#### `012-add-caps-for-deckhouse-provisioners.patch`
+
+Add capabilities for deckhouse provisioners to cdi StorageProfile.
+
+#### `013-support-converting-images-to-different-formats.patch`
+
+Support for converting images to different formats.

images/virtualization-artifact/pkg/common/pvc/util.go

@@ -26,40 +26,31 @@ import (
 // ForceDVReadWriteOnceVar enables accessMode ReadWriteOnce for testing in local environments.
 const ForceDVReadWriteOnceVar = "FORCE_DV_READ_WRITE_ONCE"
 
-// CreateSpecReadWriteMany returns pvc spec with accessMode ReadWriteMany and volumeMode Block.
-func CreateSpecReadWriteMany(storageClassName *string, size resource.Quantity) *corev1.PersistentVolumeClaimSpec {
-	mode := corev1.PersistentVolumeBlock
-
-	return &corev1.PersistentVolumeClaimSpec{
-		StorageClassName: storageClassName,
-		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany},
-		Resources: corev1.VolumeResourceRequirements{
-			Requests: corev1.ResourceList{
-				corev1.ResourceStorage: size,
-			},
-		},
-		VolumeMode: &mode,
-	}
-}
-
-// CreateSpecReadWriteOnce returns pvc spec with accessMode ReadWriteOnce.
-func CreateSpecReadWriteOnce(storageClassName *string, size resource.Quantity) *corev1.PersistentVolumeClaimSpec {
+func CreateSpec(storageClassName *string,
+	size resource.Quantity,
+	accessMode corev1.PersistentVolumeAccessMode,
+	volumeMode corev1.PersistentVolumeMode) *corev1.PersistentVolumeClaimSpec {
 	return &corev1.PersistentVolumeClaimSpec{
 		StorageClassName: storageClassName,
-		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+		AccessModes:      []corev1.PersistentVolumeAccessMode{accessMode},
 		Resources: corev1.VolumeResourceRequirements{
 			Requests: corev1.ResourceList{
 				corev1.ResourceStorage: size,
 			},
 		},
+		VolumeMode: &volumeMode,
 	}
 }
 
 // CreateSpecForDataVolume returns pvc spec with accessMode ReadWriteMany or
 // with accessMode ReadWriteOnce, depending on environment variable.
-func CreateSpecForDataVolume(storageClassName *string, size resource.Quantity) *corev1.PersistentVolumeClaimSpec {
+func CreateSpecForDataVolume(storageClassName *string,
+	size resource.Quantity,
+	accessMode corev1.PersistentVolumeAccessMode,
+	volumeMode corev1.PersistentVolumeMode,
+) *corev1.PersistentVolumeClaimSpec {
 	if os.Getenv(ForceDVReadWriteOnceVar) == "yes" {
-		return CreateSpecReadWriteOnce(storageClassName, size)
+		return CreateSpec(storageClassName, size, corev1.ReadWriteOnce, volumeMode)
 	}
-	return CreateSpecReadWriteMany(storageClassName, size)
+	return CreateSpec(storageClassName, size, accessMode, volumeMode)
 }

images/virtualization-artifact/pkg/controller/common/util.go

@@ -187,6 +187,9 @@ const (
 	// AnnVMLastAppliedSpec is an annotation on KVVM. It contains a JSON with VM spec.
 	AnnVMLastAppliedSpec = AnnAPIGroup + "/vm.last-applied-spec"
 
+	// AnnUnpackFormat  is an annotation on DataVolume. It contains type of image. qcow2,raw,etc..
+	AnnUnpackFormat = AnnAPIGroup + "/unpack.format"
+
 	// LastPropagatedVMAnnotationsAnnotation is a marshalled map of previously applied virtual machine annotations.
 	LastPropagatedVMAnnotationsAnnotation = AnnAPIGroup + "/last-propagated-vm-annotations"
 	// LastPropagatedVMLabelsAnnotation is a marshalled map of previously applied virtual machine labels.

images/virtualization-artifact/pkg/controller/kvbuilder/dv.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/deckhouse/virtualization-controller/pkg/common"
 	"github.com/deckhouse/virtualization-controller/pkg/common/pvc"
+	cc "github.com/deckhouse/virtualization-controller/pkg/controller/common"
 	"github.com/deckhouse/virtualization-controller/pkg/sdk/framework/helper"
 )
 
@@ -46,6 +47,7 @@ func NewDV(name types.NamespacedName) *DV {
 					Name:      name.Name,
 					Annotations: map[string]string{
 						"cdi.kubevirt.io/storage.deleteAfterCompletion": "false",
+						cc.AnnUnpackFormat: "qcow2",
 					},
 				},
 				Spec: cdiv1.DataVolumeSpec{
@@ -56,18 +58,15 @@ func NewDV(name types.NamespacedName) *DV {
 	}
 }
 
-func (b *DV) SetAccessMode(accessMode corev1.PersistentVolumeAccessMode) {
-	b.Resource.Spec.PVC.AccessModes = []corev1.PersistentVolumeAccessMode{accessMode}
-}
-
-func (b *DV) SetBindingMode(wffc bool) {
-	if !wffc {
-		b.AddAnnotation("cdi.kubevirt.io/storage.bind.immediate.requested", "true")
-	}
-}
-
-func (b *DV) SetPVC(storageClassName *string, size resource.Quantity) {
-	b.Resource.Spec.PVC = pvc.CreateSpecForDataVolume(storageClassName, size)
+func (b *DV) SetPVC(storageClassName *string,
+	size resource.Quantity,
+	accessMode corev1.PersistentVolumeAccessMode,
+	volumeMode corev1.PersistentVolumeMode) {
+	b.Resource.Spec.PVC = pvc.CreateSpecForDataVolume(storageClassName,
+		size,
+		accessMode,
+		volumeMode,
+	)
 }
 
 func (b *DV) SetDataSource(source *cdiv1.DataVolumeSource) {