diff --git a/.ko-kubetest2.yaml b/.ko-kubetest2.yaml new file mode 100644 index 00000000..b5121c7f --- /dev/null +++ b/.ko-kubetest2.yaml @@ -0,0 +1,7 @@ +builds: +- id: stackit-csi-plugin + main: ./cmd/stackit-csi-plugin + ldflags: + - -s -w + - -X github.com/stackitcloud/cloud-provider-stackit/pkg/version.Version={{.Env.VERSION}} + - -X github.com/stackitcloud/cloud-provider-stackit/pkg/csi/blockstorage.DriverName=kubetest2.csi.stackit.cloud diff --git a/Makefile b/Makefile index 0466e1e5..b7adfd51 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ build: $(BUILD_IMAGES) $(BUILD_IMAGES): $(SOURCES) CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=${GOPROXY} go build \ -trimpath \ - -ldflags $(LDFLAGS) \ + -ldflags "$(LDFLAGS)" \ -o $@ \ cmd/$@/main.go @@ -46,6 +46,10 @@ image-%: $(APKO) $(KO) IS_DEV=$(IS_DEV) \ ./hack/build.sh $* +.PHONY: image-stackit-csi-plugin-test +image-stackit-csi-plugin-test: export KO_CONFIG_PATH = .ko-kubetest2.yaml +image-stackit-csi-plugin-test: image-stackit-csi-plugin + .PHONY: clean-tools-bin clean-tools-bin: ## Empty the tools binary directory. rm -rf $(TOOLS_BIN_DIR)/* $(TOOLS_BIN_DIR)/.version_* diff --git a/deploy/overlays/kubetest2/cloud-config.yaml b/deploy/overlays/kubetest2/cloud-config.yaml new file mode 100644 index 00000000..a96763e0 --- /dev/null +++ b/deploy/overlays/kubetest2/cloud-config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: stackit-cloud-config + namespace: kube-system +data: + cloud.yaml: |- + global: + projectId: REPLACE_WITH_PROJECTID + region: eu01 + blockStorage: + rescanOnResize: true diff --git a/deploy/overlays/kubetest2/cloud-secret.yaml b/deploy/overlays/kubetest2/cloud-secret.yaml new file mode 100644 index 00000000..305e390d --- /dev/null +++ b/deploy/overlays/kubetest2/cloud-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: stackit-cloud-secret + namespace: kube-system +type: Opaque +stringData: + sa_key.json: |- + REPLACE_WITH_SERVICEACCOUNT_JSON diff --git a/deploy/overlays/kubetest2/controllerplugin-network-labels-patch.yaml b/deploy/overlays/kubetest2/controllerplugin-network-labels-patch.yaml new file mode 100644 index 00000000..160ebd6c --- /dev/null +++ b/deploy/overlays/kubetest2/controllerplugin-network-labels-patch.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csi-stackit-controllerplugin + namespace: kube-system +spec: + template: + metadata: + labels: + networking.gardener.cloud/to-apiserver: allowed + networking.gardener.cloud/to-dns: allowed diff --git a/deploy/overlays/kubetest2/delete-storageclass-patch.yaml b/deploy/overlays/kubetest2/delete-storageclass-patch.yaml new file mode 100644 index 00000000..963bf129 --- /dev/null +++ b/deploy/overlays/kubetest2/delete-storageclass-patch.yaml @@ -0,0 +1,5 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: premium-perf4-stackit +$patch: delete diff --git a/deploy/overlays/kubetest2/driver-name-patch.yaml b/deploy/overlays/kubetest2/driver-name-patch.yaml new file mode 100644 index 00000000..7942943e --- /dev/null +++ b/deploy/overlays/kubetest2/driver-name-patch.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-stackit-nodeplugin + namespace: kube-system +spec: + template: + spec: + containers: + - name: node-driver-registrar + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/kubetest2.csi.stackit.cloud/csi.sock + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/kubetest2.csi.stackit.cloud + type: DirectoryOrCreate diff --git a/deploy/overlays/kubetest2/kustomization.yaml b/deploy/overlays/kubetest2/kustomization.yaml new file mode 100644 index 00000000..e30bfe8d --- /dev/null +++ b/deploy/overlays/kubetest2/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../csi-plugin +- cloud-config.yaml +- cloud-secret.yaml +- storageclass.yaml + +images: +- name: ghcr.io/stackitcloud/cloud-provider-stackit/stackit-csi-plugin + newName: REPLACE_WITH_IMAGE_NAME + newTag: REPLACE_WITH_TAG + +patches: +- path: controllerplugin-network-labels-patch.yaml +- path: delete-storageclass-patch.yaml +- path: driver-name-patch.yaml +- path: nodeplugin-patch.yaml +- target: + group: storage.k8s.io + version: v1 + kind: CSIDriver + name: block-storage.csi.stackit.cloud + patch: |- + - op: replace + path: /metadata/name + value: kubetest2.csi.stackit.cloud diff --git a/deploy/overlays/kubetest2/nodeplugin-patch.yaml b/deploy/overlays/kubetest2/nodeplugin-patch.yaml new file mode 100644 index 00000000..28cb8381 --- /dev/null +++ b/deploy/overlays/kubetest2/nodeplugin-patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-stackit-nodeplugin + namespace: kube-system +spec: + template: + spec: + containers: + - name: stackit-csi-plugin + livenessProbe: + httpGet: + port: 19808 + ports: + - $patch: replace + - containerPort: 19808 + name: healthz + protocol: TCP + - name: liveness-probe + args: + - --csi-address=/csi/csi.sock + - --health-port=19808 diff --git a/deploy/overlays/kubetest2/storageclass.yaml b/deploy/overlays/kubetest2/storageclass.yaml new file mode 100644 index 00000000..2bba6f99 --- /dev/null +++ b/deploy/overlays/kubetest2/storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: premium-perf4-stackit-kubetest2 +provisioner: kubetest2.csi.stackit.cloud +parameters: + type: "storage_premium_perf4" diff --git a/pkg/csi/blockstorage/controllerserver.go b/pkg/csi/blockstorage/controllerserver.go index 367d8340..fdf778d9 100644 --- a/pkg/csi/blockstorage/controllerserver.go +++ b/pkg/csi/blockstorage/controllerserver.go @@ -58,9 +58,8 @@ type stackitParameterConfig struct { } const ( - blockStorageCSIClusterIDKey = "block-storage.csi.stackit.cloud/cluster" - snapshotTypeSnapshot = "snapshot" - snapshotTypeBackup = "backup" + snapshotTypeSnapshot = "snapshot" + snapshotTypeBackup = "backup" ) func (cs *controllerServer) validateVolumeCapabilities(req []*csi.VolumeCapability) error { @@ -117,11 +116,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol accessibleTopologyReq := req.GetAccessibilityRequirements() // Check from topology if accessibleTopologyReq != nil { - if cs.Driver.legacyDriver { - volAvailability = sharedcsi.GetAZFromTopology(legacyTopologyKey, accessibleTopologyReq) - } else { - volAvailability = sharedcsi.GetAZFromTopology(topologyKey, accessibleTopologyReq) - } + volAvailability = sharedcsi.GetAZFromTopology(activeTopologyKey(cs.Driver.legacyDriver), accessibleTopologyReq) } } @@ -148,7 +143,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // Volume Create // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{driverClusterIDKey(): cs.Driver.clusterID} properties := map[string]string{} // Tag volume with metadata if present: https://github.com/kubernetes-csi/external-provisioner/pull/399 for _, mKey := range sharedcsi.RecognizedCSIProvisionerParams { @@ -678,7 +673,7 @@ func (cs *controllerServer) createSnapshot(ctx context.Context, name, volumeID s // Add cluster ID to the snapshot metadata // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{driverClusterIDKey(): cs.Driver.clusterID} properties := map[string]string{} // see https://github.com/kubernetes-csi/external-snapshotter/pull/375/ @@ -711,7 +706,7 @@ func (cs *controllerServer) createSnapshot(ctx context.Context, name, volumeID s func (cs *controllerServer) createBackup(ctx context.Context, cloud stackitclient.IaaSClient, name, volumeID string, snap *iaas.Snapshot, parameters map[string]string) (*iaas.Backup, error) { //nolint:lll // looks weird when shortened // Add cluster ID to the snapshot metadata // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{driverClusterIDKey(): cs.Driver.clusterID} properties := map[string]string{} // see https://github.com/kubernetes-csi/external-snapshotter/pull/375/ @@ -1023,7 +1018,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat volumeSourceType = stackitclient.VolumeSourceTypes(vol.Source.Type) switch volumeSourceType { case stackitclient.VolumeSource: - volCnx[ResizeRequired] = "true" + volCnx[driverResizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Volume{ @@ -1033,7 +1028,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat }, } case stackitclient.BackupSource: - volCnx[ResizeRequired] = "true" + volCnx[driverResizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Snapshot{ @@ -1043,7 +1038,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat }, } case stackitclient.SnapshotSource: - volCnx[ResizeRequired] = "true" + volCnx[driverResizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Snapshot{ @@ -1055,14 +1050,9 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat } } - topoKey := topologyKey - if cs.Driver.legacyDriver { - topoKey = legacyTopologyKey - } - accessibleTopology := []*csi.Topology{ { - Segments: map[string]string{topoKey: vol.AvailabilityZone}, + Segments: map[string]string{activeTopologyKey(cs.Driver.legacyDriver): vol.AvailabilityZone}, }, } diff --git a/pkg/csi/blockstorage/controllerserver_test.go b/pkg/csi/blockstorage/controllerserver_test.go index c4f91be9..c3ec0fc1 100644 --- a/pkg/csi/blockstorage/controllerserver_test.go +++ b/pkg/csi/blockstorage/controllerserver_test.go @@ -115,7 +115,7 @@ var _ = Describe("ControllerServer test", Ordered, func() { }, AccessibilityRequirements: &csi.TopologyRequirement{ Requisite: []*csi.Topology{ - {Segments: map[string]string{topologyKey: "zone-from-accessibility-reqs"}}, + {Segments: map[string]string{activeTopologyKey(false): "zone-from-accessibility-reqs"}}, }, }, } @@ -143,7 +143,7 @@ var _ = Describe("ControllerServer test", Ordered, func() { }, AccessibilityRequirements: &csi.TopologyRequirement{ Requisite: []*csi.Topology{ - {Segments: map[string]string{topologyKey: "zone-from-accessibility-reqs"}}, + {Segments: map[string]string{activeTopologyKey(false): "zone-from-accessibility-reqs"}}, }, }, } @@ -279,7 +279,7 @@ var _ = Describe("ControllerServer test", Ordered, func() { AccessibilityRequirements: &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { - Segments: map[string]string{topologyKey: "eu01"}, + Segments: map[string]string{activeTopologyKey(false): "eu01"}, }, }, }, @@ -412,7 +412,7 @@ var _ = Describe("ControllerServer test", Ordered, func() { } req.AccessibilityRequirements = &csi.TopologyRequirement{ Requisite: []*csi.Topology{ - {Segments: map[string]string{topologyKey: "some-other-zone"}}, + {Segments: map[string]string{activeTopologyKey(false): "some-other-zone"}}, }, } diff --git a/pkg/csi/blockstorage/driver.go b/pkg/csi/blockstorage/driver.go index 60e3b6f9..c89e40d6 100644 --- a/pkg/csi/blockstorage/driver.go +++ b/pkg/csi/blockstorage/driver.go @@ -15,21 +15,42 @@ import ( ) const ( - driverName = "block-storage.csi.stackit.cloud" - legacyDriverName = "cinder.csi.openstack.org" - topologyKey = "topology." + driverName + "/zone" - legacyTopologyKey = "topology." + legacyDriverName + "/zone" - - // ResizeRequired parameter, if set to true, will trigger a resize on mount operation - ResizeRequired = driverName + "/resizeRequired" + legacyDriverName = "cinder.csi.openstack.org" ) var ( + // DriverName is the default CSI driver name and can be overridden at build time + // using `go build -ldflags "-X github.com/stackitcloud/cloud-provider-stackit/pkg/csi/blockstorage.DriverName=..."`. + DriverName = "block-storage.csi.stackit.cloud" // CSI spec version specVersion = "1.12.0" Version = "1.0.0" ) +func topologyKeyForDriver(name string) string { + return "topology." + name + "/zone" +} + +func driverResizeRequiredKey() string { + return DriverName + "/resizeRequired" +} + +func driverClusterIDKey() string { + return DriverName + "/cluster" +} + +func activeDriverName(legacy bool) string { + if legacy { + return legacyDriverName + } + + return DriverName +} + +func activeTopologyKey(legacy bool) string { + return topologyKeyForDriver(activeDriverName(legacy)) +} + type Driver struct { name string fqVersion string // Fully qualified version in format {Version}@{CPO version} @@ -61,16 +82,12 @@ type DriverOpts struct { func NewDriver(o *DriverOpts) *Driver { d := &Driver{ - name: driverName, - fqVersion: fmt.Sprintf("%s@%s", Version, version.Version), - endpoint: o.Endpoint, - clusterID: o.ClusterID, - pvcLister: o.PVCLister, - } - - if o.LegacyDriverName { - d.name = legacyDriverName - d.legacyDriver = true + name: activeDriverName(o.LegacyDriverName), + fqVersion: fmt.Sprintf("%s@%s", Version, version.Version), + endpoint: o.Endpoint, + clusterID: o.ClusterID, + legacyDriver: o.LegacyDriverName, + pvcLister: o.PVCLister, } if o.BlockVolumeCreation { diff --git a/pkg/csi/blockstorage/driver_name_test.go b/pkg/csi/blockstorage/driver_name_test.go new file mode 100644 index 00000000..7e615ec7 --- /dev/null +++ b/pkg/csi/blockstorage/driver_name_test.go @@ -0,0 +1,28 @@ +package blockstorage + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Driver scoped keys", func() { + var originalDriverName string + + BeforeEach(func() { + originalDriverName = DriverName + DriverName = "kubetest2.csi.stackit.cloud" + }) + + AfterEach(func() { + DriverName = originalDriverName + }) + + It("uses the active driver name and derived keys for the non-legacy driver", func() { + Expect(activeDriverName(false)).To(Equal("kubetest2.csi.stackit.cloud")) + Expect(activeDriverName(true)).To(Equal(legacyDriverName)) + Expect(activeTopologyKey(false)).To(Equal("topology.kubetest2.csi.stackit.cloud/zone")) + Expect(activeTopologyKey(true)).To(Equal("topology.cinder.csi.openstack.org/zone")) + Expect(driverResizeRequiredKey()).To(Equal("kubetest2.csi.stackit.cloud/resizeRequired")) + Expect(driverClusterIDKey()).To(Equal("kubetest2.csi.stackit.cloud/cluster")) + }) +}) diff --git a/pkg/csi/blockstorage/nodeserver.go b/pkg/csi/blockstorage/nodeserver.go index 5b3cc838..f2ed95ee 100644 --- a/pkg/csi/blockstorage/nodeserver.go +++ b/pkg/csi/blockstorage/nodeserver.go @@ -230,7 +230,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } } - if required, ok := volumeContext[ResizeRequired]; ok && strings.EqualFold(required, "true") { + if required, ok := volumeContext[driverResizeRequiredKey()]; ok && strings.EqualFold(required, "true") { r := mountutil.NewResizeFs(ns.Mount.Mounter().Exec) needResize, err := r.NeedResize(devicePath, stagingTarget) @@ -335,14 +335,9 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest return nil, status.Errorf(codes.Internal, "[NodeGetInfo] Unable to retrieve availability zone of node %v", err) } - topoKey := topologyKey - if ns.Driver.legacyDriver { - topoKey = legacyTopologyKey - } - //TODO: support well-known topology key "topology.kubernetes.io/zone" segments := map[string]string{ - topoKey: zone, + activeTopologyKey(ns.Driver.legacyDriver): zone, } nodeInfo.AccessibleTopology = &csi.Topology{Segments: segments} @@ -357,13 +352,7 @@ func (ns *nodeServer) calculateMaxVolumesPerNode() int64 { freePCIeRootPorts = 0 } - csiDriverName := driverName - if ns.Driver.legacyDriver { - // If driver launched in legacy-mode use "cinder.csi.openstack.org" - csiDriverName = legacyDriverName - } - - mountedCSIVolumes, err := mount.CountLocalCSIVolumes(csiDriverName) + mountedCSIVolumes, err := mount.CountLocalCSIVolumes(activeDriverName(ns.Driver.legacyDriver)) if err != nil { klog.Errorf("[NodeGetInfo] unable to retrieve volume count: %v", err) mountedCSIVolumes = 0