From 69e33498c567d86928a6cc630546e62db9e903a7 Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Thu, 22 Oct 2026 18:42:36 -0700 Subject: [PATCH 1/2] Revert "Updated Red Hat mounts for air-gapped clusters" This reverts commit 835c3c0d0, reversing changes made to its first parent. Restore RHEL host subscription mounts when custom repositories are configured. Retain shared test helpers required by subsequent tests. Signed-off-by: Rahul Sharma (cherry picked from commit bfd5046b5054a45213b9214e26cc6f862902b979) --- controllers/object_controls.go | 13 ++-- controllers/transforms_test.go | 110 ------------------------------- internal/state/driver_test.go | 83 ----------------------- internal/state/driver_volumes.go | 17 ++--- 4 files changed, 8 insertions(+), 215 deletions(-) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index e8f243e516..c5fea7216d 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3777,15 +3777,10 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy // set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" { - pathToVolumeSource := MountPathToVolumeSource{} - if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" && osID == "rhel" { - n.logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", osID) - } else { - n.logger.Info("Mounting subscriptions into the driver container", "OS", osID) - pathToVolumeSource, err = n.getSubscriptionPathsToVolumeSources() - if err != nil { - return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err) - } + n.logger.Info("Mounting subscriptions into the driver container", "OS", osID) + pathToVolumeSource, err := n.getSubscriptionPathsToVolumeSources() + if err != nil { + return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) } // sort host path volumes to ensure ordering is preserved when adding to pod spec diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 23b99aa3cc..4e9c4b73b2 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -20,7 +20,6 @@ import ( "os" "path" "path/filepath" - "strings" "testing" "github.com/go-logr/logr/funcr" @@ -4704,115 +4703,6 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) { } } -func TestTransformDriverSubscriptionMounts(t *testing.T) { - repoConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo-config", - Namespace: "test-ns", - }, - Data: map[string]string{ - "redhat.repo": "[test-repo]", - }, - } - mockClient := fake.NewFakeClient(repoConfigMap) - - testCases := []struct { - description string - osRelease string - osTag string - repoConfigEnabled bool - expectSubscriptionMounts bool - expectedSubscriptionHostMap map[string]corev1.HostPathType - }{ - { - description: "rhel with repo config skips host subscription mounts", - osRelease: "rhel", - osTag: "rhel8.10", - repoConfigEnabled: true, - expectSubscriptionMounts: false, - }, - { - description: "rhel without repo config mounts host subscription paths", - osRelease: "rhel", - osTag: "rhel8.10", - expectSubscriptionMounts: true, - expectedSubscriptionHostMap: map[string]corev1.HostPathType{ - "/etc/pki/entitlement": corev1.HostPathDirectory, - "/etc/yum.repos.d/redhat.repo": corev1.HostPathFile, - "/etc/rhsm": corev1.HostPathDirectory, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - ds := NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}). - WithInitContainer(corev1.Container{Name: "k8s-driver-manager"}) - cpSpec := &gpuv1.ClusterPolicySpec{ - Driver: gpuv1.DriverSpec{ - Repository: "nvcr.io/nvidia", - Image: "driver", - ImagePullPolicy: "IfNotPresent", - Version: "580.126.16", - Manager: gpuv1.DriverManagerSpec{ - Repository: "nvcr.io/nvidia/cloud-native", - Image: "k8s-driver-manager", - ImagePullPolicy: "IfNotPresent", - Version: "v0.8.0", - }, - }, - } - if tc.repoConfigEnabled { - cpSpec.Driver.RepoConfig = &gpuv1.DriverRepoConfigSpec{ConfigMapName: "test-repo-config"} - } - - err := TransformDriver(ds.DaemonSet, cpSpec, ClusterPolicyController{ - client: mockClient, - runtime: gpuv1.Containerd, - operatorNamespace: "test-ns", - logger: ctrl.Log.WithName("test"), - gpuNodeOSRelease: tc.osRelease, - gpuNodeOSTag: tc.osTag, - }) - require.NoError(t, err) - - driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr") - require.NotNil(t, driverContainer) - assertSubscriptionHostPathVolumesForTransform(t, ds.Spec.Template.Spec.Volumes, tc.expectedSubscriptionHostMap) - assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMountForTransform(driverContainer.VolumeMounts)) - }) - } -} - -func hasSubscriptionVolumeMountForTransform(volumeMounts []corev1.VolumeMount) bool { - for _, volumeMount := range volumeMounts { - if strings.HasPrefix(volumeMount.Name, "subscription-config-") { - return true - } - } - return false -} - -func assertSubscriptionHostPathVolumesForTransform(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { - t.Helper() - - if expected == nil { - expected = map[string]corev1.HostPathType{} - } - - actual := map[string]corev1.HostPathType{} - for _, volume := range volumes { - if !strings.HasPrefix(volume.Name, "subscription-config-") { - continue - } - require.NotNil(t, volume.HostPath) - require.NotNil(t, volume.HostPath.Type) - actual[volume.HostPath.Path] = *volume.HostPath.Type - } - - assert.Equal(t, expected, actual) -} - // baseDriverDaemonSetSpec returns a minimal DaemonSetSpec representative of // the post-transformation driver DaemonSet in the ClusterPolicy path. // Only fields relevant to extractDriverInstallConfig extraction are diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index 92907fb62c..3da548cd7d 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -127,26 +127,6 @@ func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool { return false } -func assertSubscriptionHostPathVolumes(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { - t.Helper() - - if expected == nil { - expected = map[string]corev1.HostPathType{} - } - - actual := map[string]corev1.HostPathType{} - for _, volume := range volumes { - if !strings.HasPrefix(volume.Name, "subscription-config-") { - continue - } - require.NotNil(t, volume.HostPath) - require.NotNil(t, volume.HostPath.Type) - actual[volume.HostPath.Path] = *volume.HostPath.Type - } - - assert.Equal(t, expected, actual) -} - func TestDriverRenderMinimal(t *testing.T) { // Construct a sample driver state manager const ( @@ -621,69 +601,6 @@ func TestDriverAdditionalConfigs(t *testing.T) { require.Equal(t, string(o), actual) } -func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { - repoConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo-config", - Namespace: "test-ns", - }, - Data: map[string]string{ - "redhat.repo": "[test-repo]", - }, - } - - testCases := []struct { - description string - osRelease string - repoConfigEnabled bool - expectSubscriptionMounts bool - expectedSubscriptionHostMap map[string]corev1.HostPathType - }{ - { - description: "rhel with repo config skips host subscription mounts", - osRelease: "rhel", - repoConfigEnabled: true, - expectSubscriptionMounts: false, - }, - { - description: "rhel without repo config mounts host subscription paths", - osRelease: "rhel", - expectSubscriptionMounts: true, - expectedSubscriptionHostMap: map[string]corev1.HostPathType{ - "/etc/pki/entitlement": corev1.HostPathDirectory, - "/etc/yum.repos.d/redhat.repo": corev1.HostPathFile, - "/etc/rhsm": corev1.HostPathDirectory, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - stateDriver := &stateDriver{ - stateSkel: stateSkel{ - client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap).Build(), - namespace: "test-ns", - }, - } - driver := &nvidiav1alpha1.NVIDIADriver{} - if tc.repoConfigEnabled { - driver.Spec.RepoConfig = &nvidiav1alpha1.DriverRepoConfigSpec{Name: "test-repo-config"} - } - - configs, err := stateDriver.getDriverAdditionalConfigs( - context.Background(), - driver, - testClusterInfo{runtime: consts.Containerd}, - nodePool{osRelease: tc.osRelease, osVersion: tc.osRelease}, - ) - require.NoError(t, err) - - assertSubscriptionHostPathVolumes(t, configs.Volumes, tc.expectedSubscriptionHostMap) - assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMount(configs.VolumeMounts)) - }) - } -} - func TestDriverPrecompiledLibModules(t *testing.T) { const ( libModulesVolumeName = "lib-modules" diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index 34f60ea609..8b15ca0cce 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -184,19 +184,10 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph // set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES if (pool.osRelease == "rhel" && openshiftVersion == "" && runtime != consts.CRIO) || pool.osRelease == "sles" || pool.osRelease == "sl-micro" { - pathToVolumeSource := MountPathToVolumeSource{} - // Custom repo ConfigMap supplies yum repos in offline/air-gapped installs. Skip - // mounting host RHSM paths (/etc/pki/entitlement, redhat.repo, /etc/rhsm): they may - // be missing or not directories on minimal nodes, and are not needed when packages - // come only from the mounted repo ConfigMap. - if cr.Spec.IsRepoConfigEnabled() && pool.osRelease == "rhel" { - logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", pool.osVersion) - } else { - logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion) - pathToVolumeSource, err = getSubscriptionPathsToVolumeSources(pool.osRelease) - if err != nil { - return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err) - } + logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion) + pathToVolumeSource, err := getSubscriptionPathsToVolumeSources(pool.osRelease) + if err != nil { + return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) } // sort host path volumes to ensure ordering is preserved when adding to pod spec From 30ac50f8acef3dab8d0f96eb26461061f20c2adc Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Thu, 22 Oct 2026 18:42:36 -0700 Subject: [PATCH 2/2] Handle missing host subscription paths in driver pods Create missing RHEL subscription files and directories, and missing SUSE credentials directories, so hosts using mirrored repositories can mount subscription paths without new API fields or driver images. Mount the RHEL repository parent directory only in the driver-manager init container to support file creation without exposing additional host repository files to regular driver containers. Apply the changes to both ClusterPolicy and NVIDIADriver rendering, preserving existing OS/runtime eligibility checks. Add regression tests for exact subscription mounts and rendered custom repository mounts. Signed-off-by: Rahul Sharma (cherry picked from commit 21c6d32e2fafcb29b02ef748c52e8f90dcab8778) Signed-off-by: Rahul Sharma --- controllers/object_controls.go | 41 +++-- controllers/transforms_test.go | 166 +++++++++++++++++++ internal/state/driver.go | 5 +- internal/state/driver_test.go | 184 +++++++++++++++++++++ internal/state/driver_volumes.go | 37 +++-- manifests/state-driver/0500_daemonset.yaml | 7 + 6 files changed, 414 insertions(+), 26 deletions(-) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c5fea7216d..6679cf028c 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -263,19 +263,27 @@ var SubscriptionPathMap = map[string](MountPathToVolumeSource){ "/run/secrets/etc-pki-entitlement": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/pki/entitlement", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/run/secrets/redhat.repo": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/yum.repos.d/redhat.repo", - Type: ptr.To(corev1.HostPathFile), + Type: new(corev1.HostPathFileOrCreate), }, }, "/run/secrets/rhsm": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/rhsm", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), + }, + }, + // FileOrCreate does not create parents. A separate directory volume + // lets kubelet create the repo directory and retry file setup if needed. + "/run/secrets/yum.repos.d": corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/yum.repos.d", + Type: new(corev1.HostPathDirectoryOrCreate), }, }, }, @@ -283,19 +291,19 @@ var SubscriptionPathMap = map[string](MountPathToVolumeSource){ "/run/secrets/etc-pki-entitlement": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/pki/entitlement", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectory), }, }, "/run/secrets/redhat.repo": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/yum.repos.d/redhat.repo", - Type: ptr.To(corev1.HostPathFile), + Type: new(corev1.HostPathFile), }, }, "/run/secrets/rhsm": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/rhsm", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectory), }, }, }, @@ -303,13 +311,13 @@ var SubscriptionPathMap = map[string](MountPathToVolumeSource){ "/etc/zypp/credentials.d": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/zypp/credentials.d", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/etc/SUSEConnect": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/SUSEConnect", - Type: ptr.To(corev1.HostPathFileOrCreate), + Type: new(corev1.HostPathFileOrCreate), }, }, }, @@ -317,13 +325,13 @@ var SubscriptionPathMap = map[string](MountPathToVolumeSource){ "/etc/zypp/credentials.d": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/zypp/credentials.d", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/etc/SUSEConnect": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/SUSEConnect", - Type: ptr.To(corev1.HostPathFileOrCreate), + Type: new(corev1.HostPathFileOrCreate), }, }, }, @@ -3780,7 +3788,7 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy n.logger.Info("Mounting subscriptions into the driver container", "OS", osID) pathToVolumeSource, err := n.getSubscriptionPathsToVolumeSources() if err != nil { - return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err) } // sort host path volumes to ensure ordering is preserved when adding to pod spec @@ -3798,7 +3806,16 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy MountPath: mountPath, ReadOnly: true, } - driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volMountSubscription) + if mountPath == "/run/secrets/yum.repos.d" { + // The parent directory is needed only for kubelet's file creation. + manager := findContainerByName(podSpec.InitContainers, "k8s-driver-manager") + if manager == nil { + return fmt.Errorf("k8s-driver-manager initContainer is missing from the driver daemonset manifest") + } + manager.VolumeMounts = append(manager.VolumeMounts, volMountSubscription) + } else { + driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volMountSubscription) + } subscriptionVol := corev1.Volume{Name: volMountSubscriptionName, VolumeSource: pathToVolumeSource[mountPath]} podSpec.Volumes = append(podSpec.Volumes, subscriptionVol) diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 4e9c4b73b2..cc4a36dfe1 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -20,6 +20,7 @@ import ( "os" "path" "path/filepath" + "strings" "testing" "github.com/go-logr/logr/funcr" @@ -4703,6 +4704,171 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) { } } +func TestTransformDriverSubscriptionMounts(t *testing.T) { + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.repo": "[test-repo]", + }, + } + mockClient := fake.NewFakeClient(repoConfigMap) + + rhelHostPaths := map[string]corev1.HostPathType{ + "/etc/pki/entitlement": corev1.HostPathDirectoryOrCreate, + "/etc/yum.repos.d/redhat.repo": corev1.HostPathFileOrCreate, + "/etc/rhsm": corev1.HostPathDirectoryOrCreate, + "/etc/yum.repos.d": corev1.HostPathDirectoryOrCreate, + } + rhelDriverMounts := []corev1.VolumeMount{ + {Name: "subscription-config-0", MountPath: "/run/secrets/etc-pki-entitlement", ReadOnly: true}, + {Name: "subscription-config-1", MountPath: "/run/secrets/redhat.repo", ReadOnly: true}, + {Name: "subscription-config-2", MountPath: "/run/secrets/rhsm", ReadOnly: true}, + } + suseDriverMounts := []corev1.VolumeMount{ + {Name: "subscription-config-0", MountPath: "/etc/SUSEConnect", ReadOnly: true}, + {Name: "subscription-config-1", MountPath: "/etc/zypp/credentials.d", ReadOnly: true}, + } + testCases := map[string]struct { + osRelease string + osTag string + runtime string + repoConfigEnabled bool + expectedSubscriptionHostMap map[string]corev1.HostPathType + expectedSubscriptionMounts []corev1.VolumeMount + }{ + "rhel with custom repositories retains subscription mounts": { + osRelease: "rhel", osTag: "rhel8.10", runtime: "containerd", + repoConfigEnabled: true, expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel without custom repositories mounts subscription paths": { + osRelease: "rhel", osTag: "rhel8.10", runtime: "containerd", + expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel with docker mounts subscription paths": { + osRelease: "rhel", osTag: "rhel8.10", runtime: "docker", + repoConfigEnabled: true, expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel with crio skips subscription mounts": { + osRelease: "rhel", osTag: "rhel8.10", runtime: "crio", repoConfigEnabled: true, + }, + "ubuntu skips subscription mounts": { + osRelease: "ubuntu", osTag: "ubuntu22.04", runtime: "containerd", repoConfigEnabled: true, + }, + "sles creates missing subscription paths": { + osRelease: "sles", osTag: "sles15.6", runtime: "containerd", repoConfigEnabled: true, + expectedSubscriptionMounts: suseDriverMounts, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/zypp/credentials.d": corev1.HostPathDirectoryOrCreate, + "/etc/SUSEConnect": corev1.HostPathFileOrCreate, + }, + }, + "sl-micro creates missing subscription paths": { + osRelease: "sl-micro", osTag: "sl-micro6.1", runtime: "containerd", repoConfigEnabled: true, + expectedSubscriptionMounts: suseDriverMounts, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/zypp/credentials.d": corev1.HostPathDirectoryOrCreate, + "/etc/SUSEConnect": corev1.HostPathFileOrCreate, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ds := NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}). + WithInitContainer(corev1.Container{Name: "k8s-driver-manager"}) + cpSpec := &gpuv1.ClusterPolicySpec{ + Driver: gpuv1.DriverSpec{ + Repository: "nvcr.io/nvidia", + Image: "driver", + ImagePullPolicy: "IfNotPresent", + Version: "580.126.16", + Manager: gpuv1.DriverManagerSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-driver-manager", + ImagePullPolicy: "IfNotPresent", + Version: "v0.8.0", + }, + }, + } + if tc.repoConfigEnabled { + cpSpec.Driver.RepoConfig = &gpuv1.DriverRepoConfigSpec{ConfigMapName: "test-repo-config"} + } + + err := TransformDriver(ds.DaemonSet, cpSpec, ClusterPolicyController{ + client: mockClient, + runtime: gpuv1.Runtime(tc.runtime), + operatorNamespace: "test-ns", + logger: ctrl.Log.WithName("test"), + gpuNodeOSRelease: tc.osRelease, + gpuNodeOSTag: tc.osTag, + }) + require.NoError(t, err) + + driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr") + require.NotNil(t, driverContainer) + assertSubscriptionHostPathVolumesForTransform(t, ds.Spec.Template.Spec.Volumes, tc.expectedSubscriptionHostMap) + expectedAdditionalMounts := append([]corev1.VolumeMount{}, tc.expectedSubscriptionMounts...) + if tc.repoConfigEnabled { + repoPaths := map[string]string{ + "rhel": "/etc/yum.repos.d/custom.repo", + "ubuntu": "/etc/apt/sources.list.d/custom.repo", + "sles": "/etc/zypp/repos.d/custom.repo", + "sl-micro": "/etc/zypp/repos.d/custom.repo", + } + expectedAdditionalMounts = append(expectedAdditionalMounts, corev1.VolumeMount{ + Name: "test-repo-config", MountPath: repoPaths[tc.osRelease], + SubPath: "custom.repo", ReadOnly: true, + }) + } + assert.ElementsMatch(t, expectedAdditionalMounts, driverContainer.VolumeMounts) + expectedManagerMounts := []corev1.VolumeMount{} + if tc.osRelease == "rhel" && len(tc.expectedSubscriptionMounts) > 0 { + expectedManagerMounts = append(expectedManagerMounts, corev1.VolumeMount{ + Name: "subscription-config-3", MountPath: "/run/secrets/yum.repos.d", ReadOnly: true, + }) + } + + manager := findContainerByName(ds.Spec.Template.Spec.InitContainers, "k8s-driver-manager") + require.NotNil(t, manager) + assert.ElementsMatch(t, expectedManagerMounts, manager.VolumeMounts) + + assert.Equal(t, len(tc.expectedSubscriptionHostMap) > 0, hasSubscriptionVolumeMountForTransform(driverContainer.VolumeMounts)) + }) + } +} + +func hasSubscriptionVolumeMountForTransform(volumeMounts []corev1.VolumeMount) bool { + for _, volumeMount := range volumeMounts { + if strings.HasPrefix(volumeMount.Name, "subscription-config-") { + return true + } + } + return false +} + +func assertSubscriptionHostPathVolumesForTransform(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { + t.Helper() + + if expected == nil { + expected = map[string]corev1.HostPathType{} + } + + actual := map[string]corev1.HostPathType{} + for _, volume := range volumes { + if !strings.HasPrefix(volume.Name, "subscription-config-") { + continue + } + require.NotNil(t, volume.HostPath) + require.NotNil(t, volume.HostPath.Type) + actual[volume.HostPath.Path] = *volume.HostPath.Type + } + + assert.Equal(t, expected, actual) +} + // baseDriverDaemonSetSpec returns a minimal DaemonSetSpec representative of // the post-transformation driver DaemonSet in the ClusterPolicy path. // Only fields relevant to extractDriverInstallConfig extraction are diff --git a/internal/state/driver.go b/internal/state/driver.go index 2046bbf815..03e416d0ef 100644 --- a/internal/state/driver.go +++ b/internal/state/driver.go @@ -84,8 +84,9 @@ type precompiledSpec struct { } type additionalConfigs struct { - VolumeMounts []corev1.VolumeMount - Volumes []corev1.Volume + DriverManagerVolumeMounts []corev1.VolumeMount + VolumeMounts []corev1.VolumeMount + Volumes []corev1.Volume } type driverRenderData struct { diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index 3da548cd7d..0328193902 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/json" apitypes "k8s.io/apimachinery/pkg/types" @@ -127,6 +128,26 @@ func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool { return false } +func assertSubscriptionHostPathVolumes(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) { + t.Helper() + + if expected == nil { + expected = map[string]corev1.HostPathType{} + } + + actual := map[string]corev1.HostPathType{} + for _, volume := range volumes { + if !strings.HasPrefix(volume.Name, "subscription-config-") { + continue + } + require.NotNil(t, volume.HostPath) + require.NotNil(t, volume.HostPath.Type) + actual[volume.HostPath.Path] = *volume.HostPath.Type + } + + assert.Equal(t, expected, actual) +} + func TestDriverRenderMinimal(t *testing.T) { // Construct a sample driver state manager const ( @@ -601,6 +622,158 @@ func TestDriverAdditionalConfigs(t *testing.T) { require.Equal(t, string(o), actual) } +func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.repo": "[test-repo]", + }, + } + + rhelHostPaths := map[string]corev1.HostPathType{ + "/etc/pki/entitlement": corev1.HostPathDirectoryOrCreate, + "/etc/yum.repos.d/redhat.repo": corev1.HostPathFileOrCreate, + "/etc/rhsm": corev1.HostPathDirectoryOrCreate, + "/etc/yum.repos.d": corev1.HostPathDirectoryOrCreate, + } + rhelDriverMounts := []corev1.VolumeMount{ + {Name: "subscription-config-0", MountPath: "/run/secrets/etc-pki-entitlement", ReadOnly: true}, + {Name: "subscription-config-1", MountPath: "/run/secrets/redhat.repo", ReadOnly: true}, + {Name: "subscription-config-2", MountPath: "/run/secrets/rhsm", ReadOnly: true}, + } + suseDriverMounts := []corev1.VolumeMount{ + {Name: "subscription-config-0", MountPath: "/etc/SUSEConnect", ReadOnly: true}, + {Name: "subscription-config-1", MountPath: "/etc/zypp/credentials.d", ReadOnly: true}, + } + testCases := map[string]struct { + osRelease string + runtime string + openshift string + repoConfigEnabled bool + expectedSubscriptionHostMap map[string]corev1.HostPathType + expectedSubscriptionMounts []corev1.VolumeMount + }{ + "rhel with custom repositories retains subscription mounts": { + osRelease: "rhel", runtime: "containerd", + repoConfigEnabled: true, expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel without custom repositories mounts subscription paths": { + osRelease: "rhel", runtime: "containerd", + expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel with docker mounts subscription paths": { + osRelease: "rhel", runtime: "docker", + repoConfigEnabled: true, expectedSubscriptionHostMap: rhelHostPaths, expectedSubscriptionMounts: rhelDriverMounts, + }, + "rhel with crio skips subscription mounts": { + osRelease: "rhel", runtime: "crio", repoConfigEnabled: true, + }, + "ubuntu skips subscription mounts": { + osRelease: "ubuntu", runtime: "containerd", repoConfigEnabled: true, + }, + "sles creates missing subscription paths": { + osRelease: "sles", runtime: "containerd", repoConfigEnabled: true, + expectedSubscriptionMounts: suseDriverMounts, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/zypp/credentials.d": corev1.HostPathDirectoryOrCreate, + "/etc/SUSEConnect": corev1.HostPathFileOrCreate, + }, + }, + "sl-micro creates missing subscription paths": { + osRelease: "sl-micro", runtime: "containerd", repoConfigEnabled: true, + expectedSubscriptionMounts: suseDriverMounts, + expectedSubscriptionHostMap: map[string]corev1.HostPathType{ + "/etc/zypp/credentials.d": corev1.HostPathDirectoryOrCreate, + "/etc/SUSEConnect": corev1.HostPathFileOrCreate, + }, + }, + "openshift skips explicit rhel subscription mounts": { + osRelease: "rhel", runtime: "containerd", openshift: "4.18", repoConfigEnabled: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + stateDriver := &stateDriver{ + stateSkel: stateSkel{ + client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap).Build(), + namespace: "test-ns", + }, + } + driver := &nvidiav1alpha1.NVIDIADriver{} + if tc.repoConfigEnabled { + driver.Spec.RepoConfig = &nvidiav1alpha1.DriverRepoConfigSpec{Name: "test-repo-config"} + } + + configs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + testClusterInfo{runtime: tc.runtime, openshiftVersion: tc.openshift}, + nodePool{osRelease: tc.osRelease, osVersion: tc.osRelease}, + ) + require.NoError(t, err) + + assertSubscriptionHostPathVolumes(t, configs.Volumes, tc.expectedSubscriptionHostMap) + expectedAdditionalMounts := append([]corev1.VolumeMount{}, tc.expectedSubscriptionMounts...) + if tc.repoConfigEnabled { + repoPaths := map[string]string{ + "rhel": "/etc/yum.repos.d/custom.repo", + "ubuntu": "/etc/apt/sources.list.d/custom.repo", + "sles": "/etc/zypp/repos.d/custom.repo", + "sl-micro": "/etc/zypp/repos.d/custom.repo", + } + expectedAdditionalMounts = append(expectedAdditionalMounts, corev1.VolumeMount{ + Name: "test-repo-config", MountPath: repoPaths[tc.osRelease], + SubPath: "custom.repo", ReadOnly: true, + }) + } + assert.ElementsMatch(t, expectedAdditionalMounts, configs.VolumeMounts) + expectedManagerMounts := []corev1.VolumeMount{} + if tc.osRelease == "rhel" && len(tc.expectedSubscriptionMounts) > 0 { + expectedManagerMounts = append(expectedManagerMounts, corev1.VolumeMount{ + Name: "subscription-config-3", MountPath: "/run/secrets/yum.repos.d", ReadOnly: true, + }) + } + + assert.ElementsMatch(t, expectedManagerMounts, configs.DriverManagerVolumeMounts) + + renderData := getMinimalDriverRenderData() + renderData.AdditionalConfigs = configs + objects := renderDriverObjects(t, scheme.Scheme, renderData) + ds, err := getDaemonsetFromObjects(objects) + require.NoError(t, err) + renderedDriver := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr") + require.NotNil(t, renderedDriver) + for _, mount := range expectedAdditionalMounts { + assert.Contains(t, renderedDriver.VolumeMounts, mount) + } + var renderedSubscriptionMounts []corev1.VolumeMount + for _, mount := range renderedDriver.VolumeMounts { + if strings.HasPrefix(mount.Name, "subscription-config-") { + renderedSubscriptionMounts = append(renderedSubscriptionMounts, mount) + } + } + assert.ElementsMatch(t, tc.expectedSubscriptionMounts, renderedSubscriptionMounts) + + manager := findContainerByName(ds.Spec.Template.Spec.InitContainers, "k8s-driver-manager") + require.NotNil(t, manager) + for _, mount := range expectedManagerMounts { + assert.Contains(t, manager.VolumeMounts, mount) + } + for _, container := range ds.Spec.Template.Spec.Containers { + for _, mount := range container.VolumeMounts { + assert.NotEqual(t, "/run/secrets/yum.repos.d", mount.MountPath) + } + } + + assert.Equal(t, len(tc.expectedSubscriptionHostMap) > 0, hasSubscriptionVolumeMount(configs.VolumeMounts)) + }) + } +} + func TestDriverPrecompiledLibModules(t *testing.T) { const ( libModulesVolumeName = "lib-modules" @@ -1006,6 +1179,17 @@ func TestVGPUHostManagerDaemonsetOpenShift(t *testing.T) { require.Equal(t, string(o), actual) } +func renderDriverObjects(t *testing.T, scheme *runtime.Scheme, renderData *driverRenderData) []*unstructured.Unstructured { + t.Helper() + state, err := NewStateDriver(nil, "test-operator", scheme, manifestDir) + require.NoError(t, err) + driverState, ok := state.(*stateDriver) + require.True(t, ok) + objects, err := driverState.renderManifestObjects(t.Context(), renderData) + require.NoError(t, err) + return objects +} + func getMinimalDriverRenderData() *driverRenderData { return &driverRenderData{ Driver: &driverSpec{ diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index 8b15ca0cce..1363f99b77 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -71,19 +71,27 @@ var SubscriptionPathMap = map[string]MountPathToVolumeSource{ "/run/secrets/etc-pki-entitlement": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/pki/entitlement", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/run/secrets/redhat.repo": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/yum.repos.d/redhat.repo", - Type: ptr.To(corev1.HostPathFile), + Type: new(corev1.HostPathFileOrCreate), }, }, "/run/secrets/rhsm": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/rhsm", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), + }, + }, + // FileOrCreate does not create parents. A separate directory volume + // lets kubelet create the repo directory and retry file setup if needed. + "/run/secrets/yum.repos.d": corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/yum.repos.d", + Type: new(corev1.HostPathDirectoryOrCreate), }, }, }, @@ -91,19 +99,19 @@ var SubscriptionPathMap = map[string]MountPathToVolumeSource{ "/run/secrets/etc-pki-entitlement": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/pki/entitlement", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectory), }, }, "/run/secrets/redhat.repo": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/yum.repos.d/redhat.repo", - Type: ptr.To(corev1.HostPathFile), + Type: new(corev1.HostPathFile), }, }, "/run/secrets/rhsm": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/rhsm", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectory), }, }, }, @@ -111,13 +119,13 @@ var SubscriptionPathMap = map[string]MountPathToVolumeSource{ "/etc/zypp/credentials.d": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/zypp/credentials.d", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/etc/SUSEConnect": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/SUSEConnect", - Type: ptr.To(corev1.HostPathFileOrCreate), + Type: new(corev1.HostPathFileOrCreate), }, }, }, @@ -125,13 +133,13 @@ var SubscriptionPathMap = map[string]MountPathToVolumeSource{ "/etc/zypp/credentials.d": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/zypp/credentials.d", - Type: ptr.To(corev1.HostPathDirectory), + Type: new(corev1.HostPathDirectoryOrCreate), }, }, "/etc/SUSEConnect": corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: "/etc/SUSEConnect", - Type: ptr.To(corev1.HostPathFileOrCreate), + Type: new(corev1.HostPathFileOrCreate), }, }, }, @@ -187,7 +195,7 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion) pathToVolumeSource, err := getSubscriptionPathsToVolumeSources(pool.osRelease) if err != nil { - return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err) + return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err) } // sort host path volumes to ensure ordering is preserved when adding to pod spec @@ -205,7 +213,12 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph MountPath: mountPath, ReadOnly: true, } - additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volMountSubscription) + if mountPath == "/run/secrets/yum.repos.d" { + // Keep the parent directory out of the driver containers. + additionalCfgs.DriverManagerVolumeMounts = append(additionalCfgs.DriverManagerVolumeMounts, volMountSubscription) + } else { + additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volMountSubscription) + } subscriptionVol := corev1.Volume{Name: volMountSubscriptionName, VolumeSource: pathToVolumeSource[mountPath]} additionalCfgs.Volumes = append(additionalCfgs.Volumes, subscriptionVol) diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml index 65e1eb74a5..1532a5ceae 100644 --- a/manifests/state-driver/0500_daemonset.yaml +++ b/manifests/state-driver/0500_daemonset.yaml @@ -196,6 +196,13 @@ spec: - name: run-mellanox-drivers mountPath: /run/mellanox/drivers mountPropagation: HostToContainer + {{- if and .AdditionalConfigs .AdditionalConfigs.DriverManagerVolumeMounts }} + {{- range .AdditionalConfigs.DriverManagerVolumeMounts }} + - name: {{ .Name }} + mountPath: {{ .MountPath }} + readOnly: {{ .ReadOnly }} + {{- end }} + {{- end }} containers: - image: {{ .Driver.ImagePath }} imagePullPolicy: {{ default "IfNotPresent" .Driver.Spec.ImagePullPolicy }}