From 460189193f1f7e6dbc30ae98c63b25375155d0c5 Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Fri, 23 Oct 2026 13:37:01 -0700 Subject: [PATCH] ci: publish a daily main chart and OLM bundle, test forward compat on it Add publish-main-chart.yaml, which runs daily and can be started by hand on the default branch only. It resolves the newest main image of every operand that publishes per-commit images, prepares the chart values once, and publishes the chart, the OLM bundle and a build record, all versioned 0.0.0-main-, for example 0.0.0-main-20261016. There is one build per date: the run stops when that date's record already exists, unless it was started by hand with overwrite. - get-latest-images.sh covers driver-manager, vgpu-device-manager, the kata and kubevirt sandbox device plugins and cc-manager, takes several components and resolves them concurrently. - generate-values-overrides.sh takes VALUES_PATH=IMAGE pairs, and apply-values-overrides.sh merges them into values.yaml with merge-yaml.py, rejects paths that are not image values and checks that every image renders unchanged into the ClusterPolicy. The operator, validator and node status exporter use the operator image of the same commit. dcgm, dcgm-exporter, gds, gdrcopy, the driver and the DRA driver stay on their released versions. - The chart goes into the existing charts/gpu-operator package through a new main_chart_version input of publish-helm-oci-chart.yaml. - The bundle CSV is updated with update-csv-images.py and validated and pushed through two composite actions, setup-olm-bundle-tools and validate-and-push-olm-bundle, which release-rc-assets.yaml now uses too. The operator-sdk version comes from OPERATOR_SDK_VERSION in versions.mk rather than the builder label in the bundle annotations, which only records the SDK that generated the bundle. The CSV name and version are unchanged, so a main bundle is installed on its own. - push-main-build-record.sh pushes a record of the chart, bundle and image digests to gpu-operator/main-builds only when both were published. forward-compatibility.yaml now finds the newest build record with get-latest-main-build.sh and runs e2e against that build's chart, with the operator version taken from the record. It fails if the newest build is dated before yesterday (UTC) or the chart's digest no longer matches the record. e2e-tests.yaml gets chart_reference and chart_version inputs to install a published chart instead of publishing one from the current commit; callers that pass neither behave as before. Signed-off-by: Abrar Shivani --- .../setup-olm-bundle-tools/action.yaml | 67 ++++ .../validate-and-push-olm-bundle/action.yaml | 84 +++++ .github/scripts/apply-values-overrides.sh | 68 ++++ .github/scripts/generate-values-overrides.sh | 81 +++-- .github/scripts/get-latest-images.sh | 204 ++++++++---- .github/scripts/get-latest-main-build.sh | 66 ++++ .github/scripts/prepare-main-values.sh | 104 ++++++ .github/scripts/push-main-build-record.sh | 83 +++++ .github/workflows/e2e-tests.yaml | 109 ++++--- .github/workflows/forward-compatibility.yaml | 87 +++-- .github/workflows/publish-helm-oci-chart.yaml | 23 +- .github/workflows/publish-main-chart.yaml | 300 ++++++++++++++++++ .github/workflows/release-rc-assets.yaml | 87 +---- versions.mk | 2 + 14 files changed, 1093 insertions(+), 272 deletions(-) create mode 100644 .github/actions/setup-olm-bundle-tools/action.yaml create mode 100644 .github/actions/validate-and-push-olm-bundle/action.yaml create mode 100755 .github/scripts/apply-values-overrides.sh create mode 100755 .github/scripts/get-latest-main-build.sh create mode 100755 .github/scripts/prepare-main-values.sh create mode 100755 .github/scripts/push-main-build-record.sh create mode 100644 .github/workflows/publish-main-chart.yaml diff --git a/.github/actions/setup-olm-bundle-tools/action.yaml b/.github/actions/setup-olm-bundle-tools/action.yaml new file mode 100644 index 0000000000..855baf8f16 --- /dev/null +++ b/.github/actions/setup-olm-bundle-tools/action.yaml @@ -0,0 +1,67 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Set up OLM bundle tools +description: Install Python 3.12 with PyYAML, and Go, regctl, and Operator SDK at the versions the repository pins + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Get Golang version + shell: bash + run: | + GOLANG_VERSION=$(grep "GOLANG_VERSION ?=" versions.mk) + echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Get regctl version + shell: bash + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Get Operator SDK version + shell: bash + run: | + OPERATOR_SDK_VERSION=$(grep "OPERATOR_SDK_VERSION ?=" versions.mk) + echo "OPERATOR_SDK_VERSION=${OPERATOR_SDK_VERSION##OPERATOR_SDK_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GOLANG_VERSION }} + + - name: Install PyYAML + shell: bash + run: pip install --quiet pyyaml + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Install Operator SDK + shell: bash + run: | + set -euo pipefail + + curl -fsSLo operator-sdk \ + "https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64" + chmod +x operator-sdk + sudo mv operator-sdk /usr/local/bin/operator-sdk + operator-sdk version diff --git a/.github/actions/validate-and-push-olm-bundle/action.yaml b/.github/actions/validate-and-push-olm-bundle/action.yaml new file mode 100644 index 0000000000..d5d608846d --- /dev/null +++ b/.github/actions/validate-and-push-olm-bundle/action.yaml @@ -0,0 +1,84 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate and push OLM bundle +description: Validate the OLM bundle, then build the multi-arch bundle image and push it to GHCR + +inputs: + release-tag: + description: Release tag the CSV release metadata must match; the release metadata is not validated when empty + required: false + default: "" + bundle-image-tag: + description: Tag of the pushed bundle image + required: true + github-token: + description: Token used to log in to GHCR + required: true + +outputs: + bundle-image: + description: Reference of the pushed bundle image + value: ${{ steps.build.outputs.bundle_image }} + +runs: + using: composite + steps: + - name: Validate OLM release metadata + if: ${{ inputs.release-tag != '' }} + shell: bash + env: + RELEASE_TAG: ${{ inputs.release-tag }} + CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + run: | + python3 .github/scripts/validate-olm-release-metadata.py \ + --release-tag "${RELEASE_TAG}" \ + --csv "${CSV}" + + - name: Validate OLM bundle + shell: bash + run: | + set -euo pipefail + + operator-sdk bundle validate ./bundle + make validate-csv + + - name: Login to GitHub Container Registry for Docker + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github-token }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + + - name: Build and push OLM bundle image + id: build + shell: bash + env: + BUNDLE_IMAGE_TAG: ${{ inputs.bundle-image-tag }} + GH_REPOSITORY: ${{ github.repository }} + VERSION: "" + DEFAULT_CHANNEL: "stable" + CHANNELS: "stable" + BUNDLE_IMAGE_BUILD_PLATFORM_OPTIONS: "--platform=linux/amd64,linux/arm64 --push" + run: | + set -euo pipefail + + repository="$(echo "${GH_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + bundle_image="ghcr.io/${repository}/gpu-operator-bundle:${BUNDLE_IMAGE_TAG}" + make build-bundle-image DOCKER="docker buildx" BUNDLE_IMAGE="${bundle_image}" + echo "bundle_image=${bundle_image}" >> "${GITHUB_OUTPUT}" + echo "::notice::Published OLM bundle image ${bundle_image}" diff --git a/.github/scripts/apply-values-overrides.sh b/.github/scripts/apply-values-overrides.sh new file mode 100755 index 0000000000..68206ddbb8 --- /dev/null +++ b/.github/scripts/apply-values-overrides.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +readonly IMAGE_REFERENCE_EXPRESSION='eval(strenv(VALUES_PATH)) | (.repository // "") + "/" + (.image // "") + ":" + (.version // "")' + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 CHART_DIRECTORY VALUES_OVERRIDES_FILE" >&2 + exit 1 +fi + +chart_directory="$1" +values_overrides_file="$2" +chart_values_file="${chart_directory}/values.yaml" +script_directory=$(dirname "${BASH_SOURCE[0]}") + +override_path_list=$(yq '.. | select(tag == "!!map" and has("image")) | path | join(".")' "${values_overrides_file}") +if [[ -z "${override_path_list}" ]]; then + echo "Error: ${values_overrides_file} has no image overrides" >&2 + exit 1 +fi +mapfile -t override_paths <<< "${override_path_list}" + +missing_path_count=0 +for override_path in "${override_paths[@]}"; do + if ! VALUES_PATH=".${override_path}" yq -e 'eval(strenv(VALUES_PATH)) | has("image")' "${chart_values_file}" &> /dev/null; then + echo "Error: ${override_path}.image is not a value in ${chart_values_file}" >&2 + missing_path_count=$((missing_path_count + 1)) + fi +done +if [[ ${missing_path_count} -gt 0 ]]; then + exit 1 +fi + +python3 "${script_directory}/merge-yaml.py" \ + --base "${chart_values_file}" \ + --override "${values_overrides_file}" \ + --output "${chart_values_file}" + +rendered_cluster_policy_spec=$(helm template gpu-operator "${chart_directory}" | yq 'select(.kind == "ClusterPolicy") | .spec') + +mismatched_path_count=0 +for override_path in "${override_paths[@]}"; do + expected_image_reference=$(VALUES_PATH=".${override_path}" yq "${IMAGE_REFERENCE_EXPRESSION}" "${values_overrides_file}") + rendered_image_reference=$(VALUES_PATH=".${override_path}" yq "${IMAGE_REFERENCE_EXPRESSION}" <<< "${rendered_cluster_policy_spec}") + if [[ "${rendered_image_reference}" != "${expected_image_reference}" ]]; then + echo "Error: ${override_path} renders as '${rendered_image_reference}' in the ClusterPolicy, expected '${expected_image_reference}'" >&2 + mismatched_path_count=$((mismatched_path_count + 1)) + fi +done +if [[ ${mismatched_path_count} -gt 0 ]]; then + exit 1 +fi + +echo "Applied ${#override_paths[@]} image overrides to ${chart_values_file}" diff --git a/.github/scripts/generate-values-overrides.sh b/.github/scripts/generate-values-overrides.sh index f79413ce55..7b86c2fd94 100755 --- a/.github/scripts/generate-values-overrides.sh +++ b/.github/scripts/generate-values-overrides.sh @@ -16,58 +16,53 @@ set -euo pipefail -# Usage: generate-values-overrides.sh OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE -# -# Generates a Helm values override file for GPU Operator component images. -# This file can be used with `helm install -f values-overrides.yaml` to -# override default component image versions. - -if [[ $# -ne 4 ]]; then - echo "Usage: $0 OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE" >&2 +if [[ $# -lt 2 ]]; then + echo "Usage: $0 OUTPUT_FILE VALUES_PATH=IMAGE [VALUES_PATH=IMAGE ...]" >&2 echo "" >&2 echo "Example:" >&2 - echo " $0 values.yaml \\" >&2 - echo " ghcr.io/nvidia/container-toolkit:v1.18.0-ubuntu20.04 \\" >&2 - echo " ghcr.io/nvidia/k8s-device-plugin:v0.17.0-ubi8 \\" >&2 - echo " ghcr.io/nvidia/k8s-mig-manager:v0.10.0-ubuntu20.04" >&2 + echo " $0 values-overrides.yaml \\" >&2 + echo " toolkit=ghcr.io/nvidia/container-toolkit:3e112b4a \\" >&2 + echo " driver.manager=ghcr.io/nvidia/k8s-driver-manager:357e757f" >&2 exit 1 fi -OUTPUT_FILE="$1" -TOOLKIT_IMAGE="$2" -DEVICE_PLUGIN_IMAGE="$3" -MIG_MANAGER_IMAGE="$4" +output_file="$1" +shift -# Generate values override file -cat > "${OUTPUT_FILE}" < "${output_file}" -toolkit: - repository: "" - version: "" - image: "${TOOLKIT_IMAGE}" +for image_override in "$@"; do + if [[ "${image_override}" != *=* ]]; then + echo "Error: '${image_override}' is not in VALUES_PATH=IMAGE form" >&2 + exit 1 + fi -devicePlugin: - repository: "" - version: "" - image: "${DEVICE_PLUGIN_IMAGE}" + values_path="${image_override%%=*}" + image_reference="${image_override#*=}" -migManager: - repository: "" - version: "" - image: "${MIG_MANAGER_IMAGE}" -EOF + if [[ ! "${values_path}" =~ ^[A-Za-z][A-Za-z0-9]*(\.[A-Za-z][A-Za-z0-9]*)*$ ]]; then + echo "Error: invalid values path '${values_path}'" >&2 + exit 1 + fi -echo "Generated values override file: ${OUTPUT_FILE}" -echo "" -echo "=== Component Images ===" -echo "Container Toolkit: ${TOOLKIT_IMAGE}" -echo "Device Plugin: ${DEVICE_PLUGIN_IMAGE}" -echo "MIG Manager: ${MIG_MANAGER_IMAGE}" + if [[ ! "${image_reference}" =~ ^(.+)/([^/:@]+):([^/:@]+)$ ]]; then + echo "Error: image '${image_reference}' for '${values_path}' is not in REPOSITORY/IMAGE:TAG form" >&2 + exit 1 + fi + + VALUES_PATH=".${values_path}" \ + IMAGE_REPOSITORY="${BASH_REMATCH[1]}" \ + IMAGE_NAME="${BASH_REMATCH[2]}" \ + IMAGE_VERSION="${BASH_REMATCH[3]}" \ + yq -i ' + eval(strenv(VALUES_PATH) + ".repository") = strenv(IMAGE_REPOSITORY) | + eval(strenv(VALUES_PATH) + ".image") = strenv(IMAGE_NAME) | + eval(strenv(VALUES_PATH) + ".version") = strenv(IMAGE_VERSION) | + eval(strenv(VALUES_PATH) + ".version") style="double" + ' "${output_file}" +done + +echo "Generated values override file: ${output_file}" echo "" echo "=== File Contents ===" -cat "${OUTPUT_FILE}" - +cat "${output_file}" diff --git a/.github/scripts/get-latest-images.sh b/.github/scripts/get-latest-images.sh index 2f9c3a7cd7..d38384c0ee 100755 --- a/.github/scripts/get-latest-images.sh +++ b/.github/scripts/get-latest-images.sh @@ -15,91 +15,157 @@ set -euo pipefail -COMPONENT=${1:-} +readonly COMPONENT_SOURCES='{ + "toolkit": { + "imageRepository": "ghcr.io/nvidia/container-toolkit", + "githubRepository": "NVIDIA/nvidia-container-toolkit", + "githubBranch": "main" + }, + "device-plugin": { + "imageRepository": "ghcr.io/nvidia/k8s-device-plugin", + "githubRepository": "NVIDIA/k8s-device-plugin", + "githubBranch": "main" + }, + "mig-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-mig-manager", + "githubRepository": "NVIDIA/mig-parted", + "githubBranch": "main" + }, + "driver-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-driver-manager", + "githubRepository": "NVIDIA/k8s-driver-manager", + "githubBranch": "main" + }, + "vgpu-device-manager": { + "imageRepository": "ghcr.io/nvidia/vgpu-device-manager", + "githubRepository": "NVIDIA/vgpu-device-manager", + "githubBranch": "main" + }, + "kata-sandbox-device-plugin": { + "imageRepository": "ghcr.io/nvidia/nvidia-sandbox-device-plugin", + "githubRepository": "NVIDIA/sandbox-device-plugin", + "githubBranch": "main" + }, + "kubevirt-gpu-device-plugin": { + "imageRepository": "ghcr.io/nvidia/kubevirt-gpu-device-plugin", + "githubRepository": "NVIDIA/kubevirt-gpu-device-plugin", + "githubBranch": "master" + }, + "cc-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-cc-manager", + "githubRepository": "NVIDIA/k8s-cc-manager", + "githubBranch": "main", + "tagSuffix": "-distroless" + } +}' +VALID_COMPONENTS=$(jq -r 'keys_unsorted | join(" ")' <<< "${COMPONENT_SOURCES}") +readonly VALID_COMPONENTS +readonly MAX_RETRIES=5 -if [[ -z "${COMPONENT}" ]]; then - echo "Usage: $0 " >&2 +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [...]" >&2 + echo "Valid components: ${VALID_COMPONENTS}" >&2 exit 1 fi -# Verify regctl is available if ! command -v regctl &> /dev/null; then echo "Error: regctl not found. Please install regctl first." >&2 exit 1 fi -# Map component names to GHCR image repositories and GitHub source repositories -case "${COMPONENT}" in - toolkit) - IMAGE_REPO="ghcr.io/nvidia/container-toolkit" - GITHUB_REPO="NVIDIA/nvidia-container-toolkit" - ;; - device-plugin) - IMAGE_REPO="ghcr.io/nvidia/k8s-device-plugin" - GITHUB_REPO="NVIDIA/k8s-device-plugin" - ;; - mig-manager) - IMAGE_REPO="ghcr.io/nvidia/k8s-mig-manager" - GITHUB_REPO="NVIDIA/mig-parted" - ;; - *) - echo "Error: Unknown component '${COMPONENT}'" >&2 - echo "Valid components: toolkit, device-plugin, mig-manager" >&2 - exit 1 - ;; -esac - -echo "Fetching latest commit from ${GITHUB_REPO}..." >&2 - -# Get the latest commit SHA from the main branch using GitHub API. -# NOTE: We use 8-char truncated SHAs as image tags. This must match the -# tag convention used by each component's CI pipeline when publishing images. -GITHUB_API_URL="https://api.github.com/repos/${GITHUB_REPO}/commits/main" - -# Use GITHUB_TOKEN if available for authentication (higher rate limits) -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - LATEST_COMMIT=$(curl -sSL \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "Accept: application/vnd.github.v3+json" \ - "${GITHUB_API_URL}" | \ - jq -r '.sha[0:8]') -else - LATEST_COMMIT=$(curl -sSL \ - -H "Accept: application/vnd.github.v3+json" \ - "${GITHUB_API_URL}" | \ - jq -r '.sha[0:8]') -fi +get_latest_image() { + local component=$1 -if [[ -z "${LATEST_COMMIT}" || "${LATEST_COMMIT}" == "null" ]]; then - echo "Error: Failed to fetch latest commit from ${GITHUB_REPO}" >&2 - exit 1 -fi + local component_source + if ! component_source=$(jq -e --arg component "${component}" '.[$component]' <<< "${COMPONENT_SOURCES}"); then + echo "Error: Unknown component '${component}'" >&2 + echo "Valid components: ${VALID_COMPONENTS}" >&2 + return 1 + fi -echo "Latest commit SHA: ${LATEST_COMMIT}" >&2 + local image_repo github_repo github_branch tag_suffix + image_repo=$(jq -er '.imageRepository' <<< "${component_source}") + github_repo=$(jq -er '.githubRepository' <<< "${component_source}") + github_branch=$(jq -er '.githubBranch' <<< "${component_source}") + tag_suffix=$(jq -r '.tagSuffix // ""' <<< "${component_source}") -# Construct full image path with commit tag -FULL_IMAGE="${IMAGE_REPO}:${LATEST_COMMIT}" + echo "Fetching latest commit from ${github_repo}@${github_branch}..." >&2 -echo "Verifying image exists: ${FULL_IMAGE}" >&2 + # NOTE: We use 8-char truncated SHAs as image tags. This must match the + # tag convention used by each component's CI pipeline when publishing images. + local github_api_url="https://api.github.com/repos/${github_repo}/commits/${github_branch}" -# Verify the image exists using regctl with retry -MAX_RETRIES=5 -RETRY_DELAY=30 -for i in $(seq 1 ${MAX_RETRIES}); do - if regctl manifest head "${FULL_IMAGE}" &> /dev/null; then - echo "Verified ${COMPONENT} image: ${FULL_IMAGE}" >&2 - echo "${FULL_IMAGE}" - exit 0 + local latest_commit + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + latest_commit=$(curl -sSL \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_api_url}" | \ + jq -r '.sha[0:8]') + else + latest_commit=$(curl -sSL \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_api_url}" | \ + jq -r '.sha[0:8]') fi - if [[ $i -lt ${MAX_RETRIES} ]]; then - echo "Image not found (attempt $i/${MAX_RETRIES}), waiting ${RETRY_DELAY}s for CI to build..." >&2 - sleep ${RETRY_DELAY} - # Exponential backoff: 30s, 60s, 120s, 240s - RETRY_DELAY=$((RETRY_DELAY * 2)) + if [[ -z "${latest_commit}" || "${latest_commit}" == "null" ]]; then + echo "Error: Failed to fetch latest commit from ${github_repo}" >&2 + return 1 + fi + + echo "Latest commit SHA: ${latest_commit}" >&2 + + local full_image="${image_repo}:${latest_commit}${tag_suffix}" + + echo "Verifying image exists: ${full_image}" >&2 + + local retry_delay=30 + local i + for i in $(seq 1 ${MAX_RETRIES}); do + if regctl manifest head "${full_image}" &> /dev/null; then + echo "Verified ${component} image: ${full_image}" >&2 + echo "${full_image}" + return 0 + fi + + if [[ $i -lt ${MAX_RETRIES} ]]; then + echo "Image not found (attempt $i/${MAX_RETRIES}), waiting ${retry_delay}s for CI to build..." >&2 + sleep ${retry_delay} + retry_delay=$((retry_delay * 2)) + fi + done + + echo "Error: Image ${full_image} does not exist after ${MAX_RETRIES} attempts" >&2 + echo "The image may not have been built yet for commit ${latest_commit}" >&2 + return 1 +} + +components=("$@") +image_lookup_dir=$(mktemp -d) +trap 'rm -rf "${image_lookup_dir}"' EXIT + +lookup_pids=() +for index in "${!components[@]}"; do + component=${components[index]} + get_latest_image "${component}" \ + > "${image_lookup_dir}/${index}.image" \ + 2> >(sed -u "s/^/[${component}] /" >&2) & + lookup_pids+=($!) +done + +failed_components=() +for index in "${!components[@]}"; do + if ! wait "${lookup_pids[index]}"; then + failed_components+=("${components[index]}") fi done -echo "Error: Image ${FULL_IMAGE} does not exist after ${MAX_RETRIES} attempts" >&2 -echo "The image may not have been built yet for commit ${LATEST_COMMIT}" >&2 -exit 1 +if [[ ${#failed_components[@]} -gt 0 ]]; then + echo "::error::Image lookup failed for ${failed_components[*]}" >&2 + exit 1 +fi + +for index in "${!components[@]}"; do + echo "${components[index]}=$(< "${image_lookup_dir}/${index}.image")" +done diff --git a/.github/scripts/get-latest-main-build.sh b/.github/scripts/get-latest-main-build.sh new file mode 100755 index 0000000000..ed922cd168 --- /dev/null +++ b/.github/scripts/get-latest-main-build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +readonly MAIN_VERSION_PATTERN='^0\.0\.0-main-([0-9]{8})$' + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 MAIN_BUILD_REPOSITORY CHART_REPOSITORY MAX_AGE_DAYS" >&2 + exit 1 +fi + +main_build_repository="$1" +chart_repository="$2" +max_age_days="$3" + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +main_build_tags=$(regctl tag ls "${main_build_repository}") +main_version=$(grep -E "${MAIN_VERSION_PATTERN}" <<< "${main_build_tags}" | sort | tail -n 1 || true) + +if [[ -z "${main_version}" ]]; then + echo "Error: ${main_build_repository} has no main build" >&2 + exit 1 +fi + +[[ "${main_version}" =~ ${MAIN_VERSION_PATTERN} ]] +published_date="${BASH_REMATCH[1]}" + +oldest_accepted_date=$(date -u -d "${max_age_days} days ago" +%Y%m%d) +if (( published_date < oldest_accepted_date )); then + echo "Error: newest main build ${main_version} is older than ${max_age_days} day(s); the scheduled publish did not run or failed" >&2 + exit 1 +fi + +main_build_record=$(regctl artifact get "${main_build_repository}:${main_version}") +operator_version=$(jq -er '.operatorVersion' <<< "${main_build_record}") +recorded_chart_digest=$(jq -r '.chart.digest' <<< "${main_build_record}") +current_chart_digest=$(regctl image digest "${chart_repository}:${main_version}") +if [[ "${current_chart_digest}" != "${recorded_chart_digest}" ]]; then + echo "Error: ${chart_repository}:${main_version} is ${current_chart_digest}, but main build ${main_version} recorded ${recorded_chart_digest}" >&2 + exit 1 +fi + +echo "chart_version=${main_version}" +echo "operator_version=${operator_version}" diff --git a/.github/scripts/prepare-main-values.sh b/.github/scripts/prepare-main-values.sh new file mode 100755 index 0000000000..5443a2a2fc --- /dev/null +++ b/.github/scripts/prepare-main-values.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "Usage: $0 CHART_DIRECTORY VALUES_OVERRIDES_FILE OPERATOR_IMAGE OPERATOR_VERSION" >&2 + exit 1 +fi + +chart_directory="$1" +values_overrides_file="$2" +operator_image="$3" +operator_version="$4" + +script_directory=$(dirname "${BASH_SOURCE[0]}") + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +if ! command -v yq &> /dev/null; then + echo "Error: yq not found. Please install yq first." >&2 + exit 1 +fi + +if ! command -v helm &> /dev/null; then + echo "Error: helm not found. Please install helm first." >&2 + exit 1 +fi + +latest_image_lines=$("${script_directory}/get-latest-images.sh" \ + toolkit device-plugin mig-manager driver-manager vgpu-device-manager \ + kata-sandbox-device-plugin kubevirt-gpu-device-plugin cc-manager) + +declare -A latest_images +while IFS='=' read -r component image; do + latest_images[${component}]=${image} +done <<< "${latest_image_lines}" + +toolkit_image=${latest_images[toolkit]} +device_plugin_image=${latest_images[device-plugin]} +mig_manager_image=${latest_images[mig-manager]} +driver_manager_image=${latest_images[driver-manager]} +vgpu_device_manager_image=${latest_images[vgpu-device-manager]} +kata_sandbox_device_plugin_image=${latest_images[kata-sandbox-device-plugin]} +kubevirt_gpu_device_plugin_image=${latest_images[kubevirt-gpu-device-plugin]} +cc_manager_image=${latest_images[cc-manager]} + +"${script_directory}/generate-values-overrides.sh" "${values_overrides_file}" \ + "toolkit=${toolkit_image}" \ + "devicePlugin=${device_plugin_image}" \ + "gfd=${device_plugin_image}" \ + "migManager=${mig_manager_image}" \ + "driver.manager=${driver_manager_image}" \ + "vgpuManager.driverManager=${driver_manager_image}" \ + "vfioManager=${driver_manager_image}" \ + "vfioManager.driverManager=${driver_manager_image}" \ + "vgpuDeviceManager=${vgpu_device_manager_image}" \ + "kataSandboxDevicePlugin=${kata_sandbox_device_plugin_image}" \ + "sandboxDevicePlugin=${kubevirt_gpu_device_plugin_image}" \ + "ccManager=${cc_manager_image}" \ + "nodeStatusExporter=${operator_image}:${operator_version}" >&2 + +"${script_directory}/apply-values-overrides.sh" "${chart_directory}" "${values_overrides_file}" >&2 + +OPERATOR_REPOSITORY="${operator_image%/*}" \ +OPERATOR_IMAGE_NAME="${operator_image##*/}" \ +OPERATOR_VERSION="${operator_version}" \ +yq -i ' + (.operator, .validator) |= ( + .repository = strenv(OPERATOR_REPOSITORY) | + .image = strenv(OPERATOR_IMAGE_NAME) | + .version = strenv(OPERATOR_VERSION) + ) +' "${chart_directory}/values.yaml" + +printf '%s\n' \ + "${toolkit_image}" \ + "${device_plugin_image}" \ + "${mig_manager_image}" \ + "${driver_manager_image}" \ + "${vgpu_device_manager_image}" \ + "${kata_sandbox_device_plugin_image}" \ + "${kubevirt_gpu_device_plugin_image}" \ + "${cc_manager_image}" diff --git a/.github/scripts/push-main-build-record.sh b/.github/scripts/push-main-build-record.sh new file mode 100755 index 0000000000..d85e95c1c3 --- /dev/null +++ b/.github/scripts/push-main-build-record.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -lt 8 ]]; then + echo "Usage: $0 MAIN_BUILD_REPOSITORY MAIN_VERSION OPERATOR_IMAGE OPERATOR_VERSION CHART_REFERENCE CHART_VERSION BUNDLE_IMAGE OPERAND_IMAGE..." >&2 + exit 1 +fi + +main_build_repository="$1" +main_version="$2" +operator_image="$3" +operator_version="$4" +chart_reference="$5" +chart_version="$6" +bundle_image="$7" +operand_images=("${@:8}") + +source_commit="${GITHUB_SHA}" +run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +main_build_record_reference="${main_build_repository}:${main_version}" +tagged_chart_reference="${chart_reference#oci://}:${chart_version}" + +recorded_images=$(printf '%s\n' "${operator_image}:${operator_version}" "${operand_images[@]}" | sort -u) + +image_records="[]" +while read -r image; do + image_digest=$(regctl image digest "${image}") + image_records=$(jq -c --arg reference "${image}" --arg digest "${image_digest}" '. + [{$reference, $digest}]' <<< "${image_records}") +done <<< "${recorded_images}" + +chart_digest=$(regctl image digest "${tagged_chart_reference}") +bundle_digest=$(regctl image digest "${bundle_image}") + +main_build_record=$(jq -n \ + --arg mainVersion "${main_version}" \ + --arg operatorVersion "${operator_version}" \ + --arg sourceCommit "${source_commit}" \ + --arg runUrl "${run_url}" \ + --arg chartReference "${tagged_chart_reference}" \ + --arg chartDigest "${chart_digest}" \ + --arg bundleReference "${bundle_image}" \ + --arg bundleDigest "${bundle_digest}" \ + --argjson images "${image_records}" \ + '{ + $mainVersion, + $operatorVersion, + $sourceCommit, + $runUrl, + chart: {reference: $chartReference, digest: $chartDigest}, + bundle: {reference: $bundleReference, digest: $bundleDigest}, + $images + }') + +regctl artifact put \ + --artifact-type application/vnd.nvidia.gpu-operator.main-build.v1+json \ + --file-media-type application/json \ + "${main_build_record_reference}" <<< "${main_build_record}" +echo "::notice::Recorded main build ${main_build_record_reference}" diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 0e452648a8..3a389fdeed 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -23,11 +23,16 @@ on: operator_version: required: true type: string - use_values_override: + chart_reference: + description: 'Published Helm chart to test instead of publishing one from this commit (set chart_version, and operator_version to its appVersion)' required: false - type: boolean - default: false - description: 'Use values-overrides artifact for component image configuration' + type: string + default: '' + chart_version: + description: 'Version of chart_reference' + required: false + type: string + default: '' secrets: AWS_ACCESS_KEY_ID: required: true @@ -49,11 +54,16 @@ on: description: 'Operator version to test (override)' required: false type: string - use_values_override: - description: 'Use values-overrides artifact for component image configuration' + chart_reference: + description: 'Published Helm chart to test instead of publishing one from this commit (set chart_version, and operator_version to its appVersion)' required: false - type: boolean - default: false + type: string + default: '' + chart_version: + description: 'Version of chart_reference' + required: false + type: string + default: '' permissions: contents: read @@ -67,6 +77,7 @@ jobs: publish-helm-oci-chart: needs: [variables] + if: ${{ !inputs.chart_reference }} permissions: contents: read packages: write @@ -77,29 +88,43 @@ jobs: e2e-tests-containerd: needs: [variables, publish-helm-oci-chart] + if: ${{ !cancelled() && needs.variables.result == 'success' && (needs.publish-helm-oci-chart.result == 'success' || needs.publish-helm-oci-chart.result == 'skipped') }} runs-on: linux-amd64-cpu4 timeout-minutes: 90 permissions: contents: read id-token: write + env: + HELM_CHART: ${{ inputs.chart_reference || needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ inputs.chart_reference && inputs.chart_version || needs.publish-helm-oci-chart.outputs.chart_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 name: Check out code with: persist-credentials: false - - name: Download values override file - if: ${{ inputs.use_values_override }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: values-overrides - path: ${{ github.workspace }} - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Verify the published Helm chart is available env: - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} - run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" + INPUT_CHART_REFERENCE: ${{ inputs.chart_reference }} + INPUT_CHART_VERSION: ${{ inputs.chart_version }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if [[ -n "${INPUT_CHART_VERSION}" && -z "${INPUT_CHART_REFERENCE}" ]]; then + echo "::error::chart_version requires chart_reference" + exit 1 + fi + if [[ -z "${HELM_CHART_VERSION}" ]]; then + echo "::error::chart_reference requires chart_version" + exit 1 + fi + chart_metadata=$(helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}") + echo "${chart_metadata}" + chart_app_version=$(awk '/^appVersion:/ {print $2}' <<< "${chart_metadata}" | tr -d '"') + if [[ -n "${INPUT_CHART_REFERENCE}" && "${chart_app_version}" != "${OPERATOR_VERSION}" ]]; then + echo "::error::operator_version ${OPERATOR_VERSION} does not match the chart's appVersion ${chart_app_version}" + exit 1 + fi - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -123,19 +148,10 @@ jobs: AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} run: | echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key} - - name: Copy values override file to remote - if: ${{ inputs.use_values_override }} - run: | - scp -i ${private_key} -o StrictHostKeyChecking=no \ - ${{ github.workspace }}/values-overrides.yaml \ - ${instance_hostname}:/tmp/values-overrides.yaml - echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV - name: Run e2e tests env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" @@ -154,29 +170,43 @@ jobs: e2e-tests-nvidiadriver: needs: [variables, publish-helm-oci-chart] + if: ${{ !cancelled() && needs.variables.result == 'success' && (needs.publish-helm-oci-chart.result == 'success' || needs.publish-helm-oci-chart.result == 'skipped') }} runs-on: linux-amd64-cpu4 timeout-minutes: 90 permissions: contents: read id-token: write + env: + HELM_CHART: ${{ inputs.chart_reference || needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ inputs.chart_reference && inputs.chart_version || needs.publish-helm-oci-chart.outputs.chart_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 name: Check out code with: persist-credentials: false - - name: Download values override file - if: ${{ inputs.use_values_override }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: values-overrides - path: ${{ github.workspace }} - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Verify the published Helm chart is available env: - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} - run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" + INPUT_CHART_REFERENCE: ${{ inputs.chart_reference }} + INPUT_CHART_VERSION: ${{ inputs.chart_version }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if [[ -n "${INPUT_CHART_VERSION}" && -z "${INPUT_CHART_REFERENCE}" ]]; then + echo "::error::chart_version requires chart_reference" + exit 1 + fi + if [[ -z "${HELM_CHART_VERSION}" ]]; then + echo "::error::chart_reference requires chart_version" + exit 1 + fi + chart_metadata=$(helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}") + echo "${chart_metadata}" + chart_app_version=$(awk '/^appVersion:/ {print $2}' <<< "${chart_metadata}" | tr -d '"') + if [[ -n "${INPUT_CHART_REFERENCE}" && "${chart_app_version}" != "${OPERATOR_VERSION}" ]]; then + echo "::error::operator_version ${OPERATOR_VERSION} does not match the chart's appVersion ${chart_app_version}" + exit 1 + fi - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -200,19 +230,10 @@ jobs: AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} run: | echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key} - - name: Copy values override file to remote - if: ${{ inputs.use_values_override }} - run: | - scp -i ${private_key} -o StrictHostKeyChecking=no \ - ${{ github.workspace }}/values-overrides.yaml \ - ${instance_hostname}:/tmp/values-overrides.yaml - echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV - name: Run e2e tests env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" diff --git a/.github/workflows/forward-compatibility.yaml b/.github/workflows/forward-compatibility.yaml index 33807d4937..04f5c0be6c 100644 --- a/.github/workflows/forward-compatibility.yaml +++ b/.github/workflows/forward-compatibility.yaml @@ -16,8 +16,8 @@ name: Forward Compatibility on: schedule: - - cron: '0 2 * * 1' # Weekly on Monday at 2 AM UTC - workflow_dispatch: # Manual trigger + - cron: '17 4 * * 1' + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -25,12 +25,26 @@ concurrency: permissions: {} +env: + MAIN_BUILD_MAX_AGE_DAYS: 1 + jobs: - fetch-latest-images: + variables: + permissions: + contents: read + uses: ./.github/workflows/variables.yaml + + find-main-chart: + needs: [variables] runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 10 permissions: contents: read + packages: read + outputs: + chart_reference: ${{ steps.main-chart.outputs.chart_reference }} + chart_version: ${{ steps.main-chart.outputs.chart_version }} + operator_version: ${{ steps.main-chart.outputs.operator_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -46,45 +60,38 @@ jobs: with: release: ${{ env.REGCTL_VERSION }} - - name: Get latest component images and generate values override file + - name: Login to GitHub Container Registry env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Fetch latest images from component repositories - echo "::notice::Fetching latest container-toolkit image..." - TOOLKIT=$(.github/scripts/get-latest-images.sh toolkit) - - echo "::notice::Fetching latest device-plugin image..." - DEVICE_PLUGIN=$(.github/scripts/get-latest-images.sh device-plugin) + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin - echo "::notice::Fetching latest mig-manager image..." - MIG_MANAGER=$(.github/scripts/get-latest-images.sh mig-manager) + - name: Find the chart of the newest complete main build + id: main-chart + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + run: | + set -euo pipefail - # Generate values override file - .github/scripts/generate-values-overrides.sh \ - values-overrides.yaml \ - "${TOOLKIT}" \ - "${DEVICE_PLUGIN}" \ - "${MIG_MANAGER}" + main_build_repository="${OPERATOR_IMAGE_BASE}/main-builds" + chart_repository="${OPERATOR_IMAGE_BASE}/charts/gpu-operator" - - name: Upload values override file - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: values-overrides - path: values-overrides.yaml - retention-days: 30 + { + .github/scripts/get-latest-main-build.sh "${main_build_repository}" "${chart_repository}" "${MAIN_BUILD_MAX_AGE_DAYS}" + echo "chart_reference=oci://${chart_repository}" + } >> "${GITHUB_OUTPUT}" run-e2e-tests: - needs: [fetch-latest-images] + needs: [variables, find-main-chart] permissions: contents: read id-token: write packages: write uses: ./.github/workflows/e2e-tests.yaml with: - operator_image: ghcr.io/nvidia/gpu-operator - operator_version: main-latest - use_values_override: true + operator_image: ${{ needs.variables.outputs.operator_image_base }} + operator_version: ${{ needs.find-main-chart.outputs.operator_version }} + chart_reference: ${{ needs.find-main-chart.outputs.chart_reference }} + chart_version: ${{ needs.find-main-chart.outputs.chart_version }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -96,11 +103,23 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 permissions: {} - needs: [fetch-latest-images, run-e2e-tests] - if: ${{ always() && (needs.fetch-latest-images.result == 'failure' || needs.run-e2e-tests.result == 'failure') }} + needs: [variables, find-main-chart, run-e2e-tests] + if: ${{ always() && (needs.variables.result == 'failure' || needs.find-main-chart.result == 'failure' || needs.run-e2e-tests.result == 'failure') }} env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} steps: + - name: Describe the tested chart + id: tested-chart + env: + CHART_REFERENCE: ${{ needs.find-main-chart.outputs.chart_reference }} + CHART_VERSION: ${{ needs.find-main-chart.outputs.chart_version }} + run: | + if [[ -n "${CHART_VERSION}" ]]; then + echo "description=\`${CHART_REFERENCE} --version ${CHART_VERSION}\` (run \`helm show values\` on it to see the component images)" >> "${GITHUB_OUTPUT}" + else + echo "description=No complete main build from the last ${MAIN_BUILD_MAX_AGE_DAYS} day(s) (none recorded, its chart changed since, or the registry could not be read); see the run log" >> "${GITHUB_OUTPUT}" + fi + - name: Send Slack alert notification if: ${{ env.SLACK_BOT_TOKEN != '' }} uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 @@ -110,5 +129,5 @@ jobs: payload: | { "channel": "${{ secrets.SLACK_CHANNEL_ID }}", - "text": ":x: *Forward Compatibility Test Failed for GPU Operator*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Tested Components:*\nDownload `values-overrides` artifact to see tested component versions\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" + "text": ":x: *Forward Compatibility Test Failed for GPU Operator*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Tested Chart:* ${{ steps.tested-chart.outputs.description }}\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" } diff --git a/.github/workflows/publish-helm-oci-chart.yaml b/.github/workflows/publish-helm-oci-chart.yaml index de65f4141e..53a90ba6c5 100644 --- a/.github/workflows/publish-helm-oci-chart.yaml +++ b/.github/workflows/publish-helm-oci-chart.yaml @@ -23,6 +23,11 @@ on: operator_version: required: true type: string + main_chart_version: + required: false + type: string + default: '' + description: "Package the chart with the merged values.yaml from the main-values artifact and publish it with this 0.0.0-main version instead of the per-commit one" outputs: chart_reference: description: "The OCI reference of the published Helm chart" @@ -57,6 +62,13 @@ jobs: - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Download merged chart values + if: ${{ inputs.main_chart_version != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: main-values + path: ${{ github.workspace }} + - name: Set chart image repository env: OPERATOR_IMAGE: ${{ inputs.operator_image }} @@ -76,12 +88,17 @@ jobs: id: package env: OPERATOR_VERSION: ${{ inputs.operator_version }} + MAIN_CHART_VERSION: ${{ inputs.main_chart_version }} run: | set -euo pipefail - # Helm chart versions must be SemVer. Prefix the image version so a - # numeric short SHA with a leading zero remains a valid prerelease. - CHART_VERSION="0.0.0-git-${OPERATOR_VERSION}" + if [[ -n "${MAIN_CHART_VERSION}" ]]; then + CHART_VERSION="${MAIN_CHART_VERSION}" + else + # Helm chart versions must be SemVer. Prefix the image version so a + # numeric short SHA with a leading zero remains a valid prerelease. + CHART_VERSION="0.0.0-git-${OPERATOR_VERSION}" + fi mkdir -p dist helm lint deployments/gpu-operator diff --git a/.github/workflows/publish-main-chart.yaml b/.github/workflows/publish-main-chart.yaml new file mode 100644 index 0000000000..4ccf9c14a9 --- /dev/null +++ b/.github/workflows/publish-main-chart.yaml @@ -0,0 +1,300 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Publish Main Chart and Bundle + +on: + schedule: + - cron: '17 2 * * *' + workflow_dispatch: + inputs: + overwrite: + description: "Replace today's main build if it is already published" + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: {} + +jobs: + variables: + permissions: + contents: read + uses: ./.github/workflows/variables.yaml + + main-version: + needs: [variables] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: read + outputs: + main_version: ${{ steps.main-version.outputs.main_version }} + publish: ${{ steps.main-version.outputs.publish }} + steps: + - name: Require the default branch + if: ${{ github.event_name == 'workflow_dispatch' && github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + echo "::error::Main charts are published only from ${DEFAULT_BRANCH}" + exit 1 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Login to GitHub Container Registry + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin + + - name: Set main version and check whether it is published + id: main-version + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + OVERWRITE: ${{ inputs.overwrite }} + run: | + set -euo pipefail + + main_version="0.0.0-main-$(date -u +%Y%m%d)" + echo "main_version=${main_version}" >> "${GITHUB_OUTPUT}" + + if record_lookup_error=$(regctl manifest head "${OPERATOR_IMAGE_BASE}/main-builds:${main_version}" 2>&1 > /dev/null); then + if [[ "${OVERWRITE}" != "true" ]]; then + echo "::notice::Main build ${main_version} is already published; run this workflow with overwrite to replace it" + echo "publish=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + elif [[ "${record_lookup_error}" != *"[http 404]"* ]]; then + echo "::error::Could not check whether main build ${main_version} is published: ${record_lookup_error}" + exit 1 + fi + echo "publish=true" >> "${GITHUB_OUTPUT}" + + prepare-main-values: + needs: [variables, main-version] + if: ${{ needs.main-version.outputs.publish == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + resolved_images: ${{ steps.images.outputs.resolved_images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Verify the operator image exists + env: + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if ! regctl manifest head "${OPERATOR_IMAGE}:${OPERATOR_VERSION}" > /dev/null; then + echo "::error::${OPERATOR_IMAGE}:${OPERATOR_VERSION} is not built yet; rerun once CI has published it" + exit 1 + fi + + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Install PyYAML + run: pip install --quiet pyyaml + + - name: Prepare main chart values + id: images + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + set -euo pipefail + + resolved_image_lines=$(.github/scripts/prepare-main-values.sh \ + deployments/gpu-operator values-overrides.yaml "${OPERATOR_IMAGE}" "${OPERATOR_VERSION}") + echo "resolved_images=${resolved_image_lines//$'\n'/ }" >> "${GITHUB_OUTPUT}" + + - name: Upload values override file and merged chart values + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: main-values + path: | + values-overrides.yaml + deployments/gpu-operator/values.yaml + retention-days: 30 + + publish-main-chart: + needs: [variables, main-version, prepare-main-values] + permissions: + contents: read + packages: write + uses: ./.github/workflows/publish-helm-oci-chart.yaml + with: + operator_image: ${{ needs.variables.outputs.operator_image }} + operator_version: ${{ needs.variables.outputs.operator_version }} + main_chart_version: ${{ needs.main-version.outputs.main_version }} + + publish-main-bundle: + needs: [variables, main-version, prepare-main-values] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + outputs: + bundle_image: ${{ steps.bundle.outputs.bundle-image }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up OLM bundle tools + uses: ./.github/actions/setup-olm-bundle-tools + + - name: Download values override file and merged chart values + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: main-values + path: ${{ github.workspace }} + + - name: Update OLM bundle images + run: | + python3 .github/scripts/update-csv-images.py \ + --values deployments/gpu-operator/values.yaml \ + --chart deployments/gpu-operator/Chart.yaml \ + --csv bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + + - name: Check the CSV related images include every override image + env: + CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + run: | + set -euo pipefail + + override_images=$(yq '.. | select(tag == "!!map" and has("image")) | .repository + "/" + .image + ":" + .version' values-overrides.yaml | sort -u) + csv_related_images=$(yq '.spec.relatedImages[].image | sub("@sha256:[0-9a-f]+$", "")' "${CSV}" | sort -u) + missing_images=$(comm -23 <(echo "${override_images}") <(echo "${csv_related_images}")) + if [[ -n "${missing_images}" ]]; then + echo "::error::CSV relatedImages are missing override images:" + echo "${missing_images}" + exit 1 + fi + + - name: Validate and push OLM bundle + id: bundle + uses: ./.github/actions/validate-and-push-olm-bundle + with: + bundle-image-tag: ${{ needs.main-version.outputs.main_version }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + record-main-build: + needs: [variables, main-version, prepare-main-values, publish-main-chart, publish-main-bundle] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Login to GitHub Container Registry + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin + + - name: Push the main build record + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + MAIN_VERSION: ${{ needs.main-version.outputs.main_version }} + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + CHART_REFERENCE: ${{ needs.publish-main-chart.outputs.chart_reference }} + CHART_VERSION: ${{ needs.publish-main-chart.outputs.chart_version }} + BUNDLE_IMAGE: ${{ needs.publish-main-bundle.outputs.bundle_image }} + RESOLVED_IMAGES: ${{ needs.prepare-main-values.outputs.resolved_images }} + run: | + set -euo pipefail + + read -ra operand_images <<< "${RESOLVED_IMAGES}" + .github/scripts/push-main-build-record.sh \ + "${OPERATOR_IMAGE_BASE}/main-builds" \ + "${MAIN_VERSION}" \ + "${OPERATOR_IMAGE}" \ + "${OPERATOR_VERSION}" \ + "${CHART_REFERENCE}" \ + "${CHART_VERSION}" \ + "${BUNDLE_IMAGE}" \ + "${operand_images[@]}" + + notify-failure: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: {} + needs: [variables, main-version, prepare-main-values, publish-main-chart, publish-main-bundle, record-main-build] + if: ${{ always() && github.event_name == 'schedule' && (needs.variables.result == 'failure' || needs.main-version.result == 'failure' || needs.prepare-main-values.result == 'failure' || needs.publish-main-chart.result == 'failure' || needs.publish-main-bundle.result == 'failure' || needs.record-main-build.result == 'failure') }} + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + steps: + - name: Send Slack alert notification + if: ${{ env.SLACK_BOT_TOKEN != '' }} + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": ":x: *GPU Operator Main Build Publish Failed*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Resolved Images:* ${{ needs.prepare-main-values.outputs.resolved_images || 'not resolved; see the run log' }}\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" + } diff --git a/.github/workflows/release-rc-assets.yaml b/.github/workflows/release-rc-assets.yaml index 7a91ec875f..5d7d7dfe30 100644 --- a/.github/workflows/release-rc-assets.yaml +++ b/.github/workflows/release-rc-assets.yaml @@ -121,49 +121,8 @@ jobs: - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Get Golang version - run: | - GOLANG_VERSION=$(grep "GOLANG_VERSION ?=" versions.mk) - echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> "${GITHUB_ENV}" - - - name: Get regctl version - run: | - REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) - echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" - - - name: Install Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version: ${{ env.GOLANG_VERSION }} - - - name: Install PyYAML - run: pip install --quiet pyyaml - - - name: Install regctl - uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 - with: - release: ${{ env.REGCTL_VERSION }} - - - name: Install Operator SDK - run: | - set -euo pipefail - - OPERATOR_SDK_VERSION="$(awk -F': ' '/operators.operatorframework.io.metrics.builder:/ { print $2 }' bundle/metadata/annotations.yaml | sed 's/^operator-sdk-//')" - if [[ -z "${OPERATOR_SDK_VERSION}" ]]; then - echo "::error::OPERATOR_SDK_VERSION not found in bundle metadata" - exit 1 - fi - - curl -sSLo operator-sdk \ - "https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64" - chmod +x operator-sdk - sudo mv operator-sdk /usr/local/bin/operator-sdk - operator-sdk version + - name: Set up OLM bundle tools + uses: ./.github/actions/setup-olm-bundle-tools - name: Login to GitHub Container Registry env: @@ -175,16 +134,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "${GH_TOKEN}" | helm registry login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - - name: Login to GitHub Container Registry for Docker - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - name: Apply Helm values override if: ${{ steps.package-inputs.outputs.helm_values_override != '' }} env: @@ -210,32 +159,12 @@ jobs: --chart deployments/gpu-operator/Chart.yaml \ --csv bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml - - name: Validate updated OLM bundle - env: - RELEASE_TAG: ${{ steps.validate.outputs.release_tag }} - CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml - run: | - set -euo pipefail - - python3 .github/scripts/validate-olm-release-metadata.py \ - --release-tag "${RELEASE_TAG}" \ - --csv "${CSV}" - - operator-sdk bundle validate ./bundle - make validate-csv - - - name: Build and push OLM bundle image - env: - RELEASE_TAG: ${{ steps.validate.outputs.release_tag }} - GH_REPOSITORY: ${{ github.repository }} - VERSION: "" - DEFAULT_CHANNEL: "stable" - CHANNELS: "stable" - BUNDLE_IMAGE_BUILD_PLATFORM_OPTIONS: "--platform=linux/amd64,linux/arm64 --push" - run: | - REPOSITORY="$(echo "${GH_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" - BUNDLE_IMAGE_BASE="ghcr.io/${REPOSITORY}/gpu-operator-bundle" - make build-bundle-image DOCKER="docker buildx" BUNDLE_IMAGE="${BUNDLE_IMAGE_BASE}:${RELEASE_TAG}" + - name: Validate and push OLM bundle + uses: ./.github/actions/validate-and-push-olm-bundle + with: + release-tag: ${{ steps.validate.outputs.release_tag }} + bundle-image-tag: ${{ steps.validate.outputs.release_tag }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Package RC assets id: package diff --git a/versions.mk b/versions.mk index dc4f765605..51b28fa009 100644 --- a/versions.mk +++ b/versions.mk @@ -25,5 +25,7 @@ GOLANGCI_LINT_VERSION ?= v2.14.0 REGCTL_VERSION ?= v0.11.5 +OPERATOR_SDK_VERSION ?= v1.4.0 + GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always 2> /dev/null || echo "") GIT_COMMIT_SHA_FULL ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")