diff --git a/.github/actions/setup-olm-bundle-tools/action.yaml b/.github/actions/setup-olm-bundle-tools/action.yaml new file mode 100644 index 0000000000..855baf8f16 --- /dev/null +++ b/.github/actions/setup-olm-bundle-tools/action.yaml @@ -0,0 +1,67 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Set up OLM bundle tools +description: Install Python 3.12 with PyYAML, and Go, regctl, and Operator SDK at the versions the repository pins + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Get Golang version + shell: bash + run: | + GOLANG_VERSION=$(grep "GOLANG_VERSION ?=" versions.mk) + echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Get regctl version + shell: bash + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Get Operator SDK version + shell: bash + run: | + OPERATOR_SDK_VERSION=$(grep "OPERATOR_SDK_VERSION ?=" versions.mk) + echo "OPERATOR_SDK_VERSION=${OPERATOR_SDK_VERSION##OPERATOR_SDK_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GOLANG_VERSION }} + + - name: Install PyYAML + shell: bash + run: pip install --quiet pyyaml + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Install Operator SDK + shell: bash + run: | + set -euo pipefail + + curl -fsSLo operator-sdk \ + "https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64" + chmod +x operator-sdk + sudo mv operator-sdk /usr/local/bin/operator-sdk + operator-sdk version diff --git a/.github/actions/validate-and-push-olm-bundle/action.yaml b/.github/actions/validate-and-push-olm-bundle/action.yaml new file mode 100644 index 0000000000..d5d608846d --- /dev/null +++ b/.github/actions/validate-and-push-olm-bundle/action.yaml @@ -0,0 +1,84 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate and push OLM bundle +description: Validate the OLM bundle, then build the multi-arch bundle image and push it to GHCR + +inputs: + release-tag: + description: Release tag the CSV release metadata must match; the release metadata is not validated when empty + required: false + default: "" + bundle-image-tag: + description: Tag of the pushed bundle image + required: true + github-token: + description: Token used to log in to GHCR + required: true + +outputs: + bundle-image: + description: Reference of the pushed bundle image + value: ${{ steps.build.outputs.bundle_image }} + +runs: + using: composite + steps: + - name: Validate OLM release metadata + if: ${{ inputs.release-tag != '' }} + shell: bash + env: + RELEASE_TAG: ${{ inputs.release-tag }} + CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + run: | + python3 .github/scripts/validate-olm-release-metadata.py \ + --release-tag "${RELEASE_TAG}" \ + --csv "${CSV}" + + - name: Validate OLM bundle + shell: bash + run: | + set -euo pipefail + + operator-sdk bundle validate ./bundle + make validate-csv + + - name: Login to GitHub Container Registry for Docker + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github-token }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + + - name: Build and push OLM bundle image + id: build + shell: bash + env: + BUNDLE_IMAGE_TAG: ${{ inputs.bundle-image-tag }} + GH_REPOSITORY: ${{ github.repository }} + VERSION: "" + DEFAULT_CHANNEL: "stable" + CHANNELS: "stable" + BUNDLE_IMAGE_BUILD_PLATFORM_OPTIONS: "--platform=linux/amd64,linux/arm64 --push" + run: | + set -euo pipefail + + repository="$(echo "${GH_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + bundle_image="ghcr.io/${repository}/gpu-operator-bundle:${BUNDLE_IMAGE_TAG}" + make build-bundle-image DOCKER="docker buildx" BUNDLE_IMAGE="${bundle_image}" + echo "bundle_image=${bundle_image}" >> "${GITHUB_OUTPUT}" + echo "::notice::Published OLM bundle image ${bundle_image}" diff --git a/.github/scripts/apply-values-overrides.sh b/.github/scripts/apply-values-overrides.sh new file mode 100755 index 0000000000..68206ddbb8 --- /dev/null +++ b/.github/scripts/apply-values-overrides.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +readonly IMAGE_REFERENCE_EXPRESSION='eval(strenv(VALUES_PATH)) | (.repository // "") + "/" + (.image // "") + ":" + (.version // "")' + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 CHART_DIRECTORY VALUES_OVERRIDES_FILE" >&2 + exit 1 +fi + +chart_directory="$1" +values_overrides_file="$2" +chart_values_file="${chart_directory}/values.yaml" +script_directory=$(dirname "${BASH_SOURCE[0]}") + +override_path_list=$(yq '.. | select(tag == "!!map" and has("image")) | path | join(".")' "${values_overrides_file}") +if [[ -z "${override_path_list}" ]]; then + echo "Error: ${values_overrides_file} has no image overrides" >&2 + exit 1 +fi +mapfile -t override_paths <<< "${override_path_list}" + +missing_path_count=0 +for override_path in "${override_paths[@]}"; do + if ! VALUES_PATH=".${override_path}" yq -e 'eval(strenv(VALUES_PATH)) | has("image")' "${chart_values_file}" &> /dev/null; then + echo "Error: ${override_path}.image is not a value in ${chart_values_file}" >&2 + missing_path_count=$((missing_path_count + 1)) + fi +done +if [[ ${missing_path_count} -gt 0 ]]; then + exit 1 +fi + +python3 "${script_directory}/merge-yaml.py" \ + --base "${chart_values_file}" \ + --override "${values_overrides_file}" \ + --output "${chart_values_file}" + +rendered_cluster_policy_spec=$(helm template gpu-operator "${chart_directory}" | yq 'select(.kind == "ClusterPolicy") | .spec') + +mismatched_path_count=0 +for override_path in "${override_paths[@]}"; do + expected_image_reference=$(VALUES_PATH=".${override_path}" yq "${IMAGE_REFERENCE_EXPRESSION}" "${values_overrides_file}") + rendered_image_reference=$(VALUES_PATH=".${override_path}" yq "${IMAGE_REFERENCE_EXPRESSION}" <<< "${rendered_cluster_policy_spec}") + if [[ "${rendered_image_reference}" != "${expected_image_reference}" ]]; then + echo "Error: ${override_path} renders as '${rendered_image_reference}' in the ClusterPolicy, expected '${expected_image_reference}'" >&2 + mismatched_path_count=$((mismatched_path_count + 1)) + fi +done +if [[ ${mismatched_path_count} -gt 0 ]]; then + exit 1 +fi + +echo "Applied ${#override_paths[@]} image overrides to ${chart_values_file}" diff --git a/.github/scripts/generate-values-overrides.sh b/.github/scripts/generate-values-overrides.sh index f79413ce55..7b86c2fd94 100755 --- a/.github/scripts/generate-values-overrides.sh +++ b/.github/scripts/generate-values-overrides.sh @@ -16,58 +16,53 @@ set -euo pipefail -# Usage: generate-values-overrides.sh OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE -# -# Generates a Helm values override file for GPU Operator component images. -# This file can be used with `helm install -f values-overrides.yaml` to -# override default component image versions. - -if [[ $# -ne 4 ]]; then - echo "Usage: $0 OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE" >&2 +if [[ $# -lt 2 ]]; then + echo "Usage: $0 OUTPUT_FILE VALUES_PATH=IMAGE [VALUES_PATH=IMAGE ...]" >&2 echo "" >&2 echo "Example:" >&2 - echo " $0 values.yaml \\" >&2 - echo " ghcr.io/nvidia/container-toolkit:v1.18.0-ubuntu20.04 \\" >&2 - echo " ghcr.io/nvidia/k8s-device-plugin:v0.17.0-ubi8 \\" >&2 - echo " ghcr.io/nvidia/k8s-mig-manager:v0.10.0-ubuntu20.04" >&2 + echo " $0 values-overrides.yaml \\" >&2 + echo " toolkit=ghcr.io/nvidia/container-toolkit:3e112b4a \\" >&2 + echo " driver.manager=ghcr.io/nvidia/k8s-driver-manager:357e757f" >&2 exit 1 fi -OUTPUT_FILE="$1" -TOOLKIT_IMAGE="$2" -DEVICE_PLUGIN_IMAGE="$3" -MIG_MANAGER_IMAGE="$4" +output_file="$1" +shift -# Generate values override file -cat > "${OUTPUT_FILE}" < "${output_file}" -toolkit: - repository: "" - version: "" - image: "${TOOLKIT_IMAGE}" +for image_override in "$@"; do + if [[ "${image_override}" != *=* ]]; then + echo "Error: '${image_override}' is not in VALUES_PATH=IMAGE form" >&2 + exit 1 + fi -devicePlugin: - repository: "" - version: "" - image: "${DEVICE_PLUGIN_IMAGE}" + values_path="${image_override%%=*}" + image_reference="${image_override#*=}" -migManager: - repository: "" - version: "" - image: "${MIG_MANAGER_IMAGE}" -EOF + if [[ ! "${values_path}" =~ ^[A-Za-z][A-Za-z0-9]*(\.[A-Za-z][A-Za-z0-9]*)*$ ]]; then + echo "Error: invalid values path '${values_path}'" >&2 + exit 1 + fi -echo "Generated values override file: ${OUTPUT_FILE}" -echo "" -echo "=== Component Images ===" -echo "Container Toolkit: ${TOOLKIT_IMAGE}" -echo "Device Plugin: ${DEVICE_PLUGIN_IMAGE}" -echo "MIG Manager: ${MIG_MANAGER_IMAGE}" + if [[ ! "${image_reference}" =~ ^(.+)/([^/:@]+):([^/:@]+)$ ]]; then + echo "Error: image '${image_reference}' for '${values_path}' is not in REPOSITORY/IMAGE:TAG form" >&2 + exit 1 + fi + + VALUES_PATH=".${values_path}" \ + IMAGE_REPOSITORY="${BASH_REMATCH[1]}" \ + IMAGE_NAME="${BASH_REMATCH[2]}" \ + IMAGE_VERSION="${BASH_REMATCH[3]}" \ + yq -i ' + eval(strenv(VALUES_PATH) + ".repository") = strenv(IMAGE_REPOSITORY) | + eval(strenv(VALUES_PATH) + ".image") = strenv(IMAGE_NAME) | + eval(strenv(VALUES_PATH) + ".version") = strenv(IMAGE_VERSION) | + eval(strenv(VALUES_PATH) + ".version") style="double" + ' "${output_file}" +done + +echo "Generated values override file: ${output_file}" echo "" echo "=== File Contents ===" -cat "${OUTPUT_FILE}" - +cat "${output_file}" diff --git a/.github/scripts/get-latest-images.sh b/.github/scripts/get-latest-images.sh index 2f9c3a7cd7..d38384c0ee 100755 --- a/.github/scripts/get-latest-images.sh +++ b/.github/scripts/get-latest-images.sh @@ -15,91 +15,157 @@ set -euo pipefail -COMPONENT=${1:-} +readonly COMPONENT_SOURCES='{ + "toolkit": { + "imageRepository": "ghcr.io/nvidia/container-toolkit", + "githubRepository": "NVIDIA/nvidia-container-toolkit", + "githubBranch": "main" + }, + "device-plugin": { + "imageRepository": "ghcr.io/nvidia/k8s-device-plugin", + "githubRepository": "NVIDIA/k8s-device-plugin", + "githubBranch": "main" + }, + "mig-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-mig-manager", + "githubRepository": "NVIDIA/mig-parted", + "githubBranch": "main" + }, + "driver-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-driver-manager", + "githubRepository": "NVIDIA/k8s-driver-manager", + "githubBranch": "main" + }, + "vgpu-device-manager": { + "imageRepository": "ghcr.io/nvidia/vgpu-device-manager", + "githubRepository": "NVIDIA/vgpu-device-manager", + "githubBranch": "main" + }, + "kata-sandbox-device-plugin": { + "imageRepository": "ghcr.io/nvidia/nvidia-sandbox-device-plugin", + "githubRepository": "NVIDIA/sandbox-device-plugin", + "githubBranch": "main" + }, + "kubevirt-gpu-device-plugin": { + "imageRepository": "ghcr.io/nvidia/kubevirt-gpu-device-plugin", + "githubRepository": "NVIDIA/kubevirt-gpu-device-plugin", + "githubBranch": "master" + }, + "cc-manager": { + "imageRepository": "ghcr.io/nvidia/k8s-cc-manager", + "githubRepository": "NVIDIA/k8s-cc-manager", + "githubBranch": "main", + "tagSuffix": "-distroless" + } +}' +VALID_COMPONENTS=$(jq -r 'keys_unsorted | join(" ")' <<< "${COMPONENT_SOURCES}") +readonly VALID_COMPONENTS +readonly MAX_RETRIES=5 -if [[ -z "${COMPONENT}" ]]; then - echo "Usage: $0 " >&2 +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [...]" >&2 + echo "Valid components: ${VALID_COMPONENTS}" >&2 exit 1 fi -# Verify regctl is available if ! command -v regctl &> /dev/null; then echo "Error: regctl not found. Please install regctl first." >&2 exit 1 fi -# Map component names to GHCR image repositories and GitHub source repositories -case "${COMPONENT}" in - toolkit) - IMAGE_REPO="ghcr.io/nvidia/container-toolkit" - GITHUB_REPO="NVIDIA/nvidia-container-toolkit" - ;; - device-plugin) - IMAGE_REPO="ghcr.io/nvidia/k8s-device-plugin" - GITHUB_REPO="NVIDIA/k8s-device-plugin" - ;; - mig-manager) - IMAGE_REPO="ghcr.io/nvidia/k8s-mig-manager" - GITHUB_REPO="NVIDIA/mig-parted" - ;; - *) - echo "Error: Unknown component '${COMPONENT}'" >&2 - echo "Valid components: toolkit, device-plugin, mig-manager" >&2 - exit 1 - ;; -esac - -echo "Fetching latest commit from ${GITHUB_REPO}..." >&2 - -# Get the latest commit SHA from the main branch using GitHub API. -# NOTE: We use 8-char truncated SHAs as image tags. This must match the -# tag convention used by each component's CI pipeline when publishing images. -GITHUB_API_URL="https://api.github.com/repos/${GITHUB_REPO}/commits/main" - -# Use GITHUB_TOKEN if available for authentication (higher rate limits) -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - LATEST_COMMIT=$(curl -sSL \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "Accept: application/vnd.github.v3+json" \ - "${GITHUB_API_URL}" | \ - jq -r '.sha[0:8]') -else - LATEST_COMMIT=$(curl -sSL \ - -H "Accept: application/vnd.github.v3+json" \ - "${GITHUB_API_URL}" | \ - jq -r '.sha[0:8]') -fi +get_latest_image() { + local component=$1 -if [[ -z "${LATEST_COMMIT}" || "${LATEST_COMMIT}" == "null" ]]; then - echo "Error: Failed to fetch latest commit from ${GITHUB_REPO}" >&2 - exit 1 -fi + local component_source + if ! component_source=$(jq -e --arg component "${component}" '.[$component]' <<< "${COMPONENT_SOURCES}"); then + echo "Error: Unknown component '${component}'" >&2 + echo "Valid components: ${VALID_COMPONENTS}" >&2 + return 1 + fi -echo "Latest commit SHA: ${LATEST_COMMIT}" >&2 + local image_repo github_repo github_branch tag_suffix + image_repo=$(jq -er '.imageRepository' <<< "${component_source}") + github_repo=$(jq -er '.githubRepository' <<< "${component_source}") + github_branch=$(jq -er '.githubBranch' <<< "${component_source}") + tag_suffix=$(jq -r '.tagSuffix // ""' <<< "${component_source}") -# Construct full image path with commit tag -FULL_IMAGE="${IMAGE_REPO}:${LATEST_COMMIT}" + echo "Fetching latest commit from ${github_repo}@${github_branch}..." >&2 -echo "Verifying image exists: ${FULL_IMAGE}" >&2 + # NOTE: We use 8-char truncated SHAs as image tags. This must match the + # tag convention used by each component's CI pipeline when publishing images. + local github_api_url="https://api.github.com/repos/${github_repo}/commits/${github_branch}" -# Verify the image exists using regctl with retry -MAX_RETRIES=5 -RETRY_DELAY=30 -for i in $(seq 1 ${MAX_RETRIES}); do - if regctl manifest head "${FULL_IMAGE}" &> /dev/null; then - echo "Verified ${COMPONENT} image: ${FULL_IMAGE}" >&2 - echo "${FULL_IMAGE}" - exit 0 + local latest_commit + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + latest_commit=$(curl -sSL \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_api_url}" | \ + jq -r '.sha[0:8]') + else + latest_commit=$(curl -sSL \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_api_url}" | \ + jq -r '.sha[0:8]') fi - if [[ $i -lt ${MAX_RETRIES} ]]; then - echo "Image not found (attempt $i/${MAX_RETRIES}), waiting ${RETRY_DELAY}s for CI to build..." >&2 - sleep ${RETRY_DELAY} - # Exponential backoff: 30s, 60s, 120s, 240s - RETRY_DELAY=$((RETRY_DELAY * 2)) + if [[ -z "${latest_commit}" || "${latest_commit}" == "null" ]]; then + echo "Error: Failed to fetch latest commit from ${github_repo}" >&2 + return 1 + fi + + echo "Latest commit SHA: ${latest_commit}" >&2 + + local full_image="${image_repo}:${latest_commit}${tag_suffix}" + + echo "Verifying image exists: ${full_image}" >&2 + + local retry_delay=30 + local i + for i in $(seq 1 ${MAX_RETRIES}); do + if regctl manifest head "${full_image}" &> /dev/null; then + echo "Verified ${component} image: ${full_image}" >&2 + echo "${full_image}" + return 0 + fi + + if [[ $i -lt ${MAX_RETRIES} ]]; then + echo "Image not found (attempt $i/${MAX_RETRIES}), waiting ${retry_delay}s for CI to build..." >&2 + sleep ${retry_delay} + retry_delay=$((retry_delay * 2)) + fi + done + + echo "Error: Image ${full_image} does not exist after ${MAX_RETRIES} attempts" >&2 + echo "The image may not have been built yet for commit ${latest_commit}" >&2 + return 1 +} + +components=("$@") +image_lookup_dir=$(mktemp -d) +trap 'rm -rf "${image_lookup_dir}"' EXIT + +lookup_pids=() +for index in "${!components[@]}"; do + component=${components[index]} + get_latest_image "${component}" \ + > "${image_lookup_dir}/${index}.image" \ + 2> >(sed -u "s/^/[${component}] /" >&2) & + lookup_pids+=($!) +done + +failed_components=() +for index in "${!components[@]}"; do + if ! wait "${lookup_pids[index]}"; then + failed_components+=("${components[index]}") fi done -echo "Error: Image ${FULL_IMAGE} does not exist after ${MAX_RETRIES} attempts" >&2 -echo "The image may not have been built yet for commit ${LATEST_COMMIT}" >&2 -exit 1 +if [[ ${#failed_components[@]} -gt 0 ]]; then + echo "::error::Image lookup failed for ${failed_components[*]}" >&2 + exit 1 +fi + +for index in "${!components[@]}"; do + echo "${components[index]}=$(< "${image_lookup_dir}/${index}.image")" +done diff --git a/.github/scripts/get-latest-main-build.sh b/.github/scripts/get-latest-main-build.sh new file mode 100755 index 0000000000..ed922cd168 --- /dev/null +++ b/.github/scripts/get-latest-main-build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +readonly MAIN_VERSION_PATTERN='^0\.0\.0-main-([0-9]{8})$' + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 MAIN_BUILD_REPOSITORY CHART_REPOSITORY MAX_AGE_DAYS" >&2 + exit 1 +fi + +main_build_repository="$1" +chart_repository="$2" +max_age_days="$3" + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +main_build_tags=$(regctl tag ls "${main_build_repository}") +main_version=$(grep -E "${MAIN_VERSION_PATTERN}" <<< "${main_build_tags}" | sort | tail -n 1 || true) + +if [[ -z "${main_version}" ]]; then + echo "Error: ${main_build_repository} has no main build" >&2 + exit 1 +fi + +[[ "${main_version}" =~ ${MAIN_VERSION_PATTERN} ]] +published_date="${BASH_REMATCH[1]}" + +oldest_accepted_date=$(date -u -d "${max_age_days} days ago" +%Y%m%d) +if (( published_date < oldest_accepted_date )); then + echo "Error: newest main build ${main_version} is older than ${max_age_days} day(s); the scheduled publish did not run or failed" >&2 + exit 1 +fi + +main_build_record=$(regctl artifact get "${main_build_repository}:${main_version}") +operator_version=$(jq -er '.operatorVersion' <<< "${main_build_record}") +recorded_chart_digest=$(jq -r '.chart.digest' <<< "${main_build_record}") +current_chart_digest=$(regctl image digest "${chart_repository}:${main_version}") +if [[ "${current_chart_digest}" != "${recorded_chart_digest}" ]]; then + echo "Error: ${chart_repository}:${main_version} is ${current_chart_digest}, but main build ${main_version} recorded ${recorded_chart_digest}" >&2 + exit 1 +fi + +echo "chart_version=${main_version}" +echo "operator_version=${operator_version}" diff --git a/.github/scripts/prepare-main-values.sh b/.github/scripts/prepare-main-values.sh new file mode 100755 index 0000000000..5443a2a2fc --- /dev/null +++ b/.github/scripts/prepare-main-values.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "Usage: $0 CHART_DIRECTORY VALUES_OVERRIDES_FILE OPERATOR_IMAGE OPERATOR_VERSION" >&2 + exit 1 +fi + +chart_directory="$1" +values_overrides_file="$2" +operator_image="$3" +operator_version="$4" + +script_directory=$(dirname "${BASH_SOURCE[0]}") + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +if ! command -v yq &> /dev/null; then + echo "Error: yq not found. Please install yq first." >&2 + exit 1 +fi + +if ! command -v helm &> /dev/null; then + echo "Error: helm not found. Please install helm first." >&2 + exit 1 +fi + +latest_image_lines=$("${script_directory}/get-latest-images.sh" \ + toolkit device-plugin mig-manager driver-manager vgpu-device-manager \ + kata-sandbox-device-plugin kubevirt-gpu-device-plugin cc-manager) + +declare -A latest_images +while IFS='=' read -r component image; do + latest_images[${component}]=${image} +done <<< "${latest_image_lines}" + +toolkit_image=${latest_images[toolkit]} +device_plugin_image=${latest_images[device-plugin]} +mig_manager_image=${latest_images[mig-manager]} +driver_manager_image=${latest_images[driver-manager]} +vgpu_device_manager_image=${latest_images[vgpu-device-manager]} +kata_sandbox_device_plugin_image=${latest_images[kata-sandbox-device-plugin]} +kubevirt_gpu_device_plugin_image=${latest_images[kubevirt-gpu-device-plugin]} +cc_manager_image=${latest_images[cc-manager]} + +"${script_directory}/generate-values-overrides.sh" "${values_overrides_file}" \ + "toolkit=${toolkit_image}" \ + "devicePlugin=${device_plugin_image}" \ + "gfd=${device_plugin_image}" \ + "migManager=${mig_manager_image}" \ + "driver.manager=${driver_manager_image}" \ + "vgpuManager.driverManager=${driver_manager_image}" \ + "vfioManager=${driver_manager_image}" \ + "vfioManager.driverManager=${driver_manager_image}" \ + "vgpuDeviceManager=${vgpu_device_manager_image}" \ + "kataSandboxDevicePlugin=${kata_sandbox_device_plugin_image}" \ + "sandboxDevicePlugin=${kubevirt_gpu_device_plugin_image}" \ + "ccManager=${cc_manager_image}" \ + "nodeStatusExporter=${operator_image}:${operator_version}" >&2 + +"${script_directory}/apply-values-overrides.sh" "${chart_directory}" "${values_overrides_file}" >&2 + +OPERATOR_REPOSITORY="${operator_image%/*}" \ +OPERATOR_IMAGE_NAME="${operator_image##*/}" \ +OPERATOR_VERSION="${operator_version}" \ +yq -i ' + (.operator, .validator) |= ( + .repository = strenv(OPERATOR_REPOSITORY) | + .image = strenv(OPERATOR_IMAGE_NAME) | + .version = strenv(OPERATOR_VERSION) + ) +' "${chart_directory}/values.yaml" + +printf '%s\n' \ + "${toolkit_image}" \ + "${device_plugin_image}" \ + "${mig_manager_image}" \ + "${driver_manager_image}" \ + "${vgpu_device_manager_image}" \ + "${kata_sandbox_device_plugin_image}" \ + "${kubevirt_gpu_device_plugin_image}" \ + "${cc_manager_image}" diff --git a/.github/scripts/push-main-build-record.sh b/.github/scripts/push-main-build-record.sh new file mode 100755 index 0000000000..d85e95c1c3 --- /dev/null +++ b/.github/scripts/push-main-build-record.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -lt 8 ]]; then + echo "Usage: $0 MAIN_BUILD_REPOSITORY MAIN_VERSION OPERATOR_IMAGE OPERATOR_VERSION CHART_REFERENCE CHART_VERSION BUNDLE_IMAGE OPERAND_IMAGE..." >&2 + exit 1 +fi + +main_build_repository="$1" +main_version="$2" +operator_image="$3" +operator_version="$4" +chart_reference="$5" +chart_version="$6" +bundle_image="$7" +operand_images=("${@:8}") + +source_commit="${GITHUB_SHA}" +run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + +if ! command -v regctl &> /dev/null; then + echo "Error: regctl not found. Please install regctl first." >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: jq not found. Please install jq first." >&2 + exit 1 +fi + +main_build_record_reference="${main_build_repository}:${main_version}" +tagged_chart_reference="${chart_reference#oci://}:${chart_version}" + +recorded_images=$(printf '%s\n' "${operator_image}:${operator_version}" "${operand_images[@]}" | sort -u) + +image_records="[]" +while read -r image; do + image_digest=$(regctl image digest "${image}") + image_records=$(jq -c --arg reference "${image}" --arg digest "${image_digest}" '. + [{$reference, $digest}]' <<< "${image_records}") +done <<< "${recorded_images}" + +chart_digest=$(regctl image digest "${tagged_chart_reference}") +bundle_digest=$(regctl image digest "${bundle_image}") + +main_build_record=$(jq -n \ + --arg mainVersion "${main_version}" \ + --arg operatorVersion "${operator_version}" \ + --arg sourceCommit "${source_commit}" \ + --arg runUrl "${run_url}" \ + --arg chartReference "${tagged_chart_reference}" \ + --arg chartDigest "${chart_digest}" \ + --arg bundleReference "${bundle_image}" \ + --arg bundleDigest "${bundle_digest}" \ + --argjson images "${image_records}" \ + '{ + $mainVersion, + $operatorVersion, + $sourceCommit, + $runUrl, + chart: {reference: $chartReference, digest: $chartDigest}, + bundle: {reference: $bundleReference, digest: $bundleDigest}, + $images + }') + +regctl artifact put \ + --artifact-type application/vnd.nvidia.gpu-operator.main-build.v1+json \ + --file-media-type application/json \ + "${main_build_record_reference}" <<< "${main_build_record}" +echo "::notice::Recorded main build ${main_build_record_reference}" diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 0e452648a8..3a389fdeed 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -23,11 +23,16 @@ on: operator_version: required: true type: string - use_values_override: + chart_reference: + description: 'Published Helm chart to test instead of publishing one from this commit (set chart_version, and operator_version to its appVersion)' required: false - type: boolean - default: false - description: 'Use values-overrides artifact for component image configuration' + type: string + default: '' + chart_version: + description: 'Version of chart_reference' + required: false + type: string + default: '' secrets: AWS_ACCESS_KEY_ID: required: true @@ -49,11 +54,16 @@ on: description: 'Operator version to test (override)' required: false type: string - use_values_override: - description: 'Use values-overrides artifact for component image configuration' + chart_reference: + description: 'Published Helm chart to test instead of publishing one from this commit (set chart_version, and operator_version to its appVersion)' required: false - type: boolean - default: false + type: string + default: '' + chart_version: + description: 'Version of chart_reference' + required: false + type: string + default: '' permissions: contents: read @@ -67,6 +77,7 @@ jobs: publish-helm-oci-chart: needs: [variables] + if: ${{ !inputs.chart_reference }} permissions: contents: read packages: write @@ -77,29 +88,43 @@ jobs: e2e-tests-containerd: needs: [variables, publish-helm-oci-chart] + if: ${{ !cancelled() && needs.variables.result == 'success' && (needs.publish-helm-oci-chart.result == 'success' || needs.publish-helm-oci-chart.result == 'skipped') }} runs-on: linux-amd64-cpu4 timeout-minutes: 90 permissions: contents: read id-token: write + env: + HELM_CHART: ${{ inputs.chart_reference || needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ inputs.chart_reference && inputs.chart_version || needs.publish-helm-oci-chart.outputs.chart_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 name: Check out code with: persist-credentials: false - - name: Download values override file - if: ${{ inputs.use_values_override }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: values-overrides - path: ${{ github.workspace }} - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Verify the published Helm chart is available env: - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} - run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" + INPUT_CHART_REFERENCE: ${{ inputs.chart_reference }} + INPUT_CHART_VERSION: ${{ inputs.chart_version }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if [[ -n "${INPUT_CHART_VERSION}" && -z "${INPUT_CHART_REFERENCE}" ]]; then + echo "::error::chart_version requires chart_reference" + exit 1 + fi + if [[ -z "${HELM_CHART_VERSION}" ]]; then + echo "::error::chart_reference requires chart_version" + exit 1 + fi + chart_metadata=$(helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}") + echo "${chart_metadata}" + chart_app_version=$(awk '/^appVersion:/ {print $2}' <<< "${chart_metadata}" | tr -d '"') + if [[ -n "${INPUT_CHART_REFERENCE}" && "${chart_app_version}" != "${OPERATOR_VERSION}" ]]; then + echo "::error::operator_version ${OPERATOR_VERSION} does not match the chart's appVersion ${chart_app_version}" + exit 1 + fi - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -123,19 +148,10 @@ jobs: AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} run: | echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key} - - name: Copy values override file to remote - if: ${{ inputs.use_values_override }} - run: | - scp -i ${private_key} -o StrictHostKeyChecking=no \ - ${{ github.workspace }}/values-overrides.yaml \ - ${instance_hostname}:/tmp/values-overrides.yaml - echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV - name: Run e2e tests env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" @@ -154,29 +170,43 @@ jobs: e2e-tests-nvidiadriver: needs: [variables, publish-helm-oci-chart] + if: ${{ !cancelled() && needs.variables.result == 'success' && (needs.publish-helm-oci-chart.result == 'success' || needs.publish-helm-oci-chart.result == 'skipped') }} runs-on: linux-amd64-cpu4 timeout-minutes: 90 permissions: contents: read id-token: write + env: + HELM_CHART: ${{ inputs.chart_reference || needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ inputs.chart_reference && inputs.chart_version || needs.publish-helm-oci-chart.outputs.chart_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 name: Check out code with: persist-credentials: false - - name: Download values override file - if: ${{ inputs.use_values_override }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: values-overrides - path: ${{ github.workspace }} - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Verify the published Helm chart is available env: - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} - run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" + INPUT_CHART_REFERENCE: ${{ inputs.chart_reference }} + INPUT_CHART_VERSION: ${{ inputs.chart_version }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if [[ -n "${INPUT_CHART_VERSION}" && -z "${INPUT_CHART_REFERENCE}" ]]; then + echo "::error::chart_version requires chart_reference" + exit 1 + fi + if [[ -z "${HELM_CHART_VERSION}" ]]; then + echo "::error::chart_reference requires chart_version" + exit 1 + fi + chart_metadata=$(helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}") + echo "${chart_metadata}" + chart_app_version=$(awk '/^appVersion:/ {print $2}' <<< "${chart_metadata}" | tr -d '"') + if [[ -n "${INPUT_CHART_REFERENCE}" && "${chart_app_version}" != "${OPERATOR_VERSION}" ]]; then + echo "::error::operator_version ${OPERATOR_VERSION} does not match the chart's appVersion ${chart_app_version}" + exit 1 + fi - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -200,19 +230,10 @@ jobs: AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} run: | echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key} - - name: Copy values override file to remote - if: ${{ inputs.use_values_override }} - run: | - scp -i ${private_key} -o StrictHostKeyChecking=no \ - ${{ github.workspace }}/values-overrides.yaml \ - ${instance_hostname}:/tmp/values-overrides.yaml - echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV - name: Run e2e tests env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} - HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} - HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" diff --git a/.github/workflows/forward-compatibility.yaml b/.github/workflows/forward-compatibility.yaml index 33807d4937..04f5c0be6c 100644 --- a/.github/workflows/forward-compatibility.yaml +++ b/.github/workflows/forward-compatibility.yaml @@ -16,8 +16,8 @@ name: Forward Compatibility on: schedule: - - cron: '0 2 * * 1' # Weekly on Monday at 2 AM UTC - workflow_dispatch: # Manual trigger + - cron: '17 4 * * 1' + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -25,12 +25,26 @@ concurrency: permissions: {} +env: + MAIN_BUILD_MAX_AGE_DAYS: 1 + jobs: - fetch-latest-images: + variables: + permissions: + contents: read + uses: ./.github/workflows/variables.yaml + + find-main-chart: + needs: [variables] runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 10 permissions: contents: read + packages: read + outputs: + chart_reference: ${{ steps.main-chart.outputs.chart_reference }} + chart_version: ${{ steps.main-chart.outputs.chart_version }} + operator_version: ${{ steps.main-chart.outputs.operator_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -46,45 +60,38 @@ jobs: with: release: ${{ env.REGCTL_VERSION }} - - name: Get latest component images and generate values override file + - name: Login to GitHub Container Registry env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Fetch latest images from component repositories - echo "::notice::Fetching latest container-toolkit image..." - TOOLKIT=$(.github/scripts/get-latest-images.sh toolkit) - - echo "::notice::Fetching latest device-plugin image..." - DEVICE_PLUGIN=$(.github/scripts/get-latest-images.sh device-plugin) + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin - echo "::notice::Fetching latest mig-manager image..." - MIG_MANAGER=$(.github/scripts/get-latest-images.sh mig-manager) + - name: Find the chart of the newest complete main build + id: main-chart + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + run: | + set -euo pipefail - # Generate values override file - .github/scripts/generate-values-overrides.sh \ - values-overrides.yaml \ - "${TOOLKIT}" \ - "${DEVICE_PLUGIN}" \ - "${MIG_MANAGER}" + main_build_repository="${OPERATOR_IMAGE_BASE}/main-builds" + chart_repository="${OPERATOR_IMAGE_BASE}/charts/gpu-operator" - - name: Upload values override file - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: values-overrides - path: values-overrides.yaml - retention-days: 30 + { + .github/scripts/get-latest-main-build.sh "${main_build_repository}" "${chart_repository}" "${MAIN_BUILD_MAX_AGE_DAYS}" + echo "chart_reference=oci://${chart_repository}" + } >> "${GITHUB_OUTPUT}" run-e2e-tests: - needs: [fetch-latest-images] + needs: [variables, find-main-chart] permissions: contents: read id-token: write packages: write uses: ./.github/workflows/e2e-tests.yaml with: - operator_image: ghcr.io/nvidia/gpu-operator - operator_version: main-latest - use_values_override: true + operator_image: ${{ needs.variables.outputs.operator_image_base }} + operator_version: ${{ needs.find-main-chart.outputs.operator_version }} + chart_reference: ${{ needs.find-main-chart.outputs.chart_reference }} + chart_version: ${{ needs.find-main-chart.outputs.chart_version }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -96,11 +103,23 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 permissions: {} - needs: [fetch-latest-images, run-e2e-tests] - if: ${{ always() && (needs.fetch-latest-images.result == 'failure' || needs.run-e2e-tests.result == 'failure') }} + needs: [variables, find-main-chart, run-e2e-tests] + if: ${{ always() && (needs.variables.result == 'failure' || needs.find-main-chart.result == 'failure' || needs.run-e2e-tests.result == 'failure') }} env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} steps: + - name: Describe the tested chart + id: tested-chart + env: + CHART_REFERENCE: ${{ needs.find-main-chart.outputs.chart_reference }} + CHART_VERSION: ${{ needs.find-main-chart.outputs.chart_version }} + run: | + if [[ -n "${CHART_VERSION}" ]]; then + echo "description=\`${CHART_REFERENCE} --version ${CHART_VERSION}\` (run \`helm show values\` on it to see the component images)" >> "${GITHUB_OUTPUT}" + else + echo "description=No complete main build from the last ${MAIN_BUILD_MAX_AGE_DAYS} day(s) (none recorded, its chart changed since, or the registry could not be read); see the run log" >> "${GITHUB_OUTPUT}" + fi + - name: Send Slack alert notification if: ${{ env.SLACK_BOT_TOKEN != '' }} uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 @@ -110,5 +129,5 @@ jobs: payload: | { "channel": "${{ secrets.SLACK_CHANNEL_ID }}", - "text": ":x: *Forward Compatibility Test Failed for GPU Operator*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Tested Components:*\nDownload `values-overrides` artifact to see tested component versions\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" + "text": ":x: *Forward Compatibility Test Failed for GPU Operator*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Tested Chart:* ${{ steps.tested-chart.outputs.description }}\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" } diff --git a/.github/workflows/publish-helm-oci-chart.yaml b/.github/workflows/publish-helm-oci-chart.yaml index de65f4141e..53a90ba6c5 100644 --- a/.github/workflows/publish-helm-oci-chart.yaml +++ b/.github/workflows/publish-helm-oci-chart.yaml @@ -23,6 +23,11 @@ on: operator_version: required: true type: string + main_chart_version: + required: false + type: string + default: '' + description: "Package the chart with the merged values.yaml from the main-values artifact and publish it with this 0.0.0-main version instead of the per-commit one" outputs: chart_reference: description: "The OCI reference of the published Helm chart" @@ -57,6 +62,13 @@ jobs: - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Download merged chart values + if: ${{ inputs.main_chart_version != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: main-values + path: ${{ github.workspace }} + - name: Set chart image repository env: OPERATOR_IMAGE: ${{ inputs.operator_image }} @@ -76,12 +88,17 @@ jobs: id: package env: OPERATOR_VERSION: ${{ inputs.operator_version }} + MAIN_CHART_VERSION: ${{ inputs.main_chart_version }} run: | set -euo pipefail - # Helm chart versions must be SemVer. Prefix the image version so a - # numeric short SHA with a leading zero remains a valid prerelease. - CHART_VERSION="0.0.0-git-${OPERATOR_VERSION}" + if [[ -n "${MAIN_CHART_VERSION}" ]]; then + CHART_VERSION="${MAIN_CHART_VERSION}" + else + # Helm chart versions must be SemVer. Prefix the image version so a + # numeric short SHA with a leading zero remains a valid prerelease. + CHART_VERSION="0.0.0-git-${OPERATOR_VERSION}" + fi mkdir -p dist helm lint deployments/gpu-operator diff --git a/.github/workflows/publish-main-chart.yaml b/.github/workflows/publish-main-chart.yaml new file mode 100644 index 0000000000..4ccf9c14a9 --- /dev/null +++ b/.github/workflows/publish-main-chart.yaml @@ -0,0 +1,300 @@ +# Copyright NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Publish Main Chart and Bundle + +on: + schedule: + - cron: '17 2 * * *' + workflow_dispatch: + inputs: + overwrite: + description: "Replace today's main build if it is already published" + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: {} + +jobs: + variables: + permissions: + contents: read + uses: ./.github/workflows/variables.yaml + + main-version: + needs: [variables] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: read + outputs: + main_version: ${{ steps.main-version.outputs.main_version }} + publish: ${{ steps.main-version.outputs.publish }} + steps: + - name: Require the default branch + if: ${{ github.event_name == 'workflow_dispatch' && github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + echo "::error::Main charts are published only from ${DEFAULT_BRANCH}" + exit 1 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Login to GitHub Container Registry + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin + + - name: Set main version and check whether it is published + id: main-version + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + OVERWRITE: ${{ inputs.overwrite }} + run: | + set -euo pipefail + + main_version="0.0.0-main-$(date -u +%Y%m%d)" + echo "main_version=${main_version}" >> "${GITHUB_OUTPUT}" + + if record_lookup_error=$(regctl manifest head "${OPERATOR_IMAGE_BASE}/main-builds:${main_version}" 2>&1 > /dev/null); then + if [[ "${OVERWRITE}" != "true" ]]; then + echo "::notice::Main build ${main_version} is already published; run this workflow with overwrite to replace it" + echo "publish=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + elif [[ "${record_lookup_error}" != *"[http 404]"* ]]; then + echo "::error::Could not check whether main build ${main_version} is published: ${record_lookup_error}" + exit 1 + fi + echo "publish=true" >> "${GITHUB_OUTPUT}" + + prepare-main-values: + needs: [variables, main-version] + if: ${{ needs.main-version.outputs.publish == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + resolved_images: ${{ steps.images.outputs.resolved_images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Verify the operator image exists + env: + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + if ! regctl manifest head "${OPERATOR_IMAGE}:${OPERATOR_VERSION}" > /dev/null; then + echo "::error::${OPERATOR_IMAGE}:${OPERATOR_VERSION} is not built yet; rerun once CI has published it" + exit 1 + fi + + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Install PyYAML + run: pip install --quiet pyyaml + + - name: Prepare main chart values + id: images + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + run: | + set -euo pipefail + + resolved_image_lines=$(.github/scripts/prepare-main-values.sh \ + deployments/gpu-operator values-overrides.yaml "${OPERATOR_IMAGE}" "${OPERATOR_VERSION}") + echo "resolved_images=${resolved_image_lines//$'\n'/ }" >> "${GITHUB_OUTPUT}" + + - name: Upload values override file and merged chart values + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: main-values + path: | + values-overrides.yaml + deployments/gpu-operator/values.yaml + retention-days: 30 + + publish-main-chart: + needs: [variables, main-version, prepare-main-values] + permissions: + contents: read + packages: write + uses: ./.github/workflows/publish-helm-oci-chart.yaml + with: + operator_image: ${{ needs.variables.outputs.operator_image }} + operator_version: ${{ needs.variables.outputs.operator_version }} + main_chart_version: ${{ needs.main-version.outputs.main_version }} + + publish-main-bundle: + needs: [variables, main-version, prepare-main-values] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + outputs: + bundle_image: ${{ steps.bundle.outputs.bundle-image }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up OLM bundle tools + uses: ./.github/actions/setup-olm-bundle-tools + + - name: Download values override file and merged chart values + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: main-values + path: ${{ github.workspace }} + + - name: Update OLM bundle images + run: | + python3 .github/scripts/update-csv-images.py \ + --values deployments/gpu-operator/values.yaml \ + --chart deployments/gpu-operator/Chart.yaml \ + --csv bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + + - name: Check the CSV related images include every override image + env: + CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml + run: | + set -euo pipefail + + override_images=$(yq '.. | select(tag == "!!map" and has("image")) | .repository + "/" + .image + ":" + .version' values-overrides.yaml | sort -u) + csv_related_images=$(yq '.spec.relatedImages[].image | sub("@sha256:[0-9a-f]+$", "")' "${CSV}" | sort -u) + missing_images=$(comm -23 <(echo "${override_images}") <(echo "${csv_related_images}")) + if [[ -n "${missing_images}" ]]; then + echo "::error::CSV relatedImages are missing override images:" + echo "${missing_images}" + exit 1 + fi + + - name: Validate and push OLM bundle + id: bundle + uses: ./.github/actions/validate-and-push-olm-bundle + with: + bundle-image-tag: ${{ needs.main-version.outputs.main_version }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + record-main-build: + needs: [variables, main-version, prepare-main-values, publish-main-chart, publish-main-bundle] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Get regctl version + run: | + REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) + echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" + + - name: Install regctl + uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 + with: + release: ${{ env.REGCTL_VERSION }} + + - name: Login to GitHub Container Registry + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "${GH_TOKEN}" | regctl registry login ghcr.io -u "${GITHUB_ACTOR}" --pass-stdin + + - name: Push the main build record + env: + OPERATOR_IMAGE_BASE: ${{ needs.variables.outputs.operator_image_base }} + MAIN_VERSION: ${{ needs.main-version.outputs.main_version }} + OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} + CHART_REFERENCE: ${{ needs.publish-main-chart.outputs.chart_reference }} + CHART_VERSION: ${{ needs.publish-main-chart.outputs.chart_version }} + BUNDLE_IMAGE: ${{ needs.publish-main-bundle.outputs.bundle_image }} + RESOLVED_IMAGES: ${{ needs.prepare-main-values.outputs.resolved_images }} + run: | + set -euo pipefail + + read -ra operand_images <<< "${RESOLVED_IMAGES}" + .github/scripts/push-main-build-record.sh \ + "${OPERATOR_IMAGE_BASE}/main-builds" \ + "${MAIN_VERSION}" \ + "${OPERATOR_IMAGE}" \ + "${OPERATOR_VERSION}" \ + "${CHART_REFERENCE}" \ + "${CHART_VERSION}" \ + "${BUNDLE_IMAGE}" \ + "${operand_images[@]}" + + notify-failure: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: {} + needs: [variables, main-version, prepare-main-values, publish-main-chart, publish-main-bundle, record-main-build] + if: ${{ always() && github.event_name == 'schedule' && (needs.variables.result == 'failure' || needs.main-version.result == 'failure' || needs.prepare-main-values.result == 'failure' || needs.publish-main-chart.result == 'failure' || needs.publish-main-bundle.result == 'failure' || needs.record-main-build.result == 'failure') }} + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + steps: + - name: Send Slack alert notification + if: ${{ env.SLACK_BOT_TOKEN != '' }} + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": ":x: *GPU Operator Main Build Publish Failed*\n\n*Workflow:* ${{ github.workflow }}\n*Repository:* ${{ github.repository }}\n*Trigger:* ${{ github.event_name }}\n\n*Resolved Images:* ${{ needs.prepare-main-values.outputs.resolved_images || 'not resolved; see the run log' }}\n\n*Details:* \n\n${{ secrets.SLACK_MENTION_LIST }}" + } diff --git a/.github/workflows/release-rc-assets.yaml b/.github/workflows/release-rc-assets.yaml index 7a91ec875f..5d7d7dfe30 100644 --- a/.github/workflows/release-rc-assets.yaml +++ b/.github/workflows/release-rc-assets.yaml @@ -121,49 +121,8 @@ jobs: - name: Set up Helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Get Golang version - run: | - GOLANG_VERSION=$(grep "GOLANG_VERSION ?=" versions.mk) - echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> "${GITHUB_ENV}" - - - name: Get regctl version - run: | - REGCTL_VERSION=$(grep "REGCTL_VERSION ?=" versions.mk) - echo "REGCTL_VERSION=${REGCTL_VERSION##REGCTL_VERSION ?= }" >> "${GITHUB_ENV}" - - - name: Install Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version: ${{ env.GOLANG_VERSION }} - - - name: Install PyYAML - run: pip install --quiet pyyaml - - - name: Install regctl - uses: regclient/actions/regctl-installer@148669fe4b19151fcab6e00c6df2db43b9e2b097 - with: - release: ${{ env.REGCTL_VERSION }} - - - name: Install Operator SDK - run: | - set -euo pipefail - - OPERATOR_SDK_VERSION="$(awk -F': ' '/operators.operatorframework.io.metrics.builder:/ { print $2 }' bundle/metadata/annotations.yaml | sed 's/^operator-sdk-//')" - if [[ -z "${OPERATOR_SDK_VERSION}" ]]; then - echo "::error::OPERATOR_SDK_VERSION not found in bundle metadata" - exit 1 - fi - - curl -sSLo operator-sdk \ - "https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64" - chmod +x operator-sdk - sudo mv operator-sdk /usr/local/bin/operator-sdk - operator-sdk version + - name: Set up OLM bundle tools + uses: ./.github/actions/setup-olm-bundle-tools - name: Login to GitHub Container Registry env: @@ -175,16 +134,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "${GH_TOKEN}" | helm registry login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - - name: Login to GitHub Container Registry for Docker - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - name: Apply Helm values override if: ${{ steps.package-inputs.outputs.helm_values_override != '' }} env: @@ -210,32 +159,12 @@ jobs: --chart deployments/gpu-operator/Chart.yaml \ --csv bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml - - name: Validate updated OLM bundle - env: - RELEASE_TAG: ${{ steps.validate.outputs.release_tag }} - CSV: bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml - run: | - set -euo pipefail - - python3 .github/scripts/validate-olm-release-metadata.py \ - --release-tag "${RELEASE_TAG}" \ - --csv "${CSV}" - - operator-sdk bundle validate ./bundle - make validate-csv - - - name: Build and push OLM bundle image - env: - RELEASE_TAG: ${{ steps.validate.outputs.release_tag }} - GH_REPOSITORY: ${{ github.repository }} - VERSION: "" - DEFAULT_CHANNEL: "stable" - CHANNELS: "stable" - BUNDLE_IMAGE_BUILD_PLATFORM_OPTIONS: "--platform=linux/amd64,linux/arm64 --push" - run: | - REPOSITORY="$(echo "${GH_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" - BUNDLE_IMAGE_BASE="ghcr.io/${REPOSITORY}/gpu-operator-bundle" - make build-bundle-image DOCKER="docker buildx" BUNDLE_IMAGE="${BUNDLE_IMAGE_BASE}:${RELEASE_TAG}" + - name: Validate and push OLM bundle + uses: ./.github/actions/validate-and-push-olm-bundle + with: + release-tag: ${{ steps.validate.outputs.release_tag }} + bundle-image-tag: ${{ steps.validate.outputs.release_tag }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Package RC assets id: package diff --git a/versions.mk b/versions.mk index dc4f765605..51b28fa009 100644 --- a/versions.mk +++ b/versions.mk @@ -25,5 +25,7 @@ GOLANGCI_LINT_VERSION ?= v2.14.0 REGCTL_VERSION ?= v0.11.5 +OPERATOR_SDK_VERSION ?= v1.4.0 + GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always 2> /dev/null || echo "") GIT_COMMIT_SHA_FULL ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")