24.3 Stable: use the clickhouse/minio-minio mirror image

docker/test/stateless/Dockerfile

@@ -1,10 +1,13 @@
-# rebuild in #33610
+# rebuild in #2701
 # docker build -t altinityinfra/stateless-test .
 ARG FROM_TAG=latest
 FROM altinityinfra/test-base:$FROM_TAG
 
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
 
+# The test-base image carries an outdated Kitware apt key
+RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg
+
 # golang version 1.13 on Ubuntu 20 is enough for tests
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \
@@ -67,17 +70,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 ENV NUM_TRIES=1
 ENV MAX_RUN_TIME=0
 
-# Unrelated to vars in setup_minio.sh, but should be the same there
-# to have the same binaries for local running scenario
-ARG MINIO_SERVER_VERSION=2022-01-03T18-22-58Z
-ARG MINIO_CLIENT_VERSION=2022-01-05T23-52-51Z
-ARG TARGETARCH
-
-# Download Minio-related binaries
-RUN arch=${TARGETARCH:-amd64} \
-    && curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o ./minio \
-    && curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \
-    && chmod +x ./mc ./minio
+# MinIO no longer publishes binaries on dl.min.io, take them from the mirror image
+COPY --from=clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z /usr/bin/minio /usr/bin/mc ./
 
 RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
     && tar -xvf hadoop-3.3.1.tar.gz \
@@ -87,8 +81,10 @@ ENV MINIO_ROOT_USER="clickhouse"
 ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 
+# Current azurite dependencies need a newer Node.js than the Ubuntu one
 RUN npm install -g azurite@3.28.0 \
-    && npm install -g tslib@2.6.2
+    && npm install -g tslib@2.6.2 \
+    && npm install -g node@20.20.2
 
 COPY run.sh /
 COPY setup_minio.sh /

docker/test/stateless/setup_minio.sh

@@ -82,10 +82,10 @@ setup_minio() {
   local test_type=$1
   ./mc alias set clickminio http://localhost:11111 clickhouse clickhouse
   ./mc admin user add clickminio test testtest
-  ./mc admin policy set clickminio readwrite user=test
+  ./mc admin policy attach clickminio readwrite --user=test
   ./mc mb clickminio/test
   if [ "$test_type" = "stateless" ]; then
-    ./mc policy set public clickminio/test
+    ./mc anonymous set public clickminio/test
   fi
 }
 

docker/test/stress/Dockerfile

@@ -1,7 +1,7 @@
 # rebuild in #33610
 # docker build -t altinityinfra/stress-test .
 ARG FROM_TAG=latest
-FROM altinityinfra/stateful-test:ce3c0a6ed901
+FROM altinityinfra/stateful-test:$FROM_TAG
 
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -281,7 +281,8 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet
 
     S3::ListObjectsV2Request request;
     request.SetBucket(uri.bucket);
-    request.SetPrefix(path);
+    if (path != "/")
+        request.SetPrefix(path);
     if (max_keys)
         request.SetMaxKeys(max_keys);
     else

tests/ci/build_report_check.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import json
 import logging
 import os
 import sys
@@ -8,6 +9,7 @@
 
 from ci_config import CI_CONFIG, Build
 from env_helper import (
+    CI_CONFIG_PATH,
     GITHUB_JOB_URL,
     GITHUB_REPOSITORY,
     GITHUB_SERVER_URL,
@@ -56,6 +58,12 @@ def main():
     required_builds = len(builds_for_check)
     missing_builds = 0
 
+    ci_jobs = None
+    if Path(CI_CONFIG_PATH).is_file():
+        with open(CI_CONFIG_PATH, encoding="utf-8") as jfd:
+            jobs_data = json.load(jfd)["jobs_data"]
+        ci_jobs = jobs_data["jobs_to_do"] + jobs_data["jobs_to_skip"]
+
     # Collect reports from json artifacts
     build_results = []  # type: List[BuildResult]
     for build_name in builds_for_check:
@@ -66,6 +74,11 @@ def main():
             if build_name == Build.FUZZERS:
                 logging.info("Build [%s] is missing - skip", Build.FUZZERS)
                 continue
+            if ci_jobs is not None and build_name not in ci_jobs:
+                # e.g. excluded with a ci_exclude_ tag
+                logging.info("Build [%s] is not in this CI run - skip", build_name)
+                required_builds -= 1
+                continue
             logging.warning("Build results for %s is missing", build_name)
             build_result = BuildResult.missing_result("missing")
             build_result.job_name = build_name

tests/ci/ci.py

@@ -40,6 +40,7 @@
 from digest_helper import DockerDigester, JobDigester
 from env_helper import (
     CI,
+    CI_CONFIG_PATH,
     GITHUB_JOB_API_URL,
     GITHUB_RUN_URL,
     REPO_COPY,
@@ -2084,6 +2085,10 @@ def main() -> int:
             else:
                 exit_code = 1
         else:
+            # Build reports use it to know which builds are part of this CI run
+            Path(TEMP_PATH).mkdir(parents=True, exist_ok=True)
+            with open(CI_CONFIG_PATH, "w", encoding="utf-8") as jfd:
+                json.dump(indata, jfd)
             exit_code = _run_test(check_name, args.run_command)
     ### RUN action: end
 

tests/ci/docker_images_check.py

@@ -61,6 +61,8 @@ def build_and_push_one_image(
 
     cmd = (
         "docker buildx build --builder default "
+        # A single image is required by `docker manifest create`, not an index with an attestation
+        "--provenance=false "
         f"--label build-url={GITHUB_RUN_URL} "
         f"{from_tag_arg}"
         # A hack to invalidate cache, grep for it in docker/ dir

tests/ci/env_helper.py

@@ -13,6 +13,7 @@
 CI = bool(os.getenv("CI"))
 TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp")))
 REPORT_PATH = f"{TEMP_PATH}/reports"
+CI_CONFIG_PATH = f"{TEMP_PATH}/ci_config.json"
 # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag"
 DOCKER_TAG = os.getenv("DOCKER_TAG", "latest")
 DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD")

tests/integration/compose/docker_compose_ldap.yml

@@ -1,7 +1,7 @@
 version: '2.3'
 services:
     openldap:
-        image: bitnami/openldap:2.6.6
+        image: bitnamilegacy/openldap:2.6.6
         restart: always
         environment:
             LDAP_ROOT: dc=example,dc=org

tests/integration/compose/docker_compose_minio.yml

@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2023-09-30T07-02-29Z
+    image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z
     volumes:
       - data1-1:/data1
       - ${MINIO_CERTS_DIR:-}:/certs