diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 72a64fa635c00..57a391e3aefc2 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,10 +1,13 @@ -# rebuild in #33610 +# rebuild in #2701 # docker build -t altinityinfra/stateless-test . ARG FROM_TAG=latest FROM altinityinfra/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" +# The test-base image carries an outdated Kitware apt key +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg + # golang version 1.13 on Ubuntu 20 is enough for tests RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ @@ -67,17 +70,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV NUM_TRIES=1 ENV MAX_RUN_TIME=0 -# Unrelated to vars in setup_minio.sh, but should be the same there -# to have the same binaries for local running scenario -ARG MINIO_SERVER_VERSION=2022-01-03T18-22-58Z -ARG MINIO_CLIENT_VERSION=2022-01-05T23-52-51Z -ARG TARGETARCH - -# Download Minio-related binaries -RUN arch=${TARGETARCH:-amd64} \ - && curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o ./minio \ - && curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \ - && chmod +x ./mc ./minio +# MinIO no longer publishes binaries on dl.min.io, take them from the mirror image +COPY --from=clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z /usr/bin/minio /usr/bin/mc ./ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \ && tar -xvf hadoop-3.3.1.tar.gz \ @@ -87,8 +81,10 @@ ENV MINIO_ROOT_USER="clickhouse" ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 +# Current azurite dependencies need a newer Node.js than the Ubuntu one RUN npm install -g azurite@3.28.0 \ - && npm install -g tslib@2.6.2 + && npm install -g tslib@2.6.2 \ + && npm install -g node@20.20.2 COPY run.sh / COPY setup_minio.sh / diff --git a/docker/test/stateless/setup_minio.sh b/docker/test/stateless/setup_minio.sh index c756ce4669dad..99ce22b5bd033 100755 --- a/docker/test/stateless/setup_minio.sh +++ b/docker/test/stateless/setup_minio.sh @@ -82,10 +82,10 @@ setup_minio() { local test_type=$1 ./mc alias set clickminio http://localhost:11111 clickhouse clickhouse ./mc admin user add clickminio test testtest - ./mc admin policy set clickminio readwrite user=test + ./mc admin policy attach clickminio readwrite --user=test ./mc mb clickminio/test if [ "$test_type" = "stateless" ]; then - ./mc policy set public clickminio/test + ./mc anonymous set public clickminio/test fi } diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 359a05b207135..507bf90bc2dde 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 # docker build -t altinityinfra/stress-test . ARG FROM_TAG=latest -FROM altinityinfra/stateful-test:ce3c0a6ed901 +FROM altinityinfra/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 7307e0d03efd4..dfb0b92a2d591 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -281,7 +281,8 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet S3::ListObjectsV2Request request; request.SetBucket(uri.bucket); - request.SetPrefix(path); + if (path != "/") + request.SetPrefix(path); if (max_keys) request.SetMaxKeys(max_keys); else diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 971be978fca3f..e006cc68e75c6 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import logging import os import sys @@ -8,6 +9,7 @@ from ci_config import CI_CONFIG, Build from env_helper import ( + CI_CONFIG_PATH, GITHUB_JOB_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL, @@ -56,6 +58,12 @@ def main(): required_builds = len(builds_for_check) missing_builds = 0 + ci_jobs = None + if Path(CI_CONFIG_PATH).is_file(): + with open(CI_CONFIG_PATH, encoding="utf-8") as jfd: + jobs_data = json.load(jfd)["jobs_data"] + ci_jobs = jobs_data["jobs_to_do"] + jobs_data["jobs_to_skip"] + # Collect reports from json artifacts build_results = [] # type: List[BuildResult] for build_name in builds_for_check: @@ -66,6 +74,11 @@ def main(): if build_name == Build.FUZZERS: logging.info("Build [%s] is missing - skip", Build.FUZZERS) continue + if ci_jobs is not None and build_name not in ci_jobs: + # e.g. excluded with a ci_exclude_ tag + logging.info("Build [%s] is not in this CI run - skip", build_name) + required_builds -= 1 + continue logging.warning("Build results for %s is missing", build_name) build_result = BuildResult.missing_result("missing") build_result.job_name = build_name diff --git a/tests/ci/ci.py b/tests/ci/ci.py index ea0c87047c059..d23527d504a39 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -40,6 +40,7 @@ from digest_helper import DockerDigester, JobDigester from env_helper import ( CI, + CI_CONFIG_PATH, GITHUB_JOB_API_URL, GITHUB_RUN_URL, REPO_COPY, @@ -2084,6 +2085,10 @@ def main() -> int: else: exit_code = 1 else: + # Build reports use it to know which builds are part of this CI run + Path(TEMP_PATH).mkdir(parents=True, exist_ok=True) + with open(CI_CONFIG_PATH, "w", encoding="utf-8") as jfd: + json.dump(indata, jfd) exit_code = _run_test(check_name, args.run_command) ### RUN action: end diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c401f218df562..d2d0d34bf2d2e 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -61,6 +61,8 @@ def build_and_push_one_image( cmd = ( "docker buildx build --builder default " + # A single image is required by `docker manifest create`, not an index with an attestation + "--provenance=false " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" # A hack to invalidate cache, grep for it in docker/ dir diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index c1946eb5aa2a1..d85b25d93d76e 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -13,6 +13,7 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) REPORT_PATH = f"{TEMP_PATH}/reports" +CI_CONFIG_PATH = f"{TEMP_PATH}/ci_config.json" # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag" DOCKER_TAG = os.getenv("DOCKER_TAG", "latest") DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") diff --git a/tests/integration/compose/docker_compose_ldap.yml b/tests/integration/compose/docker_compose_ldap.yml index f199516f315e5..afc7f812759e9 100644 --- a/tests/integration/compose/docker_compose_ldap.yml +++ b/tests/integration/compose/docker_compose_ldap.yml @@ -1,7 +1,7 @@ version: '2.3' services: openldap: - image: bitnami/openldap:2.6.6 + image: bitnamilegacy/openldap:2.6.6 restart: always environment: LDAP_ROOT: dc=example,dc=org diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index 53031fb1d94e0..de48c6b24f2a7 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -2,7 +2,7 @@ version: '2.3' services: minio1: - image: minio/minio:RELEASE.2023-09-30T07-02-29Z + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z volumes: - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs