From d33b9234a55585efba4a8898ed7617f34923b1c1 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:53:25 -0400 Subject: [PATCH 01/14] Add a persistent ThinLTO cache for release builds The ThinLTO link of `clickhouse` and `clickhouse-keeper` dominates the release build: ~82 min of a ~117 min `amd_release` build on a 16-core, 62 GB builder. lld can cache the optimized native object of every module (`--thinlto-cache-dir`), keyed by the module's bitcode hash, its import set and the LTO configuration, and reuse it on the next link when nothing in that key changed. Only the parallel backend phase is cached; the single-threaded thin-link and the final native link still run in full. Measured on the same builder class with the production `-g` configuration (sha e3f85bc69bf, same commit built twice): 114.5 min cold, 65.5 min warm. Forty-six commits of churn on top of a warm cache cost about 3 min, since a header change only invalidates modules whose emitted IR changed. The cache lives on an attached volume at `/mnt/clickhouse_thinlto`. praktika gains `--volume-if-exists=HOST:CONTAINER` in a job's docker settings, which mounts the host path only on runners that have it rather than creating an empty directory everywhere as `--volume` does, so the build job enables the cache by the directory's presence and builders without the volume run unchanged. lld prunes inside the linker after each link's backend has run, removing the least recently used entries until the cache fits the size target, so the policy is size-only (`prune_after=0s`, lld's own default is 7 days) at 90% of the volume. The volume must be mounted `relatime` so that hits refresh access times. One `amd_release` `-g` population is ~15 GB. The build job also prints the output of its `sccache --show-stats` and cache stats commands, which `Shell.check` otherwise discards. Co-Authored-By: Claude Mythos 5.1 --- CMakeLists.txt | 16 ++++++++++++++++ ci/defs/job_configs.py | 4 ++++ ci/jobs/build_clickhouse.py | 19 +++++++++++++++++++ ci/praktika/runner.py | 13 +++++++++++++ 4 files changed, 52 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b194ad629c3..54d294a529e99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,22 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility") endif() + # Persistent ThinLTO cache. lld stores the optimized native object of every + # module keyed by the module hash, its import set and the LTO flags, and + # reuses it on the next link when nothing in that key changed. Only the + # backend codegen of the link is cached; the thin-link and the final native + # link still run in full. Entries are content-addressed, so concurrent + # writers cannot corrupt each other. + set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + if (THINLTO_CACHE_DIR) + if (NOT LINKER_NAME MATCHES "lld") + message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") + endif () + message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}") + endif () + if (ENABLE_CFI) # CFI checks: virtual dispatch type safety and bad downcasts via static_cast. # cfi-icall is omitted because C function pointers in contrib libraries (compiled without diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..9811e6f39f60c 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,6 +29,10 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" + # Attached volume holding the ThinLTO cache of release builds, see + # `use_thinlto_cache` in build_clickhouse.py. Mounted only on builders that + # have it, so the build job can key on the directory's presence. + "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 23eec7551006d..02ed4a0636cd9 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -105,6 +105,7 @@ def parse_args(): def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") + kwargs.setdefault("verbose", True) Shell.check(command, **kwargs) print(f"\n<<<< {name}\n") @@ -214,6 +215,19 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" + # ThinLTO cache: the link reuses the optimized native objects of modules + # that did not change since the previous link in this directory. It is an + # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that + # have it, so a builder whose disk is wiped between jobs has no directory + # here and does not pay for writing a cache it will never read. + thinlto_cache_dir = "/thinlto-cache" + use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir( + thinlto_cache_dir + ) + if use_thinlto_cache: + print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. @@ -463,6 +477,11 @@ def do_checkout(): results.append(retry_cmake) run_shell("sccache stats", "sccache --show-stats") + if use_thinlto_cache: + run_shell( + "thinlto cache stats", + f"du -sh {thinlto_cache_dir} && find {thinlto_cache_dir} -type f | wc -l && df -h {thinlto_cache_dir}", + ) if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log" diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..5185f4b81640b 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -480,6 +480,19 @@ def _run( if not job.timeout_shell_cleanup: job.timeout_shell_cleanup = f"docker rm -f {container_name}" workdir = f"--workdir={current_dir}" + # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners + # that have it (e.g. an attached cache volume), instead of creating + # an empty directory everywhere like `--volume` does. + optional_volumes = [ + s for s in settings if s.startswith("--volume-if-exists=") + ] + for setting in optional_volumes: + settings.remove(setting) + volume = setting.removeprefix("--volume-if-exists=") + if Path(volume.split(":")[0]).exists(): + settings.append(f"--volume={volume}") + else: + print(f"NOTE: optional volume [{volume}] is not present on this runner") for setting in settings: if setting.startswith("--volume"): volume = setting.removeprefix("--volume=").split(":")[0] From d92a33d2bf38430826e9ffef802b8c1463da6826 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:53:25 -0400 Subject: [PATCH 02/14] Link clickhouse and clickhouse-keeper concurrently in ThinLTO builds Each ThinLTO link spends most of its time in a single-threaded thin-link phase, so two links overlap almost entirely. Measured on a 16-core, 62 GB builder with zram: 114.5 -> 83 min cold and 65.5 -> 48.5 min warm (with the ThinLTO cache), with a combined memory peak of ~48 GB. `limit_jobs.cmake` caps ThinLTO links from a budget of ~100 GB per `-g` link and applied that cap even to an explicit `PARALLEL_LINK_JOBS`, so a 62 GB builder could never link both binaries at once. The cap now applies only to the auto-calculated value, and the build job sets two link jobs for every ThinLTO flavour. Co-Authored-By: Claude Mythos 5.1 --- ci/jobs/build_clickhouse.py | 9 +++++++++ cmake/limit_jobs.cmake | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 02ed4a0636cd9..981b2a64e5c72 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -228,6 +228,15 @@ def main(): print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link + # spends most of its time in a single-threaded thin-link phase, so the two + # overlap almost entirely: measured on a 16-core, 62 GB builder with zram, + # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived + # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would + # allow only one; the real peak of both links together was ~48 GB. + if "ENABLE_THINLTO=1" in cmake_cmd: + cmake_cmd += " -DPARALLEL_LINK_JOBS=2" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8be747c6f27f0..366e3f71a8ba4 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) endif() endif() +# Remember whether the link-job limit was set explicitly: the ThinLTO memory +# cap below only applies to the auto-calculated value. +set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS}) + if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) @@ -54,7 +58,7 @@ endif () # links than physically fit in memory - otherwise the linker is OOM-killed. # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only # ~50 GB, so two still fit and we keep them parallel. -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT) if (DISABLE_ALL_DEBUG_SYMBOLS) set (MAX_THINLTO_LINK_MEMORY 51200) # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0 else () From 02e720836ff43fdc35367cc2ec7b359ac64b948e Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:13:24 -0400 Subject: [PATCH 03/14] Reduce runner swappiness, we are wasting available ram --- .github/actions/runner_setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index e978d8f57a741..a9f9d0d341f41 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -25,7 +25,7 @@ runs: ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 - sudo sysctl vm.swappiness=200 + sudo sysctl vm.swappiness=80 - name: Install awscli shell: bash run: | From 5b2a13913502d7603024a26debbed405d25a3e38 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:32:45 -0400 Subject: [PATCH 04/14] enable lto cache volume on builders Cache does not belong in meta label since builder instances are also used for some tests --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- ci/defs/job_configs.py | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..7b6b245777e36 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..b88decb436b4e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..5f2c4ba6cde87 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index e3c6626cbac83..6cdd7707d4d1b 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index cd3681eece7e4..68d93cbed1b60 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 9811e6f39f60c..63707a3c7a692 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -462,7 +462,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, timeout=4 * 3600, ), ) @@ -505,12 +505,12 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, timeout=3 * 3600, ), Job.ParamSet( parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, ), ) ] From d151e0f7417c4dda63bc4e508a8737f6d91e8a30 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:47:53 -0400 Subject: [PATCH 05/14] Cap the ThinLTO cache at 180 GB as well as 90% of its filesystem --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54d294a529e99..971a108900ed6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,7 +500,7 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) # link still run in full. Entries are content-addressed, so concurrent # writers cannot corrupt each other. set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") - set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") if (THINLTO_CACHE_DIR) if (NOT LINKER_NAME MATCHES "lld") message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") From 1e896cbf4dbf70663d433f8f577b220ebe932418 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:53:59 -0400 Subject: [PATCH 06/14] fix volume labels --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 7b6b245777e36..61899d96e066b 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b88decb436b4e..7ed2d154c9a94 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5f2c4ba6cde87..51a52c4b607ce 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 6cdd7707d4d1b..a41b454610754 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 68d93cbed1b60..139f9d543cd42 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", From 62318a6b5dbb3217c737214c8a4999dbdf489892 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:13:02 -0400 Subject: [PATCH 07/14] Make the ThinLTO cache stats work on a volume with `lost+found` On the builder volume the root-owned `lost+found` directory is unreadable for the build user, so `du` failed and the `&&` chain stopped before the entry count and `df`. Exclude it from `du`, count only lld's own `llvmcache-*` entries, and run the three commands independently. https://github.com/Altinity/ClickHouse/actions/runs/40545556921/job/122623854938 Co-Authored-By: Claude Opus 5.5 --- ci/jobs/build_clickhouse.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 981b2a64e5c72..8a67a904e96f1 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -489,7 +489,11 @@ def do_checkout(): if use_thinlto_cache: run_shell( "thinlto cache stats", - f"du -sh {thinlto_cache_dir} && find {thinlto_cache_dir} -type f | wc -l && df -h {thinlto_cache_dir}", + # The volume's root-owned `lost+found` is unreadable for the + # build user, so count only lld's own entries. + f"du -sh --exclude=lost+found {thinlto_cache_dir}; " + f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; " + f"df -h {thinlto_cache_dir}", ) if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") From 0d56da2db82589aa5dac5929c71407bb811e12ea Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:10:48 -0400 Subject: [PATCH 08/14] Split the ThinLTO cache volumes by target architecture amd and arm cache entries never share keys, so a build only benefits from a volume where a build for the same target ran recently. With one volume name, the scaler hands each build whichever detached volume it finds first, and a master push runs two amd and two arm ThinLTO builds at once. Give each target its own meta label, `altinity-thinlto-cache-amd` and `altinity-thinlto-cache-arm`, which the runner controller expands into the volume (`clickhouse_thinlto_amd`, `clickhouse_thinlto_arm`) and its location, so neither size nor location is fixed in the repository. The build job mounts whichever of the per-target volumes is present, or the single `/mnt/clickhouse_thinlto` directory that static builders keep on their own disk for both targets. The 180 GB cap in the cache policy stays, since those builders hold both targets' entries. `arm_release` now runs on `BUILDER_ARM`, and `amd_cfi` and the amd cache warmup build on `BUILDER_AMD`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- ci/defs/job_configs.py | 17 +++++++++++------ 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 61899d96e066b..ff8a7c80dc874 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7ed2d154c9a94..b65ab01074036 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 51a52c4b607ce..e8244433825c2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index a41b454610754..0844cfb532a7a 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 139f9d543cd42..12d32fd34d46f 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 63707a3c7a692..bcb528e8d5259 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,9 +29,14 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - # Attached volume holding the ThinLTO cache of release builds, see - # `use_thinlto_cache` in build_clickhouse.py. Mounted only on builders that - # have it, so the build job can key on the directory's presence. + # ThinLTO cache of release builds, see `use_thinlto_cache` in + # build_clickhouse.py. Mounted only on builders that have it, so the build + # job can key on the directory's presence. Cloud builders get one attached + # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static + # builders keep a single directory on their own disk for both targets. A + # builder has exactly one of the three. + "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache" + "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache" "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' ) @@ -447,7 +452,7 @@ class JobConfigs: ArtifactNames.RPM_ARM_RELEASE, ArtifactNames.TGZ_ARM_RELEASE, ], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.BUILDER_ARM, ), ) release_build_jobs_with_examples = [ @@ -462,7 +467,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.BUILDER_AMD, timeout=4 * 3600, ), ) @@ -505,7 +510,7 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.BUILDER_AMD, timeout=3 * 3600, ), Job.ParamSet( From 23cc2c9cefba660336d03dc65d888afeaf5d9fde Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:42:29 -0400 Subject: [PATCH 09/14] Store the submodule cache in Hetzner Object Storage Most runners are in Hetzner, so every job that downloads the submodule archive from AWS S3 pays egress. Keep the archive in a Hetzner bucket instead. Buckets listed in `S3_EXTERNAL_BUCKETS` are served by a boto3 client with their own endpoint and credentials. `S3.put` and `S3.head_object` use it for those buckets, and keep using the AWS CLI for every other bucket. With `public_read`, reads are unsigned, so community PRs can use the archive without credentials; they never create it. A missing credential is an error, so ambient AWS credentials are never sent to the Hetzner endpoint. The archive is public, so `_prepare_submodule_cache` refuses to upload it if the clone token ended up in a git config file. The workflows export `HETZNER_S3_ACCESS_KEY` and `HETZNER_S3_SECRET_KEY` from GitHub secrets. The bucket is `altinity-builds-cache` in hel1. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/fast_builds.yml | 2 + .github/workflows/master.yml | 2 + .github/workflows/pull_request.yml | 2 + ci/praktika/cache.py | 5 + ci/praktika/native_jobs.py | 24 +++- ci/praktika/runner.py | 3 +- ci/praktika/s3.py | 147 +++++++++++++++---- ci/praktika/settings.py | 9 +- ci/praktika/yaml_additional_templates.py | 2 + ci/settings/altinity_overrides.py | 13 ++ ci/tests/test_s3_external_bucket.py | 173 +++++++++++++++++++++++ 11 files changed, 353 insertions(+), 29 deletions(-) create mode 100644 ci/tests/test_s3_external_bucket.py diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..26a61911e0462 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -45,6 +45,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..90d8b9e15d717 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -60,6 +60,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..e16505d822493 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,6 +35,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/ci/praktika/cache.py b/ci/praktika/cache.py index 148f4edfbc521..6c1545eef742e 100644 --- a/ci/praktika/cache.py +++ b/ci/praktika/cache.py @@ -39,6 +39,11 @@ def from_fs(cls, path): def from_dict(cls, obj): return Cache.CacheRecord(**obj) + @staticmethod + def submodule_archive_s3_path(cache_hash): + base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH + return f"{base}/submodules/{cache_hash}.tar.zst" + def __init__(self): self.digest = Digest() self.success = {} # type Dict[str, Any] diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index e6c9a673f43cd..d907dc574795b 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -10,6 +10,7 @@ from . import Job, Workflow from ._environment import _Environment +from .cache import Cache from .cidb import CIDB from .digest import Digest from .docker import Docker @@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: ) cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16] - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) print(f"Submodule cache hash: {cache_hash}") # Check if cache already exists in S3 if S3.head_object(s3_path): print(f"Submodule cache hit: {s3_path}") info = f"cache hit: {cache_hash}" + elif Info().is_community_pr: + # Like sccache, community PRs only read; jobs clone without a hash. + print(f"Submodule cache miss, community PR will not create it: {s3_path}") + return Result.create_from( + name="Submodule Cache", + status=Result.Status.OK, + stopwatch=stop_watch, + info=f"cache miss, not creating (read-only): {cache_hash}", + ) else: print(f"Submodule cache miss, creating: {s3_path}") Shell.check("git submodule sync", verbose=True, strict=True) @@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: retries=3, env=_submodule_auth_env(workflow), ) + # The archive is public: never upload it with the clone token inside. + leaked = Shell.get_output( + "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects" + ) + if leaked: + raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}") archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" Shell.check( f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}", @@ -892,7 +908,11 @@ def check_affected_jobs(): ) ) - if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE: + if ( + results[-1].is_ok() + and (workflow.enable_cache or Info().is_community_pr) + and Settings.ENABLE_SUBMODULE_CACHE + ): result = _prepare_submodule_cache(workflow, workflow_config) results.append(result) diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -11,6 +11,7 @@ from pathlib import Path from ._environment import _Environment +from .cache import Cache from .artifact import Artifact from .cidb import CIDB from .digest import Digest @@ -362,7 +363,7 @@ def _restore_submodule_cache(): if not cache_hash: print("NOTE: No submodule cache hash in workflow config, skipping restore") return - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" print(f"Restoring submodule cache: {s3_path}") S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index c064f513e635b..aac08ddd5d7bb 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -15,6 +15,8 @@ try: import boto3 + from botocore import UNSIGNED + from botocore.config import Config from botocore.exceptions import ClientError, NoCredentialsError BOTO3_AVAILABLE = True @@ -58,9 +60,45 @@ def clean_line(line): class S3: _boto3_client = None + _external_clients = {} @classmethod - def _get_boto3_client(cls): + def _external_bucket_config(cls, bucket): + return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + + @classmethod + def _get_external_client(cls, bucket, config, anonymous): + assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" + client = cls._external_clients.get((bucket, anonymous)) + if client is None: + if anonymous: + credentials = {"config": Config(signature_version=UNSIGNED)} + else: + access_key = os.environ.get(config["access_key_env"], "") + secret_key = os.environ.get(config["secret_key_env"], "") + # boto3 would otherwise send the ambient AWS credentials to this endpoint + assert ( + access_key and secret_key + ), f"S3 bucket [{bucket}] needs [{config['access_key_env']}] and [{config['secret_key_env']}] set" + credentials = { + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key, + } + client = boto3.client( + "s3", + endpoint_url=config["endpoint_url"], + region_name=config["region"], + **credentials, + ) + cls._external_clients[(bucket, anonymous)] = client + return client + + @classmethod + def _get_boto3_client(cls, bucket=None, read=False): + config = cls._external_bucket_config(bucket) + if config: + anonymous = read and bool(config.get("public_read")) + return cls._get_external_client(bucket, config, anonymous) if not BOTO3_AVAILABLE: return None if cls._boto3_client is None: @@ -188,7 +226,7 @@ def copy_file_to_s3( extra_args["Tagging"] = urlencode(tags) def _upload(): - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket) if extra_args: client.upload_file( str(local_path), bucket, key, ExtraArgs=extra_args @@ -260,24 +298,69 @@ def put( s3_full_path = str(s3_full_path).removeprefix("s3://") bucket, key = s3_full_path.split("/", maxsplit=1) - command = ( - f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" - ) - if if_none_matched: - command += ' --if-none-match "*"' - if if_match: - command += f' --if-match "{if_match}"' - if metadata: - for k, v in metadata.items(): - command += f" --metadata {k}={v}" - - if text: - command += ' --content-type "text/plain; charset=utf-8"' - res = cls.run_command_with_retries(command, no_strict=no_strict) + if cls._external_bucket_config(bucket): + res = cls._put_object_boto3( + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ) + else: + command = f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" + if if_none_matched: + command += ' --if-none-match "*"' + if if_match: + command += f' --if-match "{if_match}"' + if metadata: + for k, v in metadata.items(): + command += f" --metadata {k}={v}" + + if text: + command += ' --content-type "text/plain; charset=utf-8"' + res = cls.run_command_with_retries(command, no_strict=no_strict) if res: StorageUsage.add_uploaded(local_path) return res + @classmethod + def _put_object_boto3( + cls, + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ): + args = {"Bucket": bucket, "Key": key} + if metadata: + args["Metadata"] = metadata + if text: + args["ContentType"] = "text/plain; charset=utf-8" + if if_none_matched: + args["IfNoneMatch"] = "*" + if if_match: + args["IfMatch"] = if_match + try: + with open(local_path, "rb") as f: + cls._get_boto3_client(bucket).put_object(Body=f, **args) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + if error_code in ("PreconditionFailed", "ConditionalRequestConflict"): + print(f"S3 conditional request failed (concurrent write detected): {error_code}") + return False + print(f"ERROR: Failed to put S3 object [{bucket}/{key}]: {error_code}") + if not no_strict: + raise + return False + return True + @classmethod def run_command_with_retries( cls, @@ -354,7 +437,7 @@ def copy_file_from_s3( local_file.parent.mkdir(parents=True, exist_ok=True) def _download(): - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket, read=True) client.download_file(bucket, key, str(local_file)) # Retry on transient credential failures @@ -426,18 +509,32 @@ def copy_file_from_s3_matching_pattern( def head_object(cls, s3_path): s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - output = Shell.get_output( - f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True - ) - if not output: - return None - data = json.loads(output) + if cls._external_bucket_config(bucket): + try: + data = cls._get_boto3_client(bucket, read=True).head_object( + Bucket=bucket, Key=key + ) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "") in ( + "404", + "NoSuchKey", + "NotFound", + ): + return None + raise + else: + output = Shell.get_output( + f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True + ) + if not output: + return None + data = json.loads(output) # HeadObject omits optional fields (Expiration) unless a lifecycle Expire # rule applies, and emits extra keys Object does not declare. return cls.Object( AcceptRanges=data.get("AcceptRanges", ""), Expiration=data.get("Expiration", ""), - LastModified=data.get("LastModified", ""), + LastModified=str(data.get("LastModified", "")), ContentLength=data.get("ContentLength", 0), ETag=data.get("ETag", ""), ContentType=data.get("ContentType", ""), @@ -456,7 +553,7 @@ def assert_read_access(cls, s3_path): """ s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket, read=True) assert client, "boto3 client is required for S3 read-access probe" def _probe(): diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 760a0caa90748..b96fcb9888451 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -1,7 +1,7 @@ import dataclasses import importlib.util from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional @dataclasses.dataclass @@ -111,6 +111,8 @@ class _Settings: # If enabled, submodule clones authenticate with the App installation token # (required for private submodules); otherwise they run anonymously. ENABLE_SUBMODULE_CLONE_AUTH: bool = False + # bucket/prefix of the submodule archive, defaults to CACHE_S3_PATH + SUBMODULE_CACHE_S3_PATH: str = "" # v2: records carry the producing workflow event, used as the reuse trust # signal instead of the branch (see CacheRunnerHooks.configure). @@ -127,6 +129,9 @@ class _Settings: S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None + # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read}} + # The keys are read from the named env vars; with public_read, reads are unsigned. + S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) COMPRESS_THRESHOLD_MB: int = 0 @@ -177,6 +182,8 @@ class _Settings: "EVENT_FEED_S3_PATH", "AWS_REGION", "S3_BUCKET_TO_HTTP_ENDPOINT", + "S3_EXTERNAL_BUCKETS", + "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", "TEMP_DIR", "OUTPUT_DIR", diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 8731d5e5d5c17..2869279da9962 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -8,6 +8,8 @@ class AltinityWorkflowTemplates: AZURE_CONTAINER_NAME: ${{{{ secrets.AZURE_CONTAINER_NAME }}}} AZURE_STORAGE_ACCOUNT_URL: "https://${{{{ secrets.AZURE_ACCOUNT_NAME }}}}.blob.core.windows.net/" ROBOT_TOKEN: ${{{{ secrets.ROBOT_TOKEN }}}} + HETZNER_S3_ACCESS_KEY: ${{{{ secrets.HETZNER_S3_ACCESS_KEY }}}} + HETZNER_S3_SECRET_KEY: ${{{{ secrets.HETZNER_S3_SECRET_KEY }}}} """ # Additional pre steps for all jobs JOB_SETUP_STEPS = """ diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 9e965980b8ef5..2013e153cce29 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -63,6 +63,19 @@ class RunnerLabels: # after the burst of ~150 clones; send the ambient gh token instead. ENABLE_SUBMODULE_CLONE_AUTH = True +# Most runners are in Hetzner (hel1): cache there to avoid AWS egress. +CI_CACHE_BUCKET = "altinity-builds-cache" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" +S3_EXTERNAL_BUCKETS = { + CI_CACHE_BUCKET: { + "endpoint_url": "https://hel1.your-objectstorage.com", + "region": "hel1", + "access_key_env": "HETZNER_S3_ACCESS_KEY", + "secret_key_env": "HETZNER_S3_SECRET_KEY", + "public_read": True, + } +} + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py new file mode 100644 index 0000000000000..852f12d66f7c8 --- /dev/null +++ b/ci/tests/test_s3_external_bucket.py @@ -0,0 +1,173 @@ +""" +Buckets listed in Settings.S3_EXTERNAL_BUCKETS live on an S3-compatible store other than AWS. +S3.put and S3.head_object must reach them through a boto3 client with their own endpoint and credentials, +and must keep using the AWS CLI for every other bucket. +""" + +import datetime +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +pytest.importorskip("boto3") +from botocore.exceptions import ClientError + +from ci.praktika import s3 as s3_module +from ci.praktika.cache import Cache +from ci.praktika.s3 import S3 +from ci.praktika.settings import Settings + +BUCKET = "external-bucket" +EXTERNAL = { + BUCKET: { + "endpoint_url": "https://s3.example.test", + "region": "example-1", + "access_key_env": "EXTERNAL_S3_ACCESS_KEY", + "secret_key_env": "EXTERNAL_S3_SECRET_KEY", + "public_read": True, + } +} + + +class _FakeClient: + def __init__(self, error_code=None): + self.error_code = error_code + self.put_args = None + + def _maybe_fail(self, operation): + if self.error_code: + raise ClientError({"Error": {"Code": self.error_code}}, operation) + + def put_object(self, Body, **kwargs): + self._maybe_fail("PutObject") + self.put_args = kwargs + + def head_object(self, Bucket, Key): + self._maybe_fail("HeadObject") + return { + "ETag": '"etag"', + "ContentLength": 3, + "LastModified": datetime.datetime(2026, 1, 2, 3, 4, 5), + } + + +@pytest.fixture(autouse=True) +def external_bucket(monkeypatch): + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", EXTERNAL, raising=False) + monkeypatch.setattr(S3, "_external_clients", {}) + + def no_cli(*args, **kwargs): + raise AssertionError("the AWS CLI must not be used for an external bucket") + + monkeypatch.setattr(s3_module.Shell, "get_output", no_cli) + monkeypatch.setattr(s3_module.Shell, "get_res_stdout_stderr", no_cli) + + +def _use_client(monkeypatch, client): + """Serve the same fake for signed and anonymous clients.""" + monkeypatch.setattr( + S3, "_external_clients", {(BUCKET, False): client, (BUCKET, True): client} + ) + + +def _put(tmp_path, **kwargs): + local_file = tmp_path / "archive" + local_file.write_text("abc") + return S3.put(f"{BUCKET}/prefix/archive", str(local_file), **kwargs) + + +def test_put_sends_if_none_match(tmp_path, monkeypatch): + client = _FakeClient() + _use_client(monkeypatch, client) + assert _put(tmp_path, if_none_matched=True) is True + assert client.put_args == { + "Bucket": BUCKET, + "Key": "prefix/archive", + "IfNoneMatch": "*", + } + + +@pytest.mark.parametrize("code", ["PreconditionFailed", "ConditionalRequestConflict"]) +def test_put_lost_race_returns_false(tmp_path, monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert _put(tmp_path, if_none_matched=True) is False + + +def test_put_unexpected_error(tmp_path, monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + _put(tmp_path) + assert _put(tmp_path, no_strict=True) is False + + +def test_head_object(monkeypatch): + _use_client(monkeypatch, _FakeClient()) + obj = S3.head_object(f"{BUCKET}/prefix/archive") + assert obj.ContentLength == 3 + assert obj.LastModified == "2026-01-02 03:04:05" + + +@pytest.mark.parametrize("code", ["404", "NoSuchKey", "NotFound"]) +def test_head_object_missing_is_none(monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + + +def test_head_object_other_error_raises(monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +@pytest.fixture +def no_credentials(monkeypatch): + monkeypatch.delenv("EXTERNAL_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("EXTERNAL_S3_SECRET_KEY", raising=False) + + +def test_writes_need_credentials(tmp_path, no_credentials): + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + _put(tmp_path) + + +def test_reads_of_a_public_bucket_need_no_credentials(no_credentials): + client = S3._get_boto3_client(BUCKET, read=True) + assert client.meta.config.signature_version is s3_module.UNSIGNED + assert client.meta.endpoint_url == "https://s3.example.test" + + +def test_reads_of_a_private_bucket_need_credentials(monkeypatch, no_credentials): + private = {BUCKET: {**EXTERNAL[BUCKET], "public_read": False}} + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", private, raising=False) + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_signed_client_uses_the_configured_endpoint(monkeypatch): + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + client = S3._get_boto3_client(BUCKET) + assert client.meta.endpoint_url == "https://s3.example.test" + assert client.meta.region_name == "example-1" + assert S3._get_boto3_client(BUCKET) is client + assert S3._get_boto3_client(BUCKET, read=True) is not client + + +def test_other_buckets_still_use_the_aws_cli(monkeypatch): + calls = [] + monkeypatch.setattr( + s3_module.Shell, "get_output", lambda cmd, **kw: calls.append(cmd) or "" + ) + assert S3.head_object("aws-bucket/key") is None + assert calls == ["aws s3api head-object --bucket aws-bucket --key key"] + + +def test_submodule_archive_path(monkeypatch): + monkeypatch.setattr(Settings, "CACHE_S3_PATH", "bucket/cache") + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", "") + assert Cache.submodule_archive_s3_path("abc") == "bucket/cache/submodules/abc.tar.zst" + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", f"{BUCKET}/cache") + assert Cache.submodule_archive_s3_path("abc") == f"{BUCKET}/cache/submodules/abc.tar.zst" From d6eae03e91db139f51eb7dd5ac0b49f12f76a035 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:04:38 -0400 Subject: [PATCH 10/14] Sign reads of the Hetzner cache bucket when the keys are present The first CI run failed with a 403 on the existence check for the submodule archive. The bucket is public for object reads, but answers a HEAD of a missing key with 403 to callers who cannot list it. The config job read the bucket unsigned because the bucket is marked `public_read`, even though it has the keys. Reads are now unsigned only for jobs without the bucket's keys. Those jobs treat a 403 from `head_object` as a miss; jobs with the keys use a signed client, so a 403 is still an error for them. Co-Authored-By: Claude Sonnet 5.5 --- ci/praktika/s3.py | 18 +++++++++++++++++- ci/praktika/settings.py | 2 +- ci/tests/test_s3_external_bucket.py | 11 ++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index aac08ddd5d7bb..bf8a3e5e8b2b4 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -66,6 +66,19 @@ class S3: def _external_bucket_config(cls, bucket): return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + @classmethod + def _reads_anonymously(cls, bucket): + """Reads of a public_read bucket are unsigned unless this job has the bucket's keys.""" + config = cls._external_bucket_config(bucket) + return bool( + config + and config.get("public_read") + and not ( + os.environ.get(config["access_key_env"]) + and os.environ.get(config["secret_key_env"]) + ) + ) + @classmethod def _get_external_client(cls, bucket, config, anonymous): assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" @@ -97,7 +110,7 @@ def _get_external_client(cls, bucket, config, anonymous): def _get_boto3_client(cls, bucket=None, read=False): config = cls._external_bucket_config(bucket) if config: - anonymous = read and bool(config.get("public_read")) + anonymous = read and cls._reads_anonymously(bucket) return cls._get_external_client(bucket, config, anonymous) if not BOTO3_AVAILABLE: return None @@ -521,6 +534,9 @@ def head_object(cls, s3_path): "NotFound", ): return None + # A public bucket answers a missing key with 403 to callers who may not list it + if e.response.get("Error", {}).get("Code", "") == "403" and cls._reads_anonymously(bucket): + return None raise else: output = Shell.get_output( diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index b96fcb9888451..094962b385d01 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -130,7 +130,7 @@ class _Settings: HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read}} - # The keys are read from the named env vars; with public_read, reads are unsigned. + # The keys are read from the named env vars; with public_read, jobs without them read unsigned. S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py index 852f12d66f7c8..3bffdff401f92 100644 --- a/ci/tests/test_s3_external_bucket.py +++ b/ci/tests/test_s3_external_bucket.py @@ -153,7 +153,16 @@ def test_signed_client_uses_the_configured_endpoint(monkeypatch): assert client.meta.endpoint_url == "https://s3.example.test" assert client.meta.region_name == "example-1" assert S3._get_boto3_client(BUCKET) is client - assert S3._get_boto3_client(BUCKET, read=True) is not client + assert S3._get_boto3_client(BUCKET, read=True) is client + + +def test_forbidden_is_a_miss_only_for_anonymous_reads(monkeypatch, no_credentials): + _use_client(monkeypatch, _FakeClient("403")) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") def test_other_buckets_still_use_the_aws_cli(monkeypatch): From 0084fb8f879ea44e23e0959cdc1c491076c552d8 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:50:38 -0400 Subject: [PATCH 11/14] Download the Hetzner cache with 32 parallel streams Co-Authored-By: Claude Sonnet 5.5 --- ci/praktika/s3.py | 21 +++++++++++++++++++-- ci/praktika/settings.py | 3 ++- ci/settings/altinity_overrides.py | 2 ++ ci/tests/test_s3_external_bucket.py | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index bf8a3e5e8b2b4..863718e8b915d 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -18,6 +18,7 @@ from botocore import UNSIGNED from botocore.config import Config from botocore.exceptions import ClientError, NoCredentialsError + from boto3.s3.transfer import TransferConfig BOTO3_AVAILABLE = True except ImportError: @@ -84,8 +85,13 @@ def _get_external_client(cls, bucket, config, anonymous): assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" client = cls._external_clients.get((bucket, anonymous)) if client is None: + client_config = { + "max_pool_connections": config.get("download_concurrency", 10) + } if anonymous: - credentials = {"config": Config(signature_version=UNSIGNED)} + credentials = { + "config": Config(signature_version=UNSIGNED, **client_config) + } else: access_key = os.environ.get(config["access_key_env"], "") secret_key = os.environ.get(config["secret_key_env"], "") @@ -96,6 +102,7 @@ def _get_external_client(cls, bucket, config, anonymous): credentials = { "aws_access_key_id": access_key, "aws_secret_access_key": secret_key, + "config": Config(**client_config), } client = boto3.client( "s3", @@ -451,7 +458,17 @@ def copy_file_from_s3( def _download(): client = cls._get_boto3_client(bucket, read=True) - client.download_file(bucket, key, str(local_file)) + concurrency = (cls._external_bucket_config(bucket) or {}).get( + "download_concurrency" + ) + client.download_file( + bucket, + key, + str(local_file), + Config=TransferConfig(max_concurrency=concurrency) + if concurrency + else None, + ) # Retry on transient credential failures cls._retry_on_no_credentials(_download) diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 094962b385d01..3385cd6b57368 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -129,8 +129,9 @@ class _Settings: S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None - # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read}} + # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 2013e153cce29..101c6f492a047 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -73,6 +73,8 @@ class RunnerLabels: "access_key_env": "HETZNER_S3_ACCESS_KEY", "secret_key_env": "HETZNER_S3_SECRET_KEY", "public_read": True, + # netcup runners get ~2.5 MB/s per connection from hel1 + "download_concurrency": 32, } } diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py index 3bffdff401f92..9746cbeb7a115 100644 --- a/ci/tests/test_s3_external_bucket.py +++ b/ci/tests/test_s3_external_bucket.py @@ -180,3 +180,18 @@ def test_submodule_archive_path(monkeypatch): assert Cache.submodule_archive_s3_path("abc") == "bucket/cache/submodules/abc.tar.zst" monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", f"{BUCKET}/cache") assert Cache.submodule_archive_s3_path("abc") == f"{BUCKET}/cache/submodules/abc.tar.zst" + + +def test_download_concurrency(monkeypatch, tmp_path, no_credentials): + seen = {} + + class Client: + def download_file(self, bucket, key, filename, Config=None): + seen["concurrency"] = Config.max_concurrency + + monkeypatch.setattr(S3, "_external_clients", {(BUCKET, True): Client()}) + monkeypatch.setitem(EXTERNAL[BUCKET], "download_concurrency", 32) + assert S3.copy_file_from_s3(f"{BUCKET}/prefix/archive", str(tmp_path / "out")) + assert seen["concurrency"] == 32 + monkeypatch.setattr(S3, "_external_clients", {}) + assert S3._get_boto3_client(BUCKET, read=True).meta.config.max_pool_connections == 32 From 8451c27beaa7d74b86e1fcd2a0026bc3de4c6d74 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:18:14 -0400 Subject: [PATCH 12/14] Allow the `cas` sanitizer jobs in `test_selected_tests_run` The Altinity PR workflow has full-suite sanitizer jobs that use CAS storage (`Stateless tests (amd_asan_ubsan, cas s3 storage, ...)`), which the test rejected as unexpected full-suite jobs. It failed in every CI run, including without other changes. Related: https://github.com/Altinity/ClickHouse/actions/runs/41066953900 Co-Authored-By: Claude Sonnet 5.5 --- ci/tests/test_selected_tests_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/tests/test_selected_tests_run.py b/ci/tests/test_selected_tests_run.py index 28c4d2d809da0..489823d625327 100644 --- a/ci/tests/test_selected_tests_run.py +++ b/ci/tests/test_selected_tests_run.py @@ -342,10 +342,10 @@ def test_pr_workflow_keeps_full_suite_msan_wasmedge_functional_tests(): if "amd_msan, WasmEdge" in name: full_suite_msan_wasmedge_jobs.append(name) continue - # The flaky, targeted and azure jobs run their own selection or config. + # The flaky, targeted, azure and cas jobs run their own selection or config. assert any( marker in name - for marker in ("selected tests", "flaky check", "targeted", "azure") + for marker in ("selected tests", "flaky check", "targeted", "azure", "cas ") ), f"full-suite sanitizer functional test job in the PR workflow: {name}" assert len(full_suite_msan_wasmedge_jobs) == 10 From fe70d03bdc2d007c3f0035a5a2c40a38fae7aa0c Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:42:53 -0400 Subject: [PATCH 13/14] Move the ThinLTO cache from builder volumes to Hetzner Object Storage Attached volumes tied the cache to whichever builder a job landed on, and the runner scaler never reused a volume for builders made from a snapshot image. The cache now travels as one archive per branch, build type and debug-info flavour in the Hetzner bucket from the submodule cache: `altinity-builds-cache/ci_ch_cache/thinlto//-.tar.zst`. Every ThinLTO build that links restores the archive before `ninja`. PR builds use their base branch's archive. Pushes and internal PRs then upload it again, so `-g` archives come from `MasterCI` and `-g0` ones from PR builds; community PRs read anonymously and write nothing, like the shared compiler cache. The cache-warmup builds only compile, so they skip it. The upload keeps exactly the entries the build used: restored entries get an access time of zero, so under `relatime` the next read sets it, and the entries still at zero are dropped before compressing with `zstd --long`. That keeps an archive at one build's working set, about 15 GB raw for `-g`, and lld's own pruning is off. A restore that fails leaves an empty cache rather than a partly extracted one, and an archive over 20 GB (`-g`) or 5 GB (`-g0`) is not uploaded. This removes the praktika `--volume-if-exists` setting, the cache mounts and the per-target builder labels, and puts `amd_cfi` and the cache-warmup builds back on `ARM_LARGE`. Related: https://github.com/Altinity/ClickHouse/pull/2740 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fast_builds.yml | 4 +- .github/workflows/master.yml | 8 +- .github/workflows/pull_request.yml | 4 +- .github/workflows/pull_request_community.yml | 4 +- CMakeLists.txt | 4 +- ci/defs/defs.py | 4 +- ci/defs/job_configs.py | 17 +- ci/jobs/build_clickhouse.py | 172 ++++++++++++++++--- ci/praktika/runner.py | 13 -- 9 files changed, 171 insertions(+), 59 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index c5538b8a7e8fb..26a61911e0462 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -802,7 +802,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -846,7 +846,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3672c1c649003..90d8b9e15d717 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -861,7 +861,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -905,7 +905,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -949,7 +949,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -993,7 +993,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2949239dfbcdf..e16505d822493 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -849,7 +849,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -891,7 +891,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 0844cfb532a7a..e3c6626cbac83 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 971a108900ed6..ffcd02c5d0696 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,7 +500,9 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) # link still run in full. Entries are content-addressed, so concurrent # writers cannot corrupt each other. set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") - set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + # lld prunes after each link by default. That is off here, since CI keeps + # exactly the entries a build used; set a policy for a long-lived cache dir. + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=0%:cache_size_bytes=0:cache_size_files=0" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") if (THINLTO_CACHE_DIR) if (NOT LINKER_NAME MATCHES "lld") message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 12d32fd34d46f..cd3681eece7e4 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index bcb528e8d5259..b79745981beb2 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,16 +29,9 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - # ThinLTO cache of release builds, see `use_thinlto_cache` in - # build_clickhouse.py. Mounted only on builders that have it, so the build - # job can key on the directory's presence. Cloud builders get one attached - # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static - # builders keep a single directory on their own disk for both targets. A - # builder has exactly one of the three. - "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache" - "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache" - "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' + # Writes to the ThinLTO cache in Hetzner Object Storage; jobs without them read anonymously + '+--env=HETZNER_S3_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"+--env=HETZNER_S3_SECRET_KEY="$HETZNER_S3_SECRET_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined" @@ -467,7 +460,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.ARM_LARGE, timeout=4 * 3600, ), ) @@ -510,12 +503,12 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.ARM_LARGE, timeout=3 * 3600, ), Job.ParamSet( parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.ARM_LARGE, ), ) ] diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 8a67a904e96f1..f8207263d0b04 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -6,8 +6,10 @@ from ci.jobs.scripts.clickhouse_version import CHVersion from ci.praktika.info import Info from ci.praktika.result import Result +from ci.praktika.s3 import S3 from ci.praktika.settings import Settings from ci.praktika.utils import MetaClasses, Shell, Utils +from ci.settings.altinity_overrides import CI_CACHE_BUCKET current_directory = Utils.cwd() build_dir = f"{current_directory}/ci/tmp/build" @@ -15,6 +17,18 @@ repo_path_normalized = "/ClickHouse" build_dir_normalized = f"{repo_path_normalized}/ci/tmp/build" +# ThinLTO cache: lld stores the optimized native object of every module it +# links, and the next link of an unchanged module reuses it instead of running +# the backend again. The cache travels between builders as one archive per +# branch, build type and debug-info flavour. +THINLTO_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache/thinlto" +thinlto_cache_dir = f"{temp_dir}/thinlto_cache" +thinlto_cache_dir_normalized = f"{repo_path_normalized}/ci/tmp/thinlto_cache" +thinlto_cache_archive = f"{temp_dir}/thinlto_cache.tar.zst" +# An archive holds the entries one build used, about 15 GB for a `-g` build of +# `clickhouse` and `clickhouse-keeper`. Much more means pruning went wrong. +THINLTO_CACHE_MAX_BYTES = {"g": 20 << 30, "g0": 5 << 30} + BUILD_TYPE_TO_CMAKE = { BuildTypes.AMD_DEBUG: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_LEXER_TEST=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_BUZZHOUSE=1", BuildTypes.AMD_RELEASE: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None -DENABLE_THINLTO=1 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_LEXER_TEST=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1", @@ -103,6 +117,111 @@ def parse_args(): return parser.parse_args() +def thinlto_cache_flavour(cmake_cmd): + return "g0" if "-DDISABLE_ALL_DEBUG_SYMBOLS=1" in cmake_cmd else "g" + + +def thinlto_cache_s3_path(info, build_type, cmake_cmd): + """PR builds share the cache of their base branch, other builds use their own branch.""" + branch = info.base_branch if info.pr_number > 0 else info.git_branch + if not branch: + return None + return f"{THINLTO_CACHE_S3_PATH}/{branch}/{build_type}-{thinlto_cache_flavour(cmake_cmd)}.tar.zst" + + +def restore_thinlto_cache(s3_path): + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + if not S3.head_object(s3_path): + print(f"ThinLTO cache miss: {s3_path}") + return True + try: + S3.copy_file_from_s3(s3_path=s3_path, local_path=thinlto_cache_archive) + Shell.check( + f"set -o pipefail; zstd -d --long=27 -c {thinlto_cache_archive} | tar -xf - -C {thinlto_cache_dir}", + verbose=True, + strict=True, + ) + except BaseException: + # A partly extracted entry would be linked as is: start cold instead. + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + raise + finally: + if os.path.exists(thinlto_cache_archive): + os.remove(thinlto_cache_archive) + # With an access time of zero, the kernel records the next read of an + # entry even under `relatime`, which is how the upload step tells the + # entries this build used from the ones it did not. + count = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + os.utime(entry.path, (0, stat.st_mtime)) + count += 1 + size += stat.st_size + print(f"ThinLTO cache restored: {count} entries, {size / (1 << 30):.1f} GiB") + return True + + +def thinlto_cache_tracks_reads(): + probe = f"{thinlto_cache_dir}/.atime-probe" + with open(probe, "w") as f: + f.write("probe") + os.utime(probe, (0, os.stat(probe).st_mtime)) + with open(probe) as f: + f.read() + tracked = os.stat(probe).st_atime > 0 + os.remove(probe) + return tracked + + +def upload_thinlto_cache(s3_path, flavour): + """Keep the entries this build read or wrote, then upload them.""" + assert thinlto_cache_tracks_reads(), ( + f"{thinlto_cache_dir} does not record file reads (mounted with noatime?), " + "so the entries this build used are unknown" + ) + kept = 0 + removed = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + # Anything else is a temporary file of an interrupted link. + used = entry.name.startswith("llvmcache-") and entry.is_file( + follow_symlinks=False + ) and stat.st_atime > 0 + if used: + kept += 1 + size += stat.st_size + else: + os.remove(entry.path) + removed += 1 + print( + f"ThinLTO cache: kept {kept} entries ({size / (1 << 30):.1f} GiB), " + f"removed {removed} unused" + ) + assert size <= THINLTO_CACHE_MAX_BYTES[flavour], ( + f"ThinLTO cache of {size} bytes exceeds the {THINLTO_CACHE_MAX_BYTES[flavour]} " + f"bytes expected for a `-{flavour}` build, not uploading it" + ) + Shell.check( + f"set -o pipefail; tar -cf - -C {thinlto_cache_dir} . | zstd -T0 -3 --long=27 -c > {thinlto_cache_archive}", + verbose=True, + strict=True, + ) + print( + f"ThinLTO cache archive: {os.path.getsize(thinlto_cache_archive) / (1 << 30):.1f} GiB" + ) + S3.copy_file_to_s3( + s3_path=s3_path, local_path=thinlto_cache_archive, with_rename=True + ) + os.remove(thinlto_cache_archive) + return True + + def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") kwargs.setdefault("verbose", True) @@ -215,18 +334,16 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" - # ThinLTO cache: the link reuses the optimized native objects of modules - # that did not change since the previous link in this directory. It is an - # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that - # have it, so a builder whose disk is wiped between jobs has no directory - # here and does not pay for writing a cache it will never read. - thinlto_cache_dir = "/thinlto-cache" - use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir( - thinlto_cache_dir - ) - if use_thinlto_cache: - print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") - cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # ThinLTO cache for every build that links with ThinLTO. The cache-warmup + # builds only compile. Everyone reads the archive (community PRs anonymously); + # pushes and internal PRs write it, like the shared compiler cache. + thinlto_cache_s3 = None + if "ENABLE_THINLTO=1" in cmake_cmd and not cache_warmup: + thinlto_cache_s3 = thinlto_cache_s3_path(info, build_type, cmake_cmd) + thinlto_cache_writable = not info.is_local_run and not info.is_community_pr + if thinlto_cache_s3: + print(f"ThinLTO cache: {thinlto_cache_s3}, writable: {thinlto_cache_writable}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir_normalized}" # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link # spends most of its time in a single-threaded thin-link phase, so the two @@ -445,6 +562,15 @@ def do_checkout(): else: build_command = f"command time -v ninja {targets}" + if thinlto_cache_s3: + results.append( + Result.from_commands_run( + name="Restore ThinLTO cache", + command=restore_thinlto_cache, + command_args=[thinlto_cache_s3], + ) + ) + build_result_index = len(results) results.append( Result.from_commands_run( @@ -485,16 +611,20 @@ def do_checkout(): else: results.append(retry_cmake) - run_shell("sccache stats", "sccache --show-stats") - if use_thinlto_cache: - run_shell( - "thinlto cache stats", - # The volume's root-owned `lost+found` is unreadable for the - # build user, so count only lld's own entries. - f"du -sh --exclude=lost+found {thinlto_cache_dir}; " - f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; " - f"df -h {thinlto_cache_dir}", + if ( + thinlto_cache_s3 + and thinlto_cache_writable + and results[build_result_index].is_ok() + ): + results.append( + Result.from_commands_run( + name="Upload ThinLTO cache", + command=upload_thinlto_cache, + command_args=[thinlto_cache_s3, thinlto_cache_flavour(cmake_cmd)], + ) ) + + run_shell("sccache stats", "sccache --show-stats") if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log" diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 3d6013ce2d9f1..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -481,19 +481,6 @@ def _run( if not job.timeout_shell_cleanup: job.timeout_shell_cleanup = f"docker rm -f {container_name}" workdir = f"--workdir={current_dir}" - # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners - # that have it (e.g. an attached cache volume), instead of creating - # an empty directory everywhere like `--volume` does. - optional_volumes = [ - s for s in settings if s.startswith("--volume-if-exists=") - ] - for setting in optional_volumes: - settings.remove(setting) - volume = setting.removeprefix("--volume-if-exists=") - if Path(volume.split(":")[0]).exists(): - settings.append(f"--volume={volume}") - else: - print(f"NOTE: optional volume [{volume}] is not present on this runner") for setting in settings: if setting.startswith("--volume"): volume = setting.removeprefix("--volume=").split(":")[0] From d55de5fde5f8f442fe07b220e4ed96fefacea3a3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:36:34 -0400 Subject: [PATCH 14/14] Map the Hetzner cache bucket to its public endpoint `S3.copy_file_to_s3` uploaded the ThinLTO cache archive, then raised `KeyError: 'altinity-builds-cache'` while building the link it returns, which failed the release builds after a successful upload. https://github.com/Altinity/ClickHouse/actions/runs/41109373051/job/124345996580 Co-Authored-By: Claude Opus 5.5 --- ci/settings/altinity_overrides.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 101c6f492a047..aac84ca4835dd 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -29,6 +29,8 @@ class RunnerLabels: S3_BUCKET_TO_HTTP_ENDPOINT = { S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, + # S3.copy_file_to_s3 returns a public link to every upload + "altinity-builds-cache": "altinity-builds-cache.hel1.your-objectstorage.com", } ENABLE_ARTIFACTS_REPORT = True