diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index e978d8f57a741..a9f9d0d341f41 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -25,7 +25,7 @@ runs: ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 - sudo sysctl vm.swappiness=200 + sudo sysctl vm.swappiness=80 - name: Install awscli shell: bash run: | diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..ff8a7c80dc874 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..b65ab01074036 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..e8244433825c2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index e3c6626cbac83..0844cfb532a7a 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b194ad629c3..971a108900ed6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,22 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility") endif() + # Persistent ThinLTO cache. lld stores the optimized native object of every + # module keyed by the module hash, its import set and the LTO flags, and + # reuses it on the next link when nothing in that key changed. Only the + # backend codegen of the link is cached; the thin-link and the final native + # link still run in full. Entries are content-addressed, so concurrent + # writers cannot corrupt each other. + set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + if (THINLTO_CACHE_DIR) + if (NOT LINKER_NAME MATCHES "lld") + message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") + endif () + message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}") + endif () + if (ENABLE_CFI) # CFI checks: virtual dispatch type safety and bad downcasts via static_cast. # cfi-icall is omitted because C function pointers in contrib libraries (compiled without diff --git a/ci/defs/defs.py b/ci/defs/defs.py index cd3681eece7e4..12d32fd34d46f 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..bcb528e8d5259 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,6 +29,15 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" + # ThinLTO cache of release builds, see `use_thinlto_cache` in + # build_clickhouse.py. Mounted only on builders that have it, so the build + # job can key on the directory's presence. Cloud builders get one attached + # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static + # builders keep a single directory on their own disk for both targets. A + # builder has exactly one of the three. + "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache" + "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache" + "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( @@ -443,7 +452,7 @@ class JobConfigs: ArtifactNames.RPM_ARM_RELEASE, ArtifactNames.TGZ_ARM_RELEASE, ], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.BUILDER_ARM, ), ) release_build_jobs_with_examples = [ @@ -458,7 +467,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_AMD, timeout=4 * 3600, ), ) @@ -501,12 +510,12 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_AMD, timeout=3 * 3600, ), Job.ParamSet( parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, ), ) ] diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 23eec7551006d..8a67a904e96f1 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -105,6 +105,7 @@ def parse_args(): def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") + kwargs.setdefault("verbose", True) Shell.check(command, **kwargs) print(f"\n<<<< {name}\n") @@ -214,6 +215,28 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" + # ThinLTO cache: the link reuses the optimized native objects of modules + # that did not change since the previous link in this directory. It is an + # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that + # have it, so a builder whose disk is wiped between jobs has no directory + # here and does not pay for writing a cache it will never read. + thinlto_cache_dir = "/thinlto-cache" + use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir( + thinlto_cache_dir + ) + if use_thinlto_cache: + print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + + # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link + # spends most of its time in a single-threaded thin-link phase, so the two + # overlap almost entirely: measured on a 16-core, 62 GB builder with zram, + # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived + # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would + # allow only one; the real peak of both links together was ~48 GB. + if "ENABLE_THINLTO=1" in cmake_cmd: + cmake_cmd += " -DPARALLEL_LINK_JOBS=2" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. @@ -463,6 +486,15 @@ def do_checkout(): results.append(retry_cmake) run_shell("sccache stats", "sccache --show-stats") + if use_thinlto_cache: + run_shell( + "thinlto cache stats", + # The volume's root-owned `lost+found` is unreadable for the + # build user, so count only lld's own entries. + f"du -sh --exclude=lost+found {thinlto_cache_dir}; " + f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; " + f"df -h {thinlto_cache_dir}", + ) if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log" diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..5185f4b81640b 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -480,6 +480,19 @@ def _run( if not job.timeout_shell_cleanup: job.timeout_shell_cleanup = f"docker rm -f {container_name}" workdir = f"--workdir={current_dir}" + # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners + # that have it (e.g. an attached cache volume), instead of creating + # an empty directory everywhere like `--volume` does. + optional_volumes = [ + s for s in settings if s.startswith("--volume-if-exists=") + ] + for setting in optional_volumes: + settings.remove(setting) + volume = setting.removeprefix("--volume-if-exists=") + if Path(volume.split(":")[0]).exists(): + settings.append(f"--volume={volume}") + else: + print(f"NOTE: optional volume [{volume}] is not present on this runner") for setting in settings: if setting.startswith("--volume"): volume = setting.removeprefix("--volume=").split(":")[0] diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8be747c6f27f0..366e3f71a8ba4 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) endif() endif() +# Remember whether the link-job limit was set explicitly: the ThinLTO memory +# cap below only applies to the auto-calculated value. +set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS}) + if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) @@ -54,7 +58,7 @@ endif () # links than physically fit in memory - otherwise the linker is OOM-killed. # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only # ~50 GB, so two still fit and we keep them parallel. -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT) if (DISABLE_ALL_DEBUG_SYMBOLS) set (MAX_THINLTO_LINK_MEMORY 51200) # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0 else ()