Antalya 26.8 - ThinLTO link cache

.github/actions/runner_setup/action.yml

@@ -25,7 +25,7 @@ runs:
           ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
           .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
           sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
-          sudo sysctl vm.swappiness=200
+          sudo sysctl vm.swappiness=80
     - name: Install awscli
       shell: bash
       run: |

.github/workflows/fast_builds.yml

@@ -800,7 +800,7 @@ jobs:
         run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
 
   build_amd_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
     name: "Build (amd_release)"
@@ -844,7 +844,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp
 
   build_arm_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
     name: "Build (arm_release)"

.github/workflows/master.yml

@@ -859,7 +859,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp
 
   build_amd_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
     name: "Build (amd_release)"
@@ -903,7 +903,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp
 
   build_arm_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
     name: "Build (arm_release)"
@@ -947,7 +947,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp
 
   build_amd_release_pr_cache_warmup:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }}
     name: "Build (amd_release_pr_cache_warmup)"
@@ -991,7 +991,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp
 
   build_arm_release_pr_cache_warmup:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }}
     name: "Build (arm_release_pr_cache_warmup)"

.github/workflows/pull_request.yml

@@ -847,7 +847,7 @@ jobs:
         run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
 
   build_amd_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd]
     needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
     name: "Build (amd_release)"
@@ -889,7 +889,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp
 
   build_arm_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm]
     needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
     name: "Build (arm_release)"

.github/workflows/pull_request_community.yml

@@ -817,7 +817,7 @@ jobs:
           retention-days: 14
 
   build_amd_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd]
     needs: [config_workflow, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
     name: "Build (amd_release)"
@@ -909,7 +909,7 @@ jobs:
           retention-days: 14
 
   build_arm_release:
-    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm]
     needs: [config_workflow, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
     name: "Build (arm_release)"

CMakeLists.txt

@@ -493,6 +493,22 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
         set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility")
     endif()
 
+    # Persistent ThinLTO cache. lld stores the optimized native object of every
+    # module keyed by the module hash, its import set and the LTO flags, and
+    # reuses it on the next link when nothing in that key changed. Only the
+    # backend codegen of the link is cached; the thin-link and the final native
+    # link still run in full. Entries are content-addressed, so concurrent
+    # writers cannot corrupt each other.
+    set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache")
+    set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)")
+    if (THINLTO_CACHE_DIR)
+        if (NOT LINKER_NAME MATCHES "lld")
+            message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'")
+        endif ()
+        message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'")
+        set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}")
+    endif ()
+
     if (ENABLE_CFI)
         # CFI checks: virtual dispatch type safety and bad downcasts via static_cast.
         # cfi-icall is omitted because C function pointers in contrib libraries (compiled without

ci/defs/defs.py

@@ -18,8 +18,8 @@
 class RunnerLabels:
     CI_SERVICES = "ci_services"
     CI_SERVICES_EBS = "ci_services_ebs"
-    BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"]
-    BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"]
+    BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"]
+    BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"]
     FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"]
     FUNC_TESTER_ARM = [
         "self-hosted",

ci/defs/job_configs.py

@@ -29,6 +29,15 @@
     f"+--memory={Utils.physical_memory() * 95 // 100}"
     f"+--memory-reservation={Utils.physical_memory() * 9 // 10}"
     f"+--volume=.:/ClickHouse"
+    # ThinLTO cache of release builds, see `use_thinlto_cache` in
+    # build_clickhouse.py. Mounted only on builders that have it, so the build
+    # job can key on the directory's presence. Cloud builders get one attached
+    # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static
+    # builders keep a single directory on their own disk for both targets. A
+    # builder has exactly one of the three.
+    "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache"
+    "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache"
+    "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache"
     '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"'
 )
 BINARY_DOCKER_COMMAND_SANITIZER = (
@@ -443,7 +452,7 @@ class JobConfigs:
                 ArtifactNames.RPM_ARM_RELEASE,
                 ArtifactNames.TGZ_ARM_RELEASE,
             ],
-            runs_on=RunnerLabels.BUILDER_AMD,
+            runs_on=RunnerLabels.BUILDER_ARM,
         ),
     )
     release_build_jobs_with_examples = [
@@ -458,7 +467,7 @@ class JobConfigs:
         Job.ParamSet(
             parameter=BuildTypes.AMD_CFI,
             provides=[ArtifactNames.CH_AMD_CFI],
-            runs_on=RunnerLabels.ARM_LARGE,
+            runs_on=RunnerLabels.BUILDER_AMD,
             timeout=4 * 3600,
         ),
     )
@@ -501,12 +510,12 @@ class JobConfigs:
         ).parametrize(
             Job.ParamSet(
                 parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP,
-                runs_on=RunnerLabels.ARM_LARGE,
+                runs_on=RunnerLabels.BUILDER_AMD,
                 timeout=3 * 3600,
             ),
             Job.ParamSet(
                 parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP,
-                runs_on=RunnerLabels.ARM_LARGE,
+                runs_on=RunnerLabels.BUILDER_ARM,
             ),
         )
     ]

ci/jobs/build_clickhouse.py

@@ -105,6 +105,7 @@ def parse_args():
 
 def run_shell(name, command, **kwargs):
     print(f"\n>>>> {name}\n")
+    kwargs.setdefault("verbose", True)
     Shell.check(command, **kwargs)
     print(f"\n<<<< {name}\n")
 
@@ -214,6 +215,28 @@ def main():
     ):
         cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1"
 
+    # ThinLTO cache: the link reuses the optimized native objects of modules
+    # that did not change since the previous link in this directory. It is an
+    # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that
+    # have it, so a builder whose disk is wiped between jobs has no directory
+    # here and does not pay for writing a cache it will never read.
+    thinlto_cache_dir = "/thinlto-cache"
+    use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir(
+        thinlto_cache_dir
+    )
+    if use_thinlto_cache:
+        print(f"ThinLTO cache is enabled in {thinlto_cache_dir}")
+        cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}"
+
+    # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link
+    # spends most of its time in a single-threaded thin-link phase, so the two
+    # overlap almost entirely: measured on a 16-core, 62 GB builder with zram,
+    # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived
+    # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would
+    # allow only one; the real peak of both links together was ~48 GB.
+    if "ENABLE_THINLTO=1" in cmake_cmd:
+        cmake_cmd += " -DPARALLEL_LINK_JOBS=2"
+
     # PGO/BOLT profile integration for release builds. The sccache-warmup builds
     # are deliberately excluded: they exist only to populate the shared compiler
     # cache, and PGO/BOLT belong to the real release builds.
@@ -463,6 +486,15 @@ def do_checkout():
                 results.append(retry_cmake)
 
         run_shell("sccache stats", "sccache --show-stats")
+        if use_thinlto_cache:
+            run_shell(
+                "thinlto cache stats",
+                # The volume's root-owned `lost+found` is unreadable for the
+                # build user, so count only lld's own entries.
+                f"du -sh --exclude=lost+found {thinlto_cache_dir}; "
+                f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; "
+                f"df -h {thinlto_cache_dir}",
+            )
         if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY):
             run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats")
             clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log"

ci/praktika/runner.py

@@ -480,6 +480,19 @@ def _run(
             if not job.timeout_shell_cleanup:
                 job.timeout_shell_cleanup = f"docker rm -f {container_name}"
             workdir = f"--workdir={current_dir}"
+            # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners
+            # that have it (e.g. an attached cache volume), instead of creating
+            # an empty directory everywhere like `--volume` does.
+            optional_volumes = [
+                s for s in settings if s.startswith("--volume-if-exists=")
+            ]
+            for setting in optional_volumes:
+                settings.remove(setting)
+                volume = setting.removeprefix("--volume-if-exists=")
+                if Path(volume.split(":")[0]).exists():
+                    settings.append(f"--volume={volume}")
+                else:
+                    print(f"NOTE: optional volume [{volume}] is not present on this runner")
             for setting in settings:
                 if setting.startswith("--volume"):
                     volume = setting.removeprefix("--volume=").split(":")[0]

cmake/limit_jobs.cmake

@@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY)
     endif()
 endif()
 
+# Remember whether the link-job limit was set explicitly: the ThinLTO memory
+# cap below only applies to the auto-calculated value.
+set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS})
+
 if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY)
     math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY})
 
@@ -54,7 +58,7 @@ endif ()
 # links than physically fit in memory - otherwise the linker is OOM-killed.
 # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only
 # ~50 GB, so two still fit and we keep them parallel.
-if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1)
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT)
     if (DISABLE_ALL_DEBUG_SYMBOLS)
         set (MAX_THINLTO_LINK_MEMORY 51200)  # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0
     else ()