diff --git a/.github/actions/create_workflow_report/action.yml b/.github/actions/create_workflow_report/action.yml index 7ede97fbaed16..4953860c6b603 100644 --- a/.github/actions/create_workflow_report/action.yml +++ b/.github/actions/create_workflow_report/action.yml @@ -1,6 +1,10 @@ name: Create and Upload Combined Report description: Create and upload a combined CI report inputs: + workflow_config: + description: "Workflow config (26.3 FinishCIReport still passes this)" + required: false + default: "" final: description: "Control whether the report is final or a preview" required: false @@ -8,6 +12,15 @@ inputs: runs: using: "composite" steps: + - name: Create workflow config + if: ${{ inputs.workflow_config != '' }} + shell: bash + run: | + mkdir -p ./ci/tmp + cat > ./ci/tmp/workflow_config.json << 'EOF' + ${{ inputs.workflow_config }} + EOF + - name: Create and upload workflow report env: PR_NUMBER: ${{ github.event.pull_request.number || 0 }} @@ -16,7 +29,10 @@ runs: FINAL: ${{ inputs.final }} shell: bash run: | - pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 jinja2==3.1.5 + # Newer runners have these deps preinstalled; only install on Ubuntu 22. + if [[ -f /etc/os-release ]] && . /etc/os-release && [[ "$VERSION_ID" == "22.04" ]]; then + pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 jinja2==3.1.5 + fi CMD="python3 .github/actions/create_workflow_report/create_workflow_report.py" ARGS="--actions-run-url $ACTIONS_RUN_URL --known-fails --cves --pr-number $PR_NUMBER" diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 5dd7c7bc664c9..ce4b342012a45 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -3,6 +3,7 @@ import html import os import sys +from glob import glob import time from pathlib import Path from itertools import combinations @@ -29,7 +30,10 @@ DATABASE_PASSWORD_VAR = "CLICKHOUSE_TEST_STAT_PASSWORD" S3_BUCKET = "altinity-build-artifacts" GITHUB_REPO = "Altinity/ClickHouse" +GITHUB_GIT_URL = f"https://github.com/{GITHUB_REPO}.git" GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") +# Release tags are fetched here rather than refs/tags/ so local tags are never touched. +RELEASE_TAG_REF_PREFIX = "refs/workflow-report/tags/" # Resolve from this file so discovery works from repo root or the action folder. KNOWN_FAILS_FILE = ( Path(__file__).resolve().parents[3] / "tests" / "broken_tests.yaml" @@ -61,13 +65,28 @@ def query_dataframe_with_retry( wait = backoff_seconds * attempt time.sleep(wait) -def get_commit_statuses(sha: str) -> pd.DataFrame: +def _results_link_has_ref(target_url: str | None, ref: str) -> bool: + """ + Whether a CI status results link has `ref` in it:`json.html?REF=&...` for + praktika jobs, `/REFs//...` for regression jobs. + """ + target_url = target_url or "" + return f"REF={ref}&" in target_url or f"/REFs/{ref}/" in target_url + + +def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: """ Fetch commit statuses for a given SHA and return as a pandas DataFrame. Handles pagination to get all statuses. + Statuses belong to the commit, not the ref, so a release tag and its branch + can write to the same job names. For release runs (`pr_number` 0), only + statuses of `ref` are kept. + Args: sha (str): Commit SHA to fetch statuses for. + pr_number (int): Pull request number, 0 for release runs. + ref (str): Ref of the run being reported. Returns: pd.DataFrame: DataFrame containing all statuses. @@ -116,6 +135,7 @@ def get_commit_statuses(sha: str) -> pd.DataFrame: "results_link": item["target_url"], } for item in all_data + if pr_number != 0 or _results_link_has_ref(item["target_url"], ref) ] # Create DataFrame @@ -192,6 +212,9 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram f"https://api.github.com/repos/{repo}/pulls/{pr_number}", headers=headers, ) + if response.status_code == 404: + # NOTE (strtgbb): not in this repo — upstream PR merged from a fork + continue if response.status_code != 200: raise Exception( f"Failed to fetch pull request info: {response.status_code} {response.text}" @@ -207,12 +230,16 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram "pr_labels": html.escape(", ".join(sorted(label_names)), quote=True), } ) + if not rows: + return pd.DataFrame(columns=["pr_number", "pr_name", "pr_labels"]) return pd.DataFrame(rows) def _git_rev_parse(ref: str, cwd: str | None) -> str | None: + # Peel to a commit so a raw SHA that is not in the object store fails. + # git rev-parse --verify <40-hex> succeeds even when the object is missing. p = subprocess.run( - ["git", "rev-parse", "--verify", ref], + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=cwd, capture_output=True, text=True, @@ -232,7 +259,7 @@ def _git_is_ancestor(ancestor: str, descendant: str, cwd: str | None) -> bool: def _git_log_merge_prs( - baseline: str, branch_ref: str, cwd: str | None, repo: str + baseline: str, commit_sha: str, cwd: str | None, repo: str ) -> pd.DataFrame: p = subprocess.run( [ @@ -240,7 +267,7 @@ def _git_log_merge_prs( "-c", "core.quotepath=false", "log", - f"{baseline}..{branch_ref}", + f"{baseline}..{commit_sha}", "--merges", "--format=%H%x09%s", ], @@ -260,7 +287,7 @@ def _git_log_merge_prs( if not m: continue pr_number, head_owner = int(m.group(1)), m.group(2) - if head_owner.lower() != repo.split("/")[0].lower(): + if head_owner == 'ClickHouse': continue rows.append( { @@ -276,13 +303,11 @@ def _git_log_merge_prs( return df -def _find_release_baseline( - branch_ref: str, repo: str, cwd: str | None -) -> tuple[str | None, str | None, str | None]: - """Return (tag_name, tag_sha, published_date YYYY-MM-DD) for the latest - non-draft release tag that is an ancestor of branch_ref.""" +def _get_github_releases(repo: str) -> list[dict]: + """Non-draft GitHub releases of repo, newest first.""" if not GITHUB_TOKEN: - return None, None, None + print("GITHUB_TOKEN is not set; skipping GitHub releases lookup") + return [] headers = { "Authorization": f"token {GITHUB_TOKEN}", "Accept": "application/vnd.github.v3+json", @@ -295,16 +320,31 @@ def _find_release_baseline( raise Exception( f"GitHub API request failed: {response.status_code} {response.text}" ) - for rel in response.json(): - if rel.get("draft"): - continue + return [rel for rel in response.json() if not rel.get("draft")] + + +def _find_release_baseline( + branch_ref: str, + releases: list[dict], + cwd: str | None, + commit_sha: str, +) -> tuple[str | None, str | None, str | None]: + """Return (tag_name, tag_sha, published_date YYYY-MM-DD) for the latest + release tag in releases that is an ancestor of commit_sha. + Version-tag refs skip the matching release so the range is previous→this tag. + """ + parsed = _parse_release_ref(branch_ref) + skip_current = parsed is not None and parsed[2] is not None + for rel in releases: tag_name = rel.get("tag_name") if not tag_name: continue - tag_sha = _git_rev_parse(tag_name, cwd) + tag_sha = _release_tag_sha(tag_name, cwd) if not tag_sha: continue - if not _git_is_ancestor(tag_sha, branch_ref, cwd): + if not _git_is_ancestor(tag_sha, commit_sha, cwd): + continue + if skip_current and _git_is_ancestor(commit_sha, tag_sha, cwd): continue published_at = rel.get("published_at") or rel.get("created_at") or "" published_date = published_at[:10] if published_at else None @@ -312,12 +352,12 @@ def _find_release_baseline( return None, None, None -def _find_rebase_baseline(branch_ref: str, cwd: str | None) -> str | None: +def _find_rebase_baseline(commit_sha: str, cwd: str | None) -> str | None: p = subprocess.run( [ "git", "log", - branch_ref, + commit_sha, "--reverse", "-i", "-E", @@ -339,9 +379,9 @@ def _find_rebase_baseline(branch_ref: str, cwd: str | None) -> str | None: return lines[0] -def _git_commit_date(sha: str, cwd: str | None) -> str | None: +def _git_commit_time(sha: str, cwd: str | None) -> str | None: p = subprocess.run( - ["git", "show", "-s", "--format=%cs", sha], + ["git", "show", "-s", "--format=%ct", sha], cwd=cwd, capture_output=True, text=True, @@ -349,8 +389,113 @@ def _git_commit_date(sha: str, cwd: str | None) -> str | None: ) if p.returncode != 0: return None - date = p.stdout.strip() - return date or None + timestamp = p.stdout.strip() + if not timestamp: + return None + return datetime.fromtimestamp(int(timestamp), timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + + +def _git_is_shallow(cwd: str | None) -> bool: + p = subprocess.run( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return p.stdout.strip() == "true" + + +def _git_fetch(options: list[str], refspecs: list[str], cwd: str | None) -> None: + p = subprocess.run( + ["git", "fetch", *options, GITHUB_GIT_URL, *refspecs], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + if p.returncode != 0: + raise Exception(f"git fetch failed: {p.stderr.strip()}") + + +def _release_tag_sha(tag_name: str, cwd: str | None) -> str | None: + return _git_rev_parse(f"{RELEASE_TAG_REF_PREFIX}{tag_name}", cwd) + + +def _read_autogenerated_versions(sha: str, cwd: str | None) -> tuple[int, int]: + p = subprocess.run( + ["git", "show", f"{sha}:cmake/autogenerated_versions.txt"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + major = re.search(r"SET\(VERSION_MAJOR (\d+)\)", p.stdout) + minor = re.search(r"SET\(VERSION_MINOR (\d+)\)", p.stdout) + if not major or not minor: + raise Exception(f"Cannot read version from {sha}") + return int(major.group(1)), int(minor.group(1)) + + +def _filter_releases_to_version_line( + releases: list[dict], + branch_ref: str, + commit_sha: str, + cwd: str | None, +) -> list[dict]: + """Keep GitHub releases that belong on commit_sha's version line.""" + parsed = _parse_release_ref(branch_ref) + if parsed is not None: + flavour, minor_version, _ = parsed + suffix = f"altinity{flavour}" + prefix = f"v{minor_version}." + return [ + rel + for rel in releases + if (tag := rel.get("tag_name")) + and tag.startswith(prefix) + and tag.endswith(f".{suffix}") + ] + major, minor = _read_autogenerated_versions(commit_sha, cwd) + prefix = f"v{major}.{minor}." + return [ + rel + for rel in releases + if (tag := rel.get("tag_name")) and tag.startswith(prefix) + ] + + +def _fetch_release_tags(releases: list[dict], cwd: str) -> None: + """Fetch release tags into RELEASE_TAG_REF_PREFIX; skip tags missing remotely.""" + for rel in releases: + tag_name = rel.get("tag_name") + if not tag_name: + continue + try: + _git_fetch( + ["--no-tags"], + [f"+refs/tags/{tag_name}:{RELEASE_TAG_REF_PREFIX}{tag_name}"], + cwd, + ) + except Exception as e: + # NOTE (strtgbb): one deleted remote tag must not abort the section, but + # if it is the newest release the baseline moves back to an older one + print(f"WARNING: skipping release tag {tag_name}: {e}") + + +def _fetch_commit(commit_sha: str, cwd: str) -> None: + """Fetch the workflow commit from GITHUB_GIT_URL. + + Local checkouts might be a different branch and might not have the run SHA yet. + --filter/--unshallow would convert a full clone into a partial one, so those + options are used only when the checkout is already shallow. + """ + if _git_is_shallow(cwd): + _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd) + else: + _git_fetch(["--no-tags"], [commit_sha], cwd) def get_prs_in_release_dataframe( @@ -358,88 +503,70 @@ def get_prs_in_release_dataframe( *, repo: str = GITHUB_REPO, cwd: str, + commit_sha: str, ) -> tuple[pd.DataFrame, str | None]: f""" PRs merged into branch_ref that belong in the next release notes: after the latest GitHub Release tag on this history, or after the oldest rebase bootstrap if no such tag exists. - Only merge commits whose subject has from / (e.g. from Altinity/) are included. - Columns: pr_number, pr_name, labels. Omits PRs labeled cicd. + Merge commits whose subject is from the upstream repository (ClickHouse/) are omitted. + Columns: pr_number, pr_name, labels. - Returns (dataframe, baseline_date YYYY-MM-DD or None). + commit_sha is the workflow commit from the GitHub Actions API. + + Returns (dataframe, baseline commit time as UTC YYYY-MM-DDTHH:MM:SSZ or None). """ - branch_sha = _git_rev_parse(branch_ref, cwd) - if not branch_sha: - raise Exception(f"Cannot resolve branch ref: {branch_ref!r}") - - # CI often checks out with --depth=1 and --no-tags; fetch tags once, then deepen - # in steps until a baseline is reachable. - # Observed baseline distances are ~1k commits and grow slowly, so DEEPEN_CAP sits - # well above that; the cap exists so a failed lookup fails fast instead of fetching - # the branch's entire (100k+ commit) history. - DEEPEN_STEP = 250 - DEEPEN_CAP = 10000 - - subprocess.run( - ["git", "fetch", "--tags", "origin"], - cwd=cwd, - capture_output=True, - text=True, - check=False, + _fetch_commit(commit_sha, cwd) + resolved = _git_rev_parse(commit_sha, cwd) + if not resolved: + raise Exception(f"Cannot resolve commit: {commit_sha!r}") + commit_sha = resolved + + releases = _filter_releases_to_version_line( + _get_github_releases(repo), branch_ref, commit_sha, cwd ) + _fetch_release_tags(releases, cwd) - baseline_sha = None - baseline_date = None - for _ in range(DEEPEN_CAP // DEEPEN_STEP): - subprocess.run( - ["git", "fetch", f"--deepen={DEEPEN_STEP}", "origin", branch_ref], - cwd=cwd, - capture_output=True, - text=True, - check=False, - ) - _, baseline_sha, baseline_date = _find_release_baseline( - branch_ref, repo, cwd - ) - if not baseline_sha: - baseline_sha = _find_rebase_baseline(branch_ref, cwd) - if baseline_sha: - baseline_date = _git_commit_date(baseline_sha, cwd) - if baseline_sha: - break + _, baseline_sha, _ = _find_release_baseline( + branch_ref, releases, cwd, commit_sha=commit_sha + ) + if not baseline_sha: + baseline_sha = _find_rebase_baseline(commit_sha, cwd) if not baseline_sha: raise Exception( - "No release tag on this branch and no rebase bootstrap merge found " - f"within {DEEPEN_CAP} commits" + "No release tag on this branch and no rebase bootstrap merge found" ) - df = _git_log_merge_prs(baseline_sha, branch_ref, cwd, repo) - return _enrich_prs_in_release_merge_prs(df, repo), baseline_date + df = _git_log_merge_prs(baseline_sha, commit_sha, cwd, repo) + return ( + _enrich_prs_in_release_merge_prs(df, repo), + _git_commit_time(baseline_sha, cwd), + ) def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: """Parse a release ref into (flavour, minor_version, full_version). Handles version tags (``v25.8.28.10001.altinitystable``) and release - branches (``stable-25.8``). ``full_version`` is None for branches, which + branches (``stable-25.8``, ``antalya-25.8``, ``fips-25.3``). ``full_version`` is None for branches, which carry no patch or tweak. Returns None for refs without a version. """ # Match tag match = re.match( - r"^v((\d+\.\d+)\.\d+\.\d+)\.altinity(stable|antalya|test)$", ref_name + r"^v((\d+\.\d+)\.\d+\.\d+)\.altinity(stable|antalya|fips|test)$", ref_name ) if match: - flavour = "antalya" if match.group(3) == "antalya" else "stable" + flavour = match.group(3) if match.group(3) in ("antalya", "fips") else "stable" return flavour, match.group(2), match.group(1) # Match branch - match = re.match(r"^(stable|antalya)-(\d+\.\d+)$", ref_name) + match = re.match(r"^(stable|antalya|fips)-(\d+\.\d+)$", ref_name) if match: return match.group(1), match.group(2), None return None -def _prs_in_release_search_url(branch_name: str, baseline_date: str | None) -> str | None: - if not baseline_date: +def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> str | None: + if not baseline_time: return None # Tag-triggered runs expose the tag as head_branch; search needs the PR base # branch (e.g. v25.8.28.10001.altinitystable → stable-25.8). @@ -447,7 +574,7 @@ def _prs_in_release_search_url(branch_name: str, baseline_date: str | None) -> s if parsed: flavour, minor_version, _ = parsed branch_name = f"{flavour}-{minor_version}" - query = f"is:pr base:{branch_name} merged:>{baseline_date}" + query = f"is:pr base:{branch_name} merged:>{baseline_time}" return ( f"https://github.com/{GITHUB_REPO}/pulls?" + urllib.parse.urlencode({"q": query}) @@ -470,7 +597,12 @@ def _checks_latest_test_status_cte(commit_sha: str, branch_name: str) -> str: check_start_time, maxIf( check_start_time, - NOT (check_name LIKE 'Stateless%' AND NOT match(test_name, '^[0-9]{{5}}')) + test_name != '' + AND test_name NOT IN ('Pre Hooks', 'Post Hooks') + AND NOT ( + check_name LIKE 'Stateless%' + AND NOT match(test_name, '^[0-9]{{5}}') + ) ) OVER (PARTITION BY check_name) AS latest_check_start_time FROM `gh-data`.checks WHERE commit_sha = '{commit_sha}' AND head_ref = '{branch_name}' @@ -686,6 +818,10 @@ def get_new_fails_this_pr( # Combine both types of fails and select only desired columns desired_columns = ["job_name", "test_name", "test_status", "results_link"] + if len(checks_fails) > 0 and "labels" in checks_fails.columns: + desired_columns.insert(desired_columns.index("results_link"), "labels") + if len(regression_fails) > 0: + regression_fails["labels"] = "" all_pr_fails = pd.concat([checks_fails, regression_fails], ignore_index=True)[ desired_columns ] @@ -757,10 +893,9 @@ def get_new_fails_this_pr( def _workflow_config_s3_url(pr_number: int, commit_sha: str, ref_name: str) -> str: if pr_number == 0: - # REF uploads use a double slash before config_workflow in the S3 key. return ( f"https://{S3_BUCKET}.s3.amazonaws.com/REFs/{ref_name}/{commit_sha}/" - "/config_workflow/workflow_config_masterci.json" + "config_workflow/workflow_config_masterci.json" ) return ( f"https://{S3_BUCKET}.s3.amazonaws.com/PRs/{pr_number}/{commit_sha}/" @@ -768,24 +903,48 @@ def _workflow_config_s3_url(pr_number: int, commit_sha: str, ref_name: str) -> s ) +def _workflow_config_from_local() -> dict | None: + # NOTE (strtgbb): 25.8 jobs still write ci/tmp/workflow_config*.json + # (including workflow_config_release_builds.json). 26.8 S3 lookup only knows + # masterci/pr filenames, so local wins when present. + if os.path.exists("./ci/tmp/workflow_status.json"): + with open("./ci/tmp/workflow_status.json", "r") as f: + data = json.load(f)["config_workflow"]["outputs"]["data"] + if isinstance(data, str): + data = json.loads(data) + if isinstance(data, dict) and "WORKFLOW_CONFIG" in data: + return data["WORKFLOW_CONFIG"] + if isinstance(data, dict): + return data + workflow_config_files = glob("./ci/tmp/workflow_config*.json") + if len(workflow_config_files) == 1: + with open(workflow_config_files[0], "r") as f: + return json.load(f) + return None + + @lru_cache def get_workflow_config(pr_number: int, commit_sha: str, ref_name: str) -> dict: - """Fetch workflow config published by Config Workflow from S3. + """Load workflow config from the local run, then S3. On failure, warn and return a minimal empty config (same shape recreation uses). """ + local = _workflow_config_from_local() + if local is not None: + return local url = _workflow_config_s3_url(pr_number, commit_sha, ref_name) try: response = requests.get(url, timeout=30) if response.status_code != 200: print( f"WARNING:Failed to fetch workflow config from {url}: " - f"{response.status_code}" + f"{response.status_code}", + file=sys.stderr, ) return {"cache_jobs": {}} return response.json() except Exception as e: - print(f"WARNING:Failed to fetch workflow config from {url}: {e}") + print(f"WARNING:Failed to fetch workflow config from {url}: {e}", file=sys.stderr) return {"cache_jobs": {}} @@ -972,6 +1131,82 @@ def format_test_status(text: str) -> str: return f'{text}' +def _label_names_from_ext(ext: dict) -> list[str]: + names = [] + for item in ext.get("labels") or []: + if isinstance(item, str): + name = item + elif isinstance(item, dict) and item.get("name"): + name = item["name"] + else: + continue + if name != "cidb": + names.append(name) + return names + + +def fetch_workflow_result_json( + pr_number: int, branch: str, commit_sha: str +) -> dict | None: + if pr_number == 0: + ref_param = f"REF={branch}" + workflow_name = "MasterCI" + else: + ref_param = f"PR={pr_number}" + workflow_name = "PR" + + status_file = f"result_{workflow_name.lower()}.json" + s3_path = ( + f"https://{S3_BUCKET}.s3.amazonaws.com/" + f"{ref_param.replace('=', 's/')}/{commit_sha}/{status_file}" + ) + try: + response = requests.get(s3_path, timeout=30) + if response.status_code != 200: + return None + return response.json() + except Exception as e: + print(f"WARNING:Failed to fetch workflow result from {s3_path}: {e}") + return None + + +def get_failure_labels_from_workflow(workflow_data: dict | None) -> dict: + if not workflow_data: + return {} + labels_map = {} + for job in workflow_data.get("results") or []: + job_name = job.get("name") + if not job_name: + continue + for leaf in job.get("results") or []: + test_name = leaf.get("name") + if not test_name: + continue + names = _label_names_from_ext(leaf.get("ext") or {}) + if names: + labels_map[(job_name, test_name)] = ", ".join(names) + return labels_map + + +def add_labels_to_checks_fails( + checks_fails: pd.DataFrame, workflow_data: dict | None +) -> pd.DataFrame: + if checks_fails is None or len(checks_fails) == 0: + return checks_fails + labels_map = get_failure_labels_from_workflow(workflow_data) + df = checks_fails.copy() + df["labels"] = df.apply( + lambda row: labels_map.get((row["job_name"], row["test_name"]), ""), + axis=1, + ) + cols = [c for c in df.columns if c != "labels"] + if "results_link" in cols: + cols.insert(cols.index("results_link"), "labels") + else: + cols.append("labels") + return df[cols] + + def format_results_as_html_table(results, *, branch_name: str = "") -> str: if not isinstance(results, pd.DataFrame): return results @@ -1008,47 +1243,36 @@ def format_col_name(col_name: str) -> str: "PR Labels": lambda labels: format_pr_labels_with_verification( labels, branch_name=branch_name ), + "Labels": lambda labels: html.escape(str(labels), quote=True) if labels else "", } - html = results.to_html( + return results.to_html( index=False, formatters=formatters, escape=False, border=0, classes=["test-results-table"], ) - return html def backfill_skipped_statuses( - job_statuses: pd.DataFrame, pr_number: int, branch: str, commit_sha: str + job_statuses: pd.DataFrame, + workflow_result: dict | None, ): """ Fill in the job statuses for skipped jobs. """ - - if pr_number == 0: - ref_param = f"REF={branch}" - workflow_name = "MasterCI" - else: - ref_param = f"PR={pr_number}" - workflow_name = "PR" - - status_file = f"result_{workflow_name.lower()}.json" - s3_path = f"https://{S3_BUCKET}.s3.amazonaws.com/{ref_param.replace('=', 's/')}/{commit_sha}/{status_file}" - response = requests.get(s3_path) - - if response.status_code != 200: + if workflow_result is None: return job_statuses - status_data = response.json() skipped_jobs = [] - for job in status_data["results"]: - if job["status"] == "skipped" and len(job["links"]) > 0: + for job in workflow_result["results"]: + status = job["status"].lower() + if status == "skipped" and len(job["links"]) > 0: skipped_jobs.append( { "job_name": job["name"], - "job_status": job["status"], + "job_status": status, "message": job["info"], "results_link": job["links"][0], } @@ -1188,10 +1412,15 @@ def create_workflow_report( settings={"use_numpy": True}, ) + workflow_result = fetch_workflow_result_json(pr_number, branch_name, commit_sha) + results_dfs = { "prs_in_release": [], - "job_statuses": get_commit_statuses(commit_sha), - "checks_fails": get_checks_fails(db_client, commit_sha, branch_name), + "job_statuses": get_commit_statuses(commit_sha, pr_number, branch_name), + "checks_fails": add_labels_to_checks_fails( + get_checks_fails(db_client, commit_sha, branch_name), + workflow_result, + ), "checks_known_fails": [], "pr_new_fails": [], "checks_errors": get_checks_errors(db_client, commit_sha, branch_name), @@ -1203,12 +1432,12 @@ def create_workflow_report( if pr_number == 0 and not mark_preview: try: - prs_df, baseline_date = get_prs_in_release_dataframe( - branch_name, cwd=os.getcwd() + prs_df, baseline_time = get_prs_in_release_dataframe( + branch_name, cwd=os.getcwd(), commit_sha=commit_sha ) results_dfs["prs_in_release"] = prs_df prs_in_release_search_url = _prs_in_release_search_url( - branch_name, baseline_date + branch_name, baseline_time ) except Exception as e: print(f"Error in get_prs_in_release_dataframe: {e}") @@ -1253,7 +1482,8 @@ def create_workflow_report( pr_info = {} results_dfs["job_statuses"] = backfill_skipped_statuses( - results_dfs["job_statuses"], pr_number, branch_name, commit_sha + results_dfs["job_statuses"], + workflow_result, ) high_cve_count = 0 diff --git a/.github/actions/create_workflow_report/test_prs_in_release.py b/.github/actions/create_workflow_report/test_prs_in_release.py new file mode 100644 index 0000000000000..695e92fa5330a --- /dev/null +++ b/.github/actions/create_workflow_report/test_prs_in_release.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Unit tests for PRs-in-release baseline selection and merge filtering.""" + +import io +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import MagicMock, patch + +import pandas as pd + +from create_workflow_report import ( + RELEASE_TAG_REF_PREFIX, + _checks_latest_test_status_cte, + _enrich_prs_in_release_merge_prs, + _fetch_release_tags, + _fetch_commit, + _find_release_baseline, + _get_github_releases, + _git_log_merge_prs, + _git_rev_parse, + _parse_release_ref, + _release_tag_sha, + _filter_releases_to_version_line, + get_prs_in_release_dataframe, + get_workflow_config, +) + +CURRENT_TAG = "v26.6.4.20001.altinityantalya" +PREV_TAG = "v26.6.2.20001.altinityantalya" +CURRENT_SHA = "aa" * 20 +PREV_SHA = "bb" * 20 +# Linear history: PREV_SHA is an ancestor of CURRENT_SHA (and of itself). +COMMIT_ORDER = {PREV_SHA: 0, CURRENT_SHA: 1} +REF_TO_SHA = { + CURRENT_TAG: CURRENT_SHA, + PREV_TAG: PREV_SHA, + CURRENT_SHA: CURRENT_SHA, + PREV_SHA: PREV_SHA, + "antalya-26.6": CURRENT_SHA, +} + +RELEASES = [ + { + "draft": False, + "tag_name": CURRENT_TAG, + "published_at": "2026-09-11T14:02:51Z", + }, + { + "draft": False, + "tag_name": PREV_TAG, + "published_at": "2026-08-26T23:29:39Z", + }, +] + + +def _rev_parse(ref, cwd): + return REF_TO_SHA.get(ref.removeprefix(RELEASE_TAG_REF_PREFIX)) + + +def _is_ancestor(ancestor, descendant, cwd): + a = REF_TO_SHA.get(ancestor, ancestor) + d = REF_TO_SHA.get(descendant, descendant) + if a not in COMMIT_ORDER or d not in COMMIT_ORDER: + return False + return COMMIT_ORDER[a] <= COMMIT_ORDER[d] + + +def _releases_response(): + response = MagicMock() + response.status_code = 200 + response.json.return_value = RELEASES + [ + {"draft": True, "tag_name": "v26.9.1.1.altinityantalya"} + ] + return response + + +class TestGetGithubReleases(unittest.TestCase): + def test_skips_drafts(self): + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_releases_response(), + ), + ): + releases = _get_github_releases("Altinity/ClickHouse") + self.assertEqual(releases, RELEASES) + + def test_no_token_returns_empty(self): + with ( + patch("create_workflow_report.GITHUB_TOKEN", None), + patch("builtins.print") as printed, + ): + self.assertEqual(_get_github_releases("Altinity/ClickHouse"), []) + printed.assert_called_once() + self.assertIn("GITHUB_TOKEN", printed.call_args.args[0]) + + +class TestFindReleaseBaseline(unittest.TestCase): + def test_tag_ref_skips_current_github_release(self): + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + CURRENT_TAG, RELEASES, cwd=None, commit_sha=CURRENT_SHA + ) + self.assertEqual(tag_name, PREV_TAG) + self.assertEqual(tag_sha, PREV_SHA) + self.assertEqual(published_date, "2026-08-26") + + def test_branch_ref_keeps_latest_github_release(self): + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + "antalya-26.6", RELEASES, cwd=None, commit_sha=CURRENT_SHA + ) + self.assertEqual(tag_name, CURRENT_TAG) + self.assertEqual(tag_sha, CURRENT_SHA) + self.assertEqual(published_date, "2026-09-11") + + def test_ancestry_uses_commit_not_branch_name(self): + # Local antalya-26.6 may sit on the latest release; the run commit is older. + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + "antalya-26.6", RELEASES, cwd=None, commit_sha=PREV_SHA + ) + self.assertEqual(tag_name, PREV_TAG) + self.assertEqual(tag_sha, PREV_SHA) + self.assertEqual(published_date, "2026-08-26") + + +def _version_file(major, minor): + return subprocess.CompletedProcess( + [], 0, stdout=f"SET(VERSION_MAJOR {major})\nSET(VERSION_MINOR {minor})\n" + ) + + +class TestFetchCommit(unittest.TestCase): + def test_full_clone_fetches_commit_without_clone_shape_flags(self): + with ( + patch("create_workflow_report._git_is_shallow", return_value=False), + patch("create_workflow_report._git_fetch") as fetch, + ): + _fetch_commit(CURRENT_SHA, ".") + fetch.assert_called_once_with(["--no-tags"], [CURRENT_SHA], ".") + + def test_shallow_clone_unshallows_commit(self): + with ( + patch("create_workflow_report._git_is_shallow", return_value=True), + patch("create_workflow_report._git_fetch") as fetch, + ): + _fetch_commit(PREV_SHA, ".") + fetch.assert_called_once_with( + ["--filter=tree:0", "--unshallow"], [PREV_SHA], "." + ) + + +class TestReleaseTagSha(unittest.TestCase): + def test_namespaced_only_ignores_local_tag(self): + with patch( + "create_workflow_report._git_rev_parse", + side_effect=lambda ref, cwd: ( + "private" if ref.startswith(RELEASE_TAG_REF_PREFIX) else "local" + ), + ): + self.assertEqual(_release_tag_sha(CURRENT_TAG, None), "private") + with patch( + "create_workflow_report._git_rev_parse", + side_effect=lambda ref, cwd: ( + None if ref.startswith(RELEASE_TAG_REF_PREFIX) else "local" + ), + ): + self.assertIsNone(_release_tag_sha(CURRENT_TAG, None)) + + +class TestReleasesForVersionLine(unittest.TestCase): + def test_keeps_flavour_and_minor_drops_others(self): + mixed = RELEASES + [ + { + "draft": False, + "tag_name": "v25.8.33.10001.altinitystable", + "published_at": "2026-09-01T00:00:00Z", + }, + { + "draft": False, + "tag_name": "v26.6.8.10001.altinitystable", + "published_at": "2026-10-01T00:00:00Z", + }, + ] + out = _filter_releases_to_version_line(mixed, "antalya-26.6", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [CURRENT_TAG, PREV_TAG]) + + def test_odd_branch_reads_version_file(self): + mixed = RELEASES + [ + { + "draft": False, + "tag_name": "v25.8.33.10001.altinitystable", + "published_at": "2026-09-01T00:00:00Z", + }, + ] + with patch( + "create_workflow_report._read_autogenerated_versions", + return_value=(26, 6), + ): + out = _filter_releases_to_version_line(mixed, "odd-branch", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [CURRENT_TAG, PREV_TAG]) + + +class TestFipsRefs(unittest.TestCase): + def test_parses_fips_branch_and_tag(self): + self.assertEqual(_parse_release_ref("fips-25.3"), ("fips", "25.3", None)) + self.assertEqual( + _parse_release_ref("v25.3.8.30001.altinityfips"), + ("fips", "25.3", "25.3.8.30001"), + ) + + def test_fips_branch_keeps_only_fips_tags(self): + fips_tag = "v25.3.8.30001.altinityfips" + releases = [ + {"draft": False, "tag_name": fips_tag}, + {"draft": False, "tag_name": "v25.3.8.20001.altinityantalya"}, + {"draft": False, "tag_name": "v25.3.8.10001.altinitystable"}, + ] + out = _filter_releases_to_version_line(releases, "fips-25.3", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [fips_tag]) + + +class TestFetchReleaseTagsWarning(unittest.TestCase): + def test_missing_tag_is_reported(self): + def _fetch(options, refspecs, cwd): + if CURRENT_TAG in refspecs[0]: + raise Exception("git fetch failed: not found") + + with ( + patch("create_workflow_report._git_fetch", side_effect=_fetch), + patch("builtins.print") as printed, + ): + _fetch_release_tags(RELEASES, ".") + message = printed.call_args.args[0] + self.assertIn(CURRENT_TAG, message) + self.assertIn("not found", message) + + +class TestGitRevParse(unittest.TestCase): + def test_missing_full_sha_is_unresolved(self): + with tempfile.TemporaryDirectory() as cwd: + subprocess.run(["git", "init", "-q"], cwd=cwd, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "init", + ], + cwd=cwd, + check=True, + ) + head = _git_rev_parse("HEAD", cwd) + self.assertRegex(head, r"^[0-9a-f]{40}$") + self.assertEqual(_git_rev_parse(head, cwd), head) + self.assertIsNone(_git_rev_parse("deadbeef" * 5, cwd)) + + +class TestGetPrsInReleaseCommitResolve(unittest.TestCase): + def test_raises_when_commit_cannot_resolve(self): + with ( + patch("create_workflow_report._fetch_commit"), + patch("create_workflow_report._git_rev_parse", return_value=None), + ): + with self.assertRaises(Exception) as ctx: + get_prs_in_release_dataframe( + "antalya-26.6", cwd=".", commit_sha="deadbeef" * 5 + ) + self.assertIn("Cannot resolve commit", str(ctx.exception)) + self.assertIn("deadbeef", str(ctx.exception)) + + +class TestGitLogMergePrs(unittest.TestCase): + def test_includes_fork_and_altinity_omits_upstream(self): + stdout = "\n".join( + [ + f"{'e' * 40}\tMerge pull request #2226 from strtgbb/community-pr-failure-artifacts", + f"{'a' * 40}\tMerge pull request #113355 from ClickHouse/backport/26.6/112784", + f"{'b' * 40}\tMerge pull request #2252 from Altinity/ci/antalya-26.6/container-dns", + ] + ) + with patch( + "create_workflow_report.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, stdout=stdout), + ): + df = _git_log_merge_prs("baseline", "branch", None, "Altinity/ClickHouse") + self.assertEqual(set(df["pr_number"]), {2226, 2252}) + + +def _pr_response(status_code, title=""): + response = MagicMock() + response.status_code = status_code + response.text = "Not Found" if status_code == 404 else "error" + response.json.return_value = {"title": title, "labels": []} + return response + + +class TestEnrichPrsInReleaseMergePrs(unittest.TestCase): + def test_skips_404_keeps_ours(self): + # Upstream PR merged from a personal fork (head_owner != ClickHouse) + # is not in Altinity/ClickHouse; 404 means skip, not fail. + df = pd.DataFrame( + { + "pr_number": [114814, 2287], + "merge_commit_sha": ["a" * 40, "b" * 40], + "merge_subject": [ + "Merge pull request #114814 from groeneai/groeneai/backport-112327-25.8", + "Merge pull request #2287 from Altinity/bump/antalya-25.8/25.8.33", + ], + } + ) + + def _get(url, headers=None): + if url.endswith("/114814"): + return _pr_response(404) + return _pr_response(200, title="bump 25.8.33") + + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch("create_workflow_report.requests.get", side_effect=_get), + ): + out = _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertEqual(list(out["pr_number"]), [2287]) + self.assertEqual(list(out["pr_name"]), ["bump 25.8.33"]) + + def test_all_404s_returns_empty(self): + df = pd.DataFrame( + { + "pr_number": [114814], + "merge_commit_sha": ["a" * 40], + "merge_subject": [ + "Merge pull request #114814 from groeneai/groeneai/backport-112327-25.8", + ], + } + ) + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_pr_response(404), + ), + ): + out = _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertEqual(len(out), 0) + self.assertEqual(list(out.columns), ["pr_number", "pr_name", "pr_labels"]) + + def test_non_404_error_still_raises(self): + df = pd.DataFrame( + { + "pr_number": [2287], + "merge_commit_sha": ["b" * 40], + "merge_subject": [ + "Merge pull request #2287 from Altinity/bump/antalya-25.8/25.8.33", + ], + } + ) + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_pr_response(500), + ), + ): + with self.assertRaises(Exception) as ctx: + _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertIn("500", str(ctx.exception)) + + +class TestChecksLatestTestStatusCte(unittest.TestCase): + def test_anchor_excludes_hooks_and_empty_test_name(self): + sql = _checks_latest_test_status_cte("deadbeef", "antalya-25.8") + self.assertIn("test_name != ''", sql) + self.assertIn("NOT IN ('Pre Hooks', 'Post Hooks')", sql) + + +class TestGetWorkflowConfigStdout(unittest.TestCase): + def tearDown(self): + get_workflow_config.cache_clear() + + def test_s3_404_warning_goes_to_stderr(self): + # action.yml takes the script's stdout as the report URL. A WARNING on + # stdout makes grep '^https?://' fail even after a successful upload. + mock_resp = MagicMock() + mock_resp.status_code = 404 + stdout = io.StringIO() + stderr = io.StringIO() + with ( + patch( + "create_workflow_report._workflow_config_from_local", + return_value=None, + ), + patch("create_workflow_report.requests.get", return_value=mock_resp), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + cfg = get_workflow_config(2719, "abc", "antalya-25.8") + self.assertEqual(cfg, {"cache_jobs": {}}) + self.assertEqual(stdout.getvalue(), "") + self.assertIn("WARNING", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/create_workflow_report/test_report_queries.py b/.github/actions/create_workflow_report/test_report_queries.py index 89c7f6d4bb414..dd824c1b7bda4 100755 --- a/.github/actions/create_workflow_report/test_report_queries.py +++ b/.github/actions/create_workflow_report/test_report_queries.py @@ -12,15 +12,35 @@ Set ``CHECKS_DATABASE_HOST``, ``CLICKHOUSE_TEST_STAT_LOGIN``, ``CLICKHOUSE_TEST_STAT_PASSWORD``. + +To capture ``expect`` for a commit: + + python3 -c " + import os + from clickhouse_driver import Client + from create_workflow_report import get_checks_fails + c = Client(host=os.environ['CHECKS_DATABASE_HOST'], + user=os.environ['CLICKHOUSE_TEST_STAT_LOGIN'], + password=os.environ['CLICKHOUSE_TEST_STAT_PASSWORD'], + port=9440, secure='y', verify=False, settings={'use_numpy': True}) + df = get_checks_fails(c, 'COMMIT_SHA', 'HEAD_REF') + print(repr(tuple(sorted(df['test_name'].tolist())))) + " """ import os +import subprocess +from unittest.mock import patch import pandas as pd from clickhouse_driver import Client from testflows.core import * -from create_workflow_report import get_checks_fails +from create_workflow_report import ( + _git_log_merge_prs, + get_checks_fails, + get_commit_statuses, +) def check_result_matches_expect(df: pd.DataFrame, expect: list[str]) -> None: @@ -30,7 +50,7 @@ def check_result_matches_expect(df: pd.DataFrame, expect: list[str]) -> None: actual = set(df["test_name"].tolist()) if not df.empty else set() required = set(expect) if actual != required: - fail(f"test_name mismatch: got {sorted(actual)}; required {sorted(required)}") + fail(f"test results mismatch: got {sorted(actual)}; expected {sorted(required)}") @TestOutline(Scenario) @@ -84,6 +104,25 @@ def check_result_matches_expect(df: pd.DataFrame, expect: list[str]) -> None: "antalya-25.8", (), ), + # Post Hooks (later stopwatch) used to set the check_start_time anchor and hide + # Integration FAILs, e.g. AzureQueue on https://github.com/Altinity/ClickHouse/pull/2107. + ( + "integration_fail_hidden_by_post_hooks_anchor", + "6185add695010b15869b3dc1bcb9c417621b36bc", + "bump/antalya-26.6/26.6.2.81", + ( + "00071_merge_tree_optimize_aio", + "01167_isolation_hermitage", + "01509_parallel_quorum_insert_no_replicas_long", + "03752_attach_as_replicated_transaction_metadata", + "04033_tpc_ds_q20", + "04033_tpc_ds_q39", + "04033_tpc_ds_q58", + "04033_tpc_ds_q71", + "Unknown error", + "test_storage_s3_queue/test_0.py::test_move_after_processing[another_bucket-AzureQueue]", + ), + ), ], ) def test_checks_fails_query(self, case_id, commit_sha, head_ref, expect): @@ -116,10 +155,76 @@ def test_checks_fails_query(self, case_id, commit_sha, head_ref, expect): check_result_matches_expect(df, list(expect)) +@TestScenario +def test_git_log_merge_prs_includes_fork_prs(self): + """Test PR merge history includes contributor forks.""" + stdout = "\n".join( + [ + f"{'e' * 40}\tMerge pull request #2226 from strtgbb/community-pr-failure-artifacts", + f"{'a' * 40}\tMerge pull request #113355 from ClickHouse/backport/26.6/112784", + f"{'b' * 40}\tMerge pull request #2252 from Altinity/ci/antalya-26.6/container-dns", + ] + ) + with When("I collect fork, upstream, and Altinity pull request merges"): + with patch( + "create_workflow_report.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, stdout=stdout), + ): + df = _git_log_merge_prs("baseline", "branch", None, "Altinity/ClickHouse") + + with Then("fork and Altinity pull requests are included but upstream is omitted"): + assert set(df["pr_number"]) == {2226, 2252} + + +@TestScenario +def test_commit_statuses_only_keep_requested_ref(self): + """Test statuses written by another ref on the same commit are ignored.""" + base = "https://altinity-build-artifacts.s3.amazonaws.com" + tag = "v26.6.8.20002.altinityantalya" + # GitHub returns statuses newest first. + statuses = [ + ("Stress test", "failure", f"{base}/json.html?REF=antalya-26.6&sha=abc"), + ("Regression release x", "failure", f"{base}/REFs/antalya-26.6/abc/r.html"), + ("Regression release y", "success", f"{base}/REFs/{tag}/abc/r.html"), + ("Stress test", "success", f"{base}/json.html?REF={tag}&sha=abc"), + ("PR job", "success", f"{base}/json.html?PR=1&sha=abc"), + ] + response = type( + "Response", + (), + { + "status_code": 200, + "headers": {}, + "json": lambda _: [ + {"context": c, "state": st, "description": "", "target_url": u} + for c, st, u in statuses + ], + }, + )() + with When("I fetch statuses for the release tag"): + with patch("create_workflow_report.requests.get", return_value=response): + df = get_commit_statuses("abc", 0, tag) + + with Then("only statuses of the tag remain"): + assert dict(zip(df["job_name"], df["job_status"])) == { + "Stress test": "success", + "Regression release y": "success", + } + + with When("I fetch statuses for a pull request run"): + with patch("create_workflow_report.requests.get", return_value=response): + df = get_commit_statuses("abc", 1, "some-branch") + + with Then("statuses are not filtered by ref"): + assert set(df["job_name"]) == {"Stress test", "Regression release x", "Regression release y", "PR job"} + + @Name("test report queries") @TestModule def test_report_queries(self): Scenario(run=test_checks_fails_query, flags=TE) + Scenario(run=test_git_log_merge_prs_includes_fork_prs, flags=TE) + Scenario(run=test_commit_statuses_only_keep_requested_ref, flags=TE) if main(): diff --git a/.github/actions/create_workflow_report/workflow_report_hook.sh b/.github/actions/create_workflow_report/workflow_report_hook.sh index 06edf8867fb8e..9a03b83898780 100755 --- a/.github/actions/create_workflow_report/workflow_report_hook.sh +++ b/.github/actions/create_workflow_report/workflow_report_hook.sh @@ -1,7 +1,9 @@ #!/bin/bash # This script is for generating preview reports when invoked as a post-hook from a praktika job -pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 jinja2==3.1.5 +# Newer runners have these deps preinstalled; only install on Ubuntu 22. +if [[ -f /etc/os-release ]] && . /etc/os-release && [[ "$VERSION_ID" == "22.04" ]]; then + pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 jinja2==3.1.5 +fi ARGS="--mark-preview --known-fails --cves --actions-run-url $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID --pr-number $PR_NUMBER" CMD="python3 .github/actions/create_workflow_report/create_workflow_report.py" $CMD $ARGS - diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index 5a229fdd47e72..547f2decae0bc 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -3,6 +3,13 @@ description: Setup environment runs: using: "composite" steps: + - name: Configure docker proxy and dataset mirror host + shell: bash + run: | + proxy_host=dockerhub-proxy.dockerhub-proxy-zone + if ! timeout 5 getent ahostsv4 "$proxy_host" >/dev/null; then + echo "65.108.242.32 $proxy_host" | sudo tee -a /etc/hosts + fi - name: Setup zram shell: bash run: | diff --git a/.github/workflows/release_builds.yml b/.github/workflows/fast_builds.yml similarity index 96% rename from .github/workflows/release_builds.yml rename to .github/workflows/fast_builds.yml index d5638ba27e939..a126847f4de8e 100644 --- a/.github/workflows/release_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -1,6 +1,6 @@ # generated by praktika -name: Release Builds +name: Fast Builds on: workflow_dispatch: @@ -87,7 +87,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Config Workflow' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Config Workflow' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -134,7 +134,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (amd)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (amd)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -181,7 +181,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (arm)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (arm)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -228,7 +228,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (multiplatform manifest)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (multiplatform manifest)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -275,7 +275,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -334,7 +334,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -393,7 +393,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -452,7 +452,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -511,7 +511,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_ubsan)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_ubsan)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -570,7 +570,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -629,7 +629,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -688,7 +688,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -747,7 +747,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -794,7 +794,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -841,7 +841,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Docker server image' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Docker server image' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -888,7 +888,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Docker keeper image' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Docker keeper image' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -935,7 +935,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Install packages (amd_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Install packages (amd_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -982,7 +982,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Install packages (arm_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Install packages (arm_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1029,7 +1029,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (amd_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (amd_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1076,7 +1076,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (arm_release)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (arm_release)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1123,7 +1123,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1177,7 +1177,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, parallel)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, parallel)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1231,7 +1231,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1278,7 +1278,7 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Finish Workflow' --workflow "Release Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Finish Workflow' --workflow "Fast Builds" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1347,4 +1347,5 @@ jobs: if: ${{ !cancelled() }} uses: ./.github/actions/create_workflow_report with: + workflow_config: ${{ needs.config_workflow.outputs.data }} final: true diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ab325d2fd73ec..c37f1646aedf6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -6431,4 +6431,5 @@ jobs: if: ${{ !cancelled() }} uses: ./.github/actions/create_workflow_report with: + workflow_config: ${{ needs.config_workflow.outputs.data }} final: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0905a3501722b..e7822a4a3ed9f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5899,4 +5899,5 @@ jobs: if: ${{ !cancelled() }} uses: ./.github/actions/create_workflow_report with: + workflow_config: ${{ needs.config_workflow.outputs.data }} final: true diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 2488a13e38438..a566d7c16f5c3 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -126,7 +126,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] + SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -143,6 +143,40 @@ jobs: job_name: ${{ matrix.SUITE }} secrets: inherit + LightweightDelete: + if: | + fromJson(inputs.workflow_config).JOB_KV_DATA.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).JOB_KV_DATA.ci_regression_jobs, 'common') + strategy: + fail-fast: false + matrix: + include: + - PART: 1 + FILTER: --only "/lightweight delete/disk space/*" + - PART: 2 + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: lightweight_delete + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 300 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: lightweight_delete + extra_args: ${{ matrix.FILTER }} + secrets: inherit + AggregateFunctions: if: | fromJson(inputs.workflow_config).JOB_KV_DATA.ci_regression_jobs[0] == null || diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index f1e57d7cde55a..14c3e66bef1bc 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -76,6 +76,7 @@ run_in_docker=BINARY_DOCKER_COMMAND, timeout=3600 * 4, digest_config=build_digest_config, + needs_submodules=True, ) common_ft_job_config = Job.Config( @@ -87,11 +88,12 @@ # --root/--privileged/--cgroupns=host is required for clickhouse-test --memory-limit # --ulimit nofile is raised so that azurite-rs (the in-process Azure Blob # Storage emulator) does not run out of file descriptors under parallel load - run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD", + run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--dns-search='.'+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/functional_tests.py", "./ci/jobs/scripts/clickhouse_proc.py", + "./ci/jobs/scripts/pick_endpoint.py", "./ci/jobs/scripts/functional_tests_results.py", "./ci/jobs/scripts/functional_tests/setup_log_cluster.sh", "./ci/praktika/cidb.py", @@ -126,6 +128,7 @@ "./tests/queries/0_stateless/", "./ci/jobs/stress_job.py", "./ci/jobs/scripts/clickhouse_proc.py", + "./ci/jobs/scripts/pick_endpoint.py", "./ci/jobs/scripts/stress/stress.py", "./tests/clickhouse-test", "./tests/config", @@ -193,6 +196,7 @@ class JobConfigs: run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--volume=.:/ClickHouse+--env=AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID+--env=AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY", digest_config=fast_test_digest_config, result_name_for_cidb="Tests", + needs_submodules=True, ) darwin_fast_test_jobs = Job.Config( name="Darwin fast test", diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 4f48fadfb4f33..ffd591a3db131 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -176,6 +176,10 @@ def do_checkout(): f"mkdir -p {build_dir} && git submodule sync && git submodule init" ) + if os.path.isdir(".git/modules/contrib") and os.listdir( + ".git/modules/contrib" + ): + print("Submodule cache detected, populating working trees from cache") res = res and Shell.check( "contrib/update-submodules.sh --max-procs 10", retries=3, diff --git a/ci/jobs/fast_test.py b/ci/jobs/fast_test.py index 90dae093da32f..4a26ca75e87b7 100644 --- a/ci/jobs/fast_test.py +++ b/ci/jobs/fast_test.py @@ -68,6 +68,8 @@ def clone_submodules(): res = Shell.check("git submodule sync", verbose=True, strict=True) res = res and Shell.check("git submodule init", verbose=True, strict=True) + if os.path.isdir(".git/modules/contrib") and os.listdir(".git/modules/contrib"): + print("Submodule cache detected, populating working trees from cache") res = res and Shell.check( # NOTE: max-procs was 10 before, increased to 20 to speed up checkout. # Roll back to 10 if this starts hitting GitHub rate limits. diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 425f05094991a..38d4160141429 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -722,7 +722,9 @@ def prepare_stateful_data(self, with_s3_storage, is_db_replicated): clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client < ./tests/docker_scripts/create.sql +python3 ./ci/jobs/scripts/pick_endpoint.py create-sql ./tests/docker_scripts/create.sql | clickhouse-client +S3_BASE=$(python3 ./ci/jobs/scripts/pick_endpoint.py tpcds) +export S3_BASE bash ./tests/docker_scripts/create_tpcds.sh clickhouse-client --query "SHOW TABLES FROM datasets" diff --git a/ci/jobs/scripts/pick_endpoint.py b/ci/jobs/scripts/pick_endpoint.py new file mode 100755 index 0000000000000..ac828c65baac5 --- /dev/null +++ b/ci/jobs/scripts/pick_endpoint.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Pick the faster of the dataset mirror and public S3.""" + +from __future__ import annotations + +import sys +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Dict, Optional, Tuple + +TIMEOUT = 2.0 + +PUBLIC_DATASETS_WEB = "https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com" +PUBLIC_TPCDS = "https://tpc-ds-sf1.s3.amazonaws.com" +MIRROR_DATASETS_WEB = "http://dockerhub-proxy.dockerhub-proxy-zone:6000/datasets-web" +MIRROR_TPCDS = "http://dockerhub-proxy.dockerhub-proxy-zone:6000/tpc-ds-sf1" + +ENDPOINTS: Dict[str, Tuple[str, str]] = { + "datasets-web": (MIRROR_DATASETS_WEB, PUBLIC_DATASETS_WEB), + "tpcds": (MIRROR_TPCDS, PUBLIC_TPCDS), +} + + +def _probe(url: str, timeout: float = TIMEOUT) -> Optional[float]: + box: Dict[str, float] = {} + + def run(): + t0 = time.monotonic() + try: + req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + urllib.request.urlopen(req, timeout=timeout).close() + box["elapsed"] = time.monotonic() - t0 + except urllib.error.HTTPError: + # Any HTTP response means the host answered. + box["elapsed"] = time.monotonic() - t0 + except Exception: + pass + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout + 0.5) + return box.get("elapsed") + + +def pick(name: str, timeout: float = TIMEOUT) -> str: + try: + mirror, public = ENDPOINTS[name] + except KeyError as e: + raise ValueError(f"unknown endpoint pair [{name}]") from e + + with ThreadPoolExecutor(max_workers=2) as pool: + future_mirror = pool.submit(_probe, mirror, timeout) + future_public = pool.submit(_probe, public, timeout) + t_mirror = future_mirror.result() + t_public = future_public.result() + + candidates = [] + if t_mirror is not None: + candidates.append((t_mirror, mirror)) + if t_public is not None: + candidates.append((t_public, public)) + if not candidates: + raise RuntimeError( + f"pick_endpoint [{name}]: neither [{mirror}] nor [{public}] " + f"responded within {timeout}s" + ) + candidates.sort() + winner = candidates[0][1] + print( + f"pick_endpoint [{name}]: {winner} ({candidates[0][0]:.3f}s)", + file=sys.stderr, + ) + return winner + + +def rewritten_create_sql(sql: str, web_base: str) -> str: + return sql.replace(PUBLIC_DATASETS_WEB, web_base) + + +def main(argv=None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] == "create-sql": + if len(argv) != 2: + print("usage: pick_endpoint.py create-sql ", file=sys.stderr) + return 2 + try: + sys.stdout.write( + rewritten_create_sql(Path(argv[1]).read_text(), pick("datasets-web")) + ) + except RuntimeError as e: + print(e, file=sys.stderr) + return 1 + return 0 + if len(argv) != 1 or argv[0] not in ENDPOINTS: + names = "|".join(ENDPOINTS) + print(f"usage: pick_endpoint.py {names}|create-sql ", file=sys.stderr) + return 2 + try: + print(pick(argv[0])) + except RuntimeError as e: + print(e, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/jobs/stress_job.py b/ci/jobs/stress_job.py index 68e13202ca016..85ac89aab60f1 100644 --- a/ci/jobs/stress_job.py +++ b/ci/jobs/stress_job.py @@ -2,6 +2,7 @@ import logging import os import re +import socket import subprocess import sys from pathlib import Path @@ -102,12 +103,24 @@ def get_run_command( else: run_script = "/repo/tests/docker_scripts/stress_runner.sh" + # NOTE (strtgbb): Nested docker does not inherit the runner /etc/hosts. + # --network=host would expose minio and azurite. Skip --add-host when the + # name is unknown (CI Tests e2e / local) rather than failing the job. + proxy_host = "dockerhub-proxy.dockerhub-proxy-zone" + add_host = "" + try: + proxy_ip = socket.getaddrinfo(proxy_host, None)[0][4][0] + add_host = f"--add-host={proxy_host}:{proxy_ip} " + except OSError: + logging.info("Could not resolve %s; not passing --add-host", proxy_host) + cmd = ( "docker run --cap-add=SYS_PTRACE " # For dmesg and sysctl "--privileged " # azurite-rs (in-process Azure Blob Storage emulator) needs many fds under parallel load "--ulimit nofile=1048576:1048576 " + f"{add_host}" # a static link, don't use S3_URL or S3_DOWNLOAD "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' " "--tmpfs /tmp/clickhouse:mode=1777 " diff --git a/ci/praktika/digest.py b/ci/praktika/digest.py index 62c37353090e7..431d53d7e1e66 100644 --- a/ci/praktika/digest.py +++ b/ci/praktika/digest.py @@ -64,10 +64,7 @@ def calc_job_digest(self, job_config: Job.Config, docker_digests, artifact_confi for i, file_path in enumerate(included_files): hash_md5 = self._calc_file_digest(file_path, hash_md5) if config.with_git_submodules: - submodules_shas = Shell.get_output( - "git submodule | awk '{print $1}' | sed 's/^[+-]//'", verbose=True - ) - hash_md5.update(submodules_shas.encode()) + hash_md5.update(Digest.get_submodule_shas().encode()) digest = hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN] self.digest_cache[cache_key] = digest @@ -141,6 +138,15 @@ def calc_docker_digest( return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN] + @staticmethod + def get_submodule_shas(): + return ( + Shell.get_output( + "git submodule | awk '{print $1}' | sed 's/^[+-]//'", verbose=True + ) + or "" + ) + @staticmethod def _calc_file_digest(file_path, hash_md5): # Resolve file path if it's a symbolic link diff --git a/ci/praktika/job.py b/ci/praktika/job.py index 8cc9642f6bac7..e37700a212991 100644 --- a/ci/praktika/job.py +++ b/ci/praktika/job.py @@ -85,6 +85,9 @@ class Config: # List of commands to call upon job completion post_hooks: List[str] = field(default_factory=list) + # If True, runner.py restores the submodule cache from S3 before the job starts + needs_submodules: bool = False + def parametrize(self, *param_sets: "Job.ParamSet"): res = [] for param_set in param_sets: diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index 0b25e579bc35a..a5c603c45b27e 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -1,4 +1,5 @@ import dataclasses +import hashlib import json import platform import os @@ -38,6 +39,102 @@ def _GH_Auth(workflow, force=False): GHAuth.auth(app_id=app_id, app_key=pem) +def _submodule_auth_env() -> dict: + env = os.environ.copy() + if not Settings.ENABLE_SUBMODULE_CLONE_AUTH: + return env + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or "" + if not token: + token = Shell.get_output("gh auth token") or "" + if not token: + print("WARNING: no GH token available, submodule clones run anonymously") + return env + env["GIT_CONFIG_COUNT"] = "1" + env["GIT_CONFIG_KEY_0"] = ( + f"url.https://x-access-token:{token}@github.com/.insteadOf" + ) + env["GIT_CONFIG_VALUE_0"] = "https://github.com/" + return env + + +def _prepare_submodule_cache(workflow_config: RunConfig) -> Result: + """Compute a content-addressed hash of submodule SHAs and ensure a cache + archive exists in S3. Stores the hash in workflow_config so that downstream + jobs with needs_submodules=True can restore it.""" + stop_watch = Utils.Stopwatch() + info = "" + try: + submodule_shas = Digest.get_submodule_shas() + if not submodule_shas: + print("WARNING: No submodules found, skipping submodule cache") + return Result.create_from( + name="Submodule Cache", + status=Result.Status.SUCCESS, + stopwatch=stop_watch, + info="No submodules", + ) + + cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16] + s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + print(f"Submodule cache hash: {cache_hash}") + + if S3.head_object(s3_path): + print(f"Submodule cache hit: {s3_path}") + info = f"cache hit: {cache_hash}" + else: + print(f"Submodule cache miss, creating: {s3_path}") + Shell.check("git submodule sync", verbose=True, strict=True) + Shell.check("git submodule init", verbose=True, strict=True) + Shell.check( + "git submodule update --depth=1 --single-branch --jobs 64", + verbose=True, + strict=True, + retries=3, + env=_submodule_auth_env(), + ) + archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" + Shell.check( + f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}", + verbose=True, + strict=True, + ) + # Write-once conditional create (If-None-Match: *) instead of an + # unconditional overwrite. The object is content-addressed by the + # submodule SHAs, so it never legitimately changes. + created = S3.put( + s3_path=s3_path, + local_path=archive_path, + if_none_matched=True, + no_strict=True, + ) + Shell.check(f"rm -f {archive_path}") + if not created and not S3.head_object(s3_path): + raise RuntimeError(f"failed to upload submodule cache {s3_path}") + info = ( + f"cache miss, created: {cache_hash}" + if created + else f"cache miss, created concurrently: {cache_hash}" + ) + + workflow_config.submodule_cache_hash = cache_hash + workflow_config.dump() + status = Result.Status.SUCCESS + except Exception as e: + print(f"ERROR: Submodule cache failed: {e}") + traceback.print_exc() + info = f"{e}\n{traceback.format_exc()}" + # Do not continue with an empty submodule_cache_hash. Builds would + # skip the restore and clone the same pins themselves. + status = Result.Status.FAILED + + return Result.create_from( + name="Submodule Cache", + status=status, + stopwatch=stop_watch, + info=info, + ) + + _workflow_config_job = Job.Config( name=Settings.CI_CONFIG_JOB_NAME, runs_on=Settings.CI_CONFIG_RUNS_ON, @@ -50,7 +147,9 @@ def _GH_Auth(workflow, force=False): else None ), command=f"{Settings.PYTHON_INTERPRETER} -m praktika.native_jobs '{Settings.CI_CONFIG_JOB_NAME}'", - timeout=600, + # On a submodule-cache miss this job clones all submodules + # (see _prepare_submodule_cache), which can take well over 10 minutes. + timeout=1800, ) _docker_build_manifest_job = Job.Config( @@ -360,6 +459,7 @@ def _check_db(workflow): cache_artifacts={}, cache_jobs={}, filtered_jobs={}, + submodule_cache_hash="", custom_data={}, ).dump() @@ -605,6 +705,14 @@ def check_affected_jobs(): ) ) + if ( + results[-1].is_ok() + and workflow.enable_cache + and Settings.ENABLE_SUBMODULE_CACHE + ): + result = _prepare_submodule_cache(workflow_config) + results.append(result) + if workflow.enable_slack_feed: if env.PR_NUMBER: commit_authors = set() diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 91f74511a8ad6..983640427fdb1 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -4,6 +4,7 @@ import json import os import re +import socket import sys import traceback from pathlib import Path @@ -69,6 +70,7 @@ def generate_local_run_environment(workflow, job, pr=None, sha=None, branch=None cache_artifacts={}, cache_jobs={}, filtered_jobs={}, + submodule_cache_hash="", custom_data={}, ) # Extract repository name from git remote (format: owner/repo) @@ -335,8 +337,48 @@ def _pre_run(self, workflow, job, local_run=False): if fallback_artifact.compress_zst: Utils.decompress_file(Path(Settings.INPUT_DIR) / fallback_path) + if not local_run and job.needs_submodules and Settings.ENABLE_SUBMODULE_CACHE: + self._restore_submodule_cache() + return 0 + @staticmethod + def _restore_submodule_cache(): + try: + wf_config = RunConfig.from_workflow_data() + cache_hash = wf_config.submodule_cache_hash + if not cache_hash: + print( + "NOTE: No submodule cache hash in workflow config, skipping restore" + ) + return + s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" + print(f"Restoring submodule cache: {s3_path}") + S3.copy_file_from_s3( + s3_path=s3_path, local_path=local_archive, no_strict=True + ) + if Path(local_archive).exists(): + if Shell.check( + f"zstd -d {local_archive} --stdout | tar -xf - -C .", + verbose=True, + ): + Shell.check(f"rm -f {local_archive}") + print("Submodule cache restored successfully") + else: + print("WARNING: Submodule cache extraction failed, cleaning up") + Shell.check("rm -rf .git/modules") + Shell.check(f"rm -f {local_archive}") + else: + print( + "WARNING: Submodule cache download failed, will clone from GitHub" + ) + except Exception as e: + print( + f"WARNING: Submodule cache restore failed: {e}, will clone from GitHub" + ) + traceback.print_exc() + def _run( self, workflow, @@ -386,6 +428,18 @@ def _run( ) from_root = "root" in docker_settings settings = [s for s in docker_settings if s.startswith("-")] + # NOTE (strtgbb): FT/integration docker is not --network=host, so + # the runner /etc/hosts mapping for the web-disk proxy is invisible + # inside the container. Resolve on the host and pass --add-host. + # Skip if the name is not configured on this runner (e.g. CI Tests). + _proxy_host = "dockerhub-proxy.dockerhub-proxy-zone" + try: + _proxy_ip = socket.getaddrinfo(_proxy_host, None)[0][4][0] + settings.append(f"--add-host={_proxy_host}:{_proxy_ip}") + except OSError as e: + print( + f"NOTE: skipping --add-host for {_proxy_host}: {e}" + ) if ":" in job.run_in_docker: docker_name, docker_tag = job.run_in_docker.split(":") print( diff --git a/ci/praktika/runtime.py b/ci/praktika/runtime.py index a4c95b83ca62f..5800dec8076a2 100644 --- a/ci/praktika/runtime.py +++ b/ci/praktika/runtime.py @@ -20,6 +20,7 @@ class RunConfig(MetaClasses.Serializable): cache_jobs: Dict[str, Cache.CacheRecord] filtered_jobs: Dict[str, str] sha: str + submodule_cache_hash: str custom_data: Dict[str, Any] @classmethod @@ -36,6 +37,7 @@ def from_dict(cls, obj): for job_name, cache_jobs in cache_jobs.items(): cache_jobs_deserialized[job_name] = Cache.CacheRecord.from_dict(cache_jobs) obj["cache_jobs"] = cache_artifacts_deserialized + obj.setdefault("submodule_cache_hash", "") return RunConfig(**obj) @classmethod diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index 6f185cde12469..8f5a9ed95222b 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -439,8 +439,19 @@ def head_object(cls, s3_path): ) if not output: return None - else: - return cls.Object(**json.loads(output)) + data = json.loads(output) + # HeadObject omits optional fields (Expiration) unless a lifecycle Expire + # rule applies, and emits extra keys Object does not declare. + return cls.Object( + AcceptRanges=data.get("AcceptRanges", ""), + Expiration=data.get("Expiration", ""), + LastModified=data.get("LastModified", ""), + ContentLength=data.get("ContentLength", 0), + ETag=data.get("ETag", ""), + ContentType=data.get("ContentType", ""), + ServerSideEncryption=data.get("ServerSideEncryption", ""), + Metadata=data.get("Metadata") or {}, + ) @classmethod def delete(cls, s3_path): diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index d9929dd36d725..4eeb9b8ca3ffb 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -75,6 +75,12 @@ class _Settings: ###################################### # CI Cache settings # ###################################### + # If enabled, Config Workflow creates a content-addressed .git/modules/ archive + # in S3. Jobs with needs_submodules=True download it instead of cloning from GitHub. + ENABLE_SUBMODULE_CACHE: bool = False + # If enabled, submodule clones rewrite github.com URLs with the ambient + # GITHUB_TOKEN / gh auth token; otherwise they run anonymously. + ENABLE_SUBMODULE_CLONE_AUTH: bool = False CACHE_VERSION: int = 1 CACHE_DIGEST_LEN: int = 20 CACHE_S3_PATH: str = "" @@ -171,6 +177,8 @@ class _Settings: "CI_DB_READ_USER", "CI_DB_READ_URL", "TEST_FAILURE_PATTERNS", + "ENABLE_SUBMODULE_CACHE", + "ENABLE_SUBMODULE_CLONE_AUTH", ] diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 774b7a5f6aa24..232d4d8376025 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -101,6 +101,7 @@ class AltinityWorkflowTemplates: if: ${{ !cancelled() }} uses: ./.github/actions/create_workflow_report with: + workflow_config: ${{ needs.config_workflow.outputs.data }} final: true """, } diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index c365b25ad1912..d8152070b9e63 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -48,6 +48,9 @@ class RunnerLabels: CI_DB_READ_URL: str = os.environ.get("CLICKHOUSE_TEST_STAT_URL", "") + "play" USE_CUSTOM_GH_AUTH = False +# NOTE (strtgbb): anonymous submodule fetches of non-tip pins get refused +# after the burst of ~150 clones; send the ambient gh token instead. +ENABLE_SUBMODULE_CLONE_AUTH = True # SECRET_GH_APP_ID: str = "woolenwolf_gh_app.clickhouse-app-id" # SECRET_GH_APP_PEM_KEY: str = "woolenwolf_gh_app.clickhouse-app-key" diff --git a/ci/settings/settings.py b/ci/settings/settings.py index e64910636d976..0640e9f70283a 100644 --- a/ci/settings/settings.py +++ b/ci/settings/settings.py @@ -23,6 +23,7 @@ class RunnerLabels: DOCKER_BUILD_AMD_RUNS_ON = RunnerLabels.STYLE_CHECK_AMD CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" +ENABLE_SUBMODULE_CACHE = True HTML_S3_PATH = S3_REPORT_BUCKET_NAME S3_BUCKET_TO_HTTP_ENDPOINT = { S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, diff --git a/ci/tests/test_pick_endpoint.py b/ci/tests/test_pick_endpoint.py new file mode 100644 index 0000000000000..25b3174bac4b6 --- /dev/null +++ b/ci/tests/test_pick_endpoint.py @@ -0,0 +1,102 @@ +"""Endpoint picker: Altinity dataset mirror vs Inc public S3.""" + +import os +import sys +import urllib.error +from email.message import EmailMessage + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +from ci.jobs.scripts import pick_endpoint +from ci.jobs.scripts.pick_endpoint import ( + MIRROR_DATASETS_WEB, + PUBLIC_DATASETS_WEB, + PUBLIC_TPCDS, + pick, + rewritten_create_sql, +) + + +def test_picks_faster_mirror(monkeypatch): + def fake_probe(url, timeout=2.0): + return 0.05 if url == MIRROR_DATASETS_WEB else 0.40 + + monkeypatch.setattr(pick_endpoint, "_probe", fake_probe) + assert pick("datasets-web") == MIRROR_DATASETS_WEB + + +def test_public_wins_when_mirror_is_dead(monkeypatch): + def fake_probe(url, timeout=2.0): + return None if "dockerhub-proxy" in url else 0.12 + + monkeypatch.setattr(pick_endpoint, "_probe", fake_probe) + assert pick("datasets-web") == PUBLIC_DATASETS_WEB + assert pick("tpcds") == PUBLIC_TPCDS + + +def test_mirror_wins_when_public_is_dead(monkeypatch): + def fake_probe(url, timeout=2.0): + return 0.08 if "dockerhub-proxy" in url else None + + monkeypatch.setattr(pick_endpoint, "_probe", fake_probe) + assert pick("tpcds") == pick_endpoint.MIRROR_TPCDS + + +def test_both_fail_raises(monkeypatch): + monkeypatch.setattr(pick_endpoint, "_probe", lambda *a, **k: None) + with pytest.raises(RuntimeError, match="datasets-web"): + pick("datasets-web") + + +def test_unknown_pair_raises(): + with pytest.raises(ValueError, match="unknown endpoint pair"): + pick("not-a-pair") + + +def test_http_error_counts_as_reachable(monkeypatch): + def boom(req, timeout=2.0): + raise urllib.error.HTTPError( + req.full_url, 403, "Forbidden", hdrs=EmailMessage(), fp=None + ) + + monkeypatch.setattr(pick_endpoint.urllib.request, "urlopen", boom) + elapsed = pick_endpoint._probe(PUBLIC_DATASETS_WEB, timeout=0.5) + assert elapsed is not None + assert elapsed < 0.5 + + +def test_rewritten_create_sql_swaps_only_the_web_base(): + sql = ( + "endpoint = '" + + PUBLIC_DATASETS_WEB + + "/store/abc/');\n" + "endpoint = '" + + PUBLIC_DATASETS_WEB + + "/store/def/');\n" + ) + out = rewritten_create_sql(sql, MIRROR_DATASETS_WEB) + assert PUBLIC_DATASETS_WEB not in out + assert out.count(MIRROR_DATASETS_WEB) == 2 + assert rewritten_create_sql(sql, PUBLIC_DATASETS_WEB) == sql + + +def test_create_sql_defaults_to_public_s3(): + text = open("tests/docker_scripts/create.sql", encoding="utf-8").read() + assert PUBLIC_DATASETS_WEB in text + assert "dockerhub-proxy" not in text + + +def test_create_tpcds_defaults_to_public_s3(): + text = open("tests/docker_scripts/create_tpcds.sh", encoding="utf-8").read() + assert PUBLIC_TPCDS in text + assert "dockerhub-proxy" not in text + + +def test_create_sql_cli(monkeypatch, tmp_path, capsys): + monkeypatch.setattr(pick_endpoint, "pick", lambda name, timeout=2.0: MIRROR_DATASETS_WEB) + src = tmp_path / "create.sql" + src.write_text(f"disk = disk(type = web, endpoint = '{PUBLIC_DATASETS_WEB}/store/x/'));\n") + assert pick_endpoint.main(["create-sql", str(src)]) == 0 + assert MIRROR_DATASETS_WEB in capsys.readouterr().out diff --git a/ci/tests/test_s3_head_object_optional_fields.py b/ci/tests/test_s3_head_object_optional_fields.py new file mode 100644 index 0000000000000..0d519807f9236 --- /dev/null +++ b/ci/tests/test_s3_head_object_optional_fields.py @@ -0,0 +1,40 @@ +""" +S3.head_object must tolerate HeadObject JSON that omits optional AWS fields. + +aws s3api head-object only emits Expiration when a lifecycle Expire rule +applies. Unpacking the raw JSON into S3.Object then raises TypeError and +Config Workflow leaves submodule_cache_hash empty. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +from ci.praktika import s3 as s3_module +from ci.praktika.s3 import S3 + +# Shape of a successful HeadObject on an untagged cache archive: no Expiration, +# plus keys Object does not declare. +HEAD_WITHOUT_EXPIRATION = { + "AcceptRanges": "bytes", + "LastModified": "Wed, 02 Sep 2026 14:33:49 GMT", + "ContentLength": 123, + "ETag": '"abc"', + "ContentType": "application/zstd", + "Metadata": {}, + "StorageClass": "STANDARD", +} + + +def test_head_object_without_expiration_is_a_hit(monkeypatch): + monkeypatch.setattr( + s3_module.Shell, + "get_output", + lambda *a, **kw: json.dumps(HEAD_WITHOUT_EXPIRATION), + ) + obj = S3.head_object("altinity-build-artifacts/ci_ch_cache/submodules/hash.tar.zst") + assert obj is not None + assert obj.ContentLength == 123 + assert obj.Expiration == "" diff --git a/ci/workflows/release_builds.py b/ci/workflows/fast_builds.py similarity index 99% rename from ci/workflows/release_builds.py rename to ci/workflows/fast_builds.py index 5ce43905896db..ddbf3c58c8873 100644 --- a/ci/workflows/release_builds.py +++ b/ci/workflows/fast_builds.py @@ -27,7 +27,7 @@ ] workflow = Workflow.Config( - name="Release Builds", + name="Fast Builds", event=Workflow.Event.DISPATCH, jobs=[ *[ diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 4926898872567..e877588b61fe1 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -16,7 +16,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac -FROM alpine:3.21 +FROM alpine ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ diff --git a/tests/docker_scripts/create_tpcds.sh b/tests/docker_scripts/create_tpcds.sh index 0023911c73a4b..d1ad212029703 100755 --- a/tests/docker_scripts/create_tpcds.sh +++ b/tests/docker_scripts/create_tpcds.sh @@ -5,7 +5,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" INIT_SQL="$REPO_ROOT/tests/benchmarks/tpc-ds/init.sql" -S3_BASE="https://tpc-ds-sf1.s3.amazonaws.com" +S3_BASE="${S3_BASE:-https://tpc-ds-sf1.s3.amazonaws.com}" awk -v s3="$S3_BASE" ' /^---/ { next } diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 085c79f38af77..a6675e32c6241 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -60,7 +60,13 @@ start_server || { echo "Failed to start server"; exit 1; } cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py logs_export_start || echo "ERROR: Failed to start log exports" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client < /repo/tests/docker_scripts/create.sql +# Do not pipe into clickhouse-client: `set -e` ignores a failing producer. +CREATE_SQL=$(mktemp) +python3 /repo/ci/jobs/scripts/pick_endpoint.py create-sql /repo/tests/docker_scripts/create.sql > "$CREATE_SQL" +clickhouse-client < "$CREATE_SQL" +rm -f "$CREATE_SQL" +S3_BASE=$(python3 /repo/ci/jobs/scripts/pick_endpoint.py tpcds) +export S3_BASE bash /repo/tests/docker_scripts/create_tpcds.sh clickhouse-client --query "SHOW TABLES FROM datasets"