Split landed CI and native builds onto GKE pods

.github/workflows/ci.yml

@@ -1,21 +1,14 @@
 name: CI
 
 on:
-  push:
-    branches: [main]
   pull_request:
 
 permissions:
   contents: read
   actions: read
 
-# Only a superseded pull_request run is cancelled: every other event (push to main,
-# whose run authorizes publication) gets its own run_id group and is never cancelled.
-# Every job runs on GitHub-hosted ubuntu-24.04 (owner decision CI-2 = A, 10-21 04:01, and #1050:
-# this repository is public and no job here holds a secret); dependabot counts as ours.
-# Fork PRs fail authz. Documentation/workflow-only PRs run the light checks and skip
-# the heavy suites and release/ready evidence. The required check independently proves
-# cheap skips; code PRs and every push run the full suites and produce release evidence.
+# Pull requests keep hosted runners and the independently verified cheap checks.
+# Main-branch proofs and release evidence run in ci-post.yml.
 concurrency:
   group: ci-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -236,7 +229,7 @@ jobs:
           fi
           tar -xzf "$lint_dir/$archive" -C "$lint_dir" actionlint
           "$lint_dir/actionlint" -version
-          printf '%s\n' '{"self-hosted-runner":{"labels":["pf-gcp-pr","pf-gcp-main","pf-gcp-post"]}}' > "$lint_dir/actionlint.yaml"
+          printf '%s\n' '{"self-hosted-runner":{"labels":["pf-gcp-pr","pf-gcp-main","pf-gcp-post","gke-2cpu-4gb-amd64","gke-2cpu-4gb-arm64","gke-4cpu-8gb-amd64","gke-4cpu-8gb-arm64","gke-8cpu-16gb-amd64","gke-8cpu-16gb-arm64","gke-16cpu-32gb-amd64","gke-16cpu-32gb-arm64","gke-32cpu-64gb-amd64","gke-32cpu-64gb-arm64","gkp-2cpu-4gb-amd64","gkp-2cpu-4gb-arm64","gkp-4cpu-8gb-amd64","gkp-4cpu-8gb-arm64","gkp-8cpu-16gb-amd64","gkp-8cpu-16gb-arm64","gkp-16cpu-32gb-amd64","gkp-16cpu-32gb-arm64","gkp-32cpu-64gb-amd64","gkp-32cpu-64gb-arm64"]}}' > "$lint_dir/actionlint.yaml"
           git ls-files -z -- .github/workflows > "$lint_dir/workflows"
           workflows=()
           while IFS= read -r -d '' path; do

.github/workflows/native-feed-post.yml

@@ -0,0 +1,224 @@
+# SPDX-License-Identifier: Apache-2.0
+name: Native feed
+
+on:
+  push:
+    branches: [main]
+    paths: ['native/feed/**', '.github/workflows/native-feed.yml', '.github/workflows/native-feed-post.yml']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Only main can request pods; macOS remains on its hosted runner.
+concurrency:
+  group: native-feed-post-${{ github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  authz:
+    name: authz
+    # Standard GitHub-hosted label (owner #1050); never a larger-runner label.
+    runs-on: ubuntu-24.04
+    permissions: {}
+    # No timeout-minutes: the owner's no-time-limits rule (check-no-time-limits refuses new ones).
+    steps:
+      - name: Only our own pull requests and our own events run CI
+        env:
+          EVENT: ${{ github.event_name }}
+          REPO: ${{ github.repository }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          HEAD_IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
+          ASSOCIATION: ${{ github.event.pull_request.author_association }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+          WR_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }}
+          WR_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }}
+          GH_TOKEN: ${{ github.token }}   # read-only use: one permission lookup for workflow_run
+          # Bot logins whose same-repository pull requests count as ours: dependabot[bot] (CI-DEPBOT A)
+          # and the org's own GitHub App bot logins by name (space-separated), nothing else.
+          ALLOWED_BOTS: "dependabot[bot] pineforge-app[bot]"
+        run: |
+          set -eu
+          fail() { echo "::error title=authz::$1"; echo "authz: REFUSED: $1" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
+          case "$EVENT" in
+            push|workflow_dispatch|schedule|merge_group|release|workflow_call|repository_dispatch)
+              # Only someone with write access to this repository (or our App's token) can cause these
+              # events; none of them runs pull-request code.
+              ;;
+            workflow_run)
+              # LB-437: bind identity to the triggering run and its re-runner.
+              # These two App bots are fixed canon policy; ALLOWED_BOTS is PR-only.
+              [ -n "${WR_HEAD_REPO:-}" ] || fail "workflow_run head repository is missing"
+              [ -n "${WR_ACTOR:-}" ] || fail "workflow_run triggering actor is missing"
+              [ "$WR_HEAD_REPO" = "$REPO" ] || fail "workflow_run head repository does not match this repository"
+              case "$WR_ACTOR" in
+                "pineforge-app[bot]"|"pineforge-ci-dispatcher[bot]") ;;
+                *"[bot]") fail "workflow_run bot is not on the canon allowlist" ;;
+                *)
+                  perm=$(gh api "repos/$REPO/collaborators/$WR_ACTOR/permission" --jq .permission 2>/dev/null) || fail "workflow_run permission lookup failed"
+                  case "$perm" in admin|maintain|write) ;; *) fail "workflow_run triggering actor does not have write permission" ;; esac ;;
+              esac ;;
+            pull_request)
+              [ "$HEAD_REPO" = "$REPO" ] || fail "pull request from another repository (${HEAD_REPO:-unknown})"
+              [ "$HEAD_IS_FORK" = "false" ] || fail "pull request from a fork"
+              case "$ASSOCIATION" in
+                OWNER|MEMBER|COLLABORATOR) ;;
+                *)
+                  case " $ALLOWED_BOTS " in
+                    *" $PR_AUTHOR "*) ;;
+                    *)
+                      # A member whose org membership is private shows as CONTRIBUTOR: ask GitHub for the
+                      # author's permission on this repository; anything but admin/maintain/write (or a
+                      # failed call) refuses.
+                      perm=$(gh api "repos/$REPO/collaborators/$PR_AUTHOR/permission" --jq .permission 2>/dev/null || echo none)
+                      case "$perm" in admin|maintain|write) ;; *) fail "author ${PR_AUTHOR:-unknown} (${ASSOCIATION:-NONE}) has permission '$perm', not write, and is not an allowlisted bot" ;; esac ;;
+                  esac ;;
+              esac ;;
+            *) fail "event $EVENT does not run CI (pull_request_target and other events are not allowed)" ;;
+          esac
+          echo "authz: allowed ($EVENT${PR_AUTHOR:+ by $PR_AUTHOR})" >> "$GITHUB_STEP_SUMMARY"
+
+  main-ref:
+    needs: [authz]
+    runs-on: ubuntu-24.04
+    permissions: {}
+    steps:
+      - name: Require the main branch before requesting runners
+        env:
+          REF: ${{ github.ref }}
+        run: |
+          set -eu
+          [ "$REF" = refs/heads/main ]
+
+  release-amd64:
+    name: Release (ubuntu-24.04)
+    needs: [authz, main-ref]
+    runs-on: gke-4cpu-8gb-amd64
+    env:
+      PF_CI_LABELS: '["gke-4cpu-8gb-amd64"]'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Linux prerequisites
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y cmake ninja-build libssl-dev libdigest-sha-perl
+      - name: macOS prerequisites
+        if: runner.os == 'macOS'
+        run: |
+          brew install ninja openssl@3
+          echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV"
+      - name: Pinned WebSocket curl
+        run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
+      - name: Configure, build and test
+        run: |
+          export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
+          export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
+          cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
+            -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
+          cmake --build build-feed -j4
+          ctest --test-dir build-feed --output-on-failure
+
+  release-arm64:
+    name: Release (ubuntu-24.04-arm)
+    needs: [authz, main-ref]
+    runs-on: gke-4cpu-8gb-arm64
+    env:
+      PF_CI_LABELS: '["gke-4cpu-8gb-arm64"]'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Linux prerequisites
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y cmake ninja-build libssl-dev libdigest-sha-perl
+      - name: macOS prerequisites
+        if: runner.os == 'macOS'
+        run: |
+          brew install ninja openssl@3
+          echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV"
+      - name: Pinned WebSocket curl
+        run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
+      - name: Configure, build and test
+        run: |
+          export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
+          export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
+          cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
+            -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
+          cmake --build build-feed -j4
+          ctest --test-dir build-feed --output-on-failure
+
+  release-macos:
+    name: Release (macos-14)
+    needs: [authz, main-ref]
+    runs-on: macos-14
+    env:
+      PF_CI_LABELS: '["macos-14"]'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Linux prerequisites
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y cmake ninja-build libssl-dev libdigest-sha-perl
+      - name: macOS prerequisites
+        if: runner.os == 'macOS'
+        run: |
+          brew install ninja openssl@3
+          echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV"
+      - name: Pinned WebSocket curl
+        run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
+      - name: Configure, build and test
+        run: |
+          export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
+          export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
+          cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
+            -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
+          cmake --build build-feed -j4
+          ctest --test-dir build-feed --output-on-failure
+
+  asan-ubsan:
+    name: ASan + UBSan
+    needs: [authz, main-ref]
+    runs-on: gke-4cpu-8gb-amd64
+    env:
+      PF_CI_LABELS: '["gke-4cpu-8gb-amd64"]'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 libdigest-sha-perl
+      - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
+      - name: Instrumented core and mock venue
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1
+          UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
+        run: |
+          export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
+          cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
+            -DPINEFORGE_FEED_SANITIZER=address,undefined
+          cmake --build build-feed -j4
+          ctest --test-dir build-feed --output-on-failure
+
+  tsan:
+    name: TSan
+    needs: [authz, main-ref]
+    runs-on: gke-4cpu-8gb-amd64
+    env:
+      PF_CI_LABELS: '["gke-4cpu-8gb-amd64"]'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 llvm-18 libdigest-sha-perl
+      - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
+      - name: Check TSan symbolizer
+        run: |
+          # With no symbolizer configured, TSan can fall back and still symbolize.
+          # A wrong explicit path breaks symbolization and makes race:STOP_FLAG_ASSIGN inert.
+          if ! test -x /usr/lib/llvm-18/bin/llvm-symbolizer; then
+            echo "::error::TSan requires an executable /usr/lib/llvm-18/bin/llvm-symbolizer for function-name suppressions; install llvm-18." >&2
+            exit 1
+          fi
+      - name: Threaded receive, healing, termination and serve fan-out
+        env:
+          TSAN_OPTIONS: halt_on_error=1
+        run: |
+          export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
+          cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
+            -DPINEFORGE_FEED_SANITIZER=thread \
+            -DPINEFORGE_FEED_TSAN_OPTIONS_EXTRA=external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer
+          cmake --build build-feed -j4
+          ctest --test-dir build-feed --output-on-failure

.github/workflows/native-feed.yml

@@ -1,20 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 name: Native feed
 on:
-  push:
-    branches: [main]
-    paths: ['native/feed/**', '.github/workflows/native-feed.yml']
   pull_request:
-    paths: ['native/feed/**', '.github/workflows/native-feed.yml']
-  workflow_dispatch:
+    paths: ['native/feed/**', '.github/workflows/native-feed.yml', '.github/workflows/native-feed-post.yml']
 
 permissions:
   contents: read
 
-# Only a superseded pull_request run is cancelled; push and dispatch are never cancelled.
-# authz, asan-ubsan, tsan and the release legs run on GitHub-hosted runners (owner decision
-# CI-2 = A, 10-21: ubuntu-24.04, ubuntu-24.04-arm, macos-14; no secret in any).
-# Fork PRs fail authz.
+# Pull request builds keep their existing hosted platform runners.
+# Main-branch builds run in native-feed-post.yml; macOS remains hosted.
 concurrency:
   group: native-feed-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}

docs/releasing.md

@@ -83,7 +83,7 @@ static checks, for example Ruff, mypy, Python compilation, actionlint and
 ## Publication, completion and repair
 
 Successful PR CI wakes the reconciler through `workflow_run`. The App's merge
-triggers ordinary main CI. `Release` wakes on completion of that main CI, and
+triggers `ci-post.yml` on main. `Release` wakes on completion of that main CI, and
 accepts only a landed App-created release/version/rollback PR. Ordinary main
 pushes are not publication requests. Failed main CI cannot authorize upload.
 
@@ -112,7 +112,7 @@ produce no receipt; ordinary manual package publication still uses the existing
 landed CI evidence and gates, without requiring a product receipt artifact. CI/PR-head code never builds the hub artifact.
 
 CI and E2E both reference the complete successful **pre-merge** `ci.yml` run,
-including `release/ready`. The landed main CI still authorizes the exact
+including `release/ready`. The landed `ci-post.yml` run authorizes the exact
 wheel/sdist bytes used by the existing publisher. Preview records the candidate
 head/image/facts and a SHA-256 of its tested distribution identity. After the
 `Release` run finishes, the coordinator rechecks the actual PR, tested base and

scripts/release_chain/evidence.py

@@ -13,10 +13,13 @@
 
 from .contract import gh_bytes, json_bytes, object_json, require, sha256
 
-# SHA256 of reviewed .github/workflows/ci.yml bytes: no continue-on-error, matrix leg names = JOBS.
+# SHA256 of both reviewed CI definitions: no continue-on-error, matrix leg names = JOBS.
 # Edits must update the pin in the same PR after re-review: needs.<job>.result == "success" proves
 # every leg succeeded (2026-10-23, VP-145 review P0: a lexical test alone misses escaped YAML keys).
-REVIEWED_CI_WORKFLOW_SHA256 = "de94f322b372305ccb4e5550e248c9727137f263f17dc3d6c4649a5d09ef64c5"
+REVIEWED_CI_WORKFLOW_SHA256 = "d62bd2fa2c574d336654df1a18988eb8692396cf75d21dc9f6df0d59735f3225"
+REVIEWED_CI_POST_WORKFLOW_SHA256 = (
+    "0d482244dd46ed3e4ec7ac6e5297521c81106922a93c24fde01ed1e87d2fdd5d"
+)
 
 JOBS = (
     "Python 3.11",
@@ -102,15 +105,19 @@ def validate_inventory(jobs: list[dict[str, Any]]) -> None:
 
 
 def require_no_continue_on_error(root: Path) -> None:
-    try:
-        raw = (root / ".github" / "workflows" / "ci.yml").read_bytes()
-        text = raw.decode("utf-8")
-    except (OSError, UnicodeError) as exc:
-        require(False, f"cannot read ci.yml: {exc}")
-    require(
-        sha256(raw) == REVIEWED_CI_WORKFLOW_SHA256 and "continue-on-error" not in text,
-        "ci.yml changed or uses continue-on-error: needs results require workflow review",
-    )
+    for filename, reviewed in (
+        ("ci.yml", REVIEWED_CI_WORKFLOW_SHA256),
+        ("ci-post.yml", REVIEWED_CI_POST_WORKFLOW_SHA256),
+    ):
+        try:
+            raw = (root / ".github/workflows" / filename).read_bytes()
+            text = raw.decode("utf-8")
+        except (OSError, UnicodeError) as exc:
+            require(False, f"cannot read {filename}: {exc}")
+        require(
+            sha256(raw) == reviewed and "continue-on-error" not in text,
+            f"{filename} changed or uses continue-on-error: needs results require workflow review",
+        )
 
 
 def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[str, Any]:
@@ -161,7 +168,8 @@ def validate_gate(
         run["status"] == "completed" and run["conclusion"] == "success",
         "CI has not completed successfully",
     )
-    require(run["path"] == ".github/workflows/ci.yml", "unexpected CI workflow")
+    workflow = "ci.yml" if pr else "ci-post.yml"
+    require(run["path"] == f".github/workflows/{workflow}", "unexpected CI workflow")
     require(run["head_sha"] == head and evidence["head"] == head, "stale CI head")
     require(evidence["base"] == base and evidence["pr"] == pr, "stale CI base or PR")
     require(

scripts/release_chain/publishing.py

@@ -298,7 +298,8 @@ def recover_applied(repo: str, source: Path, report: dict[str, Any]) -> None:
     report["pr"]["base_sha"] = marker(pr)["base"]
     runs = api(
         repo,
-        "actions/workflows/ci.yml/runs?event=push&per_page=100&head_sha=" + pr["merge_commit_sha"],
+        "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha="
+        + pr["merge_commit_sha"],
     )["workflow_runs"]
     runs = [run for run in runs if run["head_sha"] == pr["merge_commit_sha"]]
     if not runs or max(runs, key=lambda run: run["id"])["status"] != "completed":