ci: pin regctl release + widen toolchain freshness-guard bats coverage (#1304)

.github/workflows/quality-checks.yml

@@ -109,6 +109,11 @@ jobs:
 
       - name: Install regctl
         uses: iarekylew00t/regctl-installer@c2202c17a65fe59371c71ecc169c9e58c3710a15 # v4.0.16
+        with:
+          # Pin the regctl binary to an explicit released version (not the
+          # action's `latest` default) so this required check is reproducible
+          # and can't shift underneath us when a new regctl ships.
+          regctl-release: v0.11.6
 
       - name: Verify the pinned toolchain image matches the recipe
         env:

scripts/tests/helpers/toolchain_fixture.bash

@@ -44,11 +44,13 @@ tf_write_dockerfile() {
   local geoip2_version="${TF_GEOIP2_VERSION:-v0.0.0-20260623062220-3675c6e7e63d}"
   local caddy_get_line="${TF_CADDY_GET_LINE:-    _retry go get golang.org/x/net@v0.58.0; \\}"
   local golang_digest="${TF_GOLANG_DIGEST:-sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125}"
+  local alpine_image="${TF_ALPINE_IMAGE:-alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b}"
+  local xx_pin="${TF_XX_PIN:-tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707}"
 
   cat > "$TF_DF" <<EOF
 # syntax=docker/dockerfile:1
 ARG GO_VERSION=1.27.1
-ARG ALPINE_IMAGE=alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
+ARG ALPINE_IMAGE=${alpine_image}
 ARG CROWDSEC_VERSION=1.8.1
 ARG EXPR_LANG_VERSION=1.17.8
 ARG XNET_VERSION=0.58.0
@@ -69,7 +71,7 @@ ARG CHARON_TOOLCHAIN_IMAGE=ghcr.io/wikid82/charon-toolchain
 ARG CHARON_TOOLCHAIN_TAG=caddy-crowdsec-0000000000000000
 ARG CHARON_TOOLCHAIN_DIGEST=sha256:0000000000000000000000000000000000000000000000000000000000000000
 
-FROM --platform=\$BUILDPLATFORM tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707 AS xx
+FROM --platform=\$BUILDPLATFORM ${xx_pin} AS xx
 
 # renovate: datasource=docker depName=golang
 FROM --platform=\$BUILDPLATFORM golang:\${GO_VERSION}-alpine@${golang_digest} AS caddy-inline
@@ -149,6 +151,21 @@ EOF
   chmod +x "$TF_BIN/regctl"
 }
 
+# Install a fake `regctl` on PATH whose `image digest` call FAILS (non-zero,
+# no stdout) — simulates the pinned `:$KEY` tag not resolving in the registry.
+# `registry login` still succeeds so the script reaches the digest lookup.
+tf_stub_regctl_unresolvable() {
+  cat > "$TF_BIN/regctl" <<'EOF'
+#!/usr/bin/env bash
+case "$1 $2" in
+  "registry login") exit 0 ;;
+  "image digest")   echo "regctl: manifest unknown" >&2; exit 1 ;;
+  *) exit 0 ;;
+esac
+EOF
+  chmod +x "$TF_BIN/regctl"
+}
+
 # A minimal PATH that contains coreutils + the fixture stub dir but is
 # guaranteed NOT to contain a system `regctl`. Use for "regctl absent" tests:
 #   PATH="$(tf_min_path)" run bash "$TF_ROOT/scripts/verify-toolchain-pin.sh"

scripts/tests/toolchain-key.bats

@@ -6,6 +6,8 @@
 #   * changes when a `go get` line INSIDE caddy-inline changes
 #   * changes when a tracked ARG default (CADDY_VERSION / CADDY_GEOIP2_VERSION) moves
 #   * changes when the digest-pinned golang base moves
+#   * changes when the tonistiigi/xx pin moves
+#   * changes when the ALPINE_IMAGE pin moves
 #   * changes when .trivyignore changes
 #   * fails loudly if stage extraction breaks
 
@@ -70,6 +72,32 @@ key_of() { bash "$TF_ROOT/scripts/toolchain-key.sh" "$TF_DF"; }
   [ "$before" != "$after" ]
 }
 
+@test "changes when the tonistiigi/xx pin moves (N4)" {
+  before="$(key_of)"
+  export TF_XX_PIN='tonistiigi/xx:1.9.1@sha256:1111111111111111111111111111111111111111111111111111111111111111'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" != "$after" ]
+}
+
+@test "changes when the ALPINE_IMAGE pin moves" {
+  before="$(key_of)"
+  export TF_ALPINE_IMAGE='alpine:3.25.0@sha256:2222222222222222222222222222222222222222222222222222222222222222'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" != "$after" ]
+}
+
+@test "stable when the tonistiigi/xx and ALPINE_IMAGE pins are re-emitted unchanged" {
+  before="$(key_of)"
+  # Re-write the fixture with the exact same (default) xx / alpine pins.
+  export TF_XX_PIN='tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707'
+  export TF_ALPINE_IMAGE='alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" = "$after" ]
+}
+
 @test "changes when .trivyignore changes" {
   before="$(key_of)"
   printf 'another-cve\n' >> "$TF_ROOT/.trivyignore"

scripts/tests/verify-toolchain-pin.bats

@@ -6,6 +6,8 @@
 #   mismatched tag                               -> exit 1 (actionable)
 #   same-repo + regctl absent                    -> exit 1
 #   same-repo + GHCR_READ_TOKEN unset            -> exit 1
+#   same-repo + :$KEY tag does not resolve       -> exit 1
+#   same-repo + CHARON_TOOLCHAIN_DIGEST empty    -> exit 1
 #   same-repo + GHCR digest != pinned digest     -> exit 1
 #   same-repo + GHCR digest == pinned digest     -> exit 0
 
@@ -74,6 +76,40 @@ verify() { bash "$TF_ROOT/scripts/verify-toolchain-pin.sh" "$TF_DF"; }
   [[ "$output" == *"hand-edited or stale"* ]]
 }
 
+@test "same-repo PR + pinned :\$KEY tag does not resolve in GHCR: exit 1 (failure-closed)" {
+  tf_stub_regctl_unresolvable
+  export GITHUB_EVENT_NAME=pull_request
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"does not resolve in GHCR"* ]]
+}
+
+@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST value blank: exit 1 (failure-closed)" {
+  tf_stub_regctl "$GOOD_DIGEST"
+  sed -i "s|^ARG CHARON_TOOLCHAIN_DIGEST=.*|ARG CHARON_TOOLCHAIN_DIGEST=|" "$TF_DF"
+  export GITHUB_EVENT_NAME=pull_request
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]]
+}
+
+@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST ARG line absent: exit 1 (failure-closed)" {
+  tf_stub_regctl "$GOOD_DIGEST"
+  sed -i "/^ARG CHARON_TOOLCHAIN_DIGEST=/d" "$TF_DF"
+  export GITHUB_EVENT_NAME=push
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]]
+}
+
 @test "same-repo PR + GHCR digest matches the pinned digest: exit 0" {
   tf_stub_regctl "$GOOD_DIGEST"
   export GITHUB_EVENT_NAME=pull_request

scripts/verify-toolchain-pin.sh

@@ -43,8 +43,8 @@ if [[ ! -f "$df" ]]; then
   exit 2
 fi
 
-arg_value() { # $1 = ARG name
-  grep -E "^ARG $1=" "$df" | head -n1 | cut -d= -f2-
+arg_value() { # $1 = ARG name; empty output (never a hard error) when absent
+  { grep -E "^ARG $1=" "$df" || true; } | head -n1 | cut -d= -f2-
 }
 
 KEY="$("$here/toolchain-key.sh" "$df")"