From e938cb4b1d0a03a9f1d2e4794c63c4778e30ce63 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 08:27:20 +0000 Subject: [PATCH] ci(release): release the experiment action with a major tag A workflow that used the experiment action at main ran each change on the main branch at once. Release Please now gives the action releases such as experiment-action-v1.0.0. After each release, a new job moves experiment-action-v1 to the release commit, but never back to an earlier release. The READMEs tell users to use experiment-action-v1. --- .github/scripts/release_test.sh | 74 +++++++++++++++++++++++++++++++-- .github/workflows/release.yml | 38 +++++++++++++++++ RELEASING.md | 13 ++++++ actions/experiment/README.md | 12 ++++-- packages/sdk/README.md | 3 +- release-please-config.json | 5 +++ sdks/python/README.md | 3 +- 7 files changed, 140 insertions(+), 8 deletions(-) diff --git a/.github/scripts/release_test.sh b/.github/scripts/release_test.sh index 92dc97d..2e672c6 100755 --- a/.github/scripts/release_test.sh +++ b/.github/scripts/release_test.sh @@ -152,6 +152,19 @@ mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" run_push_discovery false '["packages/mcp"]' grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" git -C "$repo" checkout --quiet -- release-please-config.json +jq '.packages["actions/experiment"] = {"component": "experiment-action", "release-type": "simple"}' \ + "$repo/release-please-config.json" > "$repo/release-please-config.json.next" +mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" +jq '.["actions/experiment"] = "1.0.0"' "$repo/.release-please-manifest.json" > "$repo/.release-please-manifest.json.next" +mv "$repo/.release-please-manifest.json.next" "$repo/.release-please-manifest.json" +git -C "$repo" tag experiment-action-v1.0.0 +for released in '["actions/experiment","packages/mcp"]' '[]'; do + run_push_discovery false "$released" + grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" + grep -Fxq 'python=["sdks/python"]' "$tmp/output" +done +git -C "$repo" tag --delete experiment-action-v1.0.0 > /dev/null +git -C "$repo" checkout --quiet -- release-please-config.json .release-please-manifest.json expect_failure "Expected 2 configured packages for released paths, found 1" \ run_push_discovery false '["packages/mcp","packages/unknown"]' expect_failure "Release otel-v0.1.0 is still a draft" run_push_discovery true '["packages/otel"]' @@ -161,6 +174,60 @@ git -C "$repo" commit --quiet --allow-empty -m after-batch expect_failure "Release tag, event, and checkout commits do not match" \ run_push_discovery false '["packages/otel"]' "$(git -C "$repo" rev-parse HEAD)" +action_repo="$tmp/action-repo" +git init --quiet --initial-branch=main "$action_repo" +git -C "$action_repo" config user.name "Release Test" +git -C "$action_repo" config user.email "release-test@example.invalid" +action_commit() { + printf '%s\n' "$1" > "$action_repo/.release-please-manifest.json" + git -C "$action_repo" add .release-please-manifest.json + git -C "$action_repo" commit --quiet -m "$1" + git -C "$action_repo" rev-parse HEAD +} +unreleased_sha=$(action_commit '{}') +v1_0_0_sha=$(action_commit '{"actions/experiment":"1.0.0"}') +git -C "$action_repo" tag experiment-action-v1.0.0 +other_release_sha=$(action_commit '{"actions/experiment":"1.0.0","packages/mcp":"0.1.2"}') +v1_0_1_sha=$(action_commit '{"actions/experiment":"1.0.1"}') +git -C "$action_repo" tag experiment-action-v1.0.1 +v2_0_0_sha=$(action_commit '{"actions/experiment":"2.0.0"}') +git -C "$action_repo" tag experiment-action-v2.0.0 +mkdir -p "$tmp/tag-bin" +cat > "$tmp/tag-bin/gh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +SH +chmod +x "$tmp/tag-bin/gh" +extract_run tag-experiment-action > "$tmp/tag-action.sh" +run_tag_action() { + ( + cd "$action_repo" + git switch --quiet --detach "$1" + rm -f "$tmp/gh-log" + PATH="$tmp/tag-bin:$PATH" GH_LOG="$tmp/gh-log" GITHUB_REPOSITORY=telemetry-dev/telemetry GITHUB_SHA="$1" \ + bash -eo pipefail "$tmp/tag-action.sh" + ) +} +expect_gh_call() { + [[ $(wc -l < "$tmp/gh-log") -eq 1 ]] + grep -Fxq "$1" "$tmp/gh-log" +} +for sha in "$unreleased_sha" "$other_release_sha"; do + run_tag_action "$sha" + [[ ! -e "$tmp/gh-log" ]] +done +run_tag_action "$v1_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v1 -f sha=$v1_0_0_sha" +git -C "$action_repo" tag experiment-action-v1 "$v1_0_0_sha" +run_tag_action "$v1_0_1_sha" +expect_gh_call "api --method PATCH repos/telemetry-dev/telemetry/git/refs/tags/experiment-action-v1 -f sha=$v1_0_1_sha -F force=true" +git -C "$action_repo" tag --force experiment-action-v1 "$v1_0_1_sha" > /dev/null +# A rerun of an earlier release must not move the major tag back. +run_tag_action "$v1_0_0_sha" +[[ ! -e "$tmp/gh-log" ]] +run_tag_action "$v2_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v2 -f sha=$v2_0_0_sha" + release_tree=$(git -C "$repo" show --no-patch --format=%T "$release_sha") unrelated_sha=$(printf 'unrelated\n' | git -C "$repo" commit-tree "$release_tree") git -C "$repo" switch --quiet --detach "$unrelated_sha" @@ -415,7 +482,8 @@ workflow_pnpm=$( ' "$root/.github/workflows/release.yml" ) [[ "$workflow_pnpm" == "$package_pnpm" ]] -[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 2 ]] +[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 3 ]] +grep -Fxq " group: tag-experiment-action" "$root/.github/workflows/release.yml" grep -Fxq " group: release-\${{ inputs.release_tag || github.sha }}" \ "$root/.github/workflows/release.yml" grep -Fxq " group: release-please-\${{ github.ref }}" \ @@ -455,13 +523,13 @@ for path, package in packages.items(): manifest = json.loads((root / path / "package.json").read_text()) paths[manifest["name"]] = path dependencies[path] = [*manifest.get("dependencies", {}), *manifest.get("peerDependencies", {})] - else: + elif package["release-type"] == "python": project = tomllib.loads((root / path / "pyproject.toml").read_text())["project"] paths[normalize(project["name"])] = path dependencies[path] = [ normalize(re.match(r"[A-Za-z0-9._-]+", requirement)[0]) for requirement in project.get("dependencies", []) ] -for path in order: +for path in dependencies: for dependency in dependencies[path]: dependency_path = paths.get(dependency) if dependency_path and order.index(dependency_path) > order.index(path): diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac0f9bb..621be50 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,6 +96,8 @@ jobs: case "$release_type" in node) npm+=("$path") ;; python) python+=("$path") ;; + # The experiment action has no package: the tag-experiment-action job moves its major tag. + simple) continue ;; *) echo "Unsupported release type for $path: $release_type" >&2 exit 1 @@ -289,3 +291,39 @@ jobs: .github/scripts/verify-python-release-dependencies.sh "$wheel" uv publish --trusted-publishing always --check-url https://pypi.org/simple "python/$package"/* done < python/publish-order.txt + + tag-experiment-action: + needs: release-please + concurrency: + group: tag-experiment-action + cancel-in-progress: false + # The default queue cancels a waiting job when another push arrives, and that release then keeps the old major tag. + queue: max + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + - env: + GH_TOKEN: ${{ github.token }} + run: | + version=$(jq -r '.["actions/experiment"] // empty' .release-please-manifest.json) + # A rerun after release-please failed partway gets no outputs for the release, so its tag selects it. + if [[ -z "$version" || "$(git rev-parse -q --verify "refs/tags/experiment-action-v$version^{commit}" || true)" != "$GITHUB_SHA" ]]; then + exit 0 + fi + + # Workflows use the major tag, which moves to each release of its major version, but never back. + major="experiment-action-v${version%%.*}" + current=$(git rev-parse -q --verify "refs/tags/$major^{commit}" || true) + if [[ -z "$current" ]]; then + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f "ref=refs/tags/$major" -f "sha=$GITHUB_SHA" + elif git merge-base --is-ancestor "$current" "$GITHUB_SHA"; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/tags/$major" -f "sha=$GITHUB_SHA" -F force=true + else + echo "$major is at a release after experiment-action-v$version" + fi diff --git a/RELEASING.md b/RELEASING.md index 064b46b..ada9052 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -32,3 +32,16 @@ Each registry's publish job stops at the first package that fails, so the packag To retry an existing published release, run the `Release` workflow from `main` and set `release_tag` to that release's full tag. This is a real publication attempt, not a dry run. The tag must be an ancestor of the selected `main` commit. Every configured SDK directory, shared build scripts, lockfiles, release configuration, and root build configuration must be unchanged from the tag; only `packageManager` may differ within root `package.json`. Workflow-only recovery changes are allowed. Source-changing fixes need a new release, not a retry of an old version. Artifact names and versions are checked before upload. npm retries skip an existing version only when its SHA-512 integrity matches the local tarball, and publication waits about 10 minutes for published `@telemetry-dev/*` dependencies and peer dependencies to satisfy their declared ranges. Python publication likewise waits about 10 minutes for each wheel's dependencies to resolve on PyPI. PyPI retries use `uv publish --check-url` to check existing distributions. An integrity mismatch must be investigated, not bypassed or overwritten. + +## Experiment action + +The experiment action in `actions/experiment` has no package. Release Please gives it releases such as `experiment-action-v1.0.0`. The `tag-experiment-action` job of the `Release` workflow then moves the major tag, `experiment-action-v1`, to the release commit. The README of the action and the experiment page of telemetry.dev tell users to use the major tag. The job moves a major tag only forward on `main`, so a rerun of an earlier release does not move it back. + +A change that can break a workflow, for example a removed input or a changed output, must have a `feat!:` or `fix!:` commit. Release Please then gives the action a new major version, and `experiment-action-v1` stays at the last 1.x release. A `release_tag` retry does not apply to the action. If the job fails, re-run the failed job of the release run. + +The job uses `GITHUB_TOKEN`, which cannot have the `workflows` permission. Thus GitHub rejects the tag when no branch head has the same `.github/workflows` files as the release commit. If a merge changed a workflow before the job ran, move the tag with your own credentials: + +```sh +git fetch origin --tags +git push --force origin 'experiment-action-v1.0.1^{commit}:refs/tags/experiment-action-v1' +``` diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 431f2c0..5e7cbbd 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -27,7 +27,7 @@ jobs: node-version: 24 # Installs tsx too, which runs the .ts script. - run: npm ci - - uses: telemetry-dev/telemetry/actions/experiment@main + - uses: telemetry-dev/telemetry/actions/experiment@experiment-action-v1 with: dataset: support-golden script: evals/reply.eval.ts @@ -48,8 +48,14 @@ await runExperiment({ }); ``` -Pin the action to a commit SHA of this repository for a run that does not change when `main` -changes. +## Versions + +The tag `experiment-action-v1` moves to each 1.x release of the action. A change that can break a +workflow starts a new major version, `experiment-action-v2`, and `experiment-action-v1` stays at +the last 1.x release. For a run that does not change, use the tag of one release, for example +`experiment-action-v1.0.0`, or its commit SHA. The +[releases](https://github.com/telemetry-dev/telemetry/releases) of this repository list the +changes of each release. ## Inputs diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ed35293..81f726a 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -301,7 +301,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor the result on the pull request. It gives runExperiment the `dataset` input in `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the -same name. +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. ## Scores diff --git a/release-please-config.json b/release-please-config.json index e8c7e42..79b72d6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -191,6 +191,11 @@ "jsonpath": "$.package[?(@.name.value=='telemetry-dev-openrouter')].version" } ] + }, + "actions/experiment": { + "component": "experiment-action", + "release-type": "simple", + "initial-version": "1.0.0" } } } diff --git a/sdks/python/README.md b/sdks/python/README.md index c7eb87c..bd91e98 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -394,7 +394,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor the result on the pull request. It gives `run_experiment` the `dataset` input in `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the -same name. +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. ## Scores