ci(release): release the experiment action with a major tag

.github/scripts/release_test.sh

@@ -152,6 +152,19 @@ mv "$repo/release-please-config.json.next" "$repo/release-please-config.json"
 run_push_discovery false '["packages/mcp"]'
 grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output"
 git -C "$repo" checkout --quiet -- release-please-config.json
+jq '.packages["actions/experiment"] = {"component": "experiment-action", "release-type": "simple"}' \
+  "$repo/release-please-config.json" > "$repo/release-please-config.json.next"
+mv "$repo/release-please-config.json.next" "$repo/release-please-config.json"
+jq '.["actions/experiment"] = "1.0.0"' "$repo/.release-please-manifest.json" > "$repo/.release-please-manifest.json.next"
+mv "$repo/.release-please-manifest.json.next" "$repo/.release-please-manifest.json"
+git -C "$repo" tag experiment-action-v1.0.0
+for released in '["actions/experiment","packages/mcp"]' '[]'; do
+  run_push_discovery false "$released"
+  grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output"
+  grep -Fxq 'python=["sdks/python"]' "$tmp/output"
+done
+git -C "$repo" tag --delete experiment-action-v1.0.0 > /dev/null
+git -C "$repo" checkout --quiet -- release-please-config.json .release-please-manifest.json
 expect_failure "Expected 2 configured packages for released paths, found 1" \
   run_push_discovery false '["packages/mcp","packages/unknown"]'
 expect_failure "Release otel-v0.1.0 is still a draft" run_push_discovery true '["packages/otel"]'
@@ -161,6 +174,60 @@ git -C "$repo" commit --quiet --allow-empty -m after-batch
 expect_failure "Release tag, event, and checkout commits do not match" \
   run_push_discovery false '["packages/otel"]' "$(git -C "$repo" rev-parse HEAD)"
 
+action_repo="$tmp/action-repo"
+git init --quiet --initial-branch=main "$action_repo"
+git -C "$action_repo" config user.name "Release Test"
+git -C "$action_repo" config user.email "release-test@example.invalid"
+action_commit() {
+  printf '%s\n' "$1" > "$action_repo/.release-please-manifest.json"
+  git -C "$action_repo" add .release-please-manifest.json
+  git -C "$action_repo" commit --quiet -m "$1"
+  git -C "$action_repo" rev-parse HEAD
+}
+unreleased_sha=$(action_commit '{}')
+v1_0_0_sha=$(action_commit '{"actions/experiment":"1.0.0"}')
+git -C "$action_repo" tag experiment-action-v1.0.0
+other_release_sha=$(action_commit '{"actions/experiment":"1.0.0","packages/mcp":"0.1.2"}')
+v1_0_1_sha=$(action_commit '{"actions/experiment":"1.0.1"}')
+git -C "$action_repo" tag experiment-action-v1.0.1
+v2_0_0_sha=$(action_commit '{"actions/experiment":"2.0.0"}')
+git -C "$action_repo" tag experiment-action-v2.0.0
+mkdir -p "$tmp/tag-bin"
+cat > "$tmp/tag-bin/gh" <<'SH'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >> "$GH_LOG"
+SH
+chmod +x "$tmp/tag-bin/gh"
+extract_run tag-experiment-action > "$tmp/tag-action.sh"
+run_tag_action() {
+  (
+    cd "$action_repo"
+    git switch --quiet --detach "$1"
+    rm -f "$tmp/gh-log"
+    PATH="$tmp/tag-bin:$PATH" GH_LOG="$tmp/gh-log" GITHUB_REPOSITORY=telemetry-dev/telemetry GITHUB_SHA="$1" \
+      bash -eo pipefail "$tmp/tag-action.sh"
+  )
+}
+expect_gh_call() {
+  [[ $(wc -l < "$tmp/gh-log") -eq 1 ]]
+  grep -Fxq "$1" "$tmp/gh-log"
+}
+for sha in "$unreleased_sha" "$other_release_sha"; do
+  run_tag_action "$sha"
+  [[ ! -e "$tmp/gh-log" ]]
+done
+run_tag_action "$v1_0_0_sha"
+expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v1 -f sha=$v1_0_0_sha"
+git -C "$action_repo" tag experiment-action-v1 "$v1_0_0_sha"
+run_tag_action "$v1_0_1_sha"
+expect_gh_call "api --method PATCH repos/telemetry-dev/telemetry/git/refs/tags/experiment-action-v1 -f sha=$v1_0_1_sha -F force=true"
+git -C "$action_repo" tag --force experiment-action-v1 "$v1_0_1_sha" > /dev/null
+# A rerun of an earlier release must not move the major tag back.
+run_tag_action "$v1_0_0_sha"
+[[ ! -e "$tmp/gh-log" ]]
+run_tag_action "$v2_0_0_sha"
+expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v2 -f sha=$v2_0_0_sha"
+
 release_tree=$(git -C "$repo" show --no-patch --format=%T "$release_sha")
 unrelated_sha=$(printf 'unrelated\n' | git -C "$repo" commit-tree "$release_tree")
 git -C "$repo" switch --quiet --detach "$unrelated_sha"
@@ -415,7 +482,8 @@ workflow_pnpm=$(
   ' "$root/.github/workflows/release.yml"
 )
 [[ "$workflow_pnpm" == "$package_pnpm" ]]
-[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 2 ]]
+[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 3 ]]
+grep -Fxq "      group: tag-experiment-action" "$root/.github/workflows/release.yml"
 grep -Fxq "  group: release-\${{ inputs.release_tag || github.sha }}" \
   "$root/.github/workflows/release.yml"
 grep -Fxq "      group: release-please-\${{ github.ref }}" \
@@ -455,13 +523,13 @@ for path, package in packages.items():
         manifest = json.loads((root / path / "package.json").read_text())
         paths[manifest["name"]] = path
         dependencies[path] = [*manifest.get("dependencies", {}), *manifest.get("peerDependencies", {})]
-    else:
+    elif package["release-type"] == "python":
         project = tomllib.loads((root / path / "pyproject.toml").read_text())["project"]
         paths[normalize(project["name"])] = path
         dependencies[path] = [
             normalize(re.match(r"[A-Za-z0-9._-]+", requirement)[0]) for requirement in project.get("dependencies", [])
         ]
-for path in order:
+for path in dependencies:
     for dependency in dependencies[path]:
         dependency_path = paths.get(dependency)
         if dependency_path and order.index(dependency_path) > order.index(path):

.github/workflows/release.yml

@@ -96,6 +96,8 @@ jobs:
             case "$release_type" in
               node) npm+=("$path") ;;
               python) python+=("$path") ;;
+              # The experiment action has no package: the tag-experiment-action job moves its major tag.
+              simple) continue ;;
               *)
                 echo "Unsupported release type for $path: $release_type" >&2
                 exit 1
@@ -289,3 +291,39 @@ jobs:
             .github/scripts/verify-python-release-dependencies.sh "$wheel"
             uv publish --trusted-publishing always --check-url https://pypi.org/simple "python/$package"/*
           done < python/publish-order.txt
+
+  tag-experiment-action:
+    needs: release-please
+    concurrency:
+      group: tag-experiment-action
+      cancel-in-progress: false
+      # The default queue cancels a waiting job when another push arrives, and that release then keeps the old major tag.
+      queue: max
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          ref: ${{ github.sha }}
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          version=$(jq -r '.["actions/experiment"] // empty' .release-please-manifest.json)
+          # A rerun after release-please failed partway gets no outputs for the release, so its tag selects it.
+          if [[ -z "$version" || "$(git rev-parse -q --verify "refs/tags/experiment-action-v$version^{commit}" || true)" != "$GITHUB_SHA" ]]; then
+            exit 0
+          fi
+
+          # Workflows use the major tag, which moves to each release of its major version, but never back.
+          major="experiment-action-v${version%%.*}"
+          current=$(git rev-parse -q --verify "refs/tags/$major^{commit}" || true)
+          if [[ -z "$current" ]]; then
+            gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f "ref=refs/tags/$major" -f "sha=$GITHUB_SHA"
+          elif git merge-base --is-ancestor "$current" "$GITHUB_SHA"; then
+            gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/tags/$major" -f "sha=$GITHUB_SHA" -F force=true
+          else
+            echo "$major is at a release after experiment-action-v$version"
+          fi

RELEASING.md

@@ -32,3 +32,16 @@ Each registry's publish job stops at the first package that fails, so the packag
 To retry an existing published release, run the `Release` workflow from `main` and set `release_tag` to that release's full tag. This is a real publication attempt, not a dry run. The tag must be an ancestor of the selected `main` commit. Every configured SDK directory, shared build scripts, lockfiles, release configuration, and root build configuration must be unchanged from the tag; only `packageManager` may differ within root `package.json`. Workflow-only recovery changes are allowed. Source-changing fixes need a new release, not a retry of an old version.
 
 Artifact names and versions are checked before upload. npm retries skip an existing version only when its SHA-512 integrity matches the local tarball, and publication waits about 10 minutes for published `@telemetry-dev/*` dependencies and peer dependencies to satisfy their declared ranges. Python publication likewise waits about 10 minutes for each wheel's dependencies to resolve on PyPI. PyPI retries use `uv publish --check-url` to check existing distributions. An integrity mismatch must be investigated, not bypassed or overwritten.
+
+## Experiment action
+
+The experiment action in `actions/experiment` has no package. Release Please gives it releases such as `experiment-action-v1.0.0`. The `tag-experiment-action` job of the `Release` workflow then moves the major tag, `experiment-action-v1`, to the release commit. The README of the action and the experiment page of telemetry.dev tell users to use the major tag. The job moves a major tag only forward on `main`, so a rerun of an earlier release does not move it back.
+
+A change that can break a workflow, for example a removed input or a changed output, must have a `feat!:` or `fix!:` commit. Release Please then gives the action a new major version, and `experiment-action-v1` stays at the last 1.x release. A `release_tag` retry does not apply to the action. If the job fails, re-run the failed job of the release run.
+
+The job uses `GITHUB_TOKEN`, which cannot have the `workflows` permission. Thus GitHub rejects the tag when no branch head has the same `.github/workflows` files as the release commit. If a merge changed a workflow before the job ran, move the tag with your own credentials:
+
+```sh
+git fetch origin --tags
+git push --force origin 'experiment-action-v1.0.1^{commit}:refs/tags/experiment-action-v1'
+```

actions/experiment/README.md

@@ -27,7 +27,7 @@ jobs:
           node-version: 24
       # Installs tsx too, which runs the .ts script.
       - run: npm ci
-      - uses: telemetry-dev/telemetry/actions/experiment@main
+      - uses: telemetry-dev/telemetry/actions/experiment@experiment-action-v1
         with:
           dataset: support-golden
           script: evals/reply.eval.ts
@@ -48,8 +48,14 @@ await runExperiment({
 });
 ```
 
-Pin the action to a commit SHA of this repository for a run that does not change when `main`
-changes.
+## Versions
+
+The tag `experiment-action-v1` moves to each 1.x release of the action. A change that can break a
+workflow starts a new major version, `experiment-action-v2`, and `experiment-action-v1` stays at
+the last 1.x release. For a run that does not change, use the tag of one release, for example
+`experiment-action-v1.0.0`, or its commit SHA. The
+[releases](https://github.com/telemetry-dev/telemetry/releases) of this repository list the
+changes of each release.
 
 ## Inputs
 

packages/sdk/README.md

@@ -301,7 +301,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor
 the result on the pull request. It gives runExperiment the `dataset` input in
 `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the
 `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the
-same name.
+same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a
+tag that moves to each 1.x release of the action.
 
 ## Scores
 

release-please-config.json

@@ -191,6 +191,11 @@
           "jsonpath": "$.package[?(@.name.value=='telemetry-dev-openrouter')].version"
         }
       ]
+    },
+    "actions/experiment": {
+      "component": "experiment-action",
+      "release-type": "simple",
+      "initial-version": "1.0.0"
     }
   }
 }

sdks/python/README.md

@@ -394,7 +394,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor
 the result on the pull request. It gives `run_experiment` the `dataset` input in
 `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the
 `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the
-same name.
+same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a
+tag that moves to each 1.x release of the action.
 
 ## Scores