diff --git a/.github/scripts/release_test.sh b/.github/scripts/release_test.sh index 92dc97d..2e672c6 100755 --- a/.github/scripts/release_test.sh +++ b/.github/scripts/release_test.sh @@ -152,6 +152,19 @@ mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" run_push_discovery false '["packages/mcp"]' grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" git -C "$repo" checkout --quiet -- release-please-config.json +jq '.packages["actions/experiment"] = {"component": "experiment-action", "release-type": "simple"}' \ + "$repo/release-please-config.json" > "$repo/release-please-config.json.next" +mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" +jq '.["actions/experiment"] = "1.0.0"' "$repo/.release-please-manifest.json" > "$repo/.release-please-manifest.json.next" +mv "$repo/.release-please-manifest.json.next" "$repo/.release-please-manifest.json" +git -C "$repo" tag experiment-action-v1.0.0 +for released in '["actions/experiment","packages/mcp"]' '[]'; do + run_push_discovery false "$released" + grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" + grep -Fxq 'python=["sdks/python"]' "$tmp/output" +done +git -C "$repo" tag --delete experiment-action-v1.0.0 > /dev/null +git -C "$repo" checkout --quiet -- release-please-config.json .release-please-manifest.json expect_failure "Expected 2 configured packages for released paths, found 1" \ run_push_discovery false '["packages/mcp","packages/unknown"]' expect_failure "Release otel-v0.1.0 is still a draft" run_push_discovery true '["packages/otel"]' @@ -161,6 +174,60 @@ git -C "$repo" commit --quiet --allow-empty -m after-batch expect_failure "Release tag, event, and checkout commits do not match" \ run_push_discovery false '["packages/otel"]' "$(git -C "$repo" rev-parse HEAD)" +action_repo="$tmp/action-repo" +git init --quiet --initial-branch=main "$action_repo" +git -C "$action_repo" config user.name "Release Test" +git -C "$action_repo" config user.email "release-test@example.invalid" +action_commit() { + printf '%s\n' "$1" > "$action_repo/.release-please-manifest.json" + git -C "$action_repo" add .release-please-manifest.json + git -C "$action_repo" commit --quiet -m "$1" + git -C "$action_repo" rev-parse HEAD +} +unreleased_sha=$(action_commit '{}') +v1_0_0_sha=$(action_commit '{"actions/experiment":"1.0.0"}') +git -C "$action_repo" tag experiment-action-v1.0.0 +other_release_sha=$(action_commit '{"actions/experiment":"1.0.0","packages/mcp":"0.1.2"}') +v1_0_1_sha=$(action_commit '{"actions/experiment":"1.0.1"}') +git -C "$action_repo" tag experiment-action-v1.0.1 +v2_0_0_sha=$(action_commit '{"actions/experiment":"2.0.0"}') +git -C "$action_repo" tag experiment-action-v2.0.0 +mkdir -p "$tmp/tag-bin" +cat > "$tmp/tag-bin/gh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +SH +chmod +x "$tmp/tag-bin/gh" +extract_run tag-experiment-action > "$tmp/tag-action.sh" +run_tag_action() { + ( + cd "$action_repo" + git switch --quiet --detach "$1" + rm -f "$tmp/gh-log" + PATH="$tmp/tag-bin:$PATH" GH_LOG="$tmp/gh-log" GITHUB_REPOSITORY=telemetry-dev/telemetry GITHUB_SHA="$1" \ + bash -eo pipefail "$tmp/tag-action.sh" + ) +} +expect_gh_call() { + [[ $(wc -l < "$tmp/gh-log") -eq 1 ]] + grep -Fxq "$1" "$tmp/gh-log" +} +for sha in "$unreleased_sha" "$other_release_sha"; do + run_tag_action "$sha" + [[ ! -e "$tmp/gh-log" ]] +done +run_tag_action "$v1_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v1 -f sha=$v1_0_0_sha" +git -C "$action_repo" tag experiment-action-v1 "$v1_0_0_sha" +run_tag_action "$v1_0_1_sha" +expect_gh_call "api --method PATCH repos/telemetry-dev/telemetry/git/refs/tags/experiment-action-v1 -f sha=$v1_0_1_sha -F force=true" +git -C "$action_repo" tag --force experiment-action-v1 "$v1_0_1_sha" > /dev/null +# A rerun of an earlier release must not move the major tag back. +run_tag_action "$v1_0_0_sha" +[[ ! -e "$tmp/gh-log" ]] +run_tag_action "$v2_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v2 -f sha=$v2_0_0_sha" + release_tree=$(git -C "$repo" show --no-patch --format=%T "$release_sha") unrelated_sha=$(printf 'unrelated\n' | git -C "$repo" commit-tree "$release_tree") git -C "$repo" switch --quiet --detach "$unrelated_sha" @@ -415,7 +482,8 @@ workflow_pnpm=$( ' "$root/.github/workflows/release.yml" ) [[ "$workflow_pnpm" == "$package_pnpm" ]] -[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 2 ]] +[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 3 ]] +grep -Fxq " group: tag-experiment-action" "$root/.github/workflows/release.yml" grep -Fxq " group: release-\${{ inputs.release_tag || github.sha }}" \ "$root/.github/workflows/release.yml" grep -Fxq " group: release-please-\${{ github.ref }}" \ @@ -455,13 +523,13 @@ for path, package in packages.items(): manifest = json.loads((root / path / "package.json").read_text()) paths[manifest["name"]] = path dependencies[path] = [*manifest.get("dependencies", {}), *manifest.get("peerDependencies", {})] - else: + elif package["release-type"] == "python": project = tomllib.loads((root / path / "pyproject.toml").read_text())["project"] paths[normalize(project["name"])] = path dependencies[path] = [ normalize(re.match(r"[A-Za-z0-9._-]+", requirement)[0]) for requirement in project.get("dependencies", []) ] -for path in order: +for path in dependencies: for dependency in dependencies[path]: dependency_path = paths.get(dependency) if dependency_path and order.index(dependency_path) > order.index(path): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed6093f..d56afe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: with: persist-credentials: false - run: bash .github/scripts/release_test.sh - - run: shellcheck .github/scripts/*.sh + - run: shellcheck .github/scripts/*.sh actions/experiment/*.sh typescript: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac0f9bb..621be50 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,6 +96,8 @@ jobs: case "$release_type" in node) npm+=("$path") ;; python) python+=("$path") ;; + # The experiment action has no package: the tag-experiment-action job moves its major tag. + simple) continue ;; *) echo "Unsupported release type for $path: $release_type" >&2 exit 1 @@ -289,3 +291,39 @@ jobs: .github/scripts/verify-python-release-dependencies.sh "$wheel" uv publish --trusted-publishing always --check-url https://pypi.org/simple "python/$package"/* done < python/publish-order.txt + + tag-experiment-action: + needs: release-please + concurrency: + group: tag-experiment-action + cancel-in-progress: false + # The default queue cancels a waiting job when another push arrives, and that release then keeps the old major tag. + queue: max + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + - env: + GH_TOKEN: ${{ github.token }} + run: | + version=$(jq -r '.["actions/experiment"] // empty' .release-please-manifest.json) + # A rerun after release-please failed partway gets no outputs for the release, so its tag selects it. + if [[ -z "$version" || "$(git rev-parse -q --verify "refs/tags/experiment-action-v$version^{commit}" || true)" != "$GITHUB_SHA" ]]; then + exit 0 + fi + + # Workflows use the major tag, which moves to each release of its major version, but never back. + major="experiment-action-v${version%%.*}" + current=$(git rev-parse -q --verify "refs/tags/$major^{commit}" || true) + if [[ -z "$current" ]]; then + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f "ref=refs/tags/$major" -f "sha=$GITHUB_SHA" + elif git merge-base --is-ancestor "$current" "$GITHUB_SHA"; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/tags/$major" -f "sha=$GITHUB_SHA" -F force=true + else + echo "$major is at a release after experiment-action-v$version" + fi diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f008dfa..3c5dd92 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,25 +1,26 @@ { "packages/otel": "0.2.0", - "packages/sdk": "0.2.1", + "packages/sdk": "0.2.2", "packages/ai": "0.2.0", - "packages/anthropic": "0.2.1", - "packages/bedrock": "0.2.1", - "packages/cursor": "0.2.1", - "packages/eve": "0.2.1", - "packages/google-genai": "0.2.1", - "packages/mcp": "0.2.1", - "packages/omp": "0.2.1", - "packages/openai": "0.2.1", - "packages/opencode": "0.2.1", - "packages/openrouter": "0.2.1", - "packages/pi": "0.2.1", + "packages/anthropic": "0.2.2", + "packages/bedrock": "0.2.2", + "packages/cursor": "0.2.2", + "packages/eve": "0.2.2", + "packages/google-genai": "0.2.2", + "packages/mcp": "0.2.2", + "packages/omp": "0.2.2", + "packages/openai": "0.2.2", + "packages/opencode": "0.2.2", + "packages/openrouter": "0.2.2", + "packages/pi": "0.2.2", "packages/tanstack-ai": "0.2.0", - "sdks/python": "0.3.1", - "sdks/python-anthropic": "0.2.1", - "sdks/python-bedrock": "0.2.1", - "sdks/python-google-genai": "0.2.1", - "sdks/python-litellm": "0.2.1", - "sdks/python-openai": "0.2.1", - "sdks/python-openrouter": "0.2.1", - "packages/cli": "0.1.0" + "sdks/python": "0.3.2", + "sdks/python-anthropic": "0.2.2", + "sdks/python-bedrock": "0.2.2", + "sdks/python-google-genai": "0.2.2", + "sdks/python-litellm": "0.2.2", + "sdks/python-openai": "0.2.2", + "sdks/python-openrouter": "0.2.2", + "packages/cli": "0.1.2", + "actions/experiment": "1.0.0" } diff --git a/RELEASING.md b/RELEASING.md index 064b46b..ada9052 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -32,3 +32,16 @@ Each registry's publish job stops at the first package that fails, so the packag To retry an existing published release, run the `Release` workflow from `main` and set `release_tag` to that release's full tag. This is a real publication attempt, not a dry run. The tag must be an ancestor of the selected `main` commit. Every configured SDK directory, shared build scripts, lockfiles, release configuration, and root build configuration must be unchanged from the tag; only `packageManager` may differ within root `package.json`. Workflow-only recovery changes are allowed. Source-changing fixes need a new release, not a retry of an old version. Artifact names and versions are checked before upload. npm retries skip an existing version only when its SHA-512 integrity matches the local tarball, and publication waits about 10 minutes for published `@telemetry-dev/*` dependencies and peer dependencies to satisfy their declared ranges. Python publication likewise waits about 10 minutes for each wheel's dependencies to resolve on PyPI. PyPI retries use `uv publish --check-url` to check existing distributions. An integrity mismatch must be investigated, not bypassed or overwritten. + +## Experiment action + +The experiment action in `actions/experiment` has no package. Release Please gives it releases such as `experiment-action-v1.0.0`. The `tag-experiment-action` job of the `Release` workflow then moves the major tag, `experiment-action-v1`, to the release commit. The README of the action and the experiment page of telemetry.dev tell users to use the major tag. The job moves a major tag only forward on `main`, so a rerun of an earlier release does not move it back. + +A change that can break a workflow, for example a removed input or a changed output, must have a `feat!:` or `fix!:` commit. Release Please then gives the action a new major version, and `experiment-action-v1` stays at the last 1.x release. A `release_tag` retry does not apply to the action. If the job fails, re-run the failed job of the release run. + +The job uses `GITHUB_TOKEN`, which cannot have the `workflows` permission. Thus GitHub rejects the tag when no branch head has the same `.github/workflows` files as the release commit. If a merge changed a workflow before the job ran, move the tag with your own credentials: + +```sh +git fetch origin --tags +git push --force origin 'experiment-action-v1.0.1^{commit}:refs/tags/experiment-action-v1' +``` diff --git a/actions/experiment/CHANGELOG.md b/actions/experiment/CHANGELOG.md new file mode 100644 index 0000000..fb5a0a6 --- /dev/null +++ b/actions/experiment/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## 1.0.0 (2026-10-30) + + +### Features + +* **sdk:** add the experiment action for pull requests ([968a45e](https://github.com/telemetry-dev/telemetry/commit/968a45e0eeff183d2409c0336c6328ac07a2e36d)) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk:** use the tsx of the project and change only our comment ([ae0ed55](https://github.com/telemetry-dev/telemetry/commit/ae0ed556f68d1952cbe33ca8f93f29a429fab5a4)) diff --git a/actions/experiment/README.md b/actions/experiment/README.md new file mode 100644 index 0000000..5e7cbbd --- /dev/null +++ b/actions/experiment/README.md @@ -0,0 +1,109 @@ +# telemetry.dev experiment action + +This composite action runs your eval script in a GitHub Actions workflow. The script calls +`runExperiment` (`@telemetry-dev/sdk`) or `run_experiment` (`telemetry-dev`). The action writes the +result to the job summary and to a comment on the pull request. The job fails when the check of the +pass marks fails, when the experiment does not complete, or when the script exits with an error. + +```yaml +on: pull_request + +permissions: + contents: read + pull-requests: write + +jobs: + evals: + # A pull request from a fork gets no secrets, so the job runs only for branches of this + # repository. + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + env: + TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 24 + # Installs tsx too, which runs the .ts script. + - run: npm ci + - uses: telemetry-dev/telemetry/actions/experiment@experiment-action-v1 + with: + dataset: support-golden + script: evals/reply.eval.ts + min-scores: correctness=0.9 +``` + +`evals/reply.eval.ts` runs the experiment: + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init(); + +await runExperiment({ + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness"], +}); +``` + +## Versions + +The tag `experiment-action-v1` moves to each 1.x release of the action. A change that can break a +workflow starts a new major version, `experiment-action-v2`, and `experiment-action-v1` stays at +the last 1.x release. For a run that does not change, use the tag of one release, for example +`experiment-action-v1.0.0`, or its commit SHA. The +[releases](https://github.com/telemetry-dev/telemetry/releases) of this repository list the +changes of each release. + +## Inputs + +| Input | Description | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with the `tsx` of the project, from `node_modules/.bin` in the working directory or a directory above it, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://api.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | + +The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the +traces of the items stay out of your production pages. Thus the key must be for all environments +or for `experiment`. A key for another environment, for example your production key, cannot send +the traces of the items: the experiment then completes with its outputs, but with no traces and no +cost. `CI` is `true` in GitHub Actions, thus the SDK runs in CI mode: it records the repository, +the commit, the run, and the pull request with the experiment, and it exits with an error when the +check fails. + +The action does not download packages. Add `tsx` to the `devDependencies` of the project for a +`.ts` script, or give a `command`. + +The outputs, the errors, and the scorer reasons of the items go to telemetry.dev and stay with the +experiment. The capture options of the SDK and the capture setting of the environment in the +project do not apply to them. + +## Outputs + +| Output | Description | +| -------------- | --------------------------------------------------------------------- | +| `check-status` | `passed`, `failed`, or `none`, for the last experiment of the script. | +| `url` | The page of the last experiment in telemetry.dev. | + +## The comment + +The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or +`command`. A later push changes that comment. The action changes only a comment that its token +wrote: with a copy of the comment by someone else, it adds a new comment. When the token cannot +write comments, the action logs a warning, and the job summary still has the result. + +A pull request from a fork gets no secrets, thus no API key, and the action stops with the error +"Set the api-key input or the TELEMETRY_DEV_API_KEY variable." The `if` of the example above skips +the job for those pull requests. + +The comment and the summary show the name of each experiment, its status, the count of the items +that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK +writes the result for the action to a file in `RUNNER_TEMP` with mode `0600`. diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml new file mode 100644 index 0000000..5cc5fa6 --- /dev/null +++ b/actions/experiment/action.yml @@ -0,0 +1,82 @@ +name: telemetry.dev experiment +description: Run an experiment of your code on a telemetry.dev dataset, check its scores, and post the result on the pull request. +author: telemetry.dev +branding: + icon: check-circle + color: gray-dark + +inputs: + script: + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with the tsx in the node_modules of the project, a .js script with node, and a .py script with python. Give script or command. + required: false + command: + description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. + required: false + dataset: + description: The dataset to run on. When the script also names a dataset, the two must be the same. + required: false + min-scores: + description: Pass marks, as name=value pairs that commas or new lines separate, for example `correctness=0.9`. A pass rate is 0 to 1. They win over the minScores of the script for the same name. + required: false + api-key: + description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. + required: false + base-url: + description: The telemetry.dev API. Default https://api.telemetry.dev. + required: false + working-directory: + description: The directory that the script or the command runs in. + required: false + default: "." + comment: + description: Set to false to write the job summary only, with no comment on the pull request. + required: false + default: "true" + github-token: + description: The token that writes the comment. It needs the pull-requests write permission. + required: false + default: ${{ github.token }} + +outputs: + check-status: + description: passed, failed, or none, for the last experiment of the script. + value: ${{ steps.report.outputs.check-status }} + url: + description: The page of the last experiment in telemetry.dev. + value: ${{ steps.report.outputs.url }} + +runs: + using: composite + steps: + - id: run + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_MIN_SCORES: ${{ inputs.min-scores }} + INPUT_API_KEY: ${{ inputs.api-key }} + INPUT_BASE_URL: ${{ inputs.base-url }} + run: bash "$GITHUB_ACTION_PATH/run.sh" + + - id: report + if: always() + shell: bash + env: + TELEMETRY_DEV_EXPERIMENT_RESULTS: ${{ steps.run.outputs.results }} + EXPERIMENT_EXIT_CODE: ${{ steps.run.outputs.exit-code }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_COMMENT: ${{ inputs.comment }} + GITHUB_TOKEN: ${{ inputs.github-token }} + run: node "$GITHUB_ACTION_PATH/report.mjs" + + - if: always() && steps.run.outputs.exit-code != '0' + shell: bash + env: + EXIT_CODE: ${{ steps.run.outputs.exit-code }} + run: | + echo "::error title=telemetry.dev experiment::The experiment script exited with code ${EXIT_CODE:-unknown}." + exit "${EXIT_CODE:-1}" diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs new file mode 100644 index 0000000..ff94492 --- /dev/null +++ b/actions/experiment/report.mjs @@ -0,0 +1,236 @@ +// Writes the job summary of the experiment action, and the comment with the result on the pull +// request. runExperiment and run_experiment add one line of JSON to the results file for each +// experiment: the body of GET /v1/experiments/{id}. +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync } from "node:fs"; + +const RATE_KINDS = new Set(["pass_fail", "up_down"]); +/** Pages of 100 comments that the action reads to find its comment from an earlier push. */ +const MAX_COMMENT_PAGES = 10; +const STATUS = { completed: "Completed", failed: "Failed", stopped: "Stopped" }; + +const env = process.env; +const exitCode = Number.parseInt(env.EXPERIMENT_EXIT_CODE ?? "", 10); +const results = readResults(env.TELEMETRY_DEV_EXPERIMENT_RESULTS); +// The comment of this job and input, so that a new push changes it and a second step adds its own. +const key = createHash("sha256") + .update( + [ + env.GITHUB_WORKFLOW, + env.GITHUB_JOB, + env.INPUT_DATASET, + env.INPUT_SCRIPT, + env.INPUT_COMMAND, + ].join("\0"), + ) + .digest("hex") + .slice(0, 12); +const marker = ``; +const report = render(results, exitCode); + +if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, `${report}\n`); + +const last = results.at(-1); + +if (env.GITHUB_OUTPUT) { + appendFileSync( + env.GITHUB_OUTPUT, + `check-status=${last?.checkStatus ?? "none"}\nurl=${last ? (httpsUrl(last.url) ?? "") : ""}\n`, + ); +} + +if (env.INPUT_COMMENT !== "false") { + await comment(`${marker}\n${report}`).catch((error) => { + // A pull request from a fork gets a token that cannot write comments. + console.log( + `::warning title=telemetry.dev experiment::The comment on the pull request failed: ${error.message}`, + ); + }); +} + +function readResults(path) { + if (!path) return []; + + let text; + + try { + text = readFileSync(path, "utf8"); + } catch { + return []; + } + + return text + .split("\n") + .filter((line) => line.trim() !== "") + .flatMap((line) => { + try { + const result = JSON.parse(line); + + return typeof result === "object" && result !== null && typeof result.name === "string" + ? [result] + : []; + } catch { + return []; + } + }); +} + +function render(experiments, code) { + const lines = ["### telemetry.dev experiment", ""]; + + if (experiments.length === 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code} before it wrote a result. The log of the step has the error.`, + ); + } + + for (const experiment of experiments) { + const url = httpsUrl(experiment.url); + const name = url ? `[${text(experiment.name)}](${url})` : text(experiment.name); + const total = Number(experiment.itemsTotal) || 0; + const failed = Number(experiment.itemsFailed) || 0; + const ran = Math.max((Number(experiment.itemsDone) || 0) - failed, 0); + + lines.push( + `**${name}**: ${outcome(experiment)}. ${ran} of ${total} items ran${failed > 0 ? `, ${failed} failed` : ""}.`, + ); + + if (typeof experiment.error === "string" && experiment.error !== "") { + lines.push("", `> ${text(experiment.error)}`); + } + + const scores = Array.isArray(experiment.scores) ? experiment.scores : []; + + if (scores.length > 0) { + lines.push( + "", + "| Score | Value | Change | Pass mark | Result |", + "| --- | --- | --- | --- | --- |", + ); + + for (const score of scores) { + lines.push( + `| ${text(score.name)} | ${value(score.kind, score.value)} | ${change(score.kind, score.delta)} | ${value(score.kind, score.minScore)} | ${score.passed === true ? "Passed" : score.passed === false ? "Failed" : "—"} |`, + ); + } + } + + lines.push(""); + } + + if (code !== 0 && experiments.length > 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code}.`, + "", + ); + } + + return lines.join("\n").trimEnd(); +} + +function outcome(experiment) { + if (experiment.status !== "completed") + return STATUS[experiment.status] ?? text(String(experiment.status)); + + if (experiment.checkStatus === "failed") return "Check failed"; + + return experiment.checkStatus === "passed" ? "Check passed" : "Completed"; +} + +function value(kind, number) { + if (typeof number !== "number") return "—"; + + return RATE_KINDS.has(kind) + ? `${Math.round(number * 100)}%` + : String(Math.round(number * 100) / 100); +} + +function change(kind, delta) { + if (typeof delta !== "number") return "—"; + + if (delta === 0) return "0"; + + const size = RATE_KINDS.has(kind) ? `${Math.abs(delta)} pts` : String(Math.abs(delta)); + + return `${delta > 0 ? "↑" : "↓"} ${size}`; +} + +/** Text that cannot break the table or add HTML to the comment. */ +function text(value) { + return String(value) + .replace(/[\r\n]+/g, " ") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\|/g, "\\|") + .replace(/([[\]()*_`])/g, "\\$1"); +} + +function httpsUrl(url) { + return typeof url === "string" && /^https:\/\/[^\s()<>]+$/.test(url) ? url : undefined; +} + +async function comment(body) { + const pull = pullRequestNumber(); + + if (pull === undefined || !env.GITHUB_TOKEN || !env.GITHUB_REPOSITORY) return; + + const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); + const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + // Anyone can paste the marker into a comment, so the action changes only a comment that its token + // wrote. A user token knows its login. The GITHUB_TOKEN and app tokens cannot read /user, and + // they write as a bot. + const login = await github("GET", `${api}/user`).then( + (user) => (typeof user?.login === "string" ? user.login : undefined), + () => undefined, + ); + const ours = (item) => + login === undefined ? item.user?.type === "Bot" : item.user?.login === login; + let existing; + + for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { + const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); + + existing = comments.find( + (item) => typeof item.body === "string" && item.body.startsWith(marker) && ours(item), + ); + + if (comments.length < 100) break; + } + + if (existing) await github("PATCH", `${issue}/comments/${existing.id}`, { body }); + else await github("POST", `${issue}/${pull}/comments`, { body }); +} + +function pullRequestNumber() { + if (!env.GITHUB_EVENT_PATH) return undefined; + + try { + const number = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")).pull_request?.number; + + return Number.isSafeInteger(number) && number > 0 ? number : undefined; + } catch { + return undefined; + } +} + +async function github(method, url, body) { + const headers = { + accept: "application/vnd.github+json", + authorization: `Bearer ${env.GITHUB_TOKEN}`, + "x-github-api-version": "2022-11-28", + }; + const res = await fetch(url, { + method, + ...(body + ? { headers: { ...headers, "content-type": "application/json" }, body: JSON.stringify(body) } + : { headers }), + redirect: "error", + signal: AbortSignal.timeout(15_000), + }); + + if (!res.ok) + throw new Error(`GitHub returned ${res.status} for ${method} ${new URL(url).pathname}`); + + return res.json(); +} diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh new file mode 100755 index 0000000..86cf722 --- /dev/null +++ b/actions/experiment/run.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Runs the experiment script of the experiment action. A failed script does not fail this step: +# the report step reads the exit code from the output exit-code, and the last step exits with it. +set -uo pipefail + +script=${INPUT_SCRIPT:-} +command=${INPUT_COMMAND:-} + +fail() { + echo "::error title=telemetry.dev experiment::$1" + echo "exit-code=2" >>"$GITHUB_OUTPUT" + exit 0 +} + +if [ -n "$script" ] && [ -n "$command" ]; then fail "Give script or command, not the two."; fi +if [ -z "$script" ] && [ -z "$command" ]; then fail "Give the script or the command that runs the experiment."; fi + +if [ -n "${INPUT_API_KEY:-}" ]; then + echo "::add-mask::$INPUT_API_KEY" + export TELEMETRY_DEV_API_KEY=$INPUT_API_KEY +fi +if [ -z "${TELEMETRY_DEV_API_KEY:-}" ]; then fail "Set the api-key input or the TELEMETRY_DEV_API_KEY variable."; fi +if [ -n "${INPUT_BASE_URL:-}" ]; then export TELEMETRY_DEV_BASE_URL=$INPUT_BASE_URL; fi +if [ -n "${INPUT_DATASET:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_DATASET=$INPUT_DATASET; fi +if [ -n "${INPUT_MIN_SCORES:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_MIN_SCORES=$INPUT_MIN_SCORES; fi +# The traces of the items stay out of production unless the job names an environment. +export TELEMETRY_DEV_ENVIRONMENT=${TELEMETRY_DEV_ENVIRONMENT:-experiment} + +# mktemp makes the file with mode 0600, in a directory of this job. +results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail "Could not make the results file." +echo "results=$results" >>"$GITHUB_OUTPUT" +export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results + +# The tsx of the project, from node_modules/.bin here or in a directory above. The action does not +# download one: a package from npm at run time would run with the API key in its environment. +project_tsx() { + dir=$PWD + while :; do + if [ -x "$dir/node_modules/.bin/tsx" ]; then + echo "$dir/node_modules/.bin/tsx" + return 0 + fi + [ "$dir" = / ] && return 1 + dir=$(dirname "$dir") + done +} + +code=0 +if [ -n "$command" ]; then + bash -c "$command" || code=$? +else + case "$script" in + *.ts | *.mts | *.cts | *.tsx) + tsx=$(project_tsx) || fail "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command." + "$tsx" "$script" || code=$? + ;; + *.js | *.mjs | *.cjs) node "$script" || code=$? ;; + *.py) + python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." + "$python" "$script" || code=$? + ;; + *) fail "The script must end in .ts, .js, or .py. For another runner, give a command." ;; + esac +fi + +echo "exit-code=$code" >>"$GITHUB_OUTPUT" diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dc..4eb6eeb 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -41,9 +41,14 @@ repository, not this SDK workspace. Paths below refer to that product repository | error | span status ERROR + `error.type` + `exception` event (`exception.type/message/stacktrace`, `log.severity_number: 17`) | | | `log()` | OTLP log record to `/v1/logs` (always; never span events) | severity debug=5/info=9/warn=13/error=17; trace-correlated via active context | | `withPrompt(prompt, fn)` / `with with_prompt(prompt):` → `prompt.name` | `td.prompt.name` | on model-call spans that start in scope: operation `chat`, `text_completion`, or `generate_content` | +| outside a scope: a message object that `compile()` of a chat prompt returned, in the request messages of a provider integration | `td.prompt.name`, `td.prompt.version`, `td.prompt.label` | on the `chat` spans of the provider integrations listed in [Prompts](#prompts); the scope wins | | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | -| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the newest `compile()` that made a message of the request | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| `runExperiment` / `run_experiment` → each span and log record of an item run | `td.experiment.id` | the id that `POST /v1/experiments` returns; set like a propagated attribute, so the provider integration spans get it | +| the item of the run | `td.dataset.item_id` | the `id` of the dataset item: its key, the same across dataset versions | +| the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | +| the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -142,6 +147,11 @@ Concurrent calls share one fetch. and not to the full response. Only an address lookup can continue after the call stops waiting, until the system resolver gives up. Both SDKs measure the TTL and Retry-After in elapsed time, so a change of the system clock does not move them. +- Fork (Python): a process that `os.fork()` makes gets the copies of the parent, but not the fetch + threads of the parent. A call that starts in the child does not wait for a fetch of the parent. + If the call must fetch, it starts a new fetch. If a signal handler forks while a call waits for a + fetch, that call continues in the child. The call gets `timeout` 10 seconds after the start of + the fetch, because no thread of the child completes the fetch. - Failure: when a fetch fails, the SDKs keep the last copy for another TTL and send the error to `onError` / `on_error`. With no copy, the call gets the fallback. With no fallback, the call gets a `PromptError`. With no copy, the calls of the next 10 seconds (or the TTL, if it is shorter) @@ -178,6 +188,171 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. +Outside a scope, the SDKs find the prompt from the message objects of the request, not from +their text: + +- `compile()` of a chat prompt tags each message object that it makes with the prompt, a copy of + the variables of that call and of each list in them (not of the objects in a list), and a serial + number that grows with each chat compile in the process. With input capture off in the client + at that time, the tag holds no variables. It does not tag the messages of a placeholder + variable: the app passes those, and they keep the tag of their own compile. TypeScript: a + non-enumerable symbol property; the keys and the JSON of a message do not change. Python: the + messages are instances of a private `dict` subclass that holds the tag in a slot. + `isinstance(message, dict)` is true and the keys and the JSON do not change, but a check of the + exact type (`type(message) is dict`), `yaml.safe_dump`, and `marshal` reject it. Its + constructor is the one of `dict`. +- A copy has no tag: a spread, `structuredClone`, or a JSON round trip (TypeScript); `dict()`, + `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, a JSON round trip, `type(message)(message)`, or + `dataclasses.asdict` (Python). A new message with the same text has no tag either, thus a + call never records the prompt of a message from another call, task, or prompt version. +- `promptAttributesFor(messages)` / `prompt_attributes_for(messages)` take a list of messages or + one message (Python: a `list` or `tuple`, not a subclass, so the read runs no code of the app). + They look at the first 1000 and the last 1000 items and give the name, version, label, and + variables of the tag with the highest serial number: the newest compile. Thus a message that a + placeholder brings from an older compile does not win over the compile that put it in the list. + They do not look inside a message. +- In a `withPrompt` / `with_prompt` scope they give nothing. A model-call span whose parent context + is in a scope gets the attributes of the scope and drops each `td.prompt.*` attribute that the + app passes, at start and in each update (`update`, `end`, `updateActiveSpan` / + `update_current_span`), also when the app calls these functions outside the scope and passes an + explicit parent. They never throw: an error goes to `onError` / `on_error`. +- They give `td.prompt.variables` only when input capture is on in the client, at the compile and + at the call. Then they serialize the variables, which calls the `mask` of the client. A span + whose effective captureInput is off drops `td.prompt.variables` from its attributes, also when + the app passes the result of these functions to its own span. +- A tag keeps the variables of its compile for 5 minutes. A call of these functions keeps them + for 5 more minutes when the newest compile of the call is that compile. The SDKs measure this + time in elapsed time, so a change of the system clock does not move it. After that time, + they give the attributes of the tag without `td.prompt.variables`, and the next compile or call + removes the variables from the tag, at most one minute late, also when the app keeps the + message. Other compiles do not change this time. Thus a chat history keeps the variables of the + compiles of the last 5 minutes, for a history of any length or window. A guard prompt, a queue, + or a compile in other code does not remove the variables of a request before a call in these 5 + minutes. A message that more than one request sends keeps its variables while a call names its + compile at least one time in 5 minutes. A request with only messages of earlier compiles gives + the variables of the newest of them, when they are less than 5 minutes old. +- The SDKs keep a weak reference to each tag with variables, by the minute of its time, thus the + reference does not keep a tag in memory. In a runtime without `WeakRef` (Cloudflare Workers with a + compatibility date before 2025-05-05, or with `disable_weak_ref`), the TypeScript tag keeps no + variables. The SDKs keep no other reference to the variables, except those of the last compile + of each prompt object and of each running scope, for `withPrompt` / `with_prompt`. +- `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the + SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. +- The provider integrations call them with the request messages before the span starts, also + when input capture is off: OpenAI chat completions (`messages`) and responses (`input`), + Anthropic messages (`messages`; the `system` string has no tag), Amazon Bedrock Converse + (`messages`), Google GenAI `generateContent` (`contents`), LiteLLM completion (`messages`) and + responses (`input`) (Python), and OpenRouter chat (`messages`) and responses (`input`). Bedrock + Converse and Google GenAI take another message shape: a compiled message that the app changes + in place keeps its tag, and a new message does not. Bedrock InvokeModel takes a serialized body + and is not recorded outside a scope. The other spans of the integrations (embeddings, images, + audio, video, and OpenAI Realtime) do not look at the request. +- Python Anthropic `messages.stream()` starts its span at `__enter__`. It decides there, with the + messages that `.stream()` received, thus a scope around `__enter__` wins. +- A provider integration reads the function from the core module when it runs, thus it also + operates with an older core that does not have it. With such a core, the prompt is recorded + only in a scope. + +## Experiments and scores + +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` / +`score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` +header, the SDK header, and a JSON body. Python also sends +`User-Agent: telemetry-dev-python/`, because bot filters refuse the default +`Python-urllib/3.x`. The body is the JSON of `JSON.stringify`: a lone surrogate goes as a +`\uXXXX` escape. The TypeScript SDK calls `fetch` as a plain function, because Workers and +browsers reject a call whose `this` is another object. The redirect rule of getPrompt applies. An +error has the code of the `{ error: { code, message } }` body, or `http_`, or +`missing_api_key`, `timeout`, `network_error`, or `invalid_response`. + +An experiment run sends these requests in this sequence: + +1. `GET /v1/datasets/{name}?version=&split=&cursor=`. The first page names the `datasetVersion` + option, or no version. Each next page names the `version` of the first page and its + `nextCursor`. A page with a different version, a cursor that repeats, a cursor with no items, + or more than 10,000 items in all is `invalid_response`. A first page with no items is + `dataset_empty`, and no experiment starts. +2. `POST /v1/experiments` with `name`, `dataset`, `datasetVersion` (the version of the first + page), `split`, `evaluators`, `minScores`, `runsPerItem`, `ci`, and `metadata`. Keys without a + value are left out. +3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item + run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, + `null` for no value and for a value that JSON cannot hold, such as an object that holds itself, + cut to 100,000 UTF-16 units), `traceId` (when the span records), `latencyMs`, and, for a task + that failed, `error` (`: `, cut to 4,000 UTF-16 units) or, for a task that + succeeded, `scores`. Python sends a lone surrogate of an output or an error as U+FFFD. Each run + gets its own deep copy of the item, so a task that changes its input does not change the next + run. The posts go one at a time, in order. A post or a page fetch that fails stops the run: no + item starts after it. When the running items end, the SDK sends the request of step 4, sends its + failure to `onError` / `on_error`, and fails with the first error. The server then ends the + experiment with the item runs that it has, or fails it when it has none. Python does the same + after `KeyboardInterrupt` and cancellation. +4. `POST /v1/experiments/{id}/complete`, after `flush()`. +5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is + `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). + +Retries: the GET requests, the item posts, and the complete request try 3 times in all after a +timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that +makes the experiment tries again only after a 429, which the rate limit answers before any work: the +server answers 503 when its database work times out, which can be after the commit. A retry waits +for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 +second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript +request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a +score. The Python timeouts apply to each socket operation of urllib. + +Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). +`concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names +of 1 to 64 UTF-16 units. `scorers` has at most 20 scorers with trimmed names of 1 to 64 UTF-16 +units, and no scorer has the name `user_feedback` or the name of an evaluator of the run in a +different case. `minScores` / `min_scores` has at most 20 trimmed names, with finite numbers. A +Python bool is not an integer or a number for these options. Option errors are `TypeError` and +`RangeError` (TypeScript) or `TypeError` and `ValueError` (Python), before any request. + +Scorers run after the span of the item ends, one after the other. A result of `true` or `false` +is `{ value: 1, label: "pass" }` or `{ value: 0, label: "fail" }`. A number is the `value`. A +string is the trimmed `label`. An object or mapping gives `value` (a boolean there is a pass or a +fail), the trimmed `label`, and `reason` (cut to 4,000 UTF-16 units). `null` / `None` gives no +score. A result that the score rules below reject, or a scorer that throws, gives no score and +sends an error with the code `scorer_failed` to `onError` / `on_error`. + +Score rules, for `score` and for scorer results: the trimmed name has 1 to 64 UTF-16 units. A +value is a finite number. A label has 1 to 64 UTF-16 units after the trim. A score has a value, a +label, or the two. `user_feedback` has the value 0 or 1 and no label. The label `pass` goes with no +value or 1, and `fail` with no value or 0. For `score`: a `traceId` is 32 lowercase hex digits and +not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs a `traceId`, a +`sessionId` has 1 to 1,024 UTF-16 units, and a `comment` has at most 4,000. A score needs a +`traceId` or a `sessionId`. A score that breaks a rule is `invalid_score`, with no request. `score` +never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / +`on_error`. + +The result of step 5 needs `id`, `name`, `status`, `url`, `itemsTotal`, `itemsDone`, `scores`, +and `checkStatus`, else it is `invalid_response`: `checkStatus` decides CI mode, so a missing one +does not read as no check. A field of a score that can be null can also be missing. + +CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: + +- In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 + characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, +runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units, with no half of a + surrogate pair at the end, because Postgres rejects a lone surrogate in jsonb. A URL must be + `https://` with no space, at most 2,048 UTF-16 units. The run URL is + `//actions/runs/`. Keys without a value are left + out. +- After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of + `failed` is `check_failed`. The error has the `result`. + +The experiment action sets these variables: `TELEMETRY_DEV_EXPERIMENT_DATASET` is the dataset when +the `dataset` option has no value, and a different `dataset` option is a `TypeError`. +`TELEMETRY_DEV_EXPERIMENT_MIN_SCORES` holds `name=value` pairs that commas or new lines separate. +The value must match `-?\d+(\.\d+)?` (ASCII digits), else it is a `TypeError`. These marks win over +the `minScores` option for the same name. When `TELEMETRY_DEV_EXPERIMENT_RESULTS` names a file, the +SDK adds the JSON of the `GET /v1/experiments/{id}` body to it as one line. + +The SDKs print one line when the experiment starts and the result when it ends. A run with no +enabled client, or in an environment other than `experiment`, logs a warning. + ## The shared scenario Both e2e runners emit exactly this (under diff --git a/packages/anthropic/CHANGELOG.md b/packages/anthropic/CHANGELOG.md index 8e60fae..4138558 100644 --- a/packages/anthropic/CHANGELOG.md +++ b/packages/anthropic/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/anthropic-v0.2.1...anthropic-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/anthropic-v0.2.0...anthropic-v0.2.1) (2026-10-30) diff --git a/packages/anthropic/package.json b/packages/anthropic/package.json index afa055e..c52455f 100644 --- a/packages/anthropic/package.json +++ b/packages/anthropic/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/anthropic", - "version": "0.2.1", + "version": "0.2.2", "description": "Anthropic SDK instrumentation for telemetry.dev: wraps the Messages API and emits OpenTelemetry GenAI spans.", "keywords": [ "anthropic", diff --git a/packages/anthropic/src/index.ts b/packages/anthropic/src/index.ts index d9cb1f6..1082844 100644 --- a/packages/anthropic/src/index.ts +++ b/packages/anthropic/src/index.ts @@ -5,6 +5,7 @@ import { type SpanHandle, type StartSpanOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { Stream } from "@anthropic-ai/sdk/core/streaming"; import { Messages as BetaMessages } from "@anthropic-ai/sdk/resources/beta/messages"; import { Messages } from "@anthropic-ai/sdk/resources/messages"; @@ -147,6 +148,16 @@ function toolDefinitions(tools: T): Record | undefined { : undefined; } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(messages: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(messages) : undefined; +} + function messagesRequest(body: ValueRecord): RequestMapping { const model = readString(body.model); @@ -161,7 +172,10 @@ function messagesRequest(body: ValueRecord): RequestMapping { type: "generation", model, input, - attributes: toolDefinitions(body.tools), + attributes: { + ...toolDefinitions(body.tools), + ...promptAttributes(body.messages), + }, systemInstructions: body.system, temperature: readNumber(body.temperature), topP: readNumber(body.top_p), diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index e53d040..6516205 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -4,7 +4,14 @@ import { type PushMetricExporter, } from "@opentelemetry/sdk-metrics"; import { InMemorySpanExporter, type ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { flush, init, shutdown, type TelemetryOptions } from "@telemetry-dev/sdk"; +import { + type ChatMessage, + flush, + getPrompt, + init, + shutdown, + type TelemetryOptions, +} from "@telemetry-dev/sdk"; import * as telemetrySdk from "@telemetry-dev/sdk"; import Anthropic from "@anthropic-ai/sdk"; import { createRequire } from "node:module"; @@ -274,6 +281,72 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { messages: "compiled", olderCore: false }, + { messages: "copied", olderCore: false }, + { messages: "compiled", olderCore: true }, +])( + "messages.create outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ messages: kind, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: "https://prompts.anthropic.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "anthropic", name: "claude-sonnet-4-6" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const message = user as { role: "user"; content: string }; + const spans = setupSpans(); + const client = clientWith(createFakeFetch(jsonResponse(messagePayload())).fetch); + + await client.messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + system: system!.content, + messages: [kind === "compiled" ? message : { ...message }], + }); + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.input.messages"]).toBeDefined(); + + if (olderCore || kind === "copied") { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }); + }, +); + test("messages.create maps request, response, usage, finish reason, provider, and sampling attributes", async () => { const spans = setupSpans(); const messages = [{ role: "user" as const, content: "Say hello" }]; diff --git a/packages/bedrock/CHANGELOG.md b/packages/bedrock/CHANGELOG.md index 3c4a5e9..eb08036 100644 --- a/packages/bedrock/CHANGELOG.md +++ b/packages/bedrock/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/bedrock-v0.2.1...bedrock-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/bedrock-v0.2.0...bedrock-v0.2.1) (2026-10-30) diff --git a/packages/bedrock/package.json b/packages/bedrock/package.json index 2efe9cc..c53be42 100644 --- a/packages/bedrock/package.json +++ b/packages/bedrock/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/bedrock", - "version": "0.2.1", + "version": "0.2.2", "description": "AWS Bedrock instrumentation for telemetry.dev: Converse, InvokeModel, and Bedrock Agents spans for AWS SDK v3 clients.", "keywords": [ "aws", diff --git a/packages/bedrock/src/converse.ts b/packages/bedrock/src/converse.ts index 074d1b1..fd9c8e1 100644 --- a/packages/bedrock/src/converse.ts +++ b/packages/bedrock/src/converse.ts @@ -9,6 +9,7 @@ import { numberValue, omitUndefined, parseJson, + promptAttributes, PROVIDER, stringValue, StreamRetention, @@ -34,6 +35,7 @@ export function converseRequestFields(input: JsonRecord): SpanFields { model: stringValue(input.modelId), input: normalizeMessages(input.messages), systemInstructions: normalizeSystem(input.system), + attributes: promptAttributes(input.messages), temperature: numberValue(inference.temperature), topP: numberValue(inference.topP), maxTokens: numberValue(inference.maxTokens), diff --git a/packages/bedrock/src/internal.ts b/packages/bedrock/src/internal.ts index 104f595..2a89e01 100644 --- a/packages/bedrock/src/internal.ts +++ b/packages/bedrock/src/internal.ts @@ -192,6 +192,16 @@ export function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +export function promptAttributes(messages: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(messages) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(name: string, options: StartSpanOptions): SpanHandle { const withheld = options.input !== undefined && isCut(options) && captureMaskedBeforeStart(); diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index 8e94b8f..0eb0e7f 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -9,7 +9,8 @@ import { } from "@aws-sdk/client-bedrock-runtime"; import { afterEach, expect, test, vi } from "vitest"; -import type { MaskFn } from "@telemetry-dev/sdk"; +import { type ChatMessage, getPrompt, type MaskFn } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { instrumentBedrock, uninstrumentBedrock, wrapBedrock } from "../src/index.ts"; import { @@ -423,6 +424,90 @@ test("ConverseStream records stream errors with partial output", async () => { expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); }); +test.each([ + { messages: "changed in place", olderCore: false }, + { messages: "new", olderCore: false }, + { messages: "changed in place", olderCore: true }, +])( + "Converse outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ messages: kind, olderCore }) => { + const spans = setup(); + + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: "https://prompts.bedrock.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "amazon-bedrock", name: "anthropic.claude-3-haiku" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const blocks = [{ text: user!.content }]; + // Converse takes content blocks: a message changed in place keeps its tag, a new one has none. + const message = + kind === "new" + ? { role: "user", content: blocks } + : Object.assign(user!, { content: blocks }); + + const client = wrapBedrock( + new BedrockRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("prompt-1", { + output: { message: { role: "assistant", content: [{ text: "Soon." }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }), + }), + ); + + await client.send( + new ConverseCommand({ + modelId: "anthropic.claude-3-haiku", + system: [{ text: system!.content }], + messages: [message] as ConverseCommandInput["messages"], + }), + ); + + const [span] = spans.getFinishedSpans(); + expect(span!.attributes["gen_ai.response.id"]).toBe("prompt-1"); + expect( + Object.entries(span!.attributes).filter(([key]) => key.startsWith("td.prompt.")), + ).toEqual( + olderCore || kind === "new" + ? [] + : [ + ["td.prompt.name", "support/reply"], + ["td.prompt.version", 12], + ["td.prompt.label", "production"], + ["td.prompt.variables", '{"team":"billing","order":"#4"}'], + ], + ); + }, +); + test("InvokeModel captures provider-native bodies and embedding spans", async () => { const spans = setup(); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index f43ce5e..29bd94a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [0.1.2](https://github.com/telemetry-dev/telemetry/compare/cli-v0.1.1...cli-v0.1.2) (2026-10-30) + + +### Features + +* **cli:** add --json to login and logout ([#131](https://github.com/telemetry-dev/telemetry/issues/131)) ([505309b](https://github.com/telemetry-dev/telemetry/commit/505309bf84c00edf3036488348086e10a5870510)) +* **cli:** map telemetry.dev's tool error codes to cli error codes ([#134](https://github.com/telemetry-dev/telemetry/issues/134)) ([b639de9](https://github.com/telemetry-dev/telemetry/commit/b639de9f388ed3ce42335b82cff8728c61c2d322)) + +## [0.1.1](https://github.com/telemetry-dev/telemetry/compare/cli-v0.1.0...cli-v0.1.1) (2026-10-30) + + +### Features + +* **cli:** report errors as one JSON line with a stable code under --json ([#127](https://github.com/telemetry-dev/telemetry/issues/127)) ([dcad14b](https://github.com/telemetry-dev/telemetry/commit/dcad14b08926862409cfaf94c529ecfab8df26e5)) +* **cli:** summarize a trace's failures and repeated calls, and suggest next commands ([#128](https://github.com/telemetry-dev/telemetry/issues/128)) ([02f63f5](https://github.com/telemetry-dev/telemetry/commit/02f63f5bb3ac5fc3c3c28d8198c0605961768077)) + ## 0.1.0 (2026-10-30) diff --git a/packages/cli/README.md b/packages/cli/README.md index 07df7f7..d7deb0c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -34,6 +34,18 @@ In a shell with no terminal to paste into, such as an agent's, `telemetry login telemetry login --finish 'http://127.0.0.1:53124/callback?code=…' ``` +A script or agent can run `telemetry login --json` instead. It never opens a browser or waits. It prints the link to pass on to a person, then exits: + +```json +{ + "status": "pending", + "authorization_url": "https://telemetry.dev/api/auth/oauth2/authorize?…", + "next": "telemetry login --finish '
'" +} +``` + +`telemetry login --finish '
' --json` then prints the sign-in, with the same fields as `telemetry status --json` and `"status": "signed_in"`. When the workspace has exactly one project, it becomes the default. `project` is `null` otherwise, and also when the projects couldn't be loaded. + A sign-in covers one workspace. To read another workspace, run `telemetry login` again and choose it. The new sign-in replaces the old one, and the CLI asks telemetry.dev to revoke the old one. ## Use @@ -61,7 +73,7 @@ Add `--json` to any command for machine-readable output on stdout. Messages and `telemetry trace` and `telemetry payload` read every page of a trace or a captured text, up to a limit: 2,000 spans for a trace, and 1,000,000 characters for each payload field. Their JSON output has `complete: false` when the limit cut it short, and human output says so on stderr. -Captured text comes from your users, models, and tools. Human-readable output, including `payload --field`, removes terminal escape sequences and control characters from that text. JSON output escapes them, so use `--json` when you need the exact text. +Captured text comes from your users, models, and tools. telemetry.dev removes ANSI color and cursor sequences, bidirectional overrides, and Unicode tag characters from results before they reach the CLI; it does not clean the text of error messages. Human-readable output, including `payload --field`, removes the remaining escape sequences and control characters. JSON output escapes them instead of removing them, so use `--json` when you need everything the server sent. | Exit code | Meaning | | --------- | --------------------------------------------------------------- | @@ -71,6 +83,40 @@ Captured text comes from your users, models, and tools. Human-readable output, i | 4 | You are not signed in, the sign-in expired, or access is denied | | 130 | Interrupted with Ctrl-C | +With `--json`, an error is one line of JSON on stderr, and stdout stays empty. Ctrl-C exits with 130 and no JSON, and `status --json` reports a missing sign-in as data on stdout with exit code 4. + +```text +{"error":{"code":"not_signed_in","message":"You're not signed in. Run `telemetry login` first.","hint":"telemetry login","retryable":false,"exit_code":4,"cli_version":"0.1.0"}} +``` + +Branch on `code`, not on `message`, whose wording can change. `hint` is a command to run next, when there is one. `telemetry login` needs a person at a browser, so an agent should run `telemetry login --json` and pass the link on rather than run `telemetry login`. `retryable` is true when running the same command again may succeed. New codes may be added, and existing codes keep their meaning. + +| Code | Exit | Meaning | +| ---------------------- | ---- | -------------------------------------------------------------------------------- | +| `invalid_arguments` | 2 | An option or argument is not valid | +| `unknown_command` | 2 | There is no command with that name | +| `project_not_found` | 2 | No project in the workspace has that name or id | +| `invalid_cursor` | 2 | `--cursor` isn't from an earlier page of this command | +| `project_ambiguous` | 2 | Several projects have that name; pass its id | +| `project_required` | 2 | The workspace has several projects and none was chosen | +| `no_projects` | 1 | The workspace has no projects yet | +| `not_signed_in` | 4 | No sign-in is saved for this deployment | +| `sign_in_expired` | 4 | The sign-in expired or was disconnected | +| `payloads_not_allowed` | 4 | The sign-in doesn't include reading captured prompts and outputs | +| `access_denied` | 4 | The account no longer has access to the workspace | +| `sign_in_failed` | 4 | The sign-in was cancelled, refused, or used an address from another sign-in | +| `no_pending_sign_in` | 4 | `login --finish` found no sign-in waiting to finish | +| `trace_not_found` | 1 | No trace has that id, or it is past the project's retention | +| `span_not_found` | 1 | The trace has no span with that id, or it is past the project's retention | +| `unreachable` | 1 | telemetry.dev could not be reached | +| `timeout` | 1 | telemetry.dev took too long to answer | +| `server_error` | 1 | telemetry.dev failed to complete the request; try again | +| `tool_error` | 1 | telemetry.dev returned another error; `message` says what it was | +| `result_too_large` | 1 | The result was too large to read; narrow it with filters or `--limit` | +| `request_failed` | 1 | The request failed for another reason | +| `failed` | 1 | The command failed for another reason | +| `internal` | 1 | An unexpected error, such as a configuration file the CLI couldn't read or write | + ## Configuration | Variable | Default | Purpose | @@ -80,4 +126,4 @@ Captured text comes from your users, models, and tools. Human-readable output, i | `TELEMETRY_DEV_CONFIG_DIR` | `~/.config/telemetry-dev` | Where the sign-in is saved. The default follows `XDG_CONFIG_HOME` when set | | `NO_COLOR` | none | Turns off color. `--no-color` does the same | -The sign-in is saved in `session.json`, which only your user can read. A sign-in to another deployment through `TELEMETRY_DEV_URL` gets its own file next to it. The file holds a refresh token, so keep it private. `telemetry logout` revokes the token at telemetry.dev and removes it from the file, and says so when the server doesn't confirm the revocation. To disconnect the CLI from your account entirely, open account settings at telemetry.dev. +The sign-in is saved in `session.json`, which only your user can read. A sign-in to another deployment through `TELEMETRY_DEV_URL` gets its own file next to it. The file holds a refresh token, so keep it private. `telemetry logout` revokes the token at telemetry.dev and removes it from the file, and says so when the server doesn't confirm the revocation. `telemetry logout --json` prints `{"was_signed_in": true, "revoked": false}` in that case; `revoked` is `null` when there was no sign-in. To disconnect the CLI from your account entirely, open account settings at telemetry.dev. diff --git a/packages/cli/package.json b/packages/cli/package.json index daad4d3..56710fc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/cli", - "version": "0.1.0", + "version": "0.1.2", "description": "The telemetry.dev command line: search traces, sessions, logs, and AI usage from your terminal or an agent's shell.", "keywords": [ "cli", diff --git a/packages/cli/src/api.ts b/packages/cli/src/api.ts index b9ee36d..6a6d962 100644 --- a/packages/cli/src/api.ts +++ b/packages/cli/src/api.ts @@ -1,7 +1,7 @@ import type { Client } from "@modelcontextprotocol/client"; import { SessionAuthProvider } from "./auth.ts"; -import { CliError, EXIT, notSignedIn } from "./errors.ts"; +import { CliError, notSignedIn, toolError, translateError } from "./errors.ts"; import { mcpUrl, type Session } from "./session.ts"; import { withSpinner } from "./terminal.ts"; @@ -74,8 +74,7 @@ export class Api { const text = result.content.find((block) => block.type === "text")?.text ?? ""; - if (result.isError) - throw new CliError(text || "telemetry.dev could not complete this request."); + if (result.isError) throw toolError(text, result._meta); try { return JSON.parse(text) as T; @@ -83,6 +82,7 @@ export class Api { // The server cuts results over its size cap, which leaves the JSON unterminated. throw new CliError( "The result was too large to read. Narrow it with filters or a smaller --limit.", + { code: "result_too_large" }, ); } } @@ -92,39 +92,3 @@ export class Api { await client?.close().catch(() => {}); } } - -// Each is a refresh or authorization the server refused for good. -type AuthErrors = readonly (abstract new (...args: never[]) => Error)[]; - -const translateError = (error: unknown, authErrors: AuthErrors): never => { - if (error instanceof CliError) throw error; - - if (authErrors.some((type) => error instanceof type)) { - throw new CliError( - "Your sign-in has expired or was disconnected. Run `telemetry login` again.", - EXIT.auth, - ); - } - - const cause = error instanceof Error && error.cause instanceof Error ? error.cause : error; - const message = cause instanceof Error ? cause.message : String(cause); - const code = (cause as { code?: unknown } | null)?.code; - - if ( - typeof code === "string" && - ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ECONNRESET", "ETIMEDOUT"].includes(code) - ) { - throw new CliError( - `Couldn't reach telemetry.dev (${code}). Check your connection and try again.`, - ); - } - - if ( - (error as { code?: unknown } | null)?.code === "REQUEST_TIMEOUT" || - (cause instanceof Error && (cause.name === "TimeoutError" || cause.name === "AbortError")) - ) { - throw new CliError("telemetry.dev took too long to answer. Try again shortly."); - } - - throw new CliError(`telemetry.dev request failed: ${message}`); -}; diff --git a/packages/cli/src/auth.ts b/packages/cli/src/auth.ts index 1e8a14c..a486fe6 100644 --- a/packages/cli/src/auth.ts +++ b/packages/cli/src/auth.ts @@ -12,7 +12,7 @@ import type { StoredOAuthTokens, } from "@modelcontextprotocol/client"; -import { CliError, EXIT } from "./errors.ts"; +import { CliError, EXIT, translateError } from "./errors.ts"; import { loadSession, mcpUrl, type Session, updateSession } from "./session.ts"; import { oneLine } from "./terminal.ts"; @@ -264,10 +264,11 @@ export class SessionAuthProvider implements OAuthClientProvider { } const sessionExpired = () => - new CliError( - "Your sign-in has expired or was disconnected. Run `telemetry login` again.", - EXIT.auth, - ); + new CliError("Your sign-in has expired or was disconnected. Run `telemetry login` again.", { + exitCode: EXIT.auth, + code: "sign_in_expired", + hint: "telemetry login", + }); export const grantedScopes = (session: Session) => new Set((session.tokens?.scope ?? "").split(" ").filter(Boolean)); @@ -425,6 +426,8 @@ const readPastedCallback = async (state: string, signal: AbortSignal): Promise void; @@ -441,7 +444,7 @@ const authorizationParams = (address: string, state: string) => { if (params?.get("state") !== state) { throw new CliError( "That isn't the address from this sign-in. Copy the browser's whole address bar after you select Allow.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed" }, ); } @@ -457,8 +460,13 @@ const authorize = async ( provider: SessionAuthProvider, options: { authorizationCode?: string; iss?: string }, ) => { - const { auth, AuthorizationServerMismatchError, IssuerMismatchError, OAuthError } = - await import("@modelcontextprotocol/client"); + const { + auth, + AuthorizationServerMismatchError, + IssuerMismatchError, + OAuthClientFlowError, + OAuthError, + } = await import("@modelcontextprotocol/client"); try { return await auth(provider, { @@ -475,18 +483,27 @@ const authorize = async ( error.code === "invalid_grant" ? "That sign-in link was already used or has expired. Run `telemetry login` again." : `Sign-in failed: ${oneLine(error.message).slice(0, 300)}`, - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } if (error instanceof IssuerMismatchError || error instanceof AuthorizationServerMismatchError) { throw new CliError( "The sign-in answer came from a different authorization server than this sign-in started with. Run `telemetry login` again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } - throw error; + // Such as a refused client registration or a token endpoint that isn't https. + if (error instanceof OAuthClientFlowError) { + throw new CliError(`Sign-in failed: ${oneLine(error.message).slice(0, 300)}`, { + exitCode: EXIT.auth, + code: "sign_in_failed", + hint: "telemetry login", + }); + } + + return translateError(error, []); } }; @@ -499,7 +516,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => error === "access_denied" ? "Sign-in was cancelled in the browser." : `Sign-in failed: ${oneLine(params.get("error_description") ?? error)}.`, - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } @@ -508,7 +525,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => if (!code) { throw new CliError( "The browser returned without an authorization code. Run `telemetry login` again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } @@ -548,6 +565,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => export const login = async ({ session, browser, + defer = false, finish, onAuthorizationUrl, onWaiting, @@ -587,7 +605,7 @@ export const login = async ({ if (!pending) { throw new CliError( "No sign-in is waiting to finish. Run `telemetry login --no-browser` to start one.", - EXIT.auth, + { exitCode: EXIT.auth, code: "no_pending_sign_in", hint: "telemetry login --no-browser" }, ); } @@ -611,8 +629,8 @@ export const login = async ({ const state = randomBytes(16).toString("base64url"); const callback = await listenForCallback(state); const abort = new AbortController(); - const useBrowser = browser && canOpenBrowser(); - const deferred = !useBrowser && !process.stdin.isTTY; + const useBrowser = browser && !defer && canOpenBrowser(); + const deferred = defer || (!useBrowser && !process.stdin.isTTY); const provider = new SessionAuthProvider(signedIn, { redirectUrl: callback.redirectUrl, @@ -661,7 +679,7 @@ export const login = async ({ reject( new CliError( "Sign-in timed out after 10 minutes. Run `telemetry login` to try again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ), ), LOGIN_TIMEOUT_MS, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 19c6769..93d877a 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -5,6 +5,7 @@ import { VERSION } from "./api.ts"; import { type Context, home, + jsonText, loginCommand, logoutCommand, logsCommand, @@ -18,7 +19,7 @@ import { usageCommand, useCommand, } from "./commands.ts"; -import { CliError, EXIT, usageError } from "./errors.ts"; +import { CliError, type ErrorCode, EXIT, usageError } from "./errors.ts"; import { clean, disableColor, SHOW_CURSOR, style } from "./terminal.ts"; type Option = { @@ -60,7 +61,7 @@ const cursor: Option = { const COMMANDS: Readonly> = { login: { summary: "Sign in with your browser", - usage: "telemetry login [--no-browser | --finish
]", + usage: "telemetry login [--no-browser | --finish
] [--json]", options: { "no-browser": { type: "boolean", @@ -72,18 +73,23 @@ const COMMANDS: Readonly> = { description: "Finish a sign-in started without a terminal, with the browser's final address", }, + json: { + type: "boolean", + description: "Print JSON: the sign-in link without waiting, or the sign-in after --finish", + }, }, examples: [ "telemetry login", "telemetry login --no-browser # over SSH, in a container, or from an agent's shell", "telemetry login --finish 'http://127.0.0.1:53124/callback?code=…'", + "telemetry login --json # a script passes the link on, then runs --finish", ], run: loginCommand, }, logout: { summary: "Sign out on this machine", - usage: "telemetry logout", - options: {}, + usage: "telemetry logout [--json]", + options: { json }, examples: ["telemetry logout"], run: logoutCommand, }, @@ -309,8 +315,18 @@ const closest = (input: string, names: readonly string[]) => { const commandNamed = (name: string | undefined) => name !== undefined && Object.hasOwn(COMMANDS, name) ? COMMANDS[name] : undefined; +// Arguments after `--` are values, never options. +const optionArgs = (argv: readonly string[]) => { + const end = argv.indexOf("--"); + + return end === -1 ? argv : argv.slice(0, end); +}; + +const jsonOutput = optionArgs(process.argv.slice(2)).includes("--json"); + const main = async (argv: readonly string[]) => { - if (argv.includes("--no-color")) disableColor(); + // With --json, stderr carries no spinner or color, so an error line is plain JSON. + if (argv.includes("--no-color") || jsonOutput) disableColor(); // The command comes first; only --no-color may precede it. const first = argv.findIndex((arg) => arg !== "--no-color"); @@ -338,7 +354,12 @@ const main = async (argv: readonly string[]) => { const command = commandNamed(topic); - if (!command) throw usageError(`Unknown command "${topic}". Run telemetry --help.`); + if (!command) { + throw usageError(`Unknown command "${topic}". Run telemetry --help.`, { + code: "unknown_command", + hint: "telemetry --help", + }); + } return void process.stdout.write(`${commandHelp(topic, command)}\n`); } @@ -350,6 +371,7 @@ const main = async (argv: readonly string[]) => { throw usageError( `Unknown command "${name}".${suggestion ? ` Did you mean \`telemetry ${suggestion}\`?` : ""} Run telemetry --help.`, + { code: "unknown_command", hint: "telemetry --help" }, ); } @@ -373,13 +395,17 @@ const main = async (argv: readonly string[]) => { } catch (error) { const message = error instanceof Error ? error.message.split("\n")[0] : String(error); - throw usageError(`${message} Run \`telemetry ${name} --help\`.`); + throw usageError(`${message} Run \`telemetry ${name} --help\`.`, { + hint: `telemetry ${name} --help`, + }); } const extra = parsed.positionals[command.arguments ?? 0]; if (extra !== undefined) { - throw usageError(`Unexpected argument "${extra}". Run \`telemetry ${name} --help\`.`); + throw usageError(`Unexpected argument "${extra}". Run \`telemetry ${name} --help\`.`, { + hint: `telemetry ${name} --help`, + }); } await command.run({ @@ -390,21 +416,40 @@ const main = async (argv: readonly string[]) => { }; const fail = (error: unknown) => { - process.stderr.write(SHOW_CURSOR.repeat(process.stderr.isTTY ? 1 : 0)); - - // Tool errors are written by the server and can quote what the request named. - if (error instanceof CliError) { - process.stderr.write(`${style.red("error", process.stderr)} ${clean(error.message)}\n`); - process.exitCode = error.exitCode; + if (!jsonOutput) process.stderr.write(SHOW_CURSOR.repeat(process.stderr.isTTY ? 1 : 0)); + + const known = error instanceof CliError ? error : undefined; + const exitCode = known?.exitCode ?? EXIT.failure; + process.exitCode = exitCode; + + // With --json, a script or agent reads the error as one JSON line on stderr; stdout + // stays empty, so it never mistakes an error for data. + if (jsonOutput) { + process.stderr.write( + `${jsonText({ + error: { + code: known?.code ?? ("internal" satisfies ErrorCode), + message: known ? known.message : error instanceof Error ? error.message : String(error), + ...(known?.hint ? { hint: known.hint } : {}), + retryable: known?.retryable ?? false, + exit_code: exitCode, + cli_version: VERSION, + }, + })}\n`, + ); return; } - // Unexpected, so the stack is kept for a bug report; it can still quote a server. - process.stderr.write( - `${style.red("error", process.stderr)} ${clean(error instanceof Error ? (error.stack ?? error.message) : String(error))}\n`, - ); - process.exitCode = EXIT.failure; + // Tool errors are written by the server and can quote what the request named. An + // unexpected error keeps its stack for a bug report; it can still quote a server. + const message = known + ? known.message + : error instanceof Error + ? (error.stack ?? error.message) + : String(error); + + process.stderr.write(`${style.red("error", process.stderr)} ${clean(message)}\n`); }; // The MCP SDK warns app developers on the console, for instance when it retries a refresh. diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 5755555..765e57f 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -1,6 +1,6 @@ import { Api } from "./api.ts"; import { grantedScopes, login, logout, PAYLOADS_SCOPE } from "./auth.ts"; -import { CliError, EXIT, notSignedIn, usageError } from "./errors.ts"; +import { CliError, EXIT, notSignedIn, payloadsNotAllowed, usageError } from "./errors.ts"; import { printMark } from "./mark.ts"; import { DEFAULT_SERVER, @@ -15,6 +15,7 @@ import { clean, type Column, DASH, + displayWidth, formatAge, formatCost, formatCount, @@ -58,16 +59,17 @@ const note = (text = "") => process.stderr.write(`${text}\n`); /** * JSON for scripts and agents. JSON.stringify escapes C0 controls but not C1 controls - * or bidirectional overrides, which some terminals act on, so those are escaped too. + * or bidirectional overrides, which some terminals act on, or the line and paragraph + * separators some line readers split on, so those are escaped too. */ -const printJson = (value: unknown) => - out( - JSON.stringify(value, null, 2).replace( - /[\u0080-\u009f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, - (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`, - ), +export const jsonText = (value: unknown, indent?: number) => + JSON.stringify(value, null, indent).replace( + /[\u0080-\u009f\u061c\u200e\u200f\u2028\u2029\u202a-\u202e\u2066-\u2069]/g, + (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`, ); +const printJson = (value: unknown) => out(jsonText(value, 2)); + const json = (context: Context) => context.values.json === true; const string = (context: Context, name: string) => { @@ -123,10 +125,14 @@ const matchProject = (projects: ProjectList["projects"], wanted: string): Projec if (byName.length > 1) { throw usageError( `Several projects are named "${wanted}". Pass its id instead: telemetry projects`, + { code: "project_ambiguous", hint: "telemetry projects" }, ); } - throw usageError(`No project "${wanted}" in this workspace. Projects: ${names || "none"}.`); + throw usageError(`No project "${wanted}" in this workspace. Projects: ${names || "none"}.`, { + code: "project_not_found", + hint: "telemetry projects", + }); }; /** @@ -148,10 +154,13 @@ const resolveProject = async (api: Api, context: Context): Promise => { if (projects.length === 1) return toProject(projects[0]!); if (projects.length === 0) - throw new CliError("This workspace has no projects yet. Create one at telemetry.dev."); + throw new CliError("This workspace has no projects yet. Create one at telemetry.dev.", { + code: "no_projects", + }); throw usageError( `Choose a project with --project , or set a default with \`telemetry use \`. Projects: ${projectNames(projects)}.`, + { code: "project_required", hint: "telemetry projects" }, ); }; @@ -188,13 +197,59 @@ const choice = ( const DAYS = { "7d": 7, "30d": 30, "90d": 90 } as const; -/** `arg` quoted for a POSIX shell, so a suggested command runs as shown when pasted. */ +/** + * `arg` quoted for a POSIX shell, so a suggested command runs as shown when pasted. A + * leading `=` is quoted too: zsh expands `=name` to a command's path. + */ const shellQuote = (arg: string) => - /^[\w@%+=:,./-]+$/.test(arg) ? arg : `'${arg.replaceAll("'", `'\\''`)}'`; + /^[\w@%+:,./-][\w@%+=:,./-]*$/.test(arg) ? arg : `'${arg.replaceAll("'", `'\\''`)}'`; + +/** + * A telemetry command to suggest, quoted for a POSIX shell, that keeps this command's + * project so it reads the same one when pasted. Option values are joined with `=`, so a + * value starting with `-` isn't read as an option. Undefined when an id starts with `-` + * or a value holds characters the terminal line can't show as they are, since the + * printed command would then name something else. + */ +const suggestion = ( + context: Context, + command: string, + ids: readonly string[], + options: Readonly> = {}, +) => { + const project = string(context, "project"); + const values = { ...options, ...(project ? { project } : {}) }; + const shown = (text: string) => clean(text) === text && !/[\n\t]/.test(text); + + if (ids.some((id) => id.startsWith("-") || !shown(id))) return undefined; + + if (!Object.values(values).every(shown)) return undefined; + + return [ + "telemetry", + command, + ...ids, + ...Object.entries(values).map(([name, value]) => `--${name}=${value}`), + ] + .map(shellQuote) + .join(" "); +}; + +const labelled = (label: string, command: string | undefined) => + command === undefined ? undefined : `${label}: ${command}`; + +/** Prints likely next commands as one dim block on stderr, never with --json. */ +const suggest = (context: Context, ...lines: (string | undefined)[]) => { + const shown = lines.filter((line): line is string => line !== undefined); -/** Suggests the same command with the next page's cursor. */ -const suggestNextPage = (context: Context, cursor: string | null) => { - if (!cursor || json(context)) return; + if (json(context) || shown.length === 0) return; + + note(style.dim(`\n${shown.join("\n")}`, process.stderr)); +}; + +/** The same command with the next page's cursor. */ +const nextPage = (context: Context, cursor: string | null) => { + if (!cursor) return undefined; const kept: string[] = []; @@ -210,9 +265,10 @@ const suggestNextPage = (context: Context, cursor: string | null) => { kept.push(arg); } - const command = [...kept, "--cursor", cursor].map((arg) => shellQuote(clean(arg))).join(" "); + // A cursor is base64url, which can start with `-`. + const command = [...kept, `--cursor=${cursor}`].map((arg) => shellQuote(clean(arg))).join(" "); - note(style.dim(`\nMore: telemetry ${command}`, process.stderr)); + return `More: telemetry ${command}`; }; const statusCell = (status: string): Cell => @@ -267,20 +323,29 @@ export async function loginCommand(context: Context) { const server = serverUrl(); const session = await loadSession(server); const stream = process.stderr; + const asJson = json(context); const finish = typeof context.values.finish === "string" ? context.values.finish : undefined; - if (process.stderr.isTTY && finish === undefined) { + if (process.stderr.isTTY && finish === undefined && !asJson) { stream.write("\n"); await printMark(TAGLINE, { animate: true, stream }); stream.write("\n"); } + let authorizationUrl = ""; + const result = await login({ session, browser: context.values["no-browser"] !== true, + // A program reading JSON can't wait for a person, so it gets the link to pass on. + defer: asJson, finish, onAuthorizationUrl: (url, opened) => { + authorizationUrl = url.href; + + if (asJson) return; + note( opened ? "Opening your browser to sign in to telemetry.dev." @@ -294,11 +359,21 @@ export async function loginCommand(context: Context) { note(style.dim("Waiting for you to choose a workspace and select Allow…", stream)); }, }).catch((error: unknown) => { - reportStatus("error"); + if (!asJson) reportStatus("error"); throw error; }); if (result.status === "pending") { + if (asJson) { + printJson({ + status: "pending", + authorization_url: authorizationUrl, + next: "telemetry login --finish '
'", + }); + + return; + } + note( "After you select Allow, the browser opens an http://127.0.0.1 address that doesn't load. Copy the whole address and run:", ); @@ -310,64 +385,94 @@ export async function loginCommand(context: Context) { const signedIn = result.session; - reportStatus("done"); + if (!asJson) reportStatus("done"); + // The sign-in is saved by now, so a failed lookup leaves the project unset rather than + // failing a sign-in that worked. const api = new Api(signedIn); + let projects: ProjectList["projects"] | undefined; + let lookupFailure: CliError | undefined; try { - const { projects } = await api.call("list_projects", {}, "Loading projects"); + ({ projects } = await api.call("list_projects", {}, "Loading projects")); + } catch (error) { + if (!(error instanceof CliError)) throw error; - if (projects.length === 1) { - const project = toProject(projects[0]!); + lookupFailure = error; + } finally { + await api.close(); + } - signedIn.project = project; - await updateSession(signedIn.server, (saved) => ({ ...saved, project })); - } + if (projects?.length === 1) { + const project = toProject(projects[0]!); - note(); + signedIn.project = project; + await updateSession(signedIn.server, (saved) => ({ ...saved, project })); + } + + if (asJson) { + printJson({ status: "signed_in", ...sessionJson(signedIn) }); + + return; + } + + note(); + note( + `${style.green("✓", stream)} Signed in${signedIn.email ? ` as ${style.bold(signedIn.email, stream)}` : ""}`, + ); + + if (!grantedScopes(signedIn).has(PAYLOADS_SCOPE)) { note( - `${style.green("✓", stream)} Signed in${signedIn.email ? ` as ${style.bold(signedIn.email, stream)}` : ""}`, + style.dim( + " Captured prompts and outputs stay hidden. Sign in again and allow them to use `telemetry payload`.", + stream, + ), ); + } - if (!grantedScopes(signedIn).has(PAYLOADS_SCOPE)) { - note( - style.dim( - " Captured prompts and outputs stay hidden. Sign in again and allow them to use `telemetry payload`.", - stream, - ), - ); - } - - if (signedIn.project) { - note(` Project ${style.bold(signedIn.project.name, stream)}`); - note(); - note( - `Try ${style.bold("telemetry traces", stream)} or ${style.bold("telemetry usage", stream)}.`, - ); - } else if (projects.length > 1) { - note(` ${projects.length} projects: ${projectNames(projects)}`); - note(); - note( - `Choose a default: ${style.bold(`telemetry use ${shellQuote(toProject(projects[0]!).name)}`, stream)}`, - ); - } else { - note(" This workspace has no projects yet. Create one at telemetry.dev."); - } - } finally { - await api.close(); + if (lookupFailure) { + note(style.dim(` Couldn't load your projects: ${oneLine(lookupFailure.message)}`, stream)); + note(style.dim(" Run telemetry projects to list them.", stream)); + } else if (signedIn.project) { + note(` Project ${style.bold(signedIn.project.name, stream)}`); + note(); + note( + `Try ${style.bold("telemetry traces", stream)} or ${style.bold("telemetry usage", stream)}.`, + ); + } else if (projects && projects.length > 1) { + note(` ${projects.length} projects: ${projectNames(projects)}`); + note(); + note( + `Choose a default: ${style.bold(`telemetry use ${shellQuote(toProject(projects[0]!).name)}`, stream)}`, + ); + } else { + note(" This workspace has no projects yet. Create one at telemetry.dev."); } } -export async function logoutCommand() { +export async function logoutCommand(context: Context) { const session = await loadSession(serverUrl()); if (!session.tokens) { + if (json(context)) { + printJson({ was_signed_in: false, revoked: null }); + + return; + } + note("You're not signed in."); return; } const { revoked } = await logout(session); + + if (json(context)) { + printJson({ was_signed_in: true, revoked }); + + return; + } + note( `${style.green("✓", process.stderr)} Signed out${session.email ? ` of ${session.email}` : ""}.`, ); @@ -381,19 +486,22 @@ export async function logoutCommand() { ); } +/** What `status --json` and `login --json` say about a sign-in. */ +const sessionJson = (session: Session) => ({ + signed_in: Boolean(session.tokens), + server: session.server, + email: session.email ?? null, + project: session.project ?? null, + payloads_allowed: grantedScopes(session).has(PAYLOADS_SCOPE), +}); + export async function statusCommand(context: Context) { const server = serverUrl(); const session = await loadSession(server); const scopes = grantedScopes(session); if (json(context)) { - printJson({ - signed_in: Boolean(session.tokens), - server, - email: session.email ?? null, - project: session.project ?? null, - payloads_allowed: scopes.has(PAYLOADS_SCOPE), - }); + printJson(sessionJson(session)); } else if (session.tokens) { out( `${style.green("✓")} Signed in${session.email ? ` as ${style.bold(oneLine(session.email))}` : ""}`, @@ -509,7 +617,15 @@ export async function tracesCommand(context: Context) { ), ); - suggestNextPage(context, page.next_cursor); + const failure = page.traces.find((trace) => trace.status === "error"); + + suggest( + context, + failure + ? labelled("Inspect the newest failure", suggestion(context, "trace", [failure.id])) + : labelled("Inspect the newest trace", suggestion(context, "trace", [page.traces[0]!.id])), + nextPage(context, page.next_cursor), + ); }); } @@ -571,7 +687,14 @@ export async function sessionsCommand(context: Context) { ), ); - suggestNextPage(context, page.next_cursor); + suggest( + context, + labelled( + "List the most recent session's traces", + suggestion(context, "traces", [], { session: page.sessions[0]!.id }), + ), + nextPage(context, page.next_cursor), + ); }); } @@ -668,7 +791,7 @@ export async function traceCommand(context: Context) { if (!traceId) throw usageError("Name the trace to show: telemetry trace "); - await withApi(async (api) => { + await withApi(async (api, session) => { const project = await resolveProject(api, context); let page = await api.call( "get_trace", @@ -680,11 +803,24 @@ export async function traceCommand(context: Context) { let pages = 1; while (page.next_cursor && pages < MAX_TRACE_PAGES) { - page = await api.call( - "get_trace", - { project_id: project.id, trace_id: traceId, cursor: page.next_cursor }, - `Loading spans (${spans.length} of ${page.span_count})`, - ); + page = await api + .call( + "get_trace", + { project_id: project.id, trace_id: traceId, cursor: page.next_cursor }, + `Loading spans (${spans.length} of ${page.span_count})`, + ) + .catch((error: unknown) => { + // The cursor came from the server, so a refused one means the server changed + // between pages, not that the person passed a bad --cursor. + if (error instanceof CliError && error.code === "invalid_cursor") { + throw new CliError( + "telemetry.dev stopped accepting this trace's pages while it loaded. Run the command again.", + { code: "server_error", retryable: true }, + ); + } + + throw error; + }); spans.push(...page.spans); logs.push(...page.logs); pages += 1; @@ -713,9 +849,209 @@ export async function traceCommand(context: Context) { ), ); } + + // The span list carries only the start of an error; payload reads all of it. + const explained = failureCauses(spans).find((span) => span.error?.trim()); + + suggest( + context, + explained && grantedScopes(session).has(PAYLOADS_SCOPE) + ? labelled( + "Read the full error", + suggestion(context, "payload", [page.trace.id, explained.id], { field: "error" }), + ) + : undefined, + ); }); } +/** + * Failed spans with no failed child, in the order they started. Instrumentation marks + * every span an exception passes through as failed, so these are where failures began. + */ +const failureCauses = (spans: readonly Span[]) => { + const failed = spans.filter((span) => span.status === "error"); + const failedParents = new Set(failed.map((span) => span.parent_span_id)); + + return failed + .filter((span) => !failedParents.has(span.id)) + .sort((left, right) => Date.parse(left.started_at) - Date.parse(right.started_at)); +}; + +// The Errors and Repeated sections each show this many rows, then a count of the rest. +const SECTION_ROWS = 5; + +/** A section title like Logs, saying so when it covers only the spans read so far. */ +const sectionTitle = (title: string, count: number, partialOf: number | undefined) => + style.bold( + `${title} ${style.dim(`(${formatInteger(count)}${partialOf === undefined ? "" : ` in the first ${formatInteger(partialOf)} spans`})`)}`, + ); + +const more = (count: number) => + count > SECTION_ROWS ? [style.dim(` and ${formatInteger(count - SECTION_ROWS)} more`)] : []; + +/** The first line of the span's error text, after its type unless the text starts with it. */ +const errorSummary = (span: Span) => { + const type = oneLine(span.error_type ?? ""); + const first = + (span.error ?? "") + .split(/\r\n|\r|\n/) + .map(oneLine) + .find(Boolean) ?? ""; + + if (!first) return type || "error"; + + return !type || first.startsWith(type) ? first : `${type}: ${first}`; +}; + +const PATH = " › "; + +// On a narrow terminal the error text is cut to this many columns before any ancestor +// is dropped from the path, and is left out when fewer than ERROR_MIN columns remain. +const ERROR_ROOM = 30; +const ERROR_MIN = 8; +// The fewest columns worth giving the ancestors before leaving them out. +const ANCESTRY_MIN = 4; + +/** + * A failed span on one line: its ancestors, the span, and its error, fitted to `width`. + * The span's full error text is in the tree below, so the path keeps priority. + */ +const failureLine = (span: Span, byId: ReadonlyMap, width: number) => { + const ancestors: string[] = []; + const seen = new Set([span.id]); + let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined; + + while (parent && !seen.has(parent.id)) { + seen.add(parent.id); + ancestors.push(spanLabel(parent)); + parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined; + } + + ancestors.reverse(); + + const leaf = spanLabel(span); + const error = errorSummary(span); + const room = width - 2; + const errorWidth = displayWidth(error); + const reserve = Math.min(errorWidth, ERROR_ROOM); + const pathWidth = (kept: readonly string[]) => displayWidth([...kept, leaf].join(PATH)); + let kept = ancestors; + + // The root stays, so the path still says which agent or request it was under. + for ( + let dropped = 1; + dropped < ancestors.length && pathWidth(kept) + 2 + reserve > room; + dropped += 1 + ) { + kept = [ancestors[0]!, "…", ...ancestors.slice(dropped + 1)]; + } + + // When even the shortened path is too wide, the ancestors are cut, not the failed span. + const ancestry = kept.join(PATH); + const ancestryRoom = room - displayWidth(leaf) - displayWidth(PATH); + const lead = + kept.length === 0 + ? "" + : displayWidth(ancestry) <= ancestryRoom + ? `${ancestry}${PATH}` + : ancestryRoom >= ANCESTRY_MIN + ? `${fit(ancestry, ancestryRoom)}${PATH}` + : ""; + const path = fit(`${lead}${leaf}`, room); + const painted = path === `${lead}${leaf}` ? `${style.dim(lead)}${leaf}` : path; + const left = room - displayWidth(path) - 2; + const shown = left >= Math.min(errorWidth, ERROR_MIN) ? ` ${style.red(fit(error, left))}` : ""; + + return ` ${painted}${shown}`; +}; + +/** Milliseconds covered by at least one interval, so calls that overlap count once. */ +const wallTime = (intervals: readonly (readonly [number, number])[]) => { + let covered = 0; + let reached = Number.NEGATIVE_INFINITY; + + for (const [start, end] of [...intervals].sort((left, right) => left[0] - right[0])) { + if (end <= reached) continue; + covered += end - Math.max(start, reached); + reached = end; + } + + return covered; +}; + +/** + * Where a long trace went wrong and where its time went, above the span tree: failed + * spans with their path, and operations that ran more than once. + */ +const traceSummary = (spans: readonly Span[], total: number, partialOf: number | undefined) => { + const lines: string[] = []; + const byId = new Map(spans.map((span) => [span.id, span])); + const failed = failureCauses(spans); + const only = failed.length === 1 ? failed[0] : undefined; + // A root that failed with no message says nothing the header doesn't. + const rootOnly = + only !== undefined && + !only.error?.trim() && + !(only.parent_span_id && byId.has(only.parent_span_id)); + + if (failed.length > 0 && !rootOnly) { + lines.push(sectionTitle("Errors", failed.length, partialOf)); + + for (const span of failed.slice(0, SECTION_ROWS)) + lines.push(failureLine(span, byId, outputWidth())); + lines.push(...more(failed.length), ""); + } + + // Grouped by the label the span tree shows: the name with its model, tool, or agent. + const groups = new Map(); + + for (const span of spans) { + const start = Date.parse(span.started_at); + + if (!Number.isFinite(start)) continue; + + const label = spanLabel(span); + const intervals = groups.get(label) ?? []; + + intervals.push([start, start + Math.max(0, span.duration_ms ?? 0)]); + groups.set(label, intervals); + } + + const repeated = [...groups] + .filter(([, intervals]) => intervals.length > 1) + .map(([label, intervals]) => ({ label, count: intervals.length, wall: wallTime(intervals) })) + .sort((left, right) => right.wall - left.wall); + + if (repeated.length > 0) { + lines.push(sectionTitle("Repeated", repeated.length, partialOf)); + + // The empty first column indents the rows; its header line is dropped. + const rows = table( + [ + { header: "" }, + { header: "", flex: true }, + { header: "", align: "right" }, + { header: "", align: "right" }, + { header: "", align: "right" }, + ], + repeated + .slice(0, SECTION_ROWS) + .map(({ label, count, wall }) => [ + "", + label, + `×${formatInteger(count)}`, + formatDuration(wall), + { text: total > 0 ? formatPercent((wall / total) * 100) : DASH, paint: dim }, + ]), + ); + + lines.push(...rows.split("\n").slice(1), ...more(repeated.length), ""); + } + + return lines; +}; + function renderTrace(page: TraceDetailPage, spans: readonly Span[], logs: TraceDetailPage["logs"]) { const { trace } = page; const failed = trace.status === "error"; @@ -757,6 +1093,13 @@ function renderTrace(page: TraceDetailPage, spans: readonly Span[], logs: TraceD const end = Math.max(...spans.map((span, index) => starts[index]! + (span.duration_ms ?? 0))); const total = Math.max(end - origin, trace.duration_ms ?? 0); + for (const line of traceSummary( + spans, + total, + page.next_cursor === null ? undefined : spans.length, + )) + out(line); + const terminal = process.stdout.isTTY === true; const timelineWidth = terminal ? Math.min(60, Math.max(16, outputWidth() - 72)) : 0; @@ -1083,10 +1426,7 @@ export async function payloadCommand(context: Context) { await withApi(async (api, session) => { if (!grantedScopes(session).has(PAYLOADS_SCOPE)) { - throw new CliError( - 'This sign-in can\'t read captured prompts and outputs. Run `telemetry login`, check "Also read captured prompts, outputs, and tool arguments", and select Allow.', - EXIT.auth, - ); + throw payloadsNotAllowed(); } const project = await resolveProject(api, context); diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index 2973b3e..4a45f32 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -7,17 +7,189 @@ export const EXIT = { interrupted: 130, } as const; +/** What scripts and agents branch on. Listed in the README; only ever added to. */ +export type ErrorCode = + | "failed" + | "internal" + | "invalid_arguments" + | "unknown_command" + | "project_not_found" + | "project_ambiguous" + | "project_required" + | "no_projects" + | "not_signed_in" + | "sign_in_expired" + | "sign_in_failed" + | "no_pending_sign_in" + | "payloads_not_allowed" + | "access_denied" + | "invalid_cursor" + | "trace_not_found" + | "span_not_found" + | "unreachable" + | "timeout" + | "server_error" + | "tool_error" + | "result_too_large" + | "request_failed"; + +type ErrorDetails = { + readonly exitCode?: number; + readonly code?: ErrorCode; + /** A command to run next. `telemetry login` needs a person at a browser. */ + readonly hint?: string; + /** Whether running the same command again may succeed. */ + readonly retryable?: boolean; +}; + /** A failure with a message written for the person running the command. */ export class CliError extends Error { readonly exitCode: number; + readonly code: ErrorCode; + readonly hint: string | undefined; + readonly retryable: boolean; - constructor(message: string, exitCode: number = EXIT.failure) { + constructor(message: string, details: number | ErrorDetails = {}) { super(message); + const { + exitCode = EXIT.failure, + code, + hint, + retryable = false, + } = typeof details === "number" ? { exitCode: details } : details; this.exitCode = exitCode; + this.code = code ?? (exitCode === EXIT.usage ? "invalid_arguments" : "failed"); + this.hint = hint; + this.retryable = retryable; } } -export const usageError = (message: string) => new CliError(message, EXIT.usage); +export const usageError = (message: string, details: Omit = {}) => + new CliError(message, { ...details, exitCode: EXIT.usage }); + +export const payloadsNotAllowed = () => + new CliError( + 'This sign-in can\'t read captured prompts and outputs. Run `telemetry login`, check "Also read captured prompts, outputs, and tool arguments", and select Allow.', + { exitCode: EXIT.auth, code: "payloads_not_allowed", hint: "telemetry login" }, + ); + +/** + * The CliError for a tool error, from the code telemetry.dev puts in the result's + * `_meta`. A server without codes, or a code this CLI doesn't know, gives `tool_error`. + * The result's text is written for agents, so codes a person acts on get the CLI's own. + */ +export const toolError = (text: string, meta: unknown): CliError => { + const error = (meta as Record | undefined)?.["dev.telemetry/error"] as + | { code?: unknown; retryable?: unknown } + | undefined; + const retryable = error?.retryable === true; + + switch (error?.code) { + case "access_denied": + return new CliError( + "Your account no longer has access to this workspace. Run `telemetry login` and choose a workspace you belong to.", + { exitCode: EXIT.auth, code: "access_denied", hint: "telemetry login" }, + ); + case "payloads_not_allowed": + return payloadsNotAllowed(); + case "project_not_found": + return usageError( + "That project isn't in this workspace; it may have been deleted. Run `telemetry projects` to list the projects, and `telemetry use ` to change the default.", + { code: "project_not_found", hint: "telemetry projects" }, + ); + case "invalid_cursor": + return usageError( + "That cursor isn't from an earlier page of this command. Run it again without --cursor.", + { code: "invalid_cursor" }, + ); + case "trace_not_found": + case "span_not_found": + return new CliError(text, { code: error.code }); + case "server_error": + return new CliError(text, { code: "server_error", retryable }); + default: + return new CliError(text || "telemetry.dev could not complete this request.", { + code: "tool_error", + retryable, + }); + } +}; export const notSignedIn = () => - new CliError("You're not signed in. Run `telemetry login` first.", EXIT.auth); + new CliError("You're not signed in. Run `telemetry login` first.", { + exitCode: EXIT.auth, + code: "not_signed_in", + hint: "telemetry login", + }); + +// Each is a refresh or authorization the server refused for good. +type AuthErrors = readonly (abstract new (...args: never[]) => Error)[]; + +const NETWORK_CODES = new Set([ + "ECONNREFUSED", + "ECONNRESET", + "EAI_AGAIN", + "EHOSTUNREACH", + "ENETUNREACH", + "ENOTFOUND", + "ETIMEDOUT", + "UND_ERR_SOCKET", +]); + +// undici's own timeouts, raised by fetch before any of the CLI's timers fire. +const TIMEOUT_CODES = new Set([ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", +]); + +export const translateError = (error: unknown, authErrors: AuthErrors): never => { + if (error instanceof CliError) throw error; + + if (authErrors.some((type) => error instanceof type)) { + throw new CliError( + "Your sign-in has expired or was disconnected. Run `telemetry login` again.", + { exitCode: EXIT.auth, code: "sign_in_expired", hint: "telemetry login" }, + ); + } + + // The network error sits at the bottom of a chain: the SDK wraps fetch's + // "fetch failed", which wraps the socket error that has the code. + const chain: unknown[] = [error]; + + for (let link = error; link instanceof Error && link.cause !== undefined && chain.length < 8; ) { + link = link.cause; + chain.push(link); + } + + const codes = chain + .map((link) => (link as { code?: unknown } | null)?.code) + .filter((code): code is string => typeof code === "string"); + const network = codes.find((code) => NETWORK_CODES.has(code)); + // Node's AggregateError for a failed connection has an empty message. + const message = + chain.findLast((link): link is Error => link instanceof Error && link.message !== "") + ?.message ?? String(error); + + if (network) { + throw new CliError( + `Couldn't reach telemetry.dev (${network}). Check your connection and try again.`, + { code: "unreachable", retryable: true }, + ); + } + + if ( + codes.some((code) => code === "REQUEST_TIMEOUT" || TIMEOUT_CODES.has(code)) || + chain.some( + (link) => + link instanceof Error && (link.name === "TimeoutError" || link.name === "AbortError"), + ) + ) { + throw new CliError("telemetry.dev took too long to answer. Try again shortly.", { + code: "timeout", + retryable: true, + }); + } + + throw new CliError(`telemetry.dev request failed: ${message}`, { code: "request_failed" }); +}; diff --git a/packages/cli/src/session.ts b/packages/cli/src/session.ts index 644f9cf..61ffb2e 100644 --- a/packages/cli/src/session.ts +++ b/packages/cli/src/session.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from "node:crypto"; -import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; -import { homedir } from "node:os"; +import { link, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { homedir, hostname } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; @@ -117,38 +117,153 @@ const saveSession = async (session: Session) => { }; const LOCK_RETRY_MS = 25; -// A lock is held only for a read and a write, so one this old was left by a killed process. +// For a lock whose holder can't be checked: one written by CLI 0.1.0, which holds only a +// token, or by another machine sharing the config directory. A lock is held only for a +// read and a write, so one this old was most likely left by a killed process. const LOCK_STALE_MS = 3000; const LOCK_WAIT_MS = 2 * LOCK_STALE_MS; -const readLock = (path: string) => readFile(path, "utf8").catch(() => undefined); +type LockOwner = { readonly token: string; readonly pid?: number; readonly host?: string }; + +// The locks and markers this process holds, by token. +const held = new Set(); + +const readLock = async (path: string): Promise => { + const text = await readFile(path, "utf8").catch(() => undefined); + + if (text === undefined) return undefined; + + try { + const owner = JSON.parse(text) as Partial | null; + + if (typeof owner?.token === "string") return owner as LockOwner; + } catch {} + + return { token: text }; +}; + +/** Creates `path` naming this process as its owner, and returns its token; undefined when it exists. */ +const claim = async (path: string) => { + const owner: LockOwner = { + token: randomBytes(12).toString("hex"), + pid: process.pid, + host: hostname(), + }; + + const record = JSON.stringify(owner); + // Written whole first and linked into place, so where hard links work no command finds + // the file without its owner. Written fresh each time, so its modification time is when + // it was claimed. + const draft = `${path}.${owner.token}.tmp`; + + // Recorded before the file appears: another update in this process can read it before + // the claim resolves, and would take it for one left by an earlier process. + held.add(owner.token); + + try { + await writeFile(draft, record, { flag: "wx", mode: 0o600 }); + await link(draft, path).catch(async (error: NodeJS.ErrnoException) => { + if (error.code === "EEXIST") throw error; + + // A filesystem without hard links; the record is written in place instead. + await writeFile(path, record, { flag: "wx", mode: 0o600 }); + }); + } catch (error) { + held.delete(owner.token); + + if ((error as NodeJS.ErrnoException).code === "EEXIST") return undefined; + + throw error; + } finally { + // Best effort: a leftover draft has a unique name and nothing reads it, and a failure + // here must not turn a claim that succeeded into an error. + await rm(draft, { force: true }).catch(() => {}); + } + + return owner.token; +}; /** - * Takes the lock at `path` and returns its owner token. Only a lock still holding the - * token that was read is broken or released, so a command never removes another's lock. + * Whether the command holding a lock has exited without releasing it. On this machine a + * running holder keeps its lock however long it pauses, such as a laptop asleep mid-write. + * A lock from another machine or CLI 0.1.0, or one caught empty while being written on a + * filesystem without hard links, is judged by its age. */ -const lock = async (path: string) => { - const token = randomBytes(12).toString("hex"); - const deadline = Date.now() + LOCK_WAIT_MS; +const abandoned = async (path: string, owner: LockOwner) => { + const { pid } = owner; + + if ( + typeof pid === "number" && + Number.isSafeInteger(pid) && + pid > 0 && + owner.host === hostname() + ) { + // An earlier process with this one's pid left it, as when every run is pid 1 in a container. + if (pid === process.pid) return !held.has(owner.token); - for (;;) { try { - await writeFile(path, token, { flag: "wx", mode: 0o600 }); + process.kill(pid, 0); - return token; + return false; } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return (error as NodeJS.ErrnoException).code === "ESRCH"; } + } - const held = await stat(path).catch(() => undefined); - const owner = await readLock(path); + const changed = await stat(path).catch(() => undefined); - if (held && owner !== undefined && Date.now() - held.mtimeMs > LOCK_STALE_MS) { - if ((await readLock(path)) === owner) await rm(path, { force: true }); - continue; + return changed !== undefined && Date.now() - changed.mtimeMs > LOCK_STALE_MS; +}; + +/** + * Removes the lock at `path` if it still holds `token`. Every removal, whether a holder + * releasing its lock or a command breaking an abandoned one, first creates a marker named + * after the token, so while holders are judged by their pid, two commands never act on + * one lock, and one can't remove a lock another took after its check. A marker whose + * owner exited partway is removed the same way. Returns false when another command holds + * the marker. + */ +const removeLock = async (path: string, token: string): Promise => { + const marker = `${path}.${createHash("sha256").update(token).digest("hex").slice(0, 16)}`; + const mine = await claim(marker); + + if (!mine) { + const other = await readLock(marker); + + if (other && (await abandoned(marker, other))) await removeLock(marker, other.token); + + return false; + } + + try { + if ((await readLock(path))?.token === token) await rm(path, { force: true }); + + return true; + } finally { + // Held for one read and one removal. Its owner is this running process, so on this + // machine no other command removes it meanwhile. + try { + await rm(marker, { force: true }); + } finally { + held.delete(mine); } + } +}; - if (Date.now() > deadline) { +/** Takes the lock at `path` and returns its owner token. */ +const lock = async (path: string) => { + const deadline = performance.now() + LOCK_WAIT_MS; + + for (;;) { + const token = await claim(path); + + if (token) return token; + + const owner = await readLock(path); + + if (owner && (await abandoned(path, owner))) await removeLock(path, owner.token); + + if (performance.now() > deadline) { throw new CliError( `Another telemetry command is holding the sign-in file. If none is running, delete ${path} and try again.`, ); @@ -158,8 +273,23 @@ const lock = async (path: string) => { } }; +// When another command holds the marker, it is breaking this lock under the age rule, +// which can't see that this command is running. Waits for the lock or the marker to go, +// but not past LOCK_WAIT_MS: once this process exits, its lock is broken by its pid. const unlock = async (path: string, token: string) => { - if ((await readLock(path)) === token) await rm(path, { force: true }); + const deadline = performance.now() + LOCK_WAIT_MS; + + try { + while ( + !(await removeLock(path, token)) && + (await readLock(path))?.token === token && + performance.now() < deadline + ) { + await sleep(LOCK_RETRY_MS); + } + } finally { + held.delete(token); + } }; /** diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 71a80e6..ab319b3 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -1,21 +1,28 @@ -import { execFile } from "node:child_process"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { execFile, spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { type AddressInfo, createServer } from "node:net"; +import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { afterAll, beforeAll, beforeEach, expect, test } from "vitest"; +import { afterAll, beforeAll, beforeEach, expect, test, vi } from "vitest"; -import { sessionFile } from "../src/session.ts"; +import { type Session, sessionFile, updateSession } from "../src/session.ts"; import { beforeRefresh, calls, EMAIL, + PARALLEL_TRACE_ID, PAYLOADS, PROJECTS, + QUIET_TRACE_ID, REFRESH_TOKEN, + serverState, startFixture, TOKEN, + TRACE_ID, traces, } from "./fixture.ts"; @@ -34,17 +41,16 @@ afterAll(async () => { const sessionPath = () => join(home, sessionFile(fixture.url)); -const signIn = (project?: (typeof PROJECTS)[number]) => +const signIn = ( + project?: (typeof PROJECTS)[number], + scope = "openid email offline_access telemetry:read telemetry:payloads", +) => writeFile( sessionPath(), JSON.stringify({ server: fixture.url, client: { client_id: "fixture-client" }, - tokens: { - access_token: TOKEN, - token_type: "Bearer", - scope: "openid email offline_access telemetry:read telemetry:payloads", - }, + tokens: { access_token: TOKEN, token_type: "Bearer", scope }, email: "ada@acme.dev", ...(project ? { project: { id: project.id, name: "support-bot" } } : {}), }), @@ -55,6 +61,11 @@ beforeEach(async () => { home = await mkdtemp(join(tmpdir(), "telemetry-cli-")); calls.length = 0; delete beforeRefresh.run; + Object.assign(serverState, { + failProjects: false, + removedFromWorkspace: false, + errorCodes: true, + }); return () => rm(home, { recursive: true, force: true }); }); @@ -80,6 +91,15 @@ const runWith = (env: Record, ...args: string[]) => const run = (...args: string[]) => runWith({}, ...args); +/** The address of a port that was just closed, which refuses connections. */ +const closedServer = () => + new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(`http://127.0.0.1:${port}`)); + }); + }); + const CONTROL = new RegExp(String.raw`[\u0000-\u0008\u000b-\u001f\u007f-\u009f]`); test("human output never passes terminal escapes from captured text, and --json keeps the data", async () => { @@ -154,6 +174,100 @@ test("payload --field prints the whole text across pages", async () => { expect(calls.filter((call) => call.tool === "get_span_payload").length).toBeGreaterThan(1); }); +test("trace leads with where the run failed and which calls repeated, counting parallel calls once", async () => { + await signIn(PROJECTS[0]); + + const result = await run("trace", PARALLEL_TRACE_ID); + expect(result.code).toBe(0); + + const lines = result.stdout.split("\n"); + // Three 1 s searches started 50 ms apart cover 100 ms to 1,200 ms: 1.1 s of the 3 s + // trace, where adding them up would say 3 s and 100%. + expect(lines).toContain(" execute_tool web_search ×3 1.10s 37%"); + // The request that failed, under the tool that made it, under the agent. + expect(lines).toContain( + " invoke_agent researcher › execute_tool fetch_page › GET /page TimeoutError: Request timed out after 700 ms", + ); + expect(lines.indexOf("Errors (1)")).toBeGreaterThan(0); + expect(lines.indexOf("Repeated (1)")).toBeGreaterThan(lines.indexOf("Errors (1)")); + expect(lines.findIndex((line) => line.startsWith("SPAN"))).toBeGreaterThan( + lines.indexOf("Repeated (1)"), + ); + // Only the request where the failure began is listed and offered, not the tool and the + // agent it passed through. + expect(result.stderr.match(/telemetry payload .*/g)).toEqual([ + `telemetry payload ${PARALLEL_TRACE_ID} t2 --field=error`, + ]); + + // Two chat calls that don't overlap cover 1.18 s + 1.42 s of the 4.21 s trace. Measuring + // from the first start to the last end would say 2.64 s. + const sequential = await run("trace", TRACE_ID); + expect(sequential.stdout.split("\n")).toContain(" chat gpt-5 ×2 2.60s 62%"); + + // A sign-in without the payloads scope isn't pointed at a command that would refuse it. + await signIn(PROJECTS[0], "openid email offline_access telemetry:read"); + const unscoped = await run("trace", PARALLEL_TRACE_ID); + expect(unscoped.code).toBe(0); + expect(unscoped.stderr).not.toContain("telemetry payload"); +}); + +test("a trace with no failures and no repeated calls goes straight to the span tree", async () => { + await signIn(PROJECTS[0]); + + const result = await run("trace", QUIET_TRACE_ID); + expect(result.code).toBe(0); + expect(result.stdout).toMatch(/^SPAN /m); + expect(result.stdout).not.toMatch(/^(Errors|Repeated) /m); + expect(result.stderr).toBe(""); +}); + +/** The words a POSIX shell makes of a pasted command line. */ +const shellWords = (command: string) => + new Promise((resolve, reject) => { + execFile("sh", ["-c", `printf '%s\\n' ${command}`], (error, stdout) => + error ? reject(error) : resolve(stdout.trimEnd().split("\n")), + ); + }); + +test("suggested next commands run as pasted, in the same project, and never print with --json", async () => { + await signIn(); + + // The newest trace succeeded, so the suggestion names the failed one after it. + const listed = await run("traces", "-p", "support-bot"); + expect(listed.code).toBe(0); + const inspect = listed.stderr.match(/Inspect the newest failure: (.*)/)?.[1]; + expect(await shellWords(inspect!)).toEqual([ + "telemetry", + "trace", + TRACE_ID, + "--project=support-bot", + ]); + + const sessions = await run("sessions", "--project=support-bot"); + expect(sessions.code).toBe(0); + const words = await shellWords(sessions.stderr.match(/telemetry traces .*/)![0]); + expect(words).toEqual([ + "telemetry", + "traces", + "--session=Ada's thread 42", + "--project=support-bot", + ]); + + // Run as pasted, it asks for that session in that project. + calls.length = 0; + expect((await run(...words.slice(1))).code).toBe(0); + expect(calls.find((call) => call.tool === "search_traces")?.args).toMatchObject({ + project_id: PROJECTS[0]!.id, + session_id: "Ada's thread 42", + }); + + for (const args of [["traces"], ["sessions"], ["trace", PARALLEL_TRACE_ID]]) { + const result = await run(...args, "-p", "support-bot", "--json"); + expect(result.code).toBe(0); + expect(result.stderr).toBe(""); + } +}); + test("use saves the default project in a file only its owner can read", async () => { await signIn(); @@ -192,6 +306,172 @@ test("input that isn't valid exits 2 before calling telemetry.dev, even from a s expect(calls).toEqual([]); }); +test("with --json, an error is one JSON line on stderr with a code to branch on, and stdout stays empty", async () => { + const errorOf = async (result: { code: number; stdout: string; stderr: string }) => { + expect(result.stdout).toBe(""); + expect(result.stderr.trim().split("\n")).toHaveLength(1); + + return { exit: result.code, ...JSON.parse(result.stderr).error }; + }; + + expect(await errorOf(await run("traces", "--json"))).toMatchObject({ + exit: 4, + code: "not_signed_in", + hint: "telemetry login", + retryable: false, + exit_code: 4, + }); + + await signIn(PROJECTS[0]); + + expect(await errorOf(await run("traces", "--limit", "99", "--json"))).toMatchObject({ + exit: 2, + code: "invalid_arguments", + exit_code: 2, + }); + expect(await errorOf(await run("traces", "--project", "nope", "--json"))).toMatchObject({ + exit: 2, + code: "project_not_found", + hint: "telemetry projects", + }); + expect(await errorOf(await run("trace", "0000", "--json"))).toMatchObject({ + exit: 1, + code: "trace_not_found", + message: "Trace not found or outside retention", + }); + + const closed = await closedServer(); + await writeFile( + join(home, sessionFile(closed)), + JSON.stringify({ server: closed, tokens: { access_token: TOKEN, token_type: "Bearer" } }), + ); + expect( + await errorOf( + await runWith( + { TELEMETRY_DEV_URL: closed }, + "traces", + "--project", + PROJECTS[0]!.id, + "--json", + ), + ), + ).toMatchObject({ exit: 1, code: "unreachable", retryable: true }); + + // After `--`, --json is a value, so the error stays the line a person reads. + const value = await run("use", "--", "--json"); + expect(value.code).toBe(2); + expect(value.stderr).toMatch(/^error No project "--json"/); + + // The same failures without --json keep the line a person reads. + const human = await run("trace", "0000"); + expect(human.code).toBe(1); + expect(human.stderr).toBe("error Trace not found or outside retention\n"); +}); + +test("the code telemetry.dev gives a tool error becomes the CLI's code, exit code, and hint", async () => { + const errorOf = async (...args: string[]) => { + const result = await run(...args, "--json"); + expect(result.stdout).toBe(""); + + return { exit: result.code, ...JSON.parse(result.stderr).error }; + }; + + // A default project or a project id goes to telemetry.dev without a lookup first. + const deleted = "0f0e0d0c-0b0a-4908-8706-050403020100"; + await signIn({ ...PROJECTS[0]!, id: deleted }); + expect(await errorOf("traces")).toMatchObject({ + exit: 2, + code: "project_not_found", + hint: "telemetry projects", + }); + expect(await errorOf("traces", "--project", deleted)).toMatchObject({ + code: "project_not_found", + }); + // People read the CLI's own words, not the text written for an agent. + expect((await run("traces", "--project", deleted)).stderr).toMatch( + /^error That project isn't in this workspace/, + ); + + await signIn(PROJECTS[0]); + expect(await errorOf("traces", "--cursor", "bm90LWEtY3Vyc29y")).toMatchObject({ + exit: 2, + code: "invalid_cursor", + retryable: false, + }); + + serverState.failProjects = true; + expect(await errorOf("projects")).toMatchObject({ + exit: 1, + code: "server_error", + message: "telemetry.dev could not complete this request. Try again shortly.", + retryable: true, + }); + + serverState.removedFromWorkspace = true; + expect(await errorOf("traces")).toMatchObject({ + exit: 4, + code: "access_denied", + hint: "telemetry login", + }); + + // A server from before the codes, and a code this CLI doesn't know, stay tool_error. + serverState.errorCodes = false; + expect(await errorOf("traces")).toMatchObject({ + exit: 1, + code: "tool_error", + message: "Not a member of this workspace", + retryable: false, + }); + + const { toolError } = await import("../src/errors.ts"); + expect( + toolError("Plan limit reached", { "dev.telemetry/error": { code: "plan_limit" } }), + ).toMatchObject({ + code: "tool_error", + message: "Plan limit reached", + }); + expect( + toolError("…", { "dev.telemetry/error": { code: "payloads_not_allowed", retryable: false } }), + ).toMatchObject({ code: "payloads_not_allowed", exitCode: 4, hint: "telemetry login" }); + expect( + toolError("Span not found", { + "dev.telemetry/error": { code: "span_not_found", retryable: false }, + }), + ).toMatchObject({ code: "span_not_found", exitCode: 1, message: "Span not found" }); +}); + +test("a request that times out inside fetch is a retryable timeout, not a failed request", async () => { + const { Api } = await import("../src/api.ts"); + + for (const code of [ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", + ]) { + vi.stubGlobal("fetch", () => + Promise.reject( + new TypeError("fetch failed", { cause: Object.assign(new Error(""), { code }) }), + ), + ); + + const api = new Api({ + server: fixture.url, + tokens: { access_token: TOKEN, token_type: "Bearer" }, + }); + + try { + await expect(api.call("list_projects")).rejects.toMatchObject({ + code: "timeout", + retryable: true, + exitCode: 1, + }); + } finally { + vi.unstubAllGlobals(); + await api.close(); + } + } +}); + const savedSession = async () => JSON.parse(await readFile(sessionPath(), "utf8")); test("a shell without a terminal signs in with --no-browser, then --finish, keeping the old sign-in until then and revoking it after", async () => { @@ -248,6 +528,109 @@ test("a shell without a terminal signs in with --no-browser, then --finish, keep expect(again.code).toBe(4); }); +test("login --json prints the link without opening a browser or waiting, and --finish --json prints the sign-in", async () => { + // A browser that could open: an `open` and `xdg-open` that record being run. + const bin = await mkdtemp(join(tmpdir(), "telemetry-cli-bin-")); + const opened = join(bin, "opened"); + for (const name of ["open", "xdg-open"]) { + await writeFile(join(bin, name), `#!/bin/sh\ntouch '${opened}'\n`, { mode: 0o755 }); + } + const browser = { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":0" }; + + try { + const started = await runWith(browser, "login", "--json"); + expect(started).toMatchObject({ code: 0, stderr: "" }); + + const pending = JSON.parse(started.stdout); + expect(pending).toEqual({ + status: "pending", + authorization_url: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+\/authorize\?/), + next: "telemetry login --finish '
'", + }); + + const approved = await fetch(pending.authorization_url, { redirect: "manual" }); + const finished = await run("login", "--finish", approved.headers.get("location")!, "--json"); + expect(finished).toMatchObject({ code: 0, stderr: "" }); + + const status = JSON.parse((await run("status", "--json")).stdout); + expect(status).toMatchObject({ signed_in: true, email: EMAIL }); + expect(JSON.parse(finished.stdout)).toEqual({ status: "signed_in", ...status }); + + const again = await run("login", "--finish", approved.headers.get("location")!, "--json"); + expect(again).toMatchObject({ code: 4, stdout: "" }); + expect(JSON.parse(again.stderr).error.code).toBe("no_pending_sign_in"); + + expect(JSON.parse((await run("logout", "--json")).stdout)).toEqual({ + was_signed_in: true, + revoked: true, + }); + expect(JSON.parse((await run("logout", "--json")).stdout)).toEqual({ + was_signed_in: false, + revoked: null, + }); + + // `open` runs detached, so by now, several commands later, it would have left its mark. + await expect(stat(opened)).rejects.toThrow(); + } finally { + await rm(bin, { recursive: true }); + } +}); + +test("login --json returns the link at once even when stdin is a terminal", async () => { + const { login } = await import("../src/auth.ts"); + const { loadSession } = await import("../src/session.ts"); + + vi.stubEnv("TELEMETRY_DEV_CONFIG_DIR", home); + const isTTY = Object.getOwnPropertyDescriptor(process.stdin, "isTTY"); + Object.defineProperty(process.stdin, "isTTY", { value: true, configurable: true }); + + try { + const result = await login({ + session: await loadSession(fixture.url), + browser: false, + defer: true, + onAuthorizationUrl: () => {}, + onWaiting: () => { + throw new Error("login waited for a person"); + }, + }); + + expect(result).toEqual({ status: "pending" }); + expect((await loadSession(fixture.url)).pending).toBeDefined(); + } finally { + if (isTTY) Object.defineProperty(process.stdin, "isTTY", isTTY); + else delete (process.stdin as { isTTY?: boolean }).isTTY; + vi.unstubAllEnvs(); + } +}); + +test("a sign-in whose project list fails still signs in, without a default project", async () => { + serverState.failProjects = true; + + for (const format of [[], ["--json"]]) { + const started = await run("login", "--json"); + const approved = await fetch(JSON.parse(started.stdout).authorization_url, { + redirect: "manual", + }); + const finished = await run("login", "--finish", approved.headers.get("location")!, ...format); + + expect(finished.code).toBe(0); + if (format.length) expect(JSON.parse(finished.stdout)).toMatchObject({ project: null }); + else expect(finished.stderr).toContain("Couldn't load your projects"); + } + + expect(JSON.parse((await run("status", "--json")).stdout)).toMatchObject({ + signed_in: true, + project: null, + }); +}); + +test("a sign-in that can't reach the server says so, like any other command", async () => { + const result = await runWith({ TELEMETRY_DEV_URL: await closedServer() }, "login", "--json"); + expect(result).toMatchObject({ code: 1, stdout: "" }); + expect(JSON.parse(result.stderr).error).toMatchObject({ code: "unreachable", retryable: true }); +}); + test("an expired access token is refreshed and the new one saved", async () => { await writeFile( sessionPath(), @@ -367,3 +750,113 @@ test("logout revokes the refresh token and keeps the client registration and oth expect(session.tokens).toBeUndefined(); expect(await readFile(join(home, "session.json"), "utf8")).toBe(production); }); + +const lockPath = () => `${sessionPath()}.lock`; +const markerFor = (path: string, token: string) => + `${path}.${createHash("sha256").update(token).digest("hex").slice(0, 16)}`; +const holdLock = async (path: string, token: string, pid: number, host = hostname()) => + writeFile(path, JSON.stringify({ token, pid, host })); +const backdate = async (path: string) => { + const minuteAgo = new Date(Date.now() - 60_000); + await utimes(path, minuteAgo, minuteAgo); +}; +const exitedPid = () => spawnSync(process.execPath, ["-e", ""]).pid; + +const record = (write: number) => + updateSession( + fixture.url, + (saved) => + ({ + ...saved, + writes: [...((saved as { writes?: number[] }).writes ?? []), write], + }) as Session, + ); +const recorded = async () => + ((await savedSession().catch(() => ({}))) as { writes?: number[] }).writes?.toSorted( + (a, b) => a - b, + ) ?? []; + +// updateSession reads the config directory from the environment on each call. +const inConfigDir = async (run: () => Promise) => { + const configured = process.env.TELEMETRY_DEV_CONFIG_DIR; + process.env.TELEMETRY_DEV_CONFIG_DIR = home; + + try { + await run(); + } finally { + if (configured === undefined) delete process.env.TELEMETRY_DEV_CONFIG_DIR; + else process.env.TELEMETRY_DEV_CONFIG_DIR = configured; + } +}; + +test("the sign-in lock waits for a running holder however long it holds, and for a command already removing an abandoned lock", async () => { + const running = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"]); + + try { + await inConfigDir(async () => { + await holdLock(lockPath(), "running", running.pid!); + await backdate(lockPath()); + + const waiting = Promise.all([0, 1, 2, 3].map(record)); + await sleep(500); + expect(await recorded()).toEqual([]); + + await rm(lockPath()); + await waiting; + expect(await recorded()).toEqual([0, 1, 2, 3]); + + // Another machine's lock is judged by its age, and this one is fresh. + await holdLock(lockPath(), "other-machine", process.ppid, "another-machine"); + const behindOtherMachine = record(5); + await sleep(500); + expect(await recorded()).toEqual([0, 1, 2, 3]); + + await rm(lockPath()); + await behindOtherMachine; + expect(await recorded()).toEqual([0, 1, 2, 3, 5]); + + await holdLock(lockPath(), "abandoned", exitedPid()); + const marker = markerFor(lockPath(), "abandoned"); + await holdLock(marker, "breaking", running.pid!); + + const behindBreaker = record(4); + await sleep(500); + expect(await recorded()).toEqual([0, 1, 2, 3, 5]); + + await rm(marker); + await behindBreaker; + expect(await recorded()).toEqual([0, 1, 2, 3, 4, 5]); + }); + } finally { + running.kill(); + } +}); + +test("the sign-in lock is taken over at once, without lost writes, when its holder or a command breaking it has exited", async () => { + await inConfigDir(async () => { + const takeOver = async (writes: number[]) => { + const started = Date.now(); + await Promise.all(writes.map(record)); + expect(Date.now() - started).toBeLessThan(2000); + }; + + await holdLock(lockPath(), "exited", exitedPid()); + await takeOver([0, 1, 2, 3, 4, 5, 6, 7]); + + await holdLock(lockPath(), "exited-again", exitedPid()); + await holdLock(markerFor(lockPath(), "exited-again"), "exited-breaker", exitedPid()); + await takeOver([8]); + + // Left by an earlier process that had this one's pid. + await holdLock(lockPath(), "same-pid", process.pid); + await takeOver([9]); + + // Another machine's lock can only be judged by its age. + await holdLock(lockPath(), "other-machine", process.ppid, "another-machine"); + await backdate(lockPath()); + await takeOver([10]); + + expect(await recorded()).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(await readdir(home)).toEqual([sessionFile(fixture.url)]); + }); +}); diff --git a/packages/cli/tests/fixture.ts b/packages/cli/tests/fixture.ts index f0dc89c..fdb9bfd 100644 --- a/packages/cli/tests/fixture.ts +++ b/packages/cli/tests/fixture.ts @@ -24,10 +24,31 @@ export const PROJECTS = [ const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000).toISOString(); -const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +export const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +export const QUIET_TRACE_ID = "9a0b7c6d5e4f30211a2b3c4d5e6f7081"; +export const PARALLEL_TRACE_ID = "3e8a1c5b7d9f40e2a6b4c8d0e2f4a6b8"; export const traces = () => ({ traces: [ + // The newest trace succeeded, so a suggestion to inspect a failure has to look past it. + { + id: QUIET_TRACE_ID, + name: "support.answer", + status: "success", + error_type: null, + started_at: minutesAgo(1), + duration_ms: 1870, + environment: "production", + service_name: "support-api", + session_id: "conv_8f2a91", + user_id: "user_4821", + models: ["gpt-5"], + span_count: 3, + error_span_count: 0, + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }, { id: TRACE_ID, name: "support.answer", @@ -46,24 +67,6 @@ export const traces = () => ({ output_tokens: 373, cost_usd: 0.01234, }, - { - id: "9a0b7c6d5e4f30211a2b3c4d5e6f7081", - name: "support.answer", - status: "success", - error_type: null, - started_at: minutesAgo(9), - duration_ms: 1870, - environment: "production", - service_name: "support-api", - session_id: "conv_8f2a91", - user_id: "user_4821", - models: ["gpt-5"], - span_count: 5, - error_span_count: 0, - input_tokens: 1934, - output_tokens: 211, - cost_usd: 0.00718, - }, { id: "1f2e3d4c5b6a79880a9b8c7d6e5f4a3b", // Captured text can carry terminal escapes; the CLI must never pass them through. @@ -184,8 +187,116 @@ export const traceDetail = () => ({ next_cursor: null, }); +/** A run that succeeded with each operation once: nothing to summarize above the tree. */ +const quietTraceDetail = () => ({ + trace: { + id: QUIET_TRACE_ID, + name: "support.answer", + status: "success", + error_type: null, + started_at: minutesAgo(1), + duration_ms: 1870, + environment: "production", + service_name: "support-api", + framework: "ai-sdk", + session_id: "conv_8f2a91", + user_id: "user_4821", + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }, + span_count: 3, + log_count: 0, + spans: [ + span("q1", null, "invoke_agent support-bot", 0, 1870, { + agent_name: "support-bot", + operation: "invoke_agent", + }), + span("q2", "q1", "execute_tool lookup_order", 30, 240, { tool_name: "lookup_order" }), + span("q3", "q1", "chat gpt-5", 290, 1560, { + model: "gpt-5", + provider: "openai", + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }), + ], + logs: [], + next_cursor: null, +}); + +/** + * An agent that runs three 1 s searches at once, so they cover 1.1 s of a 3 s trace, then + * fails two levels down, in the request a tool made. The failure is marked on the tool + * and the agent too, as instrumentation does when an exception passes through them. + */ +const parallelTraceDetail = () => ({ + trace: { + id: PARALLEL_TRACE_ID, + name: "research.report", + status: "error", + error_type: "TimeoutError", + started_at: minutesAgo(2), + duration_ms: 3000, + environment: "production", + service_name: "research-api", + framework: "ai-sdk", + session_id: null, + user_id: null, + input_tokens: null, + output_tokens: null, + cost_usd: null, + }, + span_count: 6, + log_count: 0, + spans: [ + span("r1", null, "invoke_agent researcher", 0, 3000, { + agent_name: "researcher", + operation: "invoke_agent", + status: "error", + error_type: "TimeoutError", + error: "TimeoutError: tool fetch_page failed", + }), + span("s1", "r1", "execute_tool web_search", 100, 1000, { tool_name: "web_search" }), + span("s2", "r1", "execute_tool web_search", 150, 1000, { tool_name: "web_search" }), + span("s3", "r1", "execute_tool web_search", 200, 1000, { tool_name: "web_search" }), + span("t1", "r1", "execute_tool fetch_page", 1300, 800, { + tool_name: "fetch_page", + status: "error", + error_type: "TimeoutError", + error: "TimeoutError: Request timed out after 700 ms", + }), + span("t2", "t1", "GET /page", 1350, 700, { + status: "error", + error_type: "TimeoutError", + error: "Request timed out after 700 ms\n at fetchPage (tools.ts:41:11)", + }), + ], + logs: [], + next_cursor: null, +}); + +const TRACE_DETAILS: Record unknown> = { + [TRACE_ID]: traceDetail, + [QUIET_TRACE_ID]: quietTraceDetail, + [PARALLEL_TRACE_ID]: parallelTraceDetail, +}; + export const sessions = () => ({ sessions: [ + { + // Session ids come from the user's app and can hold spaces and quotes. + id: "Ada's thread 42", + last_active_at: minutesAgo(1), + user_id: "user_4821", + agent_name: "support-bot", + models: ["gpt-5"], + trace_count: 2, + error_trace_count: 0, + input_tokens: 3120, + output_tokens: 402, + cost_usd: 0.0114, + }, { id: "conv_8f2a91", last_active_at: minutesAgo(2), @@ -322,22 +433,62 @@ const result = (value: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(value) }], }); +/** A tool error as telemetry.dev returns it: text for the agent, a code in `_meta`. */ +class ToolFailure { + constructor( + readonly text: string, + readonly code: string, + readonly retryable = false, + ) {} +} + +const SERVER_ERROR = "telemetry.dev could not complete this request. Try again shortly."; + const server = () => { const mcp = new McpServer({ name: "telemetry.dev fixture", version: "1.0.0" }); const tool = (name: string, handler: (args: Record) => unknown) => mcp.registerTool(name, { inputSchema: loose }, (args: Record) => { calls.push({ tool: name, args }); - const value = handler(args); - - return typeof value === "string" - ? { isError: true, content: [{ type: "text" as const, text: value }] } - : result(value); + const value = serverState.removedFromWorkspace + ? new ToolFailure("Not a member of this workspace", "access_denied") + : handler(args); + + if (!(value instanceof ToolFailure)) return result(value); + + return { + isError: true, + content: [{ type: "text" as const, text: value.text }], + ...(serverState.errorCodes + ? { _meta: { "dev.telemetry/error": { code: value.code, retryable: value.retryable } } } + : {}), + }; }); - tool("list_projects", () => ({ organization_id: "org_fixture", projects: PROJECTS })); - tool("search_traces", () => traces()); + tool("list_projects", () => + serverState.failProjects + ? new ToolFailure(SERVER_ERROR, "server_error", true) + : { organization_id: "org_fixture", projects: PROJECTS }, + ); + tool("search_traces", (args) => { + if (!PROJECTS.some((project) => project.id === args.project_id)) { + return new ToolFailure("Project not found", "project_not_found"); + } + + if (args.cursor !== undefined && args.cursor !== traces().next_cursor) { + return new ToolFailure( + "cursor is not a next_cursor value from search_traces. Omit it to start over.", + "invalid_cursor", + ); + } + + return traces(); + }); tool("search_sessions", () => sessions()); - tool("get_trace", (args) => (args.trace_id === TRACE_ID ? traceDetail() : "Trace not found.")); + tool("get_trace", (args) => + Object.hasOwn(TRACE_DETAILS, String(args.trace_id)) + ? TRACE_DETAILS[String(args.trace_id)]!() + : new ToolFailure("Trace not found or outside retention", "trace_not_found"), + ); tool("list_logs", () => logs()); tool("get_usage_summary", () => usage()); tool("get_span_payload", (args) => { @@ -361,6 +512,16 @@ const server = () => { export const calls: { tool: string; args: Record }[] = []; +/** How the fixture server misbehaves; tests reset it before each run. */ +export const serverState = { + /** list_projects fails as a server failure would. */ + failProjects: false, + /** Every tool refuses, as for a user removed from the workspace. */ + removedFromWorkspace: false, + /** Tool errors carry a code in `_meta`; false acts as a server from before codes. */ + errorCodes: true, +}; + /** * Runs before the token endpoint answers a refresh, to act as another CLI process or a * server that rotates tokens. "accept" or "refuse" overrides the usual check. diff --git a/packages/cli/vite.config.ts b/packages/cli/vite.config.ts index cf8ebff..6a497a8 100644 --- a/packages/cli/vite.config.ts +++ b/packages/cli/vite.config.ts @@ -26,4 +26,9 @@ export default defineConfig({ }, }, fmt: {}, + test: { + // Each test starts the CLI as a process, several times; one start takes up to 700ms + // on a CI runner, which pushes the longer tests past the 5s default. + testTimeout: 30_000, + }, }); diff --git a/packages/cursor/CHANGELOG.md b/packages/cursor/CHANGELOG.md index 1f8628a..f03f5d4 100644 --- a/packages/cursor/CHANGELOG.md +++ b/packages/cursor/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/cursor-v0.2.1...cursor-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/cursor-v0.2.0...cursor-v0.2.1) (2026-10-30) diff --git a/packages/cursor/package.json b/packages/cursor/package.json index 6c6a7c6..7f57bdd 100644 --- a/packages/cursor/package.json +++ b/packages/cursor/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/cursor", - "version": "0.2.1", + "version": "0.2.2", "description": "Cursor coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/eve/CHANGELOG.md b/packages/eve/CHANGELOG.md index 4aaac1a..4b713d4 100644 --- a/packages/eve/CHANGELOG.md +++ b/packages/eve/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/eve-v0.2.1...eve-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/eve-v0.2.0...eve-v0.2.1) (2026-10-30) diff --git a/packages/eve/package.json b/packages/eve/package.json index 3401f18..dcf082e 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/eve", - "version": "0.2.1", + "version": "0.2.2", "description": "Eve (Vercel agent framework) telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/google-genai/CHANGELOG.md b/packages/google-genai/CHANGELOG.md index a44107e..4d8264c 100644 --- a/packages/google-genai/CHANGELOG.md +++ b/packages/google-genai/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/google-genai-v0.2.1...google-genai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/google-genai-v0.2.0...google-genai-v0.2.1) (2026-10-30) diff --git a/packages/google-genai/package.json b/packages/google-genai/package.json index 6d9fe44..9fb7e9b 100644 --- a/packages/google-genai/package.json +++ b/packages/google-genai/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/google-genai", - "version": "0.2.1", + "version": "0.2.2", "description": "Google GenAI (Gemini) SDK instrumentation for telemetry.dev: generation and embedding spans for @google/genai.", "keywords": [ "gemini", diff --git a/packages/google-genai/src/index.ts b/packages/google-genai/src/index.ts index 96ffef5..341d5e8 100644 --- a/packages/google-genai/src/index.ts +++ b/packages/google-genai/src/index.ts @@ -567,7 +567,11 @@ function responseAttributes(response: UnknownRecord, metadataContent: boolean) { function generateRequestFields(params: UnknownRecord): RequestMapping { const model = readString(params.model); const config = asRecord(params.config); - const attributes = requestAttributesFromConfig(config); + + const attributes = { + ...requestAttributesFromConfig(config), + ...promptAttributes(params.contents), + }; const input = captureEnabled("input") ? normalizeContentInput(params.contents) : undefined; const systemInstructions = captureEnabled("input") @@ -940,6 +944,16 @@ function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(messages: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(messages) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(request: RequestMapping, provider: string): SpanHandle { const fields = { ...request.fields, provider }; diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index d1d291a..19d7fbf 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -10,7 +10,8 @@ import { type PushMetricExporter, type ResourceMetrics, } from "@opentelemetry/sdk-metrics"; -import { flush, init, shutdown } from "@telemetry-dev/sdk"; +import { type ChatMessage, flush, getPrompt, init, shutdown } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { FunctionCallingConfigMode, GoogleGenAI, @@ -234,6 +235,86 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { contents: "changed in place", captureInput: true, olderCore: false }, + { contents: "changed in place", captureInput: false, olderCore: false }, + { contents: "new", captureInput: true, olderCore: false }, + { contents: "changed in place", captureInput: true, olderCore: true }, +])( + "generateContent outside withPrompt records the prompt of $contents contents, input capture $captureInput, older core $olderCore", + async ({ contents: kind, captureInput, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: "https://prompts.google-genai.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "gcp.gemini", name: "gemini-2.5-flash" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const parts = [{ text: user!.content }]; + // Gemini takes parts: a message changed in place keeps its tag, a new one has none. + const content = kind === "new" ? { role: "user", parts } : Object.assign(user!, { parts }); + const spans = setupSpans({ captureInput }); + const fake = createFakeFetch( + jsonResponse({ + candidates: [ + { content: { role: "model", parts: [{ text: "Soon." }] }, finishReason: "STOP" }, + ], + }), + ); + vi.stubGlobal("fetch", fake.fetch); + + await clientWith().models.generateContent({ + model: "gemini-2.5-flash", + contents: [content], + config: { temperature: 0.2, systemInstruction: system!.content }, + }); + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.request.temperature"]).toBe(0.2); + expect(fake.requests[0]!.body).toMatchObject({ + contents: [{ role: "user", parts: [{ text: "Where is order #4?" }] }], + }); + + if (olderCore || kind === "new") { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + expect(attributes["td.prompt.variables"]).toBe( + captureInput ? '{"team":"billing","order":"#4"}' : undefined, + ); + }, +); + test("generateContent maps request, response, usage, finish reason, provider, and sampling attributes", async () => { const spans = setupSpans(); diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index cdd7d55..c04a5a7 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/mcp-v0.2.1...mcp-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/mcp-v0.2.0...mcp-v0.2.1) (2026-10-30) diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 0a4b610..108c75b 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/mcp", - "version": "0.2.1", + "version": "0.2.2", "description": "Model Context Protocol SDK instrumentation for telemetry.dev: instruments MCP v2 transports and emits OpenTelemetry spans.", "keywords": [ "mcp", diff --git a/packages/omp/CHANGELOG.md b/packages/omp/CHANGELOG.md index 003d763..5ea2571 100644 --- a/packages/omp/CHANGELOG.md +++ b/packages/omp/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/omp-v0.2.1...omp-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/omp-v0.2.0...omp-v0.2.1) (2026-10-30) diff --git a/packages/omp/package.json b/packages/omp/package.json index 8ac9f27..792b48a 100644 --- a/packages/omp/package.json +++ b/packages/omp/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/omp", - "version": "0.2.1", + "version": "0.2.2", "description": "Oh My Pi (omp) coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/openai/CHANGELOG.md b/packages/openai/CHANGELOG.md index 842e048..a0ca87e 100644 --- a/packages/openai/CHANGELOG.md +++ b/packages/openai/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/openai-v0.2.1...openai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/openai-v0.2.0...openai-v0.2.1) (2026-10-30) diff --git a/packages/openai/package.json b/packages/openai/package.json index c338c47..d1ed5eb 100644 --- a/packages/openai/package.json +++ b/packages/openai/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/openai", - "version": "0.2.1", + "version": "0.2.2", "description": "OpenAI SDK instrumentation for telemetry.dev: generation and embedding spans for the official openai Node SDK.", "keywords": [ "genai", diff --git a/packages/openai/src/index.ts b/packages/openai/src/index.ts index 3512476..9b86e73 100644 --- a/packages/openai/src/index.ts +++ b/packages/openai/src/index.ts @@ -156,6 +156,7 @@ function stopSequences(value: T): string[] | undefined { function chatRequest(body: JsonRecord): RequestMapping { const model = readString(body.model); const capture = captureEnabled("input") ? sanitizeResponsesCapture(body.messages) : undefined; + const prompt = promptAttributes(body.messages); return { name: `chat ${model ?? "unknown"}`, @@ -170,7 +171,9 @@ function chatRequest(body: JsonRecord): RequestMapping { seed: readNumber(body.seed), frequencyPenalty: readNumber(body.frequency_penalty), presencePenalty: readNumber(body.presence_penalty), - attributes: capture?.truncated ? { "telemetry.dev.capture.truncated": true } : undefined, + attributes: capture?.truncated + ? { ...prompt, "telemetry.dev.capture.truncated": true } + : prompt, }, }; } @@ -243,6 +246,7 @@ function chatResponse(response: T): SpanFields { function responsesRequest(body: JsonRecord): RequestMapping { const model = readString(body.model); const capture = captureEnabled("input") ? sanitizeResponsesCapture(body.input) : undefined; + const prompt = promptAttributes(body.input); return { name: `chat ${model ?? "unknown"}`, @@ -254,7 +258,9 @@ function responsesRequest(body: JsonRecord): RequestMapping { temperature: readNumber(body.temperature), topP: readNumber(body.top_p), maxTokens: readNumber(body.max_output_tokens), - attributes: capture?.truncated ? { "telemetry.dev.capture.truncated": true } : undefined, + attributes: capture?.truncated + ? { ...prompt, "telemetry.dev.capture.truncated": true } + : prompt, }, }; } @@ -1626,6 +1632,16 @@ function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(messages: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(messages) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(name: string, fields: StartSpanOptions): SpanHandle { const cut = diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 4467aa3..099ec37 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -8,7 +8,14 @@ import { InMemoryMetricExporter, type PushMetricExporter, } from "@opentelemetry/sdk-metrics"; -import { flush, init, shutdown, type TelemetryOptions } from "@telemetry-dev/sdk"; +import { + type ChatMessage, + flush, + getPrompt, + init, + shutdown, + type TelemetryOptions, +} from "@telemetry-dev/sdk"; import * as telemetrySdk from "@telemetry-dev/sdk"; import { EventEmitter } from "node:events"; import { createRequire } from "node:module"; @@ -189,7 +196,7 @@ function setupSpans( metricExporter?: PushMetricExporter, options: Pick< TelemetryOptions, - "captureOutput" | "mask" | "maxAttributeLength" | "onError" | "sampler" + "captureInput" | "captureOutput" | "mask" | "maxAttributeLength" | "onError" | "sampler" > = {}, ): InMemorySpanExporter { const spanExporter = new InMemorySpanExporter(); @@ -2887,6 +2894,102 @@ test("chat streams preserve caller-requested usage chunks", async () => { expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(7); }); +test.each([ + { api: "chat completions", messages: "compiled", captureInput: false, olderCore: false }, + { api: "responses", messages: "compiled", captureInput: true, olderCore: false }, + { api: "chat completions", messages: "copied", captureInput: true, olderCore: false }, + { api: "chat completions", messages: "compiled", captureInput: true, olderCore: true }, +])( + "$api outside withPrompt records the prompt of $messages messages, input capture $captureInput, older core $olderCore", + async ({ api, messages: kind, captureInput, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: "https://prompts.openai.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "openai", name: "gpt-4o" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const messages = kind === "compiled" ? compiled : compiled.map((message) => ({ ...message })); + const spans = setupSpans(undefined, { captureInput }); + const client = clientWith( + createFakeFetch( + jsonResponse( + api === "responses" + ? { + id: "resp_1", + object: "response", + status: "completed", + model: "gpt-4.1", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + } + : { + id: "chatcmpl_1", + object: "chat.completion", + created: 1, + model: "gpt-4o", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Soon." }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ), + ).fetch, + ); + + if (api === "responses") { + await client.responses.create({ model: "gpt-4.1", input: messages }); + } else { + await client.chat.completions.create({ model: "gpt-4o", messages }); + } + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.input.messages"] !== undefined).toBe(captureInput); + + if (olderCore || kind === "copied") { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + expect(attributes["td.prompt.variables"]).toBe( + captureInput ? '{"team":"billing","order":"#4"}' : undefined, + ); + }, +); + test("responses create maps instructions to system instructions", async () => { const spans = setupSpans(); diff --git a/packages/opencode/CHANGELOG.md b/packages/opencode/CHANGELOG.md index e024f86..919db51 100644 --- a/packages/opencode/CHANGELOG.md +++ b/packages/opencode/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/opencode-v0.2.1...opencode-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/opencode-v0.2.0...opencode-v0.2.1) (2026-10-30) diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 1fe8aec..09c1043 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/opencode", - "version": "0.2.1", + "version": "0.2.2", "description": "opencode coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/openrouter/CHANGELOG.md b/packages/openrouter/CHANGELOG.md index 0b8ab81..6242db5 100644 --- a/packages/openrouter/CHANGELOG.md +++ b/packages/openrouter/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/openrouter-v0.2.1...openrouter-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/openrouter-v0.2.0...openrouter-v0.2.1) (2026-10-30) diff --git a/packages/openrouter/package.json b/packages/openrouter/package.json index ea040e2..3c75975 100644 --- a/packages/openrouter/package.json +++ b/packages/openrouter/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/openrouter", - "version": "0.2.1", + "version": "0.2.2", "description": "OpenRouter SDK instrumentation for telemetry.dev: generation and embedding spans for the official @openrouter/sdk package.", "keywords": [ "genai", diff --git a/packages/openrouter/src/index.ts b/packages/openrouter/src/index.ts index bfb0076..74e2c76 100644 --- a/packages/openrouter/src/index.ts +++ b/packages/openrouter/src/index.ts @@ -4,6 +4,7 @@ import { type SpanHandle, type StartSpanOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { activeContext } from "@telemetry-dev/otel"; import { Chat } from "@openrouter/sdk/sdk/chat.js"; import { Embeddings } from "@openrouter/sdk/sdk/embeddings.js"; @@ -384,6 +385,16 @@ function usageCost(usage: T): number | undefined { return cost ?? upstreamCost; } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(messages: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(messages) : undefined; +} + function chatRequest(request: JsonRecord): RequestMapping { const body = asRecord(request.chatRequest) ?? {}; const model = readString(body.model); @@ -403,6 +414,7 @@ function chatRequest(request: JsonRecord): RequestMapping { seed: readNumber(body.seed), frequencyPenalty: readNumber(body.frequencyPenalty), presencePenalty: readNumber(body.presencePenalty), + attributes: promptAttributes(body.messages), }, }; } @@ -472,6 +484,7 @@ function responsesRequest(request: JsonRecord): RequestMapping { maxTokens: readNumber(body.maxOutputTokens), frequencyPenalty: readNumber(body.frequencyPenalty), presencePenalty: readNumber(body.presencePenalty), + attributes: promptAttributes(body.input), }, }; } diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index 2dc9baf..0d50a1b 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -5,13 +5,16 @@ import { Embeddings } from "@openrouter/sdk/sdk/embeddings.js"; import { EventStream } from "@openrouter/sdk/lib/event-streams.js"; import { Responses } from "@openrouter/sdk/sdk/responses.js"; import { + type ChatMessage, flush, + getPrompt, init, type MaskFn, shutdown, startActiveSpan, type TelemetryOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { afterEach, expect, test, vi } from "vitest"; import { instrumentOpenRouter, uninstrumentOpenRouter, wrapOpenRouter } from "../src/index.ts"; @@ -308,6 +311,82 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { api: "chat", messages: "compiled", olderCore: false }, + { api: "streamed responses", messages: "compiled", olderCore: false }, + { api: "chat", messages: "copied", olderCore: false }, + { api: "chat", messages: "compiled", olderCore: true }, +])( + "$api outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ api, messages: kind, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: "https://prompts.openrouter.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "openrouter", name: "openai/gpt-4o" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const messages = kind === "compiled" ? compiled : compiled.map((message) => ({ ...message })); + const spans = setupSpans(); + + if (api === "chat") { + const client = clientWith(createFakeFetcher(jsonResponse(chatBody("chat_1"))).fetcher); + await client.chat.send({ chatRequest: { model: "openai/gpt-4o", messages } }); + } else { + const client = clientWith( + createFakeFetcher( + sseResponse([{ type: "response.completed", response: responsesBody("resp_1") }]), + ).fetcher, + ); + const stream = await client.responses.send({ + responsesRequest: { model: "openai/gpt-4o", input: messages, stream: true }, + }); + + if (!(stream instanceof ReadableStream)) throw new Error("expected a readable stream"); + await collectStream(stream); + } + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.request.model"]).toBe("openai/gpt-4o"); + + if (olderCore || kind === "copied") { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }); + }, +); + test("chat maps request, response, usage, primary cost, provider, and sampling fields", async () => { const spans = setupSpans(); diff --git a/packages/pi/CHANGELOG.md b/packages/pi/CHANGELOG.md index 85e36f7..8f119de 100644 --- a/packages/pi/CHANGELOG.md +++ b/packages/pi/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/pi-v0.2.1...pi-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/pi-v0.2.0...pi-v0.2.1) (2026-10-30) diff --git a/packages/pi/package.json b/packages/pi/package.json index 028a04f..dd45891 100644 --- a/packages/pi/package.json +++ b/packages/pi/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/pi", - "version": "0.2.1", + "version": "0.2.2", "description": "Pi coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 468e95e..fea0a68 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/sdk-v0.2.1...sdk-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep compiled variables for 5 minutes, not two turns ([1b09916](https://github.com/telemetry-dev/telemetry/commit/1b099167bbd8acaf5008848ad9ba2a2ab8e35b0d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of recent compiles only ([3bc6f53](https://github.com/telemetry-dev/telemetry/commit/3bc6f53a0afb0ad8b9fe19f8eaf7e5d812a93b0a)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of the last two chat turns ([f6aa1c1](https://github.com/telemetry-dev/telemetry/commit/f6aa1c159742123600d6499653916c4dbf7b2ab7)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** measure the variables time of compiled messages in elapsed time ([b2a0e09](https://github.com/telemetry-dev/telemetry/commit/b2a0e09e3a2fee5aa8c5d6951a0bac9dac019cfa)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** name the newest compile, not the last tagged message ([a6b98fe](https://github.com/telemetry-dev/telemetry/commit/a6b98fe916d5ed4b37bddd7972b07507182f2d2d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/sdk-v0.2.0...sdk-v0.2.1) (2026-10-30) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef1..81f726a 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -161,8 +161,8 @@ const pinned = await getPrompt("support/reply", { version: 13 }); its prompt. - getPrompt uses the API key and base URL of an enabled `init()`, or `TELEMETRY_DEV_API_KEY` and `TELEMETRY_DEV_BASE_URL`, or its own `apiKey`, `baseUrl`, and `fetch` options. It reads the same - host that telemetry goes to, `https://api.telemetry.dev` by default. The key needs the read - scope for prompts. + host that telemetry goes to, `https://api.telemetry.dev` by default. The key must have Read + prompts in its Access on the API keys page (the `prompts:read` scope). - When a label sends part of the traffic to a second version, each session keeps one version: the choice uses the session id of `propagateAttributes` (or of `sessionMode: "process"`). Without a session id, each call chooses at random. @@ -179,8 +179,158 @@ compiles its own. Model calls are the spans of type `generation` (operations `ch `text_completion`, and `generate_content`), including the spans of the provider integrations that use this SDK: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, and OpenRouter. Spans from the AI SDK and TanStack AI integrations, and from other OpenTelemetry tracers, do not get the attributes. -A model call outside `withPrompt` does not get the attributes, even when it sends messages from -`compile()`; with input capture on, the call still records those messages as its input. + +Outside `withPrompt`, those provider integrations record the prompt of the message objects that +`compile()` returned. `compile()` of a chat prompt tags each message that it makes with its prompt +and a copy of its variables and of each list in them, in a non-enumerable symbol property: the keys +and the JSON of the message do not change. The objects in a list are not copied, so a change to one +of them after `compile()` changes the variables that a call records. A chat call that sends tagged +messages gets `td.prompt.name`, `td.prompt.version`, and `td.prompt.label` of the newest `compile()` +among them, also when input capture is off, and `td.prompt.variables` when it is on. The SDK looks +at the first 1000 and the last 1000 messages of the call. A message from a placeholder variable +keeps the tag of its own, older `compile()`, so it does not win over the `compile()` that put it in +the list. A copy of a message (a spread, `structuredClone`, or JSON) and a new message with the same +text have no tag, so a call that sends only such messages gets no `td.prompt.*` attributes; with +input capture on, the call still records them as its input. Bedrock Converse and Google GenAI take +another message shape: change the compiled messages in place, or use `withPrompt`. A text prompt +compiles to a string, which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The +integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse, +Google GenAI `generateContent`, and OpenRouter chat and responses, but not for Bedrock InvokeModel +(its body is bytes), embeddings, images, audio, video, or OpenAI Realtime. + +With input capture on in the client when `compile()` runs, a tagged message keeps the variables of +its `compile()` for 5 minutes. A call keeps them for 5 more minutes when the newest `compile()` of +the call is that one. After that time, a call records the prompt without `td.prompt.variables`, +and the SDK removes the variables from the tag, also when you keep the message. Other compiles do +not change this time. Thus a chat history keeps the variables of the compiles of its last 5 +minutes, and a guard prompt or a queue before the call does not remove them. A message that you +compile one time and send in many calls keeps its variables while a call names its `compile()` at +least one time in 5 minutes. A call that sends only messages of earlier turns records the prompt of +the newest `compile()` among them, and its variables when they are less than 5 minutes old. With +input capture off, or in a runtime without `WeakRef` (Cloudflare Workers with a compatibility date +before 2025-05-05), the tag keeps no variables. + +For your own spans, `promptAttributesFor(messages)` gives the attributes to pass as `attributes`. +It gives nothing inside `withPrompt`, and a span with `captureInput: false` drops +`td.prompt.variables`. A model call whose parent is in `withPrompt` gets the prompt of the +`withPrompt` and none of the `td.prompt.*` attributes that you pass, at start or in `update`, +`end`, or `updateActiveSpan`, also when its `parent` option leads into a `withPrompt` that the call +is not in. + +## Experiments + +`runExperiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init({ environment: "experiment" }); + +const result = await runExperiment({ + name: "reply-v13-gpt-4o", + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness", "helpfulness"], + scorers: { short: (output) => String(output).length < 600 }, + minScores: { correctness: 0.9 }, +}); +``` + +- runExperiment reads the newest version of the dataset, or `datasetVersion`, 100 items at a time. + With `split`, it runs only the items of that split. Then it makes the experiment, with a name + that telemetry.dev picks when you give no `name`. +- `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, + `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A value that JSON + cannot hold, such as an object that holds itself, gives no output. A task that throws fails only + its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. With `runsPerItem` (default 1, + at most 5), each item runs more than one time. Each run gets its own copy of the item, so a task + that changes its input does not change the next run. The pass rate and the mean of each score + use all runs. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and + TanStack AI integrations do not get these attributes. +- Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out + the `experiment` environment by default. In a different environment, runExperiment logs a + warning. Without a client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `true` or `false` for a pass or a fail, a number, a label, or an object with + `value`, `label`, and `reason`. A scorer that throws, or returns a score that is not valid, + gives no score for that item and reports a `scorer_failed` error through `onError`. A scorer + cannot have the name of an evaluator of the run, or the name `user_feedback`. +- runExperiment sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns the result: `status`, + `checkStatus`, item counts, `scores` with the value, the baseline value, the change, and the + pass mark of each score, and the `url` of the experiment page. +- `minScores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request rejects with an `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then + ends the experiment with the item runs that it got, or fails it when it got none. +- The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same + API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` + options. With `environment: "experiment"`, the key of the telemetry client must be for all + environments or for `experiment`: a key for another environment cannot send the traces of the + items, and the items then have no trace and no cost. + +The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +and stay with the experiment, because they are its result. `captureOutput`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci: true`, runExperiment rejects when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives runExperiment the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```ts +import { score, startActiveSpan } from "@telemetry-dev/sdk"; + +// When the app replies, keep the trace id of the reply with the message: +const reply = await startActiveSpan("reply", async (span) => ({ + text: await replyToCustomer(message), + traceId: span.traceId, +})); + +// Later, when the user clicks thumbs down: +await score({ name: "user_feedback", traceId: reply.traceId, value: 0, comment: "Wrong amount" }); +``` + +- Give `traceId`, `sessionId`, or the two, and a `spanId` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, score never rejects. It resolves with the id of the score, or with + `undefined` after it reports a `ScoreError` through `onError`. It checks the score before it + sends it (`invalid_score`). The trace id of a span from a client that is not initialized is all + zeros, and score reports it as `invalid_score`. +- score makes one request, with no retry, and waits 10 seconds at most. The key needs the + `scores:write` scope. A browser must not hold the key, so call score from your server. ## API @@ -203,6 +353,9 @@ A model call outside `withPrompt` does not get the attributes, even when it send `gen_ai.conversation.id`, and `td.metadata.*` on every span and log record in scope. - `getPrompt(name, options?)` — fetch a prompt by label or version; see [Prompts](#prompts). - `withPrompt(prompt, fn)` — record the prompt on each model call in `fn`. +- `runExperiment(options)` — run your code on a dataset as an experiment; see + [Experiments](#experiments). +- `score(options)` — record a score on a trace, a span, or a session; see [Scores](#scores). - `log(message, { level, eventName, attributes }?)` — log record correlated to the active trace. - `getTraceparent()` — W3C traceparent of the active span (pass to other services; accept it via `startSpan(name, { parent })`). diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 0248c0d..2158077 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/sdk", - "version": "0.2.1", + "version": "0.2.2", "description": "Pure TypeScript SDK for telemetry.dev: LLM traces, logs, and GenAI metrics on OpenTelemetry, for Node and edge runtimes.", "keywords": [ "genai", diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts new file mode 100644 index 0000000..dd7fac6 --- /dev/null +++ b/packages/sdk/src/api.ts @@ -0,0 +1,203 @@ +import { DEFAULT_BASE_URL, resolveEnv } from "@telemetry-dev/otel"; + +import { currentClient } from "./client.ts"; + +/** Where the calls to the telemetry.dev API go, and with which key. */ +export interface ApiOptions { + /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + apiKey?: string; + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ + baseUrl?: string; + /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ + fetch?: typeof fetch; +} + +export interface Api { + apiKey: string | undefined; + baseUrl: string; + sdkName: string; + fetchImpl: typeof fetch; + onError: ((cause: Error) => void) | undefined; +} + +/** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */ +export type ApiErrorType = new ( + code: string, + message: string, + status?: number, + options?: { cause?: unknown }, +) => E; + +interface RequestOptions { + body?: unknown; + query?: Record; + timeoutMs: number; + /** + * The server takes a repeat of this request at most once, so a retry after a timeout, a network + * error, or a 5xx is safe. Other requests try again only after a 429, which the rate limit + * answers before the server does any work. A 503 can come after the work committed. + */ + idempotent: boolean; + /** Default 3. */ + attempts?: number; +} + +type Attempt = + | { ok: true; body: unknown } + | { ok: false; error: Error; status?: number; retryAfterMs?: number }; + +/** A longer Retry-After waits only this long: the API counts requests by the minute. */ +const MAX_RETRY_AFTER_MS = 60_000; +/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ +const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; + +/** The key, base URL, and fetch of a call: its options, then the initialized client, then the env. */ +export function resolveApi(options: ApiOptions): Api { + const config = currentClient().core?.config; + const env = resolveEnv(); + + return { + apiKey: options.apiKey ?? config?.apiKey ?? env.TELEMETRY_DEV_API_KEY, + // An empty value counts as unset, as in the Python SDK. + baseUrl: ( + options.baseUrl || + config?.baseUrl || + env.TELEMETRY_DEV_BASE_URL || + DEFAULT_BASE_URL + ).replace(/\/+$/, ""), + sdkName: config?.sdkName ?? "@telemetry-dev/sdk", + fetchImpl: options.fetch ?? config?.fetchImpl ?? globalThis.fetch, + onError: config?.onError, + }; +} + +/** + * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response + * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_`. + * A failure that `options.idempotent` allows tries again, after the Retry-After of the response + * or after 1 and then 2 seconds. + */ +export async function request( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + path: string, + options: RequestOptions, +): Promise { + const url = new URL(`${api.baseUrl}${path}`); + + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) url.searchParams.set(key, value); + } + + // Outside send(), so that a value JSON cannot hold fails as itself, not as a network error. + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const attempts = options.attempts ?? 3; + + for (let attempt = 1; ; attempt++) { + const result = await send(api, ErrorType, method, url, body, options.timeoutMs); + + if (result.ok) return result.body; + + if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error; + + await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS)); + } +} + +/** The wait that the Retry-After of a 429 or a 503 asks for, in seconds or as an HTTP date. */ +export function retryAfterMs(res: Response): number | undefined { + if (res.status !== 429 && res.status !== 503) return undefined; + + const value = res.headers.get("retry-after")?.trim(); + + if (!value) return undefined; + + let ms: number; + + if (/^\d+$/.test(value)) ms = Number(value) * 1000; + else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); + else return undefined; + + return ms > 0 ? ms : undefined; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function send( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + url: URL, + body: string | undefined, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + // A plain call, not a method call: Workers and browsers reject a fetch whose `this` is not the + // global object. + const { fetchImpl } = api; + + try { + const res = await fetchImpl(url, { + method, + headers: { + authorization: `Bearer ${api.apiKey}`, + "x-telemetry-dev-sdk": api.sdkName, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body, + signal: controller.signal, + // A redirect to another host must not get the API key, so a 3xx fails below like any status + // that is not OK. Cloudflare Workers reject redirect: "error". + redirect: "manual", + }); + + const json: unknown = await res.json().catch(() => undefined); + + if (res.ok) return { ok: true, body: json }; + + const error = isRecord(json) && isRecord(json.error) ? json.error : {}; + + return { + ok: false, + status: res.status, + retryAfterMs: retryAfterMs(res), + error: new ErrorType( + typeof error.code === "string" ? error.code : `http_${res.status}`, + typeof error.message === "string" ? error.message : `telemetry.dev returned ${res.status}`, + res.status, + ), + }; + } catch (error) { + // The timeout can also cut the body, which then reads as an invalid response. + if (controller.signal.aborted) { + return { + ok: false, + error: new ErrorType("timeout", `${method} ${url.pathname} took more than ${timeoutMs} ms`), + }; + } + + return { + ok: false, + error: new ErrorType("network_error", "Could not reach telemetry.dev", undefined, { + cause: error, + }), + }; + } finally { + clearTimeout(timer); + } +} + +function retryable(status: number | undefined, idempotent: boolean): boolean { + if (status === 429) return true; + + return idempotent && (status === undefined || status >= 500); +} diff --git a/packages/sdk/src/attrs.ts b/packages/sdk/src/attrs.ts index 75afcb4..6dbf34f 100644 --- a/packages/sdk/src/attrs.ts +++ b/packages/sdk/src/attrs.ts @@ -179,6 +179,9 @@ export function fieldsToAttributes( if (fields.attributes) { Object.assign(attrs, omitUndefined(fields.attributes)); + + // The variables are input, also when the app passes them from promptAttributesFor(). + if (!cfg.captureInput) delete attrs["td.prompt.variables"]; } return attrs; diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts new file mode 100644 index 0000000..33f13e9 --- /dev/null +++ b/packages/sdk/src/ci.ts @@ -0,0 +1,172 @@ +import type * as fs from "node:fs"; + +import { reportError } from "@telemetry-dev/otel"; + +/** The pull request and the workflow run of an experiment that runs in GitHub Actions. */ +export interface CiRun { + provider: "github_actions"; + /** `owner/name`. */ + repository: string; + commit: string; + prNumber?: number; + prTitle?: string; + prUrl?: string; + runUrl?: string; +} + +type Env = Record; + +type RuntimeGlobal = typeof globalThis & { + process?: { getBuiltinModule?: (id: string) => unknown }; +}; + +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const COMMIT = /^[0-9a-f]{7,64}$/i; +const HTTPS_URL = /^https:\/\/\S+$/; +/** A plain decimal, so that the TypeScript and Python SDKs read the same marks. */ +const DECIMAL = /^-?\d+(\.\d+)?$/; +const MAX_REPOSITORY_LENGTH = 200; +const MAX_URL_LENGTH = 2_048; +const MAX_PR_TITLE_LENGTH = 256; + +/** CI mode is on when `CI` is `true` or `1`, as CI services set it. */ +export function inCi(env: Env): boolean { + return env.CI === "true" || env.CI === "1"; +} + +/** + * The run that GitHub Actions describes in its variables, or undefined outside GitHub Actions. On a + * pull request the commit is the head of the pull request: `GITHUB_SHA` is then a merge commit + * that nobody pushed. + */ +export function githubRun(env: Env): CiRun | undefined { + const repository = env.GITHUB_REPOSITORY; + + if ( + env.GITHUB_ACTIONS !== "true" || + !repository || + repository.length > MAX_REPOSITORY_LENGTH || + !REPOSITORY.test(repository) + ) { + return undefined; + } + + const pull = pullRequest(env.GITHUB_EVENT_PATH); + const commit = pull?.commit ?? env.GITHUB_SHA; + + if (!commit || !COMMIT.test(commit)) return undefined; + + const server = (env.GITHUB_SERVER_URL ?? "https://github.com").replace(/\/+$/, ""); + const runId = env.GITHUB_RUN_ID; + + return { + provider: "github_actions", + repository, + commit, + prNumber: pull?.number, + prTitle: pull?.title, + prUrl: pull?.url, + runUrl: + runId && /^\d+$/.test(runId) + ? httpsUrl(`${server}/${repository}/actions/runs/${runId}`) + : undefined, + }; +} + +/** + * The pass marks in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which the experiment action sets from + * its `min-scores` input: `name=value` pairs, separated by commas or new lines. + */ +export function minScoresFromEnv(env: Env): Record { + // Pairs, so that the name __proto__ becomes a key and not the prototype of the object. + const scores: [string, number][] = []; + + for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { + if (pair.trim() === "") continue; + const [name, value, ...rest] = pair.split("=").map((part) => part.trim()); + + if (!name || !value || rest.length > 0 || !DECIMAL.test(value)) { + throw new TypeError( + `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "${pair.trim()}" is not a name=number pair`, + ); + } + + scores.push([name, Number(value)]); + } + + return Object.fromEntries(scores); +} + +/** + * Adds the result to the file in `TELEMETRY_DEV_EXPERIMENT_RESULTS` as one line of JSON. The + * experiment action reads the file to write the job summary and the comment on the pull request. + */ +export function writeResult(env: Env, result: unknown, onError?: (cause: Error) => void): void { + const path = env.TELEMETRY_DEV_EXPERIMENT_RESULTS; + + if (!path) return; + + try { + const fs = nodeFs(); + + if (!fs) throw new Error("This runtime cannot write TELEMETRY_DEV_EXPERIMENT_RESULTS"); + + fs.appendFileSync(path, `${JSON.stringify(result)}\n`, { mode: 0o600 }); + } catch (error) { + reportError(onError, error); + } +} + +function pullRequest( + eventPath: string | undefined, +): { number?: number; title?: string; url?: string; commit?: string } | undefined { + if (!eventPath) return undefined; + + let event: unknown; + + try { + event = JSON.parse(nodeFs()?.readFileSync(eventPath, "utf8") ?? "null"); + } catch { + return undefined; + } + + const pull = (event as { pull_request?: Record } | null)?.pull_request; + + if (typeof pull !== "object" || pull === null) return undefined; + + const head = pull.head as { sha?: unknown } | undefined; + + return { + number: + Number.isSafeInteger(pull.number) && (pull.number as number) > 0 + ? (pull.number as number) + : undefined, + title: typeof pull.title === "string" ? cut(pull.title, MAX_PR_TITLE_LENGTH) : undefined, + url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, + commit: typeof head?.sha === "string" ? head.sha : undefined, + }; +} + +/** + * `text` cut to `max` UTF-16 units, with no half of a surrogate pair at the end: the API stores the + * run as jsonb, and Postgres rejects a lone surrogate there. + */ +function cut(text: string, max: number): string { + if (text.length <= max) return text; + + const last = text.charCodeAt(max - 1); + + return text.slice(0, last >= 0xd800 && last <= 0xdbff ? max - 1 : max); +} + +function httpsUrl(url: string): string | undefined { + return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; +} + +// Read at call time, with no static `node:` import, so that the package still loads in runtimes +// without a file system. +function nodeFs(): typeof fs | undefined { + return (globalThis as RuntimeGlobal).process?.getBuiltinModule?.("node:fs") as + | typeof fs + | undefined; +} diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts new file mode 100644 index 0000000..9bcc963 --- /dev/null +++ b/packages/sdk/src/experiments.ts @@ -0,0 +1,972 @@ +import { type Context, trace } from "@opentelemetry/api"; +import { + activeContext, + diag, + jsonAttr, + PROPAGATED_KEY, + propagatedFromContext, + reportError, + resolveEnv, +} from "@telemetry-dev/otel"; + +import { type Api, type ApiOptions, isRecord, request, resolveApi, sleep } from "./api.ts"; +import { truncate } from "./capture.ts"; +import { type CiRun, githubRun, inCi, minScoresFromEnv, writeResult } from "./ci.ts"; +import { currentClient, flush } from "./client.ts"; +import { checkValue, MAX_REASON_LENGTH, MAX_SCORE_NAME_LENGTH, USER_FEEDBACK } from "./scores.ts"; +import { type SpanHandle, startActiveSpan } from "./span.ts"; + +/** An item of a dataset version, as `GET /v1/datasets/{name}` returns it. */ +export interface DatasetItem { + /** The key of the item. It stays the same across dataset versions. */ + readonly id: string; + readonly number: number; + readonly input: Readonly>; + readonly expectedOutput: string | null; + readonly metadata: Readonly>; + readonly tags: readonly string[]; + readonly split: string | null; +} + +/** One run of a dataset item. With `runsPerItem` above 1, an item runs more than one time. */ +export interface ExperimentItem extends DatasetItem { + /** 0 to `runsPerItem` - 1. */ + readonly runIndex: number; +} + +/** + * What a scorer returns. `true` and `false` are a pass and a fail, a number is the value, and a + * string is the label. Null or undefined gives no score. + */ +export type ScorerResult = + | boolean + | number + | string + | { value?: number | boolean; label?: string; reason?: string } + | null + | undefined; + +/** Scores the output of an item on your machine. The score has the name of the scorer. */ +export type Scorer = ( + output: unknown, + item: ExperimentItem, +) => ScorerResult | Promise; + +export interface RunExperimentOptions extends ApiOptions { + /** Runs the code under test on one item. Its return value is the output of the item. */ + task: (item: ExperimentItem) => unknown; + /** + * The name of the dataset. Default: the `dataset` input of the experiment action, which it + * sets in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When both are set, they must be the same. + */ + dataset?: string; + /** Unique in the project. telemetry.dev picks a name when it is not set. */ + name?: string; + /** Default: the newest version. */ + datasetVersion?: number; + /** Run only the items of this split. */ + split?: string; + /** Evaluators of the project that score the outputs on the server, by name. */ + evaluators?: readonly string[]; + /** Scorers that run here, by score name. */ + scorers?: Readonly>; + /** + * Pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail score, or the lowest + * mean of a number score. The `min-scores` input of the experiment action adds to them and wins + * for the same name. + */ + minScores?: Readonly>; + /** 1 to 5. Default 1. */ + runsPerItem?: number; + /** Item runs at the same time, 1 to 100. Default 4. */ + concurrency?: number; + metadata?: Readonly>; + /** + * Records the GitHub Actions run, and rejects when the check fails or the experiment does not + * complete. Default: on when the `CI` variable is `true` or `1`. + */ + ci?: boolean; +} + +export interface ExperimentScore { + readonly name: string; + /** `pass_fail`, `number`, `labels`, `a_b_tie`, or `up_down`. Null before the score has values. */ + readonly kind: string | null; + /** The pass rate (0 to 1) of a pass/fail or up/down score, or the mean of a number score. */ + readonly value: number | null; + readonly passCount: number; + readonly failCount: number; + /** The value in the experiment that this one compares with, by default the dataset baseline. */ + readonly baselineValue: number | null; + /** Whole points of a pass rate, or the change of a mean to one decimal. */ + readonly delta: number | null; + readonly minScore: number | null; + /** Null when the score has no pass mark. */ + readonly passed: boolean | null; +} + +export interface ExperimentResult { + readonly id: string; + readonly name: string; + /** `completed`, `failed`, or `stopped`. */ + readonly status: string; + /** Null when the experiment has no pass marks. */ + readonly checkStatus: "passed" | "failed" | null; + readonly itemsTotal: number; + readonly itemsDone: number; + readonly itemsFailed: number; + /** Why the experiment failed. */ + readonly error: string | null; + readonly scores: readonly ExperimentScore[]; + readonly url: string; +} + +/** + * Why runExperiment failed. `code` is the code of the API error (`dataset_not_found`, + * `insufficient_scope`, `invalid_api_key`, …), or `missing_api_key`, `timeout`, `network_error`, + * `invalid_response`, or `dataset_empty`. In CI mode it is also `check_failed` or + * `experiment_failed`, and `result` holds the result. A response with a different status and no + * API error gives `http_`. A scorer that throws or returns a value that is not a score + * reports a `scorer_failed` error through `onError`. + */ +export class ExperimentError extends Error { + override readonly name = "ExperimentError"; + readonly result?: ExperimentResult; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown; result?: ExperimentResult }, + ) { + super(message, options); + this.result = options?.result; + } +} + +interface Run { + dataset: string; + task: (item: ExperimentItem) => unknown; + name?: string; + datasetVersion?: number; + split?: string; + evaluators?: readonly string[]; + scorers: [string, Scorer][]; + minScores?: Record; + runsPerItem: number; + concurrency: number; + metadata?: Readonly>; + ci: CiRun | undefined; + inCi: boolean; +} + +interface Created { + id: string; + name: string; + itemsTotal: number; + url: string; +} + +interface DatasetPage { + version: number; + items: DatasetItem[]; + nextCursor: string | null; +} + +interface ItemScore { + name: string; + value?: number; + label?: string; + reason?: string; +} + +interface ItemResult { + itemId: string; + runIndex: number; + output: string | null; + traceId?: string; + latencyMs: number; + error?: string; + scores?: ItemScore[]; +} + +type Authed = Api & { apiKey: string }; + +/** The `experiment` environment, which pages, metrics, and alerts leave out by default. */ +const EXPERIMENT_ENVIRONMENT = "experiment"; +const MAX_RUNS_PER_ITEM = 5; +const MAX_CONCURRENCY = 100; +/** Server evaluators of one experiment, and scorers: an item takes at most 20 scores. */ +const MAX_EVALUATORS = 20; +const MAX_MIN_SCORES = 20; +const MAX_ERROR_LENGTH = 4_000; +const MAX_OUTPUT_LENGTH = 100_000; +const MAX_DATASET_ITEMS = 10_000; +/** Items in one `POST /v1/experiments/{id}/items`. */ +const MAX_BATCH_ITEMS = 100; +/** Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. */ +const MAX_BATCH_BYTES = 4 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; +/** How long runExperiment waits for the server evaluators after the last item. */ +const MAX_WAIT_MS = 30 * 60_000; +const MAX_POLL_INTERVAL_MS = 5_000; +const FINISHED = new Set(["completed", "failed", "stopped"]); +const RATE_KINDS = new Set(["pass_fail", "up_down"]); + +/** + * Runs `task` on each item of a dataset and records the result as an experiment in telemetry.dev. + * Each item run is its own trace, with `td.experiment.id` and `td.dataset.item_id` on each span. + * The outputs go to telemetry.dev in batches of at most 100, with the scores of `scorers`. Then + * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. + * + * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` + * when a request fails. When the run stops after it made the experiment, it first sends the + * complete request, so that telemetry.dev ends the experiment with the item runs that it has. A + * task that throws fails only its item. In CI mode, runExperiment rejects when the check fails, so + * the process exits with an error. + */ +export async function runExperiment(options: RunExperimentOptions): Promise { + const env = resolveEnv(); + const run = checkOptions(options, env); + const resolved = resolveApi(options); + + if (!resolved.apiKey) { + throw new ExperimentError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const api = { ...resolved, apiKey: resolved.apiKey }; + warnAboutClient(); + + const first = await fetchPage(api, run, run.datasetVersion, undefined); + + if (first.items.length === 0) { + throw new ExperimentError( + "dataset_empty", + `The dataset ${run.dataset} has no items${run.split === undefined ? "" : ` in the split ${run.split}`}`, + ); + } + + const created = parseCreated( + await request(api, ExperimentError, "POST", "/v1/experiments", { + body: { + name: run.name, + dataset: run.dataset, + datasetVersion: first.version, + split: run.split, + evaluators: run.evaluators, + minScores: run.minScores, + runsPerItem: run.runsPerItem, + ci: run.ci, + metadata: run.metadata, + }, + timeoutMs: REQUEST_TIMEOUT_MS, + // A repeat after a lost response would start a second experiment. + idempotent: false, + }), + ); + + try { + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + await runItems(api, run, created, itemRuns(api, run, first)); + } catch (error) { + // Without the complete, the experiment would stay running: the server ends it with the item + // runs that it has, or fails it when it has none. + await complete(api, created.id).catch((cause: unknown) => reportError(api.onError, cause)); + + throw error; + } + + // The spans of the items reach ingest before the server reads their cost. + await flush(); + await complete(api, created.id); + + const { result, body } = await waitForResult(api, created.id); + writeResult(env, body, api.onError); + console.log(describe(result)); + + if (run.inCi) checkResult(result); + + return result; +} + +function checkOptions(options: RunExperimentOptions, env: Record): Run { + if (typeof options.task !== "function") throw new TypeError("task must be a function"); + + for (const [key, value] of [ + ["dataset", options.dataset], + ["name", options.name], + ["split", options.split], + ] as const) { + if (value !== undefined && (typeof value !== "string" || value.length === 0)) { + throw new TypeError(`${key} must be a non-empty string`); + } + } + + const actionDataset = env.TELEMETRY_DEV_EXPERIMENT_DATASET || undefined; + const dataset = options.dataset ?? actionDataset; + + if (dataset === undefined) { + throw new TypeError("Pass dataset, or set the dataset input of the experiment action"); + } + + if (actionDataset !== undefined && actionDataset !== dataset) { + throw new TypeError( + `dataset is ${dataset}, but the experiment action names ${actionDataset}. Name one dataset.`, + ); + } + + const { datasetVersion } = options; + + if ( + datasetVersion !== undefined && + !(Number.isSafeInteger(datasetVersion) && datasetVersion >= 1) + ) { + throw new RangeError("datasetVersion must be a positive integer"); + } + + const evaluators = options.evaluators; + + if ( + evaluators !== undefined && + (!Array.isArray(evaluators) || + evaluators.length > MAX_EVALUATORS || + !evaluators.every((name) => typeof name === "string" && validScoreName(name))) + ) { + throw new TypeError(`evaluators must be a list of at most ${MAX_EVALUATORS} evaluator names`); + } + + const scorers = Object.entries(options.scorers ?? {}); + const evaluatorNames = new Set(evaluators?.map((name) => name.toLowerCase())); + + if (scorers.length > MAX_EVALUATORS) { + throw new TypeError(`scorers can have at most ${MAX_EVALUATORS} scorers`); + } + + for (const [name, scorer] of scorers) { + if (!validScoreName(name) || typeof scorer !== "function") { + throw new TypeError( + `scorers must map score names of 1 to ${MAX_SCORE_NAME_LENGTH} characters to functions`, + ); + } + + if (name === USER_FEEDBACK) { + throw new TypeError( + `${USER_FEEDBACK} is the score of user feedback. Name the scorer another way.`, + ); + } + + if (evaluatorNames.has(name.toLowerCase())) { + throw new TypeError(`The scorer ${name} has the name of an evaluator. Rename one of them.`); + } + } + + const minScores = { ...options.minScores, ...minScoresFromEnv(env) }; + + if ( + Object.keys(minScores).length > MAX_MIN_SCORES || + !Object.entries(minScores).every( + ([name, value]) => validScoreName(name) && Number.isFinite(value), + ) + ) { + throw new TypeError(`minScores must map at most ${MAX_MIN_SCORES} score names to numbers`); + } + + const runsPerItem = options.runsPerItem ?? 1; + + if (!Number.isInteger(runsPerItem) || runsPerItem < 1 || runsPerItem > MAX_RUNS_PER_ITEM) { + throw new RangeError(`runsPerItem must be an integer from 1 to ${MAX_RUNS_PER_ITEM}`); + } + + const concurrency = options.concurrency ?? 4; + + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) { + throw new RangeError(`concurrency must be an integer from 1 to ${MAX_CONCURRENCY}`); + } + + if (options.metadata !== undefined && !isRecord(options.metadata)) { + throw new TypeError("metadata must be an object"); + } + + if (options.ci !== undefined && typeof options.ci !== "boolean") { + throw new TypeError("ci must be a boolean"); + } + + const ci = options.ci ?? inCi(env); + + return { + dataset, + task: options.task, + name: options.name, + datasetVersion, + split: options.split, + evaluators, + scorers, + minScores: Object.keys(minScores).length > 0 ? minScores : undefined, + runsPerItem, + concurrency, + metadata: options.metadata, + ci: ci ? githubRun(env) : undefined, + inCi: ci, + }; +} + +/** A trimmed name of 1 to 64 characters, as the API stores score names. */ +function validScoreName(name: string): boolean { + return name.length > 0 && name.length <= MAX_SCORE_NAME_LENGTH && name.trim() === name; +} + +function warnAboutClient(): void { + const environment = currentClient().core?.config.environment; + + if (environment === undefined) { + diag.warn( + 'runExperiment: no telemetry client, so the items record no traces. Call init({ environment: "experiment" }) first.', + ); + } else if (environment.trim().toLowerCase() !== EXPERIMENT_ENVIRONMENT) { + diag.warn( + `runExperiment: the traces of the items go to the environment ${environment}. Call init({ environment: "experiment" }) to keep them out of it.`, + ); + } +} + +/** + * The runs of each item of the dataset version, page by page, so that only one page is in memory. + * Workers share the generator: an async generator answers each `next()` in turn. + */ +async function* itemRuns( + api: Authed, + run: Run, + first: DatasetPage, +): AsyncGenerator { + let page = first; + let count = 0; + + for (;;) { + count += page.items.length; + + if (count > MAX_DATASET_ITEMS) { + throw new ExperimentError( + "invalid_response", + `telemetry.dev returned more than ${MAX_DATASET_ITEMS} items`, + ); + } + + for (const item of page.items) { + // A copy for each run, so that a task that changes its input does not change the next run. + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) { + yield { ...structuredClone(item), runIndex }; + } + } + + if (page.nextCursor === null) return; + + page = await fetchPage(api, run, page.version, page.nextCursor); + } +} + +/** One page of a dataset version. A page after the first names the version of the first. */ +async function fetchPage( + api: Authed, + run: Run, + version: number | undefined, + cursor: string | undefined, +): Promise { + const page = parseDatasetPage( + await request(api, ExperimentError, "GET", `/v1/datasets/${encodeURIComponent(run.dataset)}`, { + query: { version: version?.toString(), split: run.split, cursor }, + timeoutMs: REQUEST_TIMEOUT_MS, + idempotent: true, + }), + ); + + if ( + !page || + (version !== undefined && page.version !== version) || + (page.nextCursor !== null && (page.items.length === 0 || page.nextCursor === cursor)) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned a dataset it cannot read", + ); + } + + return page; +} + +/** + * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that + * fails stops the run: no new item starts, and runItems rejects when the running items end. + */ +async function runItems( + api: Authed, + run: Run, + experiment: Created, + runs: AsyncGenerator, +): Promise { + const batch = itemBatch((results) => + request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(experiment.id)}/items`, + // The server skips an item run that it has, so a repeat is safe. + { body: { items: results }, timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ), + ); + + let failure: { error: unknown } | undefined; + + const worker = async () => { + while (failure === undefined) { + const next = await runs.next().catch((error: unknown) => { + failure ??= { error }; + + return { done: true as const, value: undefined }; + }); + + if (next.done) return; + + await runItem(run, experiment, next.value, api.onError) + .then((result) => batch.add(result)) + .catch((error: unknown) => { + failure ??= { error }; + }); + } + }; + + await Promise.all(Array.from({ length: run.concurrency }, worker)); + + if (failure === undefined) { + await batch.flush().catch((error: unknown) => { + failure = { error }; + }); + } + + if (failure !== undefined) { + await runs.return(undefined); + + throw failure.error; + } +} + +/** Collects item results and posts them in order, one post at a time. */ +function itemBatch(post: (results: ItemResult[]) => Promise) { + let pending: ItemResult[] = []; + let bytes = 0; + let queue: Promise = Promise.resolve(); + + const send = async () => { + const results = pending; + pending = []; + bytes = 0; + await post(results); + }; + + return { + add(result: ItemResult): Promise { + const size = new TextEncoder().encode(JSON.stringify(result)).length; + + queue = queue.then(async () => { + if (pending.length > 0 && bytes + size > MAX_BATCH_BYTES) await send(); + pending.push(result); + bytes += size; + + if (pending.length >= MAX_BATCH_ITEMS) await send(); + }); + + return queue; + }, + flush(): Promise { + queue = queue.then(() => (pending.length > 0 ? send() : undefined)); + + return queue; + }, + }; +} + +async function runItem( + run: Run, + experiment: Created, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const started = performance.now(); + + const outcome = await startActiveSpan( + `experiment ${experiment.name}`, + { parent: itemContext(experiment.id, item), input: item.input }, + async (span): Promise<{ output?: unknown; error?: unknown; traceId?: string }> => { + try { + const output: unknown = await run.task(item); + span.update({ output }); + + return { output, traceId: traceIdOf(span) }; + } catch (error) { + span.update({ error }); + + return { + error: error ?? new Error("The task threw a nullish value"), + traceId: traceIdOf(span), + }; + } + }, + ); + + const base = { + itemId: item.id, + runIndex: item.runIndex, + traceId: outcome.traceId, + latencyMs: Math.round(performance.now() - started), + }; + + if ("error" in outcome) { + return { ...base, output: null, error: truncate(errorText(outcome.error), MAX_ERROR_LENGTH) }; + } + + const scores = await runScorers(run, outcome.output, item, onError); + + return { + ...base, + output: outputText(outcome.output), + scores: scores.length > 0 ? scores : undefined, + }; +} + +/** + * The context of one item run. It drops the active span, so that the run starts a trace of its + * own, and gives the run a session id of its own: an ambient session or `sessionMode: "process"` + * would otherwise put all runs in one session trace. + */ +function itemContext(experimentId: string, item: ExperimentItem): Context { + const base = trace.deleteSpan(activeContext()); + + return base.setValue(PROPAGATED_KEY, { + ...propagatedFromContext(base), + "td.experiment.id": experimentId, + "td.dataset.item_id": item.id, + "gen_ai.conversation.id": `${experimentId}:${item.id}:${item.runIndex}`, + }); +} + +function traceIdOf(span: SpanHandle): string | undefined { + return span.isRecording ? span.traceId : undefined; +} + +async function runScorers( + run: Run, + output: unknown, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const scores: ItemScore[] = []; + + // After the span of the item ends, so that the model calls of a scorer do not count in the + // cost of the item. + for (const [name, scorer] of run.scorers) { + try { + const score = toScore(name, await scorer(output, item)); + + if (score) scores.push(score); + } catch (error) { + reportError( + onError, + new ExperimentError( + "scorer_failed", + `The scorer ${name} failed on item ${item.number}: ${errorText(error)}`, + undefined, + { cause: error }, + ), + ); + } + } + + return scores; +} + +/** The score of a scorer result. It throws for a result that the server would reject. */ +function toScore(name: string, result: ScorerResult): ItemScore | undefined { + if (result === null || result === undefined) return undefined; + + let score: ItemScore; + + if (typeof result === "boolean") score = { name, ...passFail(result) }; + else if (typeof result === "number") score = { name, value: result }; + else if (typeof result === "string") score = { name, label: result.trim() }; + else if (isRecord(result)) { + const { value, label, reason } = result as Record; + + for (const [key, field] of [ + ["label", label], + ["reason", reason], + ] as const) { + if (field !== undefined && typeof field !== "string") { + throw new TypeError(`returned a ${key} that is not a string`); + } + } + + score = { + name, + ...(typeof value === "boolean" ? passFail(value) : { value: value as number | undefined }), + ...(label === undefined ? {} : { label: (label as string).trim() }), + reason: reason === undefined ? undefined : truncate(reason as string, MAX_REASON_LENGTH), + }; + } else { + throw new TypeError(`returned a ${typeof result}`); + } + + const problem = checkValue(name, score.value, score.label); + + if (problem) throw new TypeError(`returned a score that is not valid: ${problem}`); + + return score; +} + +function passFail(pass: boolean): { value: number; label: string } { + return pass ? { value: 1, label: "pass" } : { value: 0, label: "fail" }; +} + +function outputText(output: unknown): string | null { + const text = output === null || output === undefined ? undefined : jsonAttr(output); + + return text === undefined ? null : truncate(text, MAX_OUTPUT_LENGTH); +} + +function errorText(error: unknown): string { + try { + return error instanceof Error ? `${error.name}: ${error.message}` : String(error); + } catch { + return "A value that cannot be converted to a string was thrown"; + } +} + +function complete(api: Authed, id: string): Promise { + return request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); +} + +/** Waits until the server evaluators finish and the experiment ends. */ +async function waitForResult( + api: Authed, + id: string, +): Promise<{ result: ExperimentResult; body: unknown }> { + const deadline = Date.now() + MAX_WAIT_MS; + + for (let interval = 1000; ; interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS)) { + const body = await request( + api, + ExperimentError, + "GET", + `/v1/experiments/${encodeURIComponent(id)}`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const result = parseResult(body); + + if (FINISHED.has(result.status)) return { result, body }; + + if (Date.now() + interval > deadline) { + throw new ExperimentError( + "timeout", + `The experiment ${result.name} did not end in ${MAX_WAIT_MS / 60_000} minutes. ${result.url}`, + undefined, + { result }, + ); + } + + await sleep(interval); + } +} + +/** In CI mode, a failed check or an experiment that did not complete fails the run. */ +function checkResult(result: ExperimentResult): void { + if (result.status !== "completed") { + throw new ExperimentError( + "experiment_failed", + `The experiment ${result.name} ${result.status}${result.error ? `: ${result.error}` : ""}. ${result.url}`, + undefined, + { result }, + ); + } + + if (result.checkStatus !== "failed") return; + + const failed = result.scores + .filter((score) => score.passed === false) + .map( + (score) => + `${score.name} is ${formatScore(score.kind, score.value)}, under the pass mark ${formatScore(score.kind, score.minScore)}`, + ); + + throw new ExperimentError( + "check_failed", + `The check of ${result.name} failed${failed.length > 0 ? `: ${failed.join("; ")}` : ""}. ${result.url}`, + undefined, + { result }, + ); +} + +function describe(result: ExperimentResult): string { + const outcome = + result.status !== "completed" + ? `${result.status}${result.error ? `: ${result.error}` : ""}` + : result.checkStatus === null + ? "completed" + : `completed. The check ${result.checkStatus}`; + + return [ + `Experiment ${result.name} ${outcome}. ${result.url}`, + ...result.scores.map( + (score) => + ` ${score.name}: ${formatScore(score.kind, score.value)}` + + (score.minScore === null ? "" : ` (pass mark ${formatScore(score.kind, score.minScore)})`), + ), + ].join("\n"); +} + +function formatScore(kind: string | null, value: number | null): string { + if (value === null) return "no score"; + + return kind !== null && RATE_KINDS.has(kind) + ? `${Math.round(value * 100)}%` + : String(Math.round(value * 100) / 100); +} + +function parseCreated(body: unknown): Created { + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.url !== "string" || + !Number.isSafeInteger(body.itemsTotal) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned an experiment it cannot read", + ); + } + + return { id: body.id, name: body.name, itemsTotal: body.itemsTotal as number, url: body.url }; +} + +function parseDatasetPage(body: unknown): DatasetPage | undefined { + if (!isRecord(body) || !isRecord(body.dataset) || !Array.isArray(body.items)) return undefined; + + const { version } = body.dataset; + const nextCursor = body.nextCursor ?? null; + + if (!Number.isSafeInteger(version) || (version as number) < 0) return undefined; + + if (nextCursor !== null && typeof nextCursor !== "string") return undefined; + + const items: DatasetItem[] = []; + + for (const item of body.items) { + if ( + !isRecord(item) || + typeof item.id !== "string" || + !Number.isSafeInteger(item.number) || + !isRecord(item.input) || + (item.expectedOutput !== null && typeof item.expectedOutput !== "string") || + !isRecord(item.metadata) || + !Array.isArray(item.tags) || + !item.tags.every((tag) => typeof tag === "string") || + (item.split !== null && typeof item.split !== "string") + ) { + return undefined; + } + + items.push({ + id: item.id, + number: item.number as number, + input: item.input, + expectedOutput: item.expectedOutput, + metadata: item.metadata, + tags: item.tags as string[], + split: item.split, + }); + } + + return { version: version as number, items, nextCursor }; +} + +function parseResult(body: unknown): ExperimentResult { + const scores = isRecord(body) && Array.isArray(body.scores) ? body.scores.map(parseScore) : []; + + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.status !== "string" || + typeof body.url !== "string" || + // Required: it decides the result of CI mode, so a missing one must not read as no check. + !( + body.checkStatus === null || + body.checkStatus === "passed" || + body.checkStatus === "failed" + ) || + typeof body.itemsTotal !== "number" || + typeof body.itemsDone !== "number" || + !(body.itemsFailed === undefined || typeof body.itemsFailed === "number") || + !(body.error === undefined || body.error === null || typeof body.error === "string") || + !Array.isArray(body.scores) || + scores.some((score) => score === undefined) + ) { + throw new ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read"); + } + + return { + id: body.id, + name: body.name, + status: body.status, + checkStatus: body.checkStatus, + itemsTotal: body.itemsTotal, + itemsDone: body.itemsDone, + itemsFailed: body.itemsFailed ?? 0, + error: body.error ?? null, + scores: scores as ExperimentScore[], + url: body.url, + }; +} + +/** A score of the result. A field that can be null can also be missing. */ +function parseScore(raw: unknown): ExperimentScore | undefined { + if (!isRecord(raw)) return undefined; + + const score = { + name: raw.name, + kind: raw.kind ?? null, + value: raw.value ?? null, + passCount: raw.passCount, + failCount: raw.failCount, + baselineValue: raw.baselineValue ?? null, + delta: raw.delta ?? null, + minScore: raw.minScore ?? null, + passed: raw.passed ?? null, + }; + const numbers = [score.value, score.baselineValue, score.delta, score.minScore]; + + if ( + typeof score.name !== "string" || + !(score.kind === null || typeof score.kind === "string") || + typeof score.passCount !== "number" || + typeof score.failCount !== "number" || + !numbers.every((field) => field === null || typeof field === "number") || + !(score.passed === null || typeof score.passed === "boolean") + ) { + return undefined; + } + + return score as ExperimentScore; +} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2fc3343..d4055c7 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -46,6 +46,20 @@ export { log, type LogOptions } from "./logs.ts"; export { observe, type ObserveOptions } from "./observe.ts"; +export { + type DatasetItem, + ExperimentError, + type ExperimentItem, + type ExperimentResult, + type ExperimentScore, + runExperiment, + type RunExperimentOptions, + type Scorer, + type ScorerResult, +} from "./experiments.ts"; + +export { score, ScoreError, type ScoreOptions } from "./scores.ts"; + export { type ChatMessage, type ChatPrompt, @@ -54,6 +68,7 @@ export { type GetPromptOptions, type GetTextPromptOptions, type Prompt, + promptAttributesFor, PromptError, type PromptMessage, type PromptModel, diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b17..93b50d9 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -1,16 +1,20 @@ -import { type Attributes, type Context, createContextKey } from "@opentelemetry/api"; +import { + type Attributes, + type AttributeValue, + type Context, + createContextKey, +} from "@opentelemetry/api"; import { activeContext, - DEFAULT_BASE_URL, jsonAttr, omitUndefined, reportError, - resolveEnv, sessionIdOf, sha256, withContext, } from "@telemetry-dev/otel"; +import { isRecord, resolveApi, retryAfterMs } from "./api.ts"; import type { FieldCaptureConfig } from "./attrs.ts"; import { prepareCaptureValue } from "./capture.ts"; import { currentClient } from "./client.ts"; @@ -92,6 +96,12 @@ export interface ChatPrompt extends PromptBase { /** * Fills in the variables and replaces each placeholder message with the messages in its * variable. `withPrompt` records the variables of the last call before it starts or inside it. + * It tags each message object that it makes with this prompt and a copy of these variables and + * of each list in them, so that a model call outside `withPrompt` that sends the message records + * them; see `promptAttributesFor`. The objects in a list are not copied. The tag keeps the + * variables for 5 minutes after this compile and after each call that names it. Other compiles + * do not change this. The tag keeps no variables when input capture is off in the client, or + * when the runtime has no `WeakRef`. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -102,7 +112,8 @@ export interface TextPrompt extends PromptBase { readonly text: string; /** * Fills in the variables. `withPrompt` records the variables of the last call before it starts - * or inside it. + * or inside it. A string cannot carry a tag, so a model call outside `withPrompt` does not + * record a text prompt. */ compile(variables?: PromptVariables): string; } @@ -116,7 +127,10 @@ interface GetPromptOptionsBase { version?: number; /** Seconds before a call fetches a kept prompt again, in the background. Default 60. */ cacheTtlSeconds?: number; - /** API key with the prompts:read scope. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + /** + * API key with Read prompts in its Access (the prompts:read scope). Falls back to the initialized + * client, then `TELEMETRY_DEV_API_KEY`. + */ apiKey?: string; /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ baseUrl?: string; @@ -193,22 +207,48 @@ interface PromptScope { variables: object | undefined; } +/** The `compile()` call that made a chat message. */ +interface CompileTag { + readonly prompt: Prompt; + /** Undefined when input capture was off, or after `expires`; see expire. */ + variables: PromptVariables | undefined; + /** Counts the compiles of the process, so a higher one is newer. */ + readonly serial: number; + /** + * When the tag removes its variables, in `performance.now()` time. A call that names the tag + * moves it. + */ + expires: number; +} + const FETCH_TIMEOUT_MS = 10_000; /** The longest Retry-After that getPrompt honors, so that a bad header cannot stop fetches. */ const MAX_RETRY_AFTER_MS = 3_600_000; /** How long a failed fetch with no copy answers calls, so that they do not each fetch again. */ const ERROR_TTL_MS = 10_000; const MAX_CACHE_ENTRIES = 1_000; -/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ -const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); +// Not enumerable, so JSON.stringify, spreads, and structuredClone leave the tag out of a copy. +const COMPILED_BY = Symbol("telemetry.dev compiled by"); +/** How many messages promptAttributesFor looks at, at each end of the list. */ +const TAG_LOOKUP = 1_000; +/** How long a tag keeps its variables after its compile, or after a call that names it. */ +const VARIABLES_TTL_MS = 300_000; +const MINUTE_MS = 60_000; const cache = new Map(); const compiledVariables = new WeakMap(); /** The Retry-After of the response that failed with this error, in milliseconds. */ const retryAfter = new WeakMap(); +let compiles = 0; +/** + * The tags that keep variables, by the minute of their deadline. A weak reference, so that the + * variables of a message that the app dropped do not stay until the deadline. The minutes are not + * sorted, since a tag that a call named goes back in at its new minute. + */ +const deadlines = new Map[]>(); /** * Fetches a prompt version from telemetry.dev and keeps it. When the copy is older than @@ -228,20 +268,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P const type = options.type ?? "chat"; checkOptions(name, options, type); - const core = currentClient().core; - const env = resolveEnv(); - const apiKey = options.apiKey ?? core?.config.apiKey ?? env.TELEMETRY_DEV_API_KEY; - - // An empty value counts as unset, as in the Python SDK. - const baseUrl = ( - options.baseUrl || - core?.config.baseUrl || - env.TELEMETRY_DEV_BASE_URL || - DEFAULT_BASE_URL - ).replace(/\/+$/, ""); - + const { apiKey, baseUrl, sdkName, fetchImpl, onError } = resolveApi(options); const label = options.version === undefined ? (options.label ?? "production") : null; - const onError = core?.config.onError; let response: PromptResponse; try { @@ -269,11 +297,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P () => fetchPrompt( url, - { - authorization: `Bearer ${apiKey}`, - "x-telemetry-dev-sdk": core?.config.sdkName ?? "@telemetry-dev/sdk", - }, - options.fetch ?? core?.config.fetchImpl ?? globalThis.fetch, + { authorization: `Bearer ${apiKey}`, "x-telemetry-dev-sdk": sdkName }, + fetchImpl, ), onError, ); @@ -344,19 +369,82 @@ export function promptAttributes( | undefined; if (!scope) return undefined; - const { prompt, variables } = scope; + return attributesOf(scope.prompt, scope.variables, capture); +} + +/** + * The td.prompt.* attributes for a model call outside `withPrompt` that sends `messages` (a list + * of messages, or one message): those of the newest `compile()` of a chat prompt that made one of + * the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the message + * object, so a copy of it gives nothing. Gives undefined inside `withPrompt`, whose prompt the span + * gets. `td.prompt.variables` needs input capture on in the client, at the compile and now, and + * the span drops it when its own input capture is off. A compile gives no variables 5 minutes + * after it or after the last call that named it; see `compile()`. It never throws. + */ +export function promptAttributesFor(messages: unknown): Record | undefined { + const config = currentClient().core?.config; + + try { + if (activeContext().getValue(PROMPT_KEY) !== undefined) return undefined; + + const now = performance.now(); + expire(now); + const tag = newestTag(messages); + + if (!tag) return undefined; + + if (tag.expires > now) tag.expires = now + VARIABLES_TTL_MS; + else tag.variables = undefined; + + return attributesOf(tag.prompt, tag.variables, config) as Record; + } catch (error) { + reportError(config?.onError, error); + + return undefined; + } +} + +function attributesOf( + prompt: Prompt, + variables: object | undefined, + capture: FieldCaptureConfig | undefined, +): Attributes { return omitUndefined({ "td.prompt.name": prompt.name, "td.prompt.version": prompt.version ?? undefined, "td.prompt.label": prompt.label ?? undefined, "td.prompt.variables": - variables !== undefined && capture.captureInput + variables !== undefined && capture?.captureInput ? prepareCaptureValue("td.prompt.variables", variables, capture) : undefined, }); } +// The messages of a placeholder come from an older compile, so the newest compile, not the last +// tagged message, names the prompt of the request. Its own messages are at the start or the end of +// the list, unless placeholders put more than 1000 messages on each side of them. +function newestTag(messages: unknown): CompileTag | undefined { + if (!Array.isArray(messages)) return tagOf(messages); + const { length } = messages; + let newest: CompileTag | undefined; + + for (let index = 0; index < length; index += 1) { + if (index === TAG_LOOKUP && length > 2 * TAG_LOOKUP) index = length - TAG_LOOKUP; + const tag = tagOf(messages[index]); + + if (tag && (newest === undefined || tag.serial > newest.serial)) newest = tag; + } + + return newest; +} + +function tagOf(message: unknown): CompileTag | undefined { + return typeof message === "object" && message !== null + ? (message as { [COMPILED_BY]?: CompileTag })[COMPILED_BY] + : undefined; +} + function checkOptions(name: string, options: GetPromptOptions, type: string): void { if (typeof name !== "string" || name.length === 0) { throw new TypeError("name must be a non-empty string"); @@ -492,7 +580,7 @@ async function fetchPrompt( ); const delay = retryAfterMs(res); - if (delay !== undefined) retryAfter.set(failure, delay); + if (delay !== undefined) retryAfter.set(failure, Math.min(delay, MAX_RETRY_AFTER_MS)); throw failure; } @@ -543,27 +631,90 @@ function createPrompt(input: { responseFormat: content.responseFormat, isFallback: input.isFallback, compile(variables: PromptVariables = {}) { - compiledVariables.set(prompt, variables); + const values = copyVariables(variables); + compiledVariables.set(prompt, values); const scope = activeContext().getValue(PROMPT_KEY) as PromptScope | undefined; - if (scope?.prompt === prompt) scope.variables = variables; - - return type === "chat" - ? compileMessages(content.messages, variables) - : fill(content.text ?? "", variables); + if (scope?.prompt === prompt) scope.variables = values; + + if (type === "text") return fill(content.text ?? "", values); + compiles += 1; + const now = performance.now(); + expire(now); + const tag: CompileTag = { + prompt, + variables: undefined, + serial: compiles, + expires: now + VARIABLES_TTL_MS, + }; + + // promptAttributesFor cannot record the variables when input capture is off in the client. + // Without WeakRef (Cloudflare Workers before 2025-05-05), expire could not remove them. + if ((currentClient().core?.config.captureInput ?? true) && typeof WeakRef === "function") { + tag.variables = values; + addDeadline(new WeakRef(tag), tag.expires); + } + + return compileMessages(content.messages, values, tag); }, }) as Prompt; return prompt; } +/** + * Removes the variables of each expired tag, also when the app keeps its messages. Thus a chat + * history keeps the variables of the last 5 minutes only, and other compiles do not change the time + * of a request. A tag that a call named gets its new deadline. + */ +function expire(now: number): void { + const minute = Math.floor(now / MINUTE_MS); + + for (const [due, refs] of deadlines) { + if (due >= minute) continue; + deadlines.delete(due); + + for (const ref of refs) { + const tag = ref.deref(); + + if (!tag) continue; + + if (tag.expires > now) addDeadline(ref, tag.expires); + else tag.variables = undefined; + } + } +} + +function addDeadline(ref: WeakRef, at: number): void { + const minute = Math.floor(at / MINUTE_MS); + const refs = deadlines.get(minute); + + if (refs) refs.push(ref); + else deadlines.set(minute, [ref]); +} + +// A copy of each list too, so that a change of the app's history after compile() does not change +// what a span records. +function copyVariables(variables: PromptVariables): PromptVariables { + return Object.fromEntries( + Object.entries(variables).map(([name, value]) => [ + name, + Array.isArray(value) ? [...value] : value, + ]), + ); +} + function compileMessages( messages: readonly PromptMessage[], variables: PromptVariables, + tag: CompileTag, ): unknown[] { return messages.flatMap((message) => { if (message.role !== "placeholder") { - return [{ role: message.role, content: fill(message.content, variables) }]; + const compiled = { role: message.role, content: fill(message.content, variables) }; + Object.defineProperty(compiled, COMPILED_BY, { value: tag }); + + return [compiled]; } const value = Object.hasOwn(variables, message.name) ? variables[message.name] : undefined; @@ -616,23 +767,6 @@ function keyFingerprint(apiKey: string): string { return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); } -/** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ -function retryAfterMs(res: Response): number | undefined { - if (res.status !== 429 && res.status !== 503) return undefined; - - const value = res.headers.get("retry-after")?.trim(); - - if (!value) return undefined; - - let ms: number; - - if (/^\d+$/.test(value)) ms = Number(value) * 1000; - else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); - else return undefined; - - return ms > 0 ? Math.min(ms, MAX_RETRY_AFTER_MS) : undefined; -} - function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; @@ -724,10 +858,6 @@ function isVersion(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 1; } -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - // Prompt objects share the cached response, so an app cannot change the cache through them. function deepFreeze(value: T): T { if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts new file mode 100644 index 0000000..bb06d76 --- /dev/null +++ b/packages/sdk/src/scores.ts @@ -0,0 +1,172 @@ +import { reportError } from "@telemetry-dev/otel"; + +import { type ApiOptions, isRecord, request, resolveApi } from "./api.ts"; + +export interface ScoreOptions extends ApiOptions { + /** The score name. `user_feedback` is a thumbs up (1) or down (0) from a user of your app. */ + name: string; + /** The trace that the score is on. Pass `traceId`, `sessionId`, or both. */ + traceId?: string; + /** A span of the trace, for a score on one span. */ + spanId?: string; + sessionId?: string; + /** Pass `value`, `label`, or both. */ + value?: number; + label?: string; + comment?: string; +} + +/** + * Why score failed. `code` is the code of the API error (`insufficient_scope`, `invalid_api_key`, + * …), or `missing_api_key`, `invalid_score`, `timeout`, `network_error`, or `invalid_response`. A + * response with a different status and no API error gives `http_`. + */ +export class ScoreError extends Error { + override readonly name = "ScoreError"; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown }, + ) { + super(message, options); + } +} + +export const USER_FEEDBACK = "user_feedback"; +export const MAX_SCORE_NAME_LENGTH = 64; +export const MAX_LABEL_LENGTH = 64; +export const MAX_REASON_LENGTH = 4_000; +const MAX_SESSION_ID_LENGTH = 1_024; +const TRACE_ID = /^(?!0+$)[0-9a-f]{32}$/; +const SPAN_ID = /^(?!0+$)[0-9a-f]{16}$/; +const TIMEOUT_MS = 10_000; + +/** + * Records a score on a trace, a span, or a session, for example the feedback of a user on a + * reply. Like the telemetry functions, score never rejects: it resolves with the id of the score, + * or with undefined after it reports the error through `onError`. It makes one request, with no + * retry, so that a call in a request handler ends in 10 seconds at most. The trace id of a span + * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. + */ +export async function score(options: ScoreOptions): Promise { + // A caller without types can pass no options, and score must not reject then either. + const fields: Partial = isRecord(options) ? options : {}; + const api = resolveApi(fields); + + try { + const name = typeof fields.name === "string" ? fields.name.trim() : fields.name; + const label = typeof fields.label === "string" ? fields.label.trim() : fields.label; + const problem = checkScore({ ...fields, name, label }); + + if (problem) throw new ScoreError("invalid_score", problem); + + if (!api.apiKey) { + throw new ScoreError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { + body: { + traceId: fields.traceId, + spanId: fields.spanId, + sessionId: fields.sessionId, + name, + value: fields.value, + label, + comment: fields.comment, + }, + timeoutMs: TIMEOUT_MS, + idempotent: false, + attempts: 1, + }); + + if (!isRecord(body) || typeof body.id !== "string") { + throw new ScoreError("invalid_response", "telemetry.dev returned a score it cannot read"); + } + + return body.id; + } catch (error) { + reportError(api.onError, error); + + return undefined; + } +} + +/** + * Why the server would reject the value and the label of a score, or undefined. A pass/fail score + * has the label `pass` with the value 1, or `fail` with 0. + */ +export function checkValue( + name: string, + value: number | undefined, + label: string | undefined, +): string | undefined { + if (value !== undefined && !(typeof value === "number" && Number.isFinite(value))) { + return "value must be a finite number"; + } + + if ( + label !== undefined && + (typeof label !== "string" || label.length === 0 || label.length > MAX_LABEL_LENGTH) + ) { + return `label must be a string of 1 to ${MAX_LABEL_LENGTH} characters`; + } + + if (value === undefined && label === undefined) return "Pass a value or a label"; + + if (name === USER_FEEDBACK && !(label === undefined && (value === 0 || value === 1))) { + return `${USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label`; + } + + if (label === "pass" && value !== undefined && value !== 1) { + return "The label pass goes with the value 1"; + } + + if (label === "fail" && value !== undefined && value !== 0) { + return "The label fail goes with the value 0"; + } + + return undefined; +} + +function checkScore(fields: Partial): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = fields; + + if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { + return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; + } + + if (traceId === undefined && sessionId === undefined) return "Pass traceId or sessionId"; + + if (traceId !== undefined && !(typeof traceId === "string" && TRACE_ID.test(traceId))) { + return "traceId must be 32 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && !(typeof spanId === "string" && SPAN_ID.test(spanId))) { + return "spanId must be 16 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && traceId === undefined) return "Pass the traceId of the span"; + + if ( + sessionId !== undefined && + (typeof sessionId !== "string" || + sessionId.length === 0 || + sessionId.length > MAX_SESSION_ID_LENGTH) + ) { + return `sessionId must be a string of 1 to ${MAX_SESSION_ID_LENGTH} characters`; + } + + if ( + comment !== undefined && + (typeof comment !== "string" || comment.length > MAX_REASON_LENGTH) + ) { + return `comment must be a string of at most ${MAX_REASON_LENGTH} characters`; + } + + return checkValue(name, value, label); +} diff --git a/packages/sdk/src/span.ts b/packages/sdk/src/span.ts index 4c4c38a..3ed9cd9 100644 --- a/packages/sdk/src/span.ts +++ b/packages/sdk/src/span.ts @@ -1,4 +1,5 @@ import { + type Attributes, type Context, defaultTextMapGetter, defaultTextMapSetter, @@ -98,6 +99,8 @@ interface SpanMeta { type: SpanType; capture: FieldCaptureConfig; onError?: (cause: Error) => void; + /** The span got its prompt from withPrompt. */ + scopedPrompt?: boolean; } // Remembers each span's type + capture flags so update()/end()/updateActiveSpan map @@ -190,11 +193,22 @@ function applyFields(span: Span, meta: SpanMeta, fields: SpanFields): void { if (fields.name) span.updateName(fields.name); const attrs = fieldsToAttributes(fields, meta.type, meta.capture); + if (meta.scopedPrompt) dropPromptAttributes(attrs); + if (Object.keys(attrs).length > 0) span.setAttributes(attrs); if (fields.error !== undefined) applyError(span, fields.error, meta.capture); } +// The prompt of withPrompt replaces each td.prompt.* attribute that the app passes, at start and in +// each update, so that a span does not mix two prompts when the app calls promptAttributesFor +// outside withPrompt. +function dropPromptAttributes(attrs: Attributes): void { + for (const key of Object.keys(attrs)) { + if (key.startsWith("td.prompt.")) delete attrs[key]; + } +} + function metaFor(core: ClientCore, options?: StartSpanOptions): SpanMeta { const cfg = core.config; @@ -231,6 +245,10 @@ export function createSpanHandle( const parentCtx = withSessionParent(sessionContext, sessionId, core.config.apiKey); const attrs = fieldsToAttributes(options, meta.type, meta.capture); const operation = attrs["gen_ai.operation.name"] ?? SPAN_TYPE_OPERATIONS[meta.type]; + const prompt = promptAttributes(parentCtx, operation, meta.capture); + meta.scopedPrompt = prompt !== undefined; + + if (meta.scopedPrompt) dropPromptAttributes(attrs); const span = core.tracer.startSpan( name, @@ -241,7 +259,7 @@ export function createSpanHandle( attributes: { "gen_ai.operation.name": SPAN_TYPE_OPERATIONS[meta.type], ...propagatedFromContext(parentCtx), - ...promptAttributes(parentCtx, operation, meta.capture), + ...prompt, ...attrs, }, }, diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts new file mode 100644 index 0000000..eb8bd99 --- /dev/null +++ b/packages/sdk/tests/experiment-action.test.ts @@ -0,0 +1,413 @@ +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +// The experiment action: run.sh runs the eval script, and report.mjs writes the job summary and +// the comment on the pull request. +const action = fileURLToPath(new URL("../../../actions/experiment/", import.meta.url)); +const closers: (() => void)[] = []; + +afterEach(() => { + for (const close of closers.splice(0)) close(); +}); + +function workdir(): string { + return mkdtempSync(join(tmpdir(), "td-action-")); +} + +function outputs(path: string): Record { + return Object.fromEntries( + readFileSync(path, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split(/=(.*)/s).slice(0, 2) as [string, string]), + ); +} + +function runScript(env: Record, files: Record = {}) { + const dir = workdir(); + const output = join(dir, "output"); + writeFileSync(output, ""); + + for (const [name, content] of Object.entries(files)) writeFileSync(join(dir, name), content); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: dir, + encoding: "utf8", + env: { PATH: process.env.PATH, RUNNER_TEMP: dir, GITHUB_OUTPUT: output, ...env }, + }); + + return { dir, run, outputs: outputs(output) }; +} + +const result = { + id: "6f9619ff-8b86-4011-b42d-00c04fc964ff", + name: "reply-v13-gpt-4o", + status: "completed", + checkStatus: "failed", + itemsTotal: 120, + itemsDone: 120, + itemsFailed: 2, + error: null, + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 100, + failCount: 18, + baselineValue: 0.88, + delta: -3, + minScore: 0.9, + passed: false, + }, + { + name: "helpfulness", + kind: "number", + value: 4.436, + passCount: 0, + failCount: 0, + baselineValue: 4.1, + delta: 0.3, + minScore: null, + passed: null, + }, + ], + url: "https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff", +}; + +const summary = `### telemetry.dev experiment + +**[reply-v13-gpt-4o](https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff)**: Check failed. 118 of 120 items ran, 2 failed. + +| Score | Value | Change | Pass mark | Result | +| --- | --- | --- | --- | --- | +| correctness | 85% | ↓ 3 pts | 90% | Failed | +| helpfulness | 4.44 | ↑ 0.3 | — | — | + +The experiment script exited with code 1.`; + +interface GithubRequest { + method: string; + url: string; + body: unknown; +} + +interface GithubComment { + id: number; + body: string; + user: { login: string; type: "Bot" | "User" }; +} + +const BOT = { login: "github-actions[bot]", type: "Bot" } as const; + +/** + * The comment routes of the GitHub REST API for one pull request. Without `user`, GET /user fails + * as it does for the GITHUB_TOKEN. + */ +async function fakeGithub( + options: { comments?: GithubComment[]; status?: number; user?: { login: string } } = {}, +) { + const requests: GithubRequest[] = []; + const server = createServer(async (req: IncomingMessage, res) => { + let raw = ""; + for await (const chunk of req) raw += chunk; + requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + + if (req.url === "/user") { + res.writeHead(options.user ? 200 : 403, { "content-type": "application/json" }); + res.end( + JSON.stringify(options.user ?? { message: "Resource not accessible by integration" }), + ); + + return; + } + + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); + res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + closers.push(() => server.close()); + + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +} + +async function report(env: Record, lines: unknown[]) { + const dir = workdir(); + const results = join(dir, "results.jsonl"); + const stepSummary = join(dir, "summary.md"); + const output = join(dir, "output"); + const event = join(dir, "event.json"); + writeFileSync(results, lines.map((line) => `${JSON.stringify(line)}\n`).join("")); + writeFileSync(stepSummary, ""); + writeFileSync(output, ""); + writeFileSync(event, JSON.stringify({ pull_request: { number: 42 } })); + + const child = spawn("node", [join(action, "report.mjs")], { + env: { + PATH: process.env.PATH, + TELEMETRY_DEV_EXPERIMENT_RESULTS: results, + GITHUB_STEP_SUMMARY: stepSummary, + GITHUB_OUTPUT: output, + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: "acme/support", + GITHUB_TOKEN: "ghs_test", + GITHUB_WORKFLOW: "evals", + GITHUB_JOB: "reply", + INPUT_DATASET: "support-golden", + EXPERIMENT_EXIT_CODE: "1", + ...env, + }, + }); + let stdout = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + const code = await new Promise((resolve) => child.on("close", resolve)); + + return { code, stdout, summary: readFileSync(stepSummary, "utf8"), outputs: outputs(output) }; +} + +test("run.sh passes the inputs to the script and records its exit code without failing", () => { + const { + dir, + run, + outputs: written, + } = runScript({ + INPUT_COMMAND: `node -e 'const fs = require("node:fs"); fs.writeFileSync("env.json", JSON.stringify(process.env)); process.exit(3)'`, + INPUT_API_KEY: "td_live_secret", + INPUT_DATASET: "support-golden", + INPUT_MIN_SCORES: "correctness=0.9", + INPUT_BASE_URL: "https://ingest.test", + }); + + expect(run.status).toBe(0); + expect(run.stdout).toContain("::add-mask::td_live_secret"); + expect(written["exit-code"]).toBe("3"); + + const env = JSON.parse(readFileSync(join(dir, "env.json"), "utf8")); + expect(env).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_secret", + TELEMETRY_DEV_BASE_URL: "https://ingest.test", + TELEMETRY_DEV_EXPERIMENT_DATASET: "support-golden", + TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0.9", + TELEMETRY_DEV_ENVIRONMENT: "experiment", + TELEMETRY_DEV_EXPERIMENT_RESULTS: written.results, + }); + expect(written.results!.startsWith(dir)).toBe(true); + expect(statSync(written.results!).mode & 0o777).toBe(0o600); +}); + +test("run.sh runs a .ts script with the tsx of the project, from a directory above", () => { + const project = workdir(); + const evals = join(project, "evals"); + const output = join(project, "output"); + mkdirSync(join(project, "node_modules", ".bin"), { recursive: true }); + mkdirSync(evals); + writeFileSync(output, ""); + // A stand-in for the tsx of the project that records the script it gets. + writeFileSync( + join(project, "node_modules", ".bin", "tsx"), + `#!/bin/sh\necho "$1" > "${join(project, "ran")}"\n`, + { mode: 0o755 }, + ); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: evals, + encoding: "utf8", + env: { + PATH: process.env.PATH, + RUNNER_TEMP: project, + GITHUB_OUTPUT: output, + INPUT_SCRIPT: "reply.eval.ts", + TELEMETRY_DEV_API_KEY: "td_live_job", + }, + }); + + expect(run.status).toBe(0); + expect(readFileSync(join(project, "ran"), "utf8")).toBe("reply.eval.ts\n"); + expect(outputs(output)["exit-code"]).toBe("0"); +}); + +test("run.sh keeps the environment of the job, and runs a .js script with node", () => { + const { run, outputs: written } = runScript( + { + INPUT_SCRIPT: "print-env.mjs", + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }, + { "print-env.mjs": "console.log(JSON.stringify(process.env));\n" }, + ); + + expect(written["exit-code"]).toBe("0"); + expect(JSON.parse(run.stdout)).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }); +}); + +test.each([ + [ + "script and command", + { INPUT_SCRIPT: "a.ts", INPUT_COMMAND: "true", TELEMETRY_DEV_API_KEY: "k" }, + "Give script or command, not the two.", + ], + [ + "no script and no command", + { TELEMETRY_DEV_API_KEY: "k" }, + "Give the script or the command that runs the experiment.", + ], + [ + "no API key", + { INPUT_COMMAND: "true" }, + "Set the api-key input or the TELEMETRY_DEV_API_KEY variable.", + ], + [ + "a script of another type", + { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, + "The script must end in .ts, .js, or .py. For another runner, give a command.", + ], + [ + "a .ts script in a project without tsx", + { INPUT_SCRIPT: "reply.eval.ts", TELEMETRY_DEV_API_KEY: "k" }, + "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command.", + ], +])("run.sh stops with an error for %s", (_, env, message) => { + const { run, outputs: written } = runScript(env); + + expect(run.stdout).toContain(`::error title=telemetry.dev experiment::${message}`); + expect(written["exit-code"]).toBe("2"); +}); + +test("report.mjs writes the summary, posts it on the pull request, and sets the outputs", async () => { + const github = await fakeGithub(); + + const { + code, + summary: written, + outputs: set, + } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(written).toBe(`${summary}\n`); + expect(set).toEqual({ "check-status": "failed", url: result.url }); + expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /user", + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "POST /repos/acme/support/issues/42/comments", + ]); + expect(github.requests[2]!.body).toEqual({ + body: expect.stringMatching( + /^\n### telemetry\.dev experiment\n/, + ), + }); + expect((github.requests[2]!.body as { body: string }).body.endsWith(summary)).toBe(true); +}); + +/** The body of the comment that the action posts for the result. */ +async function commentBody(env: Record = {}): Promise { + const github = await fakeGithub(); + await report({ GITHUB_API_URL: github.url, ...env }, [result]); + + return (github.requests.find((request) => request.method === "POST")!.body as { body: string }) + .body; +} + +function changes(github: Awaited>): string[] { + return github.requests + .filter((request) => request.method !== "GET") + .map((request) => `${request.method} ${request.url}`); +} + +test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { + const own = { id: 8, body: await commentBody(), user: BOT }; + + const other = await fakeGithub({ comments: [own] }); + await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); + + const again = await fakeGithub({ + comments: [{ id: 7, body: "Looks good", user: { login: "dana", type: "User" } }, own], + }); + await report({ GITHUB_API_URL: again.url }, [result]); + + expect(changes(other)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + expect(changes(again)).toEqual(["PATCH /repos/acme/support/issues/comments/8"]); +}); + +test.each([ + ["a person pasted under the GITHUB_TOKEN", undefined, { login: "mallory", type: "User" }], + ["another bot wrote under a user token", { login: "dana" }, BOT], +] as const)( + "report.mjs does not change a comment with its marker that %s", + async (_, user, author) => { + const github = await fakeGithub({ + user, + comments: [{ id: 7, body: await commentBody(), user: author }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + }, +); + +test("report.mjs changes the comment of the user of a user token", async () => { + const github = await fakeGithub({ + user: { login: "dana" }, + comments: [{ id: 7, body: await commentBody(), user: { login: "dana", type: "User" } }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["PATCH /repos/acme/support/issues/comments/7"]); +}); + +test("report.mjs warns and succeeds when the token cannot write the comment", async () => { + const github = await fakeGithub({ status: 403 }); + + const { code, stdout } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(stdout).toContain( + "::warning title=telemetry.dev experiment::The comment on the pull request failed: GitHub returned 403", + ); +}); + +test("report.mjs reports a script that ended before it wrote a result, and comment: false posts nothing", async () => { + const github = await fakeGithub(); + + const { summary: written, outputs: set } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "1" }, + [], + ); + + expect(github.requests).toEqual([]); + + expect(written).toBe( + "### telemetry.dev experiment\n\nThe experiment script exited with code 1 before it wrote a result. The log of the step has the error.\n", + ); + expect(set).toEqual({ "check-status": "none", url: "" }); +}); + +test("report.mjs escapes names, so that they cannot break the table or add HTML", async () => { + const github = await fakeGithub(); + const { summary: written } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "0" }, + [ + { + ...result, + name: "|*bold*", + url: "javascript:alert(1)", + scores: [{ ...result.scores[0], name: "a|b\nc" }], + }, + ], + ); + + expect(written).toContain("**<img src=x>\\|\\*bold\\***: Check failed."); + expect(written).toContain("| a\\|b c | 85% |"); + expect(written).not.toContain("javascript:"); +}); diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts new file mode 100644 index 0000000..5f555e3 --- /dev/null +++ b/packages/sdk/tests/experiments.test.ts @@ -0,0 +1,985 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { + type ExperimentItem, + ExperimentError, + log, + propagateAttributes, + runExperiment, + type RunExperimentOptions, + shutdown, + startActiveSpan, + startSpan, +} from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +interface Request { + method: string; + url: URL; + body: any; + signal: AbortSignal | undefined; +} + +const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; +const URL_OF_EXPERIMENT = `https://telemetry.dev/o/org/p/project/evals/experiments/${EXPERIMENT_ID}`; + +function itemId(number: number): string { + return `00000000-0000-4000-8000-${String(number).padStart(12, "0")}`; +} + +function datasetItem(number: number, split: string | null = null) { + return { + id: itemId(number), + number, + input: { question: `q${number}` } as Record, + expectedOutput: `a${number}`, + metadata: { source: "test" }, + tags: ["golden"], + split, + }; +} + +function result(overrides: Record = {}): Record { + return { + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 2, + itemsDone: 2, + itemsFailed: 0, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + ...overrides, + }; +} + +/** + * The Evals routes of telemetry.dev, in memory: pages of a dataset version with a cursor, an + * experiment that takes item posts of at most 100 and skips repeats, and its result. + */ +function fakeApi( + options: { + items?: ReturnType[]; + pageSize?: number; + results?: Record[]; + override?: (request: Request) => Response | Promise | undefined; + } = {}, +) { + const items = options.items ?? [datasetItem(1), datasetItem(2)]; + const pageSize = options.pageSize ?? 100; + const results = options.results ?? [result()]; + const requests: Request[] = []; + const stored = new Set(); + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = new URL(input); + const request: Request = { + method: init?.method ?? "GET", + url, + body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + signal: init?.signal ?? undefined, + }; + requests.push(request); + + const overridden = options.override?.(request); + + if (overridden) return overridden; + + const route = `${request.method} ${url.pathname}`; + + if (route === "GET /v1/datasets/support-golden") { + const split = url.searchParams.get("split"); + const start = Number(url.searchParams.get("cursor") ?? 0); + const matching = items.filter((item) => split === null || item.split === split); + + return Response.json({ + dataset: { + id: "dataset-1", + name: "support-golden", + version: Number(url.searchParams.get("version") ?? 3), + }, + items: matching.slice(start, start + pageSize), + nextCursor: start + pageSize < matching.length ? String(start + pageSize) : null, + }); + } + + if (route === "POST /v1/experiments") { + const count = items.filter( + (item) => request.body.split === undefined || item.split === request.body.split, + ).length; + + return Response.json( + { + id: EXPERIMENT_ID, + name: request.body.name ?? "support-golden-v3", + datasetVersion: request.body.datasetVersion, + itemsTotal: count * (request.body.runsPerItem ?? 1), + url: URL_OF_EXPERIMENT, + }, + { status: 201 }, + ); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/items`) { + if (request.body.items.length > 100) { + return Response.json( + { error: { code: "invalid_request", message: "at most 100 items" } }, + { status: 400 }, + ); + } + + let accepted = 0; + + for (const run of request.body.items) { + const key = `${run.itemId}:${run.runIndex}`; + + if (!stored.has(key)) accepted += 1; + stored.add(key); + } + + return Response.json({ accepted, duplicates: request.body.items.length - accepted }); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/complete`) { + return Response.json({ id: EXPERIMENT_ID, status: "running" }); + } + + if (route === `GET /v1/experiments/${EXPERIMENT_ID}`) { + return Response.json(results.length > 1 ? results.shift() : results[0]); + } + + return Response.json({ error: { code: "not_found", message: route } }, { status: 404 }); + }); + + const posts = () => + requests + .filter((request) => request.url.pathname.endsWith("/items")) + .map((request) => request.body.items); + + return { fetch: fetch as unknown as typeof globalThis.fetch, requests, posts }; +} + +function routes(requests: Request[]): string[] { + return requests.map((request) => `${request.method} ${request.url.pathname}`); +} + +function experiment( + api: ReturnType, + options: Partial = {}, +): ReturnType { + return runExperiment({ + apiKey: "td_live_test", + baseUrl: "https://ingest.test", + fetch: api.fetch, + dataset: "support-golden", + task: (item) => `answer to ${String(item.input.question)}`, + ...options, + }); +} + +beforeEach(() => { + for (const name of [ + "TELEMETRY_DEV_API_KEY", + "TELEMETRY_DEV_BASE_URL", + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", + ]) { + vi.stubEnv(name, undefined); + } + + vi.spyOn(console, "log").mockImplementation(() => {}); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("runs each item of each page, posts batches of 100 on one version, then completes", async () => { + const api = fakeApi({ + items: Array.from({ length: 75 }, (_, index) => datasetItem(index + 1)), + pageSize: 50, + results: [result({ itemsTotal: 150, itemsDone: 146, itemsFailed: 4 })], + }); + + const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); + + const sent = routes(api.requests); + expect(sent.slice(0, 2)).toEqual(["GET /v1/datasets/support-golden", "POST /v1/experiments"]); + // The second page and the first post of items run at the same time, in either order. + expect(sent.slice(2, -2).sort()).toEqual([ + "GET /v1/datasets/support-golden", + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + ]); + expect(sent.slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + `GET /v1/experiments/${EXPERIMENT_ID}`, + ]); + + // The first page names no version; the next one asks for the version of the first. + const pages = api.requests.filter((request) => request.url.pathname.startsWith("/v1/datasets/")); + expect(Object.fromEntries(pages[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(pages[1]!.url.searchParams)).toEqual({ version: "3", cursor: "50" }); + + expect(api.requests[1]!.body).toEqual({ + dataset: "support-golden", + datasetVersion: 3, + evaluators: ["Correctness"], + runsPerItem: 2, + }); + + const posts = api.posts(); + expect(posts.map((items) => items.length)).toEqual([100, 50]); + + const runs = posts + .flat() + .map((run: { itemId: string; runIndex: number }) => `${run.itemId}:${run.runIndex}`); + expect(new Set(runs).size).toBe(150); + expect(runs).toContain(`${itemId(75)}:1`); + + expect(posts[0]![0]).toEqual({ + itemId: itemId(1), + runIndex: 0, + output: "answer to q1", + latencyMs: expect.any(Number), + }); + + expect(outcome).toEqual({ + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 150, + itemsDone: 146, + itemsFailed: 4, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + }); +}); + +test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { + const { spans, logs } = setup({ environment: "experiment", sessionMode: "process" }); + const api = fakeApi(); + + // An active span, an ambient session, and the process session would each put the runs in one + // trace. + await propagateAttributes({ sessionId: "ambient-session", userId: "u-1" }, () => + startActiveSpan("eval script", () => + experiment(api, { + runsPerItem: 2, + task: (item) => { + startSpan("chat gpt-4o", { type: "generation" }).end(); + log("replied", { attributes: { number: item.number } }); + + return { reply: `r${item.number}` }; + }, + }), + ), + ); + + const posted = api.posts().flat(); + const finished = spans.getFinishedSpans(); + const chats = finished.filter((span) => span.name === "chat gpt-4o"); + const roots = finished.filter((span) => span.name === "experiment support-golden-v3"); + + expect(chats).toHaveLength(4); + expect(roots).toHaveLength(4); + expect(new Set(roots.map((span) => span.spanContext().traceId)).size).toBe(4); + expect(roots.map((span) => span.parentSpanContext?.isRemote)).toEqual([true, true, true, true]); + + for (const run of posted) { + expect(run.traceId).toMatch(/^[0-9a-f]{32}$/); + const root = roots.find((span) => span.spanContext().traceId === run.traceId)!; + const chat = chats.find((span) => span.spanContext().traceId === run.traceId)!; + + expect(chat.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + + for (const span of [root, chat]) { + expect(span.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + "gen_ai.conversation.id": `${EXPERIMENT_ID}:${run.itemId}:${run.runIndex}`, + "user.id": "u-1", + }); + } + } + + const records = logs.getFinishedLogRecords(); + expect(records).toHaveLength(4); + + for (const record of records) { + const run = posted.find( + (candidate: { traceId: string }) => candidate.traceId === record.spanContext?.traceId, + ); + + expect(record.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + }); + } + + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ]); +}); + +test("runs at most `concurrency` tasks at a time", async () => { + const api = fakeApi({ items: Array.from({ length: 12 }, (_, index) => datasetItem(index + 1)) }); + let running = 0; + let most = 0; + + await experiment(api, { + concurrency: 3, + task: async () => { + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, 5)); + running -= 1; + + return "ok"; + }, + }); + + expect(most).toBe(3); + expect(api.posts().flat()).toHaveLength(12); +}); + +test("a task that throws fails only its item", async () => { + const api = fakeApi(); + + await experiment(api, { + task: (item) => { + if (item.number === 1) throw new RangeError("no reply"); + + return "fine"; + }, + }); + + const [failed, passed] = api.posts().flat(); + + expect(failed).toMatchObject({ itemId: itemId(1), output: null, error: "RangeError: no reply" }); + expect(failed).not.toHaveProperty("scores"); + expect(passed).toMatchObject({ itemId: itemId(2), output: "fine" }); + expect(passed).not.toHaveProperty("error"); +}); + +test("an output or an error that cannot be encoded does not stop the run", async () => { + const api = fakeApi({ items: [datasetItem(1), datasetItem(2), datasetItem(3)] }); + const cycle: Record = {}; + cycle.self = cycle; + const unreadable = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("no message"); + }, + }); + + const outcome = await experiment(api, { + task: (item) => { + if (item.number === 3) throw unreadable; + + return item.number === 1 ? cycle : 10n; + }, + // A lone surrogate, which UTF-8 cannot hold. + scorers: { tone: () => "\ud83d" }, + }); + + expect(outcome.status).toBe("completed"); + expect( + api + .posts() + .flat() + .sort((a: { itemId: string }, b: { itemId: string }) => a.itemId.localeCompare(b.itemId)) + .map((run: { output: unknown; error?: unknown; scores?: unknown }) => [ + run.output, + run.error, + run.scores, + ]), + ).toEqual([ + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, "A value that cannot be converted to a string was thrown", undefined], + ]); + expect(routes(api.requests)).toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("each run of an item starts from the dataset input, also when a task changes it", async () => { + const api = fakeApi({ + items: [{ ...datasetItem(1), input: { messages: [{ role: "user", content: "Hi" }] } }], + }); + const seen: number[] = []; + + await experiment(api, { + runsPerItem: 3, + concurrency: 1, + task: (item) => { + const messages = item.input.messages as unknown[]; + seen.push(messages.length); + messages.push({ role: "assistant", content: "Hello" }); + + return "ok"; + }, + }); + + expect(seen).toEqual([1, 1, 1]); +}); + +test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ items: [datasetItem(1)] }); + + await experiment(api, { + scorers: { + exact: (output, item) => output === `answer to ${String(item.input.question)}`, + polite: () => false, + length: (output) => String(output).length, + tone: () => " friendly ", + judged: () => ({ value: 0.25, reason: "too short" }), + none: () => null, + thrown: () => { + throw new Error("judge down"); + }, + mismatched: () => ({ value: 0.5, label: "pass" }), + }, + }); + + expect(api.posts()[0]![0].scores).toEqual([ + { name: "exact", value: 1, label: "pass" }, + { name: "polite", value: 0, label: "fail" }, + { name: "length", value: 12 }, + { name: "tone", label: "friendly" }, + { name: "judged", value: 0.25, reason: "too short" }, + ]); + + expect(errors.map((error) => [(error as ExperimentError).code, error.message])).toEqual([ + ["scorer_failed", "The scorer thrown failed on item 1: Error: judge down"], + [ + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not valid: The label pass goes with the value 1", + ], + ]); +}); + +test("a failed post of items stops new items, completes the experiment, and rejects with the API error", async () => { + const api = fakeApi({ + items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), + override: (request) => + request.url.pathname.endsWith("/items") + ? Response.json( + { error: { code: "item_not_found", message: "support-golden v3 has no item" } }, + { status: 422 }, + ) + : undefined, + }); + let started = 0; + + const error = await experiment(api, { + concurrency: 1, + task: () => { + started += 1; + + return "ok"; + }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ExperimentError); + expect(error).toMatchObject({ code: "item_not_found", status: 422 }); + // The one worker waits for the post of the first 100, which fails, so no item starts after it. + expect(started).toBe(100); + // The complete ends the experiment, which would else stay running. + expect(routes(api.requests).slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + ]); +}); + +test("a complete that fails after a failed run goes to onError, and the run rejects with the first error", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") || request.url.pathname.endsWith("/complete") + ? Response.json( + { error: { code: "invalid_api_key", message: "The API key was revoked" } }, + { status: 401 }, + ) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "invalid_api_key", + message: "The API key was revoked", + }); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); + expect(errors.map((error) => (error as ExperimentError).code)).toEqual(["invalid_api_key"]); +}); + +test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { + let failures = 1; + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") && failures-- > 0 + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await experiment(api); + + expect(api.posts()).toHaveLength(2); + expect(api.posts()[1]).toEqual(api.posts()[0]); +}); + +test("a post of items holds at most 4 MiB of JSON", async () => { + const api = fakeApi({ items: Array.from({ length: 20 }, (_, index) => datasetItem(index + 1)) }); + + // 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + await experiment(api, { task: () => "中".repeat(100_000) }); + + expect(api.posts().map((items) => items.length)).toEqual([13, 7]); +}); + +test.each([ + ["10,000 items in pages run", 10_000, undefined], + ["10,001 items in pages stop the run", 10_001, "invalid_response"], +])("%s", async (_, count, code) => { + const api = fakeApi({ + items: Array.from({ length: count }, (_, index) => datasetItem(index + 1)), + pageSize: 5_000, + }); + + const outcome = await experiment(api, { concurrency: 20, task: () => "ok" }).catch( + (caught: unknown) => caught, + ); + + expect((outcome as ExperimentError).code).toBe(code); + expect(api.posts().flat()).toHaveLength(10_000); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); +}); + +test.each([ + ["a dataset page", "GET /v1/datasets/support-golden", "timeout", 2, undefined], + ["an item post", `POST /v1/experiments/${EXPERIMENT_ID}/items`, "network", 2, undefined], + ["the create", "POST /v1/experiments", "timeout", 1, "timeout"], + ["the create", "POST /v1/experiments", "network", 1, "network_error"], +])("a %s %s, after a %s, sends %i times in all", async (_, route, failure, sends, code) => { + vi.useFakeTimers(); + + try { + let failures = 1; + const api = fakeApi({ + override: (request) => { + if (`${request.method} ${request.url.pathname}` !== route || failures-- <= 0) { + return undefined; + } + + if (failure === "network") return Promise.reject(new TypeError("fetch failed")); + + // A server that never answers: the request ends when its timeout aborts it. + return new Promise((_, reject) => + request.signal!.addEventListener("abort", () => reject(request.signal!.reason)), + ); + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(60_000); + + expect(((await outcome) as ExperimentError).code).toBe(code); + expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(sends); + } finally { + vi.useRealTimers(); + } +}); + +test.each([ + ["500", () => new Response("upstream", { status: 500 }), "http_500"], + // The server answers 503 when its database work times out, which can be after the commit. + [ + "503", + () => + Response.json( + { error: { code: "evals_unavailable", message: "Evals are unavailable right now" } }, + { status: 503 }, + ), + "evals_unavailable", + ], +])( + "a create that fails with %s does not go again, so it cannot start two experiments", + async (status, reply, code) => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? reply() + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code, status: Number(status) }); + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + ]); + }, +); + +test("a create that gets a 429 goes again after its Retry-After", async () => { + let limited = 1; + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" && limited-- > 0 + ? Response.json( + { error: { code: "rate_limited", message: "Too many requests for this project" } }, + { status: 429, headers: { "retry-after": "2" } }, + ) + : undefined, + }); + const started = Date.now(); + + await experiment(api); + + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(2); + // Without the header, the first retry waits 1 second. + expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); +}); + +test("waits until the server evaluators end the experiment", async () => { + const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] }); + + const outcome = await experiment(api); + + expect(outcome.status).toBe("completed"); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/experiments/")), + ).toHaveLength(2); +}); + +test("gives up when the experiment does not end in 30 minutes, with the result", async () => { + vi.useFakeTimers(); + + try { + const polls: number[] = []; + const api = fakeApi({ + override: (request) => { + if (request.method === "GET" && request.url.pathname.startsWith("/v1/experiments/")) { + polls.push(Date.now()); + + return Response.json(result({ status: "running", itemsDone: 1 })); + } + + return undefined; + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(31 * 60_000); + const error = (await outcome) as ExperimentError; + + expect(error.code).toBe("timeout"); + expect(error.result?.status).toBe("running"); + // The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + const waited = polls.at(-1)! - polls[0]!; + expect(waited).toBeGreaterThan(30 * 60_000 - 5_000); + expect(waited).toBeLessThanOrEqual(30 * 60_000); + } finally { + vi.useRealTimers(); + } +}); + +test.each([ + [ + "the API error", + { error: { code: "dataset_not_found", message: "No dataset named support-golden" } }, + 404, + "dataset_not_found", + ], + [ + "a page that cannot be read", + { dataset: { version: 1 }, items: [{ id: 1 }] }, + 200, + "invalid_response", + ], + [ + "an empty dataset", + { dataset: { id: "d", name: "support-golden", version: 0 }, items: [], nextCursor: null }, + 200, + "dataset_empty", + ], +])("rejects with %s before it creates an experiment", async (_, body, status, code) => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json(body, { status }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code }); + expect(routes(api.requests)).toEqual(["GET /v1/datasets/support-golden"]); +}); + +test("a cursor that repeats is an invalid response, not an endless fetch", async () => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json({ + dataset: { id: "d", name: "support-golden", version: 3 }, + items: [datasetItem(1)], + nextCursor: "same", + }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/datasets/")), + ).toHaveLength(2); +}); + +const failedCheck = result({ + checkStatus: "failed", + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 17, + failCount: 3, + baselineValue: 0.9, + delta: -5, + minScore: 0.9, + passed: false, + }, + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ], +}); + +test("in CI mode a failed check rejects with the result", async () => { + vi.stubEnv("CI", "true"); + const api = fakeApi({ results: [failedCheck] }); + + const error = (await experiment(api, { minScores: { correctness: 0.9 } }).catch( + (caught: unknown) => caught, + )) as ExperimentError; + + expect(error).toBeInstanceOf(ExperimentError); + expect(error.code).toBe("check_failed"); + expect(error.message).toBe( + `The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. ${URL_OF_EXPERIMENT}`, + ); + expect(error.result?.scores[1]?.kind).toBeNull(); +}); + +test.each([ + ["CI is off", {}, undefined], + ["ci: false wins over CI", { ci: false }, "true"], +])("a failed check resolves when %s", async (_, options, ci) => { + if (ci) vi.stubEnv("CI", ci); + const api = fakeApi({ results: [failedCheck] }); + + await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); +}); + +test("a result score that leaves out its null fields reads them as null", async () => { + const api = fakeApi({ + results: [result({ scores: [{ name: "length", passCount: 0, failCount: 0 }] })], + }); + + const outcome = await experiment(api); + + expect(outcome.scores).toEqual([ + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ]); +}); + +test("a result without checkStatus is an invalid response, not a run with no check", async () => { + const { checkStatus: _, ...missing } = failedCheck; + const api = fakeApi({ results: [missing] }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); +}); + +test("in CI mode an experiment that fails rejects with its error", async () => { + vi.stubEnv("CI", "1"); + const api = fakeApi({ + results: [result({ status: "failed", error: "0 of 2 items ran. Each one failed." })], + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "experiment_failed", + message: `The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. ${URL_OF_EXPERIMENT}`, + }); +}); + +test("in GitHub Actions the experiment records the pull request, its head commit, and the run", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + const results = join(dir, "results.jsonl"); + writeFileSync( + event, + JSON.stringify({ + pull_request: { + number: 42, + title: "Shorter replies", + html_url: "https://github.com/acme/support/pull/42", + head: { sha: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" }, + }, + }), + ); + writeFileSync(results, ""); + + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "ffffffffffffffffffffffffffffffffffffffff"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + vi.stubEnv("GITHUB_RUN_ID", "991"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5,__proto__=1"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); + const api = fakeApi(); + + await experiment(api, { dataset: undefined, minScores: { correctness: 0.5, tone: 2 } }); + + expect(api.requests[1]!.body).toMatchObject({ + dataset: "support-golden", + minScores: JSON.parse('{"correctness":0.9,"tone":2,"length":12.5,"__proto__":1}'), + ci: { + provider: "github_actions", + repository: "acme/support", + commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + prNumber: 42, + prTitle: "Shorter replies", + prUrl: "https://github.com/acme/support/pull/42", + runUrl: "https://github.com/acme/support/actions/runs/991", + }, + }); + expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); +}); + +test("a pull request title is cut to 256 UTF-16 units without half of an emoji", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + // GitHub counts the emoji as one character, thus the title fits its limit of 256. + writeFileSync(event, JSON.stringify({ pull_request: { title: `${"a".repeat(255)}😀` } })); + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "a1b2c3d"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + const api = fakeApi(); + + await experiment(api); + + expect(api.requests[1]!.body.ci.prTitle).toBe("a".repeat(255)); +}); + +test.each([ + [ + "no dataset", + {}, + { dataset: undefined }, + "Pass dataset, or set the dataset input of the experiment action", + ], + [ + "a dataset that the action does not name", + { TELEMETRY_DEV_EXPERIMENT_DATASET: "support-v2" }, + {}, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ], + [ + "min-scores that do not parse", + { TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" }, + {}, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" is not a name=number pair', + ], + ["runsPerItem 6", {}, { runsPerItem: 6 }, "runsPerItem must be an integer from 1 to 5"], + ["concurrency 0", {}, { concurrency: 0 }, "concurrency must be an integer from 1 to 100"], + [ + "a scorer named user_feedback", + {}, + { scorers: { user_feedback: () => 1 } }, + "user_feedback is the score of user feedback. Name the scorer another way.", + ], + [ + "a scorer with the name of an evaluator", + {}, + { evaluators: ["correctness"], scorers: { Correctness: () => true } }, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ], +])("rejects %s before any request", async (_, env, options, message) => { + for (const [name, value] of Object.entries(env)) vi.stubEnv(name, value); + const api = fakeApi(); + + await expect(experiment(api, options as Partial)).rejects.toThrow(message); + expect(api.requests).toEqual([]); +}); + +test("without an API key it rejects with missing_api_key and sends nothing", async () => { + const api = fakeApi(); + + await expect(experiment(api, { apiKey: undefined })).rejects.toMatchObject({ + code: "missing_api_key", + }); + expect(api.requests).toEqual([]); +}); + +test("the task gets each run of an item of the split, with its run index", async () => { + const seen: ExperimentItem[] = []; + const api = fakeApi({ items: [datasetItem(7, "test")] }); + + await experiment(api, { + split: "test", + runsPerItem: 2, + concurrency: 1, + task: (item) => { + seen.push(item); + + return "ok"; + }, + }); + + expect(seen).toEqual([ + { ...datasetItem(7, "test"), runIndex: 0 }, + { ...datasetItem(7, "test"), runIndex: 1 }, + ]); + expect(api.requests[0]!.url.searchParams.get("split")).toBe("test"); + expect(api.requests[1]!.body.split).toBe("test"); +}); diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index 6048d16..d75d116 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -1,16 +1,22 @@ import { createServer, type RequestListener } from "node:http"; import type { AddressInfo } from "node:net"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { + type ChatMessage, flush, getPrompt, + type Prompt, + promptAttributesFor, propagateAttributes, PromptError, shutdown, startActiveSpan, startSpan, + updateActiveSpan, withPrompt, } from "../src/index.ts"; import { setup } from "./helpers.ts"; @@ -107,6 +113,7 @@ afterEach(async () => { await shutdown(); vi.useRealTimers(); vi.unstubAllEnvs(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); }); @@ -729,6 +736,736 @@ test("variables are omitted when input capture is off; a version fetch has no la expect(off.spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBeUndefined(); }); +async function promptOf(overrides: Record): Promise { + const { fetch } = fakeFetch(reply(overrides)); + + return getPrompt("support/reply", { + apiKey: "td_live_a", + baseUrl: newBaseUrl(), + fetch, + type: overrides.type as "chat", + }); +} + +const DESK_PROMPT = { + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "user", content: "Order {{order}}: where is it?" }, + ], +}; + +test("promptAttributesFor names the compile of a message object; a copy or a string names nothing", async () => { + setup(); + const prompt = await promptOf(DESK_PROMPT); + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const attributes = { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }; + + expect(promptAttributesFor(compiled)).toEqual(attributes); + expect(promptAttributesFor(compiled[0])).toEqual(attributes); + expect(JSON.stringify(compiled[0])).toBe( + '{"role":"system","content":"Answer as the billing desk."}', + ); + + for (const copy of [ + compiled.map((message) => ({ ...message })), + JSON.parse(JSON.stringify(compiled)) as unknown, + structuredClone(compiled), + compiled.map((message) => message.content), + ]) { + expect(promptAttributesFor(copy)).toBeUndefined(); + } + + const text = await promptOf({ type: "text", messages: [], text: "Summarize {{doc}}." }); + expect(promptAttributesFor([text.compile({ doc: "the policy" })])).toBeUndefined(); +}); + +test("an unrelated message with the text of a compiled message is not tagged", async () => { + setup(); + const prompt = await promptOf({ + messages: [{ role: "system", content: "You are a helpful assistant." }], + }); + prompt.compile(); + + expect( + promptAttributesFor([ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is 2 + 2?" }, + ]), + ).toBeUndefined(); +}); + +test("two tasks with one text and different other variables each record their own", async () => { + const { spans } = setup(); + const prompt = await promptOf(DESK_PROMPT); + + // Both compiles make the system text "Answer as the billing desk.", and each task sends only it. + const [systemA] = prompt.compile({ team: "billing", order: "#A" }) as ChatMessage[]; + const [systemB] = prompt.compile({ team: "billing", order: "#B" }) as ChatMessage[]; + startSpan("a", { type: "generation", attributes: promptAttributesFor([systemA]) }).end(); + startSpan("b", { type: "generation", attributes: promptAttributesFor([systemB]) }).end(); + await flush(); + + expect( + Object.fromEntries( + spans.getFinishedSpans().map((span) => [span.name, span.attributes["td.prompt.variables"]]), + ), + ).toEqual({ + a: '{"team":"billing","order":"#A"}', + b: '{"team":"billing","order":"#B"}', + }); +}); + +test("a rollout with two versions of the same messages records the version of each call", async () => { + setup(); + const { fetch } = fakeFetch( + reply({ + ...DESK_PROMPT, + rollout: { + percent: 16, + version: 13, + ...DESK_PROMPT, + text: null, + model: { provider: "openai", name: "gpt-4o-mini" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + }, + }), + ); + const options = { apiKey: "td_live_a", baseUrl: newBaseUrl(), fetch }; + vi.spyOn(Math, "random").mockReturnValueOnce(0.159).mockReturnValueOnce(0.16); + + const rollout = await getPrompt("support/reply", options); + const production = await getPrompt("support/reply", options); + const fromRollout = rollout.compile({ team: "billing", order: "#4" }); + const fromProduction = production.compile({ team: "billing", order: "#4" }); + + expect(promptAttributesFor(fromRollout)?.["td.prompt.version"]).toBe(13); + expect(promptAttributesFor(fromProduction)?.["td.prompt.version"]).toBe(12); +}); + +test("the newest compiled message wins, also after 170 messages of history", async () => { + setup(); + const prompt = await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], + }); + const history = Array.from({ length: 170 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: `message ${index}`, + })); + + const firstTurn = prompt.compile({ + team: "billing", + question: "Where is #4?", + history, + }) as ChatMessage[]; + const answer = { role: "assistant", content: "On its way." }; + const secondTurn = prompt.compile({ + team: "returns", + question: "Can I return it?", + }) as ChatMessage[]; + const chat = [...firstTurn, answer, ...secondTurn.slice(1)]; + + expect(promptAttributesFor(firstTurn)?.["td.prompt.variables"]).toContain('"team":"billing"'); + expect(promptAttributesFor([firstTurn[0], ...history])?.["td.prompt.variables"]).toContain( + '"team":"billing"', + ); + expect(promptAttributesFor(chat)?.["td.prompt.variables"]).toBe( + '{"team":"returns","question":"Can I return it?"}', + ); +}); + +test("a message that a placeholder brings from an older compile does not win", async () => { + setup(); + const router = await promptOf({ + name: "router", + version: 1, + messages: [{ role: "user", content: "{{question}}" }], + }); + const answer = await promptOf({ + name: "answer", + version: 7, + messages: [ + { role: "system", content: "Answer for {{team}}." }, + { role: "placeholder", name: "history" }, + ], + }); + const chat = await promptOf({ + name: "chat", + messages: [{ role: "user", content: "{{question}}" }], + }); + const summary = await promptOf({ + name: "summary", + messages: [ + { role: "system", content: "Summarize the chat." }, + { role: "placeholder", name: "history" }, + ], + }); + + const question = router.compile({ question: "Where is #4?" }) as ChatMessage[]; + const routed = answer.compile({ team: "billing", history: question }); + // The last 1000 messages are all from chat; the summary message is the first. + const conversation = Array.from( + { length: 1500 }, + (_, index) => chat.compile({ question: `question ${index}` }) as ChatMessage[], + ).flat(); + const summarized = summary.compile({ history: conversation }); + + expect(promptAttributesFor(routed)).toEqual({ + "td.prompt.name": "answer", + "td.prompt.version": 7, + "td.prompt.label": "production", + "td.prompt.variables": + '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}', + }); + expect(promptAttributesFor(summarized)?.["td.prompt.name"]).toBe("summary"); +}); + +test("promptAttributesFor looks at the first and the last 1000 messages", async () => { + setup(); + const prompt = await promptOf(DESK_PROMPT); + const [system] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const filler = (count: number) => Array.from({ length: count }, () => ({ role: "user" })); + + expect(promptAttributesFor([...filler(1000), system, ...filler(999)])).toBeDefined(); + expect(promptAttributesFor([...filler(999), system, ...filler(1001)])).toBeDefined(); + expect(promptAttributesFor([...filler(1001), system, ...filler(999)])).toBeDefined(); + expect(promptAttributesFor([...filler(1000), system, ...filler(1000)])).toBeUndefined(); +}); + +test("compile records its variables, not later changes of the object or of its lists", async () => { + const { spans } = setup(); + const prompt = await promptOf({ + messages: [ + { role: "placeholder", name: "history" }, + { role: "user", content: "Order {{order}}" }, + ], + }); + const history: ChatMessage[] = [{ role: "user", content: "Hi" }]; + const variables = { order: "#A", history }; + + const first = prompt.compile(variables); + variables.order = "#B"; + history.push({ role: "assistant", content: "Hello" }); + const second = prompt.compile(variables); + variables.order = "#C"; + history.push({ role: "user", content: "Added after compile" }); + withPrompt(prompt, () => startSpan("scope", { type: "generation" }).end()); + await flush(); + + const secondVariables = + '{"order":"#B","history":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Hello"}]}'; + expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe( + '{"order":"#A","history":[{"role":"user","content":"Hi"}]}', + ); + expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe(secondVariables); + expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBe(secondVariables); +}); + +// A WeakRef keeps its target until the current job ends, so collect in a new one. +async function collect(): Promise { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); +} + +const RAG_PROMPT = { + messages: [ + { role: "system", content: "Answer from {{documents}}." }, + { role: "user", content: "{{question}}" }, + ], +}; + +test("the variables of a compile are freed with its messages", async () => { + setup(); + const prompt = await promptOf(RAG_PROMPT); + + let document: ChatMessage | undefined = { role: "user", content: "The refund policy" }; + const freed = new WeakRef(document); + let messages = prompt.compile({ documents: [document], question: "q" }) as + | ChatMessage[] + | undefined; + document = undefined; + // withPrompt keeps the variables of the last compile of a prompt object. + prompt.compile({ documents: [], question: "other" }); + await collect(); + expect(freed.deref()).toBeDefined(); + + expect(messages).toHaveLength(2); + messages = undefined; + await collect(); + expect(freed.deref()).toBeUndefined(); +}); + +// A row removes the global WeakRef, so the test keeps its own. +const TrackedRef = WeakRef; + +test.each<[string, () => void]>([ + ["input capture is off", () => setup({ captureInput: false })], + [ + "the runtime has no WeakRef (Cloudflare Workers before 2025-05-05)", + () => { + setup(); + vi.stubGlobal("WeakRef", undefined); + }, + ], +])("when %s, compiled messages keep no variables", async (_case, start) => { + start(); + const prompt = await promptOf(RAG_PROMPT); + const documents: WeakRef[] = []; + const questions: ChatMessage[] = []; + + for (let turn = 0; turn < 200; turn += 1) { + const document: ChatMessage = { role: "user", content: `document ${turn}` }; + documents.push(new TrackedRef(document)); + const compiled = prompt.compile({ documents: [document], question: `question ${turn}` }); + questions.push(compiled.at(-1) as ChatMessage); + } + await collect(); + + // withPrompt keeps the variables of the last compile of a prompt object. + expect(documents.filter((document) => document.deref()).length).toBe(1); + + await shutdown(); + vi.unstubAllGlobals(); + setup(); + expect(promptAttributesFor(questions)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); +}); + +const CHAT_PROMPT = { + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], +}; + +interface ChatPrompts { + chat: Prompt; + question: Prompt; + rag: Prompt; + reply: Prompt; +} + +async function chatPrompts(): Promise { + return { + chat: await promptOf(CHAT_PROMPT), + question: await promptOf({ + messages: [{ role: "user", content: "Context: {{documents}}\n\n{{question}}" }], + }), + rag: await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "placeholder", name: "current" }, + ], + }), + reply: await promptOf({ messages: [{ role: "assistant", content: "{{answer}}" }] }), + }; +} + +function answer(turn: number): ChatMessage { + return { role: "assistant", content: `answer ${turn}` }; +} + +// A row runs one turn of a chat. It puts `marker` in the variables of the compile that made the +// message that the chat keeps. It gives the request, the history for the next turn, and that message. +type ChatTurn = ( + prompts: ChatPrompts, + history: ChatMessage[], + turn: number, + marker: ChatMessage, +) => [request: ChatMessage[], history: ChatMessage[], kept: ChatMessage]; + +test.each<[string, ChatTurn]>([ + [ + "a new list each turn", + ({ chat }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)], kept]; + }, + ], + [ + "a new list of the last 20 messages", + ({ chat }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)].slice(-20), kept]; + }, + ], + [ + "a full list that compile gets as copies", + ({ chat }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history: history.map((message) => ({ ...message })), + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)], kept]; + }, + ], + [ + "a question prompt for each turn, in a placeholder of the chat prompt", + ({ question, rag }, history, turn, marker) => { + const [kept] = question.compile({ + documents: `document ${turn}`, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const request = rag.compile({ team: "billing", history, current: [kept!] }) as ChatMessage[]; + + return [request, [...history, kept!, answer(turn)], kept!]; + }, + ], + [ + "raw questions and a reply prompt for each answer", + ({ chat, reply }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + }) as ChatMessage[]; + const [kept] = reply.compile({ answer: `answer ${turn}`, marker: [marker] }) as ChatMessage[]; + + return [request, [...history, { role: "user", content: `question ${turn}` }, kept!], kept!]; + }, + ], +])("a chat history in %s keeps the variables of the last 5 minutes", async (_shape, run) => { + vi.useFakeTimers({ toFake: ["Date", "performance"] }); + setup(); + const prompts = await chatPrompts(); + const markers: WeakRef[] = []; + let history: ChatMessage[] = []; + let request: ChatMessage[] = []; + let older: ChatMessage | undefined; + + // One turn in 2 minutes, for 300 turns. An unused variable marks the variables of each turn. + for (let turn = 0; turn < 300; turn += 1) { + if (turn > 0) vi.advanceTimersByTime(120_000); + const marker: ChatMessage = { role: "user", content: `turn ${turn}` }; + markers.push(new WeakRef(marker)); + let kept: ChatMessage; + [request, history, kept] = run(prompts, history, turn, marker); + // The model call of the turn. + promptAttributesFor(request); + + if (turn === 296) older = kept; + } + await collect(); + + // Turns 297, 298, and 299: 4, 2, and 0 minutes old. + expect(markers.filter((marker) => marker.deref()).length).toBe(3); + expect(promptAttributesFor(request)?.["td.prompt.variables"]).toContain("question 299"); + expect(promptAttributesFor(older)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); +}); + +function variablesOf(messages: unknown): unknown { + return promptAttributesFor(messages)?.["td.prompt.variables"]; +} + +const REFUND_VARIABLES = '{"documents":"The refund policy","question":"Where is #4?"}'; + +interface LivePrompts { + answer: Prompt; + guard: Prompt; + judge: Prompt; + other: Prompt; + chat: Prompt; + system: Prompt; + withSystem: Prompt; +} + +// A row compiles one or more requests and runs other compiles before their calls. It gives the +// variables that each call records and the variables of the compile of its request. +type LiveRequests = (prompts: LivePrompts) => Promise<[recorded: unknown, compiled: string][]>; + +test.each<[string, LiveRequests]>([ + [ + "150 compiles of other requests, then a guard", + async ({ answer, guard, other }) => { + const request = answer.compile({ documents: "The refund policy", question: "Where is #4?" }); + for (let index = 0; index < 150; index += 1) other.compile({ q: `${index}` }); + guard.compile({ conversation: request }); + + return [[variablesOf(request), REFUND_VARIABLES]]; + }, + ], + [ + "a guard before each of 150 agent steps", + async ({ answer, guard }) => { + const conversation = answer.compile({ + documents: "The refund policy", + question: "Where is #4?", + }) as ChatMessage[]; + + for (let step = 0; step < 150; step += 1) { + guard.compile({ conversation }); + conversation.push({ role: "assistant", content: `step ${step}` }); + } + + return [[variablesOf(conversation), REFUND_VARIABLES]]; + }, + ], + [ + "a guard, then a judge of the messages of the guard", + async ({ answer, guard, judge }) => { + const request = answer.compile({ documents: "The refund policy", question: "Where is #4?" }); + judge.compile({ conversation: guard.compile({ conversation: request }) }); + + return [[variablesOf(request), REFUND_VARIABLES]]; + }, + ], + [ + "three turns of a session that add their question to the history before their calls", + async ({ chat }) => { + const history: ChatMessage[] = []; + let open!: () => void; + const queue = new Promise((resolve) => { + open = resolve; + }); + const turn = async (question: string) => { + const request = chat.compile({ team: "billing", history, question }) as ChatMessage[]; + history.push(request.at(-1)!); + await queue; + + return variablesOf(request); + }; + const turns = ["q1", "q2", "q3"].map(turn); + open(); + const [first, second, third] = await Promise.all(turns); + + return [ + [first, '{"team":"billing","history":[],"question":"q1"}'], + [second, '{"team":"billing","history":[{"role":"user","content":"q1"}],"question":"q2"}'], + [ + third, + '{"team":"billing","history":[{"role":"user","content":"q1"},{"role":"user","content":"q2"}],"question":"q3"}', + ], + ]; + }, + ], + [ + "a system message compiled at startup, after 3 turns of a chat that gets it", + async ({ system, withSystem }) => { + const startup = system.compile({ product: "Acme" }) as ChatMessage[]; + let history: ChatMessage[] = []; + + for (let turn = 0; turn < 3; turn += 1) { + const request = withSystem.compile({ + system: startup, + history, + question: `question ${turn}`, + }) as ChatMessage[]; + promptAttributesFor(request); + history = [...history, request.at(-1)!, answer(turn)]; + } + + return [ + [ + variablesOf([...startup, { role: "user", content: "Where is #4?" }]), + '{"product":"Acme"}', + ], + ]; + }, + ], +])("a request keeps its variables until its call: %s", async (_case, run) => { + setup(); + const calls = await run({ + answer: await promptOf(RAG_PROMPT), + guard: await promptOf({ + name: "guard", + messages: [ + { role: "system", content: "Is this safe?" }, + { role: "placeholder", name: "conversation" }, + ], + }), + judge: await promptOf({ + name: "judge", + messages: [ + { role: "system", content: "Check the answer of the guard." }, + { role: "placeholder", name: "conversation" }, + ], + }), + other: await promptOf({ name: "other", messages: [{ role: "user", content: "{{q}}" }] }), + chat: await promptOf(CHAT_PROMPT), + system: await promptOf({ + name: "system", + messages: [{ role: "system", content: "You support {{product}}." }], + }), + withSystem: await promptOf({ + messages: [ + { role: "placeholder", name: "system" }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], + }), + }); + + for (const [recorded, compiled] of calls) expect(recorded).toBe(compiled); +}); + +test("a compile keeps its variables for 5 minutes after it or after the last call that names it", async () => { + vi.useFakeTimers({ toFake: ["Date", "performance"] }); + setup(); + const chat = await promptOf(CHAT_PROMPT); + let history: ChatMessage[] = []; + + for (let turn = 0; turn < 5; turn += 1) { + const request = chat.compile({ team: "billing", history, question: `q${turn}` }); + promptAttributesFor(request); + history = [...history, request.at(-1) as ChatMessage, answer(turn)]; + } + // These calls send only messages of the chat, so they name the newest compile among them. + const summary = [...history, { role: "user", content: "Summarize the chat." }]; + const earlier = history.slice(0, -2); + + expect(variablesOf(summary)).toContain('"question":"q4"'); + expect(variablesOf(earlier)).toContain('"question":"q3"'); + + vi.advanceTimersByTime(299_999); + expect(variablesOf(summary)).toContain('"question":"q4"'); + vi.advanceTimersByTime(1); + expect(variablesOf(earlier)).toBeUndefined(); + // The call 1 ms before moved the deadline of q4. + expect(variablesOf(summary)).toContain('"question":"q4"'); + // The system clock moves back an hour; 5 minutes still pass. + vi.setSystemTime(Date.now() - 3_600_000); + vi.advanceTimersByTime(300_000); + expect(promptAttributesFor(summary)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); +}); + +test("withPrompt wins over a compiled message of another prompt, through a parent and in updates", async () => { + const { spans } = setup(); + const scoped = await promptOf({ name: "scoped/prompt" }); + // Fetched by version, so it has no label that the label of the scope could hide. + const { fetch } = fakeFetch( + reply({ + name: "other/prompt", + label: null, + version: 3, + messages: [{ role: "user", content: "Cancel order {{order}}." }], + }), + ); + const other = await getPrompt("other/prompt", { + apiKey: "td_live_a", + baseUrl: newBaseUrl(), + version: 3, + fetch, + }); + const compiled = other.compile({ order: "#9" }); + + const parent = withPrompt(scoped, () => { + startSpan("in scope", { type: "generation", attributes: promptAttributesFor(compiled) }).end(); + + return startSpan("parent"); + }); + const attributes = promptAttributesFor(compiled); + startSpan("with parent", { type: "generation", parent, attributes }).end(); + parent.end(); + withPrompt(scoped, () => { + startSpan("updated", { type: "generation" }).update({ attributes }).end(); + startSpan("ended", { type: "generation" }).end({ attributes }); + startActiveSpan("active", { type: "generation" }, () => updateActiveSpan({ attributes })); + }); + await flush(); + + expect(attributes?.["td.prompt.name"]).toBe("other/prompt"); + for (const name of ["in scope", "with parent", "updated", "ended", "active"]) { + const span = spans.getFinishedSpans().find((finished) => finished.name === name)!; + expect( + Object.fromEntries( + Object.entries(span.attributes).filter(([key]) => key.startsWith("td.prompt.")), + ), + ).toEqual({ + "td.prompt.name": "scoped/prompt", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + } +}); + +test("td.prompt.variables follows the input capture of the client and of the span", async () => { + const prompt = await promptOf({ + messages: [{ role: "user", content: "My card is {{card}}." }], + }); + const compiled = prompt.compile({ card: "4111 1111" }); + + setup({ captureInput: false }); + expect(promptAttributesFor(compiled)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + + await shutdown(); + const { spans } = setup({ + mask: (value, { key }) => (key === "td.prompt.variables" ? { card: "[masked]" } : value), + }); + const attributes = promptAttributesFor(compiled); + expect(attributes?.["td.prompt.variables"]).toBe('{"card":"[masked]"}'); + + startSpan("private", { type: "generation", captureInput: false, attributes }).end(); + startSpan("updated", { type: "generation", captureInput: false }).update({ attributes }).end(); + await flush(); + + for (const span of spans.getFinishedSpans()) { + expect(span.attributes["td.prompt.name"]).toBe("support/reply"); + expect(span.attributes["td.prompt.variables"]).toBeUndefined(); + } +}); + +test("promptAttributesFor reports an error and gives undefined", async () => { + const errors: Error[] = []; + setup({ onError: (error) => errors.push(error) }); + + const trap = new Proxy([], { + get() { + throw new Error("trap"); + }, + }); + + expect(promptAttributesFor(trap)).toBeUndefined(); + expect(errors.map((error) => error.message)).toEqual(["trap"]); +}); + test("a fallback prompt records its name and label but no version", async () => { vi.spyOn(console, "error").mockImplementation(() => {}); const { spans } = setup(); diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts new file mode 100644 index 0000000..9b9dccc --- /dev/null +++ b/packages/sdk/tests/scores.test.ts @@ -0,0 +1,245 @@ +import { createServer, type RequestListener } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { score, ScoreError, type ScoreOptions, shutdown } from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +const SPAN_ID = "00f067aa0ba902b7"; + +function fakeFetch(reply: () => Response) { + const calls: { url: URL; init: RequestInit | undefined }[] = []; + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + calls.push({ url: new URL(input), init }); + + return reply(); + }); + + return { fetch: fetch as unknown as typeof globalThis.fetch, calls }; +} + +async function serve(handler: RequestListener): Promise<{ url: string; close: () => void }> { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + close: () => { + server.closeAllConnections(); + server.close(); + }, + }; +} + +function errorsOf() { + const errors: ScoreError[] = []; + setup({ onError: (error) => errors.push(error as ScoreError) }); + + return errors; +} + +beforeEach(() => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + vi.stubEnv("TELEMETRY_DEV_BASE_URL", undefined); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("posts the score with the key and base URL of init, and resolves with its id", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-1" }, { status: 202 })); + setup({ apiKey: "td_live_key", baseUrl: "https://ingest.test/", fetch }); + + const id = await score({ + name: " user_feedback ", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); + + expect(id).toBe("score-1"); + expect(calls).toHaveLength(1); + expect(calls[0]!.url.href).toBe("https://ingest.test/v1/scores"); + expect(calls[0]!.init).toMatchObject({ + method: "POST", + headers: { + authorization: "Bearer td_live_key", + "x-telemetry-dev-sdk": "@telemetry-dev/sdk", + "content-type": "application/json", + }, + }); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "user_feedback", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); +}); + +test("calls the global fetch as a plain function, as Workers and browsers need", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", "td_live_key"); + // A strict function called as a method of an object gets that object as `this`. + vi.stubGlobal("fetch", function (this: unknown) { + if (this !== undefined && this !== globalThis) throw new TypeError("Illegal invocation"); + + return Promise.resolve(Response.json({ id: "score-3" }, { status: 202 })); + }); + + try { + await expect(score({ name: "helpful", traceId: TRACE_ID, value: 1 })).resolves.toBe("score-3"); + } finally { + vi.unstubAllGlobals(); + } +}); + +test("score with no options reports invalid_score and resolves undefined", async () => { + const errors = errorsOf(); + + await expect((score as (options?: ScoreOptions) => Promise)()).resolves.toBeUndefined(); + expect(errors.map((error) => [error.code, error.message])).toEqual([ + ["invalid_score", "name must be a string of 1 to 64 characters"], + ]); +}); + +test("a session score needs no trace, and a label goes trimmed", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "resolved", sessionId: "chat-9", label: " yes " }), + ).resolves.toBe("score-2"); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "resolved", + sessionId: "chat-9", + label: "yes", + }); +}); + +test.each<[string, Partial, string]>([ + ["no trace and no session", { traceId: undefined }, "Pass traceId or sessionId"], + [ + "the all-zero trace id of a span that did not record", + { traceId: "0".repeat(32) }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "an uppercase trace id", + { traceId: TRACE_ID.toUpperCase() }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "a span without its trace", + { traceId: undefined, sessionId: "s", spanId: SPAN_ID }, + "Pass the traceId of the span", + ], + ["no value and no label", { value: undefined }, "Pass a value or a label"], + ["a value that is not finite", { value: Number.NaN }, "value must be a finite number"], + [ + "a label for user feedback", + { name: "user_feedback", value: 1, label: "up" }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + [ + "user feedback of 0.5", + { name: "user_feedback", value: 0.5 }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + ["a pass of 0", { value: 0, label: "pass" }, "The label pass goes with the value 1"], + ["a fail of 1", { value: 1, label: "fail" }, "The label fail goes with the value 0"], + ["a blank name", { name: " " }, "name must be a string of 1 to 64 characters"], + [ + "a comment of 4,001 characters", + { comment: "x".repeat(4_001) }, + "comment must be a string of at most 4000 characters", + ], +])("reports invalid_score for %s and sends nothing", async (_, options, message) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await expect( + score({ + apiKey: "td_live_key", + fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + ...options, + }), + ).resolves.toBeUndefined(); + expect(calls).toEqual([]); + expect(errors.map((error) => [error.code, error.message])).toEqual([["invalid_score", message]]); +}); + +test.each([ + [ + "the API error", + () => + Response.json( + { error: { code: "insufficient_scope", message: "This API key cannot send scores" } }, + { status: 403 }, + ), + ["insufficient_scope", 403], + ], + ["a status with no API error", () => new Response("down", { status: 503 }), ["http_503", 503]], + ["a body with no id", () => Response.json({}, { status: 202 }), ["invalid_response", undefined]], +])("reports %s, tries once, and resolves undefined", async (_, reply, [code, status]) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(reply); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }), + ).resolves.toBeUndefined(); + expect(calls).toHaveLength(1); + expect(errors.map((error) => [error.code, error.status])).toEqual([[code, status]]); +}); + +test("without an API key it reports missing_api_key", async () => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await score({ apiKey: "", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }); + + expect(calls).toEqual([]); + expect(errors.map((error) => error.code)).toEqual(["missing_api_key"]); +}); + +test("does not follow a redirect, so the key stays with the API host", async () => { + const errors = errorsOf(); + let reached = 0; + const other = await serve((_, res) => { + reached += 1; + res.end(JSON.stringify({ id: "stolen" })); + }); + const api = await serve((_, res) => { + res.writeHead(307, { location: `${other.url}/v1/scores` }); + res.end(); + }); + + try { + await expect( + // The real fetch: setup() gives the client a stub. + score({ + apiKey: "td_live_key", + baseUrl: api.url, + fetch: globalThis.fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + }), + ).resolves.toBeUndefined(); + } finally { + api.close(); + other.close(); + } + + expect(reached).toBe(0); + expect(errors.map((error) => [error.code, error.status])).toEqual([["http_307", 307]]); +}); diff --git a/release-please-config.json b/release-please-config.json index e8c7e42..79b72d6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -191,6 +191,11 @@ "jsonpath": "$.package[?(@.name.value=='telemetry-dev-openrouter')].version" } ] + }, + "actions/experiment": { + "component": "experiment-action", + "release-type": "simple", + "initial-version": "1.0.0" } } } diff --git a/sdks/python-anthropic/CHANGELOG.md b/sdks/python-anthropic/CHANGELOG.md index 1b31054..c21b8c1 100644 --- a/sdks/python-anthropic/CHANGELOG.md +++ b/sdks/python-anthropic/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-anthropic-v0.2.1...python-anthropic-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-anthropic-v0.2.0...python-anthropic-v0.2.1) (2026-10-30) diff --git a/sdks/python-anthropic/pyproject.toml b/sdks/python-anthropic/pyproject.toml index 13ac10c..c6298b3 100644 --- a/sdks/python-anthropic/pyproject.toml +++ b/sdks/python-anthropic/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-anthropic" -version = "0.2.1" +version = "0.2.2" description = "Anthropic integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index 0f257aa..21fc24b 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -18,7 +18,7 @@ from anthropic.resources.messages import AsyncMessages, Messages from telemetry_dev._serialize import json_dumps -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] @@ -244,6 +244,18 @@ def _offered_tools(tools: Any) -> dict[str, str] | None: return {"gen_ai.tool.definitions": json_dumps(definitions)} if definitions else None +def _add_prompt(fields: Mapping[str, Any], messages: object) -> dict[str, Any]: + """``fields`` plus the prompt tag of ``messages``, the list that the app passed: the copy that + ``_normalize_request_params`` makes has no tag. Call it where the span starts, so that a + ``with_prompt`` scope there wins.""" + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + prompt = cast("dict[str, Any]", read(messages)) if callable(read) else {} + if not prompt: + return dict(fields) + return {**fields, "attributes": {**(fields.get("attributes") or {}), **prompt}} + + def _messages_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: params = _sent_params(params) model = _string(params.get("model")) @@ -1452,16 +1464,21 @@ def __getattr__(self, name: str) -> Any: class _InstrumentedMessageStreamManager: - def __init__(self, inner: Any, name: str, fields: Mapping[str, Any], provider: str) -> None: + def __init__( + self, inner: Any, name: str, fields: Mapping[str, Any], messages: object, provider: str + ) -> None: self._inner = inner self._name = name self._fields = fields + self._messages = messages self._provider = provider self._proxy: _InstrumentedStream | None = None def __enter__(self) -> Any: handle = telemetry_dev.start_span( - self._name, provider=self._provider, **_clean_fields(self._fields) + self._name, + provider=self._provider, + **_clean_fields(_add_prompt(self._fields, self._messages)), ) end = _end_once(handle) started_at = time.perf_counter() @@ -1491,16 +1508,21 @@ def __getattr__(self, name: str) -> Any: class _InstrumentedAsyncMessageStreamManager: - def __init__(self, inner: Any, name: str, fields: Mapping[str, Any], provider: str) -> None: + def __init__( + self, inner: Any, name: str, fields: Mapping[str, Any], messages: object, provider: str + ) -> None: self._inner = inner self._name = name self._fields = fields + self._messages = messages self._provider = provider self._proxy: _InstrumentedAsyncStream | None = None async def __aenter__(self) -> Any: handle = telemetry_dev.start_span( - self._name, provider=self._provider, **_clean_fields(self._fields) + self._name, + provider=self._provider, + **_clean_fields(_add_prompt(self._fields, self._messages)), ) end = _end_once(handle) started_at = time.perf_counter() @@ -1533,9 +1555,10 @@ def __getattr__(self, name: str) -> Any: def _start_span( - params: Mapping[str, Any], provider: str + params: Mapping[str, Any], messages: object, provider: str ) -> tuple[telemetry_dev.SpanHandle, Callable[..., None], float]: name, fields = _messages_request(params) + fields = _add_prompt(fields, messages) handle = telemetry_dev.start_span(name, provider=provider, **_clean_fields(fields)) return handle, _end_once(handle), time.perf_counter() @@ -1548,7 +1571,9 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, provider(resource)) + handle, end, started_at = _start_span( + call_kwargs, _sent_params(kwargs).get("messages"), provider(resource) + ) try: result = original(*args, **call_kwargs) except BaseException as exc: @@ -1572,7 +1597,9 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return await original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, provider(resource)) + handle, end, started_at = _start_span( + call_kwargs, _sent_params(kwargs).get("messages"), provider(resource) + ) try: result = await original(*args, **call_kwargs) except BaseException as exc: @@ -1596,8 +1623,11 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) name, fields = _messages_request(call_kwargs) + messages = _sent_params(kwargs).get("messages") manager = original(*args, **call_kwargs) - return _InstrumentedMessageStreamManager(manager, name, fields, provider(resource)) + return _InstrumentedMessageStreamManager( + manager, name, fields, messages, provider(resource) + ) setattr(wrapper, _WRAPPED_ATTR, True) setattr(wrapper, _ORIGINAL_ATTR, original) @@ -1612,8 +1642,11 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) name, fields = _messages_request(call_kwargs) + messages = _sent_params(kwargs).get("messages") manager = original(*args, **call_kwargs) - return _InstrumentedAsyncMessageStreamManager(manager, name, fields, provider(resource)) + return _InstrumentedAsyncMessageStreamManager( + manager, name, fields, messages, provider(resource) + ) setattr(wrapper, _WRAPPED_ATTR, True) setattr(wrapper, _ORIGINAL_ATTR, original) diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 3911ffb..c22c555 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -421,6 +421,126 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.cache_read.input_tokens"] == 1000 +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +SCOPED_PROMPT = telemetry_dev.ChatPrompt( + name="scoped/prompt", + version=3, + label="production", + messages=[{"role": "user", "content": "Hi"}], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("messages", "capture_input", "older_core"), + [("compiled", False, False), ("copied", True, False), ("compiled", True, True)], +) +def test_create_outside_with_prompt_records_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + messages: str, + capture_input: bool, + older_core: bool, +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + client = wrapped_sync_client(lambda _request: json_response(message_payload())) + + client.messages.create( + model="claude-sonnet-4-6", + max_tokens=64, + system=system["content"], + messages=[user if messages == "compiled" else dict(user)], + ) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "claude-sonnet-4-6" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "copied" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + +@pytest.mark.parametrize("scope_around", ["stream()", "__enter__"]) +def test_messages_stream_takes_the_prompt_from_the_context_where_its_span_starts( + memory: SimpleNamespace, scope_around: str +) -> None: + _system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + client = wrapped_sync_client(lambda _request: named_sse_response(stream_events())) + request: dict[str, Any] = {"model": "claude-sonnet-4-6", "max_tokens": 64, "messages": [user]} + + if scope_around == "stream()": + with telemetry_dev.with_prompt(SCOPED_PROMPT): + manager = client.messages.stream(**request) + with manager as stream: + "".join(stream.text_stream) + else: + manager = client.messages.stream(**request) + with telemetry_dev.with_prompt(SCOPED_PROMPT), manager as stream: + "".join(stream.text_stream) + + # The span starts at __enter__, so the scope there wins, and without one the message names it. + assert attrs(only_span(memory))["td.prompt.name"] == ( + "support/reply" if scope_around == "stream()" else "scoped/prompt" + ) + + +@pytest.mark.parametrize("scope_around", ["stream()", "__aenter__"]) +async def test_async_messages_stream_takes_the_prompt_from_the_context_where_its_span_starts( + memory: SimpleNamespace, scope_around: str +) -> None: + _system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + + client = wrapped_async_client(handler) + request: dict[str, Any] = {"model": "claude-sonnet-4-6", "max_tokens": 64, "messages": [user]} + + if scope_around == "stream()": + with telemetry_dev.with_prompt(SCOPED_PROMPT): + manager = client.messages.stream(**request) + async with manager as stream: + [text async for text in stream.text_stream] + else: + manager = client.messages.stream(**request) + with telemetry_dev.with_prompt(SCOPED_PROMPT): + async with manager as stream: + [text async for text in stream.text_stream] + await client.close() + + assert attrs(only_span(memory))["td.prompt.name"] == ( + "support/reply" if scope_around == "stream()" else "scoped/prompt" + ) + + def test_create_maps_native_messages_system_params_usage_finish_and_provider( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-anthropic/uv.lock b/sdks/python-anthropic/uv.lock index cb38c4b..4956ec6 100644 --- a/sdks/python-anthropic/uv.lock +++ b/sdks/python-anthropic/uv.lock @@ -745,7 +745,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -772,7 +772,7 @@ dev = [ [[package]] name = "telemetry-dev-anthropic" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "anthropic" }, diff --git a/sdks/python-bedrock/CHANGELOG.md b/sdks/python-bedrock/CHANGELOG.md index 9c23f3a..7fab0e3 100644 --- a/sdks/python-bedrock/CHANGELOG.md +++ b/sdks/python-bedrock/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-bedrock-v0.2.1...python-bedrock-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-bedrock-v0.2.0...python-bedrock-v0.2.1) (2026-10-30) diff --git a/sdks/python-bedrock/pyproject.toml b/sdks/python-bedrock/pyproject.toml index 37fc637..de7e541 100644 --- a/sdks/python-bedrock/pyproject.toml +++ b/sdks/python-bedrock/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-bedrock" -version = "0.2.1" +version = "0.2.2" description = "AWS Bedrock integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py index 2ea5057..c748493 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py @@ -2,6 +2,6 @@ from ._instrument import instrument_bedrock, uninstrument_bedrock, wrap_bedrock -__version__ = "0.2.1" +__version__ = "0.2.2" __all__ = ["__version__", "instrument_bedrock", "uninstrument_bedrock", "wrap_bedrock"] diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py index 1017467..f6fc07d 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Any +from typing import Any, cast +import telemetry_dev from telemetry_dev._serialize import json_dumps PROVIDER = "amazon-bedrock" @@ -11,6 +12,12 @@ def clean(fields: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in fields.items() if value is not None} +def prompt_attributes(messages: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(messages)) if callable(read) else {} + + def metadata_fields(response: dict[str, Any] | BaseException | None) -> dict[str, Any]: metadata: dict[str, Any] | None = None if isinstance(response, dict) and isinstance(response.get("ResponseMetadata"), dict): diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py index 6d09848..d27304d 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py @@ -6,7 +6,14 @@ import telemetry_dev -from ._fields import PROVIDER, clean, merge_fields, metadata_fields, usage_from_converse +from ._fields import ( + PROVIDER, + clean, + merge_fields, + metadata_fields, + prompt_attributes, + usage_from_converse, +) from ._invoke_model import is_embedding_model from ._invoke_model import request_fields as invoke_request_fields from ._invoke_model import response_fields as invoke_response_fields @@ -50,6 +57,7 @@ def _converse_request(params: dict[str, Any]) -> dict[str, Any]: "model": _model(params), "input": normalize_messages(params.get("messages")), "system_instructions": normalize_content_list(params.get("system")) or None, + "attributes": prompt_attributes(params.get("messages")) or None, "temperature": inference.get("temperature"), "top_p": inference.get("topP"), "max_tokens": inference.get("maxTokens"), diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 3b6acac..505d4d6 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -14,6 +14,7 @@ from botocore.client import BaseClient from botocore.exceptions import ClientError, EventStreamError from botocore.response import StreamingBody +from botocore.stub import Stubber import telemetry_dev_bedrock._streams as bedrock_streams from telemetry_dev_bedrock import instrument_bedrock, uninstrument_bedrock, wrap_bedrock @@ -114,6 +115,74 @@ def _json_attr(span: Any, key: str) -> Any: return json.loads(value) +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("messages", "older_core"), + [("changed in place", False), ("new", False), ("changed in place", True)], +) +def test_converse_outside_with_prompt_records_the_prompt_of_compiled_messages( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, messages: str, older_core: bool +) -> None: + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + blocks = [{"text": user["content"]}] + # Converse takes content blocks: a message changed in place keeps its tag, a new one has none. + if messages == "new": + message = {"role": "user", "content": blocks} + else: + user["content"] = blocks + message = user + client = _client("bedrock-runtime") + # The stubber runs botocore's validation of the request. + stubber = Stubber(client) + stubber.add_response( + "converse", + { + "output": {"message": {"role": "assistant", "content": [{"text": "Soon."}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + "metrics": {"latencyMs": 1}, + }, + ) + + with stubber: + cast(Any, wrap_bedrock(client)).converse( + modelId="anthropic.claude-3-haiku", + system=[{"text": system["content"]}], + messages=[message], + ) + + [span] = memory.span_exporter.get_finished_spans() + a = dict(span.attributes or {}) + assert a["gen_ai.request.model"] == "anthropic.claude-3-haiku" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "new" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + } + ) + + def test_converse_happy_path(memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch) -> None: client = _client("bedrock-runtime") request = { diff --git a/sdks/python-bedrock/uv.lock b/sdks/python-bedrock/uv.lock index 6f460cb..92d00ce 100644 --- a/sdks/python-bedrock/uv.lock +++ b/sdks/python-bedrock/uv.lock @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -478,7 +478,7 @@ dev = [ [[package]] name = "telemetry-dev-bedrock" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "boto3" }, diff --git a/sdks/python-google-genai/CHANGELOG.md b/sdks/python-google-genai/CHANGELOG.md index db866b9..d8cf120 100644 --- a/sdks/python-google-genai/CHANGELOG.md +++ b/sdks/python-google-genai/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-google-genai-v0.2.1...python-google-genai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-google-genai-v0.2.0...python-google-genai-v0.2.1) (2026-10-30) diff --git a/sdks/python-google-genai/pyproject.toml b/sdks/python-google-genai/pyproject.toml index b9a2fb3..4c4cde0 100644 --- a/sdks/python-google-genai/pyproject.toml +++ b/sdks/python-google-genai/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-google-genai" -version = "0.2.1" +version = "0.2.2" description = "Google GenAI (Gemini) integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index 1d8e0ae..0d4fcb5 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -24,7 +24,7 @@ from google.genai.operations import AsyncOperations, Operations from telemetry_dev._serialize import json_dumps -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] RequestMapper = Callable[[Any, Any, Any], tuple[str, dict[str, Any]]] @@ -353,6 +353,12 @@ def _request_attributes(config: Any) -> dict[str, Any]: return attrs +def _prompt_attributes(messages: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(messages)) if callable(read) else {} + + def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[str, dict[str, Any]]: model_name = _string(model) or "unknown" fields: dict[str, Any] = { @@ -383,7 +389,7 @@ def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[st output_type = _output_type(config) if output_type is not None: fields["output_type"] = output_type - attrs = _request_attributes(config) + attrs = {**_request_attributes(config), **_prompt_attributes(contents)} if attrs: fields["attributes"] = attrs return f"chat {model_name}", fields diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index f045e68..e397763 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -656,6 +656,75 @@ async def run() -> Any: assert attrs(span)["google_genai.response.operation_done"] is True +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("contents", "capture_input", "older_core"), + [ + ("changed in place", True, False), + ("changed in place", False, False), + ("new", True, False), + ("changed in place", True, True), + ], +) +def test_generate_content_outside_with_prompt_records_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + contents: str, + capture_input: bool, + older_core: bool, +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + parts = [{"text": user["content"]}] + # Gemini takes parts: a message changed in place keeps its tag, a new one has none. + if contents == "new": + content: dict[str, Any] = {"role": "user", "parts": parts} + else: + del user["content"] + user["parts"] = parts + content = user + client, transport = client_with_transport([happy_payload()]) + + wrap_google_genai(client).models.generate_content( + model="gemini-2.5-flash", + contents=[content], + config=types.GenerateContentConfig(system_instruction=system["content"]), + ) + + assert cast(Any, transport.calls[0])["contents"] == [ + {"role": "user", "parts": [{"text": "Where is order #4?"}]} + ] + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "gemini-2.5-flash" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or contents == "new" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_generate_content_maps_request_response_usage_and_sampling( memory: SimpleNamespace, async_mode: bool diff --git a/sdks/python-google-genai/uv.lock b/sdks/python-google-genai/uv.lock index cda5be3..f6f8594 100644 --- a/sdks/python-google-genai/uv.lock +++ b/sdks/python-google-genai/uv.lock @@ -821,7 +821,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -848,7 +848,7 @@ dev = [ [[package]] name = "telemetry-dev-google-genai" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "google-genai" }, diff --git a/sdks/python-litellm/CHANGELOG.md b/sdks/python-litellm/CHANGELOG.md index f7280f4..fce6210 100644 --- a/sdks/python-litellm/CHANGELOG.md +++ b/sdks/python-litellm/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-litellm-v0.2.1...python-litellm-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-litellm-v0.2.0...python-litellm-v0.2.1) (2026-10-30) diff --git a/sdks/python-litellm/pyproject.toml b/sdks/python-litellm/pyproject.toml index fd88c5a..b9d45a5 100644 --- a/sdks/python-litellm/pyproject.toml +++ b/sdks/python-litellm/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-litellm" -version = "0.2.1" +version = "0.2.2" description = "LiteLLM integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index b2f6680..0033de1 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -15,7 +15,7 @@ import telemetry_dev from opentelemetry import trace -__version__ = "0.2.1" +__version__ = "0.2.2" RequestMapper = Callable[[tuple[Any, ...], Mapping[str, Any]], tuple[str, dict[str, Any]]] ResponseMapper = Callable[[Any], dict[str, Any]] @@ -455,6 +455,12 @@ def _output_type(response_format: Any) -> str | None: return kind +def _prompt_attributes(messages: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(messages)) if callable(read) else {} + + def _completion_request( args: tuple[Any, ...], kwargs: Mapping[str, Any] ) -> tuple[str, dict[str, Any]]: @@ -478,6 +484,7 @@ def _completion_request( "presence_penalty": _number(kwargs.get("presence_penalty")), "output_type": _output_type(kwargs.get("response_format")), "metadata": _request_metadata(kwargs), + "attributes": _prompt_attributes(_arg(args, kwargs, "messages", 1)) or None, }, ) @@ -545,7 +552,10 @@ def _responses_request( input_budget: telemetry_dev.CaptureBudget | None = None if capture_input: input_value, input_budget = _bounded_responses_native(_arg(args, kwargs, "input", 0)) - attributes: dict[str, Any] = {"gen_ai.operation.name": "chat"} + attributes: dict[str, Any] = { + "gen_ai.operation.name": "chat", + **_prompt_attributes(_arg(args, kwargs, "input", 0)), + } if input_budget is not None and input_budget.truncated: attributes["telemetry.dev.capture.truncated"] = True return ( diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index f7b08a1..a4c8dd1 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -57,6 +57,69 @@ def finished_spans(env: SimpleNamespace) -> list[ReadableSpan]: return list(env.span_exporter.get_finished_spans()) +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("api", "messages", "capture_input", "older_core"), + [ + ("completion", "compiled", False, False), + ("responses", "compiled", True, False), + ("completion", "copied", True, False), + ("completion", "compiled", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] + + if api == "completion": + telemetry_dev_litellm.completion(model="gpt-4o-mini", messages=sent, mock_response="Soon.") + else: + + def fake_responses(*_args: Any, **_kwargs: Any) -> SimpleNamespace: + return responses_result() + + monkeypatch.setattr(litellm, "responses", fake_responses) + telemetry_dev_litellm.responses(sent, "openai/gpt-4.1-mini") + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == ("gpt-4o-mini" if api == "completion" else "gpt-4.1-mini") + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "copied" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + def test_completion_maps_model_messages_usage_finish_provider_and_sampling( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-litellm/uv.lock b/sdks/python-litellm/uv.lock index 5a9e4ba..acdff9d 100644 --- a/sdks/python-litellm/uv.lock +++ b/sdks/python-litellm/uv.lock @@ -2889,7 +2889,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api", version = "1.43.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -2919,7 +2919,7 @@ dev = [ [[package]] name = "telemetry-dev-litellm" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "litellm" }, diff --git a/sdks/python-openai/CHANGELOG.md b/sdks/python-openai/CHANGELOG.md index 89cb655..f0b94aa 100644 --- a/sdks/python-openai/CHANGELOG.md +++ b/sdks/python-openai/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-openai-v0.2.1...python-openai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-openai-v0.2.0...python-openai-v0.2.1) (2026-10-30) diff --git a/sdks/python-openai/pyproject.toml b/sdks/python-openai/pyproject.toml index 79c6435..8a3fbf9 100644 --- a/sdks/python-openai/pyproject.toml +++ b/sdks/python-openai/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-openai" -version = "0.2.1" +version = "0.2.2" description = "OpenAI integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index af2a498..5475801 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -20,7 +20,7 @@ from openai.resources.images import AsyncImages, Images from openai.resources.responses.responses import AsyncResponses, Responses -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] @@ -349,9 +349,18 @@ def _capture_text(value: str, budget: telemetry_dev.CaptureBudget) -> str: return value +def _prompt_attributes(messages: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(messages)) if callable(read) else {} + + def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) + attributes = _prompt_attributes(params.get("messages")) captured, truncated = _bounded_responses_capture(params.get("messages"), "input") + if truncated: + attributes["telemetry.dev.capture.truncated"] = True return ( f"chat {model or 'unknown'}", { @@ -366,7 +375,7 @@ def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "seed": _number(params.get("seed")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), - "attributes": {"telemetry.dev.capture.truncated": True} if truncated else None, + "attributes": attributes or None, }, ) @@ -432,7 +441,10 @@ def _chat_response(response: Any) -> dict[str, Any]: def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) + attributes = _prompt_attributes(params.get("input")) captured, truncated = _bounded_responses_capture(params.get("input"), "input") + if truncated: + attributes["telemetry.dev.capture.truncated"] = True return ( f"chat {model or 'unknown'}", { @@ -443,7 +455,7 @@ def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "temperature": _number(params.get("temperature")), "top_p": _number(params.get("top_p")), "max_tokens": _number(params.get("max_output_tokens")), - "attributes": {"telemetry.dev.capture.truncated": True} if truncated else None, + "attributes": attributes or None, }, ) diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 238c7e6..55ab3c6 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1206,6 +1206,70 @@ def text(self) -> str: assert a["gen_ai.usage.text.output_tokens"] == 2 +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("api", "messages", "capture_input", "older_core"), + [ + ("chat", "compiled", False, False), + ("responses", "compiled", True, False), + ("chat", "copied", True, False), + ("chat", "compiled", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] + + def handler(_request: httpx.Request) -> httpx.Response: + return json_response(chat_completion() if api == "chat" else response_payload()) + + client = wrap_openai(sync_client(handler)) + if api == "chat": + cast(Any, client.chat.completions.create)(model="gpt-4o", messages=sent) + else: + cast(Any, client.responses.create)(model="gpt-4.1", input=sent) + + span = only_span(memory) + a = attrs(span) + assert a["gen_ai.request.model"] == ("gpt-4o" if api == "chat" else "gpt-4.1") + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "copied" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + assert ("gen_ai.input.messages" in a) is capture_input + + def test_chat_completion_maps_native_messages_usage_finish_provider_and_sampling( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-openai/uv.lock b/sdks/python-openai/uv.lock index 6369336..ffd4d32 100644 --- a/sdks/python-openai/uv.lock +++ b/sdks/python-openai/uv.lock @@ -735,7 +735,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -762,7 +762,7 @@ dev = [ [[package]] name = "telemetry-dev-openai" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "openai" }, diff --git a/sdks/python-openrouter/CHANGELOG.md b/sdks/python-openrouter/CHANGELOG.md index 56ac8c2..d9f6061 100644 --- a/sdks/python-openrouter/CHANGELOG.md +++ b/sdks/python-openrouter/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-openrouter-v0.2.1...python-openrouter-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-openrouter-v0.2.0...python-openrouter-v0.2.1) (2026-10-30) diff --git a/sdks/python-openrouter/pyproject.toml b/sdks/python-openrouter/pyproject.toml index 98379ec..d17d978 100644 --- a/sdks/python-openrouter/pyproject.toml +++ b/sdks/python-openrouter/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-openrouter" -version = "0.2.1" +version = "0.2.2" description = "OpenRouter integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index 8b46b90..0f7c2b4 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -12,7 +12,7 @@ from openrouter.embeddings import Embeddings from openrouter.responses import Responses -__version__ = "0.2.1" +__version__ = "0.2.2" RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] ResponseMapper = Callable[[Any], dict[str, Any]] @@ -174,6 +174,12 @@ def _cost_usd(raw_usage: Any) -> float | None: return cost if cost is not None else upstream_cost +def _prompt_attributes(messages: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(messages)) if callable(read) else {} + + def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) max_completion_tokens = _number(params.get("max_completion_tokens")) @@ -193,6 +199,7 @@ def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "seed": _number(params.get("seed")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), + "attributes": _prompt_attributes(params.get("messages")) or None, }, ) @@ -260,6 +267,7 @@ def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "max_tokens": _number(params.get("max_output_tokens")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), + "attributes": _prompt_attributes(params.get("input")) or None, }, ) diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 1efe3a5..400c4f5 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -221,6 +221,68 @@ def embeddings_payload() -> dict[str, Any]: } +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("api", "messages", "capture_input", "older_core"), + [ + ("chat", "compiled", False, False), + ("responses", "compiled", True, False), + ("chat", "copied", True, False), + ("chat", "compiled", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] + + def handler(_request: httpx.Request) -> httpx.Response: + return json_response(chat_payload() if api == "chat" else responses_payload()) + + client = wrap_open_router(sync_client(handler)) + if api == "chat": + client.chat.send(model="openai/gpt-4o-mini", messages=cast(Any, sent)) + else: + client.responses.send(model="openai/gpt-4o-mini", input=cast(Any, sent)) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "openai/gpt-4o-mini" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "copied" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + def test_chat_send_maps_messages_usage_cost_and_sampling(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return json_response(chat_payload()) diff --git a/sdks/python-openrouter/uv.lock b/sdks/python-openrouter/uv.lock index d317aa7..05f8f67 100644 --- a/sdks/python-openrouter/uv.lock +++ b/sdks/python-openrouter/uv.lock @@ -603,7 +603,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -630,7 +630,7 @@ dev = [ [[package]] name = "telemetry-dev-openrouter" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "openrouter" }, diff --git a/sdks/python/CHANGELOG.md b/sdks/python/CHANGELOG.md index c950f82..f72714c 100644 --- a/sdks/python/CHANGELOG.md +++ b/sdks/python/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [0.3.2](https://github.com/telemetry-dev/telemetry/compare/python-v0.3.1...python-v0.3.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk-python:** fetch prompts again in a forked process ([#105](https://github.com/telemetry-dev/telemetry/issues/105)) ([c6b599e](https://github.com/telemetry-dev/telemetry/commit/c6b599ed1d0311ff2c367c0e64e47bd7c2b3b402)) +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep compiled variables for 5 minutes, not two turns ([1b09916](https://github.com/telemetry-dev/telemetry/commit/1b099167bbd8acaf5008848ad9ba2a2ab8e35b0d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of recent compiles only ([3bc6f53](https://github.com/telemetry-dev/telemetry/commit/3bc6f53a0afb0ad8b9fe19f8eaf7e5d812a93b0a)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of the last two chat turns ([f6aa1c1](https://github.com/telemetry-dev/telemetry/commit/f6aa1c159742123600d6499653916c4dbf7b2ab7)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** name the newest compile, not the last tagged message ([a6b98fe](https://github.com/telemetry-dev/telemetry/commit/a6b98fe916d5ed4b37bddd7972b07507182f2d2d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Documentation + +* **sdk:** name the read prompts access of the prompt api key ([#103](https://github.com/telemetry-dev/telemetry/issues/103)) ([1195b11](https://github.com/telemetry-dev/telemetry/commit/1195b11581dc729e346e888cf8406ab3dd917d96)) +* **sdk:** tell that a call with no compiled message records its input ([0c2f2b6](https://github.com/telemetry-dev/telemetry/commit/0c2f2b6e17a534d403179dd277db9136ab17f815)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + ## [0.3.1](https://github.com/telemetry-dev/telemetry/compare/python-v0.3.0...python-v0.3.1) (2026-10-30) diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d10..bd91e98 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -89,6 +89,8 @@ later `init()` generates a new ID. It is not persisted across processes. | `get_traceparent() -> str \| None` | W3C traceparent of the current context. | | `get_prompt(name, *, label=, version=, type="chat", cache_ttl_seconds=60, fallback=, api_key=, base_url=)` / `get_prompt_async(...)` | Fetch a prompt by label or version; see [Prompts](#prompts). | | `with_prompt(prompt)` | Context manager recording the prompt on each model call started inside. | +| `run_experiment(*, task, dataset=, name=, dataset_version=, split=, evaluators=, scorers=, min_scores=, runs_per_item=1, concurrency=4, metadata=, ci=, api_key=, base_url=)` / `run_experiment_async(...)` | Run your code on a dataset as an experiment; see [Experiments](#experiments). | +| `score(*, name, trace_id=, span_id=, session_id=, value=, label=, comment=, api_key=, base_url=)` / `score_async(...)` | Record a score on a trace, a span, or a session; see [Scores](#scores). | | `flush(timeout_s=10.0)` / `shutdown(timeout_s=10.0)` | Force-flush / tear down traces + logs + metrics. Shutdown also runs atexit unless `disable_atexit=True`. | | `TelemetrySpanProcessor` / `telemetry_dev.otel.create_telemetry_span_exporter` | Bring-your-own-OTel helpers (below). | | `MaskContext`, `Usage`, `SpanHandle`, `Client`, `NOT_GIVEN` | Supporting types. | @@ -219,6 +221,13 @@ pinned = telemetry_dev.get_prompt("support/reply", version=13) call returns it at once and fetches a new copy in a background thread. Only a call with no copy waits for a fetch, for 10 seconds at most, also when the network stops. Calls for the same prompt share one fetch. `get_prompt_async` runs the same function in a worker thread. +- A process that `os.fork()` makes keeps the copies of its parent. This includes a process of + `multiprocessing` with the `fork` start method. A call that starts in the child does not wait + for a fetch of the parent. If the call must fetch, it starts a new fetch. +- If a signal handler forks while a call waits for a fetch, that call continues in the child. No + thread of the child completes that fetch. Thus the call waits until 10 seconds after the start + of the fetch. Then the call returns the `fallback`, or raises `PromptError` with the code + `timeout`. - If a fetch fails, `get_prompt` returns the last copy that it fetched and reports the error through `on_error`. With no copy, it returns the `fallback` (`version` is `None` and `is_fallback` is `True`). With no fallback, it raises `PromptError` whose `code` tells why, for @@ -232,8 +241,8 @@ pinned = telemetry_dev.get_prompt("support/reply", version=13) without its prompt. - `get_prompt` uses the API key and base URL of an enabled `init()`, or `TELEMETRY_DEV_API_KEY` and `TELEMETRY_DEV_BASE_URL`, or its own `api_key` and `base_url`. It reads the same host that - telemetry goes to, `https://api.telemetry.dev` by default. The key needs the read scope for - prompts. + telemetry goes to, `https://api.telemetry.dev` by default. The key must have Read prompts in + its Access on the API keys page (the `prompts:read` scope). - When a label sends part of the traffic to a second version, each session keeps one version: the choice uses the session id of `propagate_attributes` (or of `session_mode="process"`). Without a session id, each call chooses at random. @@ -251,9 +260,169 @@ ran in it, or before it started in the code that started it; with none, it recor for code that has not compiled in its own task. Model calls are the spans of type `generation` (operations `chat`, `text_completion`, and `generate_content`), including the spans of the provider integrations: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, LiteLLM, and OpenRouter. Spans from -other OpenTelemetry tracers do not get the attributes. A model call outside `with_prompt` does not -get the attributes, even when it sends messages from `compile()`; with input capture on, the call -still records those messages as its input. +other OpenTelemetry tracers do not get the attributes. + +Outside `with_prompt`, those provider integrations record the prompt of the message dicts that +`compile()` returned. `compile()` of a chat prompt returns instances of a private `dict` subclass +that also hold its prompt and a copy of its variables and of each list in them. The objects in a +list are not copied, so a change to one of them after `compile()` changes the variables that a call +records. A chat call that sends such messages gets `td.prompt.name`, `td.prompt.version`, and +`td.prompt.label` of the newest `compile()` among them, also when input capture is off, and +`td.prompt.variables` when it is on. The SDK looks at the first 1000 and the last 1000 messages of +the call. A message from a placeholder variable keeps the prompt of its own, older `compile()`, so +it does not win over the `compile()` that put it in the list. A copy of a message (`dict()`, +`.copy()`, `copy.copy`, `copy.deepcopy`, pickle, JSON, `type(message)(message)`, or +`dataclasses.asdict`) and a new message with the same text do not hold the prompt, so a call that +sends only such messages gets no `td.prompt.*` attributes; with input capture on, the call still +records them as its input. Bedrock Converse and Google GenAI take another message shape: change the +compiled dicts in place, or use `with_prompt`. A text prompt compiles to a string, which cannot hold +its prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations do it for OpenAI +chat completions and responses, Anthropic messages (for `messages.stream()`, at `__enter__`), +Bedrock Converse, Google GenAI `generate_content`, LiteLLM completion and responses, and OpenRouter +chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, +video, or OpenAI Realtime. With an older `telemetry-dev` that has no `prompt_attributes_for`, a +provider integration records the prompt only in `with_prompt`. + +A compiled message is a `dict` for `isinstance`, and its keys and its JSON (`json`, `orjson`, +pydantic) do not change. A check of the exact type (`type(message) is dict`), `yaml.safe_dump`, +and `marshal` reject it: give them `dict(message)`. + +With input capture on in the client when `compile()` runs, a compiled message keeps the variables +of its `compile()` for 5 minutes. A call keeps them for 5 more minutes when the newest `compile()` +of the call is that one. After that time, a call records the prompt without +`td.prompt.variables`, and the SDK removes the variables from the message, also when you keep the +message. Other compiles do not change this time. Thus a chat history keeps the variables of the +compiles of its last 5 minutes, and a guard prompt, a queue, or a compile in another thread before +the call does not remove them. A message that you compile one time and send in many calls keeps its +variables while a call names its `compile()` at least one time in 5 minutes. A call that sends only +messages of earlier turns records the prompt of the newest `compile()` among them, and its +variables when they are less than 5 minutes old. With input capture off, the message keeps no +variables. + +For your own spans, `prompt_attributes_for(messages)` gives the attributes to pass as +`attributes`. It gives `{}` inside `with_prompt`, and a span with `capture_input=False` drops +`td.prompt.variables`. A model call whose parent is in `with_prompt` gets the prompt of the +`with_prompt` and none of the `td.prompt.*` attributes that you pass, at start or in `update`, +`end`, or `update_current_span`, also when its `parent` context comes from a `with_prompt` that the +call is not in. + +## Experiments + +`run_experiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```python +import telemetry_dev + +telemetry_dev.init(environment="experiment") + +result = telemetry_dev.run_experiment( + name="reply-v13-gpt-4o", + dataset="support-golden", + task=lambda item: reply_to_customer(item.input), + evaluators=["correctness", "helpfulness"], + scorers={"short": lambda output, item: len(output) < 600}, + min_scores={"correctness": 0.9}, +) +``` + +- `run_experiment` reads the newest version of the dataset, or `dataset_version`, 100 items at a + time. With `split`, it runs only the items of that split. Then it makes the experiment, with a + name that telemetry.dev picks when you give no `name`. +- `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, + `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a + different value becomes JSON, cut to 100,000 UTF-16 units. A value that JSON cannot hold, such + as a dict that holds itself, gives no output. A task that raises fails only its item, and the + item records the type and the message of the exception. +- `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the + caller, so `propagate_attributes` around the call applies to each item. With `runs_per_item` + (default 1, at most 5), each item runs more than one time. Each run gets its own copy of the + item, so a task that changes its input does not change the next run. The pass rate and the mean + of each score use all runs. +- `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, + and the requests go in worker threads. A task that is not async runs on the event loop, so use + `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. +- Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the + `experiment` environment by default. In a different environment, `run_experiment` logs a + warning. Without an enabled client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `True` or `False` for a pass or a fail, a number, a label, or a mapping with + `value`, `label`, and `reason`. A scorer that raises, or returns a score that is not valid, gives + no score for that item and sends a `scorer_failed` error to `on_error`. A scorer cannot have the + name of an evaluator of the run, or the name `user_feedback`. +- `run_experiment` sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns an `ExperimentResult`: + `status`, `check_status`, item counts, `scores` with the value, the baseline value, the change, + and the pass mark of each score, and the `url` of the experiment page. +- `min_scores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request raises `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion + before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails + it when it got none. +- The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. With + `environment="experiment"`, the key of the telemetry client must be for all environments or for + `experiment`: a key for another environment cannot send the traces of the items, and the items + then have no trace and no cost. + +The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +and stay with the experiment, because they are its result. `capture_output`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci=True`, `run_experiment` raises when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives `run_experiment` the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```python +import telemetry_dev + +# When the app replies, keep the trace id of the reply with the message: +with telemetry_dev.start_span("reply") as span: + text = reply_to_customer(message) + trace_id = f"{span.span.get_span_context().trace_id:032x}" + +# Later, when the user clicks thumbs down: +telemetry_dev.score(name="user_feedback", trace_id=trace_id, value=0, comment="Wrong amount") +``` + +- Give `trace_id`, `session_id`, or the two, and a `span_id` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, `score` never raises. It returns the id of the score, or `None` + after it sends a `ScoreError` to `on_error`. It checks the score before it sends it + (`invalid_score`). The trace id of a span from a client that is not enabled is all zeros, and + `score` reports it as `invalid_score`. +- `score` makes one request, with no retry. `score_async` runs it in a worker thread. The key + needs the `scores:write` scope. ## Auto-metrics diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 9204b53..1cffcd5 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" description = "telemetry.dev SDK for Python — OpenTelemetry-native GenAI tracing, logs, and metrics" readme = "README.md" license = "MIT" diff --git a/sdks/python/src/telemetry_dev/__init__.py b/sdks/python/src/telemetry_dev/__init__.py index 9ed7aab..f6ba26c 100644 --- a/sdks/python/src/telemetry_dev/__init__.py +++ b/sdks/python/src/telemetry_dev/__init__.py @@ -17,6 +17,18 @@ from ._config import SDK_VERSION as __version__ from ._config import LogLevelOption from ._context import get_traceparent, propagate_attributes +from ._experiments import ( + AsyncScorer, + DatasetItem, + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + ScorerResult, + run_experiment, + run_experiment_async, +) from ._logs import log from ._observe import observe from ._prompts import ( @@ -27,8 +39,10 @@ TextPrompt, get_prompt, get_prompt_async, + prompt_attributes_for, with_prompt, ) +from ._scores import ScoreError, score, score_async from ._semconv import LogLevel, SpanType from ._serialize import AttributeValue, Mask, MaskContext from ._spans import ( @@ -42,10 +56,16 @@ __all__ = [ "NOT_GIVEN", + "AsyncScorer", "AttributeValue", "CaptureBudget", "ChatPrompt", "Client", + "DatasetItem", + "ExperimentError", + "ExperimentItem", + "ExperimentResult", + "ExperimentScore", "LogLevel", "LogLevelOption", "Mask", @@ -53,6 +73,9 @@ "Prompt", "PromptError", "PromptModel", + "ScoreError", + "Scorer", + "ScorerResult", "SpanHandle", "SpanType", "TelemetrySpanProcessor", @@ -67,7 +90,12 @@ "init", "log", "observe", + "prompt_attributes_for", "propagate_attributes", + "run_experiment", + "run_experiment_async", + "score", + "score_async", "shutdown", "start_span", "update_current_span", diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py new file mode 100644 index 0000000..5dc3065 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_api.py @@ -0,0 +1,203 @@ +"""Calls to the telemetry.dev API: the key and base URL, one JSON request, and its errors.""" + +from __future__ import annotations + +import http.client +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from email.message import Message +from email.utils import parsedate_to_datetime +from typing import IO, Any, Literal, TypeVar, cast + +from ._client import get_client +from ._config import SDK_VERSION, resolve_config +from ._serialize import json_dumps + +# A longer Retry-After waits only this long: the API counts requests by the minute. +_MAX_RETRY_AFTER_S = 60.0 +# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. +_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) + + +# An error class with the shape of PromptError: a code, a message, and the status. +E = TypeVar("E", bound=Exception) + + +@dataclass(frozen=True) +class Api: + api_key: str | None + base_url: str + on_error: Callable[[BaseException], None] | None + + +def resolve_api(api_key: str | None, base_url: str | None) -> Api: + """The key and base URL of a call: its arguments, then the enabled client, then the env.""" + client = get_client() + enabled = client if client is not None and client.enabled else None + config = enabled.config if enabled is not None else resolve_config() + return Api( + api_key=api_key if api_key is not None else config.api_key, + base_url=(base_url or config.base_url).rstrip("/"), + on_error=enabled.on_error if enabled is not None else None, + ) + + +@dataclass(frozen=True) +class _Failure: + error: Exception + status: int | None + retry_after: float | None + + +def request( + api: Api, + error_type: Callable[[str, str, int | None], E], + method: Literal["GET", "POST"], + path: str, + *, + timeout_s: float, + idempotent: bool, + body: object = None, + query: Mapping[str, str | None] | None = None, + attempts: int = 3, +) -> Any: + """Send one JSON request and return the JSON of the response. + + A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or + ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a + network error, or a 5xx tries again. Other requests try again only after a 429, which the rate + limit answers before the server does any work: a 503 can come after the work committed. A + retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout + applies to each socket operation of urllib. + """ + url = f"{api.base_url}{path}" + params = {key: value for key, value in (query or {}).items() if value is not None} + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + headers = auth_headers(api.api_key) + data = None + if body is not None: + # The JSON of JSON.stringify, which escapes a lone surrogate that UTF-8 cannot hold. + data = json_dumps(body, default=None).encode() + headers["Content-Type"] = "application/json" + attempt = 1 + while True: + result = _send(error_type, method, url, path, data, headers, timeout_s) + if not isinstance(result, _Failure): + return result + if attempt >= attempts or not _retryable(result.status, idempotent): + raise result.error + time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S)) + attempt += 1 + + +def auth_headers(api_key: str | None) -> dict[str, str]: + return { + "Authorization": f"Bearer {api_key}", + "X-Telemetry-Dev-SDK": "telemetry-dev", + # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters such as + # Cloudflare's Browser Integrity Check. + "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", + } + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # A redirect to another host must not get the API key, so a 3xx fails like any status that is + # not 2xx. + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: Message, + newurl: str, + ) -> urllib.request.Request | None: + return None + + +opener = urllib.request.build_opener(NoRedirect) + + +def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E: + """The error of a response that is not 2xx: its API code and message, or ``http_``.""" + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} + code = fields.get("code") + message = fields.get("message") + return error_type( + code if isinstance(code, str) else f"http_{status}", + message if isinstance(message, str) else f"telemetry.dev returned {status}", + status, + ) + + +def retry_after(status: int, value: str | None) -> float | None: + """The seconds that the Retry-After of a 429 or a 503 asks for: seconds or an HTTP date.""" + value = (value or "").strip() + if status not in (429, 503) or not value: + return None + if value.isascii() and value.isdigit(): + seconds = float(value) + elif _HTTP_DATE.fullmatch(value): + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + seconds = (date - datetime.now(timezone.utc)).total_seconds() + else: + return None + return seconds if seconds > 0 else None + + +def _send( + error_type: Callable[[str, str, int | None], E], + method: str, + url: str, + path: str, + data: bytes | None, + headers: dict[str, str], + timeout_s: float, +) -> Any: + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(request, timeout=timeout_s) as response: + body: bytes = response.read() + except urllib.error.HTTPError as exc: + with exc: + return _Failure( + api_error(error_type, exc.code, exc.read()), + exc.code, + retry_after(exc.code, exc.headers.get("Retry-After")), + ) + except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: + timed_out = isinstance(exc, TimeoutError) or isinstance( + getattr(exc, "reason", None), TimeoutError + ) + error = ( + error_type("timeout", f"{method} {path} took more than {timeout_s:g} s", None) + if timed_out + else error_type("network_error", "Could not reach telemetry.dev", None) + ) + error.__cause__ = exc + return _Failure(error, None, None) + try: + return json.loads(body) + except ValueError: + return None + + +def _retryable(status: int | None, idempotent: bool) -> bool: + if status == 429: + return True + return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py new file mode 100644 index 0000000..e27701f --- /dev/null +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -0,0 +1,136 @@ +"""The GitHub Actions run of an experiment, and the variables of the experiment action.""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Mapping +from typing import Any, cast + +from ._config import report_error +from ._serialize import utf16_length + +_REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +_COMMIT = re.compile(r"[0-9a-fA-F]{7,64}") +_HTTPS_URL = re.compile(r"https://\S+") +# A plain decimal, so that the TypeScript and Python SDKs read the same marks. +_DECIMAL = re.compile(r"-?\d+(\.\d+)?", re.ASCII) +_RUN_ID = re.compile(r"\d+", re.ASCII) +_MAX_REPOSITORY_LENGTH = 200 +_MAX_URL_LENGTH = 2048 +_MAX_PR_TITLE_LENGTH = 256 + + +def in_ci(env: Mapping[str, str]) -> bool: + """CI mode is on when ``CI`` is ``true`` or ``1``, as CI services set it.""" + return env.get("CI") in ("true", "1") + + +def github_run(env: Mapping[str, str]) -> dict[str, Any] | None: + """The run that GitHub Actions describes in its variables, or None outside GitHub Actions. + + On a pull request the commit is the head of the pull request: ``GITHUB_SHA`` is then a merge + commit that nobody pushed. + """ + repository = env.get("GITHUB_REPOSITORY") or "" + if ( + env.get("GITHUB_ACTIONS") != "true" + or len(repository) > _MAX_REPOSITORY_LENGTH + or not _REPOSITORY.fullmatch(repository) + ): + return None + pull = _pull_request(env.get("GITHUB_EVENT_PATH")) + commit = pull.get("commit") or env.get("GITHUB_SHA") + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + return None + server = (env.get("GITHUB_SERVER_URL") or "https://github.com").rstrip("/") + run_id = env.get("GITHUB_RUN_ID") or "" + run: dict[str, Any] = { + "provider": "github_actions", + "repository": repository, + "commit": commit, + "prNumber": pull.get("number"), + "prTitle": pull.get("title"), + "prUrl": pull.get("url"), + "runUrl": _https_url(f"{server}/{repository}/actions/runs/{run_id}") + if _RUN_ID.fullmatch(run_id) + else None, + } + return {key: value for key, value in run.items() if value is not None} + + +def min_scores_from_env(env: Mapping[str, str]) -> dict[str, float]: + """The pass marks in ``TELEMETRY_DEV_EXPERIMENT_MIN_SCORES``, which the experiment action + sets from its ``min-scores`` input: ``name=value`` pairs, separated by commas or new lines.""" + scores: dict[str, float] = {} + for pair in re.split(r"[,\n]", env.get("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES") or ""): + if not pair.strip(): + continue + parts = [part.strip() for part in pair.split("=")] + if len(parts) != 2 or not parts[0] or not _DECIMAL.fullmatch(parts[1]): + raise TypeError( + f'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "{pair.strip()}" is not a name=number pair' + ) + scores[parts[0]] = float(parts[1]) + return scores + + +def write_result( + env: Mapping[str, str], + result: object, + on_error: Callable[[BaseException], None] | None, +) -> None: + """Add the result to the file in ``TELEMETRY_DEV_EXPERIMENT_RESULTS`` as one line of JSON. + The experiment action reads the file to write the job summary and the pull request comment.""" + path = env.get("TELEMETRY_DEV_EXPERIMENT_RESULTS") + if not path: + return + line = json.dumps(result, ensure_ascii=False, separators=(",", ":")) + "\n" + try: + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + with os.fdopen(fd, "a", encoding="utf-8") as file: + file.write(line) + except OSError as exc: + report_error(on_error, "writing TELEMETRY_DEV_EXPERIMENT_RESULTS failed", exc) + + +def _pull_request(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + try: + with open(event_path, encoding="utf-8") as file: + event = json.load(file) + except (OSError, ValueError): + return {} + pull = cast("dict[str, Any]", event).get("pull_request") if isinstance(event, dict) else None + if not isinstance(pull, dict): + return {} + fields = cast("dict[str, Any]", pull) + number = fields.get("number") + title = fields.get("title") + url = fields.get("html_url") + head = fields.get("head") + sha = cast("dict[str, Any]", head).get("sha") if isinstance(head, dict) else None + return { + "number": number + if isinstance(number, int) and not isinstance(number, bool) and number > 0 + else None, + "title": _cut(title, _MAX_PR_TITLE_LENGTH) if isinstance(title, str) else None, + "url": _https_url(url) if isinstance(url, str) else None, + "commit": sha if isinstance(sha, str) else None, + } + + +def _https_url(url: str) -> str | None: + return url if utf16_length(url) <= _MAX_URL_LENGTH and _HTTPS_URL.fullmatch(url) else None + + +def _cut(text: str, max_units: int) -> str: + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts, with no half of + a surrogate pair at the end: the API stores the run as jsonb, and Postgres rejects a lone + surrogate there.""" + encoded = text.encode("utf-16-le", "surrogatepass") + if len(encoded) <= max_units * 2: + return text + return encoded[: max_units * 2].decode("utf-16-le", "ignore") diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py new file mode 100644 index 0000000..303dc89 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -0,0 +1,1138 @@ +"""Experiments of your code: run a task on each item of a dataset and record the result.""" + +from __future__ import annotations + +import asyncio +import contextvars +import copy +import inspect +import math +import os +import threading +import time +import urllib.parse +from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Literal, TypeAlias, TypeGuard, cast + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.context import Context + +from ._api import Api, request, resolve_api +from ._ci import github_run, in_ci, min_scores_from_env, write_result +from ._client import flush, get_client +from ._config import logger, report_error +from ._context import context_with_propagated_attributes, propagated_attributes +from ._scores import ( + MAX_REASON_LENGTH, + MAX_SCORE_NAME_LENGTH, + USER_FEEDBACK, + check_value, + is_finite, +) +from ._semconv import ATTR_DATASET_ITEM_ID, ATTR_EXPERIMENT_ID, ATTR_SESSION_ID +from ._serialize import json_dumps, truncate, utf16_length +from ._spans import SpanHandle, start_span + +# The experiment environment, which pages, metrics, and alerts leave out by default. +_EXPERIMENT_ENVIRONMENT = "experiment" +_MAX_RUNS_PER_ITEM = 5 +_MAX_CONCURRENCY = 100 +# Server evaluators of one experiment, and scorers: an item takes at most 20 scores. +_MAX_EVALUATORS = 20 +_MAX_MIN_SCORES = 20 +_MAX_ERROR_LENGTH = 4000 +_MAX_OUTPUT_LENGTH = 100_000 +_MAX_DATASET_ITEMS = 10_000 +# Items in one POST /v1/experiments/{id}/items. +_MAX_BATCH_ITEMS = 100 +# Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. +_MAX_BATCH_BYTES = 4 * 1024 * 1024 +_REQUEST_TIMEOUT_S = 30.0 +# How long run_experiment waits for the server evaluators after the last item. +_MAX_WAIT_S = 30 * 60.0 +_MAX_POLL_INTERVAL_S = 5.0 +_FINISHED = frozenset({"completed", "failed", "stopped"}) +_RATE_KINDS = frozenset({"pass_fail", "up_down"}) + + +@dataclass(frozen=True) +class DatasetItem: + """An item of a dataset version, as ``GET /v1/datasets/{name}`` returns it. ``id`` is the key + of the item: it stays the same across dataset versions.""" + + id: str + number: int + input: Mapping[str, Any] + expected_output: str | None + metadata: Mapping[str, Any] + tags: tuple[str, ...] + split: str | None + + +@dataclass(frozen=True) +class ExperimentItem(DatasetItem): + """One run of a dataset item, ``run_index`` 0 to ``runs_per_item`` - 1.""" + + run_index: int + + +ScorerResult: TypeAlias = "bool | int | float | str | Mapping[str, Any] | None" +"""What a scorer returns. True and False are a pass and a fail, a number is the value, and a +string is the label. A mapping can have ``value``, ``label``, and ``reason``. None gives no +score.""" + +Scorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult]" +"""Scores the output of an item on your machine. The score has the name of the scorer.""" + +AsyncScorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult | Awaitable[ScorerResult]]" +"""A scorer of ``run_experiment_async``, which can be async.""" + + +@dataclass(frozen=True) +class ExperimentScore: + """One score name of an experiment. ``value`` is the pass rate (0 to 1) of a pass/fail or + up/down score, or the mean of a number score. ``kind`` is None before the score has values. + ``baseline_value`` is the value in the experiment that this one compares with, by default the + dataset baseline. ``delta`` is whole points of a pass rate, or the change of a mean to one + decimal. ``passed`` is None when the score has no pass mark.""" + + name: str + kind: str | None + value: float | None + pass_count: int + fail_count: int + baseline_value: float | None + delta: float | None + min_score: float | None + passed: bool | None + + +@dataclass(frozen=True) +class ExperimentResult: + """The result of an experiment. ``status`` is ``completed``, ``failed``, or ``stopped``. + ``check_status`` is None when the experiment has no pass marks. ``error`` tells why the + experiment failed.""" + + id: str + name: str + status: str + check_status: Literal["passed", "failed"] | None + items_total: int + items_done: int + items_failed: int + error: str | None + scores: tuple[ExperimentScore, ...] + url: str + + +class ExperimentError(Exception): + """Why run_experiment failed. ``code`` is the code of the API error (``dataset_not_found``, + ``insufficient_scope``, ``invalid_api_key``, ...), or ``missing_api_key``, ``timeout``, + ``network_error``, ``invalid_response``, or ``dataset_empty``. In CI mode it is also + ``check_failed`` or ``experiment_failed``, and ``result`` holds the result. A response with a + different status and no API error gives ``http_``. A scorer that raises or returns a + value that is not a score sends a ``scorer_failed`` error to ``on_error``.""" + + def __init__( + self, + code: str, + message: str, + status: int | None = None, + *, + result: ExperimentResult | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.result = result + + +def run_experiment( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, Scorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """Run ``task`` on each item of a dataset and record the result as an experiment. + + Each item run is its own trace, with ``td.experiment.id`` and ``td.dataset.item_id`` on each + span. The outputs go to telemetry.dev in batches of at most 100, with the scores of + ``scorers``. Then run_experiment waits for the ``evaluators`` of the server, prints the result, + and returns it. ``concurrency`` (1 to 100, default 4) threads run the items, each in a copy of + the context of the caller. + + ``dataset`` defaults to the ``dataset`` input of the experiment action, which it sets in + ``TELEMETRY_DEV_EXPERIMENT_DATASET``. When the two are set, they must be the same. + ``min_scores`` holds pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. The ``min-scores`` input of the action adds to + them and wins for the same name. ``runs_per_item`` is 1 to 5. + + The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a + request fails. When the run stops after it made the experiment, it first sends the complete + request, so that telemetry.dev ends the experiment with the item runs that it has. A task that + raises fails only its item. In CI mode (``ci``, default on when ``CI`` is ``true`` or ``1``), + it records the GitHub Actions run, and it raises when the check fails or the experiment does + not complete, so the process exits with an error. + """ + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=False, + ) + runs, created = _start(api, run) + try: + _announce(run, created) + _run_items(api, run, created, runs) + except BaseException: + _complete_after_failure(api, created) + raise + return _finish(api, run, created) + + +async def run_experiment_async( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, AsyncScorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """``run_experiment`` for async tasks and scorers. ``concurrency`` asyncio tasks run the + items, and the requests go in worker threads. A task that is not async runs on the event + loop, so it blocks the other items while it runs: use ``run_experiment`` for it.""" + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=True, + ) + runs, created = await asyncio.to_thread(_start, api, run) + try: + _announce(run, created) + await _run_items_async(api, run, created, runs) + except BaseException: + await asyncio.to_thread(_complete_after_failure, api, created) + raise + return await asyncio.to_thread(_finish, api, run, created) + + +@dataclass(frozen=True) +class _Run: + dataset: str + task: Callable[[ExperimentItem], Any] + name: str | None + dataset_version: int | None + split: str | None + evaluators: list[str] | None + scorers: list[tuple[str, Callable[[Any, ExperimentItem], Any]]] + min_scores: dict[str, float] | None + runs_per_item: int + concurrency: int + metadata: dict[str, Any] | None + ci: dict[str, Any] | None + in_ci: bool + + +@dataclass(frozen=True) +class _Created: + id: str + name: str + items_total: int + url: str + + +@dataclass(frozen=True) +class _Page: + version: int + items: list[DatasetItem] + next_cursor: str | None + + +def _prepare( + task: object, + dataset: object, + name: object, + dataset_version: object, + split: object, + evaluators: object, + scorers: object, + min_scores: object, + runs_per_item: object, + concurrency: object, + metadata: object, + ci: object, + api_key: str | None, + base_url: str | None, + *, + asynchronous: bool, +) -> tuple[_Run, Api]: + env = os.environ + if not callable(task): + raise TypeError("task must be a function") + if not asynchronous and inspect.iscoroutinefunction(task): + raise TypeError("task is async: use run_experiment_async") + for key, value in (("dataset", dataset), ("name", name), ("split", split)): + if value is not None and (not isinstance(value, str) or not value): + raise TypeError(f"{key} must be a non-empty string") + action_dataset = env.get("TELEMETRY_DEV_EXPERIMENT_DATASET") or None + chosen = cast("str | None", dataset) if dataset is not None else action_dataset + if chosen is None: + raise TypeError("Pass dataset, or set the dataset input of the experiment action") + if action_dataset is not None and action_dataset != chosen: + raise TypeError( + f"dataset is {chosen}, but the experiment action names {action_dataset}. " + "Name one dataset." + ) + if dataset_version is not None and not (_is_int(dataset_version) and dataset_version >= 1): + raise ValueError("dataset_version must be a positive integer") + + evaluator_names: list[str] | None = None + if evaluators is not None: + names = ( + list(cast("Sequence[object]", evaluators)) + if isinstance(evaluators, Sequence) and not isinstance(evaluators, str) + else None + ) + if ( + names is None + or len(names) > _MAX_EVALUATORS + or not all(isinstance(n, str) and _valid_score_name(n) for n in names) + ): + raise TypeError( + f"evaluators must be a list of at most {_MAX_EVALUATORS} evaluator names" + ) + evaluator_names = cast("list[str]", names) + + if scorers is not None and not isinstance(scorers, Mapping): + raise TypeError("scorers must be a mapping of score names to functions") + scorer_list = list(cast("Mapping[object, object]", scorers or {}).items()) + if len(scorer_list) > _MAX_EVALUATORS: + raise TypeError(f"scorers can have at most {_MAX_EVALUATORS} scorers") + folded = {n.lower() for n in evaluator_names or []} + for scorer_name, scorer in scorer_list: + if ( + not isinstance(scorer_name, str) + or not _valid_score_name(scorer_name) + or not callable(scorer) + ): + raise TypeError( + f"scorers must map score names of 1 to {MAX_SCORE_NAME_LENGTH} characters " + "to functions" + ) + if not asynchronous and inspect.iscoroutinefunction(scorer): + raise TypeError(f"The scorer {scorer_name} is async: use run_experiment_async") + if scorer_name == USER_FEEDBACK: + raise TypeError( + f"{USER_FEEDBACK} is the score of user feedback. Name the scorer another way." + ) + if scorer_name.lower() in folded: + raise TypeError( + f"The scorer {scorer_name} has the name of an evaluator. Rename one of them." + ) + + if min_scores is not None and not isinstance(min_scores, Mapping): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + marks: dict[object, object] = { + **cast("Mapping[object, object]", min_scores or {}), + **min_scores_from_env(env), + } + if len(marks) > _MAX_MIN_SCORES or not all( + isinstance(n, str) and _valid_score_name(n) and is_finite(v) for n, v in marks.items() + ): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + + if not _is_int(runs_per_item) or not 1 <= runs_per_item <= _MAX_RUNS_PER_ITEM: + raise ValueError(f"runs_per_item must be an integer from 1 to {_MAX_RUNS_PER_ITEM}") + if not _is_int(concurrency) or not 1 <= concurrency <= _MAX_CONCURRENCY: + raise ValueError(f"concurrency must be an integer from 1 to {_MAX_CONCURRENCY}") + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + if ci is not None and not isinstance(ci, bool): + raise TypeError("ci must be a bool") + ci_mode = in_ci(env) if ci is None else ci + + api = resolve_api(api_key, base_url) + if not api.api_key: + raise ExperimentError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + run = _Run( + dataset=chosen, + task=task, + name=cast("str | None", name), + dataset_version=dataset_version, + split=cast("str | None", split), + evaluators=evaluator_names, + scorers=cast("list[tuple[str, Callable[[Any, ExperimentItem], Any]]]", scorer_list), + min_scores={str(n): float(cast("float", v)) for n, v in marks.items()} or None, + runs_per_item=runs_per_item, + concurrency=concurrency, + metadata=dict(cast("Mapping[str, Any]", metadata)) if metadata is not None else None, + ci=github_run(env) if ci_mode else None, + in_ci=ci_mode, + ) + return run, api + + +def _valid_score_name(name: str) -> bool: + """A trimmed name of 1 to 64 characters, as the API stores score names.""" + return 0 < utf16_length(name) <= MAX_SCORE_NAME_LENGTH and name.strip() == name + + +def _is_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) + + +def _warn_about_client() -> None: + client = get_client() + environment = client.config.environment if client is not None and client.enabled else None + if environment is None: + logger.warning( + "run_experiment: no telemetry client, so the items record no traces. " + 'Call init(environment="experiment") first.' + ) + elif environment.strip().lower() != _EXPERIMENT_ENVIRONMENT: + logger.warning( + "run_experiment: the traces of the items go to the environment %s. " + 'Call init(environment="experiment") to keep them out of it.', + environment, + ) + + +def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: + _warn_about_client() + first = _fetch_page(api, run, run.dataset_version, None) + if not first.items: + scope = "" if run.split is None else f" in the split {run.split}" + raise ExperimentError("dataset_empty", f"The dataset {run.dataset} has no items{scope}") + body = { + "name": run.name, + "dataset": run.dataset, + "datasetVersion": first.version, + "split": run.split, + "evaluators": run.evaluators, + "minScores": run.min_scores, + "runsPerItem": run.runs_per_item, + "ci": run.ci, + "metadata": run.metadata, + } + created = _parse_created( + request( + api, + ExperimentError, + "POST", + "/v1/experiments", + body={key: value for key, value in body.items() if value is not None}, + timeout_s=_REQUEST_TIMEOUT_S, + # A repeat after a lost response would start a second experiment. + idempotent=False, + ) + ) + return _Runs(api, run, first), created + + +def _announce(run: _Run, created: _Created) -> None: + each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" + print( + f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " + f"{created.url}", + flush=True, + ) + + +def _complete_after_failure(api: Api, created: _Created) -> None: + """Without the complete, the experiment would stay running: the server ends it with the item + runs that it has, or fails it when it has none.""" + try: + _complete(api, created) + except Exception as exc: + report_error(api.on_error, "completing the experiment after a failed run failed", exc) + + +class _Runs: + """The runs of each item of the dataset version, page by page, so that only one page is in + memory. A lock lets the workers share the generator.""" + + def __init__(self, api: Api, run: _Run, first: _Page) -> None: + self._lock = threading.Lock() + self._runs = _item_runs(api, run, first) + + def next(self) -> ExperimentItem | None: + with self._lock: + return next(self._runs, None) + + def close(self) -> None: + with self._lock: + self._runs.close() + + +def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: + page = first + count = 0 + while True: + count += len(page.items) + if count > _MAX_DATASET_ITEMS: + raise ExperimentError( + "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" + ) + for item in page.items: + # A copy for each run, so that a task that changes its input does not change the next + # run. + for run_index in range(run.runs_per_item): + yield ExperimentItem(**copy.deepcopy(vars(item)), run_index=run_index) + if page.next_cursor is None: + return + page = _fetch_page(api, run, page.version, page.next_cursor) + + +def _fetch_page(api: Api, run: _Run, version: int | None, cursor: str | None) -> _Page: + """One page of a dataset version. A page after the first names the version of the first.""" + page = _parse_page( + request( + api, + ExperimentError, + "GET", + f"/v1/datasets/{urllib.parse.quote(run.dataset, safe='')}", + query={ + "version": None if version is None else str(version), + "split": run.split, + "cursor": cursor, + }, + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + ) + if ( + page is None + or (version is not None and page.version != version) + or (page.next_cursor is not None and (not page.items or page.next_cursor == cursor)) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a dataset it cannot read") + return page + + +class _Batch: + """Collects item results and posts them in order, one post at a time.""" + + def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: + self._post = post + self._pending: list[dict[str, Any]] = [] + self._bytes = 0 + self._lock = threading.Lock() + + def add(self, result: dict[str, Any]) -> None: + size = len(json_dumps(result, default=None).encode()) + with self._lock: + if self._pending and self._bytes + size > _MAX_BATCH_BYTES: + self._send() + self._pending.append(result) + self._bytes += size + if len(self._pending) >= _MAX_BATCH_ITEMS: + self._send() + + def flush(self) -> None: + with self._lock: + if self._pending: + self._send() + + def _send(self) -> None: + results = self._pending + self._pending = [] + self._bytes = 0 + self._post(results) + + +def _batch(api: Api, created: _Created) -> _Batch: + path = f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/items" + return _Batch( + lambda results: request( + api, + ExperimentError, + "POST", + path, + body={"items": results}, + timeout_s=_REQUEST_TIMEOUT_S, + # The server skips an item run that it has, so a repeat is safe. + idempotent=True, + ) + ) + + +def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch + that fails stops the run: no new item starts, and _run_items raises when the running items + end.""" + batch = _batch(api, created) + failures: list[BaseException] = [] + caller = contextvars.copy_context() + + def worker() -> None: + while not failures: + try: + item = runs.next() + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + batch.add(caller.copy().run(_run_item, run, created, item, api.on_error)) + except Exception as exc: + failures.append(exc) + + with ThreadPoolExecutor(run.concurrency, thread_name_prefix="telemetry-dev-experiment") as pool: + futures = [pool.submit(worker) for _ in range(run.concurrency)] + try: + for future in futures: + future.result() + except BaseException as exc: + # Stops the workers, for example after Ctrl-C. + failures.append(exc) + raise + _end_items(batch, runs, failures) + + +async def _run_items_async(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + batch = _batch(api, created) + failures: list[BaseException] = [] + + async def worker() -> None: + while not failures: + try: + item = await asyncio.to_thread(runs.next) + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) + await asyncio.to_thread(batch.add, result) + except Exception as exc: + failures.append(exc) + + # gather runs each worker in its own task, with its own copy of the context. + await asyncio.gather(*(worker() for _ in range(run.concurrency))) + await asyncio.to_thread(_end_items, batch, runs, failures) + + +def _end_items(batch: _Batch, runs: _Runs, failures: list[BaseException]) -> None: + if not failures: + try: + batch.flush() + except Exception as exc: + failures.append(exc) + if failures: + runs.close() + raise failures[0] + + +def _run_item( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + _close(output) + raise TypeError("task returned an awaitable: use run_experiment_async") + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + _add_scores( + result, [_score(name, scorer, output, item, on_error) for name, scorer in run.scorers] + ) + return result + + +async def _run_item_async( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + output = await output + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + scores: list[dict[str, Any] | None] = [] + for name, scorer in run.scorers: + try: + value = scorer(output, item) + if inspect.isawaitable(value): + value = await value + scores.append(_to_score(name, value)) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + scores.append(None) + _add_scores(result, scores) + return result + + +def _item_context(experiment_id: str, item: ExperimentItem) -> Context: + """The context of one item run. It drops the active span, so that the run starts a trace of + its own, and gives the run a session id of its own: an ambient session or + ``session_mode="process"`` would otherwise put all runs in one session trace.""" + base = trace.set_span_in_context(trace.INVALID_SPAN, otel_context.get_current()) + return context_with_propagated_attributes( + { + **propagated_attributes(base), + ATTR_EXPERIMENT_ID: experiment_id, + ATTR_DATASET_ITEM_ID: item.id, + ATTR_SESSION_ID: f"{experiment_id}:{item.id}:{item.run_index}", + }, + base, + ) + + +def _trace_id(span: SpanHandle) -> str | None: + context = span.span.get_span_context() + return f"{context.trace_id:032x}" if span.span.is_recording() and context.is_valid else None + + +def _item_result( + item: ExperimentItem, + trace_id: str | None, + started: float, + output: Any, + error: Exception | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "itemId": item.id, + "runIndex": item.run_index, + "output": None if error is not None else _output_text(output), + "latencyMs": round((perf_counter() - started) * 1000), + } + if trace_id is not None: + result["traceId"] = trace_id + if error is not None: + result["error"] = truncate(_error_text(error), _MAX_ERROR_LENGTH) + return result + + +def _add_scores(result: dict[str, Any], scores: list[dict[str, Any] | None]) -> None: + kept = [score for score in scores if score is not None] + if kept: + result["scores"] = kept + + +def _score( + name: str, + scorer: Callable[[Any, ExperimentItem], Any], + output: Any, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any] | None: + # After the span of the item ends, so that the model calls of a scorer do not count in the + # cost of the item. + try: + value = scorer(output, item) + if inspect.isawaitable(value): + _close(value) + raise TypeError("returned an awaitable: use run_experiment_async") + return _to_score(name, value) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + return None + + +def _scorer_failed( + on_error: Callable[[BaseException], None] | None, + name: str, + item: ExperimentItem, + exc: Exception, +) -> None: + error = ExperimentError( + "scorer_failed", f"The scorer {name} failed on item {item.number}: {_error_text(exc)}" + ) + error.__cause__ = exc + report_error(on_error, "a scorer failed", error) + + +def _to_score(name: str, result: object) -> dict[str, Any] | None: + """The score of a scorer result. It raises for a result that the server would reject.""" + if result is None: + return None + score: dict[str, Any] = {"name": name} + if isinstance(result, bool): + score.update(_pass_fail(result)) + elif isinstance(result, (int, float)): + score["value"] = result + elif isinstance(result, str): + score["label"] = result.strip() + elif isinstance(result, Mapping): + fields = cast("Mapping[str, object]", result) + value, label, reason = fields.get("value"), fields.get("label"), fields.get("reason") + for key, text in (("label", label), ("reason", reason)): + if text is not None and not isinstance(text, str): + raise TypeError(f"returned a {key} that is not a string") + if isinstance(value, bool): + score.update(_pass_fail(value)) + elif value is not None: + score["value"] = value + if isinstance(label, str): + score["label"] = label.strip() + if isinstance(reason, str): + score["reason"] = truncate(reason, MAX_REASON_LENGTH) + else: + raise TypeError(f"returned a {type(result).__name__}") + problem = check_value(name, score.get("value"), score.get("label")) + if problem is not None: + raise TypeError(f"returned a score that is not valid: {problem}") + return score + + +def _pass_fail(passed: bool) -> dict[str, Any]: + return {"value": 1, "label": "pass"} if passed else {"value": 0, "label": "fail"} + + +def _output_text(output: Any) -> str | None: + """The output as text. An output that JSON cannot hold, such as a dict that holds itself, is + None.""" + if output is None: + return None + try: + text = output if isinstance(output, str) else json_dumps(output) + except Exception: + return None + return truncate(_well_formed(text), _MAX_OUTPUT_LENGTH) + + +def _error_text(error: BaseException) -> str: + try: + text = f"{type(error).__name__}: {error}" + except Exception: + # A __str__ that raises. + return type(error).__name__ + return _well_formed(text) + + +def _well_formed(text: str) -> str: + # truncate fails on a lone surrogate, which a str can hold, so each one becomes U+FFFD. + return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace") + + +def _close(awaitable: object) -> None: + # A coroutine that never runs warns when it is collected. + close = getattr(awaitable, "close", None) + if callable(close): + close() + + +def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: + # The spans of the items reach ingest before the server reads their cost. + flush() + _complete(api, created) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _complete(api: Api, created: _Created) -> None: + request( + api, + ExperimentError, + "POST", + f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/complete", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + + +def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: + """Wait until the server evaluators finish and the experiment ends.""" + deadline = time.monotonic() + _MAX_WAIT_S + interval = 1.0 + while True: + body = request( + api, + ExperimentError, + "GET", + f"/v1/experiments/{urllib.parse.quote(experiment_id, safe='')}", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result = _parse_result(body) + if result.status in _FINISHED: + return result, body + if time.monotonic() + interval > deadline: + raise ExperimentError( + "timeout", + f"The experiment {result.name} did not end in {_MAX_WAIT_S / 60:g} minutes. " + f"{result.url}", + result=result, + ) + time.sleep(interval) + interval = min(interval * 2, _MAX_POLL_INTERVAL_S) + + +def _check_result(result: ExperimentResult) -> None: + """In CI mode, a failed check or an experiment that did not complete fails the run.""" + if result.status != "completed": + reason = f": {result.error}" if result.error else "" + raise ExperimentError( + "experiment_failed", + f"The experiment {result.name} {result.status}{reason}. {result.url}", + result=result, + ) + if result.check_status != "failed": + return + failed = [ + f"{score.name} is {_format(score.kind, score.value)}, under the pass mark " + f"{_format(score.kind, score.min_score)}" + for score in result.scores + if score.passed is False + ] + detail = f": {'; '.join(failed)}" if failed else "" + raise ExperimentError( + "check_failed", + f"The check of {result.name} failed{detail}. {result.url}", + result=result, + ) + + +def _describe(result: ExperimentResult) -> str: + if result.status != "completed": + outcome = result.status + (f": {result.error}" if result.error else "") + elif result.check_status is None: + outcome = "completed" + else: + outcome = f"completed. The check {result.check_status}" + lines = [f"Experiment {result.name} {outcome}. {result.url}"] + for score in result.scores: + mark = ( + "" + if score.min_score is None + else f" (pass mark {_format(score.kind, score.min_score)})" + ) + lines.append(f" {score.name}: {_format(score.kind, score.value)}{mark}") + return "\n".join(lines) + + +def _format(kind: str | None, value: float | None) -> str: + if value is None: + return "no score" + if kind in _RATE_KINDS: + return f"{_js_round(value * 100)}%" + rounded = _js_round(value * 100) / 100 + return str(int(rounded)) if rounded.is_integer() else repr(rounded) + + +def _js_round(value: float) -> int: + # Math.round of JavaScript, which rounds a half up. round() of Python rounds it to even. + return math.floor(value + 0.5) + + +def _parse_created(body: object) -> _Created: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + created_id, name, url = fields.get("id"), fields.get("name"), fields.get("url") + items_total = fields.get("itemsTotal") + if ( + not isinstance(created_id, str) + or not isinstance(name, str) + or not isinstance(url, str) + or not _is_int(items_total) + ): + raise ExperimentError( + "invalid_response", "telemetry.dev returned an experiment it cannot read" + ) + return _Created(created_id, name, items_total, url) + + +def _parse_page(body: object) -> _Page | None: + if not isinstance(body, dict): + return None + fields = cast("dict[str, Any]", body) + dataset, items = fields.get("dataset"), fields.get("items") + next_cursor = fields.get("nextCursor") + if not isinstance(dataset, dict) or not isinstance(items, list): + return None + version = cast("dict[str, Any]", dataset).get("version") + if not _is_int(version) or version < 0: + return None + if next_cursor is not None and not isinstance(next_cursor, str): + return None + parsed: list[DatasetItem] = [] + for raw in cast("list[object]", items): + item = _parse_item(raw) + if item is None: + return None + parsed.append(item) + return _Page(version, parsed, next_cursor) + + +def _parse_item(raw: object) -> DatasetItem | None: + if not isinstance(raw, dict): + return None + item = cast("dict[str, Any]", raw) + item_id, number, tags = item.get("id"), item.get("number"), item.get("tags") + item_input, metadata = item.get("input"), item.get("metadata") + expected, split = item.get("expectedOutput"), item.get("split") + if ( + not isinstance(item_id, str) + or not _is_int(number) + or not isinstance(item_input, dict) + or not (expected is None or isinstance(expected, str)) + or not isinstance(metadata, dict) + or not isinstance(tags, list) + or not all(isinstance(tag, str) for tag in cast("list[object]", tags)) + or not (split is None or isinstance(split, str)) + ): + return None + return DatasetItem( + id=item_id, + number=number, + input=cast("dict[str, Any]", item_input), + expected_output=expected, + metadata=cast("dict[str, Any]", metadata), + tags=tuple(cast("list[str]", tags)), + split=split, + ) + + +def _parse_result(body: object) -> ExperimentResult: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + raw_scores = fields.get("scores") + scores = ( + [_parse_score(score) for score in cast("list[object]", raw_scores)] + if isinstance(raw_scores, list) + else None + ) + # Required: it decides the result of CI mode, so a missing one must not read as no check. + check_status = fields.get("checkStatus", "missing") + items_failed = fields.get("itemsFailed", 0) + error = fields.get("error") + if ( + not isinstance(fields.get("id"), str) + or not isinstance(fields.get("name"), str) + or not isinstance(fields.get("status"), str) + or not isinstance(fields.get("url"), str) + or check_status not in (None, "passed", "failed") + or not _is_number(fields.get("itemsTotal")) + or not _is_number(fields.get("itemsDone")) + or not _is_number(items_failed) + or not (error is None or isinstance(error, str)) + or scores is None + or any(score is None for score in scores) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read") + return ExperimentResult( + id=fields["id"], + name=fields["name"], + status=fields["status"], + check_status=check_status, + items_total=int(fields["itemsTotal"]), + items_done=int(fields["itemsDone"]), + items_failed=int(items_failed), + error=error, + scores=tuple(cast("list[ExperimentScore]", scores)), + url=fields["url"], + ) + + +def _parse_score(raw: object) -> ExperimentScore | None: + """A score of the result. A field that can be None can also be missing.""" + if not isinstance(raw, dict): + return None + score = cast("dict[str, Any]", raw) + name, kind, passed = score.get("name"), score.get("kind"), score.get("passed") + pass_count, fail_count = score.get("passCount"), score.get("failCount") + nullable = [score.get(key) for key in ("value", "baselineValue", "delta", "minScore")] + if ( + not isinstance(name, str) + or not (kind is None or isinstance(kind, str)) + or not _is_number(pass_count) + or not _is_number(fail_count) + or not all(value is None or _is_number(value) for value in nullable) + or not (passed is None or isinstance(passed, bool)) + ): + return None + value, baseline_value, delta, min_score = nullable + return ExperimentScore( + name=name, + kind=kind, + value=value, + pass_count=int(cast("float", pass_count)), + fail_count=int(cast("float", fail_count)), + baseline_value=baseline_value, + delta=delta, + min_score=min_score, + passed=passed, + ) + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 9673740..c7e9e80 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -4,7 +4,9 @@ import contextlib import copy import hashlib +import itertools import json +import os import random import re import socket @@ -17,18 +19,16 @@ from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field -from datetime import datetime, timezone -from email.message import Message -from email.utils import parsedate_to_datetime from time import monotonic -from typing import IO, Any, Literal, TypeAlias, TypedDict, cast, overload -from weakref import WeakKeyDictionary +from typing import Any, Literal, TypeAlias, TypedDict, cast, overload +from weakref import WeakKeyDictionary, ref from opentelemetry import context as otel_context from opentelemetry.context import Context +from ._api import NoRedirect, api_error, auth_headers, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error, resolve_config +from ._config import report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -45,11 +45,14 @@ # The longest Retry-After that get_prompt honors, so that a bad header cannot stop fetches. _MAX_RETRY_AFTER_S = 3600.0 _MAX_CACHE_ENTRIES = 1000 -# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. -_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") +# How many messages prompt_attributes_for looks at, at each end of the list. +_TAG_LOOKUP = 1000 +# How long a tag keeps its variables after its compile, or after a call that names it. +_VARIABLES_TTL_S = 300.0 +_MINUTE_S = 60.0 class PromptModel(TypedDict): @@ -98,14 +101,31 @@ def compile( ) -> list[dict[str, Any]]: """Fill in the variables and replace each placeholder message with the list of messages in its variable. ``with_prompt`` records the variables of the last call before it starts - or inside it.""" + or inside it. Each message that it makes is a ``dict`` subclass that also holds this + prompt and a copy of these variables and of each list in them, so that a model call + outside ``with_prompt`` that sends the message records them; see + ``prompt_attributes_for``. The objects in a list are not copied. The message keeps the + variables for 5 minutes after this compile and after each call that names it. Other + compiles do not change this. The message holds no variables when input capture is off in + the client. A check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and + ``marshal`` reject it: give them ``dict(message)``.""" values = _record(self, variables, kwargs) + now = monotonic() + _expire(now) + tag = _CompileTag(self, None, next(_compiles), now + _VARIABLES_TTL_S) + client = get_client() + # prompt_attributes_for cannot record the variables when input capture is off in the client. + if client is None or client.capture_input: + tag.variables = values + _add_deadline(ref(tag), tag.expires) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": - compiled.append( - {"role": message["role"], "content": _fill(str(message["content"]), values)} + tagged = _CompiledMessage( + role=message["role"], content=_fill(str(message["content"]), values) ) + tagged.compiled_by = tag + compiled.append(tagged) continue name = str(message["name"]) value = values.get(name) @@ -141,7 +161,8 @@ def messages(self) -> list[dict[str, Any]]: def compile(self, variables: Mapping[str, Any] | None = None, /, **kwargs: Any) -> str: """Fill in the variables. ``with_prompt`` records the variables of the last call before it - starts or inside it.""" + starts or inside it. A string cannot carry a tag, so a model call outside ``with_prompt`` + does not record a text prompt.""" return _fill(self.text, _record(self, variables, kwargs)) @@ -282,16 +303,59 @@ def prompt_attributes( scope = otel_context.get_value(_PROMPT_KEY, context) or otel_context.get_value(_PROMPT_KEY) if not isinstance(scope, _PromptScope): return {} - prompt = scope.prompt + task = _task_variables.get() + variables = task[1] if task is not None and task[0] is scope else scope.variables + return _attributes_of(scope.prompt, variables, client if capture_input else None) + + +def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: + """The td.prompt.* attributes for a model call outside ``with_prompt`` that sends ``messages`` + (a list or tuple of messages, or one message): those of the newest ``compile()`` of a chat + prompt that made one of the messages. It looks at the first 1000 and the last 1000 messages. + The tag is on the dict that ``compile()`` made, so a copy of it gives ``{}``. Gives ``{}`` + inside ``with_prompt``, whose prompt the span gets. ``td.prompt.variables`` needs input + capture on in the client, at the compile and now, and the span drops it when its own input + capture is off. A compile gives no variables 5 minutes after it or after the last call that + named it; see ``ChatPrompt.compile``. + + It reads only a ``list`` or ``tuple``, not a subclass, so the read runs no code of the app. + With input capture on, it serializes the variables, which calls the ``mask`` of the client and + can run code of a value, such as ``__repr__``. It never raises: it reports an error to + ``on_error`` and gives ``{}``.""" + client = get_client() + try: + if isinstance(otel_context.get_value(_PROMPT_KEY), _PromptScope): + return {} + now = monotonic() + _expire(now) + tag = _newest_tag(messages) + if tag is None: + return {} + # One read: _expire in another thread can remove the variables of the tag. + variables = tag.variables + if tag.expires > now: + tag.expires = now + _VARIABLES_TTL_S + else: + variables = tag.variables = None + capture = client if client is not None and client.capture_input else None + return _attributes_of(tag.prompt, variables, capture) + except Exception as exc: + if client is not None: + client.report("prompt_attributes_for failed", exc) + return {} + + +def _attributes_of( + prompt: Prompt, variables: dict[str, Any] | None, capture: Client | None +) -> dict[str, AttributeValue]: + """The attributes of ``prompt``; ``capture`` is the client when input capture is on.""" attrs: dict[str, AttributeValue] = {ATTR_PROMPT_NAME: prompt.name} if prompt.version is not None: attrs[ATTR_PROMPT_VERSION] = prompt.version if prompt.label is not None: attrs[ATTR_PROMPT_LABEL] = prompt.label - task = _task_variables.get() - variables = task[1] if task is not None and task[0] is scope else scope.variables - if variables is not None and capture_input: - serialized = client.serialize(variables, ATTR_PROMPT_VARIABLES) + if variables is not None and capture is not None: + serialized = capture.serialize(variables, ATTR_PROMPT_VARIABLES) if serialized is not None: attrs[ATTR_PROMPT_VARIABLES] = serialized return attrs @@ -310,11 +374,11 @@ def _get_prompt( _check(name, type, label, version, cache_ttl_seconds, fallback) client = get_client() enabled = client if client is not None and client.enabled else None - config = enabled.config if enabled is not None else resolve_config() - key = api_key if api_key is not None else config.api_key - url = (base_url or config.base_url).rstrip("/") + api = resolve_api(api_key, base_url) + key = api.api_key + url = api.base_url asked_label = (label if label is not None else "production") if version is None else None - on_error = enabled.on_error if enabled is not None else None + on_error = api.on_error try: if not key: @@ -336,13 +400,7 @@ def _get_prompt( cache_ttl_seconds, lambda: _fetch( f"{url}/v1/prompts?{urllib.parse.urlencode(query)}", - { - "Authorization": f"Bearer {key}", - "X-Telemetry-Dev-SDK": "telemetry-dev", - # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters - # such as Cloudflare's Browser Integrity Check. - "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", - }, + auth_headers(key), ), on_error, ) @@ -395,7 +453,12 @@ class _PromptScope: def _record( prompt: Prompt, variables: Mapping[str, Any] | None, kwargs: Mapping[str, Any] ) -> dict[str, Any]: - values = {**(variables or {}), **kwargs} + # A copy of each list too, so that a change of the app's history after compile() does not + # change what a span records. + values: dict[str, Any] = { + name: list(cast("list[Any]", value)) if isinstance(value, list) else value + for name, value in {**(variables or {}), **kwargs}.items() + } _compiled_variables[prompt] = values scope = otel_context.get_value(_PROMPT_KEY) if isinstance(scope, _PromptScope) and scope.prompt is prompt: @@ -404,6 +467,81 @@ def _record( return values +@dataclass(eq=False) +class _CompileTag: + """The ``compile()`` call that made a chat message.""" + + prompt: ChatPrompt + variables: dict[str, Any] | None + """None when input capture was off, or after ``expires``; see _expire.""" + serial: int + """Counts the compiles of the process, so a higher one is newer.""" + expires: float + """When the tag removes its variables (monotonic). A call that names the tag moves it.""" + + +_compiles = itertools.count() +# The tags that keep variables, by the minute of their deadline. A weak reference, so that the +# variables of a message that the app dropped do not stay until the deadline. +_deadlines: dict[int, list[ref[_CompileTag]]] = {} + + +class _CompiledMessage(dict[str, Any]): + """A message that ``compile()`` made: a dict with the tag in a slot. The constructor is the one + of ``dict``, so ``type(message)(message)`` and ``dataclasses.asdict`` rebuild it, without the + tag. A copy, pickle, or ``copy.deepcopy`` of it is a plain dict without the tag.""" + + __slots__ = ("compiled_by",) + compiled_by: _CompileTag + + def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: + return dict, (dict(self),) + + +def _expire(now: float) -> None: + """Remove the variables of each expired tag, also when the app keeps its messages. Thus a chat + history keeps the variables of the last 5 minutes only, and other compiles do not change the + time of a request. A tag that a call named gets its new deadline. Threads add only to minutes + that are not due, and ``list()`` and ``pop()`` of a dict are atomic, so the threads share the + minutes without a lock.""" + minute = int(now // _MINUTE_S) + for due in list(_deadlines): + if due >= minute: + continue + for tag_ref in _deadlines.pop(due, ()): + tag = tag_ref() + if tag is None: + continue + if tag.expires > now: + _add_deadline(tag_ref, tag.expires) + else: + tag.variables = None + + +def _add_deadline(tag_ref: ref[_CompileTag], at: float) -> None: + _deadlines.setdefault(int(at // _MINUTE_S), []).append(tag_ref) + + +def _newest_tag(messages: object) -> _CompileTag | None: + """The messages of a placeholder come from an older compile, so the newest compile, not the + last tagged message, names the prompt of the request. Its own messages are at the start or the + end of the list, unless placeholders put more than 1000 messages on each side of them.""" + if type(messages) is _CompiledMessage: + return getattr(messages, "compiled_by", None) + # Exact types: a subclass of list or tuple could run code of the app on each read. + if type(messages) is not list and type(messages) is not tuple: + return None + items = cast("list[object]", messages) + if len(items) > 2 * _TAG_LOOKUP: + items = [*items[:_TAG_LOOKUP], *items[-_TAG_LOOKUP:]] + newest: _CompileTag | None = None + for item in items: + tag = getattr(item, "compiled_by", None) if type(item) is _CompiledMessage else None + if tag is not None and (newest is None or tag.serial > newest.serial): + newest = tag + return newest + + def _fill(text: str, values: Mapping[str, Any]) -> str: def replace(match: re.Match[str]) -> str: value = values.get(match.group(1)) @@ -455,12 +593,57 @@ class _Entry: retry_at: float = 0.0 # Set when the running fetch ends. pending: threading.Event | None = None + # The process whose thread runs the fetch. A forked process gets the entry, but not the thread. + pending_pid: int = 0 # When calls stop waiting for the running fetch. deadline: float = 0.0 _cache: OrderedDict[tuple[str, str, str, str], _Entry] = OrderedDict() -_cache_lock = threading.Lock() +# Reentrant, so that a fork in a thread that holds the lock, for example in a signal handler, does +# not wait for itself in _lock_for_fork. _reset_in_child can replace it, so read it from the module +# each time. +_cache_lock = threading.RLock() +# Whether _lock_for_fork got the lock in this thread. os.fork() runs the after hooks also when a +# before hook raises, for example in a signal handler. +_forking = threading.local() + + +def _lock_for_fork() -> None: + _cache_lock.acquire() + _forking.locked = True + + +def _unlock_after_fork() -> None: + if getattr(_forking, "locked", False): + _forking.locked = False + _cache_lock.release() + + +def _reset_in_child() -> None: + """Unlock the cache, and drop the fetches of the parent: the child has none of its threads.""" + global _cache_lock + if getattr(_forking, "locked", False): + _forking.locked = False + _cache_lock.release() + else: + # The before hook did not get the lock, because a signal stopped its wait, or did not run, + # as in uWSGI with py-call-osafterfork. A thread of the parent can hold the lock, and only + # this thread exists here. + _cache_lock = threading.RLock() + for entry in _cache.values(): + entry.pending = None + + +# A fork that runs these hooks waits for a thread that holds the lock, so that the child gets no +# entry that a thread changed only in part. os.register_at_fork cannot remove a hook, and +# importlib.reload runs this module again with the same globals. Thus the hooks of the first run +# stay, and they find the lock and the cache when they run. Windows has no fork. +if hasattr(os, "register_at_fork") and "_fork_hooks" not in globals(): + _fork_hooks = True + os.register_at_fork( + before=_lock_for_fork, after_in_parent=_unlock_after_fork, after_in_child=_reset_in_child + ) def _load( @@ -481,15 +664,23 @@ def _load( entry = _cache[key] = _Entry() now = monotonic() ttl = ttl_seconds if entry.response is not None else min(ttl_seconds, _ERROR_TTL_S) - if now >= max(entry.checked_at + ttl, entry.retry_at) and entry.pending is None: - entry.pending = threading.Event() + if entry.pending is not None and entry.pending_pid != os.getpid(): + # A fork in C code that runs no hooks copied the fetch but not its thread. This check + # does not replace the lock. If a thread of the parent held the lock at that fork, a + # call in the child waits for the lock with no limit. + entry.pending = None + # A local, because a fork in this thread, as in a signal handler, can clear entry.pending. + pending = entry.pending + if now >= max(entry.checked_at + ttl, entry.retry_at) and pending is None: + pending = entry.pending = threading.Event() + entry.pending_pid = os.getpid() entry.deadline = now + _FETCH_TIMEOUT_S threading.Thread( - target=_refresh, args=(entry, entry.pending, fetch, on_error), daemon=True + target=_refresh, args=(entry, pending, fetch, on_error), daemon=True ).start() if entry.response is not None: return entry.response - pending, deadline = entry.pending, entry.deadline + deadline = entry.deadline if pending is not None and not pending.wait(deadline - monotonic()): raise _timeout() with _cache_lock: @@ -535,21 +726,6 @@ def _refresh( done.set() -class _NoRedirect(urllib.request.HTTPRedirectHandler): - # A redirect to another host must not get the API key, so a 3xx fails like any status that is - # not 2xx. - def redirect_request( - self, - req: urllib.request.Request, - fp: IO[bytes], - code: int, - msg: str, - headers: Message, - newurl: str, - ) -> urllib.request.Request | None: - return None - - class _Deadline: """Ends one prompt fetch after ``_FETCH_TIMEOUT_S``. The timeout of urllib applies to each socket operation, so a server that sends a byte now and then could keep a fetch reading forever, @@ -648,12 +824,12 @@ def do_open(self, http_class: Any, req: urllib.request.Request, **http_conn_args return super().do_open(_on_deadline(http_class, req), req, **http_conn_args) -_opener = urllib.request.build_opener(_NoRedirect, _HTTPHandler, _HTTPSHandler) +_opener = urllib.request.build_opener(NoRedirect, _HTTPHandler, _HTTPSHandler) def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: deadline = _Deadline() - retry_after: str | None = None + retry_header: str | None = None try: try: with _opener.open( @@ -664,7 +840,7 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: except urllib.error.HTTPError as exc: with exc: status, body = exc.code, exc.read() - retry_after = exc.headers.get("Retry-After") + retry_header = exc.headers.get("Retry-After") except Exception as exc: if ( deadline.expired @@ -681,9 +857,10 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: if deadline.expired: raise _timeout() if not 200 <= status < 300: - error = _api_error(status, body) - if status in (429, 503): - error._retry_after = _retry_after(retry_after) # pyright: ignore[reportPrivateUsage] + error = api_error(PromptError, status, body) + wait = retry_after(status, retry_header) + if wait is not None: + error._retry_after = min(wait, _MAX_RETRY_AFTER_S) # pyright: ignore[reportPrivateUsage] raise error try: parsed = _parse_response(json.loads(body)) @@ -698,40 +875,6 @@ def _timeout() -> PromptError: return PromptError("timeout", f"Fetching the prompt took more than {_FETCH_TIMEOUT_S:g} s") -def _api_error(status: int, body: bytes) -> PromptError: - try: - error = json.loads(body).get("error") - except (ValueError, AttributeError): - error = None - fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} - code = fields.get("code") - message = fields.get("message") - return PromptError( - code if isinstance(code, str) else f"http_{status}", - message if isinstance(message, str) else f"telemetry.dev returned {status}", - status, - ) - - -def _retry_after(value: str | None) -> float | None: - """Seconds to wait from a Retry-After header of seconds or an IMF-fixdate HTTP date - (``Sun, 06 Nov 1994 08:49:37 GMT``), capped.""" - value = (value or "").strip() - if not value: - return None - if value.isascii() and value.isdigit(): - seconds = float(value) - elif _HTTP_DATE.fullmatch(value): - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError): - return None - seconds = (date - datetime.now(timezone.utc)).total_seconds() - else: - return None - return min(seconds, _MAX_RETRY_AFTER_S) if seconds > 0 else None - - def _parse_response(data: object) -> dict[str, Any] | None: if not isinstance(data, dict): return None diff --git a/sdks/python/src/telemetry_dev/_scores.py b/sdks/python/src/telemetry_dev/_scores.py new file mode 100644 index 0000000..f5dbee3 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_scores.py @@ -0,0 +1,181 @@ +"""Scores on traces, spans, and sessions, as ``POST /v1/scores`` takes them.""" + +from __future__ import annotations + +import asyncio +import math +import re +from typing import Any, cast + +from ._api import request, resolve_api +from ._config import report_error +from ._serialize import utf16_length + +USER_FEEDBACK = "user_feedback" +MAX_SCORE_NAME_LENGTH = 64 +MAX_LABEL_LENGTH = 64 +MAX_REASON_LENGTH = 4000 +_MAX_SESSION_ID_LENGTH = 1024 +_TRACE_ID = re.compile(r"(?!0+$)[0-9a-f]{32}") +_SPAN_ID = re.compile(r"(?!0+$)[0-9a-f]{16}") +_TIMEOUT_S = 10.0 + + +class ScoreError(Exception): + """Why score failed. ``code`` is the code of the API error (``insufficient_scope``, + ``invalid_api_key``, ...), or ``missing_api_key``, ``invalid_score``, ``timeout``, + ``network_error``, or ``invalid_response``. A response with a different status and no API + error gives ``http_``. ``status`` is the HTTP status of the response.""" + + def __init__(self, code: str, message: str, status: int | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + + +def score( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """Record a score on a trace, a span, or a session, for example the feedback of a user on a + reply. ``user_feedback`` is a thumbs up (1) or down (0). Pass ``trace_id``, ``session_id``, or + the two, and ``value``, ``label``, or the two. + + Like the telemetry functions, score never raises: it returns the id of the score, or None + after it sends the error to ``on_error``. It makes one request, with no retry, so that a call + in a request handler waits 10 seconds at most for each socket operation. The trace id of a + span from a client that is not enabled is all zeros, and score reports it as + ``invalid_score``. + """ + api = resolve_api(api_key, base_url) + try: + problem = _check_score(name, trace_id, span_id, session_id, value, label, comment) + if problem is not None: + raise ScoreError("invalid_score", problem) + if not api.api_key: + raise ScoreError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + fields = { + "traceId": trace_id, + "spanId": span_id, + "sessionId": session_id, + "name": name.strip(), + "value": value, + "label": None if label is None else label.strip(), + "comment": comment, + } + body = request( + api, + ScoreError, + "POST", + "/v1/scores", + body={key: field for key, field in fields.items() if field is not None}, + timeout_s=_TIMEOUT_S, + idempotent=False, + attempts=1, + ) + score_id = cast("dict[str, Any]", body).get("id") if isinstance(body, dict) else None + if not isinstance(score_id, str): + raise ScoreError("invalid_response", "telemetry.dev returned a score it cannot read") + return score_id + except Exception as exc: + report_error(api.on_error, "sending the score failed", exc) + return None + + +async def score_async( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """``score`` in a worker thread, so that the request does not block the event loop.""" + return await asyncio.to_thread( + score, + name=name, + trace_id=trace_id, + span_id=span_id, + session_id=session_id, + value=value, + label=label, + comment=comment, + api_key=api_key, + base_url=base_url, + ) + + +def check_value(name: str, value: object, label: object) -> str | None: + """Why the server would reject the value and the label of a score, or None. A pass/fail score + has the label ``pass`` with the value 1, or ``fail`` with 0.""" + if value is not None and not is_finite(value): + return "value must be a finite number" + if label is not None and not ( + isinstance(label, str) and 0 < utf16_length(label) <= MAX_LABEL_LENGTH + ): + return f"label must be a string of 1 to {MAX_LABEL_LENGTH} characters" + if value is None and label is None: + return "Pass a value or a label" + if name == USER_FEEDBACK and not (label is None and value in (0, 1)): + return f"{USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label" + if label == "pass" and value is not None and value != 1: + return "The label pass goes with the value 1" + if label == "fail" and value is not None and value != 0: + return "The label fail goes with the value 0" + return None + + +def _check_score( + name: object, + trace_id: object, + span_id: object, + session_id: object, + value: object, + label: object, + comment: object, +) -> str | None: + if not isinstance(name, str) or not 0 < utf16_length(name.strip()) <= MAX_SCORE_NAME_LENGTH: + return f"name must be a string of 1 to {MAX_SCORE_NAME_LENGTH} characters" + if trace_id is None and session_id is None: + return "Pass trace_id or session_id" + if trace_id is not None and not (isinstance(trace_id, str) and _TRACE_ID.fullmatch(trace_id)): + return "trace_id must be 32 lowercase hex digits and not all zeros" + if span_id is not None and not (isinstance(span_id, str) and _SPAN_ID.fullmatch(span_id)): + return "span_id must be 16 lowercase hex digits and not all zeros" + if span_id is not None and trace_id is None: + return "Pass the trace_id of the span" + if session_id is not None and not ( + isinstance(session_id, str) and 0 < utf16_length(session_id) <= _MAX_SESSION_ID_LENGTH + ): + return f"session_id must be a string of 1 to {_MAX_SESSION_ID_LENGTH} characters" + if comment is not None and not ( + isinstance(comment, str) and utf16_length(comment) <= MAX_REASON_LENGTH + ): + return f"comment must be a string of at most {MAX_REASON_LENGTH} characters" + return check_value(name.strip(), value, label.strip() if isinstance(label, str) else label) + + +def is_finite(value: object) -> bool: + # True and False are ints in Python, but JSON sends them as booleans, which are not numbers. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + # An int too large for a float, which JavaScript reads as Infinity. + return False diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 69553c0..7d3041b 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -88,6 +88,8 @@ ATTR_PROMPT_VERSION = "td.prompt.version" ATTR_PROMPT_LABEL = "td.prompt.label" ATTR_PROMPT_VARIABLES = "td.prompt.variables" +ATTR_EXPERIMENT_ID = "td.experiment.id" +ATTR_DATASET_ITEM_ID = "td.dataset.item_id" USAGE_ATTRS: dict[str, str] = { "input_tokens": "gen_ai.usage.input_tokens", diff --git a/sdks/python/src/telemetry_dev/_serialize.py b/sdks/python/src/telemetry_dev/_serialize.py index dcf1321..f69c4a6 100644 --- a/sdks/python/src/telemetry_dev/_serialize.py +++ b/sdks/python/src/telemetry_dev/_serialize.py @@ -30,6 +30,11 @@ class MaskContext: Mask = Callable[[Any, MaskContext], Any] +def utf16_length(text: str) -> int: + """The length of ``text`` in UTF-16 units, as JavaScript and the telemetry.dev API count it.""" + return len(text.encode("utf-16-le", "surrogatepass")) // 2 + + def truncate(text: str, max_len: int, *, keep_end: bool = False) -> str: """Cap ``text`` at ``max_len`` UTF-16 units, marking the cut. diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 0ac1b26..6328fa3 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -38,6 +38,7 @@ ATTR_FINISH_REASONS, ATTR_OPERATION, ATTR_OUTPUT_TYPE, + ATTR_PROMPT_VARIABLES, ATTR_PROVIDER, ATTR_REQUEST_MODEL, ATTR_RESPONSE_ID, @@ -102,6 +103,7 @@ class _SpanState: operation: str capture_input: bool capture_output: bool + prompt_from_scope: bool = False output_chunk_last_ms: float | None = None output_chunk_count: int = 0 output_chunk_sum_s: float = 0 @@ -279,6 +281,9 @@ def _fields_to_attributes( raw = fields.get("attributes") if raw is not None: for key, value in raw.items(): + # The variables are input, also when the app passes them from prompt_attributes_for(). + if key == ATTR_PROMPT_VARIABLES and not state.capture_input: + continue attr = coerce_attr_value( value, max_len=sys.maxsize, @@ -296,6 +301,8 @@ def _apply_fields(client: Client, span: Span, state: _SpanState, fields: dict[st if name is not None: span.update_name(name) attrs = _fields_to_attributes(client, state, fields) + if state.prompt_from_scope: + attrs = _without_prompt(attrs) if attrs: span.set_attributes(attrs) @@ -304,6 +311,15 @@ def _apply_fields(client: Client, span: Span, state: _SpanState, fields: dict[st _record_error_on_span(client, span, state, error) +def _without_prompt( + attrs: dict[str, otel_types.AttributeValue], +) -> dict[str, otel_types.AttributeValue]: + """The prompt of with_prompt replaces each td.prompt.* attribute that the app passes, at start + and in each update, so that a span does not mix two prompts when the app calls + prompt_attributes_for outside with_prompt.""" + return {key: value for key, value in attrs.items() if not key.startswith("td.prompt.")} + + def _collect_fields( *, input: Any = NOT_GIVEN, @@ -744,11 +760,13 @@ def start_span( ctx, session_id if isinstance(session_id, str) else None, client.config.api_key ) initial.update(propagated) - initial.update( - prompt_attributes( - client, ctx, attrs.get(ATTR_OPERATION, operation), state.capture_input - ) + prompt = prompt_attributes( + client, ctx, attrs.get(ATTR_OPERATION, operation), state.capture_input ) + if prompt: + state.prompt_from_scope = True + attrs = _without_prompt(attrs) + initial.update(prompt) initial.update(attrs) span = client.tracer.start_span( name, diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py new file mode 100644 index 0000000..d86f4cd --- /dev/null +++ b/sdks/python/tests/fake_api.py @@ -0,0 +1,221 @@ +"""The Evals routes of telemetry.dev on a local HTTP server: pages of a dataset version with a +cursor, an experiment that takes item posts of at most 100 and skips repeats, and its result.""" + +from __future__ import annotations + +import json +import sys +import threading +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qsl, urlsplit + +EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff" +EXPERIMENT_URL = f"https://telemetry.dev/o/org/p/project/evals/experiments/{EXPERIMENT_ID}" + + +def item_id(number: int) -> str: + return f"00000000-0000-4000-8000-{number:012d}" + + +def dataset_item(number: int, split: str | None = None) -> dict[str, Any]: + return { + "id": item_id(number), + "number": number, + "input": {"question": f"q{number}"}, + "expectedOutput": f"a{number}", + "metadata": {"source": "test"}, + "tags": ["golden"], + "split": split, + } + + +def result(**overrides: Any) -> dict[str, Any]: + body: dict[str, Any] = { + "id": EXPERIMENT_ID, + "name": "support-golden-v3", + "status": "completed", + "checkStatus": None, + "itemsTotal": 2, + "itemsDone": 2, + "itemsFailed": 0, + "error": None, + "scores": [], + "url": EXPERIMENT_URL, + } + body.update(overrides) + return body + + +@dataclass +class ApiRequest: + method: str + path: str + query: dict[str, str] + headers: dict[str, str] + body: Any + + @property + def route(self) -> str: + return f"{self.method} {self.path}" + + +# (status, JSON body, headers), or None to answer as the API does. +Reply = tuple[int, object, dict[str, str]] + + +@dataclass(frozen=True) +class Drop: + """Closes the connection with no answer, as a network failure does.""" + + +@dataclass(frozen=True) +class Late: + """Answers after a wait, as a server that is slower than the timeout of the client.""" + + seconds: float + reply: Reply + + +@dataclass +class FakeApi: + url: str + items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) + page_size: int = 100 + results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) + override: Callable[[ApiRequest], Reply | Drop | Late | None] | None = None + requests: list[ApiRequest] = field(default_factory=lambda: []) + stored: set[str] = field(default_factory=lambda: set[str]()) + lock: threading.Lock = field(default_factory=threading.Lock) + + def routes(self) -> list[str]: + return [request.route for request in self.requests] + + def posts(self) -> list[list[dict[str, Any]]]: + return [ + request.body["items"] for request in self.requests if request.path.endswith("/items") + ] + + def answer(self, request: ApiRequest) -> Reply | Drop | Late: + with self.lock: + self.requests.append(request) + overridden = self.override(request) if self.override else None + if overridden is not None: + return overridden + return self._route(request) + + def _route(self, request: ApiRequest) -> Reply: + if request.route == "GET /v1/datasets/support-golden": + split = request.query.get("split") + start = int(request.query.get("cursor", "0")) + matching = [item for item in self.items if split is None or item["split"] == split] + more = start + self.page_size < len(matching) + return ( + 200, + { + "dataset": { + "id": "dataset-1", + "name": "support-golden", + "version": int(request.query.get("version", "3")), + }, + "items": matching[start : start + self.page_size], + "nextCursor": str(start + self.page_size) if more else None, + }, + {}, + ) + if request.route == "POST /v1/experiments": + split = request.body.get("split") + count = len([item for item in self.items if split is None or item["split"] == split]) + return ( + 201, + { + "id": EXPERIMENT_ID, + "name": request.body.get("name", "support-golden-v3"), + "datasetVersion": request.body.get("datasetVersion"), + "itemsTotal": count * request.body.get("runsPerItem", 1), + "url": EXPERIMENT_URL, + }, + {}, + ) + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/items": + runs = request.body["items"] + if len(runs) > 100: + return 400, {"error": {"code": "invalid_request", "message": "at most 100"}}, {} + keys = [f"{run['itemId']}:{run.get('runIndex', 0)}" for run in runs] + accepted = len([key for key in keys if key not in self.stored]) + self.stored.update(keys) + return 200, {"accepted": accepted, "duplicates": len(runs) - accepted}, {} + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/complete": + return 200, {"id": EXPERIMENT_ID, "status": "running"}, {} + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + return 200, self.results.pop(0) if len(self.results) > 1 else self.results[0], {} + return 404, {"error": {"code": "not_found", "message": request.route}}, {} + + +@contextmanager +def fake_api(**options: Any) -> Generator[FakeApi]: + api: FakeApi | None = None + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self._answer() + + def do_POST(self) -> None: + self._answer() + + def _answer(self) -> None: + assert api is not None + length = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(length) if length else b"" + target = urlsplit(self.path) + reply = api.answer( + ApiRequest( + method=self.command, + path=target.path, + query=dict(parse_qsl(target.query)), + headers={key.lower(): value for key, value in self.headers.items()}, + body=json.loads(raw) if raw else None, + ) + ) + if isinstance(reply, Drop): + self.close_connection = True + return + if isinstance(reply, Late): + time.sleep(reply.seconds) + reply = reply.reply + status, body, headers = reply + data = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: object) -> None: + pass + + class Server(ThreadingHTTPServer): + def handle_error(self, request: Any, client_address: Any) -> None: + # The answer of a Late reply can go to a client that has stopped waiting. + if not isinstance(sys.exc_info()[1], (BrokenPipeError, ConnectionResetError)): + super().handle_error(request, client_address) + + server = Server(("127.0.0.1", 0), Handler) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + host, port = server.server_address[0], server.server_address[1] + api = FakeApi(url=f"http://{host}:{port}", **options) + thread.start() + try: + yield api + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/sdks/python/tests/otlp_capture.py b/sdks/python/tests/otlp_capture.py index 0649f72..35cfce9 100644 --- a/sdks/python/tests/otlp_capture.py +++ b/sdks/python/tests/otlp_capture.py @@ -33,7 +33,8 @@ class CaptureServer: status: int = 200 # The (status, JSON body) replies to GET requests, in order; the last one repeats. replies: list[tuple[int, object]] = field(default_factory=lambda: []) - # Seconds that the server waits before it answers a GET request. + # Seconds that the server waits before it answers a GET request. The server reads it before the + # request shows in requests, so that a test that sees the request can change it for the next. delay: float = 0 # The location header of GET replies, for a redirect status. location: str | None = None @@ -66,6 +67,7 @@ def do_POST(self) -> None: self.end_headers() def do_GET(self) -> None: + delay = capture.delay captured.append( CapturedRequest( path=self.requestline.split(" ")[1], @@ -73,7 +75,7 @@ def do_GET(self) -> None: body=b"", ) ) - time.sleep(capture.delay) + time.sleep(delay) status, reply = ( capture.replies[0] if len(capture.replies) == 1 else capture.replies.pop(0) ) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py new file mode 100644 index 0000000..8668f3c --- /dev/null +++ b/sdks/python/tests/test_experiments.py @@ -0,0 +1,1012 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Generator +from pathlib import Path +from typing import Any + +import pytest + +import telemetry_dev +import telemetry_dev._experiments as experiments_module +from telemetry_dev import ( + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + log, + propagate_attributes, + run_experiment, + run_experiment_async, + start_span, +) + +from .conftest import MakeClient +from .fake_api import ( + EXPERIMENT_ID, + EXPERIMENT_URL, + ApiRequest, + Drop, + FakeApi, + Late, + Reply, + dataset_item, + fake_api, + item_id, + result, +) + +ENV_VARS = ( + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", +) + + +@pytest.fixture(autouse=True) +def clear_ci(monkeypatch: pytest.MonkeyPatch) -> None: + # GitHub Actions sets CI and GITHUB_* for this suite too. + for var in ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +@pytest.fixture +def api() -> Generator[FakeApi]: + with fake_api() as server: + yield server + + +def answer(item: ExperimentItem) -> str: + return f"answer to {item.input['question']}" + + +def experiment(api: FakeApi, **options: Any) -> ExperimentResult: + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return run_experiment(**arguments) + + +def answer_with(status: int, body: object, headers: dict[str, str] | None = None) -> Reply: + return status, body, headers or {} + + +def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_completes() -> None: + with fake_api( + items=[dataset_item(number) for number in range(1, 76)], + page_size=50, + results=[result(itemsTotal=150, itemsDone=146, itemsFailed=4)], + ) as api: + outcome = experiment(api, runs_per_item=2, evaluators=["Correctness"]) + + routes = api.routes() + assert routes[:2] == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + # The second page and the first post of items run at the same time, in either order. + assert sorted(routes[2:-2]) == [ + "GET /v1/datasets/support-golden", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + ] + assert routes[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + f"GET /v1/experiments/{EXPERIMENT_ID}", + ] + # The first page names no version; the next one asks for the version of the first. + pages = [request for request in api.requests if request.path.startswith("/v1/datasets/")] + assert pages[0].query == {} + assert pages[1].query == {"version": "3", "cursor": "50"} + assert api.requests[1].body == { + "dataset": "support-golden", + "datasetVersion": 3, + "evaluators": ["Correctness"], + "runsPerItem": 2, + } + assert api.requests[1].headers["authorization"] == "Bearer td_live_test" + assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + # Cloudflare refuses the default Python-urllib User-Agent. + assert api.requests[1].headers["user-agent"] == ( + f"telemetry-dev-python/{telemetry_dev.__version__}" + ) + + posts = api.posts() + assert [len(items) for items in posts] == [100, 50] + runs = [f"{run['itemId']}:{run['runIndex']}" for items in posts for run in items] + assert len(set(runs)) == 150 + assert f"{item_id(75)}:1" in runs + first = posts[0][0] + assert isinstance(first.pop("latencyMs"), int) + assert first == {"itemId": item_id(1), "runIndex": 0, "output": "answer to q1"} + + assert outcome == ExperimentResult( + id=EXPERIMENT_ID, + name="support-golden-v3", + status="completed", + check_status=None, + items_total=150, + items_done=146, + items_failed=4, + error=None, + scores=(), + url=EXPERIMENT_URL, + ) + + +def test_each_item_run_is_its_own_trace_and_each_span_of_the_task_carries_the_item( + make: MakeClient, api: FakeApi +) -> None: + # The session parent of a root span needs the key. + memory = make(api_key="td_live_test", environment="experiment", session_mode="process") + + def task(item: ExperimentItem) -> dict[str, str]: + with start_span("chat gpt-4o", type="generation"): + pass + log("replied") + return {"reply": f"r{item.number}"} + + # An active span, an ambient session, and the process session would each put the runs in + # one trace. + with ( + propagate_attributes(session_id="ambient-session", user_id="u-1"), + start_span("eval script"), + ): + experiment(api, runs_per_item=2, task=task) + + finished = memory.span_exporter.get_finished_spans() + chats = [span for span in finished if span.name == "chat gpt-4o"] + roots = [span for span in finished if span.name == "experiment support-golden-v3"] + assert len(chats) == 4 + assert len(roots) == 4 + + def trace_of(span: Any) -> str: + return f"{span.context.trace_id:032x}" + + assert len({trace_of(span) for span in roots}) == 4 + assert [span.parent is not None and span.parent.is_remote for span in roots] == [True] * 4 + posted = [run for items in api.posts() for run in items] + for run in posted: + root = next(span for span in roots if trace_of(span) == run["traceId"]) + chat = next(span for span in chats if trace_of(span) == run["traceId"]) + assert chat.parent is not None + assert chat.parent.span_id == root.context.span_id + for span in (root, chat): + attributes = dict(span.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert attributes["gen_ai.conversation.id"] == ( + f"{EXPERIMENT_ID}:{run['itemId']}:{run['runIndex']}" + ) + assert attributes["user.id"] == "u-1" + records = memory.log_exporter.get_finished_logs() + assert len(records) == 4 + for record in records: + trace_id = f"{record.log_record.trace_id:032x}" + run = next(run for run in posted if run["traceId"] == trace_id) + attributes = dict(record.log_record.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert sorted(run["output"] for run in posted) == [ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ] + + +def test_runs_at_most_concurrency_tasks_at_a_time() -> None: + lock = threading.Lock() + running = 0 + most = 0 + + def task(item: ExperimentItem) -> str: + nonlocal running, most + with lock: + running += 1 + most = max(most, running) + time.sleep(0.05) + with lock: + running -= 1 + return "ok" + + with fake_api(items=[dataset_item(number) for number in range(1, 13)]) as api: + experiment(api, concurrency=3, task=task) + + assert most == 3 + assert len([run for items in api.posts() for run in items]) == 12 + + +def test_a_task_that_raises_fails_only_its_item(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + if item.number == 1: + raise ValueError("no reply") + return "fine" + + experiment(api, task=task) + + failed, passed = [run for items in api.posts() for run in items] + assert failed["output"] is None + assert failed["error"] == "ValueError: no reply" + assert "scores" not in failed + assert passed["output"] == "fine" + assert "error" not in passed + + +def test_scorers_give_scores_and_a_bad_result_drops_only_its_score(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def thrown(output: Any, item: ExperimentItem) -> None: + raise RuntimeError("judge down") + + scorers: dict[str, Scorer] = { + "exact": lambda output, item: output == f"answer to {item.input['question']}", + "polite": lambda output, item: False, + "length": lambda output, item: len(output), + "tone": lambda output, item: " friendly ", + "judged": lambda output, item: {"value": 0.25, "reason": "too short"}, + "none": lambda output, item: None, + "thrown": thrown, + "mismatched": lambda output, item: {"value": 0.5, "label": "pass"}, + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + "wide": lambda output, item: "\U0001f600" * 33, + } + + with fake_api(items=[dataset_item(1)]) as api: + experiment(api, scorers=scorers) + + assert api.posts()[0][0]["scores"] == [ + {"name": "exact", "value": 1, "label": "pass"}, + {"name": "polite", "value": 0, "label": "fail"}, + {"name": "length", "value": 12}, + {"name": "tone", "label": "friendly"}, + {"name": "judged", "value": 0.25, "reason": "too short"}, + ] + assert [(getattr(error, "code", None), str(error)) for error in errors] == [ + ("scorer_failed", "The scorer thrown failed on item 1: RuntimeError: judge down"), + ( + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not " + "valid: The label pass goes with the value 1", + ), + ( + "scorer_failed", + "The scorer wide failed on item 1: TypeError: returned a score that is not valid: " + "label must be a string of 1 to 64 characters", + ), + ] + + +def test_an_output_is_cut_to_100000_utf16_units(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + return "\U0001f600" * 60_000 + + experiment(api, task=task) + + output = api.posts()[0][0]["output"] + assert len(output.encode("utf-16-le")) // 2 <= 100_000 + assert output.endswith("...[truncated]") + + +def run_both(api: FakeApi, asynchronous: bool, **options: Any) -> ExperimentResult: + """``experiment`` through run_experiment or run_experiment_async.""" + if not asynchronous: + return experiment(api, **options) + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return asyncio.run(run_experiment_async(**arguments)) + + +def unknown_item(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 422, {"error": {"code": "item_not_found", "message": "support-golden v3 has no item"}} + ) + return None + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_a_failed_post_of_items_stops_new_items_completes_the_experiment_and_raises( + asynchronous: bool, +) -> None: + started = 0 + + def task(item: ExperimentItem) -> str: + nonlocal started + started += 1 + return "ok" + + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=unknown_item) as api: + with pytest.raises(ExperimentError) as caught: + run_both(api, asynchronous, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("item_not_found", 422) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + # The complete ends the experiment, which would else stay running. + assert api.routes()[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + ] + + +def test_a_complete_that_fails_after_a_failed_run_goes_to_on_error(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def revoked(request: ApiRequest) -> Reply | None: + if request.path.endswith(("/items", "/complete")): + return answer_with( + 401, {"error": {"code": "invalid_api_key", "message": "The API key was revoked"}} + ) + return None + + with fake_api(override=revoked) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, str(caught.value)) == ("invalid_api_key", "The API key was revoked") + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + assert [getattr(error, "code", None) for error in errors] == ["invalid_api_key"] + + +def test_an_interrupted_run_completes_the_experiment(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + experiment(api, task=task) + + assert api.routes()[-1] == f"POST /v1/experiments/{EXPERIMENT_ID}/complete" + + +class UnreadableError(Exception): + def __str__(self) -> str: + raise RuntimeError("no message") + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_an_output_or_an_error_that_cannot_be_encoded_does_not_stop_the_run( + asynchronous: bool, +) -> None: + cycle: dict[str, Any] = {} + cycle["self"] = cycle + + def task(item: ExperimentItem) -> object: + if item.number == 3: + raise UnreadableError + # A lone surrogate, which UTF-8 cannot hold, becomes U+FFFD. + return cycle if item.number == 1 else "\ud83d" + + def tone(output: Any, item: ExperimentItem) -> str: + return "\ud83d" + + with fake_api(items=[dataset_item(1), dataset_item(2), dataset_item(3)]) as api: + outcome = run_both(api, asynchronous, task=task, scorers={"tone": tone}) + + assert outcome.status == "completed" + posted = sorted((run for items in api.posts() for run in items), key=lambda run: run["itemId"]) + tones = [{"name": "tone", "label": "\ud83d"}] + assert [(run["output"], run.get("error"), run.get("scores")) for run in posted] == [ + (None, None, tones), + ("\ufffd", None, tones), + (None, "UnreadableError", None), + ] + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" in api.routes() + + +def test_each_run_of_an_item_starts_from_the_dataset_input() -> None: + seen: list[int] = [] + + def task(item: ExperimentItem) -> str: + messages = item.input["messages"] + seen.append(len(messages)) + messages.append({"role": "assistant", "content": "Hello"}) + return "ok" + + item = {**dataset_item(1), "input": {"messages": [{"role": "user", "content": "Hi"}]}} + with fake_api(items=[item]) as api: + experiment(api, runs_per_item=3, concurrency=1, task=task) + + assert seen == [1, 1, 1] + + +def test_a_post_of_items_holds_at_most_4_mib_of_json() -> None: + def task(item: ExperimentItem) -> str: + # 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + return "\u4e2d" * 100_000 + + with fake_api(items=[dataset_item(number) for number in range(1, 21)]) as api: + experiment(api, task=task) + + assert [len(items) for items in api.posts()] == [13, 7] + + +@pytest.mark.parametrize( + ("count", "code"), + [(10_000, None), (10_001, "invalid_response")], + ids=["10,000 items in pages run", "10,001 items in pages stop the run"], +) +def test_a_dataset_has_at_most_10000_items(count: int, code: str | None) -> None: + items = [dataset_item(number) for number in range(1, count + 1)] + with fake_api(items=items, page_size=5_000) as api: + try: + experiment(api, concurrency=20) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert len([run for items in api.posts() for run in items]) == 10_000 + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + + +@pytest.mark.parametrize( + ("route", "failure", "sends", "code"), + [ + ("GET /v1/datasets/support-golden", "timeout", 2, None), + (f"POST /v1/experiments/{EXPERIMENT_ID}/items", "network", 2, None), + ("POST /v1/experiments", "timeout", 1, "timeout"), + ("POST /v1/experiments", "network", 1, "network_error"), + ], + ids=[ + "a timeout of a dataset page goes again", + "a network error of an item post goes again", + "a timeout of the create does not go again", + "a network error of the create does not go again", + ], +) +def test_a_request_after_a_timeout_or_a_network_error( + monkeypatch: pytest.MonkeyPatch, route: str, failure: str, sends: int, code: str | None +) -> None: + monkeypatch.setattr(experiments_module, "_REQUEST_TIMEOUT_S", 0.2) + failures = [1] + + def fail_once(request: ApiRequest) -> Reply | Drop | Late | None: + if request.route != route or not failures: + return None + failures.pop() + return Drop() if failure == "network" else Late(1.0, answer_with(500, {})) + + with fake_api(override=fail_once) as api: + try: + experiment(api) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert api.routes().count(route) == sends + + +def test_an_item_post_that_fails_with_500_goes_again() -> None: + failures = [1] + + def flaky(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items") and failures: + failures.pop() + return answer_with(500, {}) + return None + + with fake_api(override=flaky) as api: + experiment(api) + + assert len(api.posts()) == 2 + assert api.posts()[1] == api.posts()[0] + + +@pytest.mark.parametrize( + ("reply", "code"), + [ + (answer_with(500, {}), "http_500"), + # The server answers 503 when its database work times out, which can be after the commit. + ( + answer_with( + 503, + {"error": {"code": "evals_unavailable", "message": "Evals are unavailable"}}, + ), + "evals_unavailable", + ), + ], + ids=["500", "503"], +) +def test_a_create_that_fails_does_not_go_again(reply: Reply, code: str) -> None: + def broken(request: ApiRequest) -> Reply | None: + return reply if request.route == "POST /v1/experiments" else None + + with fake_api(override=broken) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, caught.value.status) == (code, reply[0]) + assert api.routes() == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + + +def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: + limited = [1] + + def limit(request: ApiRequest) -> Reply | None: + if request.route == "POST /v1/experiments" and limited: + limited.pop() + return answer_with( + 429, + {"error": {"code": "rate_limited", "message": "Too many requests"}}, + {"retry-after": "2"}, + ) + return None + + with fake_api(override=limit) as api: + started = time.monotonic() + experiment(api) + + assert api.routes().count("POST /v1/experiments") == 2 + # Without the header, the first retry waits 1 second. + assert time.monotonic() - started >= 2 + + +def test_waits_until_the_server_evaluators_end_the_experiment() -> None: + with fake_api(results=[result(status="running", itemsDone=1), result()]) as api: + outcome = experiment(api) + + assert outcome.status == "completed" + assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 + + +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + + +def test_gives_up_when_the_experiment_does_not_end_in_30_minutes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(experiments_module, "time", clock) + polls: list[float] = [] + + def running(request: ApiRequest) -> Reply | None: + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + polls.append(clock.now) + return answer_with(200, result(status="running", itemsDone=1)) + return None + + with fake_api(override=running) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "timeout" + assert caught.value.result is not None + assert caught.value.result.status == "running" + # The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + assert 30 * 60 - 5 < polls[-1] - polls[0] <= 30 * 60 + + +@pytest.mark.parametrize( + ("status", "body", "code"), + [ + ( + 404, + {"error": {"code": "dataset_not_found", "message": "No dataset"}}, + "dataset_not_found", + ), + (200, {"dataset": {"version": 1}, "items": [{"id": 1}]}, "invalid_response"), + ( + 200, + {"dataset": {"id": "d", "name": "support-golden", "version": 0}, "items": []}, + "dataset_empty", + ), + ], + ids=["the API error", "a page that cannot be read", "an empty dataset"], +) +def test_raises_before_it_creates_an_experiment(status: int, body: object, code: str) -> None: + def page(request: ApiRequest) -> Reply | None: + return answer_with(status, body) if request.path.startswith("/v1/datasets/") else None + + with fake_api(override=page) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == code + assert api.routes() == ["GET /v1/datasets/support-golden"] + + +def test_a_cursor_that_repeats_is_an_invalid_response() -> None: + def same(request: ApiRequest) -> Reply | None: + if not request.path.startswith("/v1/datasets/"): + return None + return answer_with( + 200, + { + "dataset": {"id": "d", "name": "support-golden", "version": 3}, + "items": [dataset_item(1)], + "nextCursor": "same", + }, + ) + + with fake_api(override=same) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + assert api.routes().count("GET /v1/datasets/support-golden") == 2 + + +FAILED_CHECK = result( + checkStatus="failed", + scores=[ + { + "name": "correctness", + "kind": "pass_fail", + "value": 0.85, + "passCount": 17, + "failCount": 3, + "baselineValue": 0.9, + "delta": -5, + "minScore": 0.9, + "passed": False, + }, + { + "name": "length", + "kind": None, + "value": None, + "passCount": 0, + "failCount": 0, + "baselineValue": None, + "delta": None, + "minScore": None, + "passed": None, + }, + ], +) + + +def test_in_ci_mode_a_failed_check_raises_with_the_result(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CI", "true") + with fake_api(results=[FAILED_CHECK]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, min_scores={"correctness": 0.9}) + + assert caught.value.code == "check_failed" + assert str(caught.value) == ( + "The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. " + f"{EXPERIMENT_URL}" + ) + assert caught.value.result is not None + assert caught.value.result.scores[1] == ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ) + + +@pytest.mark.parametrize( + ("options", "ci"), + [({}, None), ({"ci": False}, "true")], + ids=["CI is off", "ci=False wins over CI"], +) +def test_a_failed_check_returns_when( + monkeypatch: pytest.MonkeyPatch, options: dict[str, Any], ci: str | None +) -> None: + if ci is not None: + monkeypatch.setenv("CI", ci) + with fake_api(results=[FAILED_CHECK]) as api: + assert experiment(api, **options).check_status == "failed" + + +def test_a_result_score_that_leaves_out_its_null_fields_reads_them_as_none() -> None: + scores = [{"name": "length", "passCount": 0, "failCount": 0}] + with fake_api(results=[result(scores=scores)]) as api: + outcome = experiment(api) + + assert outcome.scores == ( + ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ), + ) + + +def test_a_result_without_check_status_is_an_invalid_response() -> None: + missing = {key: value for key, value in FAILED_CHECK.items() if key != "checkStatus"} + with fake_api(results=[missing]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + + +def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CI", "1") + with fake_api( + results=[result(status="failed", error="0 of 2 items ran. Each one failed.")] + ) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "experiment_failed" + assert str(caught.value) == ( + "The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. " + f"{EXPERIMENT_URL}" + ) + + +def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + results = tmp_path / "results.jsonl" + event.write_text( + json.dumps( + { + "pull_request": { + "number": 42, + "title": "Shorter replies", + "html_url": "https://github.com/acme/support/pull/42", + "head": {"sha": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"}, + } + } + ) + ) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "f" * 40, + "GITHUB_EVENT_PATH": str(event), + "GITHUB_RUN_ID": "991", + "TELEMETRY_DEV_EXPERIMENT_DATASET": "support-golden", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=0.9\nlength = 12.5", + "TELEMETRY_DEV_EXPERIMENT_RESULTS": str(results), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api, dataset=None, min_scores={"correctness": 0.5, "tone": 2}) + + create = api.requests[1].body + assert create["dataset"] == "support-golden" + assert create["minScores"] == {"correctness": 0.9, "length": 12.5, "tone": 2} + assert create["ci"] == { + "provider": "github_actions", + "repository": "acme/support", + "commit": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "prNumber": 42, + "prTitle": "Shorter replies", + "prUrl": "https://github.com/acme/support/pull/42", + "runUrl": "https://github.com/acme/support/actions/runs/991", + } + lines = results.read_text().splitlines() + assert [json.loads(line) for line in lines] == [result()] + assert results.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize( + ("title", "sent"), + [ + ("\U0001f600" * 200, "\U0001f600" * 128), + # GitHub counts the emoji as one character, thus the title fits its limit of 256. + ("a" * 255 + "\U0001f600", "a" * 255), + ], + ids=["200 emoji", "255 letters and an emoji"], +) +def test_a_pull_request_title_is_cut_to_256_utf16_units_without_half_of_an_emoji( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi, title: str, sent: str +) -> None: + event = tmp_path / "event.json" + event.write_text(json.dumps({"pull_request": {"title": title}})) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "a1b2c3d", + "GITHUB_EVENT_PATH": str(event), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api) + + assert api.requests[1].body["ci"]["prTitle"] == sent + + +def one(output: Any, item: ExperimentItem) -> int: + return 1 + + +REJECTED: list[tuple[dict[str, str], dict[str, Any], type[Exception], str]] = [ + ( + {}, + {"dataset": None}, + TypeError, + "Pass dataset, or set the dataset input of the experiment action", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_DATASET": "support-v2"}, + {}, + TypeError, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=1_000"}, + {}, + TypeError, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=1_000" is not a name=number pair', + ), + ({}, {"runs_per_item": 6}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"runs_per_item": True}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"concurrency": 0}, ValueError, "concurrency must be an integer from 1 to 100"), + ( + {}, + {"scorers": {"user_feedback": one}}, + TypeError, + "user_feedback is the score of user feedback. Name the scorer another way.", + ), + ( + {}, + {"evaluators": ["correctness"], "scorers": {"Correctness": one}}, + TypeError, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ), + ({}, {"evaluators": "correctness"}, TypeError, "evaluators must be a list of at most"), +] + + +@pytest.mark.parametrize( + ("env", "options", "error", "message"), + REJECTED, + ids=[ + "no dataset", + "a dataset that the action does not name", + "min-scores that do not parse", + "runs_per_item 6", + "runs_per_item True", + "concurrency 0", + "a scorer named user_feedback", + "a scorer with the name of an evaluator", + "evaluators as one string", + ], +) +def test_raises_before_any_request( + monkeypatch: pytest.MonkeyPatch, + api: FakeApi, + env: dict[str, str], + options: dict[str, Any], + error: type[Exception], + message: str, +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(error) as caught: + experiment(api, **options) + + assert str(caught.value).startswith(message) + assert api.requests == [] + + +def test_without_an_api_key_it_raises_missing_api_key(api: FakeApi) -> None: + with pytest.raises(ExperimentError) as caught: + experiment(api, api_key="") + + assert caught.value.code == "missing_api_key" + assert api.requests == [] + + +def test_the_task_gets_each_run_of_an_item_of_the_split_with_its_run_index() -> None: + seen: list[ExperimentItem] = [] + + def task(item: ExperimentItem) -> str: + seen.append(item) + return "ok" + + with fake_api(items=[dataset_item(7, "test")]) as api: + experiment(api, split="test", runs_per_item=2, concurrency=1, task=task) + + assert seen == [ + ExperimentItem( + id=item_id(7), + number=7, + input={"question": "q7"}, + expected_output="a7", + metadata={"source": "test"}, + tags=("golden",), + split="test", + run_index=index, + ) + for index in (0, 1) + ] + assert api.requests[0].query["split"] == "test" + assert api.requests[1].body["split"] == "test" + + +def test_run_experiment_rejects_an_async_task(api: FakeApi) -> None: + async def task(item: ExperimentItem) -> str: + return "ok" + + with pytest.raises(TypeError, match="task is async: use run_experiment_async"): + experiment(api, task=task) + assert api.requests == [] + + +async def test_run_experiment_async_runs_async_tasks_and_scorers_at_the_same_time( + make: MakeClient, +) -> None: + memory = make(environment="experiment") + running = 0 + most = 0 + + async def task(item: ExperimentItem) -> str: + nonlocal running, most + running += 1 + most = max(most, running) + with start_span("chat gpt-4o", type="generation"): + await asyncio.sleep(0.05) + running -= 1 + return f"r{item.number}" + + async def exact(output: Any, item: ExperimentItem) -> bool: + await asyncio.sleep(0) + return output == f"r{item.number}" + + with fake_api(items=[dataset_item(number) for number in range(1, 9)]) as api: + outcome = await run_experiment_async( + api_key="td_live_test", + base_url=api.url, + dataset="support-golden", + task=task, + scorers={"exact": exact}, + concurrency=4, + ) + + assert outcome.status == "completed" + assert most == 4 + posted = sorted( + (run for items in api.posts() for run in items), key=lambda run: str(run["itemId"]) + ) + assert [run["scores"] for run in posted] == [ + [{"name": "exact", "value": 1, "label": "pass"}] + ] * 8 + chats = [ + span for span in memory.span_exporter.get_finished_spans() if span.name == "chat gpt-4o" + ] + by_trace = {f"{span.context.trace_id:032x}": span for span in chats} + for run in posted: + attributes = dict(by_trace[run["traceId"]].attributes or {}) + assert attributes["td.dataset.item_id"] == run["itemId"] diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 892d676..74416e7 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -2,21 +2,33 @@ import asyncio import contextvars +import copy +import dataclasses +import gc +import importlib import itertools +import json +import os +import pickle +import signal import socket +import sys import threading import time +import traceback import urllib.error import urllib.request +import weakref from collections.abc import Callable, Generator from datetime import datetime, timedelta, timezone from email.utils import format_datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from types import SimpleNamespace -from typing import Any, NoReturn +from types import FrameType, SimpleNamespace +from typing import Any, NoReturn, TypeAlias from urllib.parse import parse_qsl, urlsplit import pytest +from opentelemetry import context as otel_context from opentelemetry import trace import telemetry_dev @@ -24,8 +36,10 @@ from telemetry_dev import ( ChatPrompt, PromptError, + TextPrompt, get_prompt, get_prompt_async, + prompt_attributes_for, propagate_attributes, start_span, with_prompt, @@ -34,6 +48,12 @@ from .conftest import MakeClient from .otlp_capture import CaptureServer, otlp_capture_server +# Python 3.12+ gives a warning for fork() in a process with threads, and the capture server has one. +pytestmark = pytest.mark.filterwarnings( + "ignore:This process .* is multi-threaded:DeprecationWarning" +) +needs_fork = pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") + def reply(**overrides: Any) -> dict[str, Any]: body: dict[str, Any] = { @@ -81,6 +101,34 @@ def empty_cache() -> None: prompts_module._cache.clear() # pyright: ignore[reportPrivateUsage] +def chat_prompt(*messages: dict[str, Any], name: str = "support/reply") -> ChatPrompt: + return ChatPrompt( + name=name, + version=12, + label="production", + messages=list(messages), + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, + ) + + +def text_prompt(text: str) -> TextPrompt: + return TextPrompt( + name="support/reply", + version=12, + label="production", + text=text, + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, + ) + + def query_of(path: str) -> dict[str, str]: return dict(parse_qsl(urlsplit(path).query)) @@ -93,6 +141,57 @@ def wait_for(check: Callable[[], bool]) -> None: time.sleep(0.01) +def in_fork(child: Callable[[], str], fork: Callable[[], int] | None = None) -> str: + """Run ``child`` in a process that ``fork`` (by default ``os.fork``) makes, and return its + result, or the repr of its exception or of an exception in a fork hook of the prompt module. + The parent kills a child that does not end in 10 s, and the test fails.""" + # Python ignores an exception in a fork hook and only gives it to sys.unraisablehook. The + # hooks of OpenTelemetry raise after an earlier test discards its batch processor. + hook_errors: list[BaseException | None] = [] + + def record(unraisable: sys.UnraisableHookArgs) -> None: + frames = traceback.extract_tb(unraisable.exc_traceback) + if any(frame.filename == prompts_module.__file__ for frame in frames): + hook_errors.append(unraisable.exc_value) + + previous_hook = sys.unraisablehook + parent = os.getpid() + read_end, write_end = os.pipe() + sys.unraisablehook = record + try: + pid = (fork or os.fork)() + finally: + sys.unraisablehook = previous_hook + # Not pid == 0: a fork function that forks more than one time can return a pid in a child. + if os.getpid() != parent: + try: + os.close(read_end) + try: + result = repr(hook_errors[0]) if hook_errors else child() + except BaseException as exc: + result = repr(exc) + os.write(write_end, result.encode()) + finally: + # The child must not return into pytest. + os._exit(0) + os.close(write_end) + with os.fdopen(read_end, "rb") as reader: + deadline = time.monotonic() + 10 + while os.waitpid(pid, os.WNOHANG) == (0, 0): + if time.monotonic() > deadline: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + pytest.fail("The forked process did not end in 10 s") + time.sleep(0.01) + result = reader.read().decode() + assert hook_errors == [] + return result + + +def version_of(server: CaptureServer) -> int | None: + return get_prompt("support/reply", api_key="td_live_a", base_url=server.url).version + + @pytest.mark.parametrize( ("options", "query"), [ @@ -525,6 +624,245 @@ def close(self) -> None: assert timeouts[0] is not None and timeouts[0] <= 0.2 +@needs_fork +@pytest.mark.parametrize("reused_pid", [False, True]) +def test_a_forked_process_fetches_the_prompt_that_the_parent_was_fetching( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch, reused_pid: bool +) -> None: + # This puts a limit on the wait of a child that waits for the fetch of the parent. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + server.delay = 1 + versions: list[int | None] = [] + parent = threading.Thread(target=lambda: versions.append(version_of(server))) + parent.start() + wait_for(lambda: len(server.requests) == 1) + server.delay = 0 + parent_pid = os.getpid() + + def child() -> str: + if reused_pid: + # A reused pid: the child gets the id of the process that started the fetch. + monkeypatch.setattr(os, "getpid", lambda: parent_pid) + return str(version_of(server)) + + assert in_fork(child) == "12" + parent.join() + assert versions == [12] + # The fetch of the parent, and the fetch of the child. + assert len(server.requests) == 2 + + +def test_a_call_does_not_wait_for_a_fetch_that_another_process_started( + server: CaptureServer, +) -> None: + server.delay = 1 + first = threading.Thread(target=version_of, args=(server,)) + first.start() + wait_for(lambda: len(server.requests) == 1) + server.delay = 0 + # A fork that runs no fork hooks, as a fork in C code, copies the fetch of the parent but not + # the thread that runs it. Here this process is the child: the fetch gets the id of its parent. + (entry,) = prompts_module._cache.values() # pyright: ignore[reportPrivateUsage] + entry.pending_pid = os.getppid() + + assert version_of(server) == 12 + assert len(server.requests) == 2 + first.join() + + +@needs_fork +def test_a_forked_process_refreshes_the_copy_that_the_parent_was_refreshing( + server: CaptureServer, clock: list[float] +) -> None: + assert version_of(server) == 12 + clock[0] += 60 + server.replies = [(200, reply(version=13))] + server.delay = 1 + assert version_of(server) == 12 + wait_for(lambda: len(server.requests) == 2) + server.delay = 0 + + def child() -> str: + kept = version_of(server) + wait_for(lambda: version_of(server) == 13) + return str(kept) + + # The child returns the copy of the parent. Then its refresh gets version 13. + assert in_fork(child) == "12" + + +@needs_fork +@pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning") +def test_a_fork_waits_for_a_thread_that_holds_the_cache_lock(server: CaptureServer) -> None: + held = threading.Event() + released_at: list[float] = [] + forked_at: list[float] = [] + errors: list[BaseException] = [] + + def hold() -> None: + try: + with prompts_module._cache_lock: # pyright: ignore[reportPrivateUsage] + held.set() + time.sleep(0.2) + released_at.append(time.monotonic()) + except BaseException as exc: + errors.append(exc) + + def fork() -> int: + pid = os.fork() + if pid != 0: + forked_at.append(time.monotonic()) + return pid + + holder = threading.Thread(target=hold) + holder.start() + held.wait() + # The child does not get a lock that a thread of the parent holds. + assert in_fork(lambda: str(version_of(server)), fork) == "12" + holder.join() + assert errors == [] + assert forked_at[0] > released_at[0] + + +@needs_fork +def test_a_fork_in_the_thread_that_holds_the_cache_lock_returns( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + # A process that still holds the lock after the fork waits for its fetch until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + + def fork_in_the_lock() -> int: + with prompts_module._cache_lock: # pyright: ignore[reportPrivateUsage] + return os.fork() + + def fork_and_fetch() -> str: + child = in_fork(lambda: str(version_of(server)), fork_in_the_lock) + return f"{child} {version_of(server)}" + + # The fork runs in a child, so that a fork that waits for itself stops only that child. + assert in_fork(fork_and_fetch) == "12 12" + + +@needs_fork +def test_a_fork_in_a_call_that_starts_a_fetch_returns_the_prompt( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + # A process that waits for a fetch that it does not have waits until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + pids: list[int] = [] + results: list[str] = [] + + class ForkingThread(threading.Thread): + def start(self) -> None: + # A fork in the call, as in a signal handler, after the call made its fetch and before + # the fetch thread starts. The child hook drops the fetch of the entry. + if not pids: + pids.append(os.fork()) + super().start() + + def fork() -> int: + with monkeypatch.context() as patch: + # Only the threads of the prompt module: a fork hook of OpenTelemetry starts a thread. + patch.setattr( + prompts_module, + "threading", + SimpleNamespace(**{**vars(threading), "Thread": ForkingThread}), + ) + try: + results.append(str(version_of(server))) + except Exception as exc: + # The child must return the pid to in_fork, and not raise into pytest. + results.append(repr(exc)) + return pids[0] + + # The parent and the child each start the fetch thread, and each call returns its result. + assert in_fork(lambda: results[0], fork) == "12" + assert results == ["12"] + + +@needs_fork +def test_a_fork_after_a_reload_of_the_prompt_module_returns( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + saved = dict(prompts_module.__dict__) + importlib.reload(prompts_module) + try: + # The reload sets the timeout again. A process that still holds the lock after the fork + # waits for its fetch until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + assert in_fork(lambda: str(version_of(server))) == "12" + assert version_of(server) == 12 + finally: + # The reload gave the module new classes and a new cache. + prompts_module.__dict__.update(saved) + + +@needs_fork +@pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="needs signal.setitimer") +@pytest.mark.parametrize("before_hook", ["stopped by a signal", "not run"]) +def test_the_fork_hooks_handle_a_lock_that_the_before_hook_did_not_get( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch, before_hook: str +) -> None: + class Interrupted(Exception): + pass + + lock = prompts_module._cache_lock # pyright: ignore[reportPrivateUsage] + # The child hook replaces the lock. monkeypatch puts it back. + monkeypatch.setattr(prompts_module, "_cache_lock", lock) + held, done = threading.Event(), threading.Event() + errors: list[BaseException] = [] + + def hold() -> None: + try: + with lock: + held.set() + # The fork must end while this thread holds the lock. If no signal stops the + # before hook, the hook waits for this thread, and this assert fails after 5 s. + assert done.wait(5) + except BaseException as exc: + errors.append(exc) + + def interrupt(signum: int, frame: FrameType | None) -> NoReturn: + raise Interrupted + + def fork_in_a_signal() -> int: + # SIGALRM raises while the before hook waits for the lock. os.fork() gives the error to + # sys.unraisablehook, forks, and then runs the after hooks. + in_fork_hook = sys.unraisablehook + + def record(unraisable: sys.UnraisableHookArgs) -> None: + if not isinstance(unraisable.exc_value, Interrupted): + in_fork_hook(unraisable) + + previous = signal.signal(signal.SIGALRM, interrupt) + sys.unraisablehook = record + signal.setitimer(signal.ITIMER_REAL, 0.1) + try: + return os.fork() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + sys.unraisablehook = in_fork_hook + signal.signal(signal.SIGALRM, previous) + + holder = threading.Thread(target=hold) + holder.start() + held.wait() + try: + if before_hook == "stopped by a signal": + # In the child, the holder is a thread of the parent that does not exist there. + assert in_fork(lambda: str(version_of(server)), fork_in_a_signal) == "12" + else: + # uWSGI with py-call-osafterfork forks in C and runs only the child hook. + prompts_module._reset_in_child() # pyright: ignore[reportPrivateUsage] + child_lock = prompts_module._cache_lock # pyright: ignore[reportPrivateUsage] + assert child_lock.acquire(blocking=False) + child_lock.release() + finally: + done.set() + holder.join() + assert errors == [] + + def test_get_prompt_rejects_a_prompt_of_the_other_type_and_bad_options( server: CaptureServer, ) -> None: @@ -793,6 +1131,716 @@ def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: assert "td.prompt.variables" not in (span.attributes or {}) +DESK_MESSAGES = [ + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "user", "content": "Order {{order}}: where is it?"}, +] + + +def test_prompt_attributes_for_names_the_compile_of_a_message_dict_and_a_copy_names_nothing( + make: MakeClient, +) -> None: + make() + compiled = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4") + attributes = { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + } + + assert prompt_attributes_for(compiled) == attributes + assert prompt_attributes_for(tuple(compiled)) == attributes + assert prompt_attributes_for(compiled[0]) == attributes + assert json.dumps(compiled[0]) == '{"role": "system", "content": "Answer as the billing desk."}' + + copies: list[object] = [ + [dict(message) for message in compiled], + [message.copy() for message in compiled], + [copy.copy(message) for message in compiled], + copy.deepcopy(compiled), + pickle.loads(pickle.dumps(compiled)), + json.loads(json.dumps(compiled)), + [message["content"] for message in compiled], + ] + for messages in copies: + assert prompt_attributes_for(messages) == {} + + summary = text_prompt("Summarize {{doc}}.").compile(doc="the policy") + assert prompt_attributes_for([summary]) == {} + + +def test_an_unrelated_message_with_the_text_of_a_compiled_message_is_not_tagged( + make: MakeClient, +) -> None: + make() + chat_prompt({"role": "system", "content": "You are a helpful assistant."}).compile() + + unrelated = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2 + 2?"}, + ] + assert prompt_attributes_for(unrelated) == {} + + +def test_two_tasks_with_one_text_and_different_other_variables_each_record_their_own( + make: MakeClient, +) -> None: + memory = make() + prompt = chat_prompt(*DESK_MESSAGES) + + # Both compiles make the system text "Answer as the billing desk.", and each task sends only it. + system_a = prompt.compile(team="billing", order="#A")[0] + system_b = prompt.compile(team="billing", order="#B")[0] + start_span("a", type="generation", attributes=prompt_attributes_for([system_a])).end() + start_span("b", type="generation", attributes=prompt_attributes_for([system_b])).end() + + assert { + span.name: (span.attributes or {})["td.prompt.variables"] + for span in memory.span_exporter.get_finished_spans() + } == { + "a": '{"team":"billing","order":"#A"}', + "b": '{"team":"billing","order":"#B"}', + } + + +def test_a_rollout_with_two_versions_of_the_same_messages_records_the_version_of_each_call( + server: CaptureServer, make: MakeClient, monkeypatch: pytest.MonkeyPatch +) -> None: + make() + server.replies = [ + ( + 200, + reply( + messages=DESK_MESSAGES, + rollout={ + "percent": 16, + "version": 13, + "messages": DESK_MESSAGES, + "text": None, + "model": {"provider": "openai", "name": "gpt-4o-mini"}, + "parameters": {}, + "tools": [], + "responseFormat": {"type": "text"}, + }, + ), + ) + ] + draws = iter([15, 16]) + + def draw(_stop: int) -> int: + return next(draws) + + monkeypatch.setattr(prompts_module.random, "randrange", draw) + rollout = get_prompt("support/reply", api_key="td_live_a", base_url=server.url) + production = get_prompt("support/reply", api_key="td_live_a", base_url=server.url) + from_rollout = rollout.compile(team="billing", order="#4") + from_production = production.compile(team="billing", order="#4") + + assert prompt_attributes_for(from_rollout)["td.prompt.version"] == 13 + assert prompt_attributes_for(from_production)["td.prompt.version"] == 12 + + +def test_the_newest_compiled_message_wins_also_after_170_messages_of_history( + make: MakeClient, +) -> None: + make() + prompt = chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, + ) + history = [ + {"role": "user" if index % 2 == 0 else "assistant", "content": f"message {index}"} + for index in range(170) + ] + + first_turn = prompt.compile(team="billing", question="Where is #4?", history=history) + second_turn = prompt.compile(team="returns", question="Can I return it?") + chat = [*first_turn, {"role": "assistant", "content": "On its way."}, *second_turn[1:]] + + assert '"team":"billing"' in str(prompt_attributes_for(first_turn)["td.prompt.variables"]) + assert '"team":"billing"' in str( + prompt_attributes_for([first_turn[0], *history])["td.prompt.variables"] + ) + assert ( + prompt_attributes_for(chat)["td.prompt.variables"] + == '{"team":"returns","question":"Can I return it?"}' + ) + + +def test_a_message_that_a_placeholder_brings_from_an_older_compile_does_not_win( + make: MakeClient, +) -> None: + make() + router = chat_prompt({"role": "user", "content": "{{question}}"}, name="router") + answer = chat_prompt( + {"role": "system", "content": "Answer for {{team}}."}, + {"role": "placeholder", "name": "history"}, + name="answer", + ) + chat = chat_prompt({"role": "user", "content": "{{question}}"}, name="chat") + summary = chat_prompt( + {"role": "system", "content": "Summarize the chat."}, + {"role": "placeholder", "name": "history"}, + name="summary", + ) + + question = router.compile(question="Where is #4?") + routed = answer.compile(team="billing", history=question) + # The last 1000 messages are all from chat; the summary message is the first. + conversation = [chat.compile(question=f"question {index}")[0] for index in range(1500)] + summarized = summary.compile(history=conversation) + + assert prompt_attributes_for(routed) == { + "td.prompt.name": "answer", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": ( + '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}' + ), + } + assert prompt_attributes_for(summarized)["td.prompt.name"] == "summary" + + +def test_prompt_attributes_for_looks_at_the_first_and_the_last_1000_messages( + make: MakeClient, +) -> None: + make() + system = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4")[0] + + def around(before: int, after: int) -> list[dict[str, Any]]: + return [*[{"role": "user"}] * before, system, *[{"role": "user"}] * after] + + assert prompt_attributes_for(around(1000, 999)) != {} + assert prompt_attributes_for(around(999, 1001)) != {} + assert prompt_attributes_for(tuple(around(1001, 999))) != {} + assert prompt_attributes_for(around(1000, 1000)) == {} + + +def test_compile_records_its_variables_not_later_changes_of_the_mapping_or_of_its_lists( + make: MakeClient, +) -> None: + memory = make() + prompt = chat_prompt( + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "Order {{order}}"}, + ) + history = [{"role": "user", "content": "Hi"}] + variables: dict[str, Any] = {"order": "#A", "history": history} + + first = prompt.compile(variables) + variables["order"] = "#B" + history.append({"role": "assistant", "content": "Hello"}) + second = prompt.compile(variables) + variables["order"] = "#C" + history.append({"role": "user", "content": "Added after compile"}) + with with_prompt(prompt): + start_span("scope", type="generation").end() + + second_variables = ( + '{"order":"#B","history":[{"role":"user","content":"Hi"},' + '{"role":"assistant","content":"Hello"}]}' + ) + assert ( + prompt_attributes_for(first)["td.prompt.variables"] + == '{"order":"#A","history":[{"role":"user","content":"Hi"}]}' + ) + assert prompt_attributes_for(second)["td.prompt.variables"] == second_variables + [span] = memory.span_exporter.get_finished_spans() + assert (span.attributes or {})["td.prompt.variables"] == second_variables + + +def test_a_compiled_message_rebuilds_like_a_dict_and_the_rebuilt_one_has_no_tag( + make: MakeClient, +) -> None: + make() + compiled = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4") + + @dataclasses.dataclass + class Conversation: + messages: list[dict[str, Any]] + + rebuilt = dataclasses.asdict(Conversation(messages=compiled))["messages"] + same_type = [type(message)(message) for message in compiled] + plain = [ + {"role": "system", "content": "Answer as the billing desk."}, + {"role": "user", "content": "Order #4: where is it?"}, + ] + + assert rebuilt == plain + assert same_type == plain + assert prompt_attributes_for(rebuilt) == {} + assert prompt_attributes_for(same_type) == {} + + +def test_with_prompt_wins_over_a_compiled_message_of_another_prompt_through_a_parent_and_in_updates( + make: MakeClient, +) -> None: + memory = make() + scoped = chat_prompt({"role": "user", "content": "Hi"}, name="scoped/prompt") + # Fetched by version, so it has no label that the label of the scope could hide. + other = dataclasses.replace( + chat_prompt({"role": "user", "content": "Cancel order {{order}}."}, name="other/prompt"), + version=3, + label=None, + ) + compiled = other.compile(order="#9") + + with with_prompt(scoped): + start_span("in scope", type="generation", attributes=prompt_attributes_for(compiled)).end() + parent = otel_context.get_current() + attributes = prompt_attributes_for(compiled) + start_span("with parent", type="generation", parent=parent, attributes=attributes).end() + with with_prompt(scoped): + start_span("updated", type="generation").update(attributes=attributes).end() + start_span("ended", type="generation").end(attributes=attributes) + with start_span("current", type="generation"): + telemetry_dev.update_current_span(attributes=attributes) + + assert attributes["td.prompt.name"] == "other/prompt" + spans = memory.span_exporter.get_finished_spans() + assert sorted(span.name for span in spans) == [ + "current", + "ended", + "in scope", + "updated", + "with parent", + ] + for span in spans: + assert { + key: value + for key, value in dict(span.attributes or {}).items() + if key.startswith("td.prompt.") + } == { + "td.prompt.name": "scoped/prompt", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + +def test_prompt_variables_follow_the_input_capture_of_the_client_and_of_the_span( + make: MakeClient, +) -> None: + compiled = chat_prompt({"role": "user", "content": "My card is {{card}}."}).compile( + card="4111 1111" + ) + + make(capture_input=False) + assert prompt_attributes_for(compiled) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + telemetry_dev.shutdown() + + def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: + return {"card": "[masked]"} if context.key == "td.prompt.variables" else value + + memory = make(mask=mask) + attributes = prompt_attributes_for(compiled) + assert attributes["td.prompt.variables"] == '{"card":"[masked]"}' + + start_span("private", type="generation", capture_input=False, attributes=attributes).end() + with start_span("updated", type="generation", capture_input=False): + telemetry_dev.update_current_span(attributes=attributes) + + for span in memory.span_exporter.get_finished_spans(): + assert (span.attributes or {})["td.prompt.name"] == "support/reply" + assert "td.prompt.variables" not in (span.attributes or {}) + + +def test_prompt_attributes_for_runs_no_code_of_the_app(make: MakeClient) -> None: + make() + [message] = chat_prompt({"role": "user", "content": "Refund order {{order}}."}).compile( + order="#3" + ) + reads: list[str] = [] + + class Messages(list[object]): + def __getitem__(self, index: Any) -> Any: + reads.append("__getitem__") + return super().__getitem__(index) + + def __iter__(self) -> Any: + reads.append("__iter__") + return super().__iter__() + + def __reversed__(self) -> Any: + reads.append("__reversed__") + return super().__reversed__() + + assert prompt_attributes_for(Messages([message])) == {} + assert reads == [] + + +RAG_MESSAGES = [ + {"role": "system", "content": "Answer from {{documents}}."}, + {"role": "user", "content": "{{question}}"}, +] + + +class Document(dict[str, str]): + """A dict that a weak reference can point to.""" + + +def test_the_variables_of_a_compile_are_freed_with_its_messages(make: MakeClient) -> None: + make() + prompt = chat_prompt(*RAG_MESSAGES) + + document = Document(content="The refund policy") + freed = weakref.ref(document) + messages = prompt.compile(documents=[document], question="q") + del document + # with_prompt keeps the variables of the last compile of a prompt object. + prompt.compile(documents=[], question="other") + gc.collect() + assert freed() is not None + + assert len(messages) == 2 + del messages + gc.collect() + assert freed() is None + + +def test_with_input_capture_off_compiled_messages_keep_no_variables(make: MakeClient) -> None: + make(capture_input=False) + prompt = chat_prompt(*RAG_MESSAGES) + documents: list[weakref.ref[Document]] = [] + questions: list[dict[str, Any]] = [] + + for turn in range(200): + document = Document(content=f"document {turn}") + documents.append(weakref.ref(document)) + questions.append(prompt.compile(documents=[document], question=f"question {turn}")[-1]) + gc.collect() + + # with_prompt keeps the variables of the last compile of a prompt object. + assert sum(1 for document in documents if document() is not None) == 1 + + telemetry_dev.shutdown() + make() + assert prompt_attributes_for(questions) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + +CHAT_MESSAGES = ( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, +) + + +@dataclasses.dataclass +class ChatPrompts: + chat: ChatPrompt + question: ChatPrompt + rag: ChatPrompt + reply: ChatPrompt + + +def _answer(turn: int) -> dict[str, Any]: + return {"role": "assistant", "content": f"answer {turn}"} + + +# What a row gives for one turn of a chat: the request, the history for the next turn, and the kept +# message. The row puts `marker` in the variables of the compile that made the kept message. +ChatTurn: TypeAlias = tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]] + + +def _new_list( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", history=history, question=f"question {turn}", marker=marker + ) + return request, [*history, request[-1], _answer(turn)], request[-1] + + +def _new_list_of_20( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", history=history, question=f"question {turn}", marker=marker + ) + return request, [*history, request[-1], _answer(turn)][-20:], request[-1] + + +def _full_list_as_copies( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", + history=[dict(message) for message in history], + question=f"question {turn}", + marker=marker, + ) + return request, [*history, request[-1], _answer(turn)], request[-1] + + +def _question_prompt_in_a_placeholder( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + [kept] = p.question.compile( + documents=f"document {turn}", question=f"question {turn}", marker=marker + ) + request = p.rag.compile(team="billing", history=history, current=[kept]) + return request, [*history, kept, _answer(turn)], kept + + +def _raw_questions_and_a_reply_prompt( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile(team="billing", history=history, question=f"question {turn}") + [kept] = p.reply.compile(answer=f"answer {turn}", marker=marker) + return request, [*history, {"role": "user", "content": f"question {turn}"}, kept], kept + + +@pytest.mark.parametrize( + "run", + [ + _new_list, + _new_list_of_20, + _full_list_as_copies, + _question_prompt_in_a_placeholder, + _raw_questions_and_a_reply_prompt, + ], + ids=[ + "new-list", + "new-list-of-20", + "full-list-as-copies", + "question-prompt-in-a-placeholder", + "raw-questions-and-a-reply-prompt", + ], +) +def test_a_chat_history_keeps_the_variables_of_the_last_5_minutes( + make: MakeClient, + clock: list[float], + run: Callable[[ChatPrompts, list[dict[str, Any]], int, Document], ChatTurn], +) -> None: + make() + prompts = ChatPrompts( + chat=chat_prompt(*CHAT_MESSAGES), + question=chat_prompt({"role": "user", "content": "Context: {{documents}}\n\n{{question}}"}), + rag=chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "placeholder", "name": "current"}, + ), + reply=chat_prompt({"role": "assistant", "content": "{{answer}}"}), + ) + markers: list[weakref.ref[Document]] = [] + history: list[dict[str, Any]] = [] + request: list[dict[str, Any]] = [] + older: dict[str, Any] = {} + + # One turn in 2 minutes, for 300 turns. An unused variable marks the variables of each turn. + for turn in range(300): + if turn > 0: + clock[0] += 120 + marker = Document(content=f"turn {turn}") + markers.append(weakref.ref(marker)) + request, history, kept = run(prompts, history, turn, marker) + # The model call of the turn. + prompt_attributes_for(request) + if turn == 296: + older = kept + gc.collect() + + # Turns 297, 298, and 299: 4, 2, and 0 minutes old. + assert sum(1 for marker in markers if marker() is not None) == 3 + assert "question 299" in str(prompt_attributes_for(request)["td.prompt.variables"]) + assert prompt_attributes_for(older) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + +def _variables_of(messages: object) -> object: + return prompt_attributes_for(messages).get("td.prompt.variables") + + +REFUND_VARIABLES = '{"documents":"The refund policy","question":"Where is #4?"}' + + +@dataclasses.dataclass +class LivePrompts: + answer: ChatPrompt + guard: ChatPrompt + judge: ChatPrompt + other: ChatPrompt + chat: ChatPrompt + system: ChatPrompt + with_system: ChatPrompt + + +# A row compiles one or more requests and runs other compiles before their calls. It gives the +# variables that each call records and the variables of the compile of its request. +LiveCalls: TypeAlias = list[tuple[object, str]] + + +def _other_compiles_then_a_guard(p: LivePrompts) -> LiveCalls: + request = p.answer.compile(documents="The refund policy", question="Where is #4?") + for index in range(150): + p.other.compile(q=str(index)) + p.guard.compile(conversation=request) + return [(_variables_of(request), REFUND_VARIABLES)] + + +def _a_guard_before_each_agent_step(p: LivePrompts) -> LiveCalls: + conversation = p.answer.compile(documents="The refund policy", question="Where is #4?") + for step in range(150): + p.guard.compile(conversation=conversation) + conversation.append({"role": "assistant", "content": f"step {step}"}) + return [(_variables_of(conversation), REFUND_VARIABLES)] + + +def _a_guard_then_a_judge_of_the_guard(p: LivePrompts) -> LiveCalls: + request = p.answer.compile(documents="The refund policy", question="Where is #4?") + p.judge.compile(conversation=p.guard.compile(conversation=request)) + return [(_variables_of(request), REFUND_VARIABLES)] + + +def _turns_in_threads_that_share_a_history(p: LivePrompts) -> LiveCalls: + history: list[dict[str, Any]] = [] + turns = [threading.Event() for _ in range(4)] + compiled = threading.Barrier(3, timeout=5) + recorded: list[object] = [None] * 3 + + def turn(index: int) -> None: + turns[index].wait(5) + request = p.chat.compile(team="billing", history=history, question=f"q{index + 1}") + history.append(request[-1]) + turns[index + 1].set() + # A guard call or a queue: the call comes after the other turns compiled. + compiled.wait() + recorded[index] = _variables_of(request) + + threads = [threading.Thread(target=turn, args=(index,)) for index in range(3)] + for thread in threads: + thread.start() + turns[0].set() + for thread in threads: + thread.join(5) + return list( + zip( + recorded, + [ + '{"team":"billing","history":[],"question":"q1"}', + '{"team":"billing","history":[{"role":"user","content":"q1"}],"question":"q2"}', + '{"team":"billing","history":[{"role":"user","content":"q1"},' + '{"role":"user","content":"q2"}],"question":"q3"}', + ], + strict=True, + ) + ) + + +def _a_system_message_from_startup_after_a_chat_that_gets_it(p: LivePrompts) -> LiveCalls: + startup = p.system.compile(product="Acme") + history: list[dict[str, Any]] = [] + for turn in range(3): + request = p.with_system.compile( + system=startup, history=history, question=f"question {turn}" + ) + prompt_attributes_for(request) + history = [*history, request[-1], _answer(turn)] + return [ + ( + _variables_of([*startup, {"role": "user", "content": "Where is #4?"}]), + '{"product":"Acme"}', + ) + ] + + +@pytest.mark.parametrize( + "run", + [ + _other_compiles_then_a_guard, + _a_guard_before_each_agent_step, + _a_guard_then_a_judge_of_the_guard, + _turns_in_threads_that_share_a_history, + _a_system_message_from_startup_after_a_chat_that_gets_it, + ], + ids=[ + "150-other-compiles-then-a-guard", + "a-guard-before-each-of-150-agent-steps", + "a-guard-then-a-judge-of-the-guard", + "turns-in-threads-that-share-a-history", + "a-system-message-from-startup-after-a-chat-that-gets-it", + ], +) +def test_a_request_keeps_its_variables_until_its_call( + make: MakeClient, run: Callable[[LivePrompts], LiveCalls] +) -> None: + make() + calls = run( + LivePrompts( + answer=chat_prompt(*RAG_MESSAGES), + guard=chat_prompt( + {"role": "system", "content": "Is this safe?"}, + {"role": "placeholder", "name": "conversation"}, + name="guard", + ), + judge=chat_prompt( + {"role": "system", "content": "Check the answer of the guard."}, + {"role": "placeholder", "name": "conversation"}, + name="judge", + ), + other=chat_prompt({"role": "user", "content": "{{q}}"}, name="other"), + chat=chat_prompt(*CHAT_MESSAGES), + system=chat_prompt( + {"role": "system", "content": "You support {{product}}."}, name="system" + ), + with_system=chat_prompt( + {"role": "placeholder", "name": "system"}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, + ), + ) + ) + + for recorded, compiled in calls: + assert recorded == compiled + + +def test_a_compile_keeps_its_variables_for_5_minutes_after_it_or_after_the_last_call_that_names_it( + make: MakeClient, clock: list[float] +) -> None: + make() + chat = chat_prompt(*CHAT_MESSAGES) + history: list[dict[str, Any]] = [] + for turn in range(5): + request = chat.compile(team="billing", history=history, question=f"q{turn}") + prompt_attributes_for(request) + history = [*history, request[-1], _answer(turn)] + # These calls send only messages of the chat, so they name the newest compile among them. + summary = [*history, {"role": "user", "content": "Summarize the chat."}] + earlier = history[:-2] + start = clock[0] + + assert '"question":"q4"' in str(_variables_of(summary)) + assert '"question":"q3"' in str(_variables_of(earlier)) + + clock[0] = start + 299.5 + assert '"question":"q4"' in str(_variables_of(summary)) + clock[0] = start + 300 + assert _variables_of(earlier) is None + # The call 0.5 seconds before moved the deadline of q4. + assert '"question":"q4"' in str(_variables_of(summary)) + clock[0] = start + 600 + assert prompt_attributes_for(summary) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + def test_a_fallback_prompt_records_its_name_and_label_but_no_version( server: CaptureServer, make: MakeClient ) -> None: diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py new file mode 100644 index 0000000..d72d0d2 --- /dev/null +++ b/sdks/python/tests/test_scores.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Any + +import pytest + +import telemetry_dev +from telemetry_dev import ScoreError, score, score_async + +from .conftest import MakeClient +from .fake_api import ApiRequest, FakeApi, Reply, fake_api + +TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736" +SPAN_ID = "00f067aa0ba902b7" + + +def replying(reply: Reply) -> Generator[FakeApi]: + def answer(request: ApiRequest) -> Reply: + return reply + + with fake_api(override=answer) as api: + yield api + + +@pytest.fixture +def accepted() -> Generator[FakeApi]: + yield from replying((202, {"id": "score-1"}, {})) + + +@pytest.fixture +def errors(make: MakeClient) -> list[ScoreError]: + caught: list[ScoreError] = [] + make(on_error=caught.append) + return caught + + +def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( + make: MakeClient, accepted: FakeApi +) -> None: + make(api_key="td_live_key", base_url=f"{accepted.url}/") + + assert ( + score( + name=" user_feedback ", + trace_id=TRACE_ID, + span_id=SPAN_ID, + value=0, + comment="Wrong refund amount", + ) + == "score-1" + ) + + [request] = accepted.requests + assert request.route == "POST /v1/scores" + assert request.headers["authorization"] == "Bearer td_live_key" + assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["user-agent"] == f"telemetry-dev-python/{telemetry_dev.__version__}" + assert request.headers["content-type"] == "application/json" + assert request.body == { + "name": "user_feedback", + "traceId": TRACE_ID, + "spanId": SPAN_ID, + "value": 0, + "comment": "Wrong refund amount", + } + + +async def test_a_session_score_needs_no_trace_and_a_label_goes_trimmed(accepted: FakeApi) -> None: + returned = await score_async( + api_key="td_live_key", + base_url=accepted.url, + name="resolved", + session_id="chat-9", + label=" yes ", + ) + + assert returned == "score-1" + assert accepted.requests[0].body == {"name": "resolved", "sessionId": "chat-9", "label": "yes"} + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"trace_id": None}, "Pass trace_id or session_id"), + ( + {"trace_id": "0" * 32}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": TRACE_ID.upper()}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": None, "session_id": "s", "span_id": SPAN_ID}, + "Pass the trace_id of the span", + ), + ({"value": None}, "Pass a value or a label"), + ({"value": float("nan")}, "value must be a finite number"), + ({"value": True}, "value must be a finite number"), + ({"value": 10**400}, "value must be a finite number"), + ( + {"name": "user_feedback", "value": 1, "label": "up"}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ( + {"name": "user_feedback", "value": 0.5}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ({"value": 0, "label": "pass"}, "The label pass goes with the value 1"), + ({"value": 1, "label": "fail"}, "The label fail goes with the value 0"), + ({"name": " "}, "name must be a string of 1 to 64 characters"), + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + ({"name": "\U0001f600" * 33}, "name must be a string of 1 to 64 characters"), + ({"comment": "x" * 4001}, "comment must be a string of at most 4000 characters"), + ], + ids=[ + "no trace and no session", + "the all-zero trace id of a span that did not record", + "an uppercase trace id", + "a span without its trace", + "no value and no label", + "a value that is not finite", + "a bool value", + "an int too large for a float", + "a label for user feedback", + "user feedback of 0.5", + "a pass of 0", + "a fail of 1", + "a blank name", + "a name of 66 UTF-16 units", + "a comment of 4001 characters", + ], +) +def test_reports_invalid_score_and_sends_nothing( + errors: list[ScoreError], accepted: FakeApi, options: dict[str, Any], message: str +) -> None: + arguments: dict[str, Any] = { + "api_key": "td_live_key", + "base_url": accepted.url, + "name": "helpful", + "trace_id": TRACE_ID, + "value": 1, + } + arguments.update(options) + + assert score(**arguments) is None + assert accepted.requests == [] + assert [(error.code, error.message) for error in errors] == [("invalid_score", message)] + + +FAILURES: list[tuple[Reply, str, int | None]] = [ + ( + (403, {"error": {"code": "insufficient_scope", "message": "No scores"}}, {}), + "insufficient_scope", + 403, + ), + ((503, {}, {"retry-after": "1"}), "http_503", 503), + ((202, {}, {}), "invalid_response", None), +] + + +@pytest.mark.parametrize( + ("reply", "code", "status"), + FAILURES, + ids=["the API error", "a status with no API error", "a body with no id"], +) +def test_reports_the_failure_tries_once_and_returns_none( + errors: list[ScoreError], reply: Reply, code: str, status: int | None +) -> None: + for api in replying(reply): + assert ( + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + is None + ) + assert len(api.requests) == 1 + + assert [(error.code, error.status) for error in errors] == [(code, status)] + + +def test_without_an_api_key_it_reports_missing_api_key( + errors: list[ScoreError], accepted: FakeApi +) -> None: + score(api_key="", base_url=accepted.url, name="helpful", trace_id=TRACE_ID, value=1) + + assert accepted.requests == [] + assert [error.code for error in errors] == ["missing_api_key"] + + +def test_does_not_follow_a_redirect_so_the_key_stays_with_the_api_host( + errors: list[ScoreError], +) -> None: + with fake_api() as other: + + def redirect(request: ApiRequest) -> Reply: + return 307, {}, {"location": f"{other.url}/v1/scores"} + + with fake_api(override=redirect) as api: + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + + assert other.requests == [] + assert [(error.code, error.status) for error in errors] == [("http_307", 307)] diff --git a/sdks/python/uv.lock b/sdks/python/uv.lock index ee9f2d2..ed63e15 100644 --- a/sdks/python/uv.lock +++ b/sdks/python/uv.lock @@ -404,7 +404,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "." } dependencies = [ { name = "opentelemetry-api" }, diff --git a/vite.config.ts b/vite.config.ts index 6adbb93..b77ab81 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ staged: { "*": "vp check --fix --no-error-on-unmatched-pattern", }, - fmt: { ignorePatterns: ["sdks/**", "packages/*/CHANGELOG.md"] }, + fmt: { ignorePatterns: ["sdks/**", "packages/*/CHANGELOG.md", "actions/*/CHANGELOG.md"] }, lint: { ignorePatterns: ["sdks/**"], options: { typeAware: true, typeCheck: true },