From 2f479a57767a13cdad892dc3aafa3ca0b736bfa7 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:01:41 +0000 Subject: [PATCH 01/32] feat(sdk): add runExperiment and score runExperiment gets a dataset version page by page and makes the experiment. It runs the task on each item, with concurrency and runs for each item. It sends the outputs and the local scores in batches of at most 100. Then it completes the experiment, waits for the server evaluators, and returns the summary. Each item run has its own trace. Each span of the run has td.experiment.id and td.dataset.item_id. In CI mode, runExperiment records the GitHub Actions run, and it rejects when the check is not successful. score sends a score on a trace, a span, or a session to /v1/scores. It never rejects. It sends its errors to onError. getPrompt and the new calls now use one function for the key, the base URL, and the Retry-After header. --- packages/sdk/src/api.ts | 199 +++++ packages/sdk/src/ci.ts | 159 ++++ packages/sdk/src/experiments.ts | 958 +++++++++++++++++++++++++ packages/sdk/src/index.ts | 14 + packages/sdk/src/prompts.ts | 48 +- packages/sdk/src/scores.ts | 170 +++++ packages/sdk/tests/experiments.test.ts | 723 +++++++++++++++++++ packages/sdk/tests/scores.test.ts | 220 ++++++ 8 files changed, 2448 insertions(+), 43 deletions(-) create mode 100644 packages/sdk/src/api.ts create mode 100644 packages/sdk/src/ci.ts create mode 100644 packages/sdk/src/experiments.ts create mode 100644 packages/sdk/src/scores.ts create mode 100644 packages/sdk/tests/experiments.test.ts create mode 100644 packages/sdk/tests/scores.test.ts diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts new file mode 100644 index 0000000..8274701 --- /dev/null +++ b/packages/sdk/src/api.ts @@ -0,0 +1,199 @@ +import { DEFAULT_BASE_URL, resolveEnv } from "@telemetry-dev/otel"; + +import { currentClient } from "./client.ts"; + +/** Where the calls to the telemetry.dev API go, and with which key. */ +export interface ApiOptions { + /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + apiKey?: string; + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://ingest.telemetry.dev`. */ + baseUrl?: string; + /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ + fetch?: typeof fetch; +} + +export interface Api { + apiKey: string | undefined; + baseUrl: string; + sdkName: string; + fetchImpl: typeof fetch; + onError: ((cause: Error) => void) | undefined; +} + +/** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */ +export type ApiErrorType = new ( + code: string, + message: string, + status?: number, + options?: { cause?: unknown }, +) => E; + +interface RequestOptions { + body?: unknown; + query?: Record; + timeoutMs: number; + /** + * The server takes a repeat of this request at most once, so a retry after a timeout, a network + * error, or a 5xx is safe. Other requests try again only after a 429 or a 503: the server did + * not take the request then. + */ + idempotent: boolean; + /** Default 3. */ + attempts?: number; +} + +type Attempt = + | { ok: true; body: unknown } + | { ok: false; error: Error; status?: number; retryAfterMs?: number }; + +/** A longer Retry-After waits only this long: the API counts requests by the minute. */ +const MAX_RETRY_AFTER_MS = 60_000; +/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ +const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; + +/** The key, base URL, and fetch of a call: its options, then the initialized client, then the env. */ +export function resolveApi(options: ApiOptions): Api { + const config = currentClient().core?.config; + const env = resolveEnv(); + + return { + apiKey: options.apiKey ?? config?.apiKey ?? env.TELEMETRY_DEV_API_KEY, + // An empty value counts as unset, as in the Python SDK. + baseUrl: ( + options.baseUrl || + config?.baseUrl || + env.TELEMETRY_DEV_BASE_URL || + DEFAULT_BASE_URL + ).replace(/\/+$/, ""), + sdkName: config?.sdkName ?? "@telemetry-dev/sdk", + fetchImpl: options.fetch ?? config?.fetchImpl ?? globalThis.fetch, + onError: config?.onError, + }; +} + +/** + * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response + * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_`. + * A failure that `options.idempotent` allows tries again, after the Retry-After of the response + * or after 1 and then 2 seconds. + */ +export async function request( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + path: string, + options: RequestOptions, +): Promise { + const url = new URL(`${api.baseUrl}${path}`); + + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) url.searchParams.set(key, value); + } + + // Outside send(), so that a value JSON cannot hold fails as itself, not as a network error. + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const attempts = options.attempts ?? 3; + + for (let attempt = 1; ; attempt++) { + const result = await send(api, ErrorType, method, url, body, options.timeoutMs); + + if (result.ok) return result.body; + + if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error; + + await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS)); + } +} + +/** The wait that the Retry-After of a 429 or a 503 asks for, in seconds or as an HTTP date. */ +export function retryAfterMs(res: Response): number | undefined { + if (res.status !== 429 && res.status !== 503) return undefined; + + const value = res.headers.get("retry-after")?.trim(); + + if (!value) return undefined; + + let ms: number; + + if (/^\d+$/.test(value)) ms = Number(value) * 1000; + else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); + else return undefined; + + return ms > 0 ? ms : undefined; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function send( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + url: URL, + body: string | undefined, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const res = await api.fetchImpl(url, { + method, + headers: { + authorization: `Bearer ${api.apiKey}`, + "x-telemetry-dev-sdk": api.sdkName, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body, + signal: controller.signal, + // A redirect to another host must not get the API key, so a 3xx fails below like any status + // that is not OK. Cloudflare Workers reject redirect: "error". + redirect: "manual", + }); + + const json: unknown = await res.json().catch(() => undefined); + + if (res.ok) return { ok: true, body: json }; + + const error = isRecord(json) && isRecord(json.error) ? json.error : {}; + + return { + ok: false, + status: res.status, + retryAfterMs: retryAfterMs(res), + error: new ErrorType( + typeof error.code === "string" ? error.code : `http_${res.status}`, + typeof error.message === "string" ? error.message : `telemetry.dev returned ${res.status}`, + res.status, + ), + }; + } catch (error) { + // The timeout can also cut the body, which then reads as an invalid response. + if (controller.signal.aborted) { + return { + ok: false, + error: new ErrorType("timeout", `${method} ${url.pathname} took more than ${timeoutMs} ms`), + }; + } + + return { + ok: false, + error: new ErrorType("network_error", "Could not reach telemetry.dev", undefined, { + cause: error, + }), + }; + } finally { + clearTimeout(timer); + } +} + +function retryable(status: number | undefined, idempotent: boolean): boolean { + if (status === 429 || status === 503) return true; + + return idempotent && (status === undefined || status >= 500); +} diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts new file mode 100644 index 0000000..45eb20c --- /dev/null +++ b/packages/sdk/src/ci.ts @@ -0,0 +1,159 @@ +import type * as fs from "node:fs"; + +import { reportError } from "@telemetry-dev/otel"; + +/** The pull request and the workflow run of an experiment that runs in GitHub Actions. */ +export interface CiRun { + provider: "github_actions"; + /** `owner/name`. */ + repository: string; + commit: string; + prNumber?: number; + prTitle?: string; + prUrl?: string; + runUrl?: string; +} + +type Env = Record; + +type RuntimeGlobal = typeof globalThis & { + process?: { getBuiltinModule?: (id: string) => unknown }; +}; + +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const COMMIT = /^[0-9a-f]{7,64}$/i; +const HTTPS_URL = /^https:\/\/\S+$/; +/** A plain decimal, so that the TypeScript and Python SDKs read the same marks. */ +const DECIMAL = /^-?\d+(\.\d+)?$/; +const MAX_REPOSITORY_LENGTH = 200; +const MAX_URL_LENGTH = 2_048; +const MAX_PR_TITLE_LENGTH = 256; + +/** CI mode is on when `CI` is `true` or `1`, as CI services set it. */ +export function inCi(env: Env): boolean { + return env.CI === "true" || env.CI === "1"; +} + +/** + * The run that GitHub Actions describes in its variables, or undefined outside GitHub Actions. On a + * pull request the commit is the head of the pull request: `GITHUB_SHA` is then a merge commit + * that nobody pushed. + */ +export function githubRun(env: Env): CiRun | undefined { + const repository = env.GITHUB_REPOSITORY; + + if ( + env.GITHUB_ACTIONS !== "true" || + !repository || + repository.length > MAX_REPOSITORY_LENGTH || + !REPOSITORY.test(repository) + ) { + return undefined; + } + + const pull = pullRequest(env.GITHUB_EVENT_PATH); + const commit = pull?.commit ?? env.GITHUB_SHA; + + if (!commit || !COMMIT.test(commit)) return undefined; + + const server = (env.GITHUB_SERVER_URL ?? "https://github.com").replace(/\/+$/, ""); + const runId = env.GITHUB_RUN_ID; + + return { + provider: "github_actions", + repository, + commit, + prNumber: pull?.number, + prTitle: pull?.title, + prUrl: pull?.url, + runUrl: + runId && /^\d+$/.test(runId) + ? httpsUrl(`${server}/${repository}/actions/runs/${runId}`) + : undefined, + }; +} + +/** + * The pass marks in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which the experiment action sets from + * its `min-scores` input: `name=value` pairs, separated by commas or new lines. + */ +export function minScoresFromEnv(env: Env): Record { + const scores: Record = {}; + + for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { + if (pair.trim() === "") continue; + const [name, value, ...rest] = pair.split("=").map((part) => part.trim()); + + if (!name || !value || rest.length > 0 || !DECIMAL.test(value)) { + throw new TypeError( + `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "${pair.trim()}" is not a name=number pair`, + ); + } + + scores[name] = Number(value); + } + + return scores; +} + +/** + * Adds the result to the file in `TELEMETRY_DEV_EXPERIMENT_RESULTS` as one line of JSON. The + * experiment action reads the file to write the job summary and the comment on the pull request. + */ +export function writeResult(env: Env, result: unknown, onError?: (cause: Error) => void): void { + const path = env.TELEMETRY_DEV_EXPERIMENT_RESULTS; + + if (!path) return; + + try { + const fs = nodeFs(); + + if (!fs) throw new Error("This runtime cannot write TELEMETRY_DEV_EXPERIMENT_RESULTS"); + + fs.appendFileSync(path, `${JSON.stringify(result)}\n`, { mode: 0o600 }); + } catch (error) { + reportError(onError, error); + } +} + +function pullRequest( + eventPath: string | undefined, +): { number?: number; title?: string; url?: string; commit?: string } | undefined { + if (!eventPath) return undefined; + + let event: unknown; + + try { + event = JSON.parse(nodeFs()?.readFileSync(eventPath, "utf8") ?? "null"); + } catch { + return undefined; + } + + const pull = (event as { pull_request?: Record } | null)?.pull_request; + + if (typeof pull !== "object" || pull === null) return undefined; + + const head = pull.head as { sha?: unknown } | undefined; + + return { + number: + Number.isSafeInteger(pull.number) && (pull.number as number) > 0 + ? (pull.number as number) + : undefined, + title: typeof pull.title === "string" ? pull.title.slice(0, MAX_PR_TITLE_LENGTH) : undefined, + url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, + commit: typeof head?.sha === "string" ? head.sha : undefined, + }; +} + +function httpsUrl(url: string): string | undefined { + return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; +} + +// Read at call time, with no static `node:` import, so that the package still loads in runtimes +// without a file system. +function nodeFs(): typeof fs | undefined { + return (globalThis as RuntimeGlobal).process?.getBuiltinModule?.("node:fs") as + | typeof fs + | undefined; +} diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts new file mode 100644 index 0000000..bb90d0d --- /dev/null +++ b/packages/sdk/src/experiments.ts @@ -0,0 +1,958 @@ +import { type Context, trace } from "@opentelemetry/api"; +import { + activeContext, + diag, + jsonAttr, + PROPAGATED_KEY, + propagatedFromContext, + reportError, + resolveEnv, +} from "@telemetry-dev/otel"; + +import { type Api, type ApiOptions, isRecord, request, resolveApi, sleep } from "./api.ts"; +import { truncate } from "./capture.ts"; +import { type CiRun, githubRun, inCi, minScoresFromEnv, writeResult } from "./ci.ts"; +import { currentClient, flush } from "./client.ts"; +import { checkValue, MAX_REASON_LENGTH, MAX_SCORE_NAME_LENGTH, USER_FEEDBACK } from "./scores.ts"; +import { type SpanHandle, startActiveSpan } from "./span.ts"; + +/** An item of a dataset version, as `GET /v1/datasets/{name}` returns it. */ +export interface DatasetItem { + /** The key of the item. It stays the same across dataset versions. */ + readonly id: string; + readonly number: number; + readonly input: Readonly>; + readonly expectedOutput: string | null; + readonly metadata: Readonly>; + readonly tags: readonly string[]; + readonly split: string | null; +} + +/** One run of a dataset item. With `runsPerItem` above 1, an item runs more than one time. */ +export interface ExperimentItem extends DatasetItem { + /** 0 to `runsPerItem` - 1. */ + readonly runIndex: number; +} + +/** + * What a scorer returns. `true` and `false` are a pass and a fail, a number is the value, and a + * string is the label. Null or undefined gives no score. + */ +export type ScorerResult = + | boolean + | number + | string + | { value?: number | boolean; label?: string; reason?: string } + | null + | undefined; + +/** Scores the output of an item on your machine. The score has the name of the scorer. */ +export type Scorer = ( + output: unknown, + item: ExperimentItem, +) => ScorerResult | Promise; + +export interface RunExperimentOptions extends ApiOptions { + /** Runs the code under test on one item. Its return value is the output of the item. */ + task: (item: ExperimentItem) => unknown; + /** + * The name of the dataset. Default: the `dataset` input of the experiment action, which it + * sets in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When both are set, they must be the same. + */ + dataset?: string; + /** Unique in the project. telemetry.dev picks a name when it is not set. */ + name?: string; + /** Default: the newest version. */ + datasetVersion?: number; + /** Run only the items of this split. */ + split?: string; + /** Evaluators of the project that score the outputs on the server, by name. */ + evaluators?: readonly string[]; + /** Scorers that run here, by score name. */ + scorers?: Readonly>; + /** + * Pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail score, or the lowest + * mean of a number score. The `min-scores` input of the experiment action adds to them and wins + * for the same name. + */ + minScores?: Readonly>; + /** 1 to 5. Default 1. */ + runsPerItem?: number; + /** Item runs at the same time, 1 to 100. Default 4. */ + concurrency?: number; + metadata?: Readonly>; + /** + * Records the GitHub Actions run, and rejects when the check fails or the experiment does not + * complete. Default: on when the `CI` variable is `true` or `1`. + */ + ci?: boolean; +} + +export interface ExperimentScore { + readonly name: string; + /** `pass_fail`, `number`, `labels`, `a_b_tie`, or `up_down`. Null before the score has values. */ + readonly kind: string | null; + /** The pass rate (0 to 1) of a pass/fail or up/down score, or the mean of a number score. */ + readonly value: number | null; + readonly passCount: number; + readonly failCount: number; + /** The value in the experiment that this one compares with, by default the dataset baseline. */ + readonly baselineValue: number | null; + /** Whole points of a pass rate, or the change of a mean to one decimal. */ + readonly delta: number | null; + readonly minScore: number | null; + /** Null when the score has no pass mark. */ + readonly passed: boolean | null; +} + +export interface ExperimentResult { + readonly id: string; + readonly name: string; + /** `completed`, `failed`, or `stopped`. */ + readonly status: string; + /** Null when the experiment has no pass marks. */ + readonly checkStatus: "passed" | "failed" | null; + readonly itemsTotal: number; + readonly itemsDone: number; + readonly itemsFailed: number; + /** Why the experiment failed. */ + readonly error: string | null; + readonly scores: readonly ExperimentScore[]; + readonly url: string; +} + +/** + * Why runExperiment failed. `code` is the code of the API error (`dataset_not_found`, + * `insufficient_scope`, `invalid_api_key`, …), or `missing_api_key`, `timeout`, `network_error`, + * `invalid_response`, or `dataset_empty`. In CI mode it is also `check_failed` or + * `experiment_failed`, and `result` holds the result. A response with a different status and no + * API error gives `http_`. A scorer that throws or returns a value that is not a score + * reports a `scorer_failed` error through `onError`. + */ +export class ExperimentError extends Error { + override readonly name = "ExperimentError"; + readonly result?: ExperimentResult; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown; result?: ExperimentResult }, + ) { + super(message, options); + this.result = options?.result; + } +} + +interface Run { + dataset: string; + task: (item: ExperimentItem) => unknown; + name?: string; + datasetVersion?: number; + split?: string; + evaluators?: readonly string[]; + scorers: [string, Scorer][]; + minScores?: Record; + runsPerItem: number; + concurrency: number; + metadata?: Readonly>; + ci: CiRun | undefined; + inCi: boolean; +} + +interface Created { + id: string; + name: string; + itemsTotal: number; + url: string; +} + +interface DatasetPage { + version: number; + items: DatasetItem[]; + nextCursor: string | null; +} + +interface ItemScore { + name: string; + value?: number; + label?: string; + reason?: string; +} + +interface ItemResult { + itemId: string; + runIndex: number; + output: string | null; + traceId?: string; + latencyMs: number; + error?: string; + scores?: ItemScore[]; +} + +type Authed = Api & { apiKey: string }; + +/** The `experiment` environment, which pages, metrics, and alerts leave out by default. */ +const EXPERIMENT_ENVIRONMENT = "experiment"; +const MAX_RUNS_PER_ITEM = 5; +const MAX_CONCURRENCY = 100; +/** Server evaluators of one experiment, and scorers: an item takes at most 20 scores. */ +const MAX_EVALUATORS = 20; +const MAX_MIN_SCORES = 20; +const MAX_ERROR_LENGTH = 4_000; +const MAX_OUTPUT_LENGTH = 100_000; +const MAX_DATASET_ITEMS = 10_000; +/** Items in one `POST /v1/experiments/{id}/items`. */ +const MAX_BATCH_ITEMS = 100; +/** Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. */ +const MAX_BATCH_BYTES = 4 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; +/** How long runExperiment waits for the server evaluators after the last item. */ +const MAX_WAIT_MS = 30 * 60_000; +const MAX_POLL_INTERVAL_MS = 5_000; +const FINISHED = new Set(["completed", "failed", "stopped"]); +const RATE_KINDS = new Set(["pass_fail", "up_down"]); + +/** + * Runs `task` on each item of a dataset and records the result as an experiment in telemetry.dev. + * Each item run is its own trace, with `td.experiment.id` and `td.dataset.item_id` on each span. + * The outputs go to telemetry.dev in batches of at most 100, with the scores of `scorers`. Then + * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. + * + * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` + * when a request fails. A task that throws fails only its item. In CI mode, runExperiment rejects + * when the check fails, so the process exits with an error. + */ +export async function runExperiment(options: RunExperimentOptions): Promise { + const env = resolveEnv(); + const run = checkOptions(options, env); + const resolved = resolveApi(options); + + if (!resolved.apiKey) { + throw new ExperimentError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const api = { ...resolved, apiKey: resolved.apiKey }; + warnAboutClient(); + + const first = await fetchPage(api, run, run.datasetVersion, undefined); + + if (first.items.length === 0) { + throw new ExperimentError( + "dataset_empty", + `The dataset ${run.dataset} has no items${run.split === undefined ? "" : ` in the split ${run.split}`}`, + ); + } + + const created = parseCreated( + await request(api, ExperimentError, "POST", "/v1/experiments", { + body: { + name: run.name, + dataset: run.dataset, + datasetVersion: first.version, + split: run.split, + evaluators: run.evaluators, + minScores: run.minScores, + runsPerItem: run.runsPerItem, + ci: run.ci, + metadata: run.metadata, + }, + timeoutMs: REQUEST_TIMEOUT_MS, + // A repeat after a lost response would start a second experiment. + idempotent: false, + }), + ); + + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + + await runItems(api, run, created, itemRuns(api, run, first)); + // The spans of the items reach ingest before the server reads their cost. + await flush(); + + await request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(created.id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const { result, body } = await waitForResult(api, created.id); + writeResult(env, body, api.onError); + console.log(describe(result)); + + if (run.inCi) checkResult(result); + + return result; +} + +function checkOptions(options: RunExperimentOptions, env: Record): Run { + if (typeof options.task !== "function") throw new TypeError("task must be a function"); + + for (const [key, value] of [ + ["dataset", options.dataset], + ["name", options.name], + ["split", options.split], + ] as const) { + if (value !== undefined && (typeof value !== "string" || value.length === 0)) { + throw new TypeError(`${key} must be a non-empty string`); + } + } + + const actionDataset = env.TELEMETRY_DEV_EXPERIMENT_DATASET || undefined; + const dataset = options.dataset ?? actionDataset; + + if (dataset === undefined) { + throw new TypeError("Pass dataset, or set the dataset input of the experiment action"); + } + + if (actionDataset !== undefined && actionDataset !== dataset) { + throw new TypeError( + `dataset is ${dataset}, but the experiment action names ${actionDataset}. Name one dataset.`, + ); + } + + const { datasetVersion } = options; + + if ( + datasetVersion !== undefined && + !(Number.isSafeInteger(datasetVersion) && datasetVersion >= 1) + ) { + throw new RangeError("datasetVersion must be a positive integer"); + } + + const evaluators = options.evaluators; + + if ( + evaluators !== undefined && + (!Array.isArray(evaluators) || + evaluators.length > MAX_EVALUATORS || + !evaluators.every((name) => typeof name === "string" && validScoreName(name))) + ) { + throw new TypeError(`evaluators must be a list of at most ${MAX_EVALUATORS} evaluator names`); + } + + const scorers = Object.entries(options.scorers ?? {}); + const evaluatorNames = new Set(evaluators?.map((name) => name.toLowerCase())); + + if (scorers.length > MAX_EVALUATORS) { + throw new TypeError(`scorers can have at most ${MAX_EVALUATORS} scorers`); + } + + for (const [name, scorer] of scorers) { + if (!validScoreName(name) || typeof scorer !== "function") { + throw new TypeError( + `scorers must map score names of 1 to ${MAX_SCORE_NAME_LENGTH} characters to functions`, + ); + } + + if (name === USER_FEEDBACK) { + throw new TypeError( + `${USER_FEEDBACK} is the score of user feedback. Name the scorer another way.`, + ); + } + + if (evaluatorNames.has(name.toLowerCase())) { + throw new TypeError(`The scorer ${name} has the name of an evaluator. Rename one of them.`); + } + } + + const minScores = { ...options.minScores, ...minScoresFromEnv(env) }; + + if ( + Object.keys(minScores).length > MAX_MIN_SCORES || + !Object.entries(minScores).every( + ([name, value]) => validScoreName(name) && Number.isFinite(value), + ) + ) { + throw new TypeError(`minScores must map at most ${MAX_MIN_SCORES} score names to numbers`); + } + + const runsPerItem = options.runsPerItem ?? 1; + + if (!Number.isInteger(runsPerItem) || runsPerItem < 1 || runsPerItem > MAX_RUNS_PER_ITEM) { + throw new RangeError(`runsPerItem must be an integer from 1 to ${MAX_RUNS_PER_ITEM}`); + } + + const concurrency = options.concurrency ?? 4; + + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) { + throw new RangeError(`concurrency must be an integer from 1 to ${MAX_CONCURRENCY}`); + } + + if (options.metadata !== undefined && !isRecord(options.metadata)) { + throw new TypeError("metadata must be an object"); + } + + if (options.ci !== undefined && typeof options.ci !== "boolean") { + throw new TypeError("ci must be a boolean"); + } + + const ci = options.ci ?? inCi(env); + + return { + dataset, + task: options.task, + name: options.name, + datasetVersion, + split: options.split, + evaluators, + scorers, + minScores: Object.keys(minScores).length > 0 ? minScores : undefined, + runsPerItem, + concurrency, + metadata: options.metadata, + ci: ci ? githubRun(env) : undefined, + inCi: ci, + }; +} + +/** A trimmed name of 1 to 64 characters, as the API stores score names. */ +function validScoreName(name: string): boolean { + return name.length > 0 && name.length <= MAX_SCORE_NAME_LENGTH && name.trim() === name; +} + +function warnAboutClient(): void { + const environment = currentClient().core?.config.environment; + + if (environment === undefined) { + diag.warn( + 'runExperiment: no telemetry client, so the items record no traces. Call init({ environment: "experiment" }) first.', + ); + } else if (environment.trim().toLowerCase() !== EXPERIMENT_ENVIRONMENT) { + diag.warn( + `runExperiment: the traces of the items go to the environment ${environment}. Call init({ environment: "experiment" }) to keep them out of it.`, + ); + } +} + +/** + * The runs of each item of the dataset version, page by page, so that only one page is in memory. + * Workers share the generator: an async generator answers each `next()` in turn. + */ +async function* itemRuns( + api: Authed, + run: Run, + first: DatasetPage, +): AsyncGenerator { + let page = first; + let count = 0; + + for (;;) { + count += page.items.length; + + if (count > MAX_DATASET_ITEMS) { + throw new ExperimentError( + "invalid_response", + `telemetry.dev returned more than ${MAX_DATASET_ITEMS} items`, + ); + } + + for (const item of page.items) { + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) yield { ...item, runIndex }; + } + + if (page.nextCursor === null) return; + + page = await fetchPage(api, run, page.version, page.nextCursor); + } +} + +/** One page of a dataset version. A page after the first names the version of the first. */ +async function fetchPage( + api: Authed, + run: Run, + version: number | undefined, + cursor: string | undefined, +): Promise { + const page = parseDatasetPage( + await request(api, ExperimentError, "GET", `/v1/datasets/${encodeURIComponent(run.dataset)}`, { + query: { version: version?.toString(), split: run.split, cursor }, + timeoutMs: REQUEST_TIMEOUT_MS, + idempotent: true, + }), + ); + + if ( + !page || + (version !== undefined && page.version !== version) || + (page.nextCursor !== null && (page.items.length === 0 || page.nextCursor === cursor)) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned a dataset it cannot read", + ); + } + + return page; +} + +/** + * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that + * fails stops the run: no new item starts, and runExperiment rejects when the running items end. + */ +async function runItems( + api: Authed, + run: Run, + experiment: Created, + runs: AsyncGenerator, +): Promise { + const batch = itemBatch((results) => + request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(experiment.id)}/items`, + // The server skips an item run that it has, so a repeat is safe. + { body: { items: results }, timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ), + ); + + let failure: { error: unknown } | undefined; + + const worker = async () => { + while (failure === undefined) { + const next = await runs.next().catch((error: unknown) => { + failure ??= { error }; + + return { done: true as const, value: undefined }; + }); + + if (next.done) return; + + const result = await runItem(run, experiment, next.value, api.onError); + + await batch.add(result).catch((error: unknown) => { + failure ??= { error }; + }); + } + }; + + await Promise.all(Array.from({ length: run.concurrency }, worker)); + + if (failure === undefined) { + await batch.flush().catch((error: unknown) => { + failure = { error }; + }); + } + + if (failure !== undefined) { + await runs.return(undefined); + + throw failure.error; + } +} + +/** Collects item results and posts them in order, one post at a time. */ +function itemBatch(post: (results: ItemResult[]) => Promise) { + let pending: ItemResult[] = []; + let bytes = 0; + let queue: Promise = Promise.resolve(); + + const send = async () => { + const results = pending; + pending = []; + bytes = 0; + await post(results); + }; + + return { + add(result: ItemResult): Promise { + const size = new TextEncoder().encode(JSON.stringify(result)).length; + + queue = queue.then(async () => { + if (pending.length > 0 && bytes + size > MAX_BATCH_BYTES) await send(); + pending.push(result); + bytes += size; + + if (pending.length >= MAX_BATCH_ITEMS) await send(); + }); + + return queue; + }, + flush(): Promise { + queue = queue.then(() => (pending.length > 0 ? send() : undefined)); + + return queue; + }, + }; +} + +async function runItem( + run: Run, + experiment: Created, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const started = performance.now(); + + const outcome = await startActiveSpan( + `experiment ${experiment.name}`, + { parent: itemContext(experiment.id, item), input: item.input }, + async (span): Promise<{ output?: unknown; error?: unknown; traceId?: string }> => { + try { + const output: unknown = await run.task(item); + span.update({ output }); + + return { output, traceId: traceIdOf(span) }; + } catch (error) { + span.update({ error }); + + return { + error: error ?? new Error("The task threw a nullish value"), + traceId: traceIdOf(span), + }; + } + }, + ); + + const base = { + itemId: item.id, + runIndex: item.runIndex, + traceId: outcome.traceId, + latencyMs: Math.round(performance.now() - started), + }; + + if ("error" in outcome) { + return { ...base, output: null, error: truncate(errorText(outcome.error), MAX_ERROR_LENGTH) }; + } + + const scores = await runScorers(run, outcome.output, item, onError); + + return { + ...base, + output: outputText(outcome.output), + scores: scores.length > 0 ? scores : undefined, + }; +} + +/** + * The context of one item run. It drops the active span, so that the run starts a trace of its + * own, and gives the run a session id of its own: an ambient session or `sessionMode: "process"` + * would otherwise put all runs in one session trace. + */ +function itemContext(experimentId: string, item: ExperimentItem): Context { + const base = trace.deleteSpan(activeContext()); + + return base.setValue(PROPAGATED_KEY, { + ...propagatedFromContext(base), + "td.experiment.id": experimentId, + "td.dataset.item_id": item.id, + "gen_ai.conversation.id": `${experimentId}:${item.id}:${item.runIndex}`, + }); +} + +function traceIdOf(span: SpanHandle): string | undefined { + return span.isRecording ? span.traceId : undefined; +} + +async function runScorers( + run: Run, + output: unknown, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const scores: ItemScore[] = []; + + // After the span of the item ends, so that the model calls of a scorer do not count in the + // cost of the item. + for (const [name, scorer] of run.scorers) { + try { + const score = toScore(name, await scorer(output, item)); + + if (score) scores.push(score); + } catch (error) { + reportError( + onError, + new ExperimentError( + "scorer_failed", + `The scorer ${name} failed on item ${item.number}: ${errorText(error)}`, + undefined, + { cause: error }, + ), + ); + } + } + + return scores; +} + +/** The score of a scorer result. It throws for a result that the server would reject. */ +function toScore(name: string, result: ScorerResult): ItemScore | undefined { + if (result === null || result === undefined) return undefined; + + let score: ItemScore; + + if (typeof result === "boolean") score = { name, ...passFail(result) }; + else if (typeof result === "number") score = { name, value: result }; + else if (typeof result === "string") score = { name, label: result.trim() }; + else if (isRecord(result)) { + const { value, label, reason } = result as Record; + + for (const [key, field] of [ + ["label", label], + ["reason", reason], + ] as const) { + if (field !== undefined && typeof field !== "string") { + throw new TypeError(`returned a ${key} that is not a string`); + } + } + + score = { + name, + ...(typeof value === "boolean" ? passFail(value) : { value: value as number | undefined }), + ...(label === undefined ? {} : { label: (label as string).trim() }), + reason: reason === undefined ? undefined : truncate(reason as string, MAX_REASON_LENGTH), + }; + } else { + throw new TypeError(`returned a ${typeof result}`); + } + + const problem = checkValue(name, score.value, score.label); + + if (problem) throw new TypeError(`returned a score that is not valid: ${problem}`); + + return score; +} + +function passFail(pass: boolean): { value: number; label: string } { + return pass ? { value: 1, label: "pass" } : { value: 0, label: "fail" }; +} + +function outputText(output: unknown): string | null { + const text = output === null || output === undefined ? undefined : jsonAttr(output); + + return text === undefined ? null : truncate(text, MAX_OUTPUT_LENGTH); +} + +function errorText(error: unknown): string { + if (error instanceof Error) return `${error.name}: ${error.message}`; + + try { + return String(error); + } catch { + return "A value that cannot be converted to a string was thrown"; + } +} + +/** Waits until the server evaluators finish and the experiment ends. */ +async function waitForResult( + api: Authed, + id: string, +): Promise<{ result: ExperimentResult; body: unknown }> { + const deadline = Date.now() + MAX_WAIT_MS; + + for (let interval = 1000; ; interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS)) { + const body = await request( + api, + ExperimentError, + "GET", + `/v1/experiments/${encodeURIComponent(id)}`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const result = parseResult(body); + + if (FINISHED.has(result.status)) return { result, body }; + + if (Date.now() + interval > deadline) { + throw new ExperimentError( + "timeout", + `The experiment ${result.name} did not end in ${MAX_WAIT_MS / 60_000} minutes. ${result.url}`, + undefined, + { result }, + ); + } + + await sleep(interval); + } +} + +/** In CI mode, a failed check or an experiment that did not complete fails the run. */ +function checkResult(result: ExperimentResult): void { + if (result.status !== "completed") { + throw new ExperimentError( + "experiment_failed", + `The experiment ${result.name} ${result.status}${result.error ? `: ${result.error}` : ""}. ${result.url}`, + undefined, + { result }, + ); + } + + if (result.checkStatus !== "failed") return; + + const failed = result.scores + .filter((score) => score.passed === false) + .map( + (score) => + `${score.name} is ${formatScore(score.kind, score.value)}, under the pass mark ${formatScore(score.kind, score.minScore)}`, + ); + + throw new ExperimentError( + "check_failed", + `The check of ${result.name} failed${failed.length > 0 ? `: ${failed.join("; ")}` : ""}. ${result.url}`, + undefined, + { result }, + ); +} + +function describe(result: ExperimentResult): string { + const outcome = + result.status !== "completed" + ? `${result.status}${result.error ? `: ${result.error}` : ""}` + : result.checkStatus === null + ? "completed" + : `completed. The check ${result.checkStatus}`; + + return [ + `Experiment ${result.name} ${outcome}. ${result.url}`, + ...result.scores.map( + (score) => + ` ${score.name}: ${formatScore(score.kind, score.value)}` + + (score.minScore === null ? "" : ` (pass mark ${formatScore(score.kind, score.minScore)})`), + ), + ].join("\n"); +} + +function formatScore(kind: string | null, value: number | null): string { + if (value === null) return "no score"; + + return kind !== null && RATE_KINDS.has(kind) + ? `${Math.round(value * 100)}%` + : String(Number(value.toFixed(2))); +} + +function parseCreated(body: unknown): Created { + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.url !== "string" || + !Number.isSafeInteger(body.itemsTotal) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned an experiment it cannot read", + ); + } + + return { id: body.id, name: body.name, itemsTotal: body.itemsTotal as number, url: body.url }; +} + +function parseDatasetPage(body: unknown): DatasetPage | undefined { + if (!isRecord(body) || !isRecord(body.dataset) || !Array.isArray(body.items)) return undefined; + + const { version } = body.dataset; + const nextCursor = body.nextCursor ?? null; + + if (!Number.isSafeInteger(version) || (version as number) < 0) return undefined; + + if (nextCursor !== null && typeof nextCursor !== "string") return undefined; + + const items: DatasetItem[] = []; + + for (const item of body.items) { + if ( + !isRecord(item) || + typeof item.id !== "string" || + !Number.isSafeInteger(item.number) || + !isRecord(item.input) || + (item.expectedOutput !== null && typeof item.expectedOutput !== "string") || + !isRecord(item.metadata) || + !Array.isArray(item.tags) || + !item.tags.every((tag) => typeof tag === "string") || + (item.split !== null && typeof item.split !== "string") + ) { + return undefined; + } + + items.push({ + id: item.id, + number: item.number as number, + input: item.input, + expectedOutput: item.expectedOutput, + metadata: item.metadata, + tags: item.tags as string[], + split: item.split, + }); + } + + return { version: version as number, items, nextCursor }; +} + +function parseResult(body: unknown): ExperimentResult { + const scores = isRecord(body) && Array.isArray(body.scores) ? body.scores.map(parseScore) : []; + + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.status !== "string" || + typeof body.url !== "string" || + !( + body.checkStatus === null || + body.checkStatus === "passed" || + body.checkStatus === "failed" + ) || + typeof body.itemsTotal !== "number" || + typeof body.itemsDone !== "number" || + !(body.itemsFailed === undefined || typeof body.itemsFailed === "number") || + !(body.error === undefined || body.error === null || typeof body.error === "string") || + !Array.isArray(body.scores) || + scores.some((score) => score === undefined) + ) { + throw new ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read"); + } + + return { + id: body.id, + name: body.name, + status: body.status, + checkStatus: body.checkStatus, + itemsTotal: body.itemsTotal, + itemsDone: body.itemsDone, + itemsFailed: body.itemsFailed ?? 0, + error: body.error ?? null, + scores: scores as ExperimentScore[], + url: body.url, + }; +} + +function parseScore(value: unknown): ExperimentScore | undefined { + if ( + !isRecord(value) || + typeof value.name !== "string" || + !(value.kind === null || typeof value.kind === "string") || + typeof value.passCount !== "number" || + typeof value.failCount !== "number" + ) { + return undefined; + } + + const nullable = ["value", "baselineValue", "delta", "minScore"] as const; + + if (!nullable.every((key) => value[key] === null || typeof value[key] === "number")) { + return undefined; + } + + if (value.passed !== null && typeof value.passed !== "boolean") return undefined; + + return { + name: value.name, + kind: value.kind, + value: value.value as number | null, + passCount: value.passCount, + failCount: value.failCount, + baselineValue: value.baselineValue as number | null, + delta: value.delta as number | null, + minScore: value.minScore as number | null, + passed: value.passed, + }; +} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2fc3343..2c81aee 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -46,6 +46,20 @@ export { log, type LogOptions } from "./logs.ts"; export { observe, type ObserveOptions } from "./observe.ts"; +export { + type DatasetItem, + ExperimentError, + type ExperimentItem, + type ExperimentResult, + type ExperimentScore, + runExperiment, + type RunExperimentOptions, + type Scorer, + type ScorerResult, +} from "./experiments.ts"; + +export { score, ScoreError, type ScoreOptions } from "./scores.ts"; + export { type ChatMessage, type ChatPrompt, diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b17..7b4612d 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -1,16 +1,15 @@ import { type Attributes, type Context, createContextKey } from "@opentelemetry/api"; import { activeContext, - DEFAULT_BASE_URL, jsonAttr, omitUndefined, reportError, - resolveEnv, sessionIdOf, sha256, withContext, } from "@telemetry-dev/otel"; +import { isRecord, resolveApi, retryAfterMs } from "./api.ts"; import type { FieldCaptureConfig } from "./attrs.ts"; import { prepareCaptureValue } from "./capture.ts"; import { currentClient } from "./client.ts"; @@ -199,8 +198,6 @@ const MAX_RETRY_AFTER_MS = 3_600_000; /** How long a failed fetch with no copy answers calls, so that they do not each fetch again. */ const ERROR_TTL_MS = 10_000; const MAX_CACHE_ENTRIES = 1_000; -/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ -const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); @@ -228,20 +225,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P const type = options.type ?? "chat"; checkOptions(name, options, type); - const core = currentClient().core; - const env = resolveEnv(); - const apiKey = options.apiKey ?? core?.config.apiKey ?? env.TELEMETRY_DEV_API_KEY; - - // An empty value counts as unset, as in the Python SDK. - const baseUrl = ( - options.baseUrl || - core?.config.baseUrl || - env.TELEMETRY_DEV_BASE_URL || - DEFAULT_BASE_URL - ).replace(/\/+$/, ""); - + const { apiKey, baseUrl, sdkName, fetchImpl, onError } = resolveApi(options); const label = options.version === undefined ? (options.label ?? "production") : null; - const onError = core?.config.onError; let response: PromptResponse; try { @@ -269,11 +254,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P () => fetchPrompt( url, - { - authorization: `Bearer ${apiKey}`, - "x-telemetry-dev-sdk": core?.config.sdkName ?? "@telemetry-dev/sdk", - }, - options.fetch ?? core?.config.fetchImpl ?? globalThis.fetch, + { authorization: `Bearer ${apiKey}`, "x-telemetry-dev-sdk": sdkName }, + fetchImpl, ), onError, ); @@ -492,7 +474,7 @@ async function fetchPrompt( ); const delay = retryAfterMs(res); - if (delay !== undefined) retryAfter.set(failure, delay); + if (delay !== undefined) retryAfter.set(failure, Math.min(delay, MAX_RETRY_AFTER_MS)); throw failure; } @@ -617,22 +599,6 @@ function keyFingerprint(apiKey: string): string { } /** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ -function retryAfterMs(res: Response): number | undefined { - if (res.status !== 429 && res.status !== 503) return undefined; - - const value = res.headers.get("retry-after")?.trim(); - - if (!value) return undefined; - - let ms: number; - - if (/^\d+$/.test(value)) ms = Number(value) * 1000; - else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); - else return undefined; - - return ms > 0 ? Math.min(ms, MAX_RETRY_AFTER_MS) : undefined; -} - function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; @@ -724,10 +690,6 @@ function isVersion(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 1; } -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - // Prompt objects share the cached response, so an app cannot change the cache through them. function deepFreeze(value: T): T { if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts new file mode 100644 index 0000000..e600c2b --- /dev/null +++ b/packages/sdk/src/scores.ts @@ -0,0 +1,170 @@ +import { reportError } from "@telemetry-dev/otel"; + +import { type ApiOptions, isRecord, request, resolveApi } from "./api.ts"; + +export interface ScoreOptions extends ApiOptions { + /** The score name. `user_feedback` is a thumbs up (1) or down (0) from a user of your app. */ + name: string; + /** The trace that the score is on. Pass `traceId`, `sessionId`, or both. */ + traceId?: string; + /** A span of the trace, for a score on one span. */ + spanId?: string; + sessionId?: string; + /** Pass `value`, `label`, or both. */ + value?: number; + label?: string; + comment?: string; +} + +/** + * Why score failed. `code` is the code of the API error (`insufficient_scope`, `invalid_api_key`, + * …), or `missing_api_key`, `invalid_score`, `timeout`, `network_error`, or `invalid_response`. A + * response with a different status and no API error gives `http_`. + */ +export class ScoreError extends Error { + override readonly name = "ScoreError"; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown }, + ) { + super(message, options); + } +} + +export const USER_FEEDBACK = "user_feedback"; +export const MAX_SCORE_NAME_LENGTH = 64; +export const MAX_LABEL_LENGTH = 64; +export const MAX_REASON_LENGTH = 4_000; +const MAX_SESSION_ID_LENGTH = 1_024; +const TRACE_ID = /^(?!0+$)[0-9a-f]{32}$/; +const SPAN_ID = /^(?!0+$)[0-9a-f]{16}$/; +const TIMEOUT_MS = 10_000; + +/** + * Records a score on a trace, a span, or a session, for example the feedback of a user on a + * reply. Like the telemetry functions, score never rejects: it resolves with the id of the score, + * or with undefined after it reports the error through `onError`. It makes one request, with no + * retry, so that a call in a request handler ends in 10 seconds at most. The trace id of a span + * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. + */ +export async function score(options: ScoreOptions): Promise { + const api = resolveApi(options); + + try { + const name = typeof options.name === "string" ? options.name.trim() : options.name; + const label = typeof options.label === "string" ? options.label.trim() : options.label; + const problem = checkScore({ ...options, name, label }); + + if (problem) throw new ScoreError("invalid_score", problem); + + if (!api.apiKey) { + throw new ScoreError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { + body: { + traceId: options.traceId, + spanId: options.spanId, + sessionId: options.sessionId, + name, + value: options.value, + label, + comment: options.comment, + }, + timeoutMs: TIMEOUT_MS, + idempotent: false, + attempts: 1, + }); + + if (!isRecord(body) || typeof body.id !== "string") { + throw new ScoreError("invalid_response", "telemetry.dev returned a score it cannot read"); + } + + return body.id; + } catch (error) { + reportError(api.onError, error); + + return undefined; + } +} + +/** + * Why the server would reject the value and the label of a score, or undefined. A pass/fail score + * has the label `pass` with the value 1, or `fail` with 0. + */ +export function checkValue( + name: string, + value: number | undefined, + label: string | undefined, +): string | undefined { + if (value !== undefined && !(typeof value === "number" && Number.isFinite(value))) { + return "value must be a finite number"; + } + + if ( + label !== undefined && + (typeof label !== "string" || label.length === 0 || label.length > MAX_LABEL_LENGTH) + ) { + return `label must be a string of 1 to ${MAX_LABEL_LENGTH} characters`; + } + + if (value === undefined && label === undefined) return "Pass a value or a label"; + + if (name === USER_FEEDBACK && !(label === undefined && (value === 0 || value === 1))) { + return `${USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label`; + } + + if (label === "pass" && value !== undefined && value !== 1) { + return "The label pass goes with the value 1"; + } + + if (label === "fail" && value !== undefined && value !== 0) { + return "The label fail goes with the value 0"; + } + + return undefined; +} + +function checkScore(options: ScoreOptions): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = options; + + if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { + return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; + } + + if (traceId === undefined && sessionId === undefined) return "Pass traceId or sessionId"; + + if (traceId !== undefined && !(typeof traceId === "string" && TRACE_ID.test(traceId))) { + return "traceId must be 32 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && !(typeof spanId === "string" && SPAN_ID.test(spanId))) { + return "spanId must be 16 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && traceId === undefined) return "Pass the traceId of the span"; + + if ( + sessionId !== undefined && + (typeof sessionId !== "string" || + sessionId.length === 0 || + sessionId.length > MAX_SESSION_ID_LENGTH) + ) { + return `sessionId must be a string of 1 to ${MAX_SESSION_ID_LENGTH} characters`; + } + + if ( + comment !== undefined && + (typeof comment !== "string" || comment.length > MAX_REASON_LENGTH) + ) { + return `comment must be a string of at most ${MAX_REASON_LENGTH} characters`; + } + + return checkValue(name, value, label); +} diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts new file mode 100644 index 0000000..eae10e2 --- /dev/null +++ b/packages/sdk/tests/experiments.test.ts @@ -0,0 +1,723 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { + type ExperimentItem, + ExperimentError, + propagateAttributes, + runExperiment, + type RunExperimentOptions, + shutdown, + startActiveSpan, + startSpan, +} from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +interface Request { + method: string; + url: URL; + body: any; +} + +const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; +const URL_OF_EXPERIMENT = `https://telemetry.dev/o/org/p/project/evals/experiments/${EXPERIMENT_ID}`; + +function itemId(number: number): string { + return `00000000-0000-4000-8000-${String(number).padStart(12, "0")}`; +} + +function datasetItem(number: number, split: string | null = null) { + return { + id: itemId(number), + number, + input: { question: `q${number}` }, + expectedOutput: `a${number}`, + metadata: { source: "test" }, + tags: ["golden"], + split, + }; +} + +function result(overrides: Record = {}): Record { + return { + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 2, + itemsDone: 2, + itemsFailed: 0, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + ...overrides, + }; +} + +/** + * The Evals routes of telemetry.dev, in memory: pages of a dataset version with a cursor, an + * experiment that takes item posts of at most 100 and skips repeats, and its result. + */ +function fakeApi( + options: { + items?: ReturnType[]; + pageSize?: number; + results?: Record[]; + override?: (request: Request) => Response | undefined; + } = {}, +) { + const items = options.items ?? [datasetItem(1), datasetItem(2)]; + const pageSize = options.pageSize ?? 100; + const results = options.results ?? [result()]; + const requests: Request[] = []; + const stored = new Set(); + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = new URL(input); + const request: Request = { + method: init?.method ?? "GET", + url, + body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + }; + requests.push(request); + + const overridden = options.override?.(request); + + if (overridden) return overridden; + + const route = `${request.method} ${url.pathname}`; + + if (route === "GET /v1/datasets/support-golden") { + const split = url.searchParams.get("split"); + const start = Number(url.searchParams.get("cursor") ?? 0); + const matching = items.filter((item) => split === null || item.split === split); + + return Response.json({ + dataset: { + id: "dataset-1", + name: "support-golden", + version: Number(url.searchParams.get("version") ?? 3), + }, + items: matching.slice(start, start + pageSize), + nextCursor: start + pageSize < matching.length ? String(start + pageSize) : null, + }); + } + + if (route === "POST /v1/experiments") { + const count = items.filter( + (item) => request.body.split === undefined || item.split === request.body.split, + ).length; + + return Response.json( + { + id: EXPERIMENT_ID, + name: request.body.name ?? "support-golden-v3", + datasetVersion: request.body.datasetVersion, + itemsTotal: count * (request.body.runsPerItem ?? 1), + url: URL_OF_EXPERIMENT, + }, + { status: 201 }, + ); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/items`) { + if (request.body.items.length > 100) { + return Response.json( + { error: { code: "invalid_request", message: "at most 100 items" } }, + { status: 400 }, + ); + } + + let accepted = 0; + + for (const run of request.body.items) { + const key = `${run.itemId}:${run.runIndex}`; + + if (!stored.has(key)) accepted += 1; + stored.add(key); + } + + return Response.json({ accepted, duplicates: request.body.items.length - accepted }); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/complete`) { + return Response.json({ id: EXPERIMENT_ID, status: "running" }); + } + + if (route === `GET /v1/experiments/${EXPERIMENT_ID}`) { + return Response.json(results.length > 1 ? results.shift() : results[0]); + } + + return Response.json({ error: { code: "not_found", message: route } }, { status: 404 }); + }); + + const posts = () => + requests + .filter((request) => request.url.pathname.endsWith("/items")) + .map((request) => request.body.items); + + return { fetch: fetch as unknown as typeof globalThis.fetch, requests, posts }; +} + +function routes(requests: Request[]): string[] { + return requests.map((request) => `${request.method} ${request.url.pathname}`); +} + +function experiment( + api: ReturnType, + options: Partial = {}, +): ReturnType { + return runExperiment({ + apiKey: "td_live_test", + baseUrl: "https://ingest.test", + fetch: api.fetch, + dataset: "support-golden", + task: (item) => `answer to ${String(item.input.question)}`, + ...options, + }); +} + +beforeEach(() => { + for (const name of [ + "TELEMETRY_DEV_API_KEY", + "TELEMETRY_DEV_BASE_URL", + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", + ]) { + vi.stubEnv(name, undefined); + } + + vi.spyOn(console, "log").mockImplementation(() => {}); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("runs each item of each page, posts batches of 100 on one version, then completes", async () => { + const api = fakeApi({ + items: Array.from({ length: 75 }, (_, index) => datasetItem(index + 1)), + pageSize: 50, + results: [result({ itemsTotal: 150, itemsDone: 146, itemsFailed: 4 })], + }); + + const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); + + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + "GET /v1/datasets/support-golden", + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + `GET /v1/experiments/${EXPERIMENT_ID}`, + ]); + + // The first page names no version; the next one asks for the version of the first. + expect(Object.fromEntries(api.requests[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(api.requests[2]!.url.searchParams)).toEqual({ + version: "3", + cursor: "50", + }); + + expect(api.requests[1]!.body).toEqual({ + dataset: "support-golden", + datasetVersion: 3, + evaluators: ["Correctness"], + runsPerItem: 2, + }); + + const posts = api.posts(); + expect(posts.map((items) => items.length)).toEqual([100, 50]); + + const runs = posts + .flat() + .map((run: { itemId: string; runIndex: number }) => `${run.itemId}:${run.runIndex}`); + expect(new Set(runs).size).toBe(150); + expect(runs).toContain(`${itemId(75)}:1`); + + expect(posts[0]![0]).toEqual({ + itemId: itemId(1), + runIndex: 0, + output: "answer to q1", + latencyMs: expect.any(Number), + }); + + expect(outcome).toEqual({ + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 150, + itemsDone: 146, + itemsFailed: 4, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + }); +}); + +test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { + const { spans } = setup({ environment: "experiment", sessionMode: "process" }); + const api = fakeApi(); + + // An active span, an ambient session, and the process session would each put the runs in one + // trace. + await propagateAttributes({ sessionId: "ambient-session", userId: "u-1" }, () => + startActiveSpan("eval script", () => + experiment(api, { + runsPerItem: 2, + task: (item) => { + startSpan("chat gpt-4o", { type: "generation" }).end(); + + return { reply: `r${item.number}` }; + }, + }), + ), + ); + + const posted = api.posts().flat(); + const finished = spans.getFinishedSpans(); + const chats = finished.filter((span) => span.name === "chat gpt-4o"); + const roots = finished.filter((span) => span.name === "experiment support-golden-v3"); + + expect(chats).toHaveLength(4); + expect(roots).toHaveLength(4); + expect(new Set(roots.map((span) => span.spanContext().traceId)).size).toBe(4); + expect(roots.map((span) => span.parentSpanContext?.isRemote)).toEqual([true, true, true, true]); + + for (const run of posted) { + expect(run.traceId).toMatch(/^[0-9a-f]{32}$/); + const root = roots.find((span) => span.spanContext().traceId === run.traceId)!; + const chat = chats.find((span) => span.spanContext().traceId === run.traceId)!; + + expect(chat.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + + for (const span of [root, chat]) { + expect(span.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + "gen_ai.conversation.id": `${EXPERIMENT_ID}:${run.itemId}:${run.runIndex}`, + "user.id": "u-1", + }); + } + } + + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ]); +}); + +test("runs at most `concurrency` tasks at a time", async () => { + const api = fakeApi({ items: Array.from({ length: 12 }, (_, index) => datasetItem(index + 1)) }); + let running = 0; + let most = 0; + + await experiment(api, { + concurrency: 3, + task: async () => { + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, 5)); + running -= 1; + + return "ok"; + }, + }); + + expect(most).toBe(3); + expect(api.posts().flat()).toHaveLength(12); +}); + +test("a task that throws fails only its item", async () => { + const api = fakeApi(); + + await experiment(api, { + task: (item) => { + if (item.number === 1) throw new RangeError("no reply"); + + return "fine"; + }, + }); + + const [failed, passed] = api.posts().flat(); + + expect(failed).toMatchObject({ itemId: itemId(1), output: null, error: "RangeError: no reply" }); + expect(failed).not.toHaveProperty("scores"); + expect(passed).toMatchObject({ itemId: itemId(2), output: "fine" }); + expect(passed).not.toHaveProperty("error"); +}); + +test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ items: [datasetItem(1)] }); + + await experiment(api, { + scorers: { + exact: (output, item) => output === `answer to ${String(item.input.question)}`, + polite: () => false, + length: (output) => String(output).length, + tone: () => " friendly ", + judged: () => ({ value: 0.25, reason: "too short" }), + none: () => null, + thrown: () => { + throw new Error("judge down"); + }, + mismatched: () => ({ value: 0.5, label: "pass" }), + }, + }); + + expect(api.posts()[0]![0].scores).toEqual([ + { name: "exact", value: 1, label: "pass" }, + { name: "polite", value: 0, label: "fail" }, + { name: "length", value: 12 }, + { name: "tone", label: "friendly" }, + { name: "judged", value: 0.25, reason: "too short" }, + ]); + + expect(errors.map((error) => [(error as ExperimentError).code, error.message])).toEqual([ + ["scorer_failed", "The scorer thrown failed on item 1: Error: judge down"], + [ + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not valid: The label pass goes with the value 1", + ], + ]); +}); + +test("a failed post of items stops new items, never completes, and rejects with the API error", async () => { + const api = fakeApi({ + items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), + override: (request) => + request.url.pathname.endsWith("/items") + ? Response.json( + { error: { code: "experiment_ended", message: "support-golden-v3 has ended." } }, + { status: 409 }, + ) + : undefined, + }); + let started = 0; + + const error = await experiment(api, { + concurrency: 1, + task: () => { + started += 1; + + return "ok"; + }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ExperimentError); + expect(error).toMatchObject({ code: "experiment_ended", status: 409 }); + // The one worker waits for the post of the first 100, which fails, so no item starts after it. + expect(started).toBe(100); + expect(routes(api.requests)).not.toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { + let failures = 1; + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") && failures-- > 0 + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await experiment(api); + + expect(api.posts()).toHaveLength(2); + expect(api.posts()[1]).toEqual(api.posts()[0]); +}); + +test("a create that fails with 500 does not go again, so it cannot start two experiments", async () => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "http_500", status: 500 }); + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(1); +}); + +test("a create that gets a 429 goes again after its Retry-After", async () => { + let limited = 1; + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" && limited-- > 0 + ? Response.json( + { error: { code: "rate_limited", message: "Too many requests for this project" } }, + { status: 429, headers: { "retry-after": "2" } }, + ) + : undefined, + }); + const started = Date.now(); + + await experiment(api); + + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(2); + // Without the header, the first retry waits 1 second. + expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); +}); + +test("waits until the server evaluators end the experiment", async () => { + const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] }); + + const outcome = await experiment(api); + + expect(outcome.status).toBe("completed"); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/experiments/")), + ).toHaveLength(2); +}); + +test.each([ + [ + "the API error", + { error: { code: "dataset_not_found", message: "No dataset named support-golden" } }, + 404, + "dataset_not_found", + ], + [ + "a page that cannot be read", + { dataset: { version: 1 }, items: [{ id: 1 }] }, + 200, + "invalid_response", + ], + [ + "an empty dataset", + { dataset: { id: "d", name: "support-golden", version: 0 }, items: [], nextCursor: null }, + 200, + "dataset_empty", + ], +])("rejects with %s before it creates an experiment", async (_, body, status, code) => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json(body, { status }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code }); + expect(routes(api.requests)).toEqual(["GET /v1/datasets/support-golden"]); +}); + +test("a cursor that repeats is an invalid response, not an endless fetch", async () => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json({ + dataset: { id: "d", name: "support-golden", version: 3 }, + items: [datasetItem(1)], + nextCursor: "same", + }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/datasets/")), + ).toHaveLength(2); +}); + +const failedCheck = result({ + checkStatus: "failed", + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 17, + failCount: 3, + baselineValue: 0.9, + delta: -5, + minScore: 0.9, + passed: false, + }, + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ], +}); + +test("in CI mode a failed check rejects with the result", async () => { + vi.stubEnv("CI", "true"); + const api = fakeApi({ results: [failedCheck] }); + + const error = (await experiment(api, { minScores: { correctness: 0.9 } }).catch( + (caught: unknown) => caught, + )) as ExperimentError; + + expect(error).toBeInstanceOf(ExperimentError); + expect(error.code).toBe("check_failed"); + expect(error.message).toBe( + `The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. ${URL_OF_EXPERIMENT}`, + ); + expect(error.result?.scores[1]?.kind).toBeNull(); +}); + +test.each([ + ["CI is off", {}, undefined], + ["ci: false wins over CI", { ci: false }, "true"], +])("a failed check resolves when %s", async (_, options, ci) => { + if (ci) vi.stubEnv("CI", ci); + const api = fakeApi({ results: [failedCheck] }); + + await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); +}); + +test("in CI mode an experiment that fails rejects with its error", async () => { + vi.stubEnv("CI", "1"); + const api = fakeApi({ + results: [result({ status: "failed", error: "0 of 2 items ran. Each one failed." })], + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "experiment_failed", + message: `The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. ${URL_OF_EXPERIMENT}`, + }); +}); + +test("in GitHub Actions the experiment records the pull request, its head commit, and the run", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + const results = join(dir, "results.jsonl"); + writeFileSync( + event, + JSON.stringify({ + pull_request: { + number: 42, + title: "Shorter replies", + html_url: "https://github.com/acme/support/pull/42", + head: { sha: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" }, + }, + }), + ); + writeFileSync(results, ""); + + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "ffffffffffffffffffffffffffffffffffffffff"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + vi.stubEnv("GITHUB_RUN_ID", "991"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); + const api = fakeApi(); + + await experiment(api, { dataset: undefined, minScores: { correctness: 0.5, tone: 2 } }); + + expect(api.requests[1]!.body).toMatchObject({ + dataset: "support-golden", + minScores: { correctness: 0.9, length: 12.5, tone: 2 }, + ci: { + provider: "github_actions", + repository: "acme/support", + commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + prNumber: 42, + prTitle: "Shorter replies", + prUrl: "https://github.com/acme/support/pull/42", + runUrl: "https://github.com/acme/support/actions/runs/991", + }, + }); + expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); +}); + +test.each([ + [ + "no dataset", + {}, + { dataset: undefined }, + "Pass dataset, or set the dataset input of the experiment action", + ], + [ + "a dataset that the action does not name", + { TELEMETRY_DEV_EXPERIMENT_DATASET: "support-v2" }, + {}, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ], + [ + "min-scores that do not parse", + { TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" }, + {}, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" is not a name=number pair', + ], + ["runsPerItem 6", {}, { runsPerItem: 6 }, "runsPerItem must be an integer from 1 to 5"], + ["concurrency 0", {}, { concurrency: 0 }, "concurrency must be an integer from 1 to 100"], + [ + "a scorer named user_feedback", + {}, + { scorers: { user_feedback: () => 1 } }, + "user_feedback is the score of user feedback. Name the scorer another way.", + ], + [ + "a scorer with the name of an evaluator", + {}, + { evaluators: ["correctness"], scorers: { Correctness: () => true } }, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ], +])("rejects %s before any request", async (_, env, options, message) => { + for (const [name, value] of Object.entries(env)) vi.stubEnv(name, value); + const api = fakeApi(); + + await expect(experiment(api, options as Partial)).rejects.toThrow(message); + expect(api.requests).toEqual([]); +}); + +test("without an API key it rejects with missing_api_key and sends nothing", async () => { + const api = fakeApi(); + + await expect(experiment(api, { apiKey: undefined })).rejects.toMatchObject({ + code: "missing_api_key", + }); + expect(api.requests).toEqual([]); +}); + +test("the task gets each run of an item of the split, with its run index", async () => { + const seen: ExperimentItem[] = []; + const api = fakeApi({ items: [datasetItem(7, "test")] }); + + await experiment(api, { + split: "test", + runsPerItem: 2, + concurrency: 1, + task: (item) => { + seen.push(item); + + return "ok"; + }, + }); + + expect(seen).toEqual([ + { ...datasetItem(7, "test"), runIndex: 0 }, + { ...datasetItem(7, "test"), runIndex: 1 }, + ]); + expect(api.requests[0]!.url.searchParams.get("split")).toBe("test"); + expect(api.requests[1]!.body.split).toBe("test"); +}); diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts new file mode 100644 index 0000000..d740b40 --- /dev/null +++ b/packages/sdk/tests/scores.test.ts @@ -0,0 +1,220 @@ +import { createServer, type RequestListener } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { score, ScoreError, type ScoreOptions, shutdown } from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +const SPAN_ID = "00f067aa0ba902b7"; + +function fakeFetch(reply: () => Response) { + const calls: { url: URL; init: RequestInit | undefined }[] = []; + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + calls.push({ url: new URL(input), init }); + + return reply(); + }); + + return { fetch: fetch as unknown as typeof globalThis.fetch, calls }; +} + +async function serve(handler: RequestListener): Promise<{ url: string; close: () => void }> { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + close: () => { + server.closeAllConnections(); + server.close(); + }, + }; +} + +function errorsOf() { + const errors: ScoreError[] = []; + setup({ onError: (error) => errors.push(error as ScoreError) }); + + return errors; +} + +beforeEach(() => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + vi.stubEnv("TELEMETRY_DEV_BASE_URL", undefined); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("posts the score with the key and base URL of init, and resolves with its id", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-1" }, { status: 202 })); + setup({ apiKey: "td_live_key", baseUrl: "https://ingest.test/", fetch }); + + const id = await score({ + name: " user_feedback ", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); + + expect(id).toBe("score-1"); + expect(calls).toHaveLength(1); + expect(calls[0]!.url.href).toBe("https://ingest.test/v1/scores"); + expect(calls[0]!.init).toMatchObject({ + method: "POST", + headers: { + authorization: "Bearer td_live_key", + "x-telemetry-dev-sdk": "@telemetry-dev/sdk", + "content-type": "application/json", + }, + }); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "user_feedback", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); +}); + +test("a session score needs no trace, and a label goes trimmed", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "resolved", sessionId: "chat-9", label: " yes " }), + ).resolves.toBe("score-2"); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "resolved", + sessionId: "chat-9", + label: "yes", + }); +}); + +test.each<[string, Partial, string]>([ + ["no trace and no session", { traceId: undefined }, "Pass traceId or sessionId"], + [ + "the all-zero trace id of a span that did not record", + { traceId: "0".repeat(32) }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "an uppercase trace id", + { traceId: TRACE_ID.toUpperCase() }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "a span without its trace", + { traceId: undefined, sessionId: "s", spanId: SPAN_ID }, + "Pass the traceId of the span", + ], + ["no value and no label", { value: undefined }, "Pass a value or a label"], + ["a value that is not finite", { value: Number.NaN }, "value must be a finite number"], + [ + "a label for user feedback", + { name: "user_feedback", value: 1, label: "up" }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + [ + "user feedback of 0.5", + { name: "user_feedback", value: 0.5 }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + ["a pass of 0", { value: 0, label: "pass" }, "The label pass goes with the value 1"], + ["a fail of 1", { value: 1, label: "fail" }, "The label fail goes with the value 0"], + ["a blank name", { name: " " }, "name must be a string of 1 to 64 characters"], + [ + "a comment of 4,001 characters", + { comment: "x".repeat(4_001) }, + "comment must be a string of at most 4000 characters", + ], +])("reports invalid_score for %s and sends nothing", async (_, options, message) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await expect( + score({ + apiKey: "td_live_key", + fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + ...options, + }), + ).resolves.toBeUndefined(); + expect(calls).toEqual([]); + expect(errors.map((error) => [error.code, error.message])).toEqual([["invalid_score", message]]); +}); + +test.each([ + [ + "the API error", + () => + Response.json( + { error: { code: "insufficient_scope", message: "This API key cannot send scores" } }, + { status: 403 }, + ), + ["insufficient_scope", 403], + ], + ["a status with no API error", () => new Response("down", { status: 503 }), ["http_503", 503]], + ["a body with no id", () => Response.json({}, { status: 202 }), ["invalid_response", undefined]], +])("reports %s, tries once, and resolves undefined", async (_, reply, [code, status]) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(reply); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }), + ).resolves.toBeUndefined(); + expect(calls).toHaveLength(1); + expect(errors.map((error) => [error.code, error.status])).toEqual([[code, status]]); +}); + +test("without an API key it reports missing_api_key", async () => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await score({ apiKey: "", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }); + + expect(calls).toEqual([]); + expect(errors.map((error) => error.code)).toEqual(["missing_api_key"]); +}); + +test("does not follow a redirect, so the key stays with the API host", async () => { + const errors = errorsOf(); + let reached = 0; + const other = await serve((_, res) => { + reached += 1; + res.end(JSON.stringify({ id: "stolen" })); + }); + const api = await serve((_, res) => { + res.writeHead(307, { location: `${other.url}/v1/scores` }); + res.end(); + }); + + try { + await expect( + // The real fetch: setup() gives the client a stub. + score({ + apiKey: "td_live_key", + baseUrl: api.url, + fetch: globalThis.fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + }), + ).resolves.toBeUndefined(); + } finally { + api.close(); + other.close(); + } + + expect(reached).toBe(0); + expect(errors.map((error) => [error.code, error.status])).toEqual([["http_307", 307]]); +}); From e44cf6022fc0e1938f3a11efaf97b49441ae574d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:41 +0000 Subject: [PATCH 02/32] fix(sdk): keep each min score and print scores as the python sdk does minScoresFromEnv now makes its object from pairs. Before, the name __proto__ set the prototype of the object, and its mark went away with no error. A number score now prints as Math.round(value * 100) / 100, a rule that the Python SDK can copy. The tests now also cover the experiment attributes on log records and the request order that the code holds. --- packages/sdk/src/ci.ts | 7 +++-- packages/sdk/src/experiments.ts | 2 +- packages/sdk/tests/experiments.test.ts | 39 ++++++++++++++++++-------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts index 45eb20c..3d7603e 100644 --- a/packages/sdk/src/ci.ts +++ b/packages/sdk/src/ci.ts @@ -78,7 +78,8 @@ export function githubRun(env: Env): CiRun | undefined { * its `min-scores` input: `name=value` pairs, separated by commas or new lines. */ export function minScoresFromEnv(env: Env): Record { - const scores: Record = {}; + // Pairs, so that the name __proto__ becomes a key and not the prototype of the object. + const scores: [string, number][] = []; for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { if (pair.trim() === "") continue; @@ -90,10 +91,10 @@ export function minScoresFromEnv(env: Env): Record { ); } - scores[name] = Number(value); + scores.push([name, Number(value)]); } - return scores; + return Object.fromEntries(scores); } /** diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts index bb90d0d..f3d6d4b 100644 --- a/packages/sdk/src/experiments.ts +++ b/packages/sdk/src/experiments.ts @@ -826,7 +826,7 @@ function formatScore(kind: string | null, value: number | null): string { return kind !== null && RATE_KINDS.has(kind) ? `${Math.round(value * 100)}%` - : String(Number(value.toFixed(2))); + : String(Math.round(value * 100) / 100); } function parseCreated(body: unknown): Created { diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts index eae10e2..daa3214 100644 --- a/packages/sdk/tests/experiments.test.ts +++ b/packages/sdk/tests/experiments.test.ts @@ -7,6 +7,7 @@ import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { type ExperimentItem, ExperimentError, + log, propagateAttributes, runExperiment, type RunExperimentOptions, @@ -216,22 +217,23 @@ test("runs each item of each page, posts batches of 100 on one version, then com const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); - expect(routes(api.requests)).toEqual([ - "GET /v1/datasets/support-golden", - "POST /v1/experiments", + const sent = routes(api.requests); + expect(sent.slice(0, 2)).toEqual(["GET /v1/datasets/support-golden", "POST /v1/experiments"]); + // The second page and the first post of items run at the same time, in either order. + expect(sent.slice(2, -2).sort()).toEqual([ "GET /v1/datasets/support-golden", `POST /v1/experiments/${EXPERIMENT_ID}/items`, `POST /v1/experiments/${EXPERIMENT_ID}/items`, + ]); + expect(sent.slice(-2)).toEqual([ `POST /v1/experiments/${EXPERIMENT_ID}/complete`, `GET /v1/experiments/${EXPERIMENT_ID}`, ]); // The first page names no version; the next one asks for the version of the first. - expect(Object.fromEntries(api.requests[0]!.url.searchParams)).toEqual({}); - expect(Object.fromEntries(api.requests[2]!.url.searchParams)).toEqual({ - version: "3", - cursor: "50", - }); + const pages = api.requests.filter((request) => request.url.pathname.startsWith("/v1/datasets/")); + expect(Object.fromEntries(pages[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(pages[1]!.url.searchParams)).toEqual({ version: "3", cursor: "50" }); expect(api.requests[1]!.body).toEqual({ dataset: "support-golden", @@ -271,7 +273,7 @@ test("runs each item of each page, posts batches of 100 on one version, then com }); test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { - const { spans } = setup({ environment: "experiment", sessionMode: "process" }); + const { spans, logs } = setup({ environment: "experiment", sessionMode: "process" }); const api = fakeApi(); // An active span, an ambient session, and the process session would each put the runs in one @@ -282,6 +284,7 @@ test("each item run is its own trace, and each span of the task carries the expe runsPerItem: 2, task: (item) => { startSpan("chat gpt-4o", { type: "generation" }).end(); + log("replied", { attributes: { number: item.number } }); return { reply: `r${item.number}` }; }, @@ -316,6 +319,20 @@ test("each item run is its own trace, and each span of the task carries the expe } } + const records = logs.getFinishedLogRecords(); + expect(records).toHaveLength(4); + + for (const record of records) { + const run = posted.find( + (candidate: { traceId: string }) => candidate.traceId === record.spanContext?.traceId, + ); + + expect(record.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + }); + } + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ '{"reply":"r1"}', '{"reply":"r1"}', @@ -627,7 +644,7 @@ test("in GitHub Actions the experiment records the pull request, its head commit vi.stubEnv("GITHUB_EVENT_PATH", event); vi.stubEnv("GITHUB_RUN_ID", "991"); vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); - vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5,__proto__=1"); vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); const api = fakeApi(); @@ -635,7 +652,7 @@ test("in GitHub Actions the experiment records the pull request, its head commit expect(api.requests[1]!.body).toMatchObject({ dataset: "support-golden", - minScores: { correctness: 0.9, length: 12.5, tone: 2 }, + minScores: JSON.parse('{"correctness":0.9,"tone":2,"length":12.5,"__proto__":1}'), ci: { provider: "github_actions", repository: "acme/support", From b8dc041c3907beb0bf5647b928c3c8a180e57525 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:50 +0000 Subject: [PATCH 03/32] feat(sdk-python): add run_experiment and score run_experiment and run_experiment_async are the Python twins of runExperiment, with the same options, defaults, errors, and requests. The sync call runs the items in threads, each in a copy of the context of the caller. The async call runs them as asyncio tasks. The two read the dataset page by page and post the outputs in batches of at most 100. score and score_async send a score to /v1/scores. They never raise and send their errors to on_error. The new _api module holds the key and base URL, one JSON request with its retry rules, the redirect guard, and the Retry-After parser. The prompt code now uses it too. The checks count lengths in UTF-16 units, as the API does. --- sdks/python/src/telemetry_dev/__init__.py | 26 + sdks/python/src/telemetry_dev/_api.py | 190 +++ sdks/python/src/telemetry_dev/_ci.py | 134 ++ sdks/python/src/telemetry_dev/_experiments.py | 1100 +++++++++++++++++ sdks/python/src/telemetry_dev/_prompts.py | 80 +- sdks/python/src/telemetry_dev/_scores.py | 181 +++ sdks/python/src/telemetry_dev/_semconv.py | 2 + sdks/python/src/telemetry_dev/_serialize.py | 5 + sdks/python/tests/fake_api.py | 193 +++ sdks/python/tests/test_experiments.py | 746 +++++++++++ sdks/python/tests/test_scores.py | 204 +++ 11 files changed, 2795 insertions(+), 66 deletions(-) create mode 100644 sdks/python/src/telemetry_dev/_api.py create mode 100644 sdks/python/src/telemetry_dev/_ci.py create mode 100644 sdks/python/src/telemetry_dev/_experiments.py create mode 100644 sdks/python/src/telemetry_dev/_scores.py create mode 100644 sdks/python/tests/fake_api.py create mode 100644 sdks/python/tests/test_experiments.py create mode 100644 sdks/python/tests/test_scores.py diff --git a/sdks/python/src/telemetry_dev/__init__.py b/sdks/python/src/telemetry_dev/__init__.py index 9ed7aab..bbcb230 100644 --- a/sdks/python/src/telemetry_dev/__init__.py +++ b/sdks/python/src/telemetry_dev/__init__.py @@ -17,6 +17,18 @@ from ._config import SDK_VERSION as __version__ from ._config import LogLevelOption from ._context import get_traceparent, propagate_attributes +from ._experiments import ( + AsyncScorer, + DatasetItem, + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + ScorerResult, + run_experiment, + run_experiment_async, +) from ._logs import log from ._observe import observe from ._prompts import ( @@ -29,6 +41,7 @@ get_prompt_async, with_prompt, ) +from ._scores import ScoreError, score, score_async from ._semconv import LogLevel, SpanType from ._serialize import AttributeValue, Mask, MaskContext from ._spans import ( @@ -42,10 +55,16 @@ __all__ = [ "NOT_GIVEN", + "AsyncScorer", "AttributeValue", "CaptureBudget", "ChatPrompt", "Client", + "DatasetItem", + "ExperimentError", + "ExperimentItem", + "ExperimentResult", + "ExperimentScore", "LogLevel", "LogLevelOption", "Mask", @@ -53,6 +72,9 @@ "Prompt", "PromptError", "PromptModel", + "ScoreError", + "Scorer", + "ScorerResult", "SpanHandle", "SpanType", "TelemetrySpanProcessor", @@ -68,6 +90,10 @@ "log", "observe", "propagate_attributes", + "run_experiment", + "run_experiment_async", + "score", + "score_async", "shutdown", "start_span", "update_current_span", diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py new file mode 100644 index 0000000..4f8e042 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_api.py @@ -0,0 +1,190 @@ +"""Calls to the telemetry.dev API: the key and base URL, one JSON request, and its errors.""" + +from __future__ import annotations + +import http.client +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from email.message import Message +from email.utils import parsedate_to_datetime +from typing import IO, Any, Literal, TypeVar, cast + +from ._client import get_client +from ._config import resolve_config + +# A longer Retry-After waits only this long: the API counts requests by the minute. +_MAX_RETRY_AFTER_S = 60.0 +# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. +_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) + + +# An error class with the shape of PromptError: a code, a message, and the status. +E = TypeVar("E", bound=Exception) + + +@dataclass(frozen=True) +class Api: + api_key: str | None + base_url: str + on_error: Callable[[BaseException], None] | None + + +def resolve_api(api_key: str | None, base_url: str | None) -> Api: + """The key and base URL of a call: its arguments, then the enabled client, then the env.""" + client = get_client() + enabled = client if client is not None and client.enabled else None + config = enabled.config if enabled is not None else resolve_config() + return Api( + api_key=api_key if api_key is not None else config.api_key, + base_url=(base_url or config.base_url).rstrip("/"), + on_error=enabled.on_error if enabled is not None else None, + ) + + +@dataclass(frozen=True) +class _Failure: + error: Exception + status: int | None + retry_after: float | None + + +def request( + api: Api, + error_type: Callable[[str, str, int | None], E], + method: Literal["GET", "POST"], + path: str, + *, + timeout_s: float, + idempotent: bool, + body: object = None, + query: Mapping[str, str | None] | None = None, + attempts: int = 3, +) -> Any: + """Send one JSON request and return the JSON of the response. + + A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or + ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a + network error, or a 5xx tries again. Other requests try again only after a 429 or a 503: the + server did not take the request then. A retry waits for the Retry-After of the response, or + for 1 and then 2 seconds. The timeout applies to each socket operation of urllib. + """ + url = f"{api.base_url}{path}" + params = {key: value for key, value in (query or {}).items() if value is not None} + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + headers = {"Authorization": f"Bearer {api.api_key}", "X-Telemetry-Dev-SDK": "telemetry-dev"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False, allow_nan=False).encode() + headers["Content-Type"] = "application/json" + attempt = 1 + while True: + result = _send(error_type, method, url, path, data, headers, timeout_s) + if not isinstance(result, _Failure): + return result + if attempt >= attempts or not _retryable(result.status, idempotent): + raise result.error + time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S)) + attempt += 1 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # A redirect to another host must not get the API key, so a 3xx fails like any status that is + # not 2xx. + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: Message, + newurl: str, + ) -> urllib.request.Request | None: + return None + + +opener = urllib.request.build_opener(NoRedirect) + + +def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E: + """The error of a response that is not 2xx: its API code and message, or ``http_``.""" + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} + code = fields.get("code") + message = fields.get("message") + return error_type( + code if isinstance(code, str) else f"http_{status}", + message if isinstance(message, str) else f"telemetry.dev returned {status}", + status, + ) + + +def retry_after(status: int, value: str | None) -> float | None: + """The seconds that the Retry-After of a 429 or a 503 asks for: seconds or an HTTP date.""" + value = (value or "").strip() + if status not in (429, 503) or not value: + return None + if value.isascii() and value.isdigit(): + seconds = float(value) + elif _HTTP_DATE.fullmatch(value): + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + seconds = (date - datetime.now(timezone.utc)).total_seconds() + else: + return None + return seconds if seconds > 0 else None + + +def _send( + error_type: Callable[[str, str, int | None], E], + method: str, + url: str, + path: str, + data: bytes | None, + headers: dict[str, str], + timeout_s: float, +) -> Any: + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(request, timeout=timeout_s) as response: + body: bytes = response.read() + except urllib.error.HTTPError as exc: + with exc: + return _Failure( + api_error(error_type, exc.code, exc.read()), + exc.code, + retry_after(exc.code, exc.headers.get("Retry-After")), + ) + except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: + timed_out = isinstance(exc, TimeoutError) or isinstance( + getattr(exc, "reason", None), TimeoutError + ) + error = ( + error_type("timeout", f"{method} {path} took more than {timeout_s:g} s", None) + if timed_out + else error_type("network_error", "Could not reach telemetry.dev", None) + ) + error.__cause__ = exc + return _Failure(error, None, None) + try: + return json.loads(body) + except ValueError: + return None + + +def _retryable(status: int | None, idempotent: bool) -> bool: + if status in (429, 503): + return True + return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py new file mode 100644 index 0000000..4b6cd00 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -0,0 +1,134 @@ +"""The GitHub Actions run of an experiment, and the variables of the experiment action.""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Mapping +from typing import Any, cast + +from ._config import report_error +from ._serialize import utf16_length + +_REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +_COMMIT = re.compile(r"[0-9a-fA-F]{7,64}") +_HTTPS_URL = re.compile(r"https://\S+") +# A plain decimal, so that the TypeScript and Python SDKs read the same marks. +_DECIMAL = re.compile(r"-?\d+(\.\d+)?", re.ASCII) +_RUN_ID = re.compile(r"\d+", re.ASCII) +_MAX_REPOSITORY_LENGTH = 200 +_MAX_URL_LENGTH = 2048 +_MAX_PR_TITLE_LENGTH = 256 + + +def in_ci(env: Mapping[str, str]) -> bool: + """CI mode is on when ``CI`` is ``true`` or ``1``, as CI services set it.""" + return env.get("CI") in ("true", "1") + + +def github_run(env: Mapping[str, str]) -> dict[str, Any] | None: + """The run that GitHub Actions describes in its variables, or None outside GitHub Actions. + + On a pull request the commit is the head of the pull request: ``GITHUB_SHA`` is then a merge + commit that nobody pushed. + """ + repository = env.get("GITHUB_REPOSITORY") or "" + if ( + env.get("GITHUB_ACTIONS") != "true" + or len(repository) > _MAX_REPOSITORY_LENGTH + or not _REPOSITORY.fullmatch(repository) + ): + return None + pull = _pull_request(env.get("GITHUB_EVENT_PATH")) + commit = pull.get("commit") or env.get("GITHUB_SHA") + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + return None + server = (env.get("GITHUB_SERVER_URL") or "https://github.com").rstrip("/") + run_id = env.get("GITHUB_RUN_ID") or "" + run: dict[str, Any] = { + "provider": "github_actions", + "repository": repository, + "commit": commit, + "prNumber": pull.get("number"), + "prTitle": pull.get("title"), + "prUrl": pull.get("url"), + "runUrl": _https_url(f"{server}/{repository}/actions/runs/{run_id}") + if _RUN_ID.fullmatch(run_id) + else None, + } + return {key: value for key, value in run.items() if value is not None} + + +def min_scores_from_env(env: Mapping[str, str]) -> dict[str, float]: + """The pass marks in ``TELEMETRY_DEV_EXPERIMENT_MIN_SCORES``, which the experiment action + sets from its ``min-scores`` input: ``name=value`` pairs, separated by commas or new lines.""" + scores: dict[str, float] = {} + for pair in re.split(r"[,\n]", env.get("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES") or ""): + if not pair.strip(): + continue + parts = [part.strip() for part in pair.split("=")] + if len(parts) != 2 or not parts[0] or not _DECIMAL.fullmatch(parts[1]): + raise TypeError( + f'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "{pair.strip()}" is not a name=number pair' + ) + scores[parts[0]] = float(parts[1]) + return scores + + +def write_result( + env: Mapping[str, str], + result: object, + on_error: Callable[[BaseException], None] | None, +) -> None: + """Add the result to the file in ``TELEMETRY_DEV_EXPERIMENT_RESULTS`` as one line of JSON. + The experiment action reads the file to write the job summary and the pull request comment.""" + path = env.get("TELEMETRY_DEV_EXPERIMENT_RESULTS") + if not path: + return + line = json.dumps(result, ensure_ascii=False, separators=(",", ":")) + "\n" + try: + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + with os.fdopen(fd, "a", encoding="utf-8") as file: + file.write(line) + except OSError as exc: + report_error(on_error, "writing TELEMETRY_DEV_EXPERIMENT_RESULTS failed", exc) + + +def _pull_request(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + try: + with open(event_path, encoding="utf-8") as file: + event = json.load(file) + except (OSError, ValueError): + return {} + pull = cast("dict[str, Any]", event).get("pull_request") if isinstance(event, dict) else None + if not isinstance(pull, dict): + return {} + fields = cast("dict[str, Any]", pull) + number = fields.get("number") + title = fields.get("title") + url = fields.get("html_url") + head = fields.get("head") + sha = cast("dict[str, Any]", head).get("sha") if isinstance(head, dict) else None + return { + "number": number + if isinstance(number, int) and not isinstance(number, bool) and number > 0 + else None, + "title": _cut(title, _MAX_PR_TITLE_LENGTH) if isinstance(title, str) else None, + "url": _https_url(url) if isinstance(url, str) else None, + "commit": sha if isinstance(sha, str) else None, + } + + +def _https_url(url: str) -> str | None: + return url if utf16_length(url) <= _MAX_URL_LENGTH and _HTTPS_URL.fullmatch(url) else None + + +def _cut(text: str, max_units: int) -> str: + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts.""" + encoded = text.encode("utf-16-le", "surrogatepass") + if len(encoded) <= max_units * 2: + return text + return encoded[: max_units * 2].decode("utf-16-le", "ignore") diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py new file mode 100644 index 0000000..b4e031d --- /dev/null +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -0,0 +1,1100 @@ +"""Experiments of your code: run a task on each item of a dataset and record the result.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import math +import os +import threading +import time +import urllib.parse +from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Literal, TypeAlias, TypeGuard, cast + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.context import Context + +from ._api import Api, request, resolve_api +from ._ci import github_run, in_ci, min_scores_from_env, write_result +from ._client import flush, get_client +from ._config import logger, report_error +from ._context import context_with_propagated_attributes, propagated_attributes +from ._scores import ( + MAX_REASON_LENGTH, + MAX_SCORE_NAME_LENGTH, + USER_FEEDBACK, + check_value, + is_finite, +) +from ._semconv import ATTR_DATASET_ITEM_ID, ATTR_EXPERIMENT_ID, ATTR_SESSION_ID +from ._serialize import json_dumps, truncate, utf16_length +from ._spans import SpanHandle, start_span + +# The experiment environment, which pages, metrics, and alerts leave out by default. +_EXPERIMENT_ENVIRONMENT = "experiment" +_MAX_RUNS_PER_ITEM = 5 +_MAX_CONCURRENCY = 100 +# Server evaluators of one experiment, and scorers: an item takes at most 20 scores. +_MAX_EVALUATORS = 20 +_MAX_MIN_SCORES = 20 +_MAX_ERROR_LENGTH = 4000 +_MAX_OUTPUT_LENGTH = 100_000 +_MAX_DATASET_ITEMS = 10_000 +# Items in one POST /v1/experiments/{id}/items. +_MAX_BATCH_ITEMS = 100 +# Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. +_MAX_BATCH_BYTES = 4 * 1024 * 1024 +_REQUEST_TIMEOUT_S = 30.0 +# How long run_experiment waits for the server evaluators after the last item. +_MAX_WAIT_S = 30 * 60.0 +_MAX_POLL_INTERVAL_S = 5.0 +_FINISHED = frozenset({"completed", "failed", "stopped"}) +_RATE_KINDS = frozenset({"pass_fail", "up_down"}) + + +@dataclass(frozen=True) +class DatasetItem: + """An item of a dataset version, as ``GET /v1/datasets/{name}`` returns it. ``id`` is the key + of the item: it stays the same across dataset versions.""" + + id: str + number: int + input: Mapping[str, Any] + expected_output: str | None + metadata: Mapping[str, Any] + tags: tuple[str, ...] + split: str | None + + +@dataclass(frozen=True) +class ExperimentItem(DatasetItem): + """One run of a dataset item, ``run_index`` 0 to ``runs_per_item`` - 1.""" + + run_index: int + + +ScorerResult: TypeAlias = "bool | int | float | str | Mapping[str, Any] | None" +"""What a scorer returns. True and False are a pass and a fail, a number is the value, and a +string is the label. A mapping can have ``value``, ``label``, and ``reason``. None gives no +score.""" + +Scorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult]" +"""Scores the output of an item on your machine. The score has the name of the scorer.""" + +AsyncScorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult | Awaitable[ScorerResult]]" +"""A scorer of ``run_experiment_async``, which can be async.""" + + +@dataclass(frozen=True) +class ExperimentScore: + """One score name of an experiment. ``value`` is the pass rate (0 to 1) of a pass/fail or + up/down score, or the mean of a number score. ``kind`` is None before the score has values. + ``baseline_value`` is the value in the experiment that this one compares with, by default the + dataset baseline. ``delta`` is whole points of a pass rate, or the change of a mean to one + decimal. ``passed`` is None when the score has no pass mark.""" + + name: str + kind: str | None + value: float | None + pass_count: int + fail_count: int + baseline_value: float | None + delta: float | None + min_score: float | None + passed: bool | None + + +@dataclass(frozen=True) +class ExperimentResult: + """The result of an experiment. ``status`` is ``completed``, ``failed``, or ``stopped``. + ``check_status`` is None when the experiment has no pass marks. ``error`` tells why the + experiment failed.""" + + id: str + name: str + status: str + check_status: Literal["passed", "failed"] | None + items_total: int + items_done: int + items_failed: int + error: str | None + scores: tuple[ExperimentScore, ...] + url: str + + +class ExperimentError(Exception): + """Why run_experiment failed. ``code`` is the code of the API error (``dataset_not_found``, + ``insufficient_scope``, ``invalid_api_key``, ...), or ``missing_api_key``, ``timeout``, + ``network_error``, ``invalid_response``, or ``dataset_empty``. In CI mode it is also + ``check_failed`` or ``experiment_failed``, and ``result`` holds the result. A response with a + different status and no API error gives ``http_``. A scorer that raises or returns a + value that is not a score sends a ``scorer_failed`` error to ``on_error``.""" + + def __init__( + self, + code: str, + message: str, + status: int | None = None, + *, + result: ExperimentResult | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.result = result + + +def run_experiment( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, Scorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """Run ``task`` on each item of a dataset and record the result as an experiment. + + Each item run is its own trace, with ``td.experiment.id`` and ``td.dataset.item_id`` on each + span. The outputs go to telemetry.dev in batches of at most 100, with the scores of + ``scorers``. Then run_experiment waits for the ``evaluators`` of the server, prints the result, + and returns it. ``concurrency`` (1 to 100, default 4) threads run the items, each in a copy of + the context of the caller. + + ``dataset`` defaults to the ``dataset`` input of the experiment action, which it sets in + ``TELEMETRY_DEV_EXPERIMENT_DATASET``. When the two are set, they must be the same. + ``min_scores`` holds pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. The ``min-scores`` input of the action adds to + them and wins for the same name. ``runs_per_item`` is 1 to 5. + + The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a + request fails. A task that raises fails only its item. In CI mode (``ci``, default on when + ``CI`` is ``true`` or ``1``), it records the GitHub Actions run, and it raises when the check + fails or the experiment does not complete, so the process exits with an error. + """ + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=False, + ) + runs, created = _start(api, run) + _run_items(api, run, created, runs) + return _finish(api, run, created) + + +async def run_experiment_async( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, AsyncScorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """``run_experiment`` for async tasks and scorers. ``concurrency`` asyncio tasks run the + items, and the requests go in worker threads. A task that is not async runs on the event + loop, so it blocks the other items while it runs: use ``run_experiment`` for it.""" + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=True, + ) + runs, created = await asyncio.to_thread(_start, api, run) + await _run_items_async(api, run, created, runs) + return await asyncio.to_thread(_finish, api, run, created) + + +@dataclass(frozen=True) +class _Run: + dataset: str + task: Callable[[ExperimentItem], Any] + name: str | None + dataset_version: int | None + split: str | None + evaluators: list[str] | None + scorers: list[tuple[str, Callable[[Any, ExperimentItem], Any]]] + min_scores: dict[str, float] | None + runs_per_item: int + concurrency: int + metadata: dict[str, Any] | None + ci: dict[str, Any] | None + in_ci: bool + + +@dataclass(frozen=True) +class _Created: + id: str + name: str + items_total: int + url: str + + +@dataclass(frozen=True) +class _Page: + version: int + items: list[DatasetItem] + next_cursor: str | None + + +def _prepare( + task: object, + dataset: object, + name: object, + dataset_version: object, + split: object, + evaluators: object, + scorers: object, + min_scores: object, + runs_per_item: object, + concurrency: object, + metadata: object, + ci: object, + api_key: str | None, + base_url: str | None, + *, + asynchronous: bool, +) -> tuple[_Run, Api]: + env = os.environ + if not callable(task): + raise TypeError("task must be a function") + if not asynchronous and inspect.iscoroutinefunction(task): + raise TypeError("task is async: use run_experiment_async") + for key, value in (("dataset", dataset), ("name", name), ("split", split)): + if value is not None and (not isinstance(value, str) or not value): + raise TypeError(f"{key} must be a non-empty string") + action_dataset = env.get("TELEMETRY_DEV_EXPERIMENT_DATASET") or None + chosen = cast("str | None", dataset) if dataset is not None else action_dataset + if chosen is None: + raise TypeError("Pass dataset, or set the dataset input of the experiment action") + if action_dataset is not None and action_dataset != chosen: + raise TypeError( + f"dataset is {chosen}, but the experiment action names {action_dataset}. " + "Name one dataset." + ) + if dataset_version is not None and not (_is_int(dataset_version) and dataset_version >= 1): + raise ValueError("dataset_version must be a positive integer") + + evaluator_names: list[str] | None = None + if evaluators is not None: + names = ( + list(cast("Sequence[object]", evaluators)) + if isinstance(evaluators, Sequence) and not isinstance(evaluators, str) + else None + ) + if ( + names is None + or len(names) > _MAX_EVALUATORS + or not all(isinstance(n, str) and _valid_score_name(n) for n in names) + ): + raise TypeError( + f"evaluators must be a list of at most {_MAX_EVALUATORS} evaluator names" + ) + evaluator_names = cast("list[str]", names) + + if scorers is not None and not isinstance(scorers, Mapping): + raise TypeError("scorers must be a mapping of score names to functions") + scorer_list = list(cast("Mapping[object, object]", scorers or {}).items()) + if len(scorer_list) > _MAX_EVALUATORS: + raise TypeError(f"scorers can have at most {_MAX_EVALUATORS} scorers") + folded = {n.lower() for n in evaluator_names or []} + for scorer_name, scorer in scorer_list: + if ( + not isinstance(scorer_name, str) + or not _valid_score_name(scorer_name) + or not callable(scorer) + ): + raise TypeError( + f"scorers must map score names of 1 to {MAX_SCORE_NAME_LENGTH} characters " + "to functions" + ) + if not asynchronous and inspect.iscoroutinefunction(scorer): + raise TypeError(f"The scorer {scorer_name} is async: use run_experiment_async") + if scorer_name == USER_FEEDBACK: + raise TypeError( + f"{USER_FEEDBACK} is the score of user feedback. Name the scorer another way." + ) + if scorer_name.lower() in folded: + raise TypeError( + f"The scorer {scorer_name} has the name of an evaluator. Rename one of them." + ) + + if min_scores is not None and not isinstance(min_scores, Mapping): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + marks: dict[object, object] = { + **cast("Mapping[object, object]", min_scores or {}), + **min_scores_from_env(env), + } + if len(marks) > _MAX_MIN_SCORES or not all( + isinstance(n, str) and _valid_score_name(n) and is_finite(v) for n, v in marks.items() + ): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + + if not _is_int(runs_per_item) or not 1 <= runs_per_item <= _MAX_RUNS_PER_ITEM: + raise ValueError(f"runs_per_item must be an integer from 1 to {_MAX_RUNS_PER_ITEM}") + if not _is_int(concurrency) or not 1 <= concurrency <= _MAX_CONCURRENCY: + raise ValueError(f"concurrency must be an integer from 1 to {_MAX_CONCURRENCY}") + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + if ci is not None and not isinstance(ci, bool): + raise TypeError("ci must be a bool") + ci_mode = in_ci(env) if ci is None else ci + + api = resolve_api(api_key, base_url) + if not api.api_key: + raise ExperimentError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + run = _Run( + dataset=chosen, + task=task, + name=cast("str | None", name), + dataset_version=dataset_version, + split=cast("str | None", split), + evaluators=evaluator_names, + scorers=cast("list[tuple[str, Callable[[Any, ExperimentItem], Any]]]", scorer_list), + min_scores={str(n): float(cast("float", v)) for n, v in marks.items()} or None, + runs_per_item=runs_per_item, + concurrency=concurrency, + metadata=dict(cast("Mapping[str, Any]", metadata)) if metadata is not None else None, + ci=github_run(env) if ci_mode else None, + in_ci=ci_mode, + ) + return run, api + + +def _valid_score_name(name: str) -> bool: + """A trimmed name of 1 to 64 characters, as the API stores score names.""" + return 0 < utf16_length(name) <= MAX_SCORE_NAME_LENGTH and name.strip() == name + + +def _is_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) + + +def _warn_about_client() -> None: + client = get_client() + environment = client.config.environment if client is not None and client.enabled else None + if environment is None: + logger.warning( + "run_experiment: no telemetry client, so the items record no traces. " + 'Call init(environment="experiment") first.' + ) + elif environment.strip().lower() != _EXPERIMENT_ENVIRONMENT: + logger.warning( + "run_experiment: the traces of the items go to the environment %s. " + 'Call init(environment="experiment") to keep them out of it.', + environment, + ) + + +def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: + _warn_about_client() + first = _fetch_page(api, run, run.dataset_version, None) + if not first.items: + scope = "" if run.split is None else f" in the split {run.split}" + raise ExperimentError("dataset_empty", f"The dataset {run.dataset} has no items{scope}") + body = { + "name": run.name, + "dataset": run.dataset, + "datasetVersion": first.version, + "split": run.split, + "evaluators": run.evaluators, + "minScores": run.min_scores, + "runsPerItem": run.runs_per_item, + "ci": run.ci, + "metadata": run.metadata, + } + created = _parse_created( + request( + api, + ExperimentError, + "POST", + "/v1/experiments", + body={key: value for key, value in body.items() if value is not None}, + timeout_s=_REQUEST_TIMEOUT_S, + # A repeat after a lost response would start a second experiment. + idempotent=False, + ) + ) + each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" + print( + f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " + f"{created.url}", + flush=True, + ) + return _Runs(api, run, first), created + + +class _Runs: + """The runs of each item of the dataset version, page by page, so that only one page is in + memory. A lock lets the workers share the generator.""" + + def __init__(self, api: Api, run: _Run, first: _Page) -> None: + self._lock = threading.Lock() + self._runs = _item_runs(api, run, first) + + def next(self) -> ExperimentItem | None: + with self._lock: + return next(self._runs, None) + + def close(self) -> None: + with self._lock: + self._runs.close() + + +def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: + page = first + count = 0 + while True: + count += len(page.items) + if count > _MAX_DATASET_ITEMS: + raise ExperimentError( + "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" + ) + for item in page.items: + for run_index in range(run.runs_per_item): + yield ExperimentItem(**vars(item), run_index=run_index) + if page.next_cursor is None: + return + page = _fetch_page(api, run, page.version, page.next_cursor) + + +def _fetch_page(api: Api, run: _Run, version: int | None, cursor: str | None) -> _Page: + """One page of a dataset version. A page after the first names the version of the first.""" + page = _parse_page( + request( + api, + ExperimentError, + "GET", + f"/v1/datasets/{urllib.parse.quote(run.dataset, safe='')}", + query={ + "version": None if version is None else str(version), + "split": run.split, + "cursor": cursor, + }, + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + ) + if ( + page is None + or (version is not None and page.version != version) + or (page.next_cursor is not None and (not page.items or page.next_cursor == cursor)) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a dataset it cannot read") + return page + + +class _Batch: + """Collects item results and posts them in order, one post at a time.""" + + def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: + self._post = post + self._pending: list[dict[str, Any]] = [] + self._bytes = 0 + self._lock = threading.Lock() + + def add(self, result: dict[str, Any]) -> None: + size = len(json.dumps(result, ensure_ascii=False).encode()) + with self._lock: + if self._pending and self._bytes + size > _MAX_BATCH_BYTES: + self._send() + self._pending.append(result) + self._bytes += size + if len(self._pending) >= _MAX_BATCH_ITEMS: + self._send() + + def flush(self) -> None: + with self._lock: + if self._pending: + self._send() + + def _send(self) -> None: + results = self._pending + self._pending = [] + self._bytes = 0 + self._post(results) + + +def _batch(api: Api, created: _Created) -> _Batch: + path = f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/items" + return _Batch( + lambda results: request( + api, + ExperimentError, + "POST", + path, + body={"items": results}, + timeout_s=_REQUEST_TIMEOUT_S, + # The server skips an item run that it has, so a repeat is safe. + idempotent=True, + ) + ) + + +def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch + that fails stops the run: no new item starts, and run_experiment raises when the running + items end.""" + batch = _batch(api, created) + failures: list[BaseException] = [] + caller = contextvars.copy_context() + + def worker() -> None: + while not failures: + try: + item = runs.next() + except Exception as exc: + failures.append(exc) + return + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + result = caller.copy().run(_run_item, run, created, item, api.on_error) + try: + batch.add(result) + except Exception as exc: + failures.append(exc) + + with ThreadPoolExecutor(run.concurrency, thread_name_prefix="telemetry-dev-experiment") as pool: + futures = [pool.submit(worker) for _ in range(run.concurrency)] + try: + for future in futures: + future.result() + except BaseException as exc: + # Stops the workers, for example after Ctrl-C. + failures.append(exc) + raise + _end_items(batch, runs, failures) + + +async def _run_items_async(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + batch = _batch(api, created) + failures: list[BaseException] = [] + + async def worker() -> None: + while not failures: + try: + item = await asyncio.to_thread(runs.next) + except Exception as exc: + failures.append(exc) + return + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) + try: + await asyncio.to_thread(batch.add, result) + except Exception as exc: + failures.append(exc) + + # gather runs each worker in its own task, with its own copy of the context. + await asyncio.gather(*(worker() for _ in range(run.concurrency))) + await asyncio.to_thread(_end_items, batch, runs, failures) + + +def _end_items(batch: _Batch, runs: _Runs, failures: list[BaseException]) -> None: + if not failures: + try: + batch.flush() + except Exception as exc: + failures.append(exc) + if failures: + runs.close() + raise failures[0] + + +def _run_item( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + _close(output) + raise TypeError("task returned an awaitable: use run_experiment_async") + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + _add_scores( + result, [_score(name, scorer, output, item, on_error) for name, scorer in run.scorers] + ) + return result + + +async def _run_item_async( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + output = await output + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + scores: list[dict[str, Any] | None] = [] + for name, scorer in run.scorers: + try: + value = scorer(output, item) + if inspect.isawaitable(value): + value = await value + scores.append(_to_score(name, value)) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + scores.append(None) + _add_scores(result, scores) + return result + + +def _item_context(experiment_id: str, item: ExperimentItem) -> Context: + """The context of one item run. It drops the active span, so that the run starts a trace of + its own, and gives the run a session id of its own: an ambient session or + ``session_mode="process"`` would otherwise put all runs in one session trace.""" + base = trace.set_span_in_context(trace.INVALID_SPAN, otel_context.get_current()) + return context_with_propagated_attributes( + { + **propagated_attributes(base), + ATTR_EXPERIMENT_ID: experiment_id, + ATTR_DATASET_ITEM_ID: item.id, + ATTR_SESSION_ID: f"{experiment_id}:{item.id}:{item.run_index}", + }, + base, + ) + + +def _trace_id(span: SpanHandle) -> str | None: + context = span.span.get_span_context() + return f"{context.trace_id:032x}" if span.span.is_recording() and context.is_valid else None + + +def _item_result( + item: ExperimentItem, + trace_id: str | None, + started: float, + output: Any, + error: Exception | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "itemId": item.id, + "runIndex": item.run_index, + "output": None if error is not None else _output_text(output), + "latencyMs": round((perf_counter() - started) * 1000), + } + if trace_id is not None: + result["traceId"] = trace_id + if error is not None: + result["error"] = truncate(_error_text(error), _MAX_ERROR_LENGTH) + return result + + +def _add_scores(result: dict[str, Any], scores: list[dict[str, Any] | None]) -> None: + kept = [score for score in scores if score is not None] + if kept: + result["scores"] = kept + + +def _score( + name: str, + scorer: Callable[[Any, ExperimentItem], Any], + output: Any, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any] | None: + # After the span of the item ends, so that the model calls of a scorer do not count in the + # cost of the item. + try: + value = scorer(output, item) + if inspect.isawaitable(value): + _close(value) + raise TypeError("returned an awaitable: use run_experiment_async") + return _to_score(name, value) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + return None + + +def _scorer_failed( + on_error: Callable[[BaseException], None] | None, + name: str, + item: ExperimentItem, + exc: Exception, +) -> None: + error = ExperimentError( + "scorer_failed", f"The scorer {name} failed on item {item.number}: {_error_text(exc)}" + ) + error.__cause__ = exc + report_error(on_error, "a scorer failed", error) + + +def _to_score(name: str, result: object) -> dict[str, Any] | None: + """The score of a scorer result. It raises for a result that the server would reject.""" + if result is None: + return None + score: dict[str, Any] = {"name": name} + if isinstance(result, bool): + score.update(_pass_fail(result)) + elif isinstance(result, (int, float)): + score["value"] = result + elif isinstance(result, str): + score["label"] = result.strip() + elif isinstance(result, Mapping): + fields = cast("Mapping[str, object]", result) + value, label, reason = fields.get("value"), fields.get("label"), fields.get("reason") + for key, text in (("label", label), ("reason", reason)): + if text is not None and not isinstance(text, str): + raise TypeError(f"returned a {key} that is not a string") + if isinstance(value, bool): + score.update(_pass_fail(value)) + elif value is not None: + score["value"] = value + if isinstance(label, str): + score["label"] = label.strip() + if isinstance(reason, str): + score["reason"] = truncate(reason, MAX_REASON_LENGTH) + else: + raise TypeError(f"returned a {type(result).__name__}") + problem = check_value(name, score.get("value"), score.get("label")) + if problem is not None: + raise TypeError(f"returned a score that is not valid: {problem}") + return score + + +def _pass_fail(passed: bool) -> dict[str, Any]: + return {"value": 1, "label": "pass"} if passed else {"value": 0, "label": "fail"} + + +def _output_text(output: Any) -> str | None: + if output is None: + return None + text = output if isinstance(output, str) else json_dumps(output) + return truncate(text, _MAX_OUTPUT_LENGTH) + + +def _error_text(error: BaseException) -> str: + return f"{type(error).__name__}: {error}" + + +def _close(awaitable: object) -> None: + # A coroutine that never runs warns when it is collected. + close = getattr(awaitable, "close", None) + if callable(close): + close() + + +def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: + # The spans of the items reach ingest before the server reads their cost. + flush() + request( + api, + ExperimentError, + "POST", + f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/complete", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: + """Wait until the server evaluators finish and the experiment ends.""" + deadline = time.monotonic() + _MAX_WAIT_S + interval = 1.0 + while True: + body = request( + api, + ExperimentError, + "GET", + f"/v1/experiments/{urllib.parse.quote(experiment_id, safe='')}", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result = _parse_result(body) + if result.status in _FINISHED: + return result, body + if time.monotonic() + interval > deadline: + raise ExperimentError( + "timeout", + f"The experiment {result.name} did not end in {_MAX_WAIT_S / 60:g} minutes. " + f"{result.url}", + result=result, + ) + time.sleep(interval) + interval = min(interval * 2, _MAX_POLL_INTERVAL_S) + + +def _check_result(result: ExperimentResult) -> None: + """In CI mode, a failed check or an experiment that did not complete fails the run.""" + if result.status != "completed": + reason = f": {result.error}" if result.error else "" + raise ExperimentError( + "experiment_failed", + f"The experiment {result.name} {result.status}{reason}. {result.url}", + result=result, + ) + if result.check_status != "failed": + return + failed = [ + f"{score.name} is {_format(score.kind, score.value)}, under the pass mark " + f"{_format(score.kind, score.min_score)}" + for score in result.scores + if score.passed is False + ] + detail = f": {'; '.join(failed)}" if failed else "" + raise ExperimentError( + "check_failed", + f"The check of {result.name} failed{detail}. {result.url}", + result=result, + ) + + +def _describe(result: ExperimentResult) -> str: + if result.status != "completed": + outcome = result.status + (f": {result.error}" if result.error else "") + elif result.check_status is None: + outcome = "completed" + else: + outcome = f"completed. The check {result.check_status}" + lines = [f"Experiment {result.name} {outcome}. {result.url}"] + for score in result.scores: + mark = ( + "" + if score.min_score is None + else f" (pass mark {_format(score.kind, score.min_score)})" + ) + lines.append(f" {score.name}: {_format(score.kind, score.value)}{mark}") + return "\n".join(lines) + + +def _format(kind: str | None, value: float | None) -> str: + if value is None: + return "no score" + if kind in _RATE_KINDS: + return f"{_js_round(value * 100)}%" + rounded = _js_round(value * 100) / 100 + return str(int(rounded)) if rounded.is_integer() else repr(rounded) + + +def _js_round(value: float) -> int: + # Math.round of JavaScript, which rounds a half up. round() of Python rounds it to even. + return math.floor(value + 0.5) + + +def _parse_created(body: object) -> _Created: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + created_id, name, url = fields.get("id"), fields.get("name"), fields.get("url") + items_total = fields.get("itemsTotal") + if ( + not isinstance(created_id, str) + or not isinstance(name, str) + or not isinstance(url, str) + or not _is_int(items_total) + ): + raise ExperimentError( + "invalid_response", "telemetry.dev returned an experiment it cannot read" + ) + return _Created(created_id, name, items_total, url) + + +def _parse_page(body: object) -> _Page | None: + if not isinstance(body, dict): + return None + fields = cast("dict[str, Any]", body) + dataset, items = fields.get("dataset"), fields.get("items") + next_cursor = fields.get("nextCursor") + if not isinstance(dataset, dict) or not isinstance(items, list): + return None + version = cast("dict[str, Any]", dataset).get("version") + if not _is_int(version) or version < 0: + return None + if next_cursor is not None and not isinstance(next_cursor, str): + return None + parsed: list[DatasetItem] = [] + for raw in cast("list[object]", items): + item = _parse_item(raw) + if item is None: + return None + parsed.append(item) + return _Page(version, parsed, next_cursor) + + +def _parse_item(raw: object) -> DatasetItem | None: + if not isinstance(raw, dict): + return None + item = cast("dict[str, Any]", raw) + item_id, number, tags = item.get("id"), item.get("number"), item.get("tags") + item_input, metadata = item.get("input"), item.get("metadata") + expected, split = item.get("expectedOutput"), item.get("split") + if ( + not isinstance(item_id, str) + or not _is_int(number) + or not isinstance(item_input, dict) + or not (expected is None or isinstance(expected, str)) + or not isinstance(metadata, dict) + or not isinstance(tags, list) + or not all(isinstance(tag, str) for tag in cast("list[object]", tags)) + or not (split is None or isinstance(split, str)) + ): + return None + return DatasetItem( + id=item_id, + number=number, + input=cast("dict[str, Any]", item_input), + expected_output=expected, + metadata=cast("dict[str, Any]", metadata), + tags=tuple(cast("list[str]", tags)), + split=split, + ) + + +def _parse_result(body: object) -> ExperimentResult: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + raw_scores = fields.get("scores") + scores = ( + [_parse_score(score) for score in cast("list[object]", raw_scores)] + if isinstance(raw_scores, list) + else None + ) + check_status = fields.get("checkStatus", "missing") + items_failed = fields.get("itemsFailed", 0) + error = fields.get("error") + if ( + not isinstance(fields.get("id"), str) + or not isinstance(fields.get("name"), str) + or not isinstance(fields.get("status"), str) + or not isinstance(fields.get("url"), str) + or check_status not in (None, "passed", "failed") + or not _is_number(fields.get("itemsTotal")) + or not _is_number(fields.get("itemsDone")) + or not _is_number(items_failed) + or not (error is None or isinstance(error, str)) + or scores is None + or any(score is None for score in scores) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read") + return ExperimentResult( + id=fields["id"], + name=fields["name"], + status=fields["status"], + check_status=check_status, + items_total=int(fields["itemsTotal"]), + items_done=int(fields["itemsDone"]), + items_failed=int(items_failed), + error=error, + scores=tuple(cast("list[ExperimentScore]", scores)), + url=fields["url"], + ) + + +def _parse_score(raw: object) -> ExperimentScore | None: + if not isinstance(raw, dict): + return None + score = cast("dict[str, Any]", raw) + name, kind, passed = score.get("name"), score.get("kind"), score.get("passed") + pass_count, fail_count = score.get("passCount"), score.get("failCount") + nullable = [score.get(key) for key in ("value", "baselineValue", "delta", "minScore")] + if ( + not isinstance(name, str) + or not (kind is None or isinstance(kind, str)) + or not _is_number(pass_count) + or not _is_number(fail_count) + or not all(value is None or _is_number(value) for value in nullable) + or not (passed is None or isinstance(passed, bool)) + ): + return None + value, baseline_value, delta, min_score = nullable + return ExperimentScore( + name=name, + kind=kind, + value=value, + pass_count=int(cast("float", pass_count)), + fail_count=int(cast("float", fail_count)), + baseline_value=baseline_value, + delta=delta, + min_score=min_score, + passed=passed, + ) + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 9673740..a7f3aff 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -17,18 +17,16 @@ from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field -from datetime import datetime, timezone -from email.message import Message -from email.utils import parsedate_to_datetime from time import monotonic -from typing import IO, Any, Literal, TypeAlias, TypedDict, cast, overload +from typing import Any, Literal, TypeAlias, TypedDict, cast, overload from weakref import WeakKeyDictionary from opentelemetry import context as otel_context from opentelemetry.context import Context +from ._api import NoRedirect, api_error, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error, resolve_config +from ._config import SDK_VERSION, report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -45,8 +43,6 @@ # The longest Retry-After that get_prompt honors, so that a bad header cannot stop fetches. _MAX_RETRY_AFTER_S = 3600.0 _MAX_CACHE_ENTRIES = 1000 -# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. -_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") @@ -310,11 +306,11 @@ def _get_prompt( _check(name, type, label, version, cache_ttl_seconds, fallback) client = get_client() enabled = client if client is not None and client.enabled else None - config = enabled.config if enabled is not None else resolve_config() - key = api_key if api_key is not None else config.api_key - url = (base_url or config.base_url).rstrip("/") + api = resolve_api(api_key, base_url) + key = api.api_key + url = api.base_url asked_label = (label if label is not None else "production") if version is None else None - on_error = enabled.on_error if enabled is not None else None + on_error = api.on_error try: if not key: @@ -535,21 +531,6 @@ def _refresh( done.set() -class _NoRedirect(urllib.request.HTTPRedirectHandler): - # A redirect to another host must not get the API key, so a 3xx fails like any status that is - # not 2xx. - def redirect_request( - self, - req: urllib.request.Request, - fp: IO[bytes], - code: int, - msg: str, - headers: Message, - newurl: str, - ) -> urllib.request.Request | None: - return None - - class _Deadline: """Ends one prompt fetch after ``_FETCH_TIMEOUT_S``. The timeout of urllib applies to each socket operation, so a server that sends a byte now and then could keep a fetch reading forever, @@ -648,12 +629,12 @@ def do_open(self, http_class: Any, req: urllib.request.Request, **http_conn_args return super().do_open(_on_deadline(http_class, req), req, **http_conn_args) -_opener = urllib.request.build_opener(_NoRedirect, _HTTPHandler, _HTTPSHandler) +_opener = urllib.request.build_opener(NoRedirect, _HTTPHandler, _HTTPSHandler) def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: deadline = _Deadline() - retry_after: str | None = None + retry_header: str | None = None try: try: with _opener.open( @@ -664,7 +645,7 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: except urllib.error.HTTPError as exc: with exc: status, body = exc.code, exc.read() - retry_after = exc.headers.get("Retry-After") + retry_header = exc.headers.get("Retry-After") except Exception as exc: if ( deadline.expired @@ -681,9 +662,10 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: if deadline.expired: raise _timeout() if not 200 <= status < 300: - error = _api_error(status, body) - if status in (429, 503): - error._retry_after = _retry_after(retry_after) # pyright: ignore[reportPrivateUsage] + error = api_error(PromptError, status, body) + wait = retry_after(status, retry_header) + if wait is not None: + error._retry_after = min(wait, _MAX_RETRY_AFTER_S) # pyright: ignore[reportPrivateUsage] raise error try: parsed = _parse_response(json.loads(body)) @@ -698,40 +680,6 @@ def _timeout() -> PromptError: return PromptError("timeout", f"Fetching the prompt took more than {_FETCH_TIMEOUT_S:g} s") -def _api_error(status: int, body: bytes) -> PromptError: - try: - error = json.loads(body).get("error") - except (ValueError, AttributeError): - error = None - fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} - code = fields.get("code") - message = fields.get("message") - return PromptError( - code if isinstance(code, str) else f"http_{status}", - message if isinstance(message, str) else f"telemetry.dev returned {status}", - status, - ) - - -def _retry_after(value: str | None) -> float | None: - """Seconds to wait from a Retry-After header of seconds or an IMF-fixdate HTTP date - (``Sun, 06 Nov 1994 08:49:37 GMT``), capped.""" - value = (value or "").strip() - if not value: - return None - if value.isascii() and value.isdigit(): - seconds = float(value) - elif _HTTP_DATE.fullmatch(value): - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError): - return None - seconds = (date - datetime.now(timezone.utc)).total_seconds() - else: - return None - return min(seconds, _MAX_RETRY_AFTER_S) if seconds > 0 else None - - def _parse_response(data: object) -> dict[str, Any] | None: if not isinstance(data, dict): return None diff --git a/sdks/python/src/telemetry_dev/_scores.py b/sdks/python/src/telemetry_dev/_scores.py new file mode 100644 index 0000000..f5dbee3 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_scores.py @@ -0,0 +1,181 @@ +"""Scores on traces, spans, and sessions, as ``POST /v1/scores`` takes them.""" + +from __future__ import annotations + +import asyncio +import math +import re +from typing import Any, cast + +from ._api import request, resolve_api +from ._config import report_error +from ._serialize import utf16_length + +USER_FEEDBACK = "user_feedback" +MAX_SCORE_NAME_LENGTH = 64 +MAX_LABEL_LENGTH = 64 +MAX_REASON_LENGTH = 4000 +_MAX_SESSION_ID_LENGTH = 1024 +_TRACE_ID = re.compile(r"(?!0+$)[0-9a-f]{32}") +_SPAN_ID = re.compile(r"(?!0+$)[0-9a-f]{16}") +_TIMEOUT_S = 10.0 + + +class ScoreError(Exception): + """Why score failed. ``code`` is the code of the API error (``insufficient_scope``, + ``invalid_api_key``, ...), or ``missing_api_key``, ``invalid_score``, ``timeout``, + ``network_error``, or ``invalid_response``. A response with a different status and no API + error gives ``http_``. ``status`` is the HTTP status of the response.""" + + def __init__(self, code: str, message: str, status: int | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + + +def score( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """Record a score on a trace, a span, or a session, for example the feedback of a user on a + reply. ``user_feedback`` is a thumbs up (1) or down (0). Pass ``trace_id``, ``session_id``, or + the two, and ``value``, ``label``, or the two. + + Like the telemetry functions, score never raises: it returns the id of the score, or None + after it sends the error to ``on_error``. It makes one request, with no retry, so that a call + in a request handler waits 10 seconds at most for each socket operation. The trace id of a + span from a client that is not enabled is all zeros, and score reports it as + ``invalid_score``. + """ + api = resolve_api(api_key, base_url) + try: + problem = _check_score(name, trace_id, span_id, session_id, value, label, comment) + if problem is not None: + raise ScoreError("invalid_score", problem) + if not api.api_key: + raise ScoreError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + fields = { + "traceId": trace_id, + "spanId": span_id, + "sessionId": session_id, + "name": name.strip(), + "value": value, + "label": None if label is None else label.strip(), + "comment": comment, + } + body = request( + api, + ScoreError, + "POST", + "/v1/scores", + body={key: field for key, field in fields.items() if field is not None}, + timeout_s=_TIMEOUT_S, + idempotent=False, + attempts=1, + ) + score_id = cast("dict[str, Any]", body).get("id") if isinstance(body, dict) else None + if not isinstance(score_id, str): + raise ScoreError("invalid_response", "telemetry.dev returned a score it cannot read") + return score_id + except Exception as exc: + report_error(api.on_error, "sending the score failed", exc) + return None + + +async def score_async( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """``score`` in a worker thread, so that the request does not block the event loop.""" + return await asyncio.to_thread( + score, + name=name, + trace_id=trace_id, + span_id=span_id, + session_id=session_id, + value=value, + label=label, + comment=comment, + api_key=api_key, + base_url=base_url, + ) + + +def check_value(name: str, value: object, label: object) -> str | None: + """Why the server would reject the value and the label of a score, or None. A pass/fail score + has the label ``pass`` with the value 1, or ``fail`` with 0.""" + if value is not None and not is_finite(value): + return "value must be a finite number" + if label is not None and not ( + isinstance(label, str) and 0 < utf16_length(label) <= MAX_LABEL_LENGTH + ): + return f"label must be a string of 1 to {MAX_LABEL_LENGTH} characters" + if value is None and label is None: + return "Pass a value or a label" + if name == USER_FEEDBACK and not (label is None and value in (0, 1)): + return f"{USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label" + if label == "pass" and value is not None and value != 1: + return "The label pass goes with the value 1" + if label == "fail" and value is not None and value != 0: + return "The label fail goes with the value 0" + return None + + +def _check_score( + name: object, + trace_id: object, + span_id: object, + session_id: object, + value: object, + label: object, + comment: object, +) -> str | None: + if not isinstance(name, str) or not 0 < utf16_length(name.strip()) <= MAX_SCORE_NAME_LENGTH: + return f"name must be a string of 1 to {MAX_SCORE_NAME_LENGTH} characters" + if trace_id is None and session_id is None: + return "Pass trace_id or session_id" + if trace_id is not None and not (isinstance(trace_id, str) and _TRACE_ID.fullmatch(trace_id)): + return "trace_id must be 32 lowercase hex digits and not all zeros" + if span_id is not None and not (isinstance(span_id, str) and _SPAN_ID.fullmatch(span_id)): + return "span_id must be 16 lowercase hex digits and not all zeros" + if span_id is not None and trace_id is None: + return "Pass the trace_id of the span" + if session_id is not None and not ( + isinstance(session_id, str) and 0 < utf16_length(session_id) <= _MAX_SESSION_ID_LENGTH + ): + return f"session_id must be a string of 1 to {_MAX_SESSION_ID_LENGTH} characters" + if comment is not None and not ( + isinstance(comment, str) and utf16_length(comment) <= MAX_REASON_LENGTH + ): + return f"comment must be a string of at most {MAX_REASON_LENGTH} characters" + return check_value(name.strip(), value, label.strip() if isinstance(label, str) else label) + + +def is_finite(value: object) -> bool: + # True and False are ints in Python, but JSON sends them as booleans, which are not numbers. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + # An int too large for a float, which JavaScript reads as Infinity. + return False diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 69553c0..7d3041b 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -88,6 +88,8 @@ ATTR_PROMPT_VERSION = "td.prompt.version" ATTR_PROMPT_LABEL = "td.prompt.label" ATTR_PROMPT_VARIABLES = "td.prompt.variables" +ATTR_EXPERIMENT_ID = "td.experiment.id" +ATTR_DATASET_ITEM_ID = "td.dataset.item_id" USAGE_ATTRS: dict[str, str] = { "input_tokens": "gen_ai.usage.input_tokens", diff --git a/sdks/python/src/telemetry_dev/_serialize.py b/sdks/python/src/telemetry_dev/_serialize.py index dcf1321..f69c4a6 100644 --- a/sdks/python/src/telemetry_dev/_serialize.py +++ b/sdks/python/src/telemetry_dev/_serialize.py @@ -30,6 +30,11 @@ class MaskContext: Mask = Callable[[Any, MaskContext], Any] +def utf16_length(text: str) -> int: + """The length of ``text`` in UTF-16 units, as JavaScript and the telemetry.dev API count it.""" + return len(text.encode("utf-16-le", "surrogatepass")) // 2 + + def truncate(text: str, max_len: int, *, keep_end: bool = False) -> str: """Cap ``text`` at ``max_len`` UTF-16 units, marking the cut. diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py new file mode 100644 index 0000000..7df2e7a --- /dev/null +++ b/sdks/python/tests/fake_api.py @@ -0,0 +1,193 @@ +"""The Evals routes of telemetry.dev on a local HTTP server: pages of a dataset version with a +cursor, an experiment that takes item posts of at most 100 and skips repeats, and its result.""" + +from __future__ import annotations + +import json +import threading +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qsl, urlsplit + +EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff" +EXPERIMENT_URL = f"https://telemetry.dev/o/org/p/project/evals/experiments/{EXPERIMENT_ID}" + + +def item_id(number: int) -> str: + return f"00000000-0000-4000-8000-{number:012d}" + + +def dataset_item(number: int, split: str | None = None) -> dict[str, Any]: + return { + "id": item_id(number), + "number": number, + "input": {"question": f"q{number}"}, + "expectedOutput": f"a{number}", + "metadata": {"source": "test"}, + "tags": ["golden"], + "split": split, + } + + +def result(**overrides: Any) -> dict[str, Any]: + body: dict[str, Any] = { + "id": EXPERIMENT_ID, + "name": "support-golden-v3", + "status": "completed", + "checkStatus": None, + "itemsTotal": 2, + "itemsDone": 2, + "itemsFailed": 0, + "error": None, + "scores": [], + "url": EXPERIMENT_URL, + } + body.update(overrides) + return body + + +@dataclass +class ApiRequest: + method: str + path: str + query: dict[str, str] + headers: dict[str, str] + body: Any + + @property + def route(self) -> str: + return f"{self.method} {self.path}" + + +# (status, JSON body, headers), or None to answer as the API does. +Reply = tuple[int, object, dict[str, str]] + + +@dataclass +class FakeApi: + url: str + items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) + page_size: int = 100 + results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) + override: Callable[[ApiRequest], Reply | None] | None = None + requests: list[ApiRequest] = field(default_factory=lambda: []) + stored: set[str] = field(default_factory=lambda: set[str]()) + lock: threading.Lock = field(default_factory=threading.Lock) + + def routes(self) -> list[str]: + return [request.route for request in self.requests] + + def posts(self) -> list[list[dict[str, Any]]]: + return [ + request.body["items"] for request in self.requests if request.path.endswith("/items") + ] + + def answer(self, request: ApiRequest) -> Reply: + with self.lock: + self.requests.append(request) + overridden = self.override(request) if self.override else None + if overridden is not None: + return overridden + return self._route(request) + + def _route(self, request: ApiRequest) -> Reply: + if request.route == "GET /v1/datasets/support-golden": + split = request.query.get("split") + start = int(request.query.get("cursor", "0")) + matching = [item for item in self.items if split is None or item["split"] == split] + more = start + self.page_size < len(matching) + return ( + 200, + { + "dataset": { + "id": "dataset-1", + "name": "support-golden", + "version": int(request.query.get("version", "3")), + }, + "items": matching[start : start + self.page_size], + "nextCursor": str(start + self.page_size) if more else None, + }, + {}, + ) + if request.route == "POST /v1/experiments": + split = request.body.get("split") + count = len([item for item in self.items if split is None or item["split"] == split]) + return ( + 201, + { + "id": EXPERIMENT_ID, + "name": request.body.get("name", "support-golden-v3"), + "datasetVersion": request.body.get("datasetVersion"), + "itemsTotal": count * request.body.get("runsPerItem", 1), + "url": EXPERIMENT_URL, + }, + {}, + ) + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/items": + runs = request.body["items"] + if len(runs) > 100: + return 400, {"error": {"code": "invalid_request", "message": "at most 100"}}, {} + keys = [f"{run['itemId']}:{run.get('runIndex', 0)}" for run in runs] + accepted = len([key for key in keys if key not in self.stored]) + self.stored.update(keys) + return 200, {"accepted": accepted, "duplicates": len(runs) - accepted}, {} + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/complete": + return 200, {"id": EXPERIMENT_ID, "status": "running"}, {} + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + return 200, self.results.pop(0) if len(self.results) > 1 else self.results[0], {} + return 404, {"error": {"code": "not_found", "message": request.route}}, {} + + +@contextmanager +def fake_api(**options: Any) -> Generator[FakeApi]: + api: FakeApi | None = None + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self._answer() + + def do_POST(self) -> None: + self._answer() + + def _answer(self) -> None: + assert api is not None + length = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(length) if length else b"" + target = urlsplit(self.path) + status, body, headers = api.answer( + ApiRequest( + method=self.command, + path=target.path, + query=dict(parse_qsl(target.query)), + headers={key.lower(): value for key, value in self.headers.items()}, + body=json.loads(raw) if raw else None, + ) + ) + data = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + host, port = server.server_address[0], server.server_address[1] + api = FakeApi(url=f"http://{host}:{port}", **options) + thread.start() + try: + yield api + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py new file mode 100644 index 0000000..a0b1807 --- /dev/null +++ b/sdks/python/tests/test_experiments.py @@ -0,0 +1,746 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Generator +from pathlib import Path +from typing import Any + +import pytest + +from telemetry_dev import ( + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + log, + propagate_attributes, + run_experiment, + run_experiment_async, + start_span, +) + +from .conftest import MakeClient +from .fake_api import ( + EXPERIMENT_ID, + EXPERIMENT_URL, + ApiRequest, + FakeApi, + Reply, + dataset_item, + fake_api, + item_id, + result, +) + +ENV_VARS = ( + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", +) + + +@pytest.fixture(autouse=True) +def clear_ci(monkeypatch: pytest.MonkeyPatch) -> None: + # GitHub Actions sets CI and GITHUB_* for this suite too. + for var in ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +@pytest.fixture +def api() -> Generator[FakeApi]: + with fake_api() as server: + yield server + + +def answer(item: ExperimentItem) -> str: + return f"answer to {item.input['question']}" + + +def experiment(api: FakeApi, **options: Any) -> ExperimentResult: + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return run_experiment(**arguments) + + +def answer_with(status: int, body: object, headers: dict[str, str] | None = None) -> Reply: + return status, body, headers or {} + + +def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_completes() -> None: + with fake_api( + items=[dataset_item(number) for number in range(1, 76)], + page_size=50, + results=[result(itemsTotal=150, itemsDone=146, itemsFailed=4)], + ) as api: + outcome = experiment(api, runs_per_item=2, evaluators=["Correctness"]) + + routes = api.routes() + assert routes[:2] == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + # The second page and the first post of items run at the same time, in either order. + assert sorted(routes[2:-2]) == [ + "GET /v1/datasets/support-golden", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + ] + assert routes[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + f"GET /v1/experiments/{EXPERIMENT_ID}", + ] + # The first page names no version; the next one asks for the version of the first. + pages = [request for request in api.requests if request.path.startswith("/v1/datasets/")] + assert pages[0].query == {} + assert pages[1].query == {"version": "3", "cursor": "50"} + assert api.requests[1].body == { + "dataset": "support-golden", + "datasetVersion": 3, + "evaluators": ["Correctness"], + "runsPerItem": 2, + } + assert api.requests[1].headers["authorization"] == "Bearer td_live_test" + assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + + posts = api.posts() + assert [len(items) for items in posts] == [100, 50] + runs = [f"{run['itemId']}:{run['runIndex']}" for items in posts for run in items] + assert len(set(runs)) == 150 + assert f"{item_id(75)}:1" in runs + first = posts[0][0] + assert isinstance(first.pop("latencyMs"), int) + assert first == {"itemId": item_id(1), "runIndex": 0, "output": "answer to q1"} + + assert outcome == ExperimentResult( + id=EXPERIMENT_ID, + name="support-golden-v3", + status="completed", + check_status=None, + items_total=150, + items_done=146, + items_failed=4, + error=None, + scores=(), + url=EXPERIMENT_URL, + ) + + +def test_each_item_run_is_its_own_trace_and_each_span_of_the_task_carries_the_item( + make: MakeClient, api: FakeApi +) -> None: + # The session parent of a root span needs the key. + memory = make(api_key="td_live_test", environment="experiment", session_mode="process") + + def task(item: ExperimentItem) -> dict[str, str]: + with start_span("chat gpt-4o", type="generation"): + pass + log("replied") + return {"reply": f"r{item.number}"} + + # An active span, an ambient session, and the process session would each put the runs in + # one trace. + with ( + propagate_attributes(session_id="ambient-session", user_id="u-1"), + start_span("eval script"), + ): + experiment(api, runs_per_item=2, task=task) + + finished = memory.span_exporter.get_finished_spans() + chats = [span for span in finished if span.name == "chat gpt-4o"] + roots = [span for span in finished if span.name == "experiment support-golden-v3"] + assert len(chats) == 4 + assert len(roots) == 4 + + def trace_of(span: Any) -> str: + return f"{span.context.trace_id:032x}" + + assert len({trace_of(span) for span in roots}) == 4 + assert [span.parent is not None and span.parent.is_remote for span in roots] == [True] * 4 + posted = [run for items in api.posts() for run in items] + for run in posted: + root = next(span for span in roots if trace_of(span) == run["traceId"]) + chat = next(span for span in chats if trace_of(span) == run["traceId"]) + assert chat.parent is not None + assert chat.parent.span_id == root.context.span_id + for span in (root, chat): + attributes = dict(span.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert attributes["gen_ai.conversation.id"] == ( + f"{EXPERIMENT_ID}:{run['itemId']}:{run['runIndex']}" + ) + assert attributes["user.id"] == "u-1" + records = memory.log_exporter.get_finished_logs() + assert len(records) == 4 + for record in records: + trace_id = f"{record.log_record.trace_id:032x}" + run = next(run for run in posted if run["traceId"] == trace_id) + attributes = dict(record.log_record.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert sorted(run["output"] for run in posted) == [ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ] + + +def test_runs_at_most_concurrency_tasks_at_a_time() -> None: + lock = threading.Lock() + running = 0 + most = 0 + + def task(item: ExperimentItem) -> str: + nonlocal running, most + with lock: + running += 1 + most = max(most, running) + time.sleep(0.05) + with lock: + running -= 1 + return "ok" + + with fake_api(items=[dataset_item(number) for number in range(1, 13)]) as api: + experiment(api, concurrency=3, task=task) + + assert most == 3 + assert len([run for items in api.posts() for run in items]) == 12 + + +def test_a_task_that_raises_fails_only_its_item(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + if item.number == 1: + raise ValueError("no reply") + return "fine" + + experiment(api, task=task) + + failed, passed = [run for items in api.posts() for run in items] + assert failed["output"] is None + assert failed["error"] == "ValueError: no reply" + assert "scores" not in failed + assert passed["output"] == "fine" + assert "error" not in passed + + +def test_scorers_give_scores_and_a_bad_result_drops_only_its_score(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def thrown(output: Any, item: ExperimentItem) -> None: + raise RuntimeError("judge down") + + scorers: dict[str, Scorer] = { + "exact": lambda output, item: output == f"answer to {item.input['question']}", + "polite": lambda output, item: False, + "length": lambda output, item: len(output), + "tone": lambda output, item: " friendly ", + "judged": lambda output, item: {"value": 0.25, "reason": "too short"}, + "none": lambda output, item: None, + "thrown": thrown, + "mismatched": lambda output, item: {"value": 0.5, "label": "pass"}, + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + "wide": lambda output, item: "\U0001f600" * 33, + } + + with fake_api(items=[dataset_item(1)]) as api: + experiment(api, scorers=scorers) + + assert api.posts()[0][0]["scores"] == [ + {"name": "exact", "value": 1, "label": "pass"}, + {"name": "polite", "value": 0, "label": "fail"}, + {"name": "length", "value": 12}, + {"name": "tone", "label": "friendly"}, + {"name": "judged", "value": 0.25, "reason": "too short"}, + ] + assert [(getattr(error, "code", None), str(error)) for error in errors] == [ + ("scorer_failed", "The scorer thrown failed on item 1: RuntimeError: judge down"), + ( + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not " + "valid: The label pass goes with the value 1", + ), + ( + "scorer_failed", + "The scorer wide failed on item 1: TypeError: returned a score that is not valid: " + "label must be a string of 1 to 64 characters", + ), + ] + + +def test_an_output_is_cut_to_100000_utf16_units(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + return "\U0001f600" * 60_000 + + experiment(api, task=task) + + output = api.posts()[0][0]["output"] + assert len(output.encode("utf-16-le")) // 2 <= 100_000 + assert output.endswith("...[truncated]") + + +def test_a_failed_post_of_items_stops_new_items_never_completes_and_raises() -> None: + started = 0 + + def task(item: ExperimentItem) -> str: + nonlocal started + started += 1 + return "ok" + + def ended(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 409, + {"error": {"code": "experiment_ended", "message": "support-golden-v3 has ended."}}, + ) + return None + + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=ended) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("experiment_ended", 409) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" not in api.routes() + + +def test_an_item_post_that_fails_with_500_goes_again() -> None: + failures = [1] + + def flaky(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items") and failures: + failures.pop() + return answer_with(500, {}) + return None + + with fake_api(override=flaky) as api: + experiment(api) + + assert len(api.posts()) == 2 + assert api.posts()[1] == api.posts()[0] + + +def test_a_create_that_fails_with_500_does_not_go_again() -> None: + def broken(request: ApiRequest) -> Reply | None: + return answer_with(500, {}) if request.route == "POST /v1/experiments" else None + + with fake_api(override=broken) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, caught.value.status) == ("http_500", 500) + assert api.routes().count("POST /v1/experiments") == 1 + + +def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: + limited = [1] + + def limit(request: ApiRequest) -> Reply | None: + if request.route == "POST /v1/experiments" and limited: + limited.pop() + return answer_with( + 429, + {"error": {"code": "rate_limited", "message": "Too many requests"}}, + {"retry-after": "2"}, + ) + return None + + with fake_api(override=limit) as api: + started = time.monotonic() + experiment(api) + + assert api.routes().count("POST /v1/experiments") == 2 + # Without the header, the first retry waits 1 second. + assert time.monotonic() - started >= 2 + + +def test_waits_until_the_server_evaluators_end_the_experiment() -> None: + with fake_api(results=[result(status="running", itemsDone=1), result()]) as api: + outcome = experiment(api) + + assert outcome.status == "completed" + assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 + + +@pytest.mark.parametrize( + ("status", "body", "code"), + [ + ( + 404, + {"error": {"code": "dataset_not_found", "message": "No dataset"}}, + "dataset_not_found", + ), + (200, {"dataset": {"version": 1}, "items": [{"id": 1}]}, "invalid_response"), + ( + 200, + {"dataset": {"id": "d", "name": "support-golden", "version": 0}, "items": []}, + "dataset_empty", + ), + ], + ids=["the API error", "a page that cannot be read", "an empty dataset"], +) +def test_raises_before_it_creates_an_experiment(status: int, body: object, code: str) -> None: + def page(request: ApiRequest) -> Reply | None: + return answer_with(status, body) if request.path.startswith("/v1/datasets/") else None + + with fake_api(override=page) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == code + assert api.routes() == ["GET /v1/datasets/support-golden"] + + +def test_a_cursor_that_repeats_is_an_invalid_response() -> None: + def same(request: ApiRequest) -> Reply | None: + if not request.path.startswith("/v1/datasets/"): + return None + return answer_with( + 200, + { + "dataset": {"id": "d", "name": "support-golden", "version": 3}, + "items": [dataset_item(1)], + "nextCursor": "same", + }, + ) + + with fake_api(override=same) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + assert api.routes().count("GET /v1/datasets/support-golden") == 2 + + +FAILED_CHECK = result( + checkStatus="failed", + scores=[ + { + "name": "correctness", + "kind": "pass_fail", + "value": 0.85, + "passCount": 17, + "failCount": 3, + "baselineValue": 0.9, + "delta": -5, + "minScore": 0.9, + "passed": False, + }, + { + "name": "length", + "kind": None, + "value": None, + "passCount": 0, + "failCount": 0, + "baselineValue": None, + "delta": None, + "minScore": None, + "passed": None, + }, + ], +) + + +def test_in_ci_mode_a_failed_check_raises_with_the_result(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CI", "true") + with fake_api(results=[FAILED_CHECK]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, min_scores={"correctness": 0.9}) + + assert caught.value.code == "check_failed" + assert str(caught.value) == ( + "The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. " + f"{EXPERIMENT_URL}" + ) + assert caught.value.result is not None + assert caught.value.result.scores[1] == ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ) + + +@pytest.mark.parametrize( + ("options", "ci"), + [({}, None), ({"ci": False}, "true")], + ids=["CI is off", "ci=False wins over CI"], +) +def test_a_failed_check_returns_when( + monkeypatch: pytest.MonkeyPatch, options: dict[str, Any], ci: str | None +) -> None: + if ci is not None: + monkeypatch.setenv("CI", ci) + with fake_api(results=[FAILED_CHECK]) as api: + assert experiment(api, **options).check_status == "failed" + + +def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CI", "1") + with fake_api( + results=[result(status="failed", error="0 of 2 items ran. Each one failed.")] + ) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "experiment_failed" + assert str(caught.value) == ( + "The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. " + f"{EXPERIMENT_URL}" + ) + + +def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + results = tmp_path / "results.jsonl" + event.write_text( + json.dumps( + { + "pull_request": { + "number": 42, + "title": "Shorter replies", + "html_url": "https://github.com/acme/support/pull/42", + "head": {"sha": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"}, + } + } + ) + ) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "f" * 40, + "GITHUB_EVENT_PATH": str(event), + "GITHUB_RUN_ID": "991", + "TELEMETRY_DEV_EXPERIMENT_DATASET": "support-golden", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=0.9\nlength = 12.5", + "TELEMETRY_DEV_EXPERIMENT_RESULTS": str(results), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api, dataset=None, min_scores={"correctness": 0.5, "tone": 2}) + + create = api.requests[1].body + assert create["dataset"] == "support-golden" + assert create["minScores"] == {"correctness": 0.9, "length": 12.5, "tone": 2} + assert create["ci"] == { + "provider": "github_actions", + "repository": "acme/support", + "commit": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "prNumber": 42, + "prTitle": "Shorter replies", + "prUrl": "https://github.com/acme/support/pull/42", + "runUrl": "https://github.com/acme/support/actions/runs/991", + } + lines = results.read_text().splitlines() + assert [json.loads(line) for line in lines] == [result()] + assert results.stat().st_mode & 0o777 == 0o600 + + +def test_a_pull_request_title_is_cut_to_256_utf16_units( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + event.write_text(json.dumps({"pull_request": {"title": "\U0001f600" * 200}})) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "a1b2c3d", + "GITHUB_EVENT_PATH": str(event), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api) + + assert api.requests[1].body["ci"]["prTitle"] == "\U0001f600" * 128 + + +def one(output: Any, item: ExperimentItem) -> int: + return 1 + + +REJECTED: list[tuple[dict[str, str], dict[str, Any], type[Exception], str]] = [ + ( + {}, + {"dataset": None}, + TypeError, + "Pass dataset, or set the dataset input of the experiment action", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_DATASET": "support-v2"}, + {}, + TypeError, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=1_000"}, + {}, + TypeError, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=1_000" is not a name=number pair', + ), + ({}, {"runs_per_item": 6}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"runs_per_item": True}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"concurrency": 0}, ValueError, "concurrency must be an integer from 1 to 100"), + ( + {}, + {"scorers": {"user_feedback": one}}, + TypeError, + "user_feedback is the score of user feedback. Name the scorer another way.", + ), + ( + {}, + {"evaluators": ["correctness"], "scorers": {"Correctness": one}}, + TypeError, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ), + ({}, {"evaluators": "correctness"}, TypeError, "evaluators must be a list of at most"), +] + + +@pytest.mark.parametrize( + ("env", "options", "error", "message"), + REJECTED, + ids=[ + "no dataset", + "a dataset that the action does not name", + "min-scores that do not parse", + "runs_per_item 6", + "runs_per_item True", + "concurrency 0", + "a scorer named user_feedback", + "a scorer with the name of an evaluator", + "evaluators as one string", + ], +) +def test_raises_before_any_request( + monkeypatch: pytest.MonkeyPatch, + api: FakeApi, + env: dict[str, str], + options: dict[str, Any], + error: type[Exception], + message: str, +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(error) as caught: + experiment(api, **options) + + assert str(caught.value).startswith(message) + assert api.requests == [] + + +def test_without_an_api_key_it_raises_missing_api_key(api: FakeApi) -> None: + with pytest.raises(ExperimentError) as caught: + experiment(api, api_key="") + + assert caught.value.code == "missing_api_key" + assert api.requests == [] + + +def test_the_task_gets_each_run_of_an_item_of_the_split_with_its_run_index() -> None: + seen: list[ExperimentItem] = [] + + def task(item: ExperimentItem) -> str: + seen.append(item) + return "ok" + + with fake_api(items=[dataset_item(7, "test")]) as api: + experiment(api, split="test", runs_per_item=2, concurrency=1, task=task) + + assert seen == [ + ExperimentItem( + id=item_id(7), + number=7, + input={"question": "q7"}, + expected_output="a7", + metadata={"source": "test"}, + tags=("golden",), + split="test", + run_index=index, + ) + for index in (0, 1) + ] + assert api.requests[0].query["split"] == "test" + assert api.requests[1].body["split"] == "test" + + +def test_run_experiment_rejects_an_async_task(api: FakeApi) -> None: + async def task(item: ExperimentItem) -> str: + return "ok" + + with pytest.raises(TypeError, match="task is async: use run_experiment_async"): + experiment(api, task=task) + assert api.requests == [] + + +async def test_run_experiment_async_runs_async_tasks_and_scorers_at_the_same_time( + make: MakeClient, +) -> None: + memory = make(environment="experiment") + running = 0 + most = 0 + + async def task(item: ExperimentItem) -> str: + nonlocal running, most + running += 1 + most = max(most, running) + with start_span("chat gpt-4o", type="generation"): + await asyncio.sleep(0.05) + running -= 1 + return f"r{item.number}" + + async def exact(output: Any, item: ExperimentItem) -> bool: + await asyncio.sleep(0) + return output == f"r{item.number}" + + with fake_api(items=[dataset_item(number) for number in range(1, 9)]) as api: + outcome = await run_experiment_async( + api_key="td_live_test", + base_url=api.url, + dataset="support-golden", + task=task, + scorers={"exact": exact}, + concurrency=4, + ) + + assert outcome.status == "completed" + assert most == 4 + posted = sorted( + (run for items in api.posts() for run in items), key=lambda run: str(run["itemId"]) + ) + assert [run["scores"] for run in posted] == [ + [{"name": "exact", "value": 1, "label": "pass"}] + ] * 8 + chats = [ + span for span in memory.span_exporter.get_finished_spans() if span.name == "chat gpt-4o" + ] + by_trace = {f"{span.context.trace_id:032x}": span for span in chats} + for run in posted: + attributes = dict(by_trace[run["traceId"]].attributes or {}) + assert attributes["td.dataset.item_id"] == run["itemId"] diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py new file mode 100644 index 0000000..44d28de --- /dev/null +++ b/sdks/python/tests/test_scores.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Any + +import pytest + +from telemetry_dev import ScoreError, score, score_async + +from .conftest import MakeClient +from .fake_api import ApiRequest, FakeApi, Reply, fake_api + +TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736" +SPAN_ID = "00f067aa0ba902b7" + + +def replying(reply: Reply) -> Generator[FakeApi]: + def answer(request: ApiRequest) -> Reply: + return reply + + with fake_api(override=answer) as api: + yield api + + +@pytest.fixture +def accepted() -> Generator[FakeApi]: + yield from replying((202, {"id": "score-1"}, {})) + + +@pytest.fixture +def errors(make: MakeClient) -> list[ScoreError]: + caught: list[ScoreError] = [] + make(on_error=caught.append) + return caught + + +def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( + make: MakeClient, accepted: FakeApi +) -> None: + make(api_key="td_live_key", base_url=f"{accepted.url}/") + + assert ( + score( + name=" user_feedback ", + trace_id=TRACE_ID, + span_id=SPAN_ID, + value=0, + comment="Wrong refund amount", + ) + == "score-1" + ) + + [request] = accepted.requests + assert request.route == "POST /v1/scores" + assert request.headers["authorization"] == "Bearer td_live_key" + assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["content-type"] == "application/json" + assert request.body == { + "name": "user_feedback", + "traceId": TRACE_ID, + "spanId": SPAN_ID, + "value": 0, + "comment": "Wrong refund amount", + } + + +async def test_a_session_score_needs_no_trace_and_a_label_goes_trimmed(accepted: FakeApi) -> None: + returned = await score_async( + api_key="td_live_key", + base_url=accepted.url, + name="resolved", + session_id="chat-9", + label=" yes ", + ) + + assert returned == "score-1" + assert accepted.requests[0].body == {"name": "resolved", "sessionId": "chat-9", "label": "yes"} + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"trace_id": None}, "Pass trace_id or session_id"), + ( + {"trace_id": "0" * 32}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": TRACE_ID.upper()}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": None, "session_id": "s", "span_id": SPAN_ID}, + "Pass the trace_id of the span", + ), + ({"value": None}, "Pass a value or a label"), + ({"value": float("nan")}, "value must be a finite number"), + ({"value": True}, "value must be a finite number"), + ({"value": 10**400}, "value must be a finite number"), + ( + {"name": "user_feedback", "value": 1, "label": "up"}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ( + {"name": "user_feedback", "value": 0.5}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ({"value": 0, "label": "pass"}, "The label pass goes with the value 1"), + ({"value": 1, "label": "fail"}, "The label fail goes with the value 0"), + ({"name": " "}, "name must be a string of 1 to 64 characters"), + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + ({"name": "\U0001f600" * 33}, "name must be a string of 1 to 64 characters"), + ({"comment": "x" * 4001}, "comment must be a string of at most 4000 characters"), + ], + ids=[ + "no trace and no session", + "the all-zero trace id of a span that did not record", + "an uppercase trace id", + "a span without its trace", + "no value and no label", + "a value that is not finite", + "a bool value", + "an int too large for a float", + "a label for user feedback", + "user feedback of 0.5", + "a pass of 0", + "a fail of 1", + "a blank name", + "a name of 66 UTF-16 units", + "a comment of 4001 characters", + ], +) +def test_reports_invalid_score_and_sends_nothing( + errors: list[ScoreError], accepted: FakeApi, options: dict[str, Any], message: str +) -> None: + arguments: dict[str, Any] = { + "api_key": "td_live_key", + "base_url": accepted.url, + "name": "helpful", + "trace_id": TRACE_ID, + "value": 1, + } + arguments.update(options) + + assert score(**arguments) is None + assert accepted.requests == [] + assert [(error.code, error.message) for error in errors] == [("invalid_score", message)] + + +FAILURES: list[tuple[Reply, str, int | None]] = [ + ( + (403, {"error": {"code": "insufficient_scope", "message": "No scores"}}, {}), + "insufficient_scope", + 403, + ), + ((503, {}, {"retry-after": "1"}), "http_503", 503), + ((202, {}, {}), "invalid_response", None), +] + + +@pytest.mark.parametrize( + ("reply", "code", "status"), + FAILURES, + ids=["the API error", "a status with no API error", "a body with no id"], +) +def test_reports_the_failure_tries_once_and_returns_none( + errors: list[ScoreError], reply: Reply, code: str, status: int | None +) -> None: + for api in replying(reply): + assert ( + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + is None + ) + assert len(api.requests) == 1 + + assert [(error.code, error.status) for error in errors] == [(code, status)] + + +def test_without_an_api_key_it_reports_missing_api_key( + errors: list[ScoreError], accepted: FakeApi +) -> None: + score(api_key="", base_url=accepted.url, name="helpful", trace_id=TRACE_ID, value=1) + + assert accepted.requests == [] + assert [error.code for error in errors] == ["missing_api_key"] + + +def test_does_not_follow_a_redirect_so_the_key_stays_with_the_api_host( + errors: list[ScoreError], +) -> None: + with fake_api() as other: + + def redirect(request: ApiRequest) -> Reply: + return 307, {}, {"location": f"{other.url}/v1/scores"} + + with fake_api(override=redirect) as api: + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + + assert other.requests == [] + assert [(error.code, error.status) for error in errors] == [("http_307", 307)] From 968a45e0eeff183d2409c0336c6328ac07a2e36d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:57 +0000 Subject: [PATCH 04/32] feat(sdk): add the experiment action for pull requests actions/experiment is a composite action. It runs the eval script of the user, a .ts, .js, or .py file or a command, with the dataset and the min-scores inputs in the variables that the SDKs read. Then it writes the result to the job summary and to one comment on the pull request, and it fails the job when the script exits with an error. The inputs go to the scripts through env, never into a run line. The results file has mode 0600, and the comment escapes the names. A token that cannot write the comment gives a warning. CI now runs shellcheck on the action script. --- .github/workflows/ci.yml | 2 +- actions/experiment/README.md | 84 +++++ actions/experiment/action.yml | 82 +++++ actions/experiment/report.mjs | 227 +++++++++++++ actions/experiment/run.sh | 49 +++ packages/sdk/tests/experiment-action.test.ts | 315 +++++++++++++++++++ 6 files changed, 758 insertions(+), 1 deletion(-) create mode 100644 actions/experiment/README.md create mode 100644 actions/experiment/action.yml create mode 100644 actions/experiment/report.mjs create mode 100755 actions/experiment/run.sh create mode 100644 packages/sdk/tests/experiment-action.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed6093f..d56afe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: with: persist-credentials: false - run: bash .github/scripts/release_test.sh - - run: shellcheck .github/scripts/*.sh + - run: shellcheck .github/scripts/*.sh actions/experiment/*.sh typescript: runs-on: ubuntu-latest diff --git a/actions/experiment/README.md b/actions/experiment/README.md new file mode 100644 index 0000000..77f5a72 --- /dev/null +++ b/actions/experiment/README.md @@ -0,0 +1,84 @@ +# telemetry.dev experiment action + +This composite action runs your eval script in a GitHub Actions workflow. The script calls +`runExperiment` (`@telemetry-dev/sdk`) or `run_experiment` (`telemetry-dev`). The action writes the +result to the job summary and to a comment on the pull request. The job fails when the check of the +pass marks fails, when the experiment does not complete, or when the script exits with an error. + +```yaml +on: pull_request + +permissions: + contents: read + pull-requests: write + +jobs: + evals: + runs-on: ubuntu-latest + env: + TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 24 + - run: npm ci + - uses: telemetry-dev/telemetry/actions/experiment@main + with: + dataset: support-golden + script: evals/reply.eval.ts + min-scores: correctness=0.9 +``` + +`evals/reply.eval.ts` runs the experiment: + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init(); + +await runExperiment({ + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness"], +}); +``` + +Pin the action to a commit SHA of this repository for a run that does not change when `main` +changes. + +## Inputs + +| Input | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with `npx --yes tsx`, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | + +The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the +traces of the items stay out of your production pages. `CI` is `true` in GitHub Actions, thus the +SDK runs in CI mode: it records the repository, the commit, the run, and the pull request with +the experiment, and it exits with an error when the check fails. + +## Outputs + +| Output | Description | +| -------------- | --------------------------------------------------------------------- | +| `check-status` | `passed`, `failed`, or `none`, for the last experiment of the script. | +| `url` | The page of the last experiment in telemetry.dev. | + +## The comment + +The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or +`command`. A later push changes that comment. A pull request from a fork gets a token that cannot +write comments: the action then logs a warning, and the job summary still has the result. + +The comment and the summary show the name of each experiment, its status, the count of the items +that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK +writes the result for the action to a file in `RUNNER_TEMP` with mode `0600`. diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml new file mode 100644 index 0000000..19a774c --- /dev/null +++ b/actions/experiment/action.yml @@ -0,0 +1,82 @@ +name: telemetry.dev experiment +description: Run an experiment of your code on a telemetry.dev dataset, check its scores, and post the result on the pull request. +author: telemetry.dev +branding: + icon: check-circle + color: gray-dark + +inputs: + script: + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with tsx, a .js script with node, and a .py script with python. Give script or command. + required: false + command: + description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. + required: false + dataset: + description: The dataset to run on. When the script also names a dataset, the two must be the same. + required: false + min-scores: + description: Pass marks, as name=value pairs that commas or new lines separate, for example `correctness=0.9`. A pass rate is 0 to 1. They win over the minScores of the script for the same name. + required: false + api-key: + description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. + required: false + base-url: + description: The telemetry.dev API. Default https://ingest.telemetry.dev. + required: false + working-directory: + description: The directory that the script or the command runs in. + required: false + default: "." + comment: + description: Set to false to write the job summary only, with no comment on the pull request. + required: false + default: "true" + github-token: + description: The token that writes the comment. It needs the pull-requests write permission. + required: false + default: ${{ github.token }} + +outputs: + check-status: + description: passed, failed, or none, for the last experiment of the script. + value: ${{ steps.report.outputs.check-status }} + url: + description: The page of the last experiment in telemetry.dev. + value: ${{ steps.report.outputs.url }} + +runs: + using: composite + steps: + - id: run + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_MIN_SCORES: ${{ inputs.min-scores }} + INPUT_API_KEY: ${{ inputs.api-key }} + INPUT_BASE_URL: ${{ inputs.base-url }} + run: bash "$GITHUB_ACTION_PATH/run.sh" + + - id: report + if: always() + shell: bash + env: + TELEMETRY_DEV_EXPERIMENT_RESULTS: ${{ steps.run.outputs.results }} + EXPERIMENT_EXIT_CODE: ${{ steps.run.outputs.exit-code }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_COMMENT: ${{ inputs.comment }} + GITHUB_TOKEN: ${{ inputs.github-token }} + run: node "$GITHUB_ACTION_PATH/report.mjs" + + - if: always() && steps.run.outputs.exit-code != '0' + shell: bash + env: + EXIT_CODE: ${{ steps.run.outputs.exit-code }} + run: | + echo "::error title=telemetry.dev experiment::The experiment script exited with code ${EXIT_CODE:-unknown}." + exit "${EXIT_CODE:-1}" diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs new file mode 100644 index 0000000..67ce615 --- /dev/null +++ b/actions/experiment/report.mjs @@ -0,0 +1,227 @@ +// Writes the job summary of the experiment action, and the comment with the result on the pull +// request. runExperiment and run_experiment add one line of JSON to the results file for each +// experiment: the body of GET /v1/experiments/{id}. +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync } from "node:fs"; + +const RATE_KINDS = new Set(["pass_fail", "up_down"]); +/** Pages of 100 comments that the action reads to find its comment from an earlier push. */ +const MAX_COMMENT_PAGES = 10; +const STATUS = { completed: "Completed", failed: "Failed", stopped: "Stopped" }; + +const env = process.env; +const exitCode = Number.parseInt(env.EXPERIMENT_EXIT_CODE ?? "", 10); +const results = readResults(env.TELEMETRY_DEV_EXPERIMENT_RESULTS); +// The comment of this job and input, so that a new push changes it and a second step adds its own. +const key = createHash("sha256") + .update( + [ + env.GITHUB_WORKFLOW, + env.GITHUB_JOB, + env.INPUT_DATASET, + env.INPUT_SCRIPT, + env.INPUT_COMMAND, + ].join("\0"), + ) + .digest("hex") + .slice(0, 12); +const marker = ``; +const report = render(results, exitCode); + +if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, `${report}\n`); + +const last = results.at(-1); + +if (env.GITHUB_OUTPUT) { + appendFileSync( + env.GITHUB_OUTPUT, + `check-status=${last?.checkStatus ?? "none"}\nurl=${last ? (httpsUrl(last.url) ?? "") : ""}\n`, + ); +} + +if (env.INPUT_COMMENT !== "false") { + await comment(`${marker}\n${report}`).catch((error) => { + // A pull request from a fork gets a token that cannot write comments. + console.log( + `::warning title=telemetry.dev experiment::The comment on the pull request failed: ${error.message}`, + ); + }); +} + +function readResults(path) { + if (!path) return []; + + let text; + + try { + text = readFileSync(path, "utf8"); + } catch { + return []; + } + + return text + .split("\n") + .filter((line) => line.trim() !== "") + .flatMap((line) => { + try { + const result = JSON.parse(line); + + return typeof result === "object" && result !== null && typeof result.name === "string" + ? [result] + : []; + } catch { + return []; + } + }); +} + +function render(experiments, code) { + const lines = ["### telemetry.dev experiment", ""]; + + if (experiments.length === 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code} before it wrote a result. The log of the step has the error.`, + ); + } + + for (const experiment of experiments) { + const url = httpsUrl(experiment.url); + const name = url ? `[${text(experiment.name)}](${url})` : text(experiment.name); + const total = Number(experiment.itemsTotal) || 0; + const failed = Number(experiment.itemsFailed) || 0; + const ran = Math.max((Number(experiment.itemsDone) || 0) - failed, 0); + + lines.push( + `**${name}**: ${outcome(experiment)}. ${ran} of ${total} items ran${failed > 0 ? `, ${failed} failed` : ""}.`, + ); + + if (typeof experiment.error === "string" && experiment.error !== "") { + lines.push("", `> ${text(experiment.error)}`); + } + + const scores = Array.isArray(experiment.scores) ? experiment.scores : []; + + if (scores.length > 0) { + lines.push( + "", + "| Score | Value | Change | Pass mark | Result |", + "| --- | --- | --- | --- | --- |", + ); + + for (const score of scores) { + lines.push( + `| ${text(score.name)} | ${value(score.kind, score.value)} | ${change(score.kind, score.delta)} | ${value(score.kind, score.minScore)} | ${score.passed === true ? "Passed" : score.passed === false ? "Failed" : "—"} |`, + ); + } + } + + lines.push(""); + } + + if (code !== 0 && experiments.length > 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code}.`, + "", + ); + } + + return lines.join("\n").trimEnd(); +} + +function outcome(experiment) { + if (experiment.status !== "completed") + return STATUS[experiment.status] ?? text(String(experiment.status)); + + if (experiment.checkStatus === "failed") return "Check failed"; + + return experiment.checkStatus === "passed" ? "Check passed" : "Completed"; +} + +function value(kind, number) { + if (typeof number !== "number") return "—"; + + return RATE_KINDS.has(kind) + ? `${Math.round(number * 100)}%` + : String(Math.round(number * 100) / 100); +} + +function change(kind, delta) { + if (typeof delta !== "number") return "—"; + + if (delta === 0) return "0"; + + const size = RATE_KINDS.has(kind) ? `${Math.abs(delta)} pts` : String(Math.abs(delta)); + + return `${delta > 0 ? "↑" : "↓"} ${size}`; +} + +/** Text that cannot break the table or add HTML to the comment. */ +function text(value) { + return String(value) + .replace(/[\r\n]+/g, " ") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\|/g, "\\|") + .replace(/([[\]()*_`])/g, "\\$1"); +} + +function httpsUrl(url) { + return typeof url === "string" && /^https:\/\/[^\s()<>]+$/.test(url) ? url : undefined; +} + +async function comment(body) { + const pull = pullRequestNumber(); + + if (pull === undefined || !env.GITHUB_TOKEN || !env.GITHUB_REPOSITORY) return; + + const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); + const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + let existing; + + for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { + const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); + + existing = comments.find( + (item) => typeof item.body === "string" && item.body.startsWith(marker), + ); + + if (comments.length < 100) break; + } + + if (existing) await github("PATCH", `${issue}/comments/${existing.id}`, { body }); + else await github("POST", `${issue}/${pull}/comments`, { body }); +} + +function pullRequestNumber() { + if (!env.GITHUB_EVENT_PATH) return undefined; + + try { + const number = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")).pull_request?.number; + + return Number.isSafeInteger(number) && number > 0 ? number : undefined; + } catch { + return undefined; + } +} + +async function github(method, url, body) { + const headers = { + accept: "application/vnd.github+json", + authorization: `Bearer ${env.GITHUB_TOKEN}`, + "x-github-api-version": "2022-11-28", + }; + const res = await fetch(url, { + method, + ...(body + ? { headers: { ...headers, "content-type": "application/json" }, body: JSON.stringify(body) } + : { headers }), + redirect: "error", + signal: AbortSignal.timeout(15_000), + }); + + if (!res.ok) + throw new Error(`GitHub returned ${res.status} for ${method} ${new URL(url).pathname}`); + + return res.json(); +} diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh new file mode 100755 index 0000000..9d6f1c2 --- /dev/null +++ b/actions/experiment/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Runs the experiment script of the experiment action. A failed script does not fail this step: +# the report step reads the exit code from the output exit-code, and the last step exits with it. +set -uo pipefail + +script=${INPUT_SCRIPT:-} +command=${INPUT_COMMAND:-} + +fail() { + echo "::error title=telemetry.dev experiment::$1" + echo "exit-code=2" >>"$GITHUB_OUTPUT" + exit 0 +} + +if [ -n "$script" ] && [ -n "$command" ]; then fail "Give script or command, not the two."; fi +if [ -z "$script" ] && [ -z "$command" ]; then fail "Give the script or the command that runs the experiment."; fi + +if [ -n "${INPUT_API_KEY:-}" ]; then + echo "::add-mask::$INPUT_API_KEY" + export TELEMETRY_DEV_API_KEY=$INPUT_API_KEY +fi +if [ -z "${TELEMETRY_DEV_API_KEY:-}" ]; then fail "Set the api-key input or the TELEMETRY_DEV_API_KEY variable."; fi +if [ -n "${INPUT_BASE_URL:-}" ]; then export TELEMETRY_DEV_BASE_URL=$INPUT_BASE_URL; fi +if [ -n "${INPUT_DATASET:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_DATASET=$INPUT_DATASET; fi +if [ -n "${INPUT_MIN_SCORES:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_MIN_SCORES=$INPUT_MIN_SCORES; fi +# The traces of the items stay out of production unless the job names an environment. +export TELEMETRY_DEV_ENVIRONMENT=${TELEMETRY_DEV_ENVIRONMENT:-experiment} + +# mktemp makes the file with mode 0600, in a directory of this job. +results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail "Could not make the results file." +echo "results=$results" >>"$GITHUB_OUTPUT" +export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results + +code=0 +if [ -n "$command" ]; then + bash -c "$command" || code=$? +else + case "$script" in + *.ts | *.mts | *.cts | *.tsx) npx --yes tsx "$script" || code=$? ;; + *.js | *.mjs | *.cjs) node "$script" || code=$? ;; + *.py) + python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." + "$python" "$script" || code=$? + ;; + *) fail "The script must end in .ts, .js, or .py. For another runner, give a command." ;; + esac +fi + +echo "exit-code=$code" >>"$GITHUB_OUTPUT" diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts new file mode 100644 index 0000000..cb7be99 --- /dev/null +++ b/packages/sdk/tests/experiment-action.test.ts @@ -0,0 +1,315 @@ +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +// The experiment action: run.sh runs the eval script, and report.mjs writes the job summary and +// the comment on the pull request. +const action = fileURLToPath(new URL("../../../actions/experiment/", import.meta.url)); +const closers: (() => void)[] = []; + +afterEach(() => { + for (const close of closers.splice(0)) close(); +}); + +function workdir(): string { + return mkdtempSync(join(tmpdir(), "td-action-")); +} + +function outputs(path: string): Record { + return Object.fromEntries( + readFileSync(path, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split(/=(.*)/s).slice(0, 2) as [string, string]), + ); +} + +function runScript(env: Record, files: Record = {}) { + const dir = workdir(); + const output = join(dir, "output"); + writeFileSync(output, ""); + + for (const [name, content] of Object.entries(files)) writeFileSync(join(dir, name), content); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: dir, + encoding: "utf8", + env: { PATH: process.env.PATH, RUNNER_TEMP: dir, GITHUB_OUTPUT: output, ...env }, + }); + + return { dir, run, outputs: outputs(output) }; +} + +const result = { + id: "6f9619ff-8b86-4011-b42d-00c04fc964ff", + name: "reply-v13-gpt-4o", + status: "completed", + checkStatus: "failed", + itemsTotal: 120, + itemsDone: 120, + itemsFailed: 2, + error: null, + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 100, + failCount: 18, + baselineValue: 0.88, + delta: -3, + minScore: 0.9, + passed: false, + }, + { + name: "helpfulness", + kind: "number", + value: 4.436, + passCount: 0, + failCount: 0, + baselineValue: 4.1, + delta: 0.3, + minScore: null, + passed: null, + }, + ], + url: "https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff", +}; + +const summary = `### telemetry.dev experiment + +**[reply-v13-gpt-4o](https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff)**: Check failed. 118 of 120 items ran, 2 failed. + +| Score | Value | Change | Pass mark | Result | +| --- | --- | --- | --- | --- | +| correctness | 85% | ↓ 3 pts | 90% | Failed | +| helpfulness | 4.44 | ↑ 0.3 | — | — | + +The experiment script exited with code 1.`; + +interface GithubRequest { + method: string; + url: string; + body: unknown; +} + +/** The comment routes of the GitHub REST API for one pull request. */ +async function fakeGithub( + options: { comments?: { id: number; body: string }[]; status?: number } = {}, +) { + const requests: GithubRequest[] = []; + const server = createServer(async (req: IncomingMessage, res) => { + let raw = ""; + for await (const chunk of req) raw += chunk; + requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); + res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + closers.push(() => server.close()); + + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +} + +async function report(env: Record, lines: unknown[]) { + const dir = workdir(); + const results = join(dir, "results.jsonl"); + const stepSummary = join(dir, "summary.md"); + const output = join(dir, "output"); + const event = join(dir, "event.json"); + writeFileSync(results, lines.map((line) => `${JSON.stringify(line)}\n`).join("")); + writeFileSync(stepSummary, ""); + writeFileSync(output, ""); + writeFileSync(event, JSON.stringify({ pull_request: { number: 42 } })); + + const child = spawn("node", [join(action, "report.mjs")], { + env: { + PATH: process.env.PATH, + TELEMETRY_DEV_EXPERIMENT_RESULTS: results, + GITHUB_STEP_SUMMARY: stepSummary, + GITHUB_OUTPUT: output, + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: "acme/support", + GITHUB_TOKEN: "ghs_test", + GITHUB_WORKFLOW: "evals", + GITHUB_JOB: "reply", + INPUT_DATASET: "support-golden", + EXPERIMENT_EXIT_CODE: "1", + ...env, + }, + }); + let stdout = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + const code = await new Promise((resolve) => child.on("close", resolve)); + + return { code, stdout, summary: readFileSync(stepSummary, "utf8"), outputs: outputs(output) }; +} + +test("run.sh passes the inputs to the script and records its exit code without failing", () => { + const { + dir, + run, + outputs: written, + } = runScript({ + INPUT_COMMAND: `node -e 'const fs = require("node:fs"); fs.writeFileSync("env.json", JSON.stringify(process.env)); process.exit(3)'`, + INPUT_API_KEY: "td_live_secret", + INPUT_DATASET: "support-golden", + INPUT_MIN_SCORES: "correctness=0.9", + INPUT_BASE_URL: "https://ingest.test", + }); + + expect(run.status).toBe(0); + expect(run.stdout).toContain("::add-mask::td_live_secret"); + expect(written["exit-code"]).toBe("3"); + + const env = JSON.parse(readFileSync(join(dir, "env.json"), "utf8")); + expect(env).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_secret", + TELEMETRY_DEV_BASE_URL: "https://ingest.test", + TELEMETRY_DEV_EXPERIMENT_DATASET: "support-golden", + TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0.9", + TELEMETRY_DEV_ENVIRONMENT: "experiment", + TELEMETRY_DEV_EXPERIMENT_RESULTS: written.results, + }); + expect(written.results!.startsWith(dir)).toBe(true); + expect(statSync(written.results!).mode & 0o777).toBe(0o600); +}); + +test("run.sh keeps the environment of the job, and runs a .js script with node", () => { + const { run, outputs: written } = runScript( + { + INPUT_SCRIPT: "print-env.mjs", + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }, + { "print-env.mjs": "console.log(JSON.stringify(process.env));\n" }, + ); + + expect(written["exit-code"]).toBe("0"); + expect(JSON.parse(run.stdout)).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }); +}); + +test.each([ + [ + "script and command", + { INPUT_SCRIPT: "a.ts", INPUT_COMMAND: "true", TELEMETRY_DEV_API_KEY: "k" }, + "Give script or command, not the two.", + ], + [ + "no script and no command", + { TELEMETRY_DEV_API_KEY: "k" }, + "Give the script or the command that runs the experiment.", + ], + [ + "no API key", + { INPUT_COMMAND: "true" }, + "Set the api-key input or the TELEMETRY_DEV_API_KEY variable.", + ], + [ + "a script of another type", + { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, + "The script must end in .ts, .js, or .py. For another runner, give a command.", + ], +])("run.sh stops with an error for %s", (_, env, message) => { + const { run, outputs: written } = runScript(env); + + expect(run.stdout).toContain(`::error title=telemetry.dev experiment::${message}`); + expect(written["exit-code"]).toBe("2"); +}); + +test("report.mjs writes the summary, posts it on the pull request, and sets the outputs", async () => { + const github = await fakeGithub(); + + const { + code, + summary: written, + outputs: set, + } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(written).toBe(`${summary}\n`); + expect(set).toEqual({ "check-status": "failed", url: result.url }); + expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "POST /repos/acme/support/issues/42/comments", + ]); + expect(github.requests[1]!.body).toEqual({ + body: expect.stringMatching( + /^\n### telemetry\.dev experiment\n/, + ), + }); + expect((github.requests[1]!.body as { body: string }).body.endsWith(summary)).toBe(true); +}); + +test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { + const first = await fakeGithub(); + await report({ GITHUB_API_URL: first.url }, [result]); + const own = { id: 8, body: (first.requests[1]!.body as { body: string }).body }; + + const other = await fakeGithub({ comments: [own] }); + await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); + + const again = await fakeGithub({ comments: [{ id: 7, body: "Looks good" }, own] }); + await report({ GITHUB_API_URL: again.url }, [result]); + + expect(other.requests.map((request) => request.method)).toEqual(["GET", "POST"]); + expect(again.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "PATCH /repos/acme/support/issues/comments/8", + ]); +}); + +test("report.mjs warns and succeeds when the token cannot write the comment", async () => { + const github = await fakeGithub({ status: 403 }); + + const { code, stdout } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(stdout).toContain( + "::warning title=telemetry.dev experiment::The comment on the pull request failed: GitHub returned 403", + ); +}); + +test("report.mjs reports a script that ended before it wrote a result, and comment: false posts nothing", async () => { + const github = await fakeGithub(); + + const { summary: written, outputs: set } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "1" }, + [], + ); + + expect(github.requests).toEqual([]); + + expect(written).toBe( + "### telemetry.dev experiment\n\nThe experiment script exited with code 1 before it wrote a result. The log of the step has the error.\n", + ); + expect(set).toEqual({ "check-status": "none", url: "" }); +}); + +test("report.mjs escapes names, so that they cannot break the table or add HTML", async () => { + const github = await fakeGithub(); + const { summary: written } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "0" }, + [ + { + ...result, + name: "|*bold*", + url: "javascript:alert(1)", + scores: [{ ...result.scores[0], name: "a|b\nc" }], + }, + ], + ); + + expect(written).toContain("**<img src=x>\\|\\*bold\\***: Check failed."); + expect(written).toContain("| a\\|b c | 85% |"); + expect(written).not.toContain("javascript:"); +}); From 473a5293b699201632b35acc1d3a868807eed7c8 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:28:10 +0000 Subject: [PATCH 05/32] docs(sdk): document experiments, scores, and the experiment action The READMEs of the two core SDKs get the Experiments and the Scores sections. They tell which data goes to telemetry.dev when output capture is off: the outputs, the task errors, the scorer reasons, and the metadata. The conformance document gets the experiment attributes and the wire contract of the two SDKs: the requests, the retry rules, the limits, the score rules, CI mode, and the variables of the action. --- docs/sdk-conformance.md | 88 +++++++++++++++++++++++++++++++ packages/sdk/README.md | 110 +++++++++++++++++++++++++++++++++++++++ sdks/python/README.md | 111 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 309 insertions(+) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dc..e81153b 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -44,6 +44,10 @@ repository, not this SDK workspace. Paths below refer to that product repository | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | | variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| `runExperiment` / `run_experiment` → each span and log record of an item run | `td.experiment.id` | the id that `POST /v1/experiments` returns; set like a propagated attribute, so the provider integration spans get it | +| the item of the run | `td.dataset.item_id` | the `id` of the dataset item: its key, the same across dataset versions | +| the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | +| the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -178,6 +182,90 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. +## Experiments and scores + +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` +/ `score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` +header, the SDK header, and a JSON body. The redirect rule of getPrompt applies. An error has the +code of the `{ error: { code, message } }` body, or `http_`, or `missing_api_key`, +`timeout`, `network_error`, or `invalid_response`. + +An experiment run sends these requests in this sequence: + +1. `GET /v1/datasets/{name}?version=&split=&cursor=`. The first page names the `datasetVersion` + option, or no version. Each next page names the `version` of the first page and its + `nextCursor`. A page with a different version, a cursor that repeats, a cursor with no items, + or more than 10,000 items in all is `invalid_response`. A first page with no items is + `dataset_empty`, and no experiment starts. +2. `POST /v1/experiments` with `name`, `dataset`, `datasetVersion` (the version of the first + page), `split`, `evaluators`, `minScores`, `runsPerItem`, `ci`, and `metadata`. Keys without a + value are left out. +3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item + run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, + `null` for no value, cut to 100,000 UTF-16 units), `traceId` (when the span records), + `latencyMs`, and, for a task that failed, `error` (`: `, cut to 4,000 UTF-16 + units) or, for a task that succeeded, `scores`. The posts go one at a time, in order. A post + that fails stops the run: no item starts after it, and the run fails when the running items end. +4. `POST /v1/experiments/{id}/complete`, after `flush()`. +5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is + `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). + +Retries: the GET requests, the item posts, and the complete request try 3 times in all after a +timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that +makes the experiment tries again only after a 429 or a 503. A retry waits for the `Retry-After` of +a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 second and then 2 seconds. +`POST /v1/scores` makes one request, with no retry. The TypeScript request timeouts abort the +request: 30 seconds for the experiment requests and 10 seconds for a score. The Python timeouts +apply to each socket operation of urllib. + +Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). +`concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names +of 1 to 64 UTF-16 units. `scorers` has at most 20 scorers with trimmed names of 1 to 64 UTF-16 +units, and no scorer has the name `user_feedback` or the name of an evaluator of the run in a +different case. `minScores` / `min_scores` has at most 20 trimmed names, with finite numbers. A +Python bool is not an integer or a number for these options. Option errors are `TypeError` and +`RangeError` (TypeScript) or `TypeError` and `ValueError` (Python), before any request. + +Scorers run after the span of the item ends, one after the other. A result of `true` or `false` +is `{ value: 1, label: "pass" }` or `{ value: 0, label: "fail" }`. A number is the `value`. A +string is the trimmed `label`. An object or mapping gives `value` (a boolean there is a pass or a +fail), the trimmed `label`, and `reason` (cut to 4,000 UTF-16 units). `null` / `None` gives no +score. A result that the score rules below reject, or a scorer that throws, gives no score and +sends an error with the code `scorer_failed` to `onError` / `on_error`. + +Score rules, for `score` and for scorer results: the trimmed name has 1 to 64 UTF-16 units. A +value is a finite number. A label has 1 to 64 UTF-16 units after the trim. A score has a value, a +label, or the two. `user_feedback` has the value 0 or 1 and no label. The label `pass` goes with no +value or 1, and `fail` with no value or 0. For `score`: a `traceId` is 32 lowercase hex digits and +not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs a `traceId`, a +`sessionId` has 1 to 1,024 UTF-16 units, and a `comment` has at most 4,000. A score needs a +`traceId` or a `sessionId`. A score that breaks a rule is `invalid_score`, with no request. `score` +never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / +`on_error`. + +CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: + +- In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 + characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, +runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units. A URL must be `https://` + with no space, at most 2,048 UTF-16 units. The run URL is + `//actions/runs/`. Keys without a value are left + out. +- After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of + `failed` is `check_failed`. The error has the `result`. + +The experiment action sets these variables: `TELEMETRY_DEV_EXPERIMENT_DATASET` is the dataset when +the `dataset` option has no value, and a different `dataset` option is a `TypeError`. +`TELEMETRY_DEV_EXPERIMENT_MIN_SCORES` holds `name=value` pairs that commas or new lines separate. +The value must match `-?\d+(\.\d+)?` (ASCII digits), else it is a `TypeError`. These marks win over +the `minScores` option for the same name. When `TELEMETRY_DEV_EXPERIMENT_RESULTS` names a file, the +SDK adds the JSON of the `GET /v1/experiments/{id}` body to it as one line. + +The SDKs print one line when the experiment starts and the result when it ends. A run with no +enabled client, or in an environment other than `experiment`, logs a warning. + ## The shared scenario Both e2e runners emit exactly this (under diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef1..c60c5ec 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -182,6 +182,113 @@ and TanStack AI integrations, and from other OpenTelemetry tracers, do not get t A model call outside `withPrompt` does not get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`runExperiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init({ environment: "experiment" }); + +const result = await runExperiment({ + name: "reply-v13-gpt-4o", + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness", "helpfulness"], + scorers: { short: (output) => String(output).length < 600 }, + minScores: { correctness: 0.9 }, +}); +``` + +- runExperiment reads the newest version of the dataset, or `datasetVersion`, 100 items at a time. + With `split`, it runs only the items of that split. Then it makes the experiment, with a name + that telemetry.dev picks when you give no `name`. +- `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, + `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A task that + throws fails only its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. `runsPerItem` (1 to 5, default + 1. runs each item more than one time. The pass rate and the mean of each score use all runs. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI integrations do not + get these attributes. +- Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out + the `experiment` environment by default. In a different environment, runExperiment logs a + warning. Without a client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `true` or `false` for a pass or a fail, a number, a label, or an object with + `value`, `label`, and `reason`. A scorer that throws, or returns a score that is not valid, + gives no score for that item and reports a `scorer_failed` error through `onError`. A scorer + cannot have the name of an evaluator of the run, or the name `user_feedback`. +- runExperiment sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns the result: `status`, + `checkStatus`, item counts, `scores` with the value, the baseline value, the change, and the + pass mark of each score, and the `url` of the experiment page. +- `minScores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request rejects with an `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a 429 + or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, + the experiment does not complete in telemetry.dev. +- The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same + API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` + options. + +The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +also when `captureOutput` is off, because they are the result of the experiment. The capture +options and `mask` apply only to the spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci: true`, runExperiment rejects when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives runExperiment the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```ts +import { score, startActiveSpan } from "@telemetry-dev/sdk"; + +// When the app replies, keep the trace id of the reply with the message: +const reply = await startActiveSpan("reply", async (span) => ({ + text: await replyToCustomer(message), + traceId: span.traceId, +})); + +// Later, when the user clicks thumbs down: +await score({ name: "user_feedback", traceId: reply.traceId, value: 0, comment: "Wrong amount" }); +``` + +- Give `traceId`, `sessionId`, or the two, and a `spanId` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, score never rejects. It resolves with the id of the score, or with + `undefined` after it reports a `ScoreError` through `onError`. It checks the score before it + sends it (`invalid_score`). The trace id of a span from a client that is not initialized is all + zeros, and score reports it as `invalid_score`. +- score makes one request, with no retry, and waits 10 seconds at most. The key needs the + `scores:write` scope. A browser must not hold the key, so call score from your server. + ## API - `init(options?)` — create the client (isolated OTel TracerProvider by default). @@ -203,6 +310,9 @@ A model call outside `withPrompt` does not get the attributes, even when it send `gen_ai.conversation.id`, and `td.metadata.*` on every span and log record in scope. - `getPrompt(name, options?)` — fetch a prompt by label or version; see [Prompts](#prompts). - `withPrompt(prompt, fn)` — record the prompt on each model call in `fn`. +- `runExperiment(options)` — run your code on a dataset as an experiment; see + [Experiments](#experiments). +- `score(options)` — record a score on a trace, a span, or a session; see [Scores](#scores). - `log(message, { level, eventName, attributes }?)` — log record correlated to the active trace. - `getTraceparent()` — W3C traceparent of the active span (pass to other services; accept it via `startSpan(name, { parent })`). diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d10..f7696a1 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -89,6 +89,8 @@ later `init()` generates a new ID. It is not persisted across processes. | `get_traceparent() -> str \| None` | W3C traceparent of the current context. | | `get_prompt(name, *, label=, version=, type="chat", cache_ttl_seconds=60, fallback=, api_key=, base_url=)` / `get_prompt_async(...)` | Fetch a prompt by label or version; see [Prompts](#prompts). | | `with_prompt(prompt)` | Context manager recording the prompt on each model call started inside. | +| `run_experiment(*, task, dataset=, name=, dataset_version=, split=, evaluators=, scorers=, min_scores=, runs_per_item=1, concurrency=4, metadata=, ci=, api_key=, base_url=)` / `run_experiment_async(...)` | Run your code on a dataset as an experiment; see [Experiments](#experiments). | +| `score(*, name, trace_id=, span_id=, session_id=, value=, label=, comment=, api_key=, base_url=)` / `score_async(...)` | Record a score on a trace, a span, or a session; see [Scores](#scores). | | `flush(timeout_s=10.0)` / `shutdown(timeout_s=10.0)` | Force-flush / tear down traces + logs + metrics. Shutdown also runs atexit unless `disable_atexit=True`. | | `TelemetrySpanProcessor` / `telemetry_dev.otel.create_telemetry_span_exporter` | Bring-your-own-OTel helpers (below). | | `MaskContext`, `Usage`, `SpanHandle`, `Client`, `NOT_GIVEN` | Supporting types. | @@ -255,6 +257,115 @@ other OpenTelemetry tracers do not get the attributes. A model call outside `wit get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`run_experiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```python +import telemetry_dev + +telemetry_dev.init(environment="experiment") + +result = telemetry_dev.run_experiment( + name="reply-v13-gpt-4o", + dataset="support-golden", + task=lambda item: reply_to_customer(item.input), + evaluators=["correctness", "helpfulness"], + scorers={"short": lambda output, item: len(output) < 600}, + min_scores={"correctness": 0.9}, +) +``` + +- `run_experiment` reads the newest version of the dataset, or `dataset_version`, 100 items at a + time. With `split`, it runs only the items of that split. Then it makes the experiment, with a + name that telemetry.dev picks when you give no `name`. +- `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, + `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a + different value becomes JSON, cut to 100,000 UTF-16 units. A task that raises fails only its + item, and the item records the type and the message of the exception. +- `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the + caller, so `propagate_attributes` around the call applies to each item. `runs_per_item` (1 to 5, + default 1) runs each item more than one time. The pass rate and the mean of each score use all + runs. +- `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, + and the requests go in worker threads. A task that is not async runs on the event loop, so use + `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. +- Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the + `experiment` environment by default. In a different environment, `run_experiment` logs a + warning. Without an enabled client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `True` or `False` for a pass or a fail, a number, a label, or a mapping with + `value`, `label`, and `reason`. A scorer that raises, or returns a score that is not valid, gives + no score for that item and sends a `scorer_failed` error to `on_error`. A scorer cannot have the + name of an evaluator of the run, or the name `user_feedback`. +- `run_experiment` sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns an `ExperimentResult`: + `status`, `check_status`, item counts, `scores` with the value, the baseline value, the change, + and the pass mark of each score, and the `url` of the experiment page. +- `min_scores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request raises `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a 429 + or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, + the experiment does not complete in telemetry.dev. +- The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. + +The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +also when `capture_output` is off, because they are the result of the experiment. The capture +options and `mask` apply only to the spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci=True`, `run_experiment` raises when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives `run_experiment` the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```python +import telemetry_dev + +# When the app replies, keep the trace id of the reply with the message: +with telemetry_dev.start_span("reply") as span: + text = reply_to_customer(message) + trace_id = f"{span.span.get_span_context().trace_id:032x}" + +# Later, when the user clicks thumbs down: +telemetry_dev.score(name="user_feedback", trace_id=trace_id, value=0, comment="Wrong amount") +``` + +- Give `trace_id`, `session_id`, or the two, and a `span_id` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, `score` never raises. It returns the id of the score, or `None` + after it sends a `ScoreError` to `on_error`. It checks the score before it sends it + (`invalid_score`). The trace id of a span from a client that is not enabled is all zeros, and + `score` reports it as `invalid_score`. +- `score` makes one request, with no retry. `score_async` runs it in a worker thread. The key + needs the `scores:write` scope. + ## Auto-metrics Ended spans automatically record histograms (DELTA temporality, exported every 60s): From 1d8ee43268e3acb782e443c8146558987e1887ef Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:26 +0000 Subject: [PATCH 06/32] fix(sdk): complete the experiment after a run stops send() called fetch as a method of the Api object. Workers and browsers reject a fetch call that has a different `this` object. Thus score and runExperiment did not operate there without a `fetch` option. send() calls fetch as a function, not as a method. When a run stops after runExperiment makes the experiment, it sends the complete request before it rejects. Examples are an item post or a page fetch that is not successful, and more than 10,000 items. The server then completes the experiment with its item runs, or sets the status to failed when it has none. An error of that request goes to onError, and runExperiment rejects with the first error. The create request tries again only after a 429. The server sends a 503 when its database work is longer than its time limit. That 503 can come after the commit, and a retry then makes a second experiment. Each run of an item gets its own copy of the item. Thus a task that changes its input does not change the next run. The SDK cuts a pull request title without half of a surrogate pair, because Postgres rejects a lone surrogate in jsonb. A score field that can be null can also be missing, as in the Python SDK. A result without checkStatus stays an invalid_response. score() without options reports invalid_score and does not reject. New tests do checks of the 4 MiB batch limit, the 10,000-item limit, the 30-minute wait, and the retry after a timeout or a network error. --- packages/sdk/README.md | 33 +-- packages/sdk/src/api.ts | 12 +- packages/sdk/src/ci.ts | 14 +- packages/sdk/src/experiments.ts | 116 ++++++----- packages/sdk/src/prompts.ts | 1 - packages/sdk/src/scores.ts | 24 ++- packages/sdk/tests/experiments.test.ts | 273 +++++++++++++++++++++++-- packages/sdk/tests/scores.test.ts | 25 +++ 8 files changed, 403 insertions(+), 95 deletions(-) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index c60c5ec..0edb8e2 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -208,15 +208,18 @@ const result = await runExperiment({ that telemetry.dev picks when you give no `name`. - `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string - stays as it is, and a different value becomes JSON, cut to 100,000 characters. A task that - throws fails only its item, and the item records the name and the message of the error. -- `concurrency` (1 to 100, default 4) items run at the same time. `runsPerItem` (1 to 5, default - 1. runs each item more than one time. The pass rate and the mean of each score use all runs. + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A value that JSON + cannot hold, such as an object that holds itself, gives no output. A task that throws fails only + its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. With `runsPerItem` (default 1, + at most 5), each item runs more than one time. Each run gets its own copy of the item, so a task + that changes its input does not change the next run. The pass rate and the mean of each score + use all runs. - Each item run is a trace of its own, with the span `experiment `. The SDK puts `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the - provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI integrations do not - get these attributes. + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and + TanStack AI integrations do not get these attributes. - Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out the `experiment` environment by default. In a different environment, runExperiment logs a warning. Without a client, the items record no traces and no cost. @@ -236,16 +239,20 @@ index>`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI inte `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server - skips an item run that it has. The request that makes the experiment goes again only after a 429 - or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, - the experiment does not complete in telemetry.dev. + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then + ends the experiment with the item runs that it got, or fails it when it got none. - The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` - options. + options. With `environment: "experiment"`, the key of the telemetry client must be for all + environments or for `experiment`: a key for another environment cannot send the traces of the + items, and the items then have no trace and no cost. The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev -also when `captureOutput` is off, because they are the result of the experiment. The capture -options and `mask` apply only to the spans of the run. +and stay with the experiment, because they are its result. `captureOutput`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. ### CI mode diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts index 8274701..573f780 100644 --- a/packages/sdk/src/api.ts +++ b/packages/sdk/src/api.ts @@ -34,8 +34,8 @@ interface RequestOptions { timeoutMs: number; /** * The server takes a repeat of this request at most once, so a retry after a timeout, a network - * error, or a 5xx is safe. Other requests try again only after a 429 or a 503: the server did - * not take the request then. + * error, or a 5xx is safe. Other requests try again only after a 429, which the rate limit + * answers before the server does any work. A 503 can come after the work committed. */ idempotent: boolean; /** Default 3. */ @@ -141,8 +141,12 @@ async function send( const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); + // A plain call, not a method call: Workers and browsers reject a fetch whose `this` is not the + // global object. + const { fetchImpl } = api; + try { - const res = await api.fetchImpl(url, { + const res = await fetchImpl(url, { method, headers: { authorization: `Bearer ${api.apiKey}`, @@ -193,7 +197,7 @@ async function send( } function retryable(status: number | undefined, idempotent: boolean): boolean { - if (status === 429 || status === 503) return true; + if (status === 429) return true; return idempotent && (status === undefined || status >= 500); } diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts index 3d7603e..33f13e9 100644 --- a/packages/sdk/src/ci.ts +++ b/packages/sdk/src/ci.ts @@ -141,12 +141,24 @@ function pullRequest( Number.isSafeInteger(pull.number) && (pull.number as number) > 0 ? (pull.number as number) : undefined, - title: typeof pull.title === "string" ? pull.title.slice(0, MAX_PR_TITLE_LENGTH) : undefined, + title: typeof pull.title === "string" ? cut(pull.title, MAX_PR_TITLE_LENGTH) : undefined, url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, commit: typeof head?.sha === "string" ? head.sha : undefined, }; } +/** + * `text` cut to `max` UTF-16 units, with no half of a surrogate pair at the end: the API stores the + * run as jsonb, and Postgres rejects a lone surrogate there. + */ +function cut(text: string, max: number): string { + if (text.length <= max) return text; + + const last = text.charCodeAt(max - 1); + + return text.slice(0, last >= 0xd800 && last <= 0xdbff ? max - 1 : max); +} + function httpsUrl(url: string): string | undefined { return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; } diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts index f3d6d4b..9bcc963 100644 --- a/packages/sdk/src/experiments.ts +++ b/packages/sdk/src/experiments.ts @@ -220,8 +220,10 @@ const RATE_KINDS = new Set(["pass_fail", "up_down"]); * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. * * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` - * when a request fails. A task that throws fails only its item. In CI mode, runExperiment rejects - * when the check fails, so the process exits with an error. + * when a request fails. When the run stops after it made the experiment, it first sends the + * complete request, so that telemetry.dev ends the experiment with the item runs that it has. A + * task that throws fails only its item. In CI mode, runExperiment rejects when the check fails, so + * the process exits with an error. */ export async function runExperiment(options: RunExperimentOptions): Promise { const env = resolveEnv(); @@ -266,23 +268,24 @@ export async function runExperiment(options: RunExperimentOptions): Promise 1 ? `, ${run.runsPerItem} runs each` : "") + - `. ${created.url}`, - ); + try { + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + await runItems(api, run, created, itemRuns(api, run, first)); + } catch (error) { + // Without the complete, the experiment would stay running: the server ends it with the item + // runs that it has, or fails it when it has none. + await complete(api, created.id).catch((cause: unknown) => reportError(api.onError, cause)); + + throw error; + } - await runItems(api, run, created, itemRuns(api, run, first)); // The spans of the items reach ingest before the server reads their cost. await flush(); - - await request( - api, - ExperimentError, - "POST", - `/v1/experiments/${encodeURIComponent(created.id)}/complete`, - { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, - ); + await complete(api, created.id); const { result, body } = await waitForResult(api, created.id); writeResult(env, body, api.onError); @@ -456,7 +459,10 @@ async function* itemRuns( } for (const item of page.items) { - for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) yield { ...item, runIndex }; + // A copy for each run, so that a task that changes its input does not change the next run. + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) { + yield { ...structuredClone(item), runIndex }; + } } if (page.nextCursor === null) return; @@ -496,7 +502,7 @@ async function fetchPage( /** * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that - * fails stops the run: no new item starts, and runExperiment rejects when the running items end. + * fails stops the run: no new item starts, and runItems rejects when the running items end. */ async function runItems( api: Authed, @@ -527,11 +533,11 @@ async function runItems( if (next.done) return; - const result = await runItem(run, experiment, next.value, api.onError); - - await batch.add(result).catch((error: unknown) => { - failure ??= { error }; - }); + await runItem(run, experiment, next.value, api.onError) + .then((result) => batch.add(result)) + .catch((error: unknown) => { + failure ??= { error }; + }); } }; @@ -733,15 +739,23 @@ function outputText(output: unknown): string | null { } function errorText(error: unknown): string { - if (error instanceof Error) return `${error.name}: ${error.message}`; - try { - return String(error); + return error instanceof Error ? `${error.name}: ${error.message}` : String(error); } catch { return "A value that cannot be converted to a string was thrown"; } } +function complete(api: Authed, id: string): Promise { + return request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); +} + /** Waits until the server evaluators finish and the experiment ends. */ async function waitForResult( api: Authed, @@ -896,6 +910,7 @@ function parseResult(body: unknown): ExperimentResult { typeof body.name !== "string" || typeof body.status !== "string" || typeof body.url !== "string" || + // Required: it decides the result of CI mode, so a missing one must not read as no check. !( body.checkStatus === null || body.checkStatus === "passed" || @@ -925,34 +940,33 @@ function parseResult(body: unknown): ExperimentResult { }; } -function parseScore(value: unknown): ExperimentScore | undefined { +/** A score of the result. A field that can be null can also be missing. */ +function parseScore(raw: unknown): ExperimentScore | undefined { + if (!isRecord(raw)) return undefined; + + const score = { + name: raw.name, + kind: raw.kind ?? null, + value: raw.value ?? null, + passCount: raw.passCount, + failCount: raw.failCount, + baselineValue: raw.baselineValue ?? null, + delta: raw.delta ?? null, + minScore: raw.minScore ?? null, + passed: raw.passed ?? null, + }; + const numbers = [score.value, score.baselineValue, score.delta, score.minScore]; + if ( - !isRecord(value) || - typeof value.name !== "string" || - !(value.kind === null || typeof value.kind === "string") || - typeof value.passCount !== "number" || - typeof value.failCount !== "number" + typeof score.name !== "string" || + !(score.kind === null || typeof score.kind === "string") || + typeof score.passCount !== "number" || + typeof score.failCount !== "number" || + !numbers.every((field) => field === null || typeof field === "number") || + !(score.passed === null || typeof score.passed === "boolean") ) { return undefined; } - const nullable = ["value", "baselineValue", "delta", "minScore"] as const; - - if (!nullable.every((key) => value[key] === null || typeof value[key] === "number")) { - return undefined; - } - - if (value.passed !== null && typeof value.passed !== "boolean") return undefined; - - return { - name: value.name, - kind: value.kind, - value: value.value as number | null, - passCount: value.passCount, - failCount: value.failCount, - baselineValue: value.baselineValue as number | null, - delta: value.delta as number | null, - minScore: value.minScore as number | null, - passed: value.passed, - }; + return score as ExperimentScore; } diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 7b4612d..2787b29 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -598,7 +598,6 @@ function keyFingerprint(apiKey: string): string { return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); } -/** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts index e600c2b..bb06d76 100644 --- a/packages/sdk/src/scores.ts +++ b/packages/sdk/src/scores.ts @@ -51,12 +51,14 @@ const TIMEOUT_MS = 10_000; * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. */ export async function score(options: ScoreOptions): Promise { - const api = resolveApi(options); + // A caller without types can pass no options, and score must not reject then either. + const fields: Partial = isRecord(options) ? options : {}; + const api = resolveApi(fields); try { - const name = typeof options.name === "string" ? options.name.trim() : options.name; - const label = typeof options.label === "string" ? options.label.trim() : options.label; - const problem = checkScore({ ...options, name, label }); + const name = typeof fields.name === "string" ? fields.name.trim() : fields.name; + const label = typeof fields.label === "string" ? fields.label.trim() : fields.label; + const problem = checkScore({ ...fields, name, label }); if (problem) throw new ScoreError("invalid_score", problem); @@ -69,13 +71,13 @@ export async function score(options: ScoreOptions): Promise const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { body: { - traceId: options.traceId, - spanId: options.spanId, - sessionId: options.sessionId, + traceId: fields.traceId, + spanId: fields.spanId, + sessionId: fields.sessionId, name, - value: options.value, + value: fields.value, label, - comment: options.comment, + comment: fields.comment, }, timeoutMs: TIMEOUT_MS, idempotent: false, @@ -131,8 +133,8 @@ export function checkValue( return undefined; } -function checkScore(options: ScoreOptions): string | undefined { - const { name, traceId, spanId, sessionId, value, label, comment } = options; +function checkScore(fields: Partial): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = fields; if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts index daa3214..5f555e3 100644 --- a/packages/sdk/tests/experiments.test.ts +++ b/packages/sdk/tests/experiments.test.ts @@ -21,6 +21,7 @@ interface Request { method: string; url: URL; body: any; + signal: AbortSignal | undefined; } const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; @@ -34,7 +35,7 @@ function datasetItem(number: number, split: string | null = null) { return { id: itemId(number), number, - input: { question: `q${number}` }, + input: { question: `q${number}` } as Record, expectedOutput: `a${number}`, metadata: { source: "test" }, tags: ["golden"], @@ -67,7 +68,7 @@ function fakeApi( items?: ReturnType[]; pageSize?: number; results?: Record[]; - override?: (request: Request) => Response | undefined; + override?: (request: Request) => Response | Promise | undefined; } = {}, ) { const items = options.items ?? [datasetItem(1), datasetItem(2)]; @@ -82,6 +83,7 @@ function fakeApi( method: init?.method ?? "GET", url, body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + signal: init?.signal ?? undefined, }; requests.push(request); @@ -381,6 +383,66 @@ test("a task that throws fails only its item", async () => { expect(passed).not.toHaveProperty("error"); }); +test("an output or an error that cannot be encoded does not stop the run", async () => { + const api = fakeApi({ items: [datasetItem(1), datasetItem(2), datasetItem(3)] }); + const cycle: Record = {}; + cycle.self = cycle; + const unreadable = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("no message"); + }, + }); + + const outcome = await experiment(api, { + task: (item) => { + if (item.number === 3) throw unreadable; + + return item.number === 1 ? cycle : 10n; + }, + // A lone surrogate, which UTF-8 cannot hold. + scorers: { tone: () => "\ud83d" }, + }); + + expect(outcome.status).toBe("completed"); + expect( + api + .posts() + .flat() + .sort((a: { itemId: string }, b: { itemId: string }) => a.itemId.localeCompare(b.itemId)) + .map((run: { output: unknown; error?: unknown; scores?: unknown }) => [ + run.output, + run.error, + run.scores, + ]), + ).toEqual([ + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, "A value that cannot be converted to a string was thrown", undefined], + ]); + expect(routes(api.requests)).toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("each run of an item starts from the dataset input, also when a task changes it", async () => { + const api = fakeApi({ + items: [{ ...datasetItem(1), input: { messages: [{ role: "user", content: "Hi" }] } }], + }); + const seen: number[] = []; + + await experiment(api, { + runsPerItem: 3, + concurrency: 1, + task: (item) => { + const messages = item.input.messages as unknown[]; + seen.push(messages.length); + messages.push({ role: "assistant", content: "Hello" }); + + return "ok"; + }, + }); + + expect(seen).toEqual([1, 1, 1]); +}); + test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { const errors: Error[] = []; setup({ environment: "experiment", onError: (error) => errors.push(error) }); @@ -418,14 +480,14 @@ test("scorers give pass/fail, numbers, labels, and reasons, and a bad result dro ]); }); -test("a failed post of items stops new items, never completes, and rejects with the API error", async () => { +test("a failed post of items stops new items, completes the experiment, and rejects with the API error", async () => { const api = fakeApi({ items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), override: (request) => request.url.pathname.endsWith("/items") ? Response.json( - { error: { code: "experiment_ended", message: "support-golden-v3 has ended." } }, - { status: 409 }, + { error: { code: "item_not_found", message: "support-golden v3 has no item" } }, + { status: 422 }, ) : undefined, }); @@ -441,10 +503,35 @@ test("a failed post of items stops new items, never completes, and rejects with }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(ExperimentError); - expect(error).toMatchObject({ code: "experiment_ended", status: 409 }); + expect(error).toMatchObject({ code: "item_not_found", status: 422 }); // The one worker waits for the post of the first 100, which fails, so no item starts after it. expect(started).toBe(100); - expect(routes(api.requests)).not.toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); + // The complete ends the experiment, which would else stay running. + expect(routes(api.requests).slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + ]); +}); + +test("a complete that fails after a failed run goes to onError, and the run rejects with the first error", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") || request.url.pathname.endsWith("/complete") + ? Response.json( + { error: { code: "invalid_api_key", message: "The API key was revoked" } }, + { status: 401 }, + ) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "invalid_api_key", + message: "The API key was revoked", + }); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); + expect(errors.map((error) => (error as ExperimentError).code)).toEqual(["invalid_api_key"]); }); test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { @@ -462,18 +549,98 @@ test("an item post that fails with 500 goes again, and the server skips the runs expect(api.posts()[1]).toEqual(api.posts()[0]); }); -test("a create that fails with 500 does not go again, so it cannot start two experiments", async () => { +test("a post of items holds at most 4 MiB of JSON", async () => { + const api = fakeApi({ items: Array.from({ length: 20 }, (_, index) => datasetItem(index + 1)) }); + + // 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + await experiment(api, { task: () => "中".repeat(100_000) }); + + expect(api.posts().map((items) => items.length)).toEqual([13, 7]); +}); + +test.each([ + ["10,000 items in pages run", 10_000, undefined], + ["10,001 items in pages stop the run", 10_001, "invalid_response"], +])("%s", async (_, count, code) => { const api = fakeApi({ - override: (request) => - request.method === "POST" && request.url.pathname === "/v1/experiments" - ? new Response("upstream", { status: 500 }) - : undefined, + items: Array.from({ length: count }, (_, index) => datasetItem(index + 1)), + pageSize: 5_000, }); - await expect(experiment(api)).rejects.toMatchObject({ code: "http_500", status: 500 }); - expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(1); + const outcome = await experiment(api, { concurrency: 20, task: () => "ok" }).catch( + (caught: unknown) => caught, + ); + + expect((outcome as ExperimentError).code).toBe(code); + expect(api.posts().flat()).toHaveLength(10_000); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); +}); + +test.each([ + ["a dataset page", "GET /v1/datasets/support-golden", "timeout", 2, undefined], + ["an item post", `POST /v1/experiments/${EXPERIMENT_ID}/items`, "network", 2, undefined], + ["the create", "POST /v1/experiments", "timeout", 1, "timeout"], + ["the create", "POST /v1/experiments", "network", 1, "network_error"], +])("a %s %s, after a %s, sends %i times in all", async (_, route, failure, sends, code) => { + vi.useFakeTimers(); + + try { + let failures = 1; + const api = fakeApi({ + override: (request) => { + if (`${request.method} ${request.url.pathname}` !== route || failures-- <= 0) { + return undefined; + } + + if (failure === "network") return Promise.reject(new TypeError("fetch failed")); + + // A server that never answers: the request ends when its timeout aborts it. + return new Promise((_, reject) => + request.signal!.addEventListener("abort", () => reject(request.signal!.reason)), + ); + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(60_000); + + expect(((await outcome) as ExperimentError).code).toBe(code); + expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(sends); + } finally { + vi.useRealTimers(); + } }); +test.each([ + ["500", () => new Response("upstream", { status: 500 }), "http_500"], + // The server answers 503 when its database work times out, which can be after the commit. + [ + "503", + () => + Response.json( + { error: { code: "evals_unavailable", message: "Evals are unavailable right now" } }, + { status: 503 }, + ), + "evals_unavailable", + ], +])( + "a create that fails with %s does not go again, so it cannot start two experiments", + async (status, reply, code) => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? reply() + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code, status: Number(status) }); + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + ]); + }, +); + test("a create that gets a 429 goes again after its Retry-After", async () => { let limited = 1; const api = fakeApi({ @@ -505,6 +672,38 @@ test("waits until the server evaluators end the experiment", async () => { ).toHaveLength(2); }); +test("gives up when the experiment does not end in 30 minutes, with the result", async () => { + vi.useFakeTimers(); + + try { + const polls: number[] = []; + const api = fakeApi({ + override: (request) => { + if (request.method === "GET" && request.url.pathname.startsWith("/v1/experiments/")) { + polls.push(Date.now()); + + return Response.json(result({ status: "running", itemsDone: 1 })); + } + + return undefined; + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(31 * 60_000); + const error = (await outcome) as ExperimentError; + + expect(error.code).toBe("timeout"); + expect(error.result?.status).toBe("running"); + // The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + const waited = polls.at(-1)! - polls[0]!; + expect(waited).toBeGreaterThan(30 * 60_000 - 5_000); + expect(waited).toBeLessThanOrEqual(30 * 60_000); + } finally { + vi.useRealTimers(); + } +}); + test.each([ [ "the API error", @@ -608,6 +807,35 @@ test.each([ await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); }); +test("a result score that leaves out its null fields reads them as null", async () => { + const api = fakeApi({ + results: [result({ scores: [{ name: "length", passCount: 0, failCount: 0 }] })], + }); + + const outcome = await experiment(api); + + expect(outcome.scores).toEqual([ + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ]); +}); + +test("a result without checkStatus is an invalid response, not a run with no check", async () => { + const { checkStatus: _, ...missing } = failedCheck; + const api = fakeApi({ results: [missing] }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); +}); + test("in CI mode an experiment that fails rejects with its error", async () => { vi.stubEnv("CI", "1"); const api = fakeApi({ @@ -666,6 +894,23 @@ test("in GitHub Actions the experiment records the pull request, its head commit expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); }); +test("a pull request title is cut to 256 UTF-16 units without half of an emoji", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + // GitHub counts the emoji as one character, thus the title fits its limit of 256. + writeFileSync(event, JSON.stringify({ pull_request: { title: `${"a".repeat(255)}😀` } })); + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "a1b2c3d"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + const api = fakeApi(); + + await experiment(api); + + expect(api.requests[1]!.body.ci.prTitle).toBe("a".repeat(255)); +}); + test.each([ [ "no dataset", diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts index d740b40..9b9dccc 100644 --- a/packages/sdk/tests/scores.test.ts +++ b/packages/sdk/tests/scores.test.ts @@ -85,6 +85,31 @@ test("posts the score with the key and base URL of init, and resolves with its i }); }); +test("calls the global fetch as a plain function, as Workers and browsers need", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", "td_live_key"); + // A strict function called as a method of an object gets that object as `this`. + vi.stubGlobal("fetch", function (this: unknown) { + if (this !== undefined && this !== globalThis) throw new TypeError("Illegal invocation"); + + return Promise.resolve(Response.json({ id: "score-3" }, { status: 202 })); + }); + + try { + await expect(score({ name: "helpful", traceId: TRACE_ID, value: 1 })).resolves.toBe("score-3"); + } finally { + vi.unstubAllGlobals(); + } +}); + +test("score with no options reports invalid_score and resolves undefined", async () => { + const errors = errorsOf(); + + await expect((score as (options?: ScoreOptions) => Promise)()).resolves.toBeUndefined(); + expect(errors.map((error) => [error.code, error.message])).toEqual([ + ["invalid_score", "name must be a string of 1 to 64 characters"], + ]); +}); + test("a session score needs no trace, and a label goes trimmed", async () => { const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); From bf7844f97df44c6d67aba99cd22dc10209de641d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:41 +0000 Subject: [PATCH 07/32] fix(sdk-python): complete the experiment after a run stops An output that JSON cannot hold stopped the full run outside the guard of its item. An example is a dict that holds itself. An exception with a __str__ that raises did the same. The SDK sends None as the output, as the TypeScript SDK does, and the name of the type as the error. A lone surrogate in an output or an error becomes U+FFFD. The request body is the JSON of JSON.stringify, which escapes a lone surrogate in a label. When a run stops after run_experiment makes the experiment, it sends the complete request before it raises. This also occurs after Ctrl-C and after a cancellation. An error of that request goes to on_error. An error of an item run stops all workers. The create request tries again only after a 429, as in the TypeScript SDK. Each run of an item gets a deep copy of the item. The requests for experiments, scores, and prompts send the User-Agent telemetry-dev-python/, because Cloudflare rejects Python-urllib. New tests do checks of the 4 MiB batch limit, the 10,000-item limit, the 30-minute wait, and the retry after a timeout or a dropped connection. --- sdks/python/README.md | 34 +- sdks/python/src/telemetry_dev/_api.py | 27 +- sdks/python/src/telemetry_dev/_ci.py | 4 +- sdks/python/src/telemetry_dev/_experiments.py | 110 ++++--- sdks/python/src/telemetry_dev/_prompts.py | 12 +- sdks/python/tests/fake_api.py | 36 ++- sdks/python/tests/test_experiments.py | 304 ++++++++++++++++-- sdks/python/tests/test_scores.py | 2 + 8 files changed, 440 insertions(+), 89 deletions(-) diff --git a/sdks/python/README.md b/sdks/python/README.md index f7696a1..bc53f67 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -283,19 +283,21 @@ result = telemetry_dev.run_experiment( name that telemetry.dev picks when you give no `name`. - `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a - different value becomes JSON, cut to 100,000 UTF-16 units. A task that raises fails only its - item, and the item records the type and the message of the exception. + different value becomes JSON, cut to 100,000 UTF-16 units. A value that JSON cannot hold, such + as a dict that holds itself, gives no output. A task that raises fails only its item, and the + item records the type and the message of the exception. - `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the - caller, so `propagate_attributes` around the call applies to each item. `runs_per_item` (1 to 5, - default 1) runs each item more than one time. The pass rate and the mean of each score use all - runs. + caller, so `propagate_attributes` around the call applies to each item. With `runs_per_item` + (default 1, at most 5), each item runs more than one time. Each run gets its own copy of the + item, so a task that changes its input does not change the next run. The pass rate and the mean + of each score use all runs. - `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, and the requests go in worker threads. A task that is not async runs on the event loop, so use `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. - Each item run is a trace of its own, with the span `experiment `. The SDK puts `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the - provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. - Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the `experiment` environment by default. In a different environment, `run_experiment` logs a warning. Without an enabled client, the items record no traces and no cost. @@ -315,15 +317,21 @@ result = telemetry_dev.run_experiment( `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server - skips an item run that it has. The request that makes the experiment goes again only after a 429 - or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, - the experiment does not complete in telemetry.dev. + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion + before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails + it when it got none. - The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same - API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. With + `environment="experiment"`, the key of the telemetry client must be for all environments or for + `experiment`: a key for another environment cannot send the traces of the items, and the items + then have no trace and no cost. The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev -also when `capture_output` is off, because they are the result of the experiment. The capture -options and `mask` apply only to the spans of the run. +and stay with the experiment, because they are its result. `capture_output`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. ### CI mode diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py index 4f8e042..5dc3065 100644 --- a/sdks/python/src/telemetry_dev/_api.py +++ b/sdks/python/src/telemetry_dev/_api.py @@ -17,7 +17,8 @@ from typing import IO, Any, Literal, TypeVar, cast from ._client import get_client -from ._config import resolve_config +from ._config import SDK_VERSION, resolve_config +from ._serialize import json_dumps # A longer Retry-After waits only this long: the API counts requests by the minute. _MAX_RETRY_AFTER_S = 60.0 @@ -71,18 +72,20 @@ def request( A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a - network error, or a 5xx tries again. Other requests try again only after a 429 or a 503: the - server did not take the request then. A retry waits for the Retry-After of the response, or - for 1 and then 2 seconds. The timeout applies to each socket operation of urllib. + network error, or a 5xx tries again. Other requests try again only after a 429, which the rate + limit answers before the server does any work: a 503 can come after the work committed. A + retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout + applies to each socket operation of urllib. """ url = f"{api.base_url}{path}" params = {key: value for key, value in (query or {}).items() if value is not None} if params: url = f"{url}?{urllib.parse.urlencode(params)}" - headers = {"Authorization": f"Bearer {api.api_key}", "X-Telemetry-Dev-SDK": "telemetry-dev"} + headers = auth_headers(api.api_key) data = None if body is not None: - data = json.dumps(body, ensure_ascii=False, allow_nan=False).encode() + # The JSON of JSON.stringify, which escapes a lone surrogate that UTF-8 cannot hold. + data = json_dumps(body, default=None).encode() headers["Content-Type"] = "application/json" attempt = 1 while True: @@ -95,6 +98,16 @@ def request( attempt += 1 +def auth_headers(api_key: str | None) -> dict[str, str]: + return { + "Authorization": f"Bearer {api_key}", + "X-Telemetry-Dev-SDK": "telemetry-dev", + # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters such as + # Cloudflare's Browser Integrity Check. + "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", + } + + class NoRedirect(urllib.request.HTTPRedirectHandler): # A redirect to another host must not get the API key, so a 3xx fails like any status that is # not 2xx. @@ -185,6 +198,6 @@ def _send( def _retryable(status: int | None, idempotent: bool) -> bool: - if status in (429, 503): + if status == 429: return True return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py index 4b6cd00..e27701f 100644 --- a/sdks/python/src/telemetry_dev/_ci.py +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -127,7 +127,9 @@ def _https_url(url: str) -> str | None: def _cut(text: str, max_units: int) -> str: - """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts.""" + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts, with no half of + a surrogate pair at the end: the API stores the run as jsonb, and Postgres rejects a lone + surrogate there.""" encoded = text.encode("utf-16-le", "surrogatepass") if len(encoded) <= max_units * 2: return text diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py index b4e031d..303dc89 100644 --- a/sdks/python/src/telemetry_dev/_experiments.py +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -4,8 +4,8 @@ import asyncio import contextvars +import copy import inspect -import json import math import os import threading @@ -184,9 +184,11 @@ def run_experiment( them and wins for the same name. ``runs_per_item`` is 1 to 5. The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a - request fails. A task that raises fails only its item. In CI mode (``ci``, default on when - ``CI`` is ``true`` or ``1``), it records the GitHub Actions run, and it raises when the check - fails or the experiment does not complete, so the process exits with an error. + request fails. When the run stops after it made the experiment, it first sends the complete + request, so that telemetry.dev ends the experiment with the item runs that it has. A task that + raises fails only its item. In CI mode (``ci``, default on when ``CI`` is ``true`` or ``1``), + it records the GitHub Actions run, and it raises when the check fails or the experiment does + not complete, so the process exits with an error. """ run, api = _prepare( task, @@ -206,7 +208,12 @@ def run_experiment( asynchronous=False, ) runs, created = _start(api, run) - _run_items(api, run, created, runs) + try: + _announce(run, created) + _run_items(api, run, created, runs) + except BaseException: + _complete_after_failure(api, created) + raise return _finish(api, run, created) @@ -248,7 +255,12 @@ async def run_experiment_async( asynchronous=True, ) runs, created = await asyncio.to_thread(_start, api, run) - await _run_items_async(api, run, created, runs) + try: + _announce(run, created) + await _run_items_async(api, run, created, runs) + except BaseException: + await asyncio.to_thread(_complete_after_failure, api, created) + raise return await asyncio.to_thread(_finish, api, run, created) @@ -464,13 +476,25 @@ def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: idempotent=False, ) ) + return _Runs(api, run, first), created + + +def _announce(run: _Run, created: _Created) -> None: each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" print( f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " f"{created.url}", flush=True, ) - return _Runs(api, run, first), created + + +def _complete_after_failure(api: Api, created: _Created) -> None: + """Without the complete, the experiment would stay running: the server ends it with the item + runs that it has, or fails it when it has none.""" + try: + _complete(api, created) + except Exception as exc: + report_error(api.on_error, "completing the experiment after a failed run failed", exc) class _Runs: @@ -500,8 +524,10 @@ def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" ) for item in page.items: + # A copy for each run, so that a task that changes its input does not change the next + # run. for run_index in range(run.runs_per_item): - yield ExperimentItem(**vars(item), run_index=run_index) + yield ExperimentItem(**copy.deepcopy(vars(item)), run_index=run_index) if page.next_cursor is None: return page = _fetch_page(api, run, page.version, page.next_cursor) @@ -543,7 +569,7 @@ def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: self._lock = threading.Lock() def add(self, result: dict[str, Any]) -> None: - size = len(json.dumps(result, ensure_ascii=False).encode()) + size = len(json_dumps(result, default=None).encode()) with self._lock: if self._pending and self._bytes + size > _MAX_BATCH_BYTES: self._send() @@ -582,8 +608,8 @@ def _batch(api: Api, created: _Created) -> _Batch: def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch - that fails stops the run: no new item starts, and run_experiment raises when the running - items end.""" + that fails stops the run: no new item starts, and _run_items raises when the running items + end.""" batch = _batch(api, created) failures: list[BaseException] = [] caller = contextvars.copy_context() @@ -592,15 +618,10 @@ def worker() -> None: while not failures: try: item = runs.next() - except Exception as exc: - failures.append(exc) - return - if item is None: - return - # A copy for each item: one Context cannot run in two threads at the same time. - result = caller.copy().run(_run_item, run, created, item, api.on_error) - try: - batch.add(result) + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + batch.add(caller.copy().run(_run_item, run, created, item, api.on_error)) except Exception as exc: failures.append(exc) @@ -624,13 +645,9 @@ async def worker() -> None: while not failures: try: item = await asyncio.to_thread(runs.next) - except Exception as exc: - failures.append(exc) - return - if item is None: - return - result = await _run_item_async(run, created, item, api.on_error) - try: + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) await asyncio.to_thread(batch.add, result) except Exception as exc: failures.append(exc) @@ -840,14 +857,29 @@ def _pass_fail(passed: bool) -> dict[str, Any]: def _output_text(output: Any) -> str | None: + """The output as text. An output that JSON cannot hold, such as a dict that holds itself, is + None.""" if output is None: return None - text = output if isinstance(output, str) else json_dumps(output) - return truncate(text, _MAX_OUTPUT_LENGTH) + try: + text = output if isinstance(output, str) else json_dumps(output) + except Exception: + return None + return truncate(_well_formed(text), _MAX_OUTPUT_LENGTH) def _error_text(error: BaseException) -> str: - return f"{type(error).__name__}: {error}" + try: + text = f"{type(error).__name__}: {error}" + except Exception: + # A __str__ that raises. + return type(error).__name__ + return _well_formed(text) + + +def _well_formed(text: str) -> str: + # truncate fails on a lone surrogate, which a str can hold, so each one becomes U+FFFD. + return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace") def _close(awaitable: object) -> None: @@ -860,6 +892,16 @@ def _close(awaitable: object) -> None: def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: # The spans of the items reach ingest before the server reads their cost. flush() + _complete(api, created) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _complete(api: Api, created: _Created) -> None: request( api, ExperimentError, @@ -868,12 +910,6 @@ def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: timeout_s=_REQUEST_TIMEOUT_S, idempotent=True, ) - result, body = _wait_for_result(api, created.id) - write_result(os.environ, body, api.on_error) - print(_describe(result), flush=True) - if run.in_ci: - _check_result(result) - return result def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: @@ -1035,6 +1071,7 @@ def _parse_result(body: object) -> ExperimentResult: if isinstance(raw_scores, list) else None ) + # Required: it decides the result of CI mode, so a missing one must not read as no check. check_status = fields.get("checkStatus", "missing") items_failed = fields.get("itemsFailed", 0) error = fields.get("error") @@ -1067,6 +1104,7 @@ def _parse_result(body: object) -> ExperimentResult: def _parse_score(raw: object) -> ExperimentScore | None: + """A score of the result. A field that can be None can also be missing.""" if not isinstance(raw, dict): return None score = cast("dict[str, Any]", raw) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index a7f3aff..4408852 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -24,9 +24,9 @@ from opentelemetry import context as otel_context from opentelemetry.context import Context -from ._api import NoRedirect, api_error, resolve_api, retry_after +from ._api import NoRedirect, api_error, auth_headers, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error +from ._config import report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -332,13 +332,7 @@ def _get_prompt( cache_ttl_seconds, lambda: _fetch( f"{url}/v1/prompts?{urllib.parse.urlencode(query)}", - { - "Authorization": f"Bearer {key}", - "X-Telemetry-Dev-SDK": "telemetry-dev", - # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters - # such as Cloudflare's Browser Integrity Check. - "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", - }, + auth_headers(key), ), on_error, ) diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py index 7df2e7a..d86f4cd 100644 --- a/sdks/python/tests/fake_api.py +++ b/sdks/python/tests/fake_api.py @@ -4,7 +4,9 @@ from __future__ import annotations import json +import sys import threading +import time from collections.abc import Callable, Generator from contextlib import contextmanager from dataclasses import dataclass, field @@ -66,13 +68,26 @@ def route(self) -> str: Reply = tuple[int, object, dict[str, str]] +@dataclass(frozen=True) +class Drop: + """Closes the connection with no answer, as a network failure does.""" + + +@dataclass(frozen=True) +class Late: + """Answers after a wait, as a server that is slower than the timeout of the client.""" + + seconds: float + reply: Reply + + @dataclass class FakeApi: url: str items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) page_size: int = 100 results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) - override: Callable[[ApiRequest], Reply | None] | None = None + override: Callable[[ApiRequest], Reply | Drop | Late | None] | None = None requests: list[ApiRequest] = field(default_factory=lambda: []) stored: set[str] = field(default_factory=lambda: set[str]()) lock: threading.Lock = field(default_factory=threading.Lock) @@ -85,7 +100,7 @@ def posts(self) -> list[list[dict[str, Any]]]: request.body["items"] for request in self.requests if request.path.endswith("/items") ] - def answer(self, request: ApiRequest) -> Reply: + def answer(self, request: ApiRequest) -> Reply | Drop | Late: with self.lock: self.requests.append(request) overridden = self.override(request) if self.override else None @@ -157,7 +172,7 @@ def _answer(self) -> None: length = int(self.headers.get("content-length") or 0) raw = self.rfile.read(length) if length else b"" target = urlsplit(self.path) - status, body, headers = api.answer( + reply = api.answer( ApiRequest( method=self.command, path=target.path, @@ -166,6 +181,13 @@ def _answer(self) -> None: body=json.loads(raw) if raw else None, ) ) + if isinstance(reply, Drop): + self.close_connection = True + return + if isinstance(reply, Late): + time.sleep(reply.seconds) + reply = reply.reply + status, body, headers = reply data = body if isinstance(body, bytes) else json.dumps(body).encode() self.send_response(status) self.send_header("content-type", "application/json") @@ -178,7 +200,13 @@ def _answer(self) -> None: def log_message(self, format: str, *args: object) -> None: pass - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + class Server(ThreadingHTTPServer): + def handle_error(self, request: Any, client_address: Any) -> None: + # The answer of a Late reply can go to a client that has stopped waiting. + if not isinstance(sys.exc_info()[1], (BrokenPipeError, ConnectionResetError)): + super().handle_error(request, client_address) + + server = Server(("127.0.0.1", 0), Handler) thread = threading.Thread( target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True ) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py index a0b1807..8668f3c 100644 --- a/sdks/python/tests/test_experiments.py +++ b/sdks/python/tests/test_experiments.py @@ -10,6 +10,8 @@ import pytest +import telemetry_dev +import telemetry_dev._experiments as experiments_module from telemetry_dev import ( ExperimentError, ExperimentItem, @@ -28,7 +30,9 @@ EXPERIMENT_ID, EXPERIMENT_URL, ApiRequest, + Drop, FakeApi, + Late, Reply, dataset_item, fake_api, @@ -114,6 +118,10 @@ def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_co } assert api.requests[1].headers["authorization"] == "Bearer td_live_test" assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + # Cloudflare refuses the default Python-urllib User-Agent. + assert api.requests[1].headers["user-agent"] == ( + f"telemetry-dev-python/{telemetry_dev.__version__}" + ) posts = api.posts() assert [len(items) for items in posts] == [100, 50] @@ -293,7 +301,32 @@ def task(item: ExperimentItem) -> str: assert output.endswith("...[truncated]") -def test_a_failed_post_of_items_stops_new_items_never_completes_and_raises() -> None: +def run_both(api: FakeApi, asynchronous: bool, **options: Any) -> ExperimentResult: + """``experiment`` through run_experiment or run_experiment_async.""" + if not asynchronous: + return experiment(api, **options) + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return asyncio.run(run_experiment_async(**arguments)) + + +def unknown_item(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 422, {"error": {"code": "item_not_found", "message": "support-golden v3 has no item"}} + ) + return None + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_a_failed_post_of_items_stops_new_items_completes_the_experiment_and_raises( + asynchronous: bool, +) -> None: started = 0 def task(item: ExperimentItem) -> str: @@ -301,22 +334,167 @@ def task(item: ExperimentItem) -> str: started += 1 return "ok" - def ended(request: ApiRequest) -> Reply | None: - if request.path.endswith("/items"): + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=unknown_item) as api: + with pytest.raises(ExperimentError) as caught: + run_both(api, asynchronous, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("item_not_found", 422) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + # The complete ends the experiment, which would else stay running. + assert api.routes()[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + ] + + +def test_a_complete_that_fails_after_a_failed_run_goes_to_on_error(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def revoked(request: ApiRequest) -> Reply | None: + if request.path.endswith(("/items", "/complete")): return answer_with( - 409, - {"error": {"code": "experiment_ended", "message": "support-golden-v3 has ended."}}, + 401, {"error": {"code": "invalid_api_key", "message": "The API key was revoked"}} ) return None - with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=ended) as api: + with fake_api(override=revoked) as api: with pytest.raises(ExperimentError) as caught: - experiment(api, concurrency=1, task=task) + experiment(api) - assert (caught.value.code, caught.value.status) == ("experiment_ended", 409) - # The one worker waits for the post of the first 100, which fails, so no item starts after. - assert started == 100 - assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" not in api.routes() + assert (caught.value.code, str(caught.value)) == ("invalid_api_key", "The API key was revoked") + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + assert [getattr(error, "code", None) for error in errors] == ["invalid_api_key"] + + +def test_an_interrupted_run_completes_the_experiment(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + experiment(api, task=task) + + assert api.routes()[-1] == f"POST /v1/experiments/{EXPERIMENT_ID}/complete" + + +class UnreadableError(Exception): + def __str__(self) -> str: + raise RuntimeError("no message") + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_an_output_or_an_error_that_cannot_be_encoded_does_not_stop_the_run( + asynchronous: bool, +) -> None: + cycle: dict[str, Any] = {} + cycle["self"] = cycle + + def task(item: ExperimentItem) -> object: + if item.number == 3: + raise UnreadableError + # A lone surrogate, which UTF-8 cannot hold, becomes U+FFFD. + return cycle if item.number == 1 else "\ud83d" + + def tone(output: Any, item: ExperimentItem) -> str: + return "\ud83d" + + with fake_api(items=[dataset_item(1), dataset_item(2), dataset_item(3)]) as api: + outcome = run_both(api, asynchronous, task=task, scorers={"tone": tone}) + + assert outcome.status == "completed" + posted = sorted((run for items in api.posts() for run in items), key=lambda run: run["itemId"]) + tones = [{"name": "tone", "label": "\ud83d"}] + assert [(run["output"], run.get("error"), run.get("scores")) for run in posted] == [ + (None, None, tones), + ("\ufffd", None, tones), + (None, "UnreadableError", None), + ] + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" in api.routes() + + +def test_each_run_of_an_item_starts_from_the_dataset_input() -> None: + seen: list[int] = [] + + def task(item: ExperimentItem) -> str: + messages = item.input["messages"] + seen.append(len(messages)) + messages.append({"role": "assistant", "content": "Hello"}) + return "ok" + + item = {**dataset_item(1), "input": {"messages": [{"role": "user", "content": "Hi"}]}} + with fake_api(items=[item]) as api: + experiment(api, runs_per_item=3, concurrency=1, task=task) + + assert seen == [1, 1, 1] + + +def test_a_post_of_items_holds_at_most_4_mib_of_json() -> None: + def task(item: ExperimentItem) -> str: + # 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + return "\u4e2d" * 100_000 + + with fake_api(items=[dataset_item(number) for number in range(1, 21)]) as api: + experiment(api, task=task) + + assert [len(items) for items in api.posts()] == [13, 7] + + +@pytest.mark.parametrize( + ("count", "code"), + [(10_000, None), (10_001, "invalid_response")], + ids=["10,000 items in pages run", "10,001 items in pages stop the run"], +) +def test_a_dataset_has_at_most_10000_items(count: int, code: str | None) -> None: + items = [dataset_item(number) for number in range(1, count + 1)] + with fake_api(items=items, page_size=5_000) as api: + try: + experiment(api, concurrency=20) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert len([run for items in api.posts() for run in items]) == 10_000 + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + + +@pytest.mark.parametrize( + ("route", "failure", "sends", "code"), + [ + ("GET /v1/datasets/support-golden", "timeout", 2, None), + (f"POST /v1/experiments/{EXPERIMENT_ID}/items", "network", 2, None), + ("POST /v1/experiments", "timeout", 1, "timeout"), + ("POST /v1/experiments", "network", 1, "network_error"), + ], + ids=[ + "a timeout of a dataset page goes again", + "a network error of an item post goes again", + "a timeout of the create does not go again", + "a network error of the create does not go again", + ], +) +def test_a_request_after_a_timeout_or_a_network_error( + monkeypatch: pytest.MonkeyPatch, route: str, failure: str, sends: int, code: str | None +) -> None: + monkeypatch.setattr(experiments_module, "_REQUEST_TIMEOUT_S", 0.2) + failures = [1] + + def fail_once(request: ApiRequest) -> Reply | Drop | Late | None: + if request.route != route or not failures: + return None + failures.pop() + return Drop() if failure == "network" else Late(1.0, answer_with(500, {})) + + with fake_api(override=fail_once) as api: + try: + experiment(api) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert api.routes().count(route) == sends def test_an_item_post_that_fails_with_500_goes_again() -> None: @@ -335,16 +513,31 @@ def flaky(request: ApiRequest) -> Reply | None: assert api.posts()[1] == api.posts()[0] -def test_a_create_that_fails_with_500_does_not_go_again() -> None: +@pytest.mark.parametrize( + ("reply", "code"), + [ + (answer_with(500, {}), "http_500"), + # The server answers 503 when its database work times out, which can be after the commit. + ( + answer_with( + 503, + {"error": {"code": "evals_unavailable", "message": "Evals are unavailable"}}, + ), + "evals_unavailable", + ), + ], + ids=["500", "503"], +) +def test_a_create_that_fails_does_not_go_again(reply: Reply, code: str) -> None: def broken(request: ApiRequest) -> Reply | None: - return answer_with(500, {}) if request.route == "POST /v1/experiments" else None + return reply if request.route == "POST /v1/experiments" else None with fake_api(override=broken) as api: with pytest.raises(ExperimentError) as caught: experiment(api) - assert (caught.value.code, caught.value.status) == ("http_500", 500) - assert api.routes().count("POST /v1/experiments") == 1 + assert (caught.value.code, caught.value.status) == (code, reply[0]) + assert api.routes() == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: @@ -377,6 +570,41 @@ def test_waits_until_the_server_evaluators_end_the_experiment() -> None: assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + + +def test_gives_up_when_the_experiment_does_not_end_in_30_minutes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(experiments_module, "time", clock) + polls: list[float] = [] + + def running(request: ApiRequest) -> Reply | None: + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + polls.append(clock.now) + return answer_with(200, result(status="running", itemsDone=1)) + return None + + with fake_api(override=running) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "timeout" + assert caught.value.result is not None + assert caught.value.result.status == "running" + # The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + assert 30 * 60 - 5 < polls[-1] - polls[0] <= 30 * 60 + + @pytest.mark.parametrize( ("status", "body", "code"), [ @@ -495,6 +723,35 @@ def test_a_failed_check_returns_when( assert experiment(api, **options).check_status == "failed" +def test_a_result_score_that_leaves_out_its_null_fields_reads_them_as_none() -> None: + scores = [{"name": "length", "passCount": 0, "failCount": 0}] + with fake_api(results=[result(scores=scores)]) as api: + outcome = experiment(api) + + assert outcome.scores == ( + ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ), + ) + + +def test_a_result_without_check_status_is_an_invalid_response() -> None: + missing = {key: value for key, value in FAILED_CHECK.items() if key != "checkStatus"} + with fake_api(results=[missing]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + + def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -561,11 +818,20 @@ def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_r assert results.stat().st_mode & 0o777 == 0o600 -def test_a_pull_request_title_is_cut_to_256_utf16_units( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +@pytest.mark.parametrize( + ("title", "sent"), + [ + ("\U0001f600" * 200, "\U0001f600" * 128), + # GitHub counts the emoji as one character, thus the title fits its limit of 256. + ("a" * 255 + "\U0001f600", "a" * 255), + ], + ids=["200 emoji", "255 letters and an emoji"], +) +def test_a_pull_request_title_is_cut_to_256_utf16_units_without_half_of_an_emoji( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi, title: str, sent: str ) -> None: event = tmp_path / "event.json" - event.write_text(json.dumps({"pull_request": {"title": "\U0001f600" * 200}})) + event.write_text(json.dumps({"pull_request": {"title": title}})) for name, value in { "CI": "true", "GITHUB_ACTIONS": "true", @@ -577,7 +843,7 @@ def test_a_pull_request_title_is_cut_to_256_utf16_units( experiment(api) - assert api.requests[1].body["ci"]["prTitle"] == "\U0001f600" * 128 + assert api.requests[1].body["ci"]["prTitle"] == sent def one(output: Any, item: ExperimentItem) -> int: diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py index 44d28de..d72d0d2 100644 --- a/sdks/python/tests/test_scores.py +++ b/sdks/python/tests/test_scores.py @@ -5,6 +5,7 @@ import pytest +import telemetry_dev from telemetry_dev import ScoreError, score, score_async from .conftest import MakeClient @@ -54,6 +55,7 @@ def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( assert request.route == "POST /v1/scores" assert request.headers["authorization"] == "Bearer td_live_key" assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["user-agent"] == f"telemetry-dev-python/{telemetry_dev.__version__}" assert request.headers["content-type"] == "application/json" assert request.body == { "name": "user_feedback", From ae0ed556f68d1952cbe33ca8f93f29a429fab5a4 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:42 +0000 Subject: [PATCH 08/32] fix(sdk): use the tsx of the project and change only our comment The action ran a .ts script with npx --yes tsx. npx then downloads the newest tsx from npm, in a step that has the API key. The action uses the tsx in node_modules/.bin of the working directory or of a directory above it. If it finds no tsx, it stops with a clear error. report.mjs used each comment that started with its marker, thus it tried to change a pasted copy. It changes only a comment of its token. For a user token, that is a comment of its login. The GITHUB_TOKEN and app tokens cannot read /user, and they write as a bot. The README tells that the key must be for all environments or for experiment. It tells that the capture settings do not apply to item outputs, and that a pull request from a fork gets no API key. The example does not run the job for those pull requests. --- actions/experiment/README.md | 51 +++++--- actions/experiment/action.yml | 2 +- actions/experiment/report.mjs | 11 +- actions/experiment/run.sh | 19 ++- packages/sdk/tests/experiment-action.test.ts | 126 ++++++++++++++++--- 5 files changed, 176 insertions(+), 33 deletions(-) diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 77f5a72..7b851fd 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -14,6 +14,9 @@ permissions: jobs: evals: + # A pull request from a fork gets no secrets, so the job runs only for branches of this + # repository. + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest env: TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} @@ -22,6 +25,7 @@ jobs: - uses: actions/setup-node@v5 with: node-version: 24 + # Installs tsx too, which runs the .ts script. - run: npm ci - uses: telemetry-dev/telemetry/actions/experiment@main with: @@ -49,22 +53,32 @@ changes. ## Inputs -| Input | Description | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with `npx --yes tsx`, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | -| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | -| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | -| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | -| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | -| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | -| `working-directory` | The directory that the script or the command runs in. Default: `.`. | -| `comment` | Set to `false` to write the job summary only. | -| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | +| Input | Description | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with the `tsx` of the project, from `node_modules/.bin` in the working directory or a directory above it, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the -traces of the items stay out of your production pages. `CI` is `true` in GitHub Actions, thus the -SDK runs in CI mode: it records the repository, the commit, the run, and the pull request with -the experiment, and it exits with an error when the check fails. +traces of the items stay out of your production pages. Thus the key must be for all environments +or for `experiment`. A key for another environment, for example your production key, cannot send +the traces of the items: the experiment then completes with its outputs, but with no traces and no +cost. `CI` is `true` in GitHub Actions, thus the SDK runs in CI mode: it records the repository, +the commit, the run, and the pull request with the experiment, and it exits with an error when the +check fails. + +The action does not download packages. Add `tsx` to the `devDependencies` of the project for a +`.ts` script, or give a `command`. + +The outputs, the errors, and the scorer reasons of the items go to telemetry.dev and stay with the +experiment. The capture options of the SDK and the capture setting of the environment in the +project do not apply to them. ## Outputs @@ -76,8 +90,13 @@ the experiment, and it exits with an error when the check fails. ## The comment The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or -`command`. A later push changes that comment. A pull request from a fork gets a token that cannot -write comments: the action then logs a warning, and the job summary still has the result. +`command`. A later push changes that comment. The action changes only a comment that its token +wrote: with a copy of the comment by someone else, it adds a new comment. When the token cannot +write comments, the action logs a warning, and the job summary still has the result. + +A pull request from a fork gets no secrets, thus no API key, and the action stops with the error +"Set the api-key input or the TELEMETRY_DEV_API_KEY variable." The `if` of the example above skips +the job for those pull requests. The comment and the summary show the name of each experiment, its status, the count of the items that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml index 19a774c..edde3ec 100644 --- a/actions/experiment/action.yml +++ b/actions/experiment/action.yml @@ -7,7 +7,7 @@ branding: inputs: script: - description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with tsx, a .js script with node, and a .py script with python. Give script or command. + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with the tsx in the node_modules of the project, a .js script with node, and a .py script with python. Give script or command. required: false command: description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs index 67ce615..ff94492 100644 --- a/actions/experiment/report.mjs +++ b/actions/experiment/report.mjs @@ -177,13 +177,22 @@ async function comment(body) { const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + // Anyone can paste the marker into a comment, so the action changes only a comment that its token + // wrote. A user token knows its login. The GITHUB_TOKEN and app tokens cannot read /user, and + // they write as a bot. + const login = await github("GET", `${api}/user`).then( + (user) => (typeof user?.login === "string" ? user.login : undefined), + () => undefined, + ); + const ours = (item) => + login === undefined ? item.user?.type === "Bot" : item.user?.login === login; let existing; for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); existing = comments.find( - (item) => typeof item.body === "string" && item.body.startsWith(marker), + (item) => typeof item.body === "string" && item.body.startsWith(marker) && ours(item), ); if (comments.length < 100) break; diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh index 9d6f1c2..86cf722 100755 --- a/actions/experiment/run.sh +++ b/actions/experiment/run.sh @@ -31,12 +31,29 @@ results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail echo "results=$results" >>"$GITHUB_OUTPUT" export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results +# The tsx of the project, from node_modules/.bin here or in a directory above. The action does not +# download one: a package from npm at run time would run with the API key in its environment. +project_tsx() { + dir=$PWD + while :; do + if [ -x "$dir/node_modules/.bin/tsx" ]; then + echo "$dir/node_modules/.bin/tsx" + return 0 + fi + [ "$dir" = / ] && return 1 + dir=$(dirname "$dir") + done +} + code=0 if [ -n "$command" ]; then bash -c "$command" || code=$? else case "$script" in - *.ts | *.mts | *.cts | *.tsx) npx --yes tsx "$script" || code=$? ;; + *.ts | *.mts | *.cts | *.tsx) + tsx=$(project_tsx) || fail "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command." + "$tsx" "$script" || code=$? + ;; *.js | *.mjs | *.cjs) node "$script" || code=$? ;; *.py) python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts index cb7be99..eb8bd99 100644 --- a/packages/sdk/tests/experiment-action.test.ts +++ b/packages/sdk/tests/experiment-action.test.ts @@ -1,5 +1,5 @@ import { spawn, spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { createServer, type IncomingMessage } from "node:http"; import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; @@ -99,15 +99,36 @@ interface GithubRequest { body: unknown; } -/** The comment routes of the GitHub REST API for one pull request. */ +interface GithubComment { + id: number; + body: string; + user: { login: string; type: "Bot" | "User" }; +} + +const BOT = { login: "github-actions[bot]", type: "Bot" } as const; + +/** + * The comment routes of the GitHub REST API for one pull request. Without `user`, GET /user fails + * as it does for the GITHUB_TOKEN. + */ async function fakeGithub( - options: { comments?: { id: number; body: string }[]; status?: number } = {}, + options: { comments?: GithubComment[]; status?: number; user?: { login: string } } = {}, ) { const requests: GithubRequest[] = []; const server = createServer(async (req: IncomingMessage, res) => { let raw = ""; for await (const chunk of req) raw += chunk; requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + + if (req.url === "/user") { + res.writeHead(options.user ? 200 : 403, { "content-type": "application/json" }); + res.end( + JSON.stringify(options.user ?? { message: "Resource not accessible by integration" }), + ); + + return; + } + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); }); @@ -181,6 +202,37 @@ test("run.sh passes the inputs to the script and records its exit code without f expect(statSync(written.results!).mode & 0o777).toBe(0o600); }); +test("run.sh runs a .ts script with the tsx of the project, from a directory above", () => { + const project = workdir(); + const evals = join(project, "evals"); + const output = join(project, "output"); + mkdirSync(join(project, "node_modules", ".bin"), { recursive: true }); + mkdirSync(evals); + writeFileSync(output, ""); + // A stand-in for the tsx of the project that records the script it gets. + writeFileSync( + join(project, "node_modules", ".bin", "tsx"), + `#!/bin/sh\necho "$1" > "${join(project, "ran")}"\n`, + { mode: 0o755 }, + ); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: evals, + encoding: "utf8", + env: { + PATH: process.env.PATH, + RUNNER_TEMP: project, + GITHUB_OUTPUT: output, + INPUT_SCRIPT: "reply.eval.ts", + TELEMETRY_DEV_API_KEY: "td_live_job", + }, + }); + + expect(run.status).toBe(0); + expect(readFileSync(join(project, "ran"), "utf8")).toBe("reply.eval.ts\n"); + expect(outputs(output)["exit-code"]).toBe("0"); +}); + test("run.sh keeps the environment of the job, and runs a .js script with node", () => { const { run, outputs: written } = runScript( { @@ -219,6 +271,11 @@ test.each([ { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, "The script must end in .ts, .js, or .py. For another runner, give a command.", ], + [ + "a .ts script in a project without tsx", + { INPUT_SCRIPT: "reply.eval.ts", TELEMETRY_DEV_API_KEY: "k" }, + "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command.", + ], ])("run.sh stops with an error for %s", (_, env, message) => { const { run, outputs: written } = runScript(env); @@ -239,33 +296,74 @@ test("report.mjs writes the summary, posts it on the pull request, and sets the expect(written).toBe(`${summary}\n`); expect(set).toEqual({ "check-status": "failed", url: result.url }); expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /user", "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", "POST /repos/acme/support/issues/42/comments", ]); - expect(github.requests[1]!.body).toEqual({ + expect(github.requests[2]!.body).toEqual({ body: expect.stringMatching( /^\n### telemetry\.dev experiment\n/, ), }); - expect((github.requests[1]!.body as { body: string }).body.endsWith(summary)).toBe(true); + expect((github.requests[2]!.body as { body: string }).body.endsWith(summary)).toBe(true); }); +/** The body of the comment that the action posts for the result. */ +async function commentBody(env: Record = {}): Promise { + const github = await fakeGithub(); + await report({ GITHUB_API_URL: github.url, ...env }, [result]); + + return (github.requests.find((request) => request.method === "POST")!.body as { body: string }) + .body; +} + +function changes(github: Awaited>): string[] { + return github.requests + .filter((request) => request.method !== "GET") + .map((request) => `${request.method} ${request.url}`); +} + test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { - const first = await fakeGithub(); - await report({ GITHUB_API_URL: first.url }, [result]); - const own = { id: 8, body: (first.requests[1]!.body as { body: string }).body }; + const own = { id: 8, body: await commentBody(), user: BOT }; const other = await fakeGithub({ comments: [own] }); await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); - const again = await fakeGithub({ comments: [{ id: 7, body: "Looks good" }, own] }); + const again = await fakeGithub({ + comments: [{ id: 7, body: "Looks good", user: { login: "dana", type: "User" } }, own], + }); await report({ GITHUB_API_URL: again.url }, [result]); - expect(other.requests.map((request) => request.method)).toEqual(["GET", "POST"]); - expect(again.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ - "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", - "PATCH /repos/acme/support/issues/comments/8", - ]); + expect(changes(other)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + expect(changes(again)).toEqual(["PATCH /repos/acme/support/issues/comments/8"]); +}); + +test.each([ + ["a person pasted under the GITHUB_TOKEN", undefined, { login: "mallory", type: "User" }], + ["another bot wrote under a user token", { login: "dana" }, BOT], +] as const)( + "report.mjs does not change a comment with its marker that %s", + async (_, user, author) => { + const github = await fakeGithub({ + user, + comments: [{ id: 7, body: await commentBody(), user: author }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + }, +); + +test("report.mjs changes the comment of the user of a user token", async () => { + const github = await fakeGithub({ + user: { login: "dana" }, + comments: [{ id: 7, body: await commentBody(), user: { login: "dana", type: "User" } }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["PATCH /repos/acme/support/issues/comments/7"]); }); test("report.mjs warns and succeeds when the token cannot write the comment", async () => { From fad2f001dc0574a5ece53efc1ca0025e311e0d4f Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:43 +0000 Subject: [PATCH 09/32] docs(sdk): tell how a stopped experiment run completes The conformance document gives the complete request after a run stops, and the create retry only after a 429. It also gives the copy of the item for each run and the outputs that JSON cannot hold. It gives the Python User-Agent, the fetch call as a function, the fields that a result must have, and the cut of the title. --- docs/sdk-conformance.md | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index e81153b..483a8c4 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -184,12 +184,16 @@ task. ## Experiments and scores -`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` -/ `score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` / +`score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` -header, the SDK header, and a JSON body. The redirect rule of getPrompt applies. An error has the -code of the `{ error: { code, message } }` body, or `http_`, or `missing_api_key`, -`timeout`, `network_error`, or `invalid_response`. +header, the SDK header, and a JSON body. Python also sends +`User-Agent: telemetry-dev-python/`, because bot filters refuse the default +`Python-urllib/3.x`. The body is the JSON of `JSON.stringify`: a lone surrogate goes as a +`\uXXXX` escape. The TypeScript SDK calls `fetch` as a plain function, because Workers and +browsers reject a call whose `this` is another object. The redirect rule of getPrompt applies. An +error has the code of the `{ error: { code, message } }` body, or `http_`, or +`missing_api_key`, `timeout`, `network_error`, or `invalid_response`. An experiment run sends these requests in this sequence: @@ -203,21 +207,28 @@ An experiment run sends these requests in this sequence: value are left out. 3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, - `null` for no value, cut to 100,000 UTF-16 units), `traceId` (when the span records), - `latencyMs`, and, for a task that failed, `error` (`: `, cut to 4,000 UTF-16 - units) or, for a task that succeeded, `scores`. The posts go one at a time, in order. A post - that fails stops the run: no item starts after it, and the run fails when the running items end. + `null` for no value and for a value that JSON cannot hold, such as an object that holds itself, + cut to 100,000 UTF-16 units), `traceId` (when the span records), `latencyMs`, and, for a task + that failed, `error` (`: `, cut to 4,000 UTF-16 units) or, for a task that + succeeded, `scores`. Python sends a lone surrogate of an output or an error as U+FFFD. Each run + gets its own deep copy of the item, so a task that changes its input does not change the next + run. The posts go one at a time, in order. A post or a page fetch that fails stops the run: no + item starts after it. When the running items end, the SDK sends the request of step 4, sends its + failure to `onError` / `on_error`, and fails with the first error. The server then ends the + experiment with the item runs that it has, or fails it when it has none. Python does the same + after `KeyboardInterrupt` and cancellation. 4. `POST /v1/experiments/{id}/complete`, after `flush()`. 5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). Retries: the GET requests, the item posts, and the complete request try 3 times in all after a timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that -makes the experiment tries again only after a 429 or a 503. A retry waits for the `Retry-After` of -a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 second and then 2 seconds. -`POST /v1/scores` makes one request, with no retry. The TypeScript request timeouts abort the -request: 30 seconds for the experiment requests and 10 seconds for a score. The Python timeouts -apply to each socket operation of urllib. +makes the experiment tries again only after a 429, which the rate limit answers before any work: the +server answers 503 when its database work times out, which can be after the commit. A retry waits +for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 +second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript +request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a +score. The Python timeouts apply to each socket operation of urllib. Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). `concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names @@ -244,13 +255,18 @@ not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / `on_error`. +The result of step 5 needs `id`, `name`, `status`, `url`, `itemsTotal`, `itemsDone`, `scores`, +and `checkStatus`, else it is `invalid_response`: `checkStatus` decides CI mode, so a missing one +does not read as no check. A field of a score that can be null can also be missing. + CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: - In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, - and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units. A URL must be `https://` - with no space, at most 2,048 UTF-16 units. The run URL is + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units, with no half of a + surrogate pair at the end, because Postgres rejects a lone surrogate in jsonb. A URL must be + `https://` with no space, at most 2,048 UTF-16 units. The run URL is `//actions/runs/`. Keys without a value are left out. - After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of From 85fa4f14b95194aa59c4ebca7a4fe8062d6bf82d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 01:01:35 +0000 Subject: [PATCH 10/32] docs(sdk): give api.telemetry.dev as the default base url The base-url input of the experiment action and the baseUrl comment in api.ts gave the old default, https://ingest.telemetry.dev. The default base URL of the SDKs changed to https://api.telemetry.dev in #109. --- actions/experiment/README.md | 2 +- actions/experiment/action.yml | 2 +- packages/sdk/src/api.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 7b851fd..431f2c0 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -60,7 +60,7 @@ changes. | `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | | `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | | `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | -| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `base-url` | The telemetry.dev API. Default: `https://api.telemetry.dev`. | | `working-directory` | The directory that the script or the command runs in. Default: `.`. | | `comment` | Set to `false` to write the job summary only. | | `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml index edde3ec..5cc5fa6 100644 --- a/actions/experiment/action.yml +++ b/actions/experiment/action.yml @@ -22,7 +22,7 @@ inputs: description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. required: false base-url: - description: The telemetry.dev API. Default https://ingest.telemetry.dev. + description: The telemetry.dev API. Default https://api.telemetry.dev. required: false working-directory: description: The directory that the script or the command runs in. diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts index 573f780..dd7fac6 100644 --- a/packages/sdk/src/api.ts +++ b/packages/sdk/src/api.ts @@ -6,7 +6,7 @@ import { currentClient } from "./client.ts"; export interface ApiOptions { /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ apiKey?: string; - /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://ingest.telemetry.dev`. */ + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ baseUrl?: string; /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ fetch?: typeof fetch; From 339eb0c01c5754191eb7d1b5ec37db21ebd28855 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 14:18:40 +0000 Subject: [PATCH 11/32] feat(sdk): record the prompt on model calls outside withPrompt compile() keeps each text that it makes, with the prompt. For a template with a variable, compile() also keeps the variables. It keeps a maximum of 1000 texts and 2,000,000 characters, and it removes the text that it kept first. It does not keep a text from a template that has only variable tags. promptAttributesFor (TypeScript) and prompt_attributes_for (Python) find a kept text in a request. They examine 6 levels and 500 values, and they do not call getters or properties. They give the td.prompt.* attributes of the first text from a template with a variable. If the request has no such text, they give the name, version, and label of the first text from a template with no variable. In a withPrompt or with_prompt scope, they give no attributes. They do not throw an error. The 11 provider integrations send the request of the app to the helper before the span starts. They do this also when input capture is off. The integrations are OpenAI chat and responses, Anthropic messages, Bedrock Converse and InvokeModel, Google GenAI generateContent, LiteLLM completion and responses, and OpenRouter chat and responses. Bedrock embedding models do not get the attributes. The spans for images, audio, video, and Realtime do not examine the request. Each integration reads the helper from the core module when it runs. Thus it can operate with a core that does not have the helper. As a result, the minimum Python core versions do not change. In Python, fork() waits for the lock of the kept texts. Thus a child process does not get a lock that a different thread holds. Refs #97 --- docs/sdk-conformance.md | 28 +- packages/anthropic/src/index.ts | 16 +- packages/anthropic/tests/anthropic.test.ts | 74 ++++- packages/bedrock/src/converse.ts | 2 + packages/bedrock/src/internal.ts | 10 + packages/bedrock/src/invoke-model.ts | 5 +- packages/bedrock/tests/bedrock.test.ts | 111 ++++++- packages/google-genai/src/index.ts | 16 +- .../google-genai/tests/google-genai.test.ts | 81 ++++- packages/openai/src/index.ts | 20 +- packages/openai/tests/openai.test.ts | 112 ++++++- packages/openrouter/src/index.ts | 13 + packages/openrouter/tests/openrouter.test.ts | 93 ++++++ packages/sdk/README.md | 14 +- packages/sdk/src/index.ts | 1 + packages/sdk/src/prompts.ts | 152 +++++++++- packages/sdk/tests/prompts.test.ts | 228 ++++++++++++++ .../src/telemetry_dev_anthropic/__init__.py | 12 +- sdks/python-anthropic/tests/test_anthropic.py | 50 ++++ .../src/telemetry_dev_bedrock/_fields.py | 9 +- .../telemetry_dev_bedrock/_invoke_model.py | 4 +- .../src/telemetry_dev_bedrock/_registry.py | 10 +- sdks/python-bedrock/tests/test_bedrock.py | 87 ++++++ .../telemetry_dev_google_genai/__init__.py | 8 +- .../tests/test_google_genai.py | 56 ++++ .../src/telemetry_dev_litellm/__init__.py | 12 +- sdks/python-litellm/tests/test_litellm.py | 64 ++++ .../src/telemetry_dev_openai/__init__.py | 16 +- sdks/python-openai/tests/test_openai.py | 66 ++++ .../src/telemetry_dev_openrouter/__init__.py | 9 + .../tests/test_openrouter.py | 67 +++++ sdks/python/README.md | 18 +- sdks/python/src/telemetry_dev/__init__.py | 2 + sdks/python/src/telemetry_dev/_prompts.py | 192 +++++++++++- sdks/python/tests/test_prompts.py | 283 +++++++++++++++++- 35 files changed, 1898 insertions(+), 43 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dc..efe4a7a 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -41,9 +41,10 @@ repository, not this SDK workspace. Paths below refer to that product repository | error | span status ERROR + `error.type` + `exception` event (`exception.type/message/stacktrace`, `log.severity_number: 17`) | | | `log()` | OTLP log record to `/v1/logs` (always; never span events) | severity debug=5/info=9/warn=13/error=17; trace-correlated via active context | | `withPrompt(prompt, fn)` / `with with_prompt(prompt):` → `prompt.name` | `td.prompt.name` | on model-call spans that start in scope: operation `chat`, `text_completion`, or `generate_content` | +| outside a scope: a text that `prompt.compile()` made, in the request of a provider integration | `td.prompt.name`, `td.prompt.version`, `td.prompt.label` | on the `chat` spans of the provider integrations listed in [Prompts](#prompts); the scope wins | | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | -| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the `compile()` that made the text, when its template has a variable | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -178,6 +179,31 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. +Outside a scope, the SDKs find the prompt from the text of the request: + +- `compile()` keeps each text that it makes (each message of a chat prompt, the text of a text + prompt), with the prompt and the variables. It keeps at most 1000 texts and 2,000,000 + characters, and drops the oldest first. A text that `compile()` makes again becomes the newest. + It does not keep a text whose template has only variable tags, for example `{{ question }}`: + that text is input of the app, and an unrelated request can hold it too. +- `promptAttributesFor(request)` / `prompt_attributes_for(request)` look for a kept text in + `request`: strings in arrays and plain objects (Python: `list`, `tuple`, and `dict` but not their + subclasses, and the fields that an object stores in `__dict__` or `__pydantic_extra__`), 6 + levels deep and 500 values in all. They run no getter or property. A string matches only when it is equal to a kept text. +- The first text from a template with a variable gives the name, version, label, and variables + of its `compile()`. Without one, the first text from a template with no variable gives the + name, version, and label, but no variables: the same static text comes from each `compile()`. +- In a `withPrompt` / `with_prompt` scope they give nothing, thus the span gets the prompt of the + scope. They never throw. +- The provider integrations call them with the request of the app before the span starts, also + when input capture is off: OpenAI chat completions and responses, Anthropic messages, Amazon + Bedrock Converse and InvokeModel (not embedding models), Google GenAI `generateContent`, + LiteLLM completion and responses (Python), and OpenRouter chat and responses. The other spans + of the integrations (images, audio, video, and OpenAI Realtime) do not look in the request. +- A provider integration reads the function from the core module when it runs, thus it also + operates with an older core that does not have it. With such a core, the prompt is recorded + only in a scope. + ## The shared scenario Both e2e runners emit exactly this (under diff --git a/packages/anthropic/src/index.ts b/packages/anthropic/src/index.ts index d9cb1f6..4e5b057 100644 --- a/packages/anthropic/src/index.ts +++ b/packages/anthropic/src/index.ts @@ -5,6 +5,7 @@ import { type SpanHandle, type StartSpanOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { Stream } from "@anthropic-ai/sdk/core/streaming"; import { Messages as BetaMessages } from "@anthropic-ai/sdk/resources/beta/messages"; import { Messages } from "@anthropic-ai/sdk/resources/messages"; @@ -147,6 +148,16 @@ function toolDefinitions(tools: T): Record | undefined { : undefined; } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(request: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(request) : undefined; +} + function messagesRequest(body: ValueRecord): RequestMapping { const model = readString(body.model); @@ -161,7 +172,10 @@ function messagesRequest(body: ValueRecord): RequestMapping { type: "generation", model, input, - attributes: toolDefinitions(body.tools), + attributes: { + ...toolDefinitions(body.tools), + ...promptAttributes([body.system, body.messages]), + }, systemInstructions: body.system, temperature: readNumber(body.temperature), topP: readNumber(body.top_p), diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index e53d040..c8cab08 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -4,7 +4,14 @@ import { type PushMetricExporter, } from "@opentelemetry/sdk-metrics"; import { InMemorySpanExporter, type ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { flush, init, shutdown, type TelemetryOptions } from "@telemetry-dev/sdk"; +import { + type ChatMessage, + flush, + getPrompt, + init, + shutdown, + type TelemetryOptions, +} from "@telemetry-dev/sdk"; import * as telemetrySdk from "@telemetry-dev/sdk"; import Anthropic from "@anthropic-ai/sdk"; import { createRequire } from "node:module"; @@ -274,6 +281,71 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { field: "system", olderCore: false }, + { field: "messages", olderCore: false }, + { field: "messages", olderCore: true }, +])( + "messages.create outside withPrompt records the prompt of a compiled text in $field, older core $olderCore", + async ({ field, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: "https://prompts.anthropic.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "anthropic", name: "claude-sonnet-4-6" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const spans = setupSpans(); + const client = clientWith(createFakeFetch(jsonResponse(messagePayload())).fetch); + + await client.messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + system: field === "system" ? system!.content : "Be terse.", + messages: [{ role: "user", content: field === "messages" ? user!.content : "Hi" }], + }); + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.input.messages"]).toBeDefined(); + + if (olderCore) { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }); + }, +); + test("messages.create maps request, response, usage, finish reason, provider, and sampling attributes", async () => { const spans = setupSpans(); const messages = [{ role: "user" as const, content: "Say hello" }]; diff --git a/packages/bedrock/src/converse.ts b/packages/bedrock/src/converse.ts index 074d1b1..e552c6d 100644 --- a/packages/bedrock/src/converse.ts +++ b/packages/bedrock/src/converse.ts @@ -9,6 +9,7 @@ import { numberValue, omitUndefined, parseJson, + promptAttributes, PROVIDER, stringValue, StreamRetention, @@ -34,6 +35,7 @@ export function converseRequestFields(input: JsonRecord): SpanFields { model: stringValue(input.modelId), input: normalizeMessages(input.messages), systemInstructions: normalizeSystem(input.system), + attributes: promptAttributes([input.system, input.messages]), temperature: numberValue(inference.temperature), topP: numberValue(inference.topP), maxTokens: numberValue(inference.maxTokens), diff --git a/packages/bedrock/src/internal.ts b/packages/bedrock/src/internal.ts index 104f595..7d82cc8 100644 --- a/packages/bedrock/src/internal.ts +++ b/packages/bedrock/src/internal.ts @@ -192,6 +192,16 @@ export function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +export function promptAttributes(request: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(request) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(name: string, options: StartSpanOptions): SpanHandle { const withheld = options.input !== undefined && isCut(options) && captureMaskedBeforeStart(); diff --git a/packages/bedrock/src/invoke-model.ts b/packages/bedrock/src/invoke-model.ts index d1e1f11..d2532d2 100644 --- a/packages/bedrock/src/invoke-model.ts +++ b/packages/bedrock/src/invoke-model.ts @@ -9,6 +9,7 @@ import { numberValue, omitUndefined, parseJson, + promptAttributes, PROVIDER, STREAM_CAPTURE_MAX_ITEMS, stringValue, @@ -35,12 +36,14 @@ export function invokeModelSpanName(input: JsonRecord): string { export function invokeModelRequestFields(input: JsonRecord): SpanFields { const model = stringValue(input.modelId); const body = parseBody(input.body, input.contentType); + const embedding = isEmbeddingModel(model); return omitUndefined({ provider: PROVIDER, model, input: body, - outputType: isEmbeddingModel(model) ? "embedding" : undefined, + outputType: embedding ? "embedding" : undefined, + attributes: embedding ? undefined : promptAttributes(body), ...nativeSamplingFields(model, body), }); } diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index 8e94b8f..d12bf19 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -9,7 +9,8 @@ import { } from "@aws-sdk/client-bedrock-runtime"; import { afterEach, expect, test, vi } from "vitest"; -import type { MaskFn } from "@telemetry-dev/sdk"; +import { type ChatMessage, getPrompt, type MaskFn } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { instrumentBedrock, uninstrumentBedrock, wrapBedrock } from "../src/index.ts"; import { @@ -423,6 +424,114 @@ test("ConverseStream records stream errors with partial output", async () => { expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); }); +test.each([false, true])( + "Converse and InvokeModel outside withPrompt record the prompt of a compiled text; embeddings do not; older core %s", + async (olderCore) => { + const spans = setup(); + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: "https://prompts.bedrock.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "amazon-bedrock", name: "anthropic.claude-3-haiku" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const converseReply = { + output: { message: { role: "assistant", content: [{ text: "Soon." }] } }, + stopReason: "end_turn", + }; + + const client = wrapBedrock( + new FakeClient([ + converseReply, + converseReply, + { + body: bytes({ content: [{ type: "text", text: "Soon." }], stop_reason: "end_turn" }), + contentType: "application/json", + }, + { + body: bytes({ embedding: [0.1, 0.2], inputTextTokenCount: 6 }), + contentType: "application/json", + }, + ]), + ); + + await client.send( + new ConverseCommand({ + modelId: "anthropic.claude-3-haiku", + system: [{ text: system!.content }], + messages: [{ role: "user", content: [{ text: "Hi" }] }], + }), + ); + await client.send( + new ConverseCommand({ + modelId: "anthropic.claude-3-haiku", + messages: [{ role: "user", content: [{ text: user!.content }] }], + }), + ); + await client.send( + new InvokeModelCommand({ + modelId: "anthropic.claude-3-haiku", + contentType: "application/json", + body: bytes({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: user!.content }], + }), + }), + ); + await client.send( + new InvokeModelCommand({ + modelId: "amazon.titan-embed-text-v2:0", + contentType: "application/json", + body: bytes({ inputText: user!.content }), + }), + ); + + const [systemSpan, messagesSpan, invokeSpan, embedSpan] = spans.getFinishedSpans(); + + for (const span of [systemSpan, messagesSpan, invokeSpan]) { + expect(span!.attributes["gen_ai.request.model"]).toBe("anthropic.claude-3-haiku"); + expect( + Object.entries(span!.attributes).filter(([key]) => key.startsWith("td.prompt.")), + ).toEqual( + olderCore + ? [] + : [ + ["td.prompt.name", "support/reply"], + ["td.prompt.version", 12], + ["td.prompt.label", "production"], + ["td.prompt.variables", '{"team":"billing","order":"#4"}'], + ], + ); + } + + expect(embedSpan!.attributes["gen_ai.operation.name"]).toBe("embeddings"); + expect(embedSpan!.attributes["td.prompt.name"]).toBeUndefined(); + }, +); + test("InvokeModel captures provider-native bodies and embedding spans", async () => { const spans = setup(); diff --git a/packages/google-genai/src/index.ts b/packages/google-genai/src/index.ts index 96ffef5..2272e46 100644 --- a/packages/google-genai/src/index.ts +++ b/packages/google-genai/src/index.ts @@ -567,7 +567,11 @@ function responseAttributes(response: UnknownRecord, metadataContent: boolean) { function generateRequestFields(params: UnknownRecord): RequestMapping { const model = readString(params.model); const config = asRecord(params.config); - const attributes = requestAttributesFromConfig(config); + + const attributes = { + ...requestAttributesFromConfig(config), + ...promptAttributes([config?.systemInstruction, params.contents]), + }; const input = captureEnabled("input") ? normalizeContentInput(params.contents) : undefined; const systemInstructions = captureEnabled("input") @@ -940,6 +944,16 @@ function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(request: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(request) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(request: RequestMapping, provider: string): SpanHandle { const fields = { ...request.fields, provider }; diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index d1d291a..e7417ae 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -10,7 +10,8 @@ import { type PushMetricExporter, type ResourceMetrics, } from "@opentelemetry/sdk-metrics"; -import { flush, init, shutdown } from "@telemetry-dev/sdk"; +import { type ChatMessage, flush, getPrompt, init, shutdown } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { FunctionCallingConfigMode, GoogleGenAI, @@ -234,6 +235,84 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { field: "systemInstruction", captureInput: true, olderCore: false }, + { field: "contents", captureInput: false, olderCore: false }, + { field: "contents", captureInput: true, olderCore: true }, +])( + "generateContent outside withPrompt records the prompt of a compiled text in $field, input capture $captureInput, older core $olderCore", + async ({ field, captureInput, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: "https://prompts.google-genai.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "gcp.gemini", name: "gemini-2.5-flash" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const spans = setupSpans({ captureInput }); + vi.stubGlobal( + "fetch", + createFakeFetch( + jsonResponse({ + candidates: [ + { content: { role: "model", parts: [{ text: "Soon." }] }, finishReason: "STOP" }, + ], + }), + ).fetch, + ); + + await clientWith().models.generateContent({ + model: "gemini-2.5-flash", + contents: [{ role: "user", parts: [{ text: field === "contents" ? user!.content : "Hi" }] }], + config: { + temperature: 0.2, + systemInstruction: field === "systemInstruction" ? system!.content : "Be terse.", + }, + }); + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.request.temperature"]).toBe(0.2); + + if (olderCore) { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + expect(attributes["td.prompt.variables"]).toBe( + captureInput ? '{"team":"billing","order":"#4"}' : undefined, + ); + }, +); + test("generateContent maps request, response, usage, finish reason, provider, and sampling attributes", async () => { const spans = setupSpans(); diff --git a/packages/openai/src/index.ts b/packages/openai/src/index.ts index 3512476..c62671c 100644 --- a/packages/openai/src/index.ts +++ b/packages/openai/src/index.ts @@ -156,6 +156,7 @@ function stopSequences(value: T): string[] | undefined { function chatRequest(body: JsonRecord): RequestMapping { const model = readString(body.model); const capture = captureEnabled("input") ? sanitizeResponsesCapture(body.messages) : undefined; + const prompt = promptAttributes(body.messages); return { name: `chat ${model ?? "unknown"}`, @@ -170,7 +171,9 @@ function chatRequest(body: JsonRecord): RequestMapping { seed: readNumber(body.seed), frequencyPenalty: readNumber(body.frequency_penalty), presencePenalty: readNumber(body.presence_penalty), - attributes: capture?.truncated ? { "telemetry.dev.capture.truncated": true } : undefined, + attributes: capture?.truncated + ? { ...prompt, "telemetry.dev.capture.truncated": true } + : prompt, }, }; } @@ -243,6 +246,7 @@ function chatResponse(response: T): SpanFields { function responsesRequest(body: JsonRecord): RequestMapping { const model = readString(body.model); const capture = captureEnabled("input") ? sanitizeResponsesCapture(body.input) : undefined; + const prompt = promptAttributes([body.instructions, body.input]); return { name: `chat ${model ?? "unknown"}`, @@ -254,7 +258,9 @@ function responsesRequest(body: JsonRecord): RequestMapping { temperature: readNumber(body.temperature), topP: readNumber(body.top_p), maxTokens: readNumber(body.max_output_tokens), - attributes: capture?.truncated ? { "telemetry.dev.capture.truncated": true } : undefined, + attributes: capture?.truncated + ? { ...prompt, "telemetry.dev.capture.truncated": true } + : prompt, }, }; } @@ -1626,6 +1632,16 @@ function captureMaskedBeforeStart(): boolean { } } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(request: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(request) : undefined; +} + // The decision is made before the span starts so samplers see the same attributes as before. function startRequestSpan(name: string, fields: StartSpanOptions): SpanHandle { const cut = diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 4467aa3..06518ee 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -8,7 +8,14 @@ import { InMemoryMetricExporter, type PushMetricExporter, } from "@opentelemetry/sdk-metrics"; -import { flush, init, shutdown, type TelemetryOptions } from "@telemetry-dev/sdk"; +import { + type ChatMessage, + flush, + getPrompt, + init, + shutdown, + type TelemetryOptions, +} from "@telemetry-dev/sdk"; import * as telemetrySdk from "@telemetry-dev/sdk"; import { EventEmitter } from "node:events"; import { createRequire } from "node:module"; @@ -189,7 +196,7 @@ function setupSpans( metricExporter?: PushMetricExporter, options: Pick< TelemetryOptions, - "captureOutput" | "mask" | "maxAttributeLength" | "onError" | "sampler" + "captureInput" | "captureOutput" | "mask" | "maxAttributeLength" | "onError" | "sampler" > = {}, ): InMemorySpanExporter { const spanExporter = new InMemorySpanExporter(); @@ -2887,6 +2894,107 @@ test("chat streams preserve caller-requested usage chunks", async () => { expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(7); }); +test.each([ + { api: "chat completions", captureInput: false, olderCore: false }, + { api: "responses instructions", captureInput: true, olderCore: false }, + { api: "responses input", captureInput: true, olderCore: false }, + { api: "chat completions", captureInput: true, olderCore: true }, +])( + "$api outside withPrompt record the prompt of compiled messages, input capture $captureInput, older core $olderCore", + async ({ api, captureInput, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: "https://prompts.openai.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "openai", name: "gpt-4o" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const spans = setupSpans(undefined, { captureInput }); + const client = clientWith( + createFakeFetch( + jsonResponse( + api.startsWith("responses") + ? { + id: "resp_1", + object: "response", + status: "completed", + model: "gpt-4.1", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + } + : { + id: "chatcmpl_1", + object: "chat.completion", + created: 1, + model: "gpt-4o", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Soon." }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ), + ).fetch, + ); + + if (api.startsWith("responses")) { + const inInput = api === "responses input"; + + await client.responses.create({ + model: "gpt-4.1", + instructions: inInput ? "Be terse." : system!.content, + input: [{ role: "user", content: inInput ? user!.content : "Hi" }], + }); + } else { + await client.chat.completions.create({ model: "gpt-4o", messages: [system!, user!] }); + } + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.input.messages"] !== undefined).toBe(captureInput); + + if (olderCore) { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + expect(attributes["td.prompt.variables"]).toBe( + captureInput ? '{"team":"billing","order":"#4"}' : undefined, + ); + }, +); + test("responses create maps instructions to system instructions", async () => { const spans = setupSpans(); diff --git a/packages/openrouter/src/index.ts b/packages/openrouter/src/index.ts index bfb0076..e0d696e 100644 --- a/packages/openrouter/src/index.ts +++ b/packages/openrouter/src/index.ts @@ -4,6 +4,7 @@ import { type SpanHandle, type StartSpanOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { activeContext } from "@telemetry-dev/otel"; import { Chat } from "@openrouter/sdk/sdk/chat.js"; import { Embeddings } from "@openrouter/sdk/sdk/embeddings.js"; @@ -384,6 +385,16 @@ function usageCost(usage: T): number | undefined { return cost ?? upstreamCost; } +// Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read +// keeps them loadable without the export. +function promptAttributes(request: unknown): SpanFields["attributes"] { + const read = ( + telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + ).promptAttributesFor; + + return typeof read === "function" ? read(request) : undefined; +} + function chatRequest(request: JsonRecord): RequestMapping { const body = asRecord(request.chatRequest) ?? {}; const model = readString(body.model); @@ -403,6 +414,7 @@ function chatRequest(request: JsonRecord): RequestMapping { seed: readNumber(body.seed), frequencyPenalty: readNumber(body.frequencyPenalty), presencePenalty: readNumber(body.presencePenalty), + attributes: promptAttributes(body.messages), }, }; } @@ -472,6 +484,7 @@ function responsesRequest(request: JsonRecord): RequestMapping { maxTokens: readNumber(body.maxOutputTokens), frequencyPenalty: readNumber(body.frequencyPenalty), presencePenalty: readNumber(body.presencePenalty), + attributes: promptAttributes([body.instructions, body.input]), }, }; } diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index 2dc9baf..dc67157 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -5,13 +5,16 @@ import { Embeddings } from "@openrouter/sdk/sdk/embeddings.js"; import { EventStream } from "@openrouter/sdk/lib/event-streams.js"; import { Responses } from "@openrouter/sdk/sdk/responses.js"; import { + type ChatMessage, flush, + getPrompt, init, type MaskFn, shutdown, startActiveSpan, type TelemetryOptions, } from "@telemetry-dev/sdk"; +import * as telemetrySdk from "@telemetry-dev/sdk"; import { afterEach, expect, test, vi } from "vitest"; import { instrumentOpenRouter, uninstrumentOpenRouter, wrapOpenRouter } from "../src/index.ts"; @@ -308,6 +311,96 @@ afterEach(async () => { vi.restoreAllMocks(); }); +test.each([ + { api: "chat", olderCore: false }, + { api: "streamed responses instructions", olderCore: false }, + { api: "responses input", olderCore: false }, + { api: "chat", olderCore: true }, +])( + "$api outside withPrompt record the prompt of compiled messages, older core $olderCore", + async ({ api, olderCore }) => { + if (olderCore) { + vi.spyOn( + telemetrySdk as { promptAttributesFor: unknown }, + "promptAttributesFor", + "get", + ).mockReturnValue(undefined); + } + + const prompt = await getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: "https://prompts.openrouter.test", + fetch: async () => + Response.json({ + name: "support/reply", + type: "chat", + label: "production", + version: 12, + messages: [ + { role: "system", content: "You answer for the {{team}} desk." }, + { role: "user", content: "Where is order {{order}}?" }, + ], + text: null, + model: { provider: "openrouter", name: "openai/gpt-4o" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + rollout: null, + }), + }); + const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const spans = setupSpans(); + + if (api === "chat") { + const client = clientWith(createFakeFetcher(jsonResponse(chatBody("chat_1"))).fetcher); + await client.chat.send({ + chatRequest: { model: "openai/gpt-4o", messages: [system!, user!] }, + }); + } else if (api === "responses input") { + const client = clientWith(createFakeFetcher(jsonResponse(responsesBody("resp_1"))).fetcher); + await client.responses.send({ + responsesRequest: { + model: "openai/gpt-4o", + input: [{ type: "message", role: "user", content: user!.content }], + }, + }); + } else { + const client = clientWith( + createFakeFetcher( + sseResponse([{ type: "response.completed", response: responsesBody("resp_1") }]), + ).fetcher, + ); + const stream = await client.responses.send({ + responsesRequest: { + model: "openai/gpt-4o", + instructions: system!.content, + input: "Hi", + stream: true, + }, + }); + + if (!(stream instanceof ReadableStream)) throw new Error("expected a readable stream"); + await collectStream(stream); + } + + const { attributes } = await exportedSpan(spans); + expect(attributes["gen_ai.request.model"]).toBe("openai/gpt-4o"); + + if (olderCore) { + expect(attributes["td.prompt.name"]).toBeUndefined(); + + return; + } + + expect(attributes).toMatchObject({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }); + }, +); + test("chat maps request, response, usage, primary cost, provider, and sampling fields", async () => { const spans = setupSpans(); diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef1..00cd298 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -179,8 +179,18 @@ compiles its own. Model calls are the spans of type `generation` (operations `ch `text_completion`, and `generate_content`), including the spans of the provider integrations that use this SDK: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, and OpenRouter. Spans from the AI SDK and TanStack AI integrations, and from other OpenTelemetry tracers, do not get the attributes. -A model call outside `withPrompt` does not get the attributes, even when it sends messages from -`compile()`; with input capture on, the call still records those messages as its input. + +Outside `withPrompt`, those provider integrations find the prompt in the request: +`compile()` keeps the texts that it makes (the last 1000, at most 2,000,000 characters), and a +chat call whose request holds one of them gets `td.prompt.name`, `td.prompt.version`, and +`td.prompt.label`, also when input capture is off. A text from a template with a variable also +gives `td.prompt.variables`, and it wins over a text from a template with none. A message of only +a variable tag, for example `{{ question }}`, is not kept, because its text is your user's input. +The text in the request must be equal to the compiled text. `withPrompt` wins over this. The +integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse +and InvokeModel, Google GenAI `generateContent`, and OpenRouter chat and responses, but not for +images, audio, video, or OpenAI Realtime. For your own spans, `promptAttributesFor(request)` +gives the attributes to pass as `attributes`; it gives nothing inside `withPrompt`. ## API diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2fc3343..18d8946 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -54,6 +54,7 @@ export { type GetPromptOptions, type GetTextPromptOptions, type Prompt, + promptAttributesFor, PromptError, type PromptMessage, type PromptModel, diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b17..afd76c2 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -1,4 +1,9 @@ -import { type Attributes, type Context, createContextKey } from "@opentelemetry/api"; +import { + type Attributes, + type AttributeValue, + type Context, + createContextKey, +} from "@opentelemetry/api"; import { activeContext, DEFAULT_BASE_URL, @@ -92,6 +97,8 @@ export interface ChatPrompt extends PromptBase { /** * Fills in the variables and replaces each placeholder message with the messages in its * variable. `withPrompt` records the variables of the last call before it starts or inside it. + * Each text is also kept with its variables (the last 1000 texts), so that a model call outside + * `withPrompt` that sends it can name the prompt; see `promptAttributesFor`. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -102,7 +109,8 @@ export interface TextPrompt extends PromptBase { readonly text: string; /** * Fills in the variables. `withPrompt` records the variables of the last call before it starts - * or inside it. + * or inside it. The text is also kept with its variables (the last 1000 texts), so that a model + * call outside `withPrompt` that sends it can name the prompt; see `promptAttributesFor`. */ compile(variables?: PromptVariables): string; } @@ -193,6 +201,13 @@ interface PromptScope { variables: object | undefined; } +/** The prompt that `compile()` made a text from. */ +interface CompiledText { + readonly prompt: Prompt; + /** Only for a template with a variable: the text holds the values, so they belong to it. */ + readonly variables: object | undefined; +} + const FETCH_TIMEOUT_MS = 10_000; /** The longest Retry-After that getPrompt honors, so that a bad header cannot stop fetches. */ const MAX_RETRY_AFTER_MS = 3_600_000; @@ -204,11 +219,19 @@ const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} G const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); +const MAX_COMPILED_TEXTS = 1_000; +const MAX_COMPILED_CHARACTERS = 2_000_000; +const MAX_SEARCH_DEPTH = 6; +const MAX_SEARCH_NODES = 500; const cache = new Map(); const compiledVariables = new WeakMap(); /** The Retry-After of the response that failed with this error, in milliseconds. */ const retryAfter = new WeakMap(); +// The texts of the last compile() calls, oldest first, for model calls outside withPrompt. +const compiledTexts = new Map(); +let compiledCharacters = 0; +let longestCompiledText = 0; /** * Fetches a prompt version from telemetry.dev and keeps it. When the copy is older than @@ -344,19 +367,97 @@ export function promptAttributes( | undefined; if (!scope) return undefined; - const { prompt, variables } = scope; + return attributesOf(scope.prompt, scope.variables, capture); +} + +/** + * The td.prompt.* attributes for a model call outside `withPrompt`, from the texts that + * `compile()` made: the first text in `request` from a template with a variable, else the first + * text from a template with none. Only the first gives `td.prompt.variables`, because only its text + * holds the values. Gives undefined inside `withPrompt`, whose prompt the span gets, and when no + * text matches. It reads own data properties of arrays and plain objects, 6 levels and 500 values + * deep, and runs no getter. It never throws. + */ +export function promptAttributesFor(request: unknown): Record | undefined { + const config = currentClient().core?.config; + + try { + if (compiledTexts.size === 0 || activeContext().getValue(PROMPT_KEY) !== undefined) { + return undefined; + } + + const found = findCompiledText(request); + + if (!found) return undefined; + + return attributesOf(found.prompt, found.variables, config) as Record; + } catch (error) { + reportError(config?.onError, error); + + return undefined; + } +} + +function attributesOf( + prompt: Prompt, + variables: object | undefined, + capture: FieldCaptureConfig | undefined, +): Attributes { return omitUndefined({ "td.prompt.name": prompt.name, "td.prompt.version": prompt.version ?? undefined, "td.prompt.label": prompt.label ?? undefined, "td.prompt.variables": - variables !== undefined && capture.captureInput + variables !== undefined && capture?.captureInput ? prepareCaptureValue("td.prompt.variables", variables, capture) : undefined, }); } +function findCompiledText(request: unknown): CompiledText | undefined { + let firstStatic: CompiledText | undefined; + let budget = MAX_SEARCH_NODES; + const stack: [value: unknown, depth: number][] = [[request, 0]]; + + while (stack.length > 0 && budget > 0) { + budget -= 1; + const [value, depth] = stack.pop()!; + + if (typeof value === "string") { + const found = value.length <= longestCompiledText ? compiledTexts.get(value) : undefined; + + if (found?.variables !== undefined) return found; + firstStatic ??= found; + } else if (depth < MAX_SEARCH_DEPTH) { + const children = ownValues(value, budget); + + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push([children[index], depth + 1]); + } + } + } + + return firstStatic; +} + +/** At most `limit` own data values of an array or a plain object, in order. */ +function ownValues(value: unknown, limit: number): unknown[] { + if (typeof value !== "object" || value === null) return []; + let keys: string[]; + + if (Array.isArray(value)) { + keys = Array.from({ length: Math.min(value.length, limit) }, (_, index) => String(index)); + } else { + const prototype: unknown = Object.getPrototypeOf(value); + + if (prototype !== Object.prototype && prototype !== null) return []; + keys = Object.keys(value).slice(0, limit); + } + + return keys.map((key) => Object.getOwnPropertyDescriptor(value, key)?.value); +} + function checkOptions(name: string, options: GetPromptOptions, type: string): void { if (typeof name !== "string" || name.length === 0) { throw new TypeError("name must be a non-empty string"); @@ -548,9 +649,12 @@ function createPrompt(input: { if (scope?.prompt === prompt) scope.variables = variables; + const compileText = (template: string) => + remember(fill(template, variables), template, prompt, variables); + return type === "chat" - ? compileMessages(content.messages, variables) - : fill(content.text ?? "", variables); + ? compileMessages(content.messages, variables, compileText) + : compileText(content.text ?? ""); }, }) as Prompt; @@ -560,10 +664,11 @@ function createPrompt(input: { function compileMessages( messages: readonly PromptMessage[], variables: PromptVariables, + compileText: (template: string) => string, ): unknown[] { return messages.flatMap((message) => { if (message.role !== "placeholder") { - return [{ role: message.role, content: fill(message.content, variables) }]; + return [{ role: message.role, content: compileText(message.content) }]; } const value = Object.hasOwn(variables, message.name) ? variables[message.name] : undefined; @@ -578,6 +683,39 @@ function compileMessages( }); } +/** + * Keeps `text` for promptAttributesFor, as the newest entry. A template of only variable tags is + * skipped: its text is input of the app, which an unrelated request can hold too. + */ +function remember(text: string, template: string, prompt: Prompt, variables: object): string { + const fixedText = template.replace(VARIABLE, ""); + + if (fixedText.trim() === "" || text.length > MAX_COMPILED_CHARACTERS) return text; + + if (compiledTexts.delete(text)) compiledCharacters -= text.length; + compiledTexts.set(text, { prompt, variables: fixedText === template ? undefined : variables }); + compiledCharacters += text.length; + longestCompiledText = Math.max(longestCompiledText, text.length); + let droppedLongest = false; + + while (compiledTexts.size > MAX_COMPILED_TEXTS || compiledCharacters > MAX_COMPILED_CHARACTERS) { + const oldest = compiledTexts.keys().next().value!; + compiledTexts.delete(oldest); + compiledCharacters -= oldest.length; + droppedLongest ||= oldest.length === longestCompiledText; + } + + if (droppedLongest) { + longestCompiledText = 0; + + for (const kept of compiledTexts.keys()) { + longestCompiledText = Math.max(longestCompiledText, kept.length); + } + } + + return text; +} + function fill(text: string, variables: PromptVariables): string { return text.replace(VARIABLE, (_tag, name: string) => { const value = Object.hasOwn(variables, name) ? variables[name] : undefined; diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index 6048d16..372150e 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -4,8 +4,11 @@ import type { AddressInfo } from "node:net"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { + type ChatMessage, flush, getPrompt, + type Prompt, + promptAttributesFor, propagateAttributes, PromptError, shutdown, @@ -729,6 +732,231 @@ test("variables are omitted when input capture is off; a version fetch has no la expect(off.spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBeUndefined(); }); +async function promptOf(overrides: Record): Promise { + const { fetch } = fakeFetch(reply(overrides)); + + return getPrompt("support/reply", { + apiKey: "td_live_a", + apiUrl: newApiUrl(), + fetch, + type: overrides.type as "chat", + }); +} + +test("promptAttributesFor finds a compiled text in the request shapes of the providers", async () => { + setup(); + const prompt = await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "user", content: "Order {{order}}: where is it?" }, + ], + }); + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const [system, user] = compiled.map((message) => message.content); + + const requests = { + same: compiled, + spread: compiled.map((message) => ({ ...message })), + json: JSON.parse(JSON.stringify(compiled)) as unknown, + anthropic: [system, compiled.slice(1)], + bedrock: [[{ text: system }], [{ role: "user", content: [{ text: user }] }]], + google: [{ parts: [{ text: system }] }, [{ role: "user", parts: [{ text: user }] }]], + }; + + for (const request of Object.values(requests)) { + expect(promptAttributesFor(request)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + }); + } + + expect( + promptAttributesFor([{ role: "user", content: "Order #5: where is it?" }]), + ).toBeUndefined(); +}); + +test("a static text gives no variables, and a template of only a variable tag is not kept", async () => { + setup(); + const chat = await promptOf({ + messages: [ + { role: "system", content: "You are the refunds agent." }, + { role: "user", content: "{{ question }}" }, + ], + }); + const compiled = chat.compile({ question: "Where is my refund?" }); + + expect(promptAttributesFor(compiled)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + expect(promptAttributesFor([{ role: "user", content: "Where is my refund?" }])).toBeUndefined(); + + const text = await promptOf({ type: "text", messages: [], text: "Summarize {{doc}} in a line." }); + const summary = text.compile({ doc: "the refund policy" }) as string; + + expect(promptAttributesFor(summary)).toMatchObject({ + "td.prompt.variables": '{"doc":"the refund policy"}', + }); +}); + +test("a text with variables wins over a static text, and the first one wins over a later one", async () => { + setup(); + const refunds = await promptOf({ + name: "refunds/system", + messages: [{ role: "system", content: "You are the refunds agent, v2." }], + }); + const order = await promptOf({ + name: "orders/question", + messages: [{ role: "user", content: "Where is order {{order}}?" }], + }); + const language = await promptOf({ + name: "style/language", + messages: [{ role: "system", content: "Reply in {{language}}." }], + }); + + const system = refunds.compile() as ChatMessage[]; + const question = order.compile({ order: "#4" }) as ChatMessage[]; + const style = language.compile({ language: "French" }) as ChatMessage[]; + + expect(promptAttributesFor([...system, ...question])).toMatchObject({ + "td.prompt.name": "orders/question", + "td.prompt.variables": '{"order":"#4"}', + }); + expect(promptAttributesFor([...system, ...style, ...question])).toMatchObject({ + "td.prompt.name": "style/language", + "td.prompt.variables": '{"language":"French"}', + }); +}); + +test("each compile keeps its own values, so two tasks with one prompt get their own variables", async () => { + setup(); + const prompt = await promptOf({ + messages: [{ role: "user", content: "Track parcel {{parcel}}." }], + }); + + const first = prompt.compile({ parcel: "A-1" }); + const second = prompt.compile({ parcel: "B-2" }); + + expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe('{"parcel":"A-1"}'); + expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe('{"parcel":"B-2"}'); +}); + +test("withPrompt wins over a compiled text of another prompt", async () => { + const { spans } = setup(); + const scoped = await promptOf({ name: "scoped/prompt" }); + const other = await promptOf({ + name: "other/prompt", + messages: [{ role: "user", content: "Cancel order {{order}}." }], + }); + const compiled = other.compile({ order: "#9" }); + + withPrompt(scoped, () => { + startSpan("chat", { type: "generation", attributes: promptAttributesFor(compiled) }).end(); + }); + await flush(); + + expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.name"]).toBe("scoped/prompt"); +}); + +test("promptAttributesFor omits variables when input capture is off and masks them when on", async () => { + const prompt = await promptOf({ + messages: [{ role: "user", content: "My card is {{card}}." }], + }); + const compiled = prompt.compile({ card: "4111 1111" }); + + setup({ captureInput: false }); + expect(promptAttributesFor(compiled)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + + await shutdown(); + setup({ + mask: (value, { key }) => (key === "td.prompt.variables" ? { card: "[masked]" } : value), + }); + expect(promptAttributesFor(compiled)?.["td.prompt.variables"]).toBe('{"card":"[masked]"}'); +}); + +test("the kept texts stop at 1000 and at 2,000,000 characters, oldest out first", async () => { + setup(); + const ticket = await promptOf({ type: "text", messages: [], text: "Ticket {{n}} is open." }); + + for (let n = 0; n <= 1000; n += 1) ticket.compile({ n }); + + expect(promptAttributesFor("Ticket 0 is open.")).toBeUndefined(); + expect(promptAttributesFor("Ticket 1 is open.")).toMatchObject({ + "td.prompt.variables": '{"n":1}', + }); + + const note = await promptOf({ type: "text", messages: [], text: "Note: {{body}}" }); + const notes = ["a", "b", "c", "d", "e"].map( + (letter) => note.compile({ body: letter.repeat(500_000 - "Note: ".length) }) as string, + ); + + expect(promptAttributesFor(notes[0])).toBeUndefined(); + expect(promptAttributesFor(notes[1])?.["td.prompt.name"]).toBe("support/reply"); + + // A text over the character limit is not kept, and it does not push the others out. + note.compile({ body: "f".repeat(2_000_001) }); + expect(promptAttributesFor(notes[1])?.["td.prompt.name"]).toBe("support/reply"); + + // A text compiled again counts once. + for (let n = 0; n < 5; n += 1) note.compile({ body: "e".repeat(500_000 - "Note: ".length) }); + expect(promptAttributesFor(notes[4])?.["td.prompt.name"]).toBe("support/reply"); +}); + +test("promptAttributesFor looks 6 levels and 500 values deep", async () => { + setup(); + const prompt = await promptOf({ + messages: [{ role: "user", content: "Find invoice {{invoice}}." }], + }); + const [{ content }] = prompt.compile({ invoice: "INV-7" }) as ChatMessage[]; + const nest = (value: unknown, levels: number): unknown => + levels === 0 ? value : nest([value], levels - 1); + + expect(promptAttributesFor(nest(content, 6))).toBeDefined(); + expect(promptAttributesFor(nest(content, 7))).toBeUndefined(); + + // The request array, 498 strings, and the text are 500 values. + expect(promptAttributesFor([...Array(498).fill("filler"), content])).toBeDefined(); + expect(promptAttributesFor([...Array(499).fill("filler"), content])).toBeUndefined(); +}); + +test("promptAttributesFor runs no getter and reports a trap that throws", async () => { + const errors: Error[] = []; + setup({ onError: (error) => errors.push(error) }); + const prompt = await promptOf({ + messages: [{ role: "user", content: "Refund order {{order}}." }], + }); + const [{ content }] = prompt.compile({ order: "#3" }) as ChatMessage[]; + let reads = 0; + + const getter = { + role: "user", + get content() { + reads += 1; + + return content; + }, + }; + + expect(promptAttributesFor([getter])).toBeUndefined(); + expect(reads).toBe(0); + + const trap = new Proxy([], { + get() { + throw new Error("trap"); + }, + }); + + expect(promptAttributesFor([trap, content])).toBeUndefined(); + expect(errors.map((error) => error.message)).toEqual(["trap"]); +}); + test("a fallback prompt records its name and label but no version", async () => { vi.spyOn(console, "error").mockImplementation(() => {}); const { spans } = setup(); diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index d842e15..956b634 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -244,6 +244,12 @@ def _offered_tools(tools: Any) -> dict[str, str] | None: return {"gen_ai.tool.definitions": json_dumps(definitions)} if definitions else None +def _prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def _messages_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: params = _sent_params(params) model = _string(params.get("model")) @@ -264,7 +270,11 @@ def _messages_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "type": "generation", "model": model, "input": input_value, - "attributes": _offered_tools(tools), + "attributes": { + **(_offered_tools(tools) or {}), + **_prompt_attributes([params.get("system"), params.get("messages")]), + } + or None, "system_instructions": _native(params.get("system")), "temperature": _number(params.get("temperature")), "top_p": _number(params.get("top_p")), diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 3911ffb..8ffbc67 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -421,6 +421,56 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.cache_read.input_tokens"] == 1000 +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("field", "capture_input", "older_core"), + [("system", False, False), ("messages", True, False), ("messages", True, True)], +) +def test_create_outside_with_prompt_records_the_prompt_of_a_compiled_text( + make: Any, monkeypatch: pytest.MonkeyPatch, field: str, capture_input: bool, older_core: bool +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + client = wrapped_sync_client(lambda _request: json_response(message_payload())) + + client.messages.create( + model="claude-sonnet-4-6", + max_tokens=64, + system=system["content"] if field == "system" else "Be terse.", + messages=[{"role": "user", "content": user["content"] if field == "messages" else "Hi"}], + ) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "claude-sonnet-4-6" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + def test_create_maps_native_messages_system_params_usage_finish_and_provider( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py index 1017467..04af28b 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Any +from typing import Any, cast +import telemetry_dev from telemetry_dev._serialize import json_dumps PROVIDER = "amazon-bedrock" @@ -11,6 +12,12 @@ def clean(fields: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in fields.items() if value is not None} +def prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def metadata_fields(response: dict[str, Any] | BaseException | None) -> dict[str, Any]: metadata: dict[str, Any] | None = None if isinstance(response, dict) and isinstance(response.get("ResponseMetadata"), dict): diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py index d22e50b..04a1162 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py @@ -4,7 +4,7 @@ import math from typing import Any -from ._fields import clean +from ._fields import clean, prompt_attributes _USAGE_ITERATION_LIMIT = 1000 _CLAUDE_USAGE_KEYS = ( @@ -93,6 +93,8 @@ def request_fields(params: dict[str, Any]) -> dict[str, Any]: fields = {"provider": "amazon-bedrock", "model": model, "input": body} if is_embedding_model(model): fields["output_type"] = "embedding" + else: + fields["attributes"] = prompt_attributes(body) or None fields.update(sampling_fields(model, body)) return clean(fields) diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py index 6d09848..0b68201 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py @@ -6,7 +6,14 @@ import telemetry_dev -from ._fields import PROVIDER, clean, merge_fields, metadata_fields, usage_from_converse +from ._fields import ( + PROVIDER, + clean, + merge_fields, + metadata_fields, + prompt_attributes, + usage_from_converse, +) from ._invoke_model import is_embedding_model from ._invoke_model import request_fields as invoke_request_fields from ._invoke_model import response_fields as invoke_response_fields @@ -50,6 +57,7 @@ def _converse_request(params: dict[str, Any]) -> dict[str, Any]: "model": _model(params), "input": normalize_messages(params.get("messages")), "system_instructions": normalize_content_list(params.get("system")) or None, + "attributes": prompt_attributes([params.get("system"), params.get("messages")]) or None, "temperature": inference.get("temperature"), "top_p": inference.get("topP"), "max_tokens": inference.get("maxTokens"), diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 3b6acac..8919186 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -114,6 +114,93 @@ def _json_attr(span: Any, key: str) -> Any: return json.loads(value) +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize("older_core", [False, True]) +def test_converse_and_invoke_model_outside_with_prompt_record_the_prompt_of_a_compiled_text( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, older_core: bool +) -> None: + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + converse_reply = { + "output": {"message": {"role": "assistant", "content": [{"text": "Soon."}]}}, + "stopReason": "end_turn", + } + answer = _body({"content": [{"type": "text", "text": "Soon."}], "stop_reason": "end_turn"}) + embedding = _body({"embedding": [0.1, 0.2], "inputTextTokenCount": 6}) + client = _client("bedrock-runtime") + _stub_api_call( + client, + [ + converse_reply, + converse_reply, + { + "body": StreamingBody(io.BytesIO(answer), len(answer)), + "contentType": "application/json", + }, + { + "body": StreamingBody(io.BytesIO(embedding), len(embedding)), + "contentType": "application/json", + }, + ], + monkeypatch, + ) + wrapped = cast(Any, wrap_bedrock(client)) + + wrapped.converse( + modelId="anthropic.claude-3-haiku", + system=[{"text": system["content"]}], + messages=[{"role": "user", "content": [{"text": "Hi"}]}], + ) + wrapped.converse( + modelId="anthropic.claude-3-haiku", + messages=[{"role": "user", "content": [{"text": user["content"]}]}], + ) + wrapped.invoke_model( + modelId="anthropic.claude-3-haiku", + contentType="application/json", + body=_body({"max_tokens": 64, "messages": [{"role": "user", "content": user["content"]}]}), + )["body"].read() + wrapped.invoke_model( + modelId="amazon.titan-embed-text-v2:0", + contentType="application/json", + body=_body({"inputText": user["content"]}), + )["body"].read() + + system_span, messages_span, invoke_span, embed_span = ( + dict(span.attributes or {}) for span in memory.span_exporter.get_finished_spans() + ) + for span in (system_span, messages_span, invoke_span): + assert span["gen_ai.request.model"] == "anthropic.claude-3-haiku" + assert {key: value for key, value in span.items() if key.startswith("td.prompt.")} == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + } + ) + assert embed_span["gen_ai.operation.name"] == "embeddings" + assert "td.prompt.name" not in embed_span + + def test_converse_happy_path(memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch) -> None: client = _client("bedrock-runtime") request = { diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index 5782425..776c9de 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -353,6 +353,12 @@ def _request_attributes(config: Any) -> dict[str, Any]: return attrs +def _prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[str, dict[str, Any]]: model_name = _string(model) or "unknown" fields: dict[str, Any] = { @@ -383,7 +389,7 @@ def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[st output_type = _output_type(config) if output_type is not None: fields["output_type"] = output_type - attrs = _request_attributes(config) + attrs = {**_request_attributes(config), **_prompt_attributes([system_instruction, contents])} if attrs: fields["attributes"] = attrs return f"chat {model_name}", fields diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index f045e68..c74c08c 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -656,6 +656,62 @@ async def run() -> Any: assert attrs(span)["google_genai.response.operation_done"] is True +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("field", "capture_input", "older_core"), + [("system_instruction", True, False), ("contents", False, False), ("contents", True, True)], +) +def test_generate_content_outside_with_prompt_records_the_prompt_of_a_compiled_text( + make: Any, monkeypatch: pytest.MonkeyPatch, field: str, capture_input: bool, older_core: bool +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + client, _transport = client_with_transport([happy_payload()]) + + wrap_google_genai(client).models.generate_content( + model="gemini-2.5-flash", + contents=[ + types.Content( + role="user", + parts=[types.Part(text=user["content"] if field == "contents" else "Hi")], + ) + ], + config=types.GenerateContentConfig( + system_instruction=system["content"] if field == "system_instruction" else "Be terse." + ), + ) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "gemini-2.5-flash" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_generate_content_maps_request_response_usage_and_sampling( memory: SimpleNamespace, async_mode: bool diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index e149a00..ee18732 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -455,6 +455,12 @@ def _output_type(response_format: Any) -> str | None: return kind +def _prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def _completion_request( args: tuple[Any, ...], kwargs: Mapping[str, Any] ) -> tuple[str, dict[str, Any]]: @@ -478,6 +484,7 @@ def _completion_request( "presence_penalty": _number(kwargs.get("presence_penalty")), "output_type": _output_type(kwargs.get("response_format")), "metadata": _request_metadata(kwargs), + "attributes": _prompt_attributes(_arg(args, kwargs, "messages", 1)) or None, }, ) @@ -545,7 +552,10 @@ def _responses_request( input_budget: telemetry_dev.CaptureBudget | None = None if capture_input: input_value, input_budget = _bounded_responses_native(_arg(args, kwargs, "input", 0)) - attributes: dict[str, Any] = {"gen_ai.operation.name": "chat"} + attributes: dict[str, Any] = { + "gen_ai.operation.name": "chat", + **_prompt_attributes([kwargs.get("instructions"), _arg(args, kwargs, "input", 0)]), + } if input_budget is not None and input_budget.truncated: attributes["telemetry.dev.capture.truncated"] = True return ( diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index f7b08a1..ae34865 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -57,6 +57,70 @@ def finished_spans(env: SimpleNamespace) -> list[ReadableSpan]: return list(env.span_exporter.get_finished_spans()) +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("api", "capture_input", "older_core"), + [ + ("completion", False, False), + ("responses instructions", True, False), + ("responses input", True, False), + ("completion", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + + if api == "completion": + telemetry_dev_litellm.completion( + model="gpt-4o-mini", messages=[system, user], mock_response="Soon." + ) + else: + + def fake_responses(*_args: Any, **_kwargs: Any) -> SimpleNamespace: + return responses_result() + + monkeypatch.setattr(litellm, "responses", fake_responses) + in_input = api == "responses input" + telemetry_dev_litellm.responses( + [{"role": "user", "content": user["content"] if in_input else "Hi"}], + "openai/gpt-4.1-mini", + instructions="Be terse." if in_input else system["content"], + ) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == ("gpt-4o-mini" if api == "completion" else "gpt-4.1-mini") + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + def test_completion_maps_model_messages_usage_finish_provider_and_sampling( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index ee02a97..eb84699 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -349,9 +349,18 @@ def _capture_text(value: str, budget: telemetry_dev.CaptureBudget) -> str: return value +def _prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) + attributes = _prompt_attributes(params.get("messages")) captured, truncated = _bounded_responses_capture(params.get("messages"), "input") + if truncated: + attributes["telemetry.dev.capture.truncated"] = True return ( f"chat {model or 'unknown'}", { @@ -366,7 +375,7 @@ def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "seed": _number(params.get("seed")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), - "attributes": {"telemetry.dev.capture.truncated": True} if truncated else None, + "attributes": attributes or None, }, ) @@ -432,7 +441,10 @@ def _chat_response(response: Any) -> dict[str, Any]: def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) + attributes = _prompt_attributes([params.get("instructions"), params.get("input")]) captured, truncated = _bounded_responses_capture(params.get("input"), "input") + if truncated: + attributes["telemetry.dev.capture.truncated"] = True return ( f"chat {model or 'unknown'}", { @@ -443,7 +455,7 @@ def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "temperature": _number(params.get("temperature")), "top_p": _number(params.get("top_p")), "max_tokens": _number(params.get("max_output_tokens")), - "attributes": {"telemetry.dev.capture.truncated": True} if truncated else None, + "attributes": attributes or None, }, ) diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 238c7e6..6f71978 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1206,6 +1206,72 @@ def text(self) -> str: assert a["gen_ai.usage.text.output_tokens"] == 2 +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +def prompt_attrs(span: ReadableSpan) -> dict[str, object]: + return {key: value for key, value in attrs(span).items() if key.startswith("td.prompt.")} + + +@pytest.mark.parametrize( + ("api", "capture_input", "older_core"), + [ + ("chat", False, False), + ("responses instructions", True, False), + ("responses input", True, False), + ("chat", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + + def handler(_request: httpx.Request) -> httpx.Response: + return json_response(chat_completion() if api == "chat" else response_payload()) + + client = wrap_openai(sync_client(handler)) + if api == "chat": + cast(Any, client.chat.completions.create)(model="gpt-4o", messages=[system, user]) + else: + in_input = api == "responses input" + cast(Any, client.responses.create)( + model="gpt-4.1", + instructions="Be terse." if in_input else system["content"], + input=[{"role": "user", "content": user["content"] if in_input else "Hi"}], + ) + + span = only_span(memory) + assert attrs(span)["gen_ai.request.model"] == ("gpt-4o" if api == "chat" else "gpt-4.1") + assert prompt_attrs(span) == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + assert ("gen_ai.input.messages" in attrs(span)) is capture_input + + def test_chat_completion_maps_native_messages_usage_finish_provider_and_sampling( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index 016014c..4af3fb7 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -174,6 +174,12 @@ def _cost_usd(raw_usage: Any) -> float | None: return cost if cost is not None else upstream_cost +def _prompt_attributes(request: object) -> dict[str, Any]: + # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. + read = getattr(telemetry_dev, "prompt_attributes_for", None) + return cast("dict[str, Any]", read(request)) if callable(read) else {} + + def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) max_completion_tokens = _number(params.get("max_completion_tokens")) @@ -193,6 +199,7 @@ def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "seed": _number(params.get("seed")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), + "attributes": _prompt_attributes(params.get("messages")) or None, }, ) @@ -260,6 +267,8 @@ def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "max_tokens": _number(params.get("max_output_tokens")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), + "attributes": _prompt_attributes([params.get("instructions"), params.get("input")]) + or None, }, ) diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 1efe3a5..01456c2 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -221,6 +221,73 @@ def embeddings_payload() -> dict[str, Any]: } +SUPPORT_PROMPT = telemetry_dev.ChatPrompt( + name="support/reply", + version=12, + label="production", + messages=[ + {"role": "system", "content": "You answer for the {{team}} desk."}, + {"role": "user", "content": "Where is order {{order}}?"}, + ], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + +@pytest.mark.parametrize( + ("api", "capture_input", "older_core"), + [ + ("chat", False, False), + ("responses instructions", True, False), + ("responses input", True, False), + ("chat", True, True), + ], +) +def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( + make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool +) -> None: + memory = make(capture_input=capture_input) + if older_core: + monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") + system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + + def handler(_request: httpx.Request) -> httpx.Response: + return json_response(chat_payload() if api == "chat" else responses_payload()) + + client = wrap_open_router(sync_client(handler)) + if api == "chat": + client.chat.send( + model="openai/gpt-4o-mini", + messages=[ + ChatSystemMessage(role="system", content=system["content"]), + ChatUserMessage(role="user", content=user["content"]), + ], + ) + else: + in_input = api == "responses input" + client.responses.send( + model="openai/gpt-4o-mini", + input=[EasyInputMessage(role="user", content=user["content"] if in_input else "Hi")], + instructions="Be terse." if in_input else system["content"], + ) + + a = attrs(only_span(memory)) + assert a["gen_ai.request.model"] == "openai/gpt-4o-mini" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), + } + ) + + def test_chat_send_maps_messages_usage_cost_and_sampling(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return json_response(chat_payload()) diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d10..dfcd2b8 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -251,9 +251,21 @@ ran in it, or before it started in the code that started it; with none, it recor for code that has not compiled in its own task. Model calls are the spans of type `generation` (operations `chat`, `text_completion`, and `generate_content`), including the spans of the provider integrations: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, LiteLLM, and OpenRouter. Spans from -other OpenTelemetry tracers do not get the attributes. A model call outside `with_prompt` does not -get the attributes, even when it sends messages from `compile()`; with input capture on, the call -still records those messages as its input. +other OpenTelemetry tracers do not get the attributes. + +Outside `with_prompt`, those provider integrations find the prompt in the request: `compile()` +keeps the texts that it makes (the last 1000, at most 2,000,000 characters), and a chat call +whose request holds one of them gets `td.prompt.name`, `td.prompt.version`, and +`td.prompt.label`, also when input capture is off. A text from a template with a variable also +gives `td.prompt.variables`, and it wins over a text from a template with none. A message of only +a variable tag, for example `{{ question }}`, is not kept, because its text is your user's input. +The text in the request must be equal to the compiled text. `with_prompt` wins over this. The +integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock +Converse and InvokeModel, Google GenAI `generate_content`, LiteLLM completion and responses, and +OpenRouter chat and responses, but not for images, audio, video, or OpenAI Realtime. With an +older `telemetry-dev` that has no `prompt_attributes_for`, a provider integration records the +prompt only in `with_prompt`. For your own spans, `prompt_attributes_for(request)` gives the +attributes to pass as `attributes`; it gives `{}` inside `with_prompt`. ## Auto-metrics diff --git a/sdks/python/src/telemetry_dev/__init__.py b/sdks/python/src/telemetry_dev/__init__.py index 9ed7aab..b0bd718 100644 --- a/sdks/python/src/telemetry_dev/__init__.py +++ b/sdks/python/src/telemetry_dev/__init__.py @@ -27,6 +27,7 @@ TextPrompt, get_prompt, get_prompt_async, + prompt_attributes_for, with_prompt, ) from ._semconv import LogLevel, SpanType @@ -67,6 +68,7 @@ "init", "log", "observe", + "prompt_attributes_for", "propagate_attributes", "shutdown", "start_span", diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 9673740..43b863e 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -4,11 +4,14 @@ import contextlib import copy import hashlib +import itertools import json +import os import random import re import socket import threading +import types import urllib.error import urllib.parse import urllib.request @@ -50,6 +53,10 @@ _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") +_MAX_COMPILED_TEXTS = 1000 +_MAX_COMPILED_CHARACTERS = 2_000_000 +_MAX_SEARCH_DEPTH = 6 +_MAX_SEARCH_NODES = 500 class PromptModel(TypedDict): @@ -98,14 +105,15 @@ def compile( ) -> list[dict[str, Any]]: """Fill in the variables and replace each placeholder message with the list of messages in its variable. ``with_prompt`` records the variables of the last call before it starts - or inside it.""" + or inside it. Each text is also kept with its variables (the last 1000 texts), so that a + model call outside ``with_prompt`` that sends it can name the prompt; see + ``prompt_attributes_for``.""" values = _record(self, variables, kwargs) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": - compiled.append( - {"role": message["role"], "content": _fill(str(message["content"]), values)} - ) + content = _compile_text(self, str(message["content"]), values) + compiled.append({"role": message["role"], "content": content}) continue name = str(message["name"]) value = values.get(name) @@ -141,8 +149,10 @@ def messages(self) -> list[dict[str, Any]]: def compile(self, variables: Mapping[str, Any] | None = None, /, **kwargs: Any) -> str: """Fill in the variables. ``with_prompt`` records the variables of the last call before it - starts or inside it.""" - return _fill(self.text, _record(self, variables, kwargs)) + starts or inside it. The text is also kept with its variables (the last 1000 texts), so + that a model call outside ``with_prompt`` that sends it can name the prompt; see + ``prompt_attributes_for``.""" + return _compile_text(self, self.text, _record(self, variables, kwargs)) Prompt: TypeAlias = ChatPrompt | TextPrompt @@ -282,16 +292,45 @@ def prompt_attributes( scope = otel_context.get_value(_PROMPT_KEY, context) or otel_context.get_value(_PROMPT_KEY) if not isinstance(scope, _PromptScope): return {} - prompt = scope.prompt + task = _task_variables.get() + variables = task[1] if task is not None and task[0] is scope else scope.variables + return _attributes_of(scope.prompt, variables, client if capture_input else None) + + +def prompt_attributes_for(request: object) -> dict[str, AttributeValue]: + """The td.prompt.* attributes for a model call outside ``with_prompt``, from the texts that + ``compile()`` made: the first text in ``request`` from a template with a variable, else the + first text from a template with none. Only the first gives ``td.prompt.variables``, because + only its text holds the values. Gives ``{}`` inside ``with_prompt``, whose prompt the span + gets, and when no text matches. It reads lists, tuples, dicts, and the fields that objects + store (pydantic models), 6 levels and 500 values deep, and runs no property. It never + raises.""" + client = get_client() + try: + if isinstance(otel_context.get_value(_PROMPT_KEY), _PromptScope): + return {} + found = _compiled_texts.find(request) + if found is None: + return {} + capture = client if client is not None and client.capture_input else None + return _attributes_of(found.prompt, found.variables, capture) + except Exception as exc: + if client is not None: + client.report("prompt_attributes_for failed", exc) + return {} + + +def _attributes_of( + prompt: Prompt, variables: dict[str, Any] | None, capture: Client | None +) -> dict[str, AttributeValue]: + """The attributes of ``prompt``; ``capture`` is the client when input capture is on.""" attrs: dict[str, AttributeValue] = {ATTR_PROMPT_NAME: prompt.name} if prompt.version is not None: attrs[ATTR_PROMPT_VERSION] = prompt.version if prompt.label is not None: attrs[ATTR_PROMPT_LABEL] = prompt.label - task = _task_variables.get() - variables = task[1] if task is not None and task[0] is scope else scope.variables - if variables is not None and capture_input: - serialized = client.serialize(variables, ATTR_PROMPT_VARIABLES) + if variables is not None and capture is not None: + serialized = capture.serialize(variables, ATTR_PROMPT_VARIABLES) if serialized is not None: attrs[ATTR_PROMPT_VARIABLES] = serialized return attrs @@ -404,6 +443,137 @@ def _record( return values +@dataclass(frozen=True) +class _CompiledText: + """The prompt that ``compile()`` made a text from.""" + + prompt: Prompt + variables: dict[str, Any] | None + """Only for a template with a variable: the text holds the values, so they belong to it.""" + + +class _CompiledTexts: + """The texts of the last ``compile()`` calls, oldest first, for model calls outside + ``with_prompt``.""" + + def __init__(self) -> None: + self.texts: OrderedDict[str, _CompiledText] = OrderedDict() + self.characters = 0 + self.longest = 0 + self.lock = threading.Lock() + + def remember(self, text: str, template: str, prompt: Prompt, values: dict[str, Any]) -> None: + """Keep ``text`` as the newest entry. A template of only variable tags is skipped: its + text is input of the app, which an unrelated request can hold too.""" + fixed_text = _VARIABLE.sub("", template) + if not fixed_text.strip() or len(text) > _MAX_COMPILED_CHARACTERS: + return + compiled = _CompiledText(prompt, None if fixed_text == template else values) + with self.lock: + if self.texts.pop(text, None) is not None: + self.characters -= len(text) + self.texts[text] = compiled + self.characters += len(text) + self.longest = max(self.longest, len(text)) + dropped_longest = False + while ( + len(self.texts) > _MAX_COMPILED_TEXTS or self.characters > _MAX_COMPILED_CHARACTERS + ): + oldest, _ = self.texts.popitem(last=False) + self.characters -= len(oldest) + dropped_longest = dropped_longest or len(oldest) == self.longest + if dropped_longest: + self.longest = max(map(len, self.texts), default=0) + + def find(self, request: object) -> _CompiledText | None: + first_static: _CompiledText | None = None + budget = _MAX_SEARCH_NODES + stack: list[tuple[object, int]] = [(request, 0)] + with self.lock: + while stack and budget > 0: + budget -= 1 + value, depth = stack.pop() + if type(value) is str: + found = self.texts.get(value) if len(value) <= self.longest else None + if found is not None and found.variables is not None: + return found + first_static = first_static or found + elif depth < _MAX_SEARCH_DEPTH: + children = _stored_values(value, budget) + stack.extend((child, depth + 1) for child in reversed(children)) + return first_static + + +_compiled_texts = _CompiledTexts() + + +def _lock_compiled_texts_before_fork() -> None: + _compiled_texts.lock.acquire() + + +def _unlock_compiled_texts_after_fork() -> None: + _compiled_texts.lock.release() + + +# fork() waits for the lock: the child has only the thread that forked, so a lock that another +# thread held at the fork would stay held there for all time. +if hasattr(os, "register_at_fork"): # absent on Windows + os.register_at_fork( + before=_lock_compiled_texts_before_fork, + after_in_parent=_unlock_compiled_texts_after_fork, + after_in_child=_unlock_compiled_texts_after_fork, + ) + + +def _compile_text(prompt: Prompt, template: str, values: dict[str, Any]) -> str: + text = _fill(template, values) + _compiled_texts.remember(text, template, prompt, values) + return text + + +_TYPE_MRO = type.__dict__["__mro__"] +_TYPE_NAMESPACE = type.__dict__["__dict__"] + + +def _stored_values(value: object, limit: int) -> list[object]: + """At most ``limit`` values that ``value`` stores, in order: the items of a list or tuple, the + values of a dict, or the fields of another object in ``__dict__`` and ``__pydantic_extra__``. + Subclasses of list, tuple, and dict are skipped and fields are read from C-level storage, so + no override or property of the app runs.""" + cls = type(value) + if cls is list or cls is tuple: + return list(cast("list[object] | tuple[object, ...]", value)[:limit]) + if cls is dict: + return list(itertools.islice(cast("dict[object, object]", value).values(), limit)) + values: list[object] = [] + for name in ("__dict__", "__pydantic_extra__"): + stored = _stored(value, name) + if type(stored) is dict: + fields = cast("dict[str, object]", stored) + values.extend(itertools.islice(fields.values(), limit - len(values))) + return values + + +def _stored(value: object, name: str) -> object: + cls = type(value) + for klass in _TYPE_MRO.__get__(cls): + namespace = _TYPE_NAMESPACE.__get__(klass) + if name not in namespace: + continue + descriptor: object = namespace[name] + kind = type(descriptor) + storage = kind is types.MemberDescriptorType or ( + name == "__dict__" and kind is types.GetSetDescriptorType + ) + if not storage or cast(Any, descriptor).__name__ != name: + return None + try: + return cast(Any, descriptor).__get__(value, cls) + except (AttributeError, TypeError): + return None + return None + + def _fill(text: str, values: Mapping[str, Any]) -> str: def replace(match: re.Match[str]) -> str: value = values.get(match.group(1)) diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 892d676..9fd8656 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -3,6 +3,9 @@ import asyncio import contextvars import itertools +import json +import os +import signal import socket import threading import time @@ -24,8 +27,10 @@ from telemetry_dev import ( ChatPrompt, PromptError, + TextPrompt, get_prompt, get_prompt_async, + prompt_attributes_for, propagate_attributes, start_span, with_prompt, @@ -76,9 +81,42 @@ def clock(monkeypatch: pytest.MonkeyPatch) -> list[float]: @pytest.fixture(autouse=True) -def empty_cache() -> None: +def empty_cache(monkeypatch: pytest.MonkeyPatch) -> None: # A fetch that runs on after its test must not answer a later test on a reused port. prompts_module._cache.clear() # pyright: ignore[reportPrivateUsage] + monkeypatch.setattr( + prompts_module, + "_compiled_texts", + prompts_module._CompiledTexts(), # pyright: ignore[reportPrivateUsage] + ) + + +def chat_prompt(*messages: dict[str, Any], name: str = "support/reply") -> ChatPrompt: + return ChatPrompt( + name=name, + version=12, + label="production", + messages=list(messages), + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, + ) + + +def text_prompt(text: str) -> TextPrompt: + return TextPrompt( + name="support/reply", + version=12, + label="production", + text=text, + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, + ) def query_of(path: str) -> dict[str, str]: @@ -793,6 +831,249 @@ def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: assert "td.prompt.variables" not in (span.attributes or {}) +class Part: + def __init__(self, text: str) -> None: + self.text = text + + +def test_prompt_attributes_for_finds_a_compiled_text_in_the_request_shapes_of_the_providers( + make: MakeClient, +) -> None: + make() + prompt = chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "user", "content": "Order {{order}}: where is it?"}, + ) + compiled = prompt.compile(team="billing", order="#4") + system, user = (message["content"] for message in compiled) + + requests = { + "same": compiled, + "copy": [dict(message) for message in compiled], + "json": json.loads(json.dumps(compiled)), + "anthropic": [system, compiled[1:]], + "bedrock": [[{"text": system}], [{"role": "user", "content": [{"text": user}]}]], + "objects": (Part(system), [SimpleNamespace(role="user", parts=[Part(user)])]), + } + for request in requests.values(): + assert prompt_attributes_for(request) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + } + + assert prompt_attributes_for([{"role": "user", "content": "Order #5: where is it?"}]) == {} + + +def test_a_static_text_gives_no_variables_and_a_template_of_only_a_variable_tag_is_not_kept( + make: MakeClient, +) -> None: + make() + chat = chat_prompt( + {"role": "system", "content": "You are the refunds agent."}, + {"role": "user", "content": "{{ question }}"}, + ) + compiled = chat.compile(question="Where is my refund?") + + assert prompt_attributes_for(compiled) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + assert prompt_attributes_for([{"role": "user", "content": "Where is my refund?"}]) == {} + + summary = text_prompt("Summarize {{doc}} in a line.").compile(doc="the refund policy") + assert prompt_attributes_for(summary)["td.prompt.variables"] == '{"doc":"the refund policy"}' + + +def test_a_text_with_variables_wins_over_a_static_text_and_the_first_one_wins_over_a_later_one( + make: MakeClient, +) -> None: + make() + system = chat_prompt( + {"role": "system", "content": "You are the refunds agent, v2."}, name="refunds/system" + ).compile() + question = chat_prompt( + {"role": "user", "content": "Where is order {{order}}?"}, name="orders/question" + ).compile(order="#4") + style = chat_prompt( + {"role": "system", "content": "Reply in {{language}}."}, name="style/language" + ).compile(language="French") + + first = prompt_attributes_for([*system, *question]) + assert (first["td.prompt.name"], first["td.prompt.variables"]) == ( + "orders/question", + '{"order":"#4"}', + ) + second = prompt_attributes_for([*system, *style, *question]) + assert (second["td.prompt.name"], second["td.prompt.variables"]) == ( + "style/language", + '{"language":"French"}', + ) + + +def test_each_compile_keeps_its_own_values_so_two_threads_with_one_prompt_get_their_own( + make: MakeClient, +) -> None: + make() + prompt = chat_prompt({"role": "user", "content": "Track parcel {{parcel}}."}) + + first = prompt.compile(parcel="A-1") + second = prompt.compile(parcel="B-2") + + assert prompt_attributes_for(first)["td.prompt.variables"] == '{"parcel":"A-1"}' + assert prompt_attributes_for(second)["td.prompt.variables"] == '{"parcel":"B-2"}' + + +def test_with_prompt_wins_over_a_compiled_text_of_another_prompt(make: MakeClient) -> None: + memory = make() + scoped = chat_prompt({"role": "user", "content": "Hi"}, name="scoped/prompt") + compiled = chat_prompt( + {"role": "user", "content": "Cancel order {{order}}."}, name="other/prompt" + ).compile(order="#9") + + with with_prompt(scoped): + start_span("chat", type="generation", attributes=prompt_attributes_for(compiled)).end() + + [span] = memory.span_exporter.get_finished_spans() + assert (span.attributes or {})["td.prompt.name"] == "scoped/prompt" + + +def test_prompt_attributes_for_omits_variables_when_input_capture_is_off_and_masks_them( + make: MakeClient, +) -> None: + compiled = chat_prompt({"role": "user", "content": "My card is {{card}}."}).compile( + card="4111 1111" + ) + + make(capture_input=False) + assert prompt_attributes_for(compiled) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + telemetry_dev.shutdown() + + def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: + return {"card": "[masked]"} if context.key == "td.prompt.variables" else value + + make(mask=mask) + assert prompt_attributes_for(compiled)["td.prompt.variables"] == '{"card":"[masked]"}' + + +def test_the_kept_texts_stop_at_1000_and_at_2000000_characters_oldest_out_first( + make: MakeClient, +) -> None: + make() + ticket = text_prompt("Ticket {{n}} is open.") + for n in range(1001): + ticket.compile(n=n) + + assert prompt_attributes_for("Ticket 0 is open.") == {} + assert prompt_attributes_for("Ticket 1 is open.")["td.prompt.variables"] == '{"n":1}' + + note = text_prompt("Note: {{body}}") + notes = [note.compile(body=letter * (500_000 - len("Note: "))) for letter in "abcde"] + + assert prompt_attributes_for(notes[0]) == {} + assert prompt_attributes_for(notes[1])["td.prompt.name"] == "support/reply" + + # A text over the character limit is not kept, and it does not push the others out. + note.compile(body="f" * 2_000_001) + assert prompt_attributes_for(notes[1])["td.prompt.name"] == "support/reply" + + # A text compiled again counts once. + for _ in range(5): + note.compile(body="e" * (500_000 - len("Note: "))) + assert prompt_attributes_for(notes[4])["td.prompt.name"] == "support/reply" + + +def test_prompt_attributes_for_looks_6_levels_and_500_values_deep(make: MakeClient) -> None: + make() + [message] = chat_prompt({"role": "user", "content": "Find invoice {{invoice}}."}).compile( + invoice="INV-7" + ) + content = message["content"] + + def nest(value: object, levels: int) -> object: + return value if levels == 0 else nest([value], levels - 1) + + assert prompt_attributes_for(nest(content, 6)) != {} + assert prompt_attributes_for(nest(content, 7)) == {} + + # The request list, 498 strings, and the text are 500 values. + assert prompt_attributes_for([*["filler"] * 498, content]) != {} + assert prompt_attributes_for([*["filler"] * 499, content]) == {} + + +def test_prompt_attributes_for_runs_no_property_or_override_of_the_app(make: MakeClient) -> None: + make() + [message] = chat_prompt({"role": "user", "content": "Refund order {{order}}."}).compile( + order="#3" + ) + content = message["content"] + reads: list[str] = [] + + class Message: + @property + def content(self) -> str: + reads.append("property") + return content + + class Messages(list[object]): + def __getitem__(self, index: Any) -> Any: + reads.append("__getitem__") + return super().__getitem__(index) + + def __iter__(self) -> Any: + reads.append("__iter__") + return super().__iter__() + + assert prompt_attributes_for([Message(), Messages([content])]) == {} + assert reads == [] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") +# Python 3.12+ warns on fork() in a process with threads. +@pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") +def test_a_fork_while_a_thread_keeps_a_compiled_text_leaves_compile_usable_in_the_child() -> None: + prompt = text_prompt("Ship order {{order}}.") + texts = prompts_module._compiled_texts # pyright: ignore[reportPrivateUsage] + holding, release = threading.Event(), threading.Event() + + def hold() -> None: + with texts.lock: + holding.set() + release.wait(0.2) + + holder = threading.Thread(target=hold) + holder.start() + holding.wait() + read_end, write_end = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_end) + os.write(write_end, prompt.compile(order="#1").encode()) + finally: + os._exit(0) + os.close(write_end) + release.set() + holder.join() + + deadline = time.monotonic() + 5 + while os.waitpid(pid, os.WNOHANG) == (0, 0): + if time.monotonic() > deadline: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + pytest.fail("compile() in the child waited for a lock held at the fork") + time.sleep(0.01) + with os.fdopen(read_end, "rb") as child: + assert child.read() == b"Ship order #1." + + def test_a_fallback_prompt_records_its_name_and_label_but_no_version( server: CaptureServer, make: MakeClient ) -> None: From ae47605555ef9f5dfed1d6f762ac56823e692edb Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 15:31:55 +0000 Subject: [PATCH 12/32] fix(sdk): find the prompt by message identity, not by text The compile() function of a chat prompt tags each message object that it makes with the prompt and the variables of that call. TypeScript uses a non-enumerable symbol property, and Python uses a dict subclass with a slot. The keys and the JSON of a message do not change. The promptAttributesFor and prompt_attributes_for functions read the tag of the newest message of the last 1000, and they compare no text. Thus a message from a different task, rollout version, turn, or call does not get the wrong prompt, and a copy gets no prompt. We removed the text registry, its limits, its lock, and its fork hooks. As a result, compile() keeps no variables alive. The integrations give only the request messages to the core: OpenAI messages and input, Anthropic messages, Bedrock Converse messages, Google GenAI contents, LiteLLM messages and input, and OpenRouter messages and input. Bedrock InvokeModel sends a serialized body, thus it does not look for a prompt. Python Anthropic reads the list that the app gave, not its own copy. Its messages.stream() decides at __enter__, where its span starts. A span with input capture off removes td.prompt.variables from the attributes that the app gives. The SDKs do not record text prompts outside withPrompt and with_prompt. --- docs/sdk-conformance.md | 54 +-- packages/anthropic/src/index.ts | 8 +- packages/anthropic/tests/anthropic.test.ts | 17 +- packages/bedrock/src/converse.ts | 2 +- packages/bedrock/src/internal.ts | 6 +- packages/bedrock/src/invoke-model.ts | 5 +- packages/bedrock/tests/bedrock.test.ts | 104 +++--- packages/google-genai/src/index.ts | 8 +- .../google-genai/tests/google-genai.test.ts | 42 +-- packages/openai/src/index.ts | 8 +- packages/openai/tests/openai.test.ts | 31 +- packages/openrouter/src/index.ts | 8 +- packages/openrouter/tests/openrouter.test.ts | 36 +- packages/sdk/README.md | 26 +- packages/sdk/src/attrs.ts | 3 + packages/sdk/src/prompts.ts | 150 +++----- packages/sdk/tests/prompts.test.ts | 272 +++++++-------- .../src/telemetry_dev_anthropic/__init__.py | 55 ++- sdks/python-anthropic/tests/test_anthropic.py | 84 ++++- .../src/telemetry_dev_bedrock/_fields.py | 4 +- .../telemetry_dev_bedrock/_invoke_model.py | 4 +- .../src/telemetry_dev_bedrock/_registry.py | 2 +- sdks/python-bedrock/tests/test_bedrock.py | 104 +++--- .../telemetry_dev_google_genai/__init__.py | 6 +- .../tests/test_google_genai.py | 43 ++- .../src/telemetry_dev_litellm/__init__.py | 6 +- sdks/python-litellm/tests/test_litellm.py | 33 +- .../src/telemetry_dev_openai/__init__.py | 6 +- sdks/python-openai/tests/test_openai.py | 38 +- .../src/telemetry_dev_openrouter/__init__.py | 7 +- .../tests/test_openrouter.py | 37 +- sdks/python/README.md | 30 +- sdks/python/src/telemetry_dev/_prompts.py | 189 +++------- sdks/python/src/telemetry_dev/_spans.py | 4 + sdks/python/tests/test_prompts.py | 328 +++++++++--------- 35 files changed, 822 insertions(+), 938 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index efe4a7a..610c0c1 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -41,10 +41,10 @@ repository, not this SDK workspace. Paths below refer to that product repository | error | span status ERROR + `error.type` + `exception` event (`exception.type/message/stacktrace`, `log.severity_number: 17`) | | | `log()` | OTLP log record to `/v1/logs` (always; never span events) | severity debug=5/info=9/warn=13/error=17; trace-correlated via active context | | `withPrompt(prompt, fn)` / `with with_prompt(prompt):` → `prompt.name` | `td.prompt.name` | on model-call spans that start in scope: operation `chat`, `text_completion`, or `generate_content` | -| outside a scope: a text that `prompt.compile()` made, in the request of a provider integration | `td.prompt.name`, `td.prompt.version`, `td.prompt.label` | on the `chat` spans of the provider integrations listed in [Prompts](#prompts); the scope wins | +| outside a scope: a message object that `compile()` of a chat prompt returned, in the request messages of a provider integration | `td.prompt.name`, `td.prompt.version`, `td.prompt.label` | on the `chat` spans of the provider integrations listed in [Prompts](#prompts); the scope wins | | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | -| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the `compile()` that made the text, when its template has a variable | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the `compile()` that returned the message | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -179,27 +179,39 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. -Outside a scope, the SDKs find the prompt from the text of the request: - -- `compile()` keeps each text that it makes (each message of a chat prompt, the text of a text - prompt), with the prompt and the variables. It keeps at most 1000 texts and 2,000,000 - characters, and drops the oldest first. A text that `compile()` makes again becomes the newest. - It does not keep a text whose template has only variable tags, for example `{{ question }}`: - that text is input of the app, and an unrelated request can hold it too. -- `promptAttributesFor(request)` / `prompt_attributes_for(request)` look for a kept text in - `request`: strings in arrays and plain objects (Python: `list`, `tuple`, and `dict` but not their - subclasses, and the fields that an object stores in `__dict__` or `__pydantic_extra__`), 6 - levels deep and 500 values in all. They run no getter or property. A string matches only when it is equal to a kept text. -- The first text from a template with a variable gives the name, version, label, and variables - of its `compile()`. Without one, the first text from a template with no variable gives the - name, version, and label, but no variables: the same static text comes from each `compile()`. +Outside a scope, the SDKs find the prompt from the message objects of the request, not from +their text: + +- `compile()` of a chat prompt tags each message object that it makes with the prompt and the + variables of that call. It does not tag the messages of a placeholder variable: the app passes + those. TypeScript: a non-enumerable symbol property. Python: the messages are instances of a + `dict` subclass that holds the tag in a slot. The keys and the JSON of a message do not change. +- A copy has no tag: a spread, `structuredClone`, or a JSON round trip (TypeScript); `dict()`, + `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, or a JSON round trip (Python). A new message + with the same text has no tag either, thus a message from another call, task, or prompt version + is never recorded. +- `promptAttributesFor(messages)` / `prompt_attributes_for(messages)` take a list of messages or + one message (Python: a `list` or `tuple`, not a subclass, so no code of the app runs). They + look at the last 1000 items, from the end, and give the name, version, label, and variables of + the newest tagged message. They do not look inside a message. - In a `withPrompt` / `with_prompt` scope they give nothing, thus the span gets the prompt of the scope. They never throw. -- The provider integrations call them with the request of the app before the span starts, also - when input capture is off: OpenAI chat completions and responses, Anthropic messages, Amazon - Bedrock Converse and InvokeModel (not embedding models), Google GenAI `generateContent`, - LiteLLM completion and responses (Python), and OpenRouter chat and responses. The other spans - of the integrations (images, audio, video, and OpenAI Realtime) do not look in the request. +- They give `td.prompt.variables` only when input capture is on in the client. A span whose + effective captureInput is off drops `td.prompt.variables` from its attributes, also when the app + passes the result of these functions to its own span. +- `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the + SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. +- The provider integrations call them with the request messages before the span starts, also + when input capture is off: OpenAI chat completions (`messages`) and responses (`input`), + Anthropic messages (`messages`; the `system` string has no tag), Amazon Bedrock Converse + (`messages`), Google GenAI `generateContent` (`contents`), LiteLLM completion (`messages`) and + responses (`input`) (Python), and OpenRouter chat (`messages`) and responses (`input`). Bedrock + Converse and Google GenAI take another message shape: a compiled message that the app changes + in place keeps its tag, and a new message does not. Bedrock InvokeModel takes a serialized body + and is not recorded outside a scope. The other spans of the integrations (embeddings, images, + audio, video, and OpenAI Realtime) do not look at the request. +- Python Anthropic `messages.stream()` starts its span at `__enter__`. It decides there, with the + messages that `.stream()` received, thus a scope around `__enter__` wins. - A provider integration reads the function from the core module when it runs, thus it also operates with an older core that does not have it. With such a core, the prompt is recorded only in a scope. diff --git a/packages/anthropic/src/index.ts b/packages/anthropic/src/index.ts index 4e5b057..1082844 100644 --- a/packages/anthropic/src/index.ts +++ b/packages/anthropic/src/index.ts @@ -150,12 +150,12 @@ function toolDefinitions(tools: T): Record | undefined { // Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read // keeps them loadable without the export. -function promptAttributes(request: unknown): SpanFields["attributes"] { +function promptAttributes(messages: unknown): SpanFields["attributes"] { const read = ( - telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } ).promptAttributesFor; - return typeof read === "function" ? read(request) : undefined; + return typeof read === "function" ? read(messages) : undefined; } function messagesRequest(body: ValueRecord): RequestMapping { @@ -174,7 +174,7 @@ function messagesRequest(body: ValueRecord): RequestMapping { input, attributes: { ...toolDefinitions(body.tools), - ...promptAttributes([body.system, body.messages]), + ...promptAttributes(body.messages), }, systemInstructions: body.system, temperature: readNumber(body.temperature), diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index c8cab08..12eeae9 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -282,12 +282,12 @@ afterEach(async () => { }); test.each([ - { field: "system", olderCore: false }, - { field: "messages", olderCore: false }, - { field: "messages", olderCore: true }, + { messages: "compiled", olderCore: false }, + { messages: "copied", olderCore: false }, + { messages: "compiled", olderCore: true }, ])( - "messages.create outside withPrompt records the prompt of a compiled text in $field, older core $olderCore", - async ({ field, olderCore }) => { + "messages.create outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ messages: kind, olderCore }) => { if (olderCore) { vi.spyOn( telemetrySdk as { promptAttributesFor: unknown }, @@ -318,20 +318,21 @@ test.each([ }), }); const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const message = user as { role: "user"; content: string }; const spans = setupSpans(); const client = clientWith(createFakeFetch(jsonResponse(messagePayload())).fetch); await client.messages.create({ model: "claude-sonnet-4-6", max_tokens: 64, - system: field === "system" ? system!.content : "Be terse.", - messages: [{ role: "user", content: field === "messages" ? user!.content : "Hi" }], + system: system!.content, + messages: [kind === "compiled" ? message : { ...message }], }); const { attributes } = await exportedSpan(spans); expect(attributes["gen_ai.input.messages"]).toBeDefined(); - if (olderCore) { + if (olderCore || kind === "copied") { expect(attributes["td.prompt.name"]).toBeUndefined(); return; diff --git a/packages/bedrock/src/converse.ts b/packages/bedrock/src/converse.ts index e552c6d..fd9c8e1 100644 --- a/packages/bedrock/src/converse.ts +++ b/packages/bedrock/src/converse.ts @@ -35,7 +35,7 @@ export function converseRequestFields(input: JsonRecord): SpanFields { model: stringValue(input.modelId), input: normalizeMessages(input.messages), systemInstructions: normalizeSystem(input.system), - attributes: promptAttributes([input.system, input.messages]), + attributes: promptAttributes(input.messages), temperature: numberValue(inference.temperature), topP: numberValue(inference.topP), maxTokens: numberValue(inference.maxTokens), diff --git a/packages/bedrock/src/internal.ts b/packages/bedrock/src/internal.ts index 7d82cc8..2a89e01 100644 --- a/packages/bedrock/src/internal.ts +++ b/packages/bedrock/src/internal.ts @@ -194,12 +194,12 @@ export function captureMaskedBeforeStart(): boolean { // Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read // keeps them loadable without the export. -export function promptAttributes(request: unknown): SpanFields["attributes"] { +export function promptAttributes(messages: unknown): SpanFields["attributes"] { const read = ( - telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } ).promptAttributesFor; - return typeof read === "function" ? read(request) : undefined; + return typeof read === "function" ? read(messages) : undefined; } // The decision is made before the span starts so samplers see the same attributes as before. diff --git a/packages/bedrock/src/invoke-model.ts b/packages/bedrock/src/invoke-model.ts index d2532d2..d1e1f11 100644 --- a/packages/bedrock/src/invoke-model.ts +++ b/packages/bedrock/src/invoke-model.ts @@ -9,7 +9,6 @@ import { numberValue, omitUndefined, parseJson, - promptAttributes, PROVIDER, STREAM_CAPTURE_MAX_ITEMS, stringValue, @@ -36,14 +35,12 @@ export function invokeModelSpanName(input: JsonRecord): string { export function invokeModelRequestFields(input: JsonRecord): SpanFields { const model = stringValue(input.modelId); const body = parseBody(input.body, input.contentType); - const embedding = isEmbeddingModel(model); return omitUndefined({ provider: PROVIDER, model, input: body, - outputType: embedding ? "embedding" : undefined, - attributes: embedding ? undefined : promptAttributes(body), + outputType: isEmbeddingModel(model) ? "embedding" : undefined, ...nativeSamplingFields(model, body), }); } diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index d12bf19..6098bea 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -424,10 +424,15 @@ test("ConverseStream records stream errors with partial output", async () => { expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); }); -test.each([false, true])( - "Converse and InvokeModel outside withPrompt record the prompt of a compiled text; embeddings do not; older core %s", - async (olderCore) => { +test.each([ + { messages: "changed in place", olderCore: false }, + { messages: "new", olderCore: false }, + { messages: "changed in place", olderCore: true }, +])( + "Converse outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ messages: kind, olderCore }) => { const spans = setup(); + if (olderCore) { vi.spyOn( telemetrySdk as { promptAttributesFor: unknown }, @@ -435,6 +440,7 @@ test.each([false, true])( "get", ).mockReturnValue(undefined); } + const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", apiUrl: "https://prompts.bedrock.test", @@ -457,78 +463,48 @@ test.each([false, true])( }), }); const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; - const converseReply = { - output: { message: { role: "assistant", content: [{ text: "Soon." }] } }, - stopReason: "end_turn", - }; + const blocks = [{ text: user!.content }]; + // Converse takes content blocks: a message changed in place keeps its tag, a new one has none. + const message = + kind === "new" + ? { role: "user", content: blocks } + : Object.assign(user!, { content: blocks }); const client = wrapBedrock( - new FakeClient([ - converseReply, - converseReply, - { - body: bytes({ content: [{ type: "text", text: "Soon." }], stop_reason: "end_turn" }), - contentType: "application/json", - }, - { - body: bytes({ embedding: [0.1, 0.2], inputTextTokenCount: 6 }), - contentType: "application/json", - }, - ]), + new BedrockRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("prompt-1", { + output: { message: { role: "assistant", content: [{ text: "Soon." }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }), + }), ); await client.send( new ConverseCommand({ modelId: "anthropic.claude-3-haiku", system: [{ text: system!.content }], - messages: [{ role: "user", content: [{ text: "Hi" }] }], - }), - ); - await client.send( - new ConverseCommand({ - modelId: "anthropic.claude-3-haiku", - messages: [{ role: "user", content: [{ text: user!.content }] }], - }), - ); - await client.send( - new InvokeModelCommand({ - modelId: "anthropic.claude-3-haiku", - contentType: "application/json", - body: bytes({ - anthropic_version: "bedrock-2023-05-31", - max_tokens: 64, - messages: [{ role: "user", content: user!.content }], - }), - }), - ); - await client.send( - new InvokeModelCommand({ - modelId: "amazon.titan-embed-text-v2:0", - contentType: "application/json", - body: bytes({ inputText: user!.content }), + messages: [message] as ConverseCommandInput["messages"], }), ); - const [systemSpan, messagesSpan, invokeSpan, embedSpan] = spans.getFinishedSpans(); - - for (const span of [systemSpan, messagesSpan, invokeSpan]) { - expect(span!.attributes["gen_ai.request.model"]).toBe("anthropic.claude-3-haiku"); - expect( - Object.entries(span!.attributes).filter(([key]) => key.startsWith("td.prompt.")), - ).toEqual( - olderCore - ? [] - : [ - ["td.prompt.name", "support/reply"], - ["td.prompt.version", 12], - ["td.prompt.label", "production"], - ["td.prompt.variables", '{"team":"billing","order":"#4"}'], - ], - ); - } - - expect(embedSpan!.attributes["gen_ai.operation.name"]).toBe("embeddings"); - expect(embedSpan!.attributes["td.prompt.name"]).toBeUndefined(); + const [span] = spans.getFinishedSpans(); + expect(span!.attributes["gen_ai.response.id"]).toBe("prompt-1"); + expect( + Object.entries(span!.attributes).filter(([key]) => key.startsWith("td.prompt.")), + ).toEqual( + olderCore || kind === "new" + ? [] + : [ + ["td.prompt.name", "support/reply"], + ["td.prompt.version", 12], + ["td.prompt.label", "production"], + ["td.prompt.variables", '{"team":"billing","order":"#4"}'], + ], + ); }, ); diff --git a/packages/google-genai/src/index.ts b/packages/google-genai/src/index.ts index 2272e46..341d5e8 100644 --- a/packages/google-genai/src/index.ts +++ b/packages/google-genai/src/index.ts @@ -570,7 +570,7 @@ function generateRequestFields(params: UnknownRecord): RequestMapping { const attributes = { ...requestAttributesFromConfig(config), - ...promptAttributes([config?.systemInstruction, params.contents]), + ...promptAttributes(params.contents), }; const input = captureEnabled("input") ? normalizeContentInput(params.contents) : undefined; @@ -946,12 +946,12 @@ function captureMaskedBeforeStart(): boolean { // Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read // keeps them loadable without the export. -function promptAttributes(request: unknown): SpanFields["attributes"] { +function promptAttributes(messages: unknown): SpanFields["attributes"] { const read = ( - telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } ).promptAttributesFor; - return typeof read === "function" ? read(request) : undefined; + return typeof read === "function" ? read(messages) : undefined; } // The decision is made before the span starts so samplers see the same attributes as before. diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index e7417ae..e7d5ec6 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -236,12 +236,13 @@ afterEach(async () => { }); test.each([ - { field: "systemInstruction", captureInput: true, olderCore: false }, - { field: "contents", captureInput: false, olderCore: false }, - { field: "contents", captureInput: true, olderCore: true }, + { contents: "changed in place", captureInput: true, olderCore: false }, + { contents: "changed in place", captureInput: false, olderCore: false }, + { contents: "new", captureInput: true, olderCore: false }, + { contents: "changed in place", captureInput: true, olderCore: true }, ])( - "generateContent outside withPrompt records the prompt of a compiled text in $field, input capture $captureInput, older core $olderCore", - async ({ field, captureInput, olderCore }) => { + "generateContent outside withPrompt records the prompt of $contents contents, input capture $captureInput, older core $olderCore", + async ({ contents: kind, captureInput, olderCore }) => { if (olderCore) { vi.spyOn( telemetrySdk as { promptAttributesFor: unknown }, @@ -272,31 +273,32 @@ test.each([ }), }); const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const parts = [{ text: user!.content }]; + // Gemini takes parts: a message changed in place keeps its tag, a new one has none. + const content = kind === "new" ? { role: "user", parts } : Object.assign(user!, { parts }); const spans = setupSpans({ captureInput }); - vi.stubGlobal( - "fetch", - createFakeFetch( - jsonResponse({ - candidates: [ - { content: { role: "model", parts: [{ text: "Soon." }] }, finishReason: "STOP" }, - ], - }), - ).fetch, + const fake = createFakeFetch( + jsonResponse({ + candidates: [ + { content: { role: "model", parts: [{ text: "Soon." }] }, finishReason: "STOP" }, + ], + }), ); + vi.stubGlobal("fetch", fake.fetch); await clientWith().models.generateContent({ model: "gemini-2.5-flash", - contents: [{ role: "user", parts: [{ text: field === "contents" ? user!.content : "Hi" }] }], - config: { - temperature: 0.2, - systemInstruction: field === "systemInstruction" ? system!.content : "Be terse.", - }, + contents: [content], + config: { temperature: 0.2, systemInstruction: system!.content }, }); const { attributes } = await exportedSpan(spans); expect(attributes["gen_ai.request.temperature"]).toBe(0.2); + expect(fake.requests[0]!.body).toMatchObject({ + contents: [{ role: "user", parts: [{ text: "Where is order #4?" }] }], + }); - if (olderCore) { + if (olderCore || kind === "new") { expect(attributes["td.prompt.name"]).toBeUndefined(); return; diff --git a/packages/openai/src/index.ts b/packages/openai/src/index.ts index c62671c..9b86e73 100644 --- a/packages/openai/src/index.ts +++ b/packages/openai/src/index.ts @@ -246,7 +246,7 @@ function chatResponse(response: T): SpanFields { function responsesRequest(body: JsonRecord): RequestMapping { const model = readString(body.model); const capture = captureEnabled("input") ? sanitizeResponsesCapture(body.input) : undefined; - const prompt = promptAttributes([body.instructions, body.input]); + const prompt = promptAttributes(body.input); return { name: `chat ${model ?? "unknown"}`, @@ -1634,12 +1634,12 @@ function captureMaskedBeforeStart(): boolean { // Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read // keeps them loadable without the export. -function promptAttributes(request: unknown): SpanFields["attributes"] { +function promptAttributes(messages: unknown): SpanFields["attributes"] { const read = ( - telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } ).promptAttributesFor; - return typeof read === "function" ? read(request) : undefined; + return typeof read === "function" ? read(messages) : undefined; } // The decision is made before the span starts so samplers see the same attributes as before. diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 06518ee..4437d75 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -2895,13 +2895,13 @@ test("chat streams preserve caller-requested usage chunks", async () => { }); test.each([ - { api: "chat completions", captureInput: false, olderCore: false }, - { api: "responses instructions", captureInput: true, olderCore: false }, - { api: "responses input", captureInput: true, olderCore: false }, - { api: "chat completions", captureInput: true, olderCore: true }, + { api: "chat completions", messages: "compiled", captureInput: false, olderCore: false }, + { api: "responses", messages: "compiled", captureInput: true, olderCore: false }, + { api: "chat completions", messages: "copied", captureInput: true, olderCore: false }, + { api: "chat completions", messages: "compiled", captureInput: true, olderCore: true }, ])( - "$api outside withPrompt record the prompt of compiled messages, input capture $captureInput, older core $olderCore", - async ({ api, captureInput, olderCore }) => { + "$api outside withPrompt records the prompt of $messages messages, input capture $captureInput, older core $olderCore", + async ({ api, messages: kind, captureInput, olderCore }) => { if (olderCore) { vi.spyOn( telemetrySdk as { promptAttributesFor: unknown }, @@ -2931,12 +2931,13 @@ test.each([ rollout: null, }), }); - const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const messages = kind === "compiled" ? compiled : compiled.map((message) => ({ ...message })); const spans = setupSpans(undefined, { captureInput }); const client = clientWith( createFakeFetch( jsonResponse( - api.startsWith("responses") + api === "responses" ? { id: "resp_1", object: "response", @@ -2963,22 +2964,16 @@ test.each([ ).fetch, ); - if (api.startsWith("responses")) { - const inInput = api === "responses input"; - - await client.responses.create({ - model: "gpt-4.1", - instructions: inInput ? "Be terse." : system!.content, - input: [{ role: "user", content: inInput ? user!.content : "Hi" }], - }); + if (api === "responses") { + await client.responses.create({ model: "gpt-4.1", input: messages }); } else { - await client.chat.completions.create({ model: "gpt-4o", messages: [system!, user!] }); + await client.chat.completions.create({ model: "gpt-4o", messages }); } const { attributes } = await exportedSpan(spans); expect(attributes["gen_ai.input.messages"] !== undefined).toBe(captureInput); - if (olderCore) { + if (olderCore || kind === "copied") { expect(attributes["td.prompt.name"]).toBeUndefined(); return; diff --git a/packages/openrouter/src/index.ts b/packages/openrouter/src/index.ts index e0d696e..74e2c76 100644 --- a/packages/openrouter/src/index.ts +++ b/packages/openrouter/src/index.ts @@ -387,12 +387,12 @@ function usageCost(usage: T): number | undefined { // Cores older than promptAttributesFor record a prompt only inside withPrompt. A namespace read // keeps them loadable without the export. -function promptAttributes(request: unknown): SpanFields["attributes"] { +function promptAttributes(messages: unknown): SpanFields["attributes"] { const read = ( - telemetrySdk as { promptAttributesFor?: (request: unknown) => SpanFields["attributes"] } + telemetrySdk as { promptAttributesFor?: (messages: unknown) => SpanFields["attributes"] } ).promptAttributesFor; - return typeof read === "function" ? read(request) : undefined; + return typeof read === "function" ? read(messages) : undefined; } function chatRequest(request: JsonRecord): RequestMapping { @@ -484,7 +484,7 @@ function responsesRequest(request: JsonRecord): RequestMapping { maxTokens: readNumber(body.maxOutputTokens), frequencyPenalty: readNumber(body.frequencyPenalty), presencePenalty: readNumber(body.presencePenalty), - attributes: promptAttributes([body.instructions, body.input]), + attributes: promptAttributes(body.input), }, }; } diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index dc67157..3f5cb68 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -312,13 +312,13 @@ afterEach(async () => { }); test.each([ - { api: "chat", olderCore: false }, - { api: "streamed responses instructions", olderCore: false }, - { api: "responses input", olderCore: false }, - { api: "chat", olderCore: true }, + { api: "chat", messages: "compiled", olderCore: false }, + { api: "streamed responses", messages: "compiled", olderCore: false }, + { api: "chat", messages: "copied", olderCore: false }, + { api: "chat", messages: "compiled", olderCore: true }, ])( - "$api outside withPrompt record the prompt of compiled messages, older core $olderCore", - async ({ api, olderCore }) => { + "$api outside withPrompt records the prompt of $messages messages, older core $olderCore", + async ({ api, messages: kind, olderCore }) => { if (olderCore) { vi.spyOn( telemetrySdk as { promptAttributesFor: unknown }, @@ -348,22 +348,13 @@ test.each([ rollout: null, }), }); - const [system, user] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const messages = kind === "compiled" ? compiled : compiled.map((message) => ({ ...message })); const spans = setupSpans(); if (api === "chat") { const client = clientWith(createFakeFetcher(jsonResponse(chatBody("chat_1"))).fetcher); - await client.chat.send({ - chatRequest: { model: "openai/gpt-4o", messages: [system!, user!] }, - }); - } else if (api === "responses input") { - const client = clientWith(createFakeFetcher(jsonResponse(responsesBody("resp_1"))).fetcher); - await client.responses.send({ - responsesRequest: { - model: "openai/gpt-4o", - input: [{ type: "message", role: "user", content: user!.content }], - }, - }); + await client.chat.send({ chatRequest: { model: "openai/gpt-4o", messages } }); } else { const client = clientWith( createFakeFetcher( @@ -371,12 +362,7 @@ test.each([ ).fetcher, ); const stream = await client.responses.send({ - responsesRequest: { - model: "openai/gpt-4o", - instructions: system!.content, - input: "Hi", - stream: true, - }, + responsesRequest: { model: "openai/gpt-4o", input: messages, stream: true }, }); if (!(stream instanceof ReadableStream)) throw new Error("expected a readable stream"); @@ -386,7 +372,7 @@ test.each([ const { attributes } = await exportedSpan(spans); expect(attributes["gen_ai.request.model"]).toBe("openai/gpt-4o"); - if (olderCore) { + if (olderCore || kind === "copied") { expect(attributes["td.prompt.name"]).toBeUndefined(); return; diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 00cd298..b73d74c 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -180,17 +180,21 @@ compiles its own. Model calls are the spans of type `generation` (operations `ch use this SDK: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, and OpenRouter. Spans from the AI SDK and TanStack AI integrations, and from other OpenTelemetry tracers, do not get the attributes. -Outside `withPrompt`, those provider integrations find the prompt in the request: -`compile()` keeps the texts that it makes (the last 1000, at most 2,000,000 characters), and a -chat call whose request holds one of them gets `td.prompt.name`, `td.prompt.version`, and -`td.prompt.label`, also when input capture is off. A text from a template with a variable also -gives `td.prompt.variables`, and it wins over a text from a template with none. A message of only -a variable tag, for example `{{ question }}`, is not kept, because its text is your user's input. -The text in the request must be equal to the compiled text. `withPrompt` wins over this. The -integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse -and InvokeModel, Google GenAI `generateContent`, and OpenRouter chat and responses, but not for -images, audio, video, or OpenAI Realtime. For your own spans, `promptAttributesFor(request)` -gives the attributes to pass as `attributes`; it gives nothing inside `withPrompt`. +Outside `withPrompt`, those provider integrations record the prompt of the message objects that +`compile()` returned. `compile()` of a chat prompt tags each message that it makes with its prompt +and variables, in a non-enumerable symbol property: the keys and the JSON of the message do not +change. A chat call that sends a tagged message gets `td.prompt.name`, `td.prompt.version`, and +`td.prompt.label` of the newest one (of the last 1000 messages), also when input capture is off, +and `td.prompt.variables` when it is on. A copy of a message (a spread, `structuredClone`, or +JSON) and a new message with the same text have no tag, so the SDK does not record them. Bedrock +Converse and Google GenAI take another message shape: change the compiled messages in place, or +use `withPrompt`. A text prompt compiles to a string, which cannot carry a tag: use `withPrompt`. +`withPrompt` wins over the tag. The integrations do it for OpenAI chat completions and responses, +Anthropic messages, Bedrock Converse, Google GenAI `generateContent`, and OpenRouter chat and +responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, video, +or OpenAI Realtime. For your own spans, `promptAttributesFor(messages)` gives the attributes to +pass as `attributes`; it gives nothing inside `withPrompt`, and a span with `captureInput: false` +drops `td.prompt.variables`. ## API diff --git a/packages/sdk/src/attrs.ts b/packages/sdk/src/attrs.ts index 75afcb4..6dbf34f 100644 --- a/packages/sdk/src/attrs.ts +++ b/packages/sdk/src/attrs.ts @@ -179,6 +179,9 @@ export function fieldsToAttributes( if (fields.attributes) { Object.assign(attrs, omitUndefined(fields.attributes)); + + // The variables are input, also when the app passes them from promptAttributesFor(). + if (!cfg.captureInput) delete attrs["td.prompt.variables"]; } return attrs; diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index afd76c2..48cfe27 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -97,8 +97,8 @@ export interface ChatPrompt extends PromptBase { /** * Fills in the variables and replaces each placeholder message with the messages in its * variable. `withPrompt` records the variables of the last call before it starts or inside it. - * Each text is also kept with its variables (the last 1000 texts), so that a model call outside - * `withPrompt` that sends it can name the prompt; see `promptAttributesFor`. + * It tags each message object that it makes with this prompt and these variables, so that a + * model call outside `withPrompt` that sends the message records them; see `promptAttributesFor`. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -109,8 +109,8 @@ export interface TextPrompt extends PromptBase { readonly text: string; /** * Fills in the variables. `withPrompt` records the variables of the last call before it starts - * or inside it. The text is also kept with its variables (the last 1000 texts), so that a model - * call outside `withPrompt` that sends it can name the prompt; see `promptAttributesFor`. + * or inside it. A string cannot carry a tag, so a model call outside `withPrompt` does not + * record a text prompt. */ compile(variables?: PromptVariables): string; } @@ -201,11 +201,10 @@ interface PromptScope { variables: object | undefined; } -/** The prompt that `compile()` made a text from. */ -interface CompiledText { +/** The `compile()` call that made a chat message. */ +interface CompileTag { readonly prompt: Prompt; - /** Only for a template with a variable: the text holds the values, so they belong to it. */ - readonly variables: object | undefined; + readonly variables: PromptVariables; } const FETCH_TIMEOUT_MS = 10_000; @@ -219,19 +218,15 @@ const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} G const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); -const MAX_COMPILED_TEXTS = 1_000; -const MAX_COMPILED_CHARACTERS = 2_000_000; -const MAX_SEARCH_DEPTH = 6; -const MAX_SEARCH_NODES = 500; +// Not enumerable, so JSON.stringify, spreads, and structuredClone leave the tag out of a copy. +const COMPILED_BY = Symbol("telemetry.dev compiled by"); +/** How many messages promptAttributesFor looks at, from the end. */ +const MAX_TAG_LOOKUP = 1_000; const cache = new Map(); const compiledVariables = new WeakMap(); /** The Retry-After of the response that failed with this error, in milliseconds. */ const retryAfter = new WeakMap(); -// The texts of the last compile() calls, oldest first, for model calls outside withPrompt. -const compiledTexts = new Map(); -let compiledCharacters = 0; -let longestCompiledText = 0; /** * Fetches a prompt version from telemetry.dev and keeps it. When the copy is older than @@ -372,26 +367,24 @@ export function promptAttributes( } /** - * The td.prompt.* attributes for a model call outside `withPrompt`, from the texts that - * `compile()` made: the first text in `request` from a template with a variable, else the first - * text from a template with none. Only the first gives `td.prompt.variables`, because only its text - * holds the values. Gives undefined inside `withPrompt`, whose prompt the span gets, and when no - * text matches. It reads own data properties of arrays and plain objects, 6 levels and 500 values - * deep, and runs no getter. It never throws. + * The td.prompt.* attributes for a model call outside `withPrompt` that sends `messages` (a list + * of messages, or one message): those of the newest message, of the last 1000, that `compile()` + * of a chat prompt returned. The tag is on the message object, so a copy of it gives nothing. Gives + * undefined inside `withPrompt`, whose prompt the span gets. `td.prompt.variables` needs input + * capture on in the client, and the span drops it when its own input capture is off. It never + * throws. */ -export function promptAttributesFor(request: unknown): Record | undefined { +export function promptAttributesFor(messages: unknown): Record | undefined { const config = currentClient().core?.config; try { - if (compiledTexts.size === 0 || activeContext().getValue(PROMPT_KEY) !== undefined) { - return undefined; - } + if (activeContext().getValue(PROMPT_KEY) !== undefined) return undefined; - const found = findCompiledText(request); + const tag = newestTag(messages); - if (!found) return undefined; + if (!tag) return undefined; - return attributesOf(found.prompt, found.variables, config) as Record; + return attributesOf(tag.prompt, tag.variables, config) as Record; } catch (error) { reportError(config?.onError, error); @@ -415,47 +408,23 @@ function attributesOf( }); } -function findCompiledText(request: unknown): CompiledText | undefined { - let firstStatic: CompiledText | undefined; - let budget = MAX_SEARCH_NODES; - const stack: [value: unknown, depth: number][] = [[request, 0]]; - - while (stack.length > 0 && budget > 0) { - budget -= 1; - const [value, depth] = stack.pop()!; - - if (typeof value === "string") { - const found = value.length <= longestCompiledText ? compiledTexts.get(value) : undefined; +function newestTag(messages: unknown): CompileTag | undefined { + if (!Array.isArray(messages)) return tagOf(messages); + const stop = Math.max(0, messages.length - MAX_TAG_LOOKUP); - if (found?.variables !== undefined) return found; - firstStatic ??= found; - } else if (depth < MAX_SEARCH_DEPTH) { - const children = ownValues(value, budget); + for (let index = messages.length - 1; index >= stop; index -= 1) { + const tag = tagOf(messages[index]); - for (let index = children.length - 1; index >= 0; index -= 1) { - stack.push([children[index], depth + 1]); - } - } + if (tag) return tag; } - return firstStatic; + return undefined; } -/** At most `limit` own data values of an array or a plain object, in order. */ -function ownValues(value: unknown, limit: number): unknown[] { - if (typeof value !== "object" || value === null) return []; - let keys: string[]; - - if (Array.isArray(value)) { - keys = Array.from({ length: Math.min(value.length, limit) }, (_, index) => String(index)); - } else { - const prototype: unknown = Object.getPrototypeOf(value); - - if (prototype !== Object.prototype && prototype !== null) return []; - keys = Object.keys(value).slice(0, limit); - } - - return keys.map((key) => Object.getOwnPropertyDescriptor(value, key)?.value); +function tagOf(message: unknown): CompileTag | undefined { + return typeof message === "object" && message !== null + ? (message as { [COMPILED_BY]?: CompileTag })[COMPILED_BY] + : undefined; } function checkOptions(name: string, options: GetPromptOptions, type: string): void { @@ -649,26 +618,24 @@ function createPrompt(input: { if (scope?.prompt === prompt) scope.variables = variables; - const compileText = (template: string) => - remember(fill(template, variables), template, prompt, variables); - return type === "chat" - ? compileMessages(content.messages, variables, compileText) - : compileText(content.text ?? ""); + ? compileMessages(content.messages, Object.freeze({ prompt, variables })) + : fill(content.text ?? "", variables); }, }) as Prompt; return prompt; } -function compileMessages( - messages: readonly PromptMessage[], - variables: PromptVariables, - compileText: (template: string) => string, -): unknown[] { +function compileMessages(messages: readonly PromptMessage[], tag: CompileTag): unknown[] { + const { variables } = tag; + return messages.flatMap((message) => { if (message.role !== "placeholder") { - return [{ role: message.role, content: compileText(message.content) }]; + const compiled = { role: message.role, content: fill(message.content, variables) }; + Object.defineProperty(compiled, COMPILED_BY, { value: tag }); + + return [compiled]; } const value = Object.hasOwn(variables, message.name) ? variables[message.name] : undefined; @@ -683,39 +650,6 @@ function compileMessages( }); } -/** - * Keeps `text` for promptAttributesFor, as the newest entry. A template of only variable tags is - * skipped: its text is input of the app, which an unrelated request can hold too. - */ -function remember(text: string, template: string, prompt: Prompt, variables: object): string { - const fixedText = template.replace(VARIABLE, ""); - - if (fixedText.trim() === "" || text.length > MAX_COMPILED_CHARACTERS) return text; - - if (compiledTexts.delete(text)) compiledCharacters -= text.length; - compiledTexts.set(text, { prompt, variables: fixedText === template ? undefined : variables }); - compiledCharacters += text.length; - longestCompiledText = Math.max(longestCompiledText, text.length); - let droppedLongest = false; - - while (compiledTexts.size > MAX_COMPILED_TEXTS || compiledCharacters > MAX_COMPILED_CHARACTERS) { - const oldest = compiledTexts.keys().next().value!; - compiledTexts.delete(oldest); - compiledCharacters -= oldest.length; - droppedLongest ||= oldest.length === longestCompiledText; - } - - if (droppedLongest) { - longestCompiledText = 0; - - for (const kept of compiledTexts.keys()) { - longestCompiledText = Math.max(longestCompiledText, kept.length); - } - } - - return text; -} - function fill(text: string, variables: PromptVariables): string { return text.replace(VARIABLE, (_tag, name: string) => { const value = Object.hasOwn(variables, name) ? variables[name] : undefined; diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index 372150e..8e68644 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -743,108 +743,154 @@ async function promptOf(overrides: Record): Promise { }); } -test("promptAttributesFor finds a compiled text in the request shapes of the providers", async () => { +const DESK_PROMPT = { + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "user", content: "Order {{order}}: where is it?" }, + ], +}; + +test("promptAttributesFor names the compile of a message object; a copy or a string names nothing", async () => { setup(); - const prompt = await promptOf({ - messages: [ - { role: "system", content: "Answer as the {{team}} desk." }, - { role: "user", content: "Order {{order}}: where is it?" }, - ], - }); + const prompt = await promptOf(DESK_PROMPT); const compiled = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; - const [system, user] = compiled.map((message) => message.content); - - const requests = { - same: compiled, - spread: compiled.map((message) => ({ ...message })), - json: JSON.parse(JSON.stringify(compiled)) as unknown, - anthropic: [system, compiled.slice(1)], - bedrock: [[{ text: system }], [{ role: "user", content: [{ text: user }] }]], - google: [{ parts: [{ text: system }] }, [{ role: "user", parts: [{ text: user }] }]], + const attributes = { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', }; - for (const request of Object.values(requests)) { - expect(promptAttributesFor(request)).toEqual({ - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", - "td.prompt.variables": '{"team":"billing","order":"#4"}', - }); + expect(promptAttributesFor(compiled)).toEqual(attributes); + expect(promptAttributesFor(compiled[0])).toEqual(attributes); + expect(JSON.stringify(compiled[0])).toBe( + '{"role":"system","content":"Answer as the billing desk."}', + ); + + for (const copy of [ + compiled.map((message) => ({ ...message })), + JSON.parse(JSON.stringify(compiled)) as unknown, + structuredClone(compiled), + compiled.map((message) => message.content), + ]) { + expect(promptAttributesFor(copy)).toBeUndefined(); } - expect( - promptAttributesFor([{ role: "user", content: "Order #5: where is it?" }]), - ).toBeUndefined(); + const text = await promptOf({ type: "text", messages: [], text: "Summarize {{doc}}." }); + expect(promptAttributesFor([text.compile({ doc: "the policy" })])).toBeUndefined(); }); -test("a static text gives no variables, and a template of only a variable tag is not kept", async () => { +test("an unrelated message with the text of a compiled message is not tagged", async () => { setup(); - const chat = await promptOf({ - messages: [ - { role: "system", content: "You are the refunds agent." }, - { role: "user", content: "{{ question }}" }, - ], + const prompt = await promptOf({ + messages: [{ role: "system", content: "You are a helpful assistant." }], }); - const compiled = chat.compile({ question: "Where is my refund?" }); + prompt.compile(); - expect(promptAttributesFor(compiled)).toEqual({ - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", - }); - expect(promptAttributesFor([{ role: "user", content: "Where is my refund?" }])).toBeUndefined(); + expect( + promptAttributesFor([ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is 2 + 2?" }, + ]), + ).toBeUndefined(); +}); - const text = await promptOf({ type: "text", messages: [], text: "Summarize {{doc}} in a line." }); - const summary = text.compile({ doc: "the refund policy" }) as string; +test("two tasks with one text and different other variables each record their own", async () => { + const { spans } = setup(); + const prompt = await promptOf(DESK_PROMPT); - expect(promptAttributesFor(summary)).toMatchObject({ - "td.prompt.variables": '{"doc":"the refund policy"}', + // Both compiles make the system text "Answer as the billing desk.", and each task sends only it. + const [systemA] = prompt.compile({ team: "billing", order: "#A" }) as ChatMessage[]; + const [systemB] = prompt.compile({ team: "billing", order: "#B" }) as ChatMessage[]; + startSpan("a", { type: "generation", attributes: promptAttributesFor([systemA]) }).end(); + startSpan("b", { type: "generation", attributes: promptAttributesFor([systemB]) }).end(); + await flush(); + + expect( + Object.fromEntries( + spans.getFinishedSpans().map((span) => [span.name, span.attributes["td.prompt.variables"]]), + ), + ).toEqual({ + a: '{"team":"billing","order":"#A"}', + b: '{"team":"billing","order":"#B"}', }); }); -test("a text with variables wins over a static text, and the first one wins over a later one", async () => { +test("a rollout with two versions of the same messages records the version of each call", async () => { setup(); - const refunds = await promptOf({ - name: "refunds/system", - messages: [{ role: "system", content: "You are the refunds agent, v2." }], - }); - const order = await promptOf({ - name: "orders/question", - messages: [{ role: "user", content: "Where is order {{order}}?" }], - }); - const language = await promptOf({ - name: "style/language", - messages: [{ role: "system", content: "Reply in {{language}}." }], - }); + const { fetch } = fakeFetch( + reply({ + ...DESK_PROMPT, + rollout: { + percent: 16, + version: 13, + ...DESK_PROMPT, + text: null, + model: { provider: "openai", name: "gpt-4o-mini" }, + parameters: {}, + tools: [], + responseFormat: { type: "text" }, + }, + }), + ); + const options = { apiKey: "td_live_a", apiUrl: newApiUrl(), fetch }; + vi.spyOn(Math, "random").mockReturnValueOnce(0.159).mockReturnValueOnce(0.16); - const system = refunds.compile() as ChatMessage[]; - const question = order.compile({ order: "#4" }) as ChatMessage[]; - const style = language.compile({ language: "French" }) as ChatMessage[]; + const rollout = await getPrompt("support/reply", options); + const production = await getPrompt("support/reply", options); + const fromRollout = rollout.compile({ team: "billing", order: "#4" }); + const fromProduction = production.compile({ team: "billing", order: "#4" }); - expect(promptAttributesFor([...system, ...question])).toMatchObject({ - "td.prompt.name": "orders/question", - "td.prompt.variables": '{"order":"#4"}', - }); - expect(promptAttributesFor([...system, ...style, ...question])).toMatchObject({ - "td.prompt.name": "style/language", - "td.prompt.variables": '{"language":"French"}', - }); + expect(promptAttributesFor(fromRollout)?.["td.prompt.version"]).toBe(13); + expect(promptAttributesFor(fromProduction)?.["td.prompt.version"]).toBe(12); }); -test("each compile keeps its own values, so two tasks with one prompt get their own variables", async () => { +test("the newest compiled message wins, also after 170 messages of history", async () => { setup(); const prompt = await promptOf({ - messages: [{ role: "user", content: "Track parcel {{parcel}}." }], + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], }); + const history = Array.from({ length: 170 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: `message ${index}`, + })); + + const firstTurn = prompt.compile({ + team: "billing", + question: "Where is #4?", + history, + }) as ChatMessage[]; + const answer = { role: "assistant", content: "On its way." }; + const secondTurn = prompt.compile({ + team: "returns", + question: "Can I return it?", + }) as ChatMessage[]; + const chat = [...firstTurn, answer, ...secondTurn.slice(1)]; + + expect(promptAttributesFor(firstTurn)?.["td.prompt.variables"]).toContain('"team":"billing"'); + expect(promptAttributesFor([firstTurn[0], ...history])?.["td.prompt.variables"]).toContain( + '"team":"billing"', + ); + expect(promptAttributesFor(chat)?.["td.prompt.variables"]).toBe( + '{"team":"returns","question":"Can I return it?"}', + ); +}); - const first = prompt.compile({ parcel: "A-1" }); - const second = prompt.compile({ parcel: "B-2" }); +test("promptAttributesFor looks at the last 1000 messages", async () => { + setup(); + const prompt = await promptOf(DESK_PROMPT); + const [system] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; + const filler = (count: number) => Array.from({ length: count }, () => ({ role: "user" })); - expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe('{"parcel":"A-1"}'); - expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe('{"parcel":"B-2"}'); + expect(promptAttributesFor([system, ...filler(999)])).toBeDefined(); + expect(promptAttributesFor([system, ...filler(1000)])).toBeUndefined(); }); -test("withPrompt wins over a compiled text of another prompt", async () => { +test("withPrompt wins over a compiled message of another prompt", async () => { const { spans } = setup(); const scoped = await promptOf({ name: "scoped/prompt" }); const other = await promptOf({ @@ -861,7 +907,7 @@ test("withPrompt wins over a compiled text of another prompt", async () => { expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.name"]).toBe("scoped/prompt"); }); -test("promptAttributesFor omits variables when input capture is off and masks them when on", async () => { +test("td.prompt.variables follows the input capture of the client and of the span", async () => { const prompt = await promptOf({ messages: [{ role: "user", content: "My card is {{card}}." }], }); @@ -875,77 +921,25 @@ test("promptAttributesFor omits variables when input capture is off and masks th }); await shutdown(); - setup({ + const { spans } = setup({ mask: (value, { key }) => (key === "td.prompt.variables" ? { card: "[masked]" } : value), }); - expect(promptAttributesFor(compiled)?.["td.prompt.variables"]).toBe('{"card":"[masked]"}'); -}); - -test("the kept texts stop at 1000 and at 2,000,000 characters, oldest out first", async () => { - setup(); - const ticket = await promptOf({ type: "text", messages: [], text: "Ticket {{n}} is open." }); - - for (let n = 0; n <= 1000; n += 1) ticket.compile({ n }); - - expect(promptAttributesFor("Ticket 0 is open.")).toBeUndefined(); - expect(promptAttributesFor("Ticket 1 is open.")).toMatchObject({ - "td.prompt.variables": '{"n":1}', - }); - - const note = await promptOf({ type: "text", messages: [], text: "Note: {{body}}" }); - const notes = ["a", "b", "c", "d", "e"].map( - (letter) => note.compile({ body: letter.repeat(500_000 - "Note: ".length) }) as string, - ); - - expect(promptAttributesFor(notes[0])).toBeUndefined(); - expect(promptAttributesFor(notes[1])?.["td.prompt.name"]).toBe("support/reply"); - - // A text over the character limit is not kept, and it does not push the others out. - note.compile({ body: "f".repeat(2_000_001) }); - expect(promptAttributesFor(notes[1])?.["td.prompt.name"]).toBe("support/reply"); - - // A text compiled again counts once. - for (let n = 0; n < 5; n += 1) note.compile({ body: "e".repeat(500_000 - "Note: ".length) }); - expect(promptAttributesFor(notes[4])?.["td.prompt.name"]).toBe("support/reply"); -}); - -test("promptAttributesFor looks 6 levels and 500 values deep", async () => { - setup(); - const prompt = await promptOf({ - messages: [{ role: "user", content: "Find invoice {{invoice}}." }], - }); - const [{ content }] = prompt.compile({ invoice: "INV-7" }) as ChatMessage[]; - const nest = (value: unknown, levels: number): unknown => - levels === 0 ? value : nest([value], levels - 1); + const attributes = promptAttributesFor(compiled); + expect(attributes?.["td.prompt.variables"]).toBe('{"card":"[masked]"}'); - expect(promptAttributesFor(nest(content, 6))).toBeDefined(); - expect(promptAttributesFor(nest(content, 7))).toBeUndefined(); + startSpan("private", { type: "generation", captureInput: false, attributes }).end(); + startSpan("updated", { type: "generation", captureInput: false }).update({ attributes }).end(); + await flush(); - // The request array, 498 strings, and the text are 500 values. - expect(promptAttributesFor([...Array(498).fill("filler"), content])).toBeDefined(); - expect(promptAttributesFor([...Array(499).fill("filler"), content])).toBeUndefined(); + for (const span of spans.getFinishedSpans()) { + expect(span.attributes["td.prompt.name"]).toBe("support/reply"); + expect(span.attributes["td.prompt.variables"]).toBeUndefined(); + } }); -test("promptAttributesFor runs no getter and reports a trap that throws", async () => { +test("promptAttributesFor reports an error and gives undefined", async () => { const errors: Error[] = []; setup({ onError: (error) => errors.push(error) }); - const prompt = await promptOf({ - messages: [{ role: "user", content: "Refund order {{order}}." }], - }); - const [{ content }] = prompt.compile({ order: "#3" }) as ChatMessage[]; - let reads = 0; - - const getter = { - role: "user", - get content() { - reads += 1; - - return content; - }, - }; - - expect(promptAttributesFor([getter])).toBeUndefined(); - expect(reads).toBe(0); const trap = new Proxy([], { get() { @@ -953,7 +947,7 @@ test("promptAttributesFor runs no getter and reports a trap that throws", async }, }); - expect(promptAttributesFor([trap, content])).toBeUndefined(); + expect(promptAttributesFor(trap)).toBeUndefined(); expect(errors.map((error) => error.message)).toEqual(["trap"]); }); diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index 956b634..d815bd7 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -244,10 +244,16 @@ def _offered_tools(tools: Any) -> dict[str, str] | None: return {"gen_ai.tool.definitions": json_dumps(definitions)} if definitions else None -def _prompt_attributes(request: object) -> dict[str, Any]: +def _add_prompt(fields: Mapping[str, Any], messages: object) -> dict[str, Any]: + """``fields`` plus the prompt tag of ``messages``, the list that the app passed: the copy that + ``_normalize_request_params`` makes has no tag. Call it where the span starts, so that a + ``with_prompt`` scope there wins.""" # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + prompt = cast("dict[str, Any]", read(messages)) if callable(read) else {} + if not prompt: + return dict(fields) + return {**fields, "attributes": {**(fields.get("attributes") or {}), **prompt}} def _messages_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: @@ -270,11 +276,7 @@ def _messages_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "type": "generation", "model": model, "input": input_value, - "attributes": { - **(_offered_tools(tools) or {}), - **_prompt_attributes([params.get("system"), params.get("messages")]), - } - or None, + "attributes": _offered_tools(tools), "system_instructions": _native(params.get("system")), "temperature": _number(params.get("temperature")), "top_p": _number(params.get("top_p")), @@ -1462,16 +1464,21 @@ def __getattr__(self, name: str) -> Any: class _InstrumentedMessageStreamManager: - def __init__(self, inner: Any, name: str, fields: Mapping[str, Any], provider: str) -> None: + def __init__( + self, inner: Any, name: str, fields: Mapping[str, Any], messages: object, provider: str + ) -> None: self._inner = inner self._name = name self._fields = fields + self._messages = messages self._provider = provider self._proxy: _InstrumentedStream | None = None def __enter__(self) -> Any: handle = telemetry_dev.start_span( - self._name, provider=self._provider, **_clean_fields(self._fields) + self._name, + provider=self._provider, + **_clean_fields(_add_prompt(self._fields, self._messages)), ) end = _end_once(handle) started_at = time.perf_counter() @@ -1501,16 +1508,21 @@ def __getattr__(self, name: str) -> Any: class _InstrumentedAsyncMessageStreamManager: - def __init__(self, inner: Any, name: str, fields: Mapping[str, Any], provider: str) -> None: + def __init__( + self, inner: Any, name: str, fields: Mapping[str, Any], messages: object, provider: str + ) -> None: self._inner = inner self._name = name self._fields = fields + self._messages = messages self._provider = provider self._proxy: _InstrumentedAsyncStream | None = None async def __aenter__(self) -> Any: handle = telemetry_dev.start_span( - self._name, provider=self._provider, **_clean_fields(self._fields) + self._name, + provider=self._provider, + **_clean_fields(_add_prompt(self._fields, self._messages)), ) end = _end_once(handle) started_at = time.perf_counter() @@ -1543,9 +1555,10 @@ def __getattr__(self, name: str) -> Any: def _start_span( - params: Mapping[str, Any], provider: str + params: Mapping[str, Any], messages: object, provider: str ) -> tuple[telemetry_dev.SpanHandle, Callable[..., None], float]: name, fields = _messages_request(params) + fields = _add_prompt(fields, messages) handle = telemetry_dev.start_span(name, provider=provider, **_clean_fields(fields)) return handle, _end_once(handle), time.perf_counter() @@ -1558,7 +1571,9 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, provider(resource)) + handle, end, started_at = _start_span( + call_kwargs, _sent_params(kwargs).get("messages"), provider(resource) + ) try: result = original(*args, **call_kwargs) except BaseException as exc: @@ -1582,7 +1597,9 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return await original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, provider(resource)) + handle, end, started_at = _start_span( + call_kwargs, _sent_params(kwargs).get("messages"), provider(resource) + ) try: result = await original(*args, **call_kwargs) except BaseException as exc: @@ -1606,8 +1623,11 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) name, fields = _messages_request(call_kwargs) + messages = _sent_params(kwargs).get("messages") manager = original(*args, **call_kwargs) - return _InstrumentedMessageStreamManager(manager, name, fields, provider(resource)) + return _InstrumentedMessageStreamManager( + manager, name, fields, messages, provider(resource) + ) setattr(wrapper, _WRAPPED_ATTR, True) setattr(wrapper, _ORIGINAL_ATTR, original) @@ -1622,8 +1642,11 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) name, fields = _messages_request(call_kwargs) + messages = _sent_params(kwargs).get("messages") manager = original(*args, **call_kwargs) - return _InstrumentedAsyncMessageStreamManager(manager, name, fields, provider(resource)) + return _InstrumentedAsyncMessageStreamManager( + manager, name, fields, messages, provider(resource) + ) setattr(wrapper, _WRAPPED_ATTR, True) setattr(wrapper, _ORIGINAL_ATTR, original) diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 8ffbc67..c22c555 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -437,12 +437,29 @@ def handler(request: httpx.Request) -> httpx.Response: ) +SCOPED_PROMPT = telemetry_dev.ChatPrompt( + name="scoped/prompt", + version=3, + label="production", + messages=[{"role": "user", "content": "Hi"}], + model=None, + parameters={}, + tools=[], + response_format={"type": "text"}, + is_fallback=False, +) + + @pytest.mark.parametrize( - ("field", "capture_input", "older_core"), - [("system", False, False), ("messages", True, False), ("messages", True, True)], + ("messages", "capture_input", "older_core"), + [("compiled", False, False), ("copied", True, False), ("compiled", True, True)], ) -def test_create_outside_with_prompt_records_the_prompt_of_a_compiled_text( - make: Any, monkeypatch: pytest.MonkeyPatch, field: str, capture_input: bool, older_core: bool +def test_create_outside_with_prompt_records_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + messages: str, + capture_input: bool, + older_core: bool, ) -> None: memory = make(capture_input=capture_input) if older_core: @@ -453,15 +470,15 @@ def test_create_outside_with_prompt_records_the_prompt_of_a_compiled_text( client.messages.create( model="claude-sonnet-4-6", max_tokens=64, - system=system["content"] if field == "system" else "Be terse.", - messages=[{"role": "user", "content": user["content"] if field == "messages" else "Hi"}], + system=system["content"], + messages=[user if messages == "compiled" else dict(user)], ) a = attrs(only_span(memory)) assert a["gen_ai.request.model"] == "claude-sonnet-4-6" assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( {} - if older_core + if older_core or messages == "copied" else { "td.prompt.name": "support/reply", "td.prompt.version": 12, @@ -471,6 +488,59 @@ def test_create_outside_with_prompt_records_the_prompt_of_a_compiled_text( ) +@pytest.mark.parametrize("scope_around", ["stream()", "__enter__"]) +def test_messages_stream_takes_the_prompt_from_the_context_where_its_span_starts( + memory: SimpleNamespace, scope_around: str +) -> None: + _system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + client = wrapped_sync_client(lambda _request: named_sse_response(stream_events())) + request: dict[str, Any] = {"model": "claude-sonnet-4-6", "max_tokens": 64, "messages": [user]} + + if scope_around == "stream()": + with telemetry_dev.with_prompt(SCOPED_PROMPT): + manager = client.messages.stream(**request) + with manager as stream: + "".join(stream.text_stream) + else: + manager = client.messages.stream(**request) + with telemetry_dev.with_prompt(SCOPED_PROMPT), manager as stream: + "".join(stream.text_stream) + + # The span starts at __enter__, so the scope there wins, and without one the message names it. + assert attrs(only_span(memory))["td.prompt.name"] == ( + "support/reply" if scope_around == "stream()" else "scoped/prompt" + ) + + +@pytest.mark.parametrize("scope_around", ["stream()", "__aenter__"]) +async def test_async_messages_stream_takes_the_prompt_from_the_context_where_its_span_starts( + memory: SimpleNamespace, scope_around: str +) -> None: + _system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + + client = wrapped_async_client(handler) + request: dict[str, Any] = {"model": "claude-sonnet-4-6", "max_tokens": 64, "messages": [user]} + + if scope_around == "stream()": + with telemetry_dev.with_prompt(SCOPED_PROMPT): + manager = client.messages.stream(**request) + async with manager as stream: + [text async for text in stream.text_stream] + else: + manager = client.messages.stream(**request) + with telemetry_dev.with_prompt(SCOPED_PROMPT): + async with manager as stream: + [text async for text in stream.text_stream] + await client.close() + + assert attrs(only_span(memory))["td.prompt.name"] == ( + "support/reply" if scope_around == "stream()" else "scoped/prompt" + ) + + def test_create_maps_native_messages_system_params_usage_finish_and_provider( memory: SimpleNamespace, ) -> None: diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py index 04af28b..f6fc07d 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_fields.py @@ -12,10 +12,10 @@ def clean(fields: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in fields.items() if value is not None} -def prompt_attributes(request: object) -> dict[str, Any]: +def prompt_attributes(messages: object) -> dict[str, Any]: # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + return cast("dict[str, Any]", read(messages)) if callable(read) else {} def metadata_fields(response: dict[str, Any] | BaseException | None) -> dict[str, Any]: diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py index 04a1162..d22e50b 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_invoke_model.py @@ -4,7 +4,7 @@ import math from typing import Any -from ._fields import clean, prompt_attributes +from ._fields import clean _USAGE_ITERATION_LIMIT = 1000 _CLAUDE_USAGE_KEYS = ( @@ -93,8 +93,6 @@ def request_fields(params: dict[str, Any]) -> dict[str, Any]: fields = {"provider": "amazon-bedrock", "model": model, "input": body} if is_embedding_model(model): fields["output_type"] = "embedding" - else: - fields["attributes"] = prompt_attributes(body) or None fields.update(sampling_fields(model, body)) return clean(fields) diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py index 0b68201..d27304d 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/_registry.py @@ -57,7 +57,7 @@ def _converse_request(params: dict[str, Any]) -> dict[str, Any]: "model": _model(params), "input": normalize_messages(params.get("messages")), "system_instructions": normalize_content_list(params.get("system")) or None, - "attributes": prompt_attributes([params.get("system"), params.get("messages")]) or None, + "attributes": prompt_attributes(params.get("messages")) or None, "temperature": inference.get("temperature"), "top_p": inference.get("topP"), "max_tokens": inference.get("maxTokens"), diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 8919186..505d4d6 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -14,6 +14,7 @@ from botocore.client import BaseClient from botocore.exceptions import ClientError, EventStreamError from botocore.response import StreamingBody +from botocore.stub import Stubber import telemetry_dev_bedrock._streams as bedrock_streams from telemetry_dev_bedrock import instrument_bedrock, uninstrument_bedrock, wrap_bedrock @@ -130,75 +131,56 @@ def _json_attr(span: Any, key: str) -> Any: ) -@pytest.mark.parametrize("older_core", [False, True]) -def test_converse_and_invoke_model_outside_with_prompt_record_the_prompt_of_a_compiled_text( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, older_core: bool +@pytest.mark.parametrize( + ("messages", "older_core"), + [("changed in place", False), ("new", False), ("changed in place", True)], +) +def test_converse_outside_with_prompt_records_the_prompt_of_compiled_messages( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, messages: str, older_core: bool ) -> None: if older_core: monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") - converse_reply = { - "output": {"message": {"role": "assistant", "content": [{"text": "Soon."}]}}, - "stopReason": "end_turn", - } - answer = _body({"content": [{"type": "text", "text": "Soon."}], "stop_reason": "end_turn"}) - embedding = _body({"embedding": [0.1, 0.2], "inputTextTokenCount": 6}) + blocks = [{"text": user["content"]}] + # Converse takes content blocks: a message changed in place keeps its tag, a new one has none. + if messages == "new": + message = {"role": "user", "content": blocks} + else: + user["content"] = blocks + message = user client = _client("bedrock-runtime") - _stub_api_call( - client, - [ - converse_reply, - converse_reply, - { - "body": StreamingBody(io.BytesIO(answer), len(answer)), - "contentType": "application/json", - }, - { - "body": StreamingBody(io.BytesIO(embedding), len(embedding)), - "contentType": "application/json", - }, - ], - monkeypatch, + # The stubber runs botocore's validation of the request. + stubber = Stubber(client) + stubber.add_response( + "converse", + { + "output": {"message": {"role": "assistant", "content": [{"text": "Soon."}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + "metrics": {"latencyMs": 1}, + }, ) - wrapped = cast(Any, wrap_bedrock(client)) - wrapped.converse( - modelId="anthropic.claude-3-haiku", - system=[{"text": system["content"]}], - messages=[{"role": "user", "content": [{"text": "Hi"}]}], - ) - wrapped.converse( - modelId="anthropic.claude-3-haiku", - messages=[{"role": "user", "content": [{"text": user["content"]}]}], - ) - wrapped.invoke_model( - modelId="anthropic.claude-3-haiku", - contentType="application/json", - body=_body({"max_tokens": 64, "messages": [{"role": "user", "content": user["content"]}]}), - )["body"].read() - wrapped.invoke_model( - modelId="amazon.titan-embed-text-v2:0", - contentType="application/json", - body=_body({"inputText": user["content"]}), - )["body"].read() - - system_span, messages_span, invoke_span, embed_span = ( - dict(span.attributes or {}) for span in memory.span_exporter.get_finished_spans() - ) - for span in (system_span, messages_span, invoke_span): - assert span["gen_ai.request.model"] == "anthropic.claude-3-haiku" - assert {key: value for key, value in span.items() if key.startswith("td.prompt.")} == ( - {} - if older_core - else { - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", - "td.prompt.variables": '{"team":"billing","order":"#4"}', - } + with stubber: + cast(Any, wrap_bedrock(client)).converse( + modelId="anthropic.claude-3-haiku", + system=[{"text": system["content"]}], + messages=[message], ) - assert embed_span["gen_ai.operation.name"] == "embeddings" - assert "td.prompt.name" not in embed_span + + [span] = memory.span_exporter.get_finished_spans() + a = dict(span.attributes or {}) + assert a["gen_ai.request.model"] == "anthropic.claude-3-haiku" + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( + {} + if older_core or messages == "new" + else { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', + } + ) def test_converse_happy_path(memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index 776c9de..8fc4b34 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -353,10 +353,10 @@ def _request_attributes(config: Any) -> dict[str, Any]: return attrs -def _prompt_attributes(request: object) -> dict[str, Any]: +def _prompt_attributes(messages: object) -> dict[str, Any]: # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + return cast("dict[str, Any]", read(messages)) if callable(read) else {} def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[str, dict[str, Any]]: @@ -389,7 +389,7 @@ def _generate_request_fields(model: Any, contents: Any, config: Any) -> tuple[st output_type = _output_type(config) if output_type is not None: fields["output_type"] = output_type - attrs = {**_request_attributes(config), **_prompt_attributes([system_instruction, contents])} + attrs = {**_request_attributes(config), **_prompt_attributes(contents)} if attrs: fields["attributes"] = attrs return f"chat {model_name}", fields diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index c74c08c..e397763 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -673,36 +673,49 @@ async def run() -> Any: @pytest.mark.parametrize( - ("field", "capture_input", "older_core"), - [("system_instruction", True, False), ("contents", False, False), ("contents", True, True)], + ("contents", "capture_input", "older_core"), + [ + ("changed in place", True, False), + ("changed in place", False, False), + ("new", True, False), + ("changed in place", True, True), + ], ) -def test_generate_content_outside_with_prompt_records_the_prompt_of_a_compiled_text( - make: Any, monkeypatch: pytest.MonkeyPatch, field: str, capture_input: bool, older_core: bool +def test_generate_content_outside_with_prompt_records_the_prompt_of_compiled_messages( + make: Any, + monkeypatch: pytest.MonkeyPatch, + contents: str, + capture_input: bool, + older_core: bool, ) -> None: memory = make(capture_input=capture_input) if older_core: monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") - client, _transport = client_with_transport([happy_payload()]) + parts = [{"text": user["content"]}] + # Gemini takes parts: a message changed in place keeps its tag, a new one has none. + if contents == "new": + content: dict[str, Any] = {"role": "user", "parts": parts} + else: + del user["content"] + user["parts"] = parts + content = user + client, transport = client_with_transport([happy_payload()]) wrap_google_genai(client).models.generate_content( model="gemini-2.5-flash", - contents=[ - types.Content( - role="user", - parts=[types.Part(text=user["content"] if field == "contents" else "Hi")], - ) - ], - config=types.GenerateContentConfig( - system_instruction=system["content"] if field == "system_instruction" else "Be terse." - ), + contents=[content], + config=types.GenerateContentConfig(system_instruction=system["content"]), ) + assert cast(Any, transport.calls[0])["contents"] == [ + {"role": "user", "parts": [{"text": "Where is order #4?"}]} + ] a = attrs(only_span(memory)) assert a["gen_ai.request.model"] == "gemini-2.5-flash" assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( {} - if older_core + if older_core or contents == "new" else { "td.prompt.name": "support/reply", "td.prompt.version": 12, diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index ee18732..1a975f4 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -455,10 +455,10 @@ def _output_type(response_format: Any) -> str | None: return kind -def _prompt_attributes(request: object) -> dict[str, Any]: +def _prompt_attributes(messages: object) -> dict[str, Any]: # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + return cast("dict[str, Any]", read(messages)) if callable(read) else {} def _completion_request( @@ -554,7 +554,7 @@ def _responses_request( input_value, input_budget = _bounded_responses_native(_arg(args, kwargs, "input", 0)) attributes: dict[str, Any] = { "gen_ai.operation.name": "chat", - **_prompt_attributes([kwargs.get("instructions"), _arg(args, kwargs, "input", 0)]), + **_prompt_attributes(_arg(args, kwargs, "input", 0)), } if input_budget is not None and input_budget.truncated: attributes["telemetry.dev.capture.truncated"] = True diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index ae34865..a4c8dd1 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -74,44 +74,43 @@ def finished_spans(env: SimpleNamespace) -> list[ReadableSpan]: @pytest.mark.parametrize( - ("api", "capture_input", "older_core"), + ("api", "messages", "capture_input", "older_core"), [ - ("completion", False, False), - ("responses instructions", True, False), - ("responses input", True, False), - ("completion", True, True), + ("completion", "compiled", False, False), + ("responses", "compiled", True, False), + ("completion", "copied", True, False), + ("completion", "compiled", True, True), ], ) def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( - make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, ) -> None: memory = make(capture_input=capture_input) if older_core: monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") - system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] if api == "completion": - telemetry_dev_litellm.completion( - model="gpt-4o-mini", messages=[system, user], mock_response="Soon." - ) + telemetry_dev_litellm.completion(model="gpt-4o-mini", messages=sent, mock_response="Soon.") else: def fake_responses(*_args: Any, **_kwargs: Any) -> SimpleNamespace: return responses_result() monkeypatch.setattr(litellm, "responses", fake_responses) - in_input = api == "responses input" - telemetry_dev_litellm.responses( - [{"role": "user", "content": user["content"] if in_input else "Hi"}], - "openai/gpt-4.1-mini", - instructions="Be terse." if in_input else system["content"], - ) + telemetry_dev_litellm.responses(sent, "openai/gpt-4.1-mini") a = attrs(only_span(memory)) assert a["gen_ai.request.model"] == ("gpt-4o-mini" if api == "completion" else "gpt-4.1-mini") assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( {} - if older_core + if older_core or messages == "copied" else { "td.prompt.name": "support/reply", "td.prompt.version": 12, diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index eb84699..fdad482 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -349,10 +349,10 @@ def _capture_text(value: str, budget: telemetry_dev.CaptureBudget) -> str: return value -def _prompt_attributes(request: object) -> dict[str, Any]: +def _prompt_attributes(messages: object) -> dict[str, Any]: # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + return cast("dict[str, Any]", read(messages)) if callable(read) else {} def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: @@ -441,7 +441,7 @@ def _chat_response(response: Any) -> dict[str, Any]: def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: model = _string(params.get("model")) - attributes = _prompt_attributes([params.get("instructions"), params.get("input")]) + attributes = _prompt_attributes(params.get("input")) captured, truncated = _bounded_responses_capture(params.get("input"), "input") if truncated: attributes["telemetry.dev.capture.truncated"] = True diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 6f71978..1ad3a36 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1227,41 +1227,43 @@ def prompt_attrs(span: ReadableSpan) -> dict[str, object]: @pytest.mark.parametrize( - ("api", "capture_input", "older_core"), + ("api", "messages", "capture_input", "older_core"), [ - ("chat", False, False), - ("responses instructions", True, False), - ("responses input", True, False), - ("chat", True, True), + ("chat", "compiled", False, False), + ("responses", "compiled", True, False), + ("chat", "copied", True, False), + ("chat", "compiled", True, True), ], ) def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( - make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, ) -> None: memory = make(capture_input=capture_input) if older_core: monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") - system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] def handler(_request: httpx.Request) -> httpx.Response: return json_response(chat_completion() if api == "chat" else response_payload()) client = wrap_openai(sync_client(handler)) if api == "chat": - cast(Any, client.chat.completions.create)(model="gpt-4o", messages=[system, user]) + cast(Any, client.chat.completions.create)(model="gpt-4o", messages=sent) else: - in_input = api == "responses input" - cast(Any, client.responses.create)( - model="gpt-4.1", - instructions="Be terse." if in_input else system["content"], - input=[{"role": "user", "content": user["content"] if in_input else "Hi"}], - ) + cast(Any, client.responses.create)(model="gpt-4.1", input=sent) span = only_span(memory) - assert attrs(span)["gen_ai.request.model"] == ("gpt-4o" if api == "chat" else "gpt-4.1") - assert prompt_attrs(span) == ( + a = attrs(span) + assert a["gen_ai.request.model"] == ("gpt-4o" if api == "chat" else "gpt-4.1") + assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( {} - if older_core + if older_core or messages == "copied" else { "td.prompt.name": "support/reply", "td.prompt.version": 12, @@ -1269,7 +1271,7 @@ def handler(_request: httpx.Request) -> httpx.Response: **({"td.prompt.variables": '{"team":"billing","order":"#4"}'} if capture_input else {}), } ) - assert ("gen_ai.input.messages" in attrs(span)) is capture_input + assert ("gen_ai.input.messages" in a) is capture_input def test_chat_completion_maps_native_messages_usage_finish_provider_and_sampling( diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index 4af3fb7..a5056d3 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -174,10 +174,10 @@ def _cost_usd(raw_usage: Any) -> float | None: return cost if cost is not None else upstream_cost -def _prompt_attributes(request: object) -> dict[str, Any]: +def _prompt_attributes(messages: object) -> dict[str, Any]: # Older cores have no prompt_attributes_for and record a prompt only inside with_prompt. read = getattr(telemetry_dev, "prompt_attributes_for", None) - return cast("dict[str, Any]", read(request)) if callable(read) else {} + return cast("dict[str, Any]", read(messages)) if callable(read) else {} def _chat_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: @@ -267,8 +267,7 @@ def _responses_request(params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: "max_tokens": _number(params.get("max_output_tokens")), "frequency_penalty": _number(params.get("frequency_penalty")), "presence_penalty": _number(params.get("presence_penalty")), - "attributes": _prompt_attributes([params.get("instructions"), params.get("input")]) - or None, + "attributes": _prompt_attributes(params.get("input")) or None, }, ) diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 01456c2..400c4f5 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -238,47 +238,42 @@ def embeddings_payload() -> dict[str, Any]: @pytest.mark.parametrize( - ("api", "capture_input", "older_core"), + ("api", "messages", "capture_input", "older_core"), [ - ("chat", False, False), - ("responses instructions", True, False), - ("responses input", True, False), - ("chat", True, True), + ("chat", "compiled", False, False), + ("responses", "compiled", True, False), + ("chat", "copied", True, False), + ("chat", "compiled", True, True), ], ) def test_calls_outside_with_prompt_record_the_prompt_of_compiled_messages( - make: Any, monkeypatch: pytest.MonkeyPatch, api: str, capture_input: bool, older_core: bool + make: Any, + monkeypatch: pytest.MonkeyPatch, + api: str, + messages: str, + capture_input: bool, + older_core: bool, ) -> None: memory = make(capture_input=capture_input) if older_core: monkeypatch.delattr(telemetry_dev, "prompt_attributes_for") - system, user = SUPPORT_PROMPT.compile(team="billing", order="#4") + compiled = SUPPORT_PROMPT.compile(team="billing", order="#4") + sent = compiled if messages == "compiled" else [dict(message) for message in compiled] def handler(_request: httpx.Request) -> httpx.Response: return json_response(chat_payload() if api == "chat" else responses_payload()) client = wrap_open_router(sync_client(handler)) if api == "chat": - client.chat.send( - model="openai/gpt-4o-mini", - messages=[ - ChatSystemMessage(role="system", content=system["content"]), - ChatUserMessage(role="user", content=user["content"]), - ], - ) + client.chat.send(model="openai/gpt-4o-mini", messages=cast(Any, sent)) else: - in_input = api == "responses input" - client.responses.send( - model="openai/gpt-4o-mini", - input=[EasyInputMessage(role="user", content=user["content"] if in_input else "Hi")], - instructions="Be terse." if in_input else system["content"], - ) + client.responses.send(model="openai/gpt-4o-mini", input=cast(Any, sent)) a = attrs(only_span(memory)) assert a["gen_ai.request.model"] == "openai/gpt-4o-mini" assert {key: value for key, value in a.items() if key.startswith("td.prompt.")} == ( {} - if older_core + if older_core or messages == "copied" else { "td.prompt.name": "support/reply", "td.prompt.version": 12, diff --git a/sdks/python/README.md b/sdks/python/README.md index dfcd2b8..e39a537 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -253,19 +253,23 @@ for code that has not compiled in its own task. Model calls are the spans of typ integrations: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, LiteLLM, and OpenRouter. Spans from other OpenTelemetry tracers do not get the attributes. -Outside `with_prompt`, those provider integrations find the prompt in the request: `compile()` -keeps the texts that it makes (the last 1000, at most 2,000,000 characters), and a chat call -whose request holds one of them gets `td.prompt.name`, `td.prompt.version`, and -`td.prompt.label`, also when input capture is off. A text from a template with a variable also -gives `td.prompt.variables`, and it wins over a text from a template with none. A message of only -a variable tag, for example `{{ question }}`, is not kept, because its text is your user's input. -The text in the request must be equal to the compiled text. `with_prompt` wins over this. The -integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock -Converse and InvokeModel, Google GenAI `generate_content`, LiteLLM completion and responses, and -OpenRouter chat and responses, but not for images, audio, video, or OpenAI Realtime. With an -older `telemetry-dev` that has no `prompt_attributes_for`, a provider integration records the -prompt only in `with_prompt`. For your own spans, `prompt_attributes_for(request)` gives the -attributes to pass as `attributes`; it gives `{}` inside `with_prompt`. +Outside `with_prompt`, those provider integrations record the prompt of the message dicts that +`compile()` returned. `compile()` of a chat prompt returns instances of a `dict` subclass that +also name its prompt and variables: the keys and the JSON of the message do not change. A chat +call that sends such a message gets `td.prompt.name`, `td.prompt.version`, and `td.prompt.label` +of the newest one (of the last 1000 messages), also when input capture is off, and +`td.prompt.variables` when it is on. A copy of a message (`dict()`, `.copy()`, `copy.copy`, +`copy.deepcopy`, pickle, or JSON) and a new message with the same text do not name the prompt, so +the SDK does not record them. Bedrock Converse and Google GenAI take another message shape: change +the compiled dicts in place, or use `with_prompt`. A text prompt compiles to a string, which +cannot name its prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations +do it for OpenAI chat completions and responses, Anthropic messages (for `messages.stream()`, at +`__enter__`), Bedrock Converse, Google GenAI `generate_content`, LiteLLM completion and responses, +and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, +images, audio, video, or OpenAI Realtime. With an older `telemetry-dev` that has no +`prompt_attributes_for`, a provider integration records the prompt only in `with_prompt`. For your +own spans, `prompt_attributes_for(messages)` gives the attributes to pass as `attributes`; it gives +`{}` inside `with_prompt`, and a span with `capture_input=False` drops `td.prompt.variables`. ## Auto-metrics diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 43b863e..cb230fd 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -4,14 +4,11 @@ import contextlib import copy import hashlib -import itertools import json -import os import random import re import socket import threading -import types import urllib.error import urllib.parse import urllib.request @@ -53,10 +50,8 @@ _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") -_MAX_COMPILED_TEXTS = 1000 -_MAX_COMPILED_CHARACTERS = 2_000_000 -_MAX_SEARCH_DEPTH = 6 -_MAX_SEARCH_NODES = 500 +# How many messages prompt_attributes_for looks at, from the end. +_MAX_TAG_LOOKUP = 1000 class PromptModel(TypedDict): @@ -105,15 +100,16 @@ def compile( ) -> list[dict[str, Any]]: """Fill in the variables and replace each placeholder message with the list of messages in its variable. ``with_prompt`` records the variables of the last call before it starts - or inside it. Each text is also kept with its variables (the last 1000 texts), so that a - model call outside ``with_prompt`` that sends it can name the prompt; see - ``prompt_attributes_for``.""" + or inside it. It tags each message dict that it makes with this prompt and these + variables, so that a model call outside ``with_prompt`` that sends the message records + them; see ``prompt_attributes_for``.""" values = _record(self, variables, kwargs) + tag = _CompileTag(self, values) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": - content = _compile_text(self, str(message["content"]), values) - compiled.append({"role": message["role"], "content": content}) + content = _fill(str(message["content"]), values) + compiled.append(_CompiledMessage(tag, role=message["role"], content=content)) continue name = str(message["name"]) value = values.get(name) @@ -149,10 +145,9 @@ def messages(self) -> list[dict[str, Any]]: def compile(self, variables: Mapping[str, Any] | None = None, /, **kwargs: Any) -> str: """Fill in the variables. ``with_prompt`` records the variables of the last call before it - starts or inside it. The text is also kept with its variables (the last 1000 texts), so - that a model call outside ``with_prompt`` that sends it can name the prompt; see - ``prompt_attributes_for``.""" - return _compile_text(self, self.text, _record(self, variables, kwargs)) + starts or inside it. A string cannot carry a tag, so a model call outside ``with_prompt`` + does not record a text prompt.""" + return _fill(self.text, _record(self, variables, kwargs)) Prompt: TypeAlias = ChatPrompt | TextPrompt @@ -297,23 +292,22 @@ def prompt_attributes( return _attributes_of(scope.prompt, variables, client if capture_input else None) -def prompt_attributes_for(request: object) -> dict[str, AttributeValue]: - """The td.prompt.* attributes for a model call outside ``with_prompt``, from the texts that - ``compile()`` made: the first text in ``request`` from a template with a variable, else the - first text from a template with none. Only the first gives ``td.prompt.variables``, because - only its text holds the values. Gives ``{}`` inside ``with_prompt``, whose prompt the span - gets, and when no text matches. It reads lists, tuples, dicts, and the fields that objects - store (pydantic models), 6 levels and 500 values deep, and runs no property. It never - raises.""" +def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: + """The td.prompt.* attributes for a model call outside ``with_prompt`` that sends ``messages`` + (a list or tuple of messages, or one message): those of the newest message, of the last 1000, + that ``compile()`` of a chat prompt returned. The tag is on the dict that ``compile()`` made, so + a copy of it gives ``{}``. Gives ``{}`` inside ``with_prompt``, whose prompt the span gets. + ``td.prompt.variables`` needs input capture on in the client, and the span drops it when its + own input capture is off. It runs no code of the app and never raises.""" client = get_client() try: if isinstance(otel_context.get_value(_PROMPT_KEY), _PromptScope): return {} - found = _compiled_texts.find(request) - if found is None: + tag = _newest_tag(messages) + if tag is None: return {} capture = client if client is not None and client.capture_input else None - return _attributes_of(found.prompt, found.variables, capture) + return _attributes_of(tag.prompt, tag.variables, capture) except Exception as exc: if client is not None: client.report("prompt_attributes_for failed", exc) @@ -444,133 +438,36 @@ def _record( @dataclass(frozen=True) -class _CompiledText: - """The prompt that ``compile()`` made a text from.""" +class _CompileTag: + """The ``compile()`` call that made a chat message.""" - prompt: Prompt - variables: dict[str, Any] | None - """Only for a template with a variable: the text holds the values, so they belong to it.""" + prompt: ChatPrompt + variables: dict[str, Any] -class _CompiledTexts: - """The texts of the last ``compile()`` calls, oldest first, for model calls outside - ``with_prompt``.""" +class _CompiledMessage(dict[str, Any]): + """A message that ``compile()`` made: a dict with the tag in a slot. A copy, pickle, or + ``copy.deepcopy`` of it is a plain dict without the tag.""" - def __init__(self) -> None: - self.texts: OrderedDict[str, _CompiledText] = OrderedDict() - self.characters = 0 - self.longest = 0 - self.lock = threading.Lock() - - def remember(self, text: str, template: str, prompt: Prompt, values: dict[str, Any]) -> None: - """Keep ``text`` as the newest entry. A template of only variable tags is skipped: its - text is input of the app, which an unrelated request can hold too.""" - fixed_text = _VARIABLE.sub("", template) - if not fixed_text.strip() or len(text) > _MAX_COMPILED_CHARACTERS: - return - compiled = _CompiledText(prompt, None if fixed_text == template else values) - with self.lock: - if self.texts.pop(text, None) is not None: - self.characters -= len(text) - self.texts[text] = compiled - self.characters += len(text) - self.longest = max(self.longest, len(text)) - dropped_longest = False - while ( - len(self.texts) > _MAX_COMPILED_TEXTS or self.characters > _MAX_COMPILED_CHARACTERS - ): - oldest, _ = self.texts.popitem(last=False) - self.characters -= len(oldest) - dropped_longest = dropped_longest or len(oldest) == self.longest - if dropped_longest: - self.longest = max(map(len, self.texts), default=0) - - def find(self, request: object) -> _CompiledText | None: - first_static: _CompiledText | None = None - budget = _MAX_SEARCH_NODES - stack: list[tuple[object, int]] = [(request, 0)] - with self.lock: - while stack and budget > 0: - budget -= 1 - value, depth = stack.pop() - if type(value) is str: - found = self.texts.get(value) if len(value) <= self.longest else None - if found is not None and found.variables is not None: - return found - first_static = first_static or found - elif depth < _MAX_SEARCH_DEPTH: - children = _stored_values(value, budget) - stack.extend((child, depth + 1) for child in reversed(children)) - return first_static - - -_compiled_texts = _CompiledTexts() - - -def _lock_compiled_texts_before_fork() -> None: - _compiled_texts.lock.acquire() - - -def _unlock_compiled_texts_after_fork() -> None: - _compiled_texts.lock.release() - - -# fork() waits for the lock: the child has only the thread that forked, so a lock that another -# thread held at the fork would stay held there for all time. -if hasattr(os, "register_at_fork"): # absent on Windows - os.register_at_fork( - before=_lock_compiled_texts_before_fork, - after_in_parent=_unlock_compiled_texts_after_fork, - after_in_child=_unlock_compiled_texts_after_fork, - ) + __slots__ = ("compiled_by",) + def __init__(self, compiled_by: _CompileTag, **fields: Any) -> None: + super().__init__(**fields) + self.compiled_by = compiled_by -def _compile_text(prompt: Prompt, template: str, values: dict[str, Any]) -> str: - text = _fill(template, values) - _compiled_texts.remember(text, template, prompt, values) - return text - - -_TYPE_MRO = type.__dict__["__mro__"] -_TYPE_NAMESPACE = type.__dict__["__dict__"] - - -def _stored_values(value: object, limit: int) -> list[object]: - """At most ``limit`` values that ``value`` stores, in order: the items of a list or tuple, the - values of a dict, or the fields of another object in ``__dict__`` and ``__pydantic_extra__``. - Subclasses of list, tuple, and dict are skipped and fields are read from C-level storage, so - no override or property of the app runs.""" - cls = type(value) - if cls is list or cls is tuple: - return list(cast("list[object] | tuple[object, ...]", value)[:limit]) - if cls is dict: - return list(itertools.islice(cast("dict[object, object]", value).values(), limit)) - values: list[object] = [] - for name in ("__dict__", "__pydantic_extra__"): - stored = _stored(value, name) - if type(stored) is dict: - fields = cast("dict[str, object]", stored) - values.extend(itertools.islice(fields.values(), limit - len(values))) - return values + def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: + return dict, (dict(self),) -def _stored(value: object, name: str) -> object: - cls = type(value) - for klass in _TYPE_MRO.__get__(cls): - namespace = _TYPE_NAMESPACE.__get__(klass) - if name not in namespace: - continue - descriptor: object = namespace[name] - kind = type(descriptor) - storage = kind is types.MemberDescriptorType or ( - name == "__dict__" and kind is types.GetSetDescriptorType - ) - if not storage or cast(Any, descriptor).__name__ != name: - return None - try: - return cast(Any, descriptor).__get__(value, cls) - except (AttributeError, TypeError): - return None +def _newest_tag(messages: object) -> _CompileTag | None: + # Exact types: a subclass of list or tuple could run code of the app on each read. + if type(messages) is _CompiledMessage: + return messages.compiled_by + if type(messages) is not list and type(messages) is not tuple: + return None + for message in reversed(cast("list[object]", messages)[-_MAX_TAG_LOOKUP:]): + if type(message) is _CompiledMessage: + return message.compiled_by return None diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 0ac1b26..d66f086 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -38,6 +38,7 @@ ATTR_FINISH_REASONS, ATTR_OPERATION, ATTR_OUTPUT_TYPE, + ATTR_PROMPT_VARIABLES, ATTR_PROVIDER, ATTR_REQUEST_MODEL, ATTR_RESPONSE_ID, @@ -279,6 +280,9 @@ def _fields_to_attributes( raw = fields.get("attributes") if raw is not None: for key, value in raw.items(): + # The variables are input, also when the app passes them from prompt_attributes_for(). + if key == ATTR_PROMPT_VARIABLES and not state.capture_input: + continue attr = coerce_attr_value( value, max_len=sys.maxsize, diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 9fd8656..29297e3 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -2,15 +2,17 @@ import asyncio import contextvars +import copy +import gc import itertools import json -import os -import signal +import pickle import socket import threading import time import urllib.error import urllib.request +import weakref from collections.abc import Callable, Generator from datetime import datetime, timedelta, timezone from email.utils import format_datetime @@ -81,14 +83,9 @@ def clock(monkeypatch: pytest.MonkeyPatch) -> list[float]: @pytest.fixture(autouse=True) -def empty_cache(monkeypatch: pytest.MonkeyPatch) -> None: +def empty_cache() -> None: # A fetch that runs on after its test must not answer a later test on a reused port. prompts_module._cache.clear() # pyright: ignore[reportPrivateUsage] - monkeypatch.setattr( - prompts_module, - "_compiled_texts", - prompts_module._CompiledTexts(), # pyright: ignore[reportPrivateUsage] - ) def chat_prompt(*messages: dict[str, Any], name: str = "support/reply") -> ChatPrompt: @@ -831,102 +828,153 @@ def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: assert "td.prompt.variables" not in (span.attributes or {}) -class Part: - def __init__(self, text: str) -> None: - self.text = text +DESK_MESSAGES = [ + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "user", "content": "Order {{order}}: where is it?"}, +] -def test_prompt_attributes_for_finds_a_compiled_text_in_the_request_shapes_of_the_providers( +def test_prompt_attributes_for_names_the_compile_of_a_message_dict_and_a_copy_names_nothing( make: MakeClient, ) -> None: make() - prompt = chat_prompt( - {"role": "system", "content": "Answer as the {{team}} desk."}, - {"role": "user", "content": "Order {{order}}: where is it?"}, - ) - compiled = prompt.compile(team="billing", order="#4") - system, user = (message["content"] for message in compiled) - - requests = { - "same": compiled, - "copy": [dict(message) for message in compiled], - "json": json.loads(json.dumps(compiled)), - "anthropic": [system, compiled[1:]], - "bedrock": [[{"text": system}], [{"role": "user", "content": [{"text": user}]}]], - "objects": (Part(system), [SimpleNamespace(role="user", parts=[Part(user)])]), + compiled = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4") + attributes = { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"team":"billing","order":"#4"}', } - for request in requests.values(): - assert prompt_attributes_for(request) == { - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", - "td.prompt.variables": '{"team":"billing","order":"#4"}', - } - assert prompt_attributes_for([{"role": "user", "content": "Order #5: where is it?"}]) == {} + assert prompt_attributes_for(compiled) == attributes + assert prompt_attributes_for(tuple(compiled)) == attributes + assert prompt_attributes_for(compiled[0]) == attributes + assert json.dumps(compiled[0]) == '{"role": "system", "content": "Answer as the billing desk."}' + + copies: list[object] = [ + [dict(message) for message in compiled], + [message.copy() for message in compiled], + [copy.copy(message) for message in compiled], + copy.deepcopy(compiled), + pickle.loads(pickle.dumps(compiled)), + json.loads(json.dumps(compiled)), + [message["content"] for message in compiled], + ] + for messages in copies: + assert prompt_attributes_for(messages) == {} + + summary = text_prompt("Summarize {{doc}}.").compile(doc="the policy") + assert prompt_attributes_for([summary]) == {} -def test_a_static_text_gives_no_variables_and_a_template_of_only_a_variable_tag_is_not_kept( +def test_an_unrelated_message_with_the_text_of_a_compiled_message_is_not_tagged( make: MakeClient, ) -> None: make() - chat = chat_prompt( - {"role": "system", "content": "You are the refunds agent."}, - {"role": "user", "content": "{{ question }}"}, - ) - compiled = chat.compile(question="Where is my refund?") + chat_prompt({"role": "system", "content": "You are a helpful assistant."}).compile() - assert prompt_attributes_for(compiled) == { - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", + unrelated = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2 + 2?"}, + ] + assert prompt_attributes_for(unrelated) == {} + + +def test_two_tasks_with_one_text_and_different_other_variables_each_record_their_own( + make: MakeClient, +) -> None: + memory = make() + prompt = chat_prompt(*DESK_MESSAGES) + + # Both compiles make the system text "Answer as the billing desk.", and each task sends only it. + system_a = prompt.compile(team="billing", order="#A")[0] + system_b = prompt.compile(team="billing", order="#B")[0] + start_span("a", type="generation", attributes=prompt_attributes_for([system_a])).end() + start_span("b", type="generation", attributes=prompt_attributes_for([system_b])).end() + + assert { + span.name: (span.attributes or {})["td.prompt.variables"] + for span in memory.span_exporter.get_finished_spans() + } == { + "a": '{"team":"billing","order":"#A"}', + "b": '{"team":"billing","order":"#B"}', } - assert prompt_attributes_for([{"role": "user", "content": "Where is my refund?"}]) == {} - summary = text_prompt("Summarize {{doc}} in a line.").compile(doc="the refund policy") - assert prompt_attributes_for(summary)["td.prompt.variables"] == '{"doc":"the refund policy"}' +def test_a_rollout_with_two_versions_of_the_same_messages_records_the_version_of_each_call( + server: CaptureServer, make: MakeClient, monkeypatch: pytest.MonkeyPatch +) -> None: + make() + server.replies = [ + ( + 200, + reply( + messages=DESK_MESSAGES, + rollout={ + "percent": 16, + "version": 13, + "messages": DESK_MESSAGES, + "text": None, + "model": {"provider": "openai", "name": "gpt-4o-mini"}, + "parameters": {}, + "tools": [], + "responseFormat": {"type": "text"}, + }, + ), + ) + ] + draws = iter([15, 16]) + + def draw(_stop: int) -> int: + return next(draws) + + monkeypatch.setattr(prompts_module.random, "randrange", draw) + rollout = get_prompt("support/reply", api_key="td_live_a", api_url=server.url) + production = get_prompt("support/reply", api_key="td_live_a", api_url=server.url) + from_rollout = rollout.compile(team="billing", order="#4") + from_production = production.compile(team="billing", order="#4") + + assert prompt_attributes_for(from_rollout)["td.prompt.version"] == 13 + assert prompt_attributes_for(from_production)["td.prompt.version"] == 12 -def test_a_text_with_variables_wins_over_a_static_text_and_the_first_one_wins_over_a_later_one( + +def test_the_newest_compiled_message_wins_also_after_170_messages_of_history( make: MakeClient, ) -> None: make() - system = chat_prompt( - {"role": "system", "content": "You are the refunds agent, v2."}, name="refunds/system" - ).compile() - question = chat_prompt( - {"role": "user", "content": "Where is order {{order}}?"}, name="orders/question" - ).compile(order="#4") - style = chat_prompt( - {"role": "system", "content": "Reply in {{language}}."}, name="style/language" - ).compile(language="French") - - first = prompt_attributes_for([*system, *question]) - assert (first["td.prompt.name"], first["td.prompt.variables"]) == ( - "orders/question", - '{"order":"#4"}', + prompt = chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, ) - second = prompt_attributes_for([*system, *style, *question]) - assert (second["td.prompt.name"], second["td.prompt.variables"]) == ( - "style/language", - '{"language":"French"}', + history = [ + {"role": "user" if index % 2 == 0 else "assistant", "content": f"message {index}"} + for index in range(170) + ] + + first_turn = prompt.compile(team="billing", question="Where is #4?", history=history) + second_turn = prompt.compile(team="returns", question="Can I return it?") + chat = [*first_turn, {"role": "assistant", "content": "On its way."}, *second_turn[1:]] + + assert '"team":"billing"' in str(prompt_attributes_for(first_turn)["td.prompt.variables"]) + assert '"team":"billing"' in str( + prompt_attributes_for([first_turn[0], *history])["td.prompt.variables"] + ) + assert ( + prompt_attributes_for(chat)["td.prompt.variables"] + == '{"team":"returns","question":"Can I return it?"}' ) -def test_each_compile_keeps_its_own_values_so_two_threads_with_one_prompt_get_their_own( - make: MakeClient, -) -> None: +def test_prompt_attributes_for_looks_at_the_last_1000_messages(make: MakeClient) -> None: make() - prompt = chat_prompt({"role": "user", "content": "Track parcel {{parcel}}."}) - - first = prompt.compile(parcel="A-1") - second = prompt.compile(parcel="B-2") + system = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4")[0] - assert prompt_attributes_for(first)["td.prompt.variables"] == '{"parcel":"A-1"}' - assert prompt_attributes_for(second)["td.prompt.variables"] == '{"parcel":"B-2"}' + assert prompt_attributes_for([system, *[{"role": "user"}] * 999]) != {} + assert prompt_attributes_for([system, *[{"role": "user"}] * 1000]) == {} -def test_with_prompt_wins_over_a_compiled_text_of_another_prompt(make: MakeClient) -> None: +def test_with_prompt_wins_over_a_compiled_message_of_another_prompt(make: MakeClient) -> None: memory = make() scoped = chat_prompt({"role": "user", "content": "Hi"}, name="scoped/prompt") compiled = chat_prompt( @@ -940,7 +988,7 @@ def test_with_prompt_wins_over_a_compiled_text_of_another_prompt(make: MakeClien assert (span.attributes or {})["td.prompt.name"] == "scoped/prompt" -def test_prompt_attributes_for_omits_variables_when_input_capture_is_off_and_masks_them( +def test_prompt_variables_follow_the_input_capture_of_the_client_and_of_the_span( make: MakeClient, ) -> None: compiled = chat_prompt({"role": "user", "content": "My card is {{card}}."}).compile( @@ -959,69 +1007,26 @@ def test_prompt_attributes_for_omits_variables_when_input_capture_is_off_and_mas def mask(value: Any, context: telemetry_dev.MaskContext) -> Any: return {"card": "[masked]"} if context.key == "td.prompt.variables" else value - make(mask=mask) - assert prompt_attributes_for(compiled)["td.prompt.variables"] == '{"card":"[masked]"}' - - -def test_the_kept_texts_stop_at_1000_and_at_2000000_characters_oldest_out_first( - make: MakeClient, -) -> None: - make() - ticket = text_prompt("Ticket {{n}} is open.") - for n in range(1001): - ticket.compile(n=n) - - assert prompt_attributes_for("Ticket 0 is open.") == {} - assert prompt_attributes_for("Ticket 1 is open.")["td.prompt.variables"] == '{"n":1}' - - note = text_prompt("Note: {{body}}") - notes = [note.compile(body=letter * (500_000 - len("Note: "))) for letter in "abcde"] - - assert prompt_attributes_for(notes[0]) == {} - assert prompt_attributes_for(notes[1])["td.prompt.name"] == "support/reply" - - # A text over the character limit is not kept, and it does not push the others out. - note.compile(body="f" * 2_000_001) - assert prompt_attributes_for(notes[1])["td.prompt.name"] == "support/reply" - - # A text compiled again counts once. - for _ in range(5): - note.compile(body="e" * (500_000 - len("Note: "))) - assert prompt_attributes_for(notes[4])["td.prompt.name"] == "support/reply" - - -def test_prompt_attributes_for_looks_6_levels_and_500_values_deep(make: MakeClient) -> None: - make() - [message] = chat_prompt({"role": "user", "content": "Find invoice {{invoice}}."}).compile( - invoice="INV-7" - ) - content = message["content"] - - def nest(value: object, levels: int) -> object: - return value if levels == 0 else nest([value], levels - 1) + memory = make(mask=mask) + attributes = prompt_attributes_for(compiled) + assert attributes["td.prompt.variables"] == '{"card":"[masked]"}' - assert prompt_attributes_for(nest(content, 6)) != {} - assert prompt_attributes_for(nest(content, 7)) == {} + start_span("private", type="generation", capture_input=False, attributes=attributes).end() + with start_span("updated", type="generation", capture_input=False): + telemetry_dev.update_current_span(attributes=attributes) - # The request list, 498 strings, and the text are 500 values. - assert prompt_attributes_for([*["filler"] * 498, content]) != {} - assert prompt_attributes_for([*["filler"] * 499, content]) == {} + for span in memory.span_exporter.get_finished_spans(): + assert (span.attributes or {})["td.prompt.name"] == "support/reply" + assert "td.prompt.variables" not in (span.attributes or {}) -def test_prompt_attributes_for_runs_no_property_or_override_of_the_app(make: MakeClient) -> None: +def test_prompt_attributes_for_runs_no_code_of_the_app(make: MakeClient) -> None: make() [message] = chat_prompt({"role": "user", "content": "Refund order {{order}}."}).compile( order="#3" ) - content = message["content"] reads: list[str] = [] - class Message: - @property - def content(self) -> str: - reads.append("property") - return content - class Messages(list[object]): def __getitem__(self, index: Any) -> Any: reads.append("__getitem__") @@ -1031,47 +1036,36 @@ def __iter__(self) -> Any: reads.append("__iter__") return super().__iter__() - assert prompt_attributes_for([Message(), Messages([content])]) == {} + def __reversed__(self) -> Any: + reads.append("__reversed__") + return super().__reversed__() + + assert prompt_attributes_for(Messages([message])) == {} assert reads == [] -@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") -# Python 3.12+ warns on fork() in a process with threads. -@pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") -def test_a_fork_while_a_thread_keeps_a_compiled_text_leaves_compile_usable_in_the_child() -> None: - prompt = text_prompt("Ship order {{order}}.") - texts = prompts_module._compiled_texts # pyright: ignore[reportPrivateUsage] - holding, release = threading.Event(), threading.Event() - - def hold() -> None: - with texts.lock: - holding.set() - release.wait(0.2) - - holder = threading.Thread(target=hold) - holder.start() - holding.wait() - read_end, write_end = os.pipe() - pid = os.fork() - if pid == 0: - try: - os.close(read_end) - os.write(write_end, prompt.compile(order="#1").encode()) - finally: - os._exit(0) - os.close(write_end) - release.set() - holder.join() +def test_the_variables_of_a_compile_are_freed_with_its_messages(make: MakeClient) -> None: + make() + prompt = chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + ) - deadline = time.monotonic() + 5 - while os.waitpid(pid, os.WNOHANG) == (0, 0): - if time.monotonic() > deadline: - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - pytest.fail("compile() in the child waited for a lock held at the fork") - time.sleep(0.01) - with os.fdopen(read_end, "rb") as child: - assert child.read() == b"Ship order #1." + class History(list[dict[str, str]]): + """A list that a weak reference can point to.""" + + history = History([{"role": "user", "content": "Hi"}]) + freed = weakref.ref(history) + messages = prompt.compile(team="billing", history=history) + del history + # with_prompt keeps the variables of the last compile of a prompt object. + prompt.compile(team="returns") + gc.collect() + assert freed() is not None + + del messages + gc.collect() + assert freed() is None def test_a_fallback_prompt_records_its_name_and_label_but_no_version( From a6b98fe916d5ed4b37bddd7972b07507182f2d2d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 16:20:12 +0000 Subject: [PATCH 13/32] fix(sdk): name the newest compile, not the last tagged message A compile tag holds a serial number that increases with each chat compile. promptAttributesFor and prompt_attributes_for give the tag with the highest number. Thus a message that a placeholder adds from an older compile of a different prompt does not win over the compile that made the list. They look at the first 1000 and the last 1000 messages. A template that ends with a placeholder has its own messages at the start. The TypeScript compile() keeps a copy of the top level of the variables, as Python does. Thus a later change to the object of the app does not change the variables that a message or a withPrompt span records. The Python compiled message keeps the constructor of dict. Thus type(message)(message) and dataclasses.asdict make a copy with all of the keys and without the tag. A model-call span with a parent context in withPrompt or with_prompt removes the td.prompt.* attributes that the app gives. Thus the span does not mix two prompts when the app calls promptAttributesFor outside the scope and gives an explicit parent. The READMEs and docs/sdk-conformance.md tell that the Python messages are a dict subclass, which yaml.safe_dump, marshal, and checks of the exact type reject. They also tell that a tagged message keeps the variables of its compile in memory. We corrected the docstring of prompt_attributes_for: with input capture on, it calls the mask of the app. Refs #97 --- docs/sdk-conformance.md | 47 ++++--- packages/sdk/README.md | 36 ++++-- packages/sdk/src/prompts.ts | 51 +++++--- packages/sdk/src/span.ts | 11 +- packages/sdk/tests/prompts.test.ts | 148 ++++++++++++++++++++-- sdks/python-openai/tests/test_openai.py | 4 - sdks/python/README.md | 46 ++++--- sdks/python/src/telemetry_dev/_prompts.py | 72 +++++++---- sdks/python/src/telemetry_dev/_spans.py | 12 +- sdks/python/tests/test_prompts.py | 123 ++++++++++++++++-- 10 files changed, 433 insertions(+), 117 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 610c0c1..e3e6fec 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -44,7 +44,7 @@ repository, not this SDK workspace. Paths below refer to that product repository | outside a scope: a message object that `compile()` of a chat prompt returned, in the request messages of a provider integration | `td.prompt.name`, `td.prompt.version`, `td.prompt.label` | on the `chat` spans of the provider integrations listed in [Prompts](#prompts); the scope wins | | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | -| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the `compile()` that returned the message | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope; outside a scope, of the newest `compile()` that made a message of the request | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -182,23 +182,38 @@ task. Outside a scope, the SDKs find the prompt from the message objects of the request, not from their text: -- `compile()` of a chat prompt tags each message object that it makes with the prompt and the - variables of that call. It does not tag the messages of a placeholder variable: the app passes - those. TypeScript: a non-enumerable symbol property. Python: the messages are instances of a - `dict` subclass that holds the tag in a slot. The keys and the JSON of a message do not change. +- `compile()` of a chat prompt tags each message object that it makes with the prompt, a copy of + the top level of the variables of that call, and a serial number that grows with each chat + compile in the process. It does not tag the messages of a placeholder variable: the app passes + those, and they keep the tag of their own compile. TypeScript: a non-enumerable symbol + property; the keys and the JSON of a message do not change. Python: the messages are instances + of a private `dict` subclass that holds the tag in a slot. `isinstance(message, dict)` is true + and the keys and the JSON do not change, but a check of the exact type + (`type(message) is dict`), `yaml.safe_dump`, and `marshal` reject it. Its constructor is the + one of `dict`. - A copy has no tag: a spread, `structuredClone`, or a JSON round trip (TypeScript); `dict()`, - `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, or a JSON round trip (Python). A new message - with the same text has no tag either, thus a message from another call, task, or prompt version - is never recorded. + `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, a JSON round trip, `type(message)(message)`, or + `dataclasses.asdict` (Python). A new message with the same text has no tag either, thus a + message from another call, task, or prompt version is never recorded. - `promptAttributesFor(messages)` / `prompt_attributes_for(messages)` take a list of messages or - one message (Python: a `list` or `tuple`, not a subclass, so no code of the app runs). They - look at the last 1000 items, from the end, and give the name, version, label, and variables of - the newest tagged message. They do not look inside a message. -- In a `withPrompt` / `with_prompt` scope they give nothing, thus the span gets the prompt of the - scope. They never throw. -- They give `td.prompt.variables` only when input capture is on in the client. A span whose - effective captureInput is off drops `td.prompt.variables` from its attributes, also when the app - passes the result of these functions to its own span. + one message (Python: a `list` or `tuple`, not a subclass, so the read runs no code of the app). + They look at the first 1000 and the last 1000 items and give the name, version, label, and + variables of the tag with the highest serial number: the newest compile. Thus a message that a + placeholder brings from an older compile does not win over the compile that put it in the list. + They do not look inside a message. +- In a `withPrompt` / `with_prompt` scope they give nothing. A model-call span whose parent context + is in a scope gets the attributes of the scope and drops each `td.prompt.*` attribute that the + app passes, also when the app calls these functions outside the scope and passes an explicit + parent. They never throw: an error goes to `onError` / `on_error`. +- They give `td.prompt.variables` only when input capture is on in the client. Then they + serialize the variables, which calls the `mask` of the client. A span whose effective + captureInput is off drops `td.prompt.variables` from its attributes, also when the app passes + the result of these functions to its own span. +- A tagged message keeps the variables of its compile alive while the message lives, with the + lists of its placeholder variables. Thus a chat history that keeps a compiled message of each + turn keeps the history list of each turn. The SDKs keep no other reference to the variables, + except those of the last compile of each prompt object and of each running scope, for + `withPrompt` / `with_prompt`. - `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. - The provider integrations call them with the request messages before the span starts, also diff --git a/packages/sdk/README.md b/packages/sdk/README.md index b73d74c..1fe9131 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -182,19 +182,29 @@ and TanStack AI integrations, and from other OpenTelemetry tracers, do not get t Outside `withPrompt`, those provider integrations record the prompt of the message objects that `compile()` returned. `compile()` of a chat prompt tags each message that it makes with its prompt -and variables, in a non-enumerable symbol property: the keys and the JSON of the message do not -change. A chat call that sends a tagged message gets `td.prompt.name`, `td.prompt.version`, and -`td.prompt.label` of the newest one (of the last 1000 messages), also when input capture is off, -and `td.prompt.variables` when it is on. A copy of a message (a spread, `structuredClone`, or -JSON) and a new message with the same text have no tag, so the SDK does not record them. Bedrock -Converse and Google GenAI take another message shape: change the compiled messages in place, or -use `withPrompt`. A text prompt compiles to a string, which cannot carry a tag: use `withPrompt`. -`withPrompt` wins over the tag. The integrations do it for OpenAI chat completions and responses, -Anthropic messages, Bedrock Converse, Google GenAI `generateContent`, and OpenRouter chat and -responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, video, -or OpenAI Realtime. For your own spans, `promptAttributesFor(messages)` gives the attributes to -pass as `attributes`; it gives nothing inside `withPrompt`, and a span with `captureInput: false` -drops `td.prompt.variables`. +and a copy of its variables, in a non-enumerable symbol property: the keys and the JSON of the +message do not change. A chat call that sends tagged messages gets `td.prompt.name`, +`td.prompt.version`, and `td.prompt.label` of the newest `compile()` among them, also when input +capture is off, and `td.prompt.variables` when it is on. The SDK looks at the first 1000 and the +last 1000 messages of the call. A message from a placeholder variable keeps the tag of its own, +older `compile()`, so it does not win over the `compile()` that put it in the list. A copy of a +message (a spread, `structuredClone`, or JSON) and a new message with the same text have no tag, +so the SDK does not record them. Bedrock Converse and Google GenAI take another message shape: +change the compiled messages in place, or use `withPrompt`. A text prompt compiles to a string, +which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The integrations do it +for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse, Google GenAI +`generateContent`, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is +bytes), embeddings, images, audio, video, or OpenAI Realtime. + +A tagged message keeps the variables of its `compile()` alive while the message lives, with the +lists of its placeholder variables. If you keep compiled messages in a long chat history, put +copies (`{ ...message }`) in the history: else each turn keeps the history list of its own turn. + +For your own spans, `promptAttributesFor(messages)` gives the attributes to pass as `attributes`. +It gives nothing inside `withPrompt`, and a span with `captureInput: false` drops +`td.prompt.variables`. A model call whose parent is in `withPrompt` gets the prompt of the +`withPrompt` and none of the `td.prompt.*` attributes that you pass, also when its `parent` option +leads into a `withPrompt` that the call is not in. ## API diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 48cfe27..8202159 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -97,8 +97,9 @@ export interface ChatPrompt extends PromptBase { /** * Fills in the variables and replaces each placeholder message with the messages in its * variable. `withPrompt` records the variables of the last call before it starts or inside it. - * It tags each message object that it makes with this prompt and these variables, so that a - * model call outside `withPrompt` that sends the message records them; see `promptAttributesFor`. + * It tags each message object that it makes with this prompt and a copy of these variables, so + * that a model call outside `withPrompt` that sends the message records them; see + * `promptAttributesFor`. The message keeps the variables alive while it lives. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -205,6 +206,8 @@ interface PromptScope { interface CompileTag { readonly prompt: Prompt; readonly variables: PromptVariables; + /** Counts the compiles of the process, so a higher one is newer. */ + readonly serial: number; } const FETCH_TIMEOUT_MS = 10_000; @@ -220,13 +223,14 @@ const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "gene const PROMPT_KEY = createContextKey("telemetry.dev prompt"); // Not enumerable, so JSON.stringify, spreads, and structuredClone leave the tag out of a copy. const COMPILED_BY = Symbol("telemetry.dev compiled by"); -/** How many messages promptAttributesFor looks at, from the end. */ -const MAX_TAG_LOOKUP = 1_000; +/** How many messages promptAttributesFor looks at, at each end of the list. */ +const TAG_LOOKUP = 1_000; const cache = new Map(); const compiledVariables = new WeakMap(); /** The Retry-After of the response that failed with this error, in milliseconds. */ const retryAfter = new WeakMap(); +let compiles = 0; /** * Fetches a prompt version from telemetry.dev and keeps it. When the copy is older than @@ -368,11 +372,11 @@ export function promptAttributes( /** * The td.prompt.* attributes for a model call outside `withPrompt` that sends `messages` (a list - * of messages, or one message): those of the newest message, of the last 1000, that `compile()` - * of a chat prompt returned. The tag is on the message object, so a copy of it gives nothing. Gives - * undefined inside `withPrompt`, whose prompt the span gets. `td.prompt.variables` needs input - * capture on in the client, and the span drops it when its own input capture is off. It never - * throws. + * of messages, or one message): those of the newest `compile()` of a chat prompt that made one of + * the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the message + * object, so a copy of it gives nothing. Gives undefined inside `withPrompt`, whose prompt the span + * gets. `td.prompt.variables` needs input capture on in the client, and the span drops it when its + * own input capture is off. It never throws. */ export function promptAttributesFor(messages: unknown): Record | undefined { const config = currentClient().core?.config; @@ -408,17 +412,22 @@ function attributesOf( }); } +// The messages of a placeholder come from an older compile, so the newest compile, not the last +// tagged message, names the prompt of the request. Its own messages are at the start or the end of +// the list, unless placeholders put more than 1000 messages on each side of them. function newestTag(messages: unknown): CompileTag | undefined { if (!Array.isArray(messages)) return tagOf(messages); - const stop = Math.max(0, messages.length - MAX_TAG_LOOKUP); + const { length } = messages; + let newest: CompileTag | undefined; - for (let index = messages.length - 1; index >= stop; index -= 1) { + for (let index = 0; index < length; index += 1) { + if (index === TAG_LOOKUP && length > 2 * TAG_LOOKUP) index = length - TAG_LOOKUP; const tag = tagOf(messages[index]); - if (tag) return tag; + if (tag && (newest === undefined || tag.serial > newest.serial)) newest = tag; } - return undefined; + return newest; } function tagOf(message: unknown): CompileTag | undefined { @@ -613,14 +622,20 @@ function createPrompt(input: { responseFormat: content.responseFormat, isFallback: input.isFallback, compile(variables: PromptVariables = {}) { - compiledVariables.set(prompt, variables); + // A copy, so that a later change of the app's object does not change what a span records. + const values = { ...variables }; + compiledVariables.set(prompt, values); const scope = activeContext().getValue(PROMPT_KEY) as PromptScope | undefined; - if (scope?.prompt === prompt) scope.variables = variables; + if (scope?.prompt === prompt) scope.variables = values; - return type === "chat" - ? compileMessages(content.messages, Object.freeze({ prompt, variables })) - : fill(content.text ?? "", variables); + if (type === "text") return fill(content.text ?? "", values); + compiles += 1; + + return compileMessages( + content.messages, + Object.freeze({ prompt, variables: values, serial: compiles }), + ); }, }) as Prompt; diff --git a/packages/sdk/src/span.ts b/packages/sdk/src/span.ts index 4c4c38a..cf0c374 100644 --- a/packages/sdk/src/span.ts +++ b/packages/sdk/src/span.ts @@ -231,6 +231,15 @@ export function createSpanHandle( const parentCtx = withSessionParent(sessionContext, sessionId, core.config.apiKey); const attrs = fieldsToAttributes(options, meta.type, meta.capture); const operation = attrs["gen_ai.operation.name"] ?? SPAN_TYPE_OPERATIONS[meta.type]; + const prompt = promptAttributes(parentCtx, operation, meta.capture); + + // The prompt of withPrompt replaces all td.prompt.* attributes of the app, so that a span never + // mixes two prompts, also when its parent is in withPrompt and promptAttributesFor ran outside. + if (prompt) { + for (const key of Object.keys(attrs)) { + if (key.startsWith("td.prompt.")) delete attrs[key]; + } + } const span = core.tracer.startSpan( name, @@ -241,7 +250,7 @@ export function createSpanHandle( attributes: { "gen_ai.operation.name": SPAN_TYPE_OPERATIONS[meta.type], ...propagatedFromContext(parentCtx), - ...promptAttributes(parentCtx, operation, meta.capture), + ...prompt, ...attrs, }, }, diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index 8e68644..a10ebb6 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -1,5 +1,7 @@ import { createServer, type RequestListener } from "node:http"; import type { AddressInfo } from "node:net"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; @@ -880,31 +882,157 @@ test("the newest compiled message wins, also after 170 messages of history", asy ); }); -test("promptAttributesFor looks at the last 1000 messages", async () => { +test("a message that a placeholder brings from an older compile does not win", async () => { + setup(); + const router = await promptOf({ + name: "router", + version: 1, + messages: [{ role: "user", content: "{{question}}" }], + }); + const answer = await promptOf({ + name: "answer", + version: 7, + messages: [ + { role: "system", content: "Answer for {{team}}." }, + { role: "placeholder", name: "history" }, + ], + }); + const chat = await promptOf({ + name: "chat", + messages: [{ role: "user", content: "{{question}}" }], + }); + const summary = await promptOf({ + name: "summary", + messages: [ + { role: "system", content: "Summarize the chat." }, + { role: "placeholder", name: "history" }, + ], + }); + + const routed = answer.compile({ + team: "billing", + history: router.compile({ question: "Where is #4?" }) as ChatMessage[], + }); + // The last 1000 messages are all from chat; the summary message is the first. + const conversation = Array.from( + { length: 1500 }, + (_, index) => chat.compile({ question: `question ${index}` }) as ChatMessage[], + ).flat(); + const summarized = summary.compile({ history: conversation }); + + expect(promptAttributesFor(routed)).toEqual({ + "td.prompt.name": "answer", + "td.prompt.version": 7, + "td.prompt.label": "production", + "td.prompt.variables": + '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}', + }); + expect(promptAttributesFor(summarized)?.["td.prompt.name"]).toBe("summary"); +}); + +test("promptAttributesFor looks at the first and the last 1000 messages", async () => { setup(); const prompt = await promptOf(DESK_PROMPT); const [system] = prompt.compile({ team: "billing", order: "#4" }) as ChatMessage[]; const filler = (count: number) => Array.from({ length: count }, () => ({ role: "user" })); - expect(promptAttributesFor([system, ...filler(999)])).toBeDefined(); - expect(promptAttributesFor([system, ...filler(1000)])).toBeUndefined(); + expect(promptAttributesFor([...filler(1000), system, ...filler(999)])).toBeDefined(); + expect(promptAttributesFor([...filler(999), system, ...filler(1001)])).toBeDefined(); + expect(promptAttributesFor([...filler(1001), system, ...filler(999)])).toBeDefined(); + expect(promptAttributesFor([...filler(1000), system, ...filler(1000)])).toBeUndefined(); +}); + +test("compile records its variables, not later changes of the object", async () => { + const { spans } = setup(); + const prompt = await promptOf({ messages: [{ role: "user", content: "Order {{order}}" }] }); + const variables = { order: "#A" }; + + const first = prompt.compile(variables); + variables.order = "#B"; + const second = prompt.compile(variables); + variables.order = "#C"; + withPrompt(prompt, () => startSpan("scope", { type: "generation" }).end()); + await flush(); + + expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe('{"order":"#A"}'); + expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe('{"order":"#B"}'); + expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBe('{"order":"#B"}'); }); -test("withPrompt wins over a compiled message of another prompt", async () => { +test("the variables of a compile are freed with its messages", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + // A WeakRef keeps its target until the current job ends, so collect in a new one. + const collect = async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + }; + setup(); + const prompt = await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + ], + }); + + let history: ChatMessage[] | undefined = [{ role: "user", content: "Hi" }]; + const freed = new WeakRef(history); + let messages = prompt.compile({ team: "billing", history }) as ChatMessage[] | undefined; + history = undefined; + // withPrompt keeps the variables of the last compile of a prompt object. + prompt.compile({ team: "returns" }); + await collect(); + expect(freed.deref()).toBeDefined(); + + expect(messages).toHaveLength(2); + messages = undefined; + await collect(); + expect(freed.deref()).toBeUndefined(); +}); + +test("withPrompt wins over a compiled message of another prompt, also through a parent", async () => { const { spans } = setup(); const scoped = await promptOf({ name: "scoped/prompt" }); - const other = await promptOf({ - name: "other/prompt", - messages: [{ role: "user", content: "Cancel order {{order}}." }], + // Fetched by version, so it has no label that the label of the scope could hide. + const { fetch } = fakeFetch( + reply({ + name: "other/prompt", + label: null, + version: 3, + messages: [{ role: "user", content: "Cancel order {{order}}." }], + }), + ); + const other = await getPrompt("other/prompt", { + apiKey: "td_live_a", + apiUrl: newApiUrl(), + version: 3, + fetch, }); const compiled = other.compile({ order: "#9" }); - withPrompt(scoped, () => { - startSpan("chat", { type: "generation", attributes: promptAttributesFor(compiled) }).end(); + const parent = withPrompt(scoped, () => { + startSpan("in scope", { type: "generation", attributes: promptAttributesFor(compiled) }).end(); + + return startSpan("parent"); }); + const attributes = promptAttributesFor(compiled); + startSpan("with parent", { type: "generation", parent, attributes }).end(); + parent.end(); await flush(); - expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.name"]).toBe("scoped/prompt"); + expect(attributes?.["td.prompt.name"]).toBe("other/prompt"); + for (const name of ["in scope", "with parent"]) { + const span = spans.getFinishedSpans().find((finished) => finished.name === name)!; + expect( + Object.fromEntries( + Object.entries(span.attributes).filter(([key]) => key.startsWith("td.prompt.")), + ), + ).toEqual({ + "td.prompt.name": "scoped/prompt", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + } }); test("td.prompt.variables follows the input capture of the client and of the span", async () => { diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 1ad3a36..55ab3c6 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1222,10 +1222,6 @@ def text(self) -> str: ) -def prompt_attrs(span: ReadableSpan) -> dict[str, object]: - return {key: value for key, value in attrs(span).items() if key.startswith("td.prompt.")} - - @pytest.mark.parametrize( ("api", "messages", "capture_input", "older_core"), [ diff --git a/sdks/python/README.md b/sdks/python/README.md index e39a537..0345367 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -254,22 +254,36 @@ integrations: OpenAI, Anthropic, Amazon Bedrock, Google GenAI, LiteLLM, and Open other OpenTelemetry tracers do not get the attributes. Outside `with_prompt`, those provider integrations record the prompt of the message dicts that -`compile()` returned. `compile()` of a chat prompt returns instances of a `dict` subclass that -also name its prompt and variables: the keys and the JSON of the message do not change. A chat -call that sends such a message gets `td.prompt.name`, `td.prompt.version`, and `td.prompt.label` -of the newest one (of the last 1000 messages), also when input capture is off, and -`td.prompt.variables` when it is on. A copy of a message (`dict()`, `.copy()`, `copy.copy`, -`copy.deepcopy`, pickle, or JSON) and a new message with the same text do not name the prompt, so -the SDK does not record them. Bedrock Converse and Google GenAI take another message shape: change -the compiled dicts in place, or use `with_prompt`. A text prompt compiles to a string, which -cannot name its prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations -do it for OpenAI chat completions and responses, Anthropic messages (for `messages.stream()`, at -`__enter__`), Bedrock Converse, Google GenAI `generate_content`, LiteLLM completion and responses, -and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, -images, audio, video, or OpenAI Realtime. With an older `telemetry-dev` that has no -`prompt_attributes_for`, a provider integration records the prompt only in `with_prompt`. For your -own spans, `prompt_attributes_for(messages)` gives the attributes to pass as `attributes`; it gives -`{}` inside `with_prompt`, and a span with `capture_input=False` drops `td.prompt.variables`. +`compile()` returned. `compile()` of a chat prompt returns instances of a private `dict` subclass +that also hold its prompt and a copy of its variables. A chat call that sends such messages gets +`td.prompt.name`, `td.prompt.version`, and `td.prompt.label` of the newest `compile()` among +them, also when input capture is off, and `td.prompt.variables` when it is on. The SDK looks at +the first 1000 and the last 1000 messages of the call. A message from a placeholder variable keeps +the prompt of its own, older `compile()`, so it does not win over the `compile()` that put it in +the list. A copy of a message (`dict()`, `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, JSON, +`type(message)(message)`, or `dataclasses.asdict`) and a new message with the same text do not +hold the prompt, so the SDK does not record them. Bedrock Converse and Google GenAI take another +message shape: change the compiled dicts in place, or use `with_prompt`. A text prompt compiles to +a string, which cannot hold its prompt: use `with_prompt`. `with_prompt` wins over the message. +The integrations do it for OpenAI chat completions and responses, Anthropic messages (for +`messages.stream()`, at `__enter__`), Bedrock Converse, Google GenAI `generate_content`, LiteLLM +completion and responses, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its +body is bytes), embeddings, images, audio, video, or OpenAI Realtime. With an older +`telemetry-dev` that has no `prompt_attributes_for`, a provider integration records the prompt +only in `with_prompt`. + +A compiled message is a `dict` for `isinstance`, and its keys and its JSON (`json`, `orjson`, +pydantic) do not change. A check of the exact type (`type(message) is dict`), `yaml.safe_dump`, +and `marshal` reject it: give them `dict(message)`. A compiled message keeps the variables of its +`compile()` alive while the message lives, with the lists of its placeholder variables. If you +keep compiled messages in a long chat history, put copies (`dict(message)`) in the history: else +each turn keeps the history list of its own turn. + +For your own spans, `prompt_attributes_for(messages)` gives the attributes to pass as +`attributes`. It gives `{}` inside `with_prompt`, and a span with `capture_input=False` drops +`td.prompt.variables`. A model call whose parent is in `with_prompt` gets the prompt of the +`with_prompt` and none of the `td.prompt.*` attributes that you pass, also when its `parent` +context comes from a `with_prompt` that the call is not in. ## Auto-metrics diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index cb230fd..28ee83e 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -4,6 +4,7 @@ import contextlib import copy import hashlib +import itertools import json import random import re @@ -50,8 +51,8 @@ _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") -# How many messages prompt_attributes_for looks at, from the end. -_MAX_TAG_LOOKUP = 1000 +# How many messages prompt_attributes_for looks at, at each end of the list. +_TAG_LOOKUP = 1000 class PromptModel(TypedDict): @@ -100,16 +101,21 @@ def compile( ) -> list[dict[str, Any]]: """Fill in the variables and replace each placeholder message with the list of messages in its variable. ``with_prompt`` records the variables of the last call before it starts - or inside it. It tags each message dict that it makes with this prompt and these - variables, so that a model call outside ``with_prompt`` that sends the message records - them; see ``prompt_attributes_for``.""" + or inside it. Each message that it makes is a ``dict`` subclass that also holds this + prompt and a copy of these variables, so that a model call outside ``with_prompt`` that + sends the message records them; see ``prompt_attributes_for``. The message keeps the + variables alive while it lives. A check of the exact type (``type(message) is dict``), + ``yaml.safe_dump``, and ``marshal`` reject it: give them ``dict(message)``.""" values = _record(self, variables, kwargs) - tag = _CompileTag(self, values) + tag = _CompileTag(self, values, next(_compiles)) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": - content = _fill(str(message["content"]), values) - compiled.append(_CompiledMessage(tag, role=message["role"], content=content)) + tagged = _CompiledMessage( + role=message["role"], content=_fill(str(message["content"]), values) + ) + tagged.compiled_by = tag + compiled.append(tagged) continue name = str(message["name"]) value = values.get(name) @@ -294,11 +300,16 @@ def prompt_attributes( def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: """The td.prompt.* attributes for a model call outside ``with_prompt`` that sends ``messages`` - (a list or tuple of messages, or one message): those of the newest message, of the last 1000, - that ``compile()`` of a chat prompt returned. The tag is on the dict that ``compile()`` made, so - a copy of it gives ``{}``. Gives ``{}`` inside ``with_prompt``, whose prompt the span gets. - ``td.prompt.variables`` needs input capture on in the client, and the span drops it when its - own input capture is off. It runs no code of the app and never raises.""" + (a list or tuple of messages, or one message): those of the newest ``compile()`` of a chat + prompt that made one of the messages. It looks at the first 1000 and the last 1000 messages. + The tag is on the dict that ``compile()`` made, so a copy of it gives ``{}``. Gives ``{}`` + inside ``with_prompt``, whose prompt the span gets. ``td.prompt.variables`` needs input + capture on in the client, and the span drops it when its own input capture is off. + + It reads only a ``list`` or ``tuple``, not a subclass, so the read runs no code of the app. + With input capture on, it serializes the variables, which calls the ``mask`` of the client and + can run code of a value, such as ``__repr__``. It never raises: it reports an error to + ``on_error`` and gives ``{}``.""" client = get_client() try: if isinstance(otel_context.get_value(_PROMPT_KEY), _PromptScope): @@ -443,32 +454,43 @@ class _CompileTag: prompt: ChatPrompt variables: dict[str, Any] + serial: int + """Counts the compiles of the process, so a higher one is newer.""" + + +_compiles = itertools.count() class _CompiledMessage(dict[str, Any]): - """A message that ``compile()`` made: a dict with the tag in a slot. A copy, pickle, or - ``copy.deepcopy`` of it is a plain dict without the tag.""" + """A message that ``compile()`` made: a dict with the tag in a slot. The constructor is the one + of ``dict``, so ``type(message)(message)`` and ``dataclasses.asdict`` rebuild it, without the + tag. A copy, pickle, or ``copy.deepcopy`` of it is a plain dict without the tag.""" __slots__ = ("compiled_by",) - - def __init__(self, compiled_by: _CompileTag, **fields: Any) -> None: - super().__init__(**fields) - self.compiled_by = compiled_by + compiled_by: _CompileTag def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: return dict, (dict(self),) def _newest_tag(messages: object) -> _CompileTag | None: - # Exact types: a subclass of list or tuple could run code of the app on each read. + """The messages of a placeholder come from an older compile, so the newest compile, not the + last tagged message, names the prompt of the request. Its own messages are at the start or the + end of the list, unless placeholders put more than 1000 messages on each side of them.""" if type(messages) is _CompiledMessage: - return messages.compiled_by + return getattr(messages, "compiled_by", None) + # Exact types: a subclass of list or tuple could run code of the app on each read. if type(messages) is not list and type(messages) is not tuple: return None - for message in reversed(cast("list[object]", messages)[-_MAX_TAG_LOOKUP:]): - if type(message) is _CompiledMessage: - return message.compiled_by - return None + items = cast("list[object]", messages) + if len(items) > 2 * _TAG_LOOKUP: + items = [*items[:_TAG_LOOKUP], *items[-_TAG_LOOKUP:]] + newest: _CompileTag | None = None + for item in items: + tag = getattr(item, "compiled_by", None) if type(item) is _CompiledMessage else None + if tag is not None and (newest is None or tag.serial > newest.serial): + newest = tag + return newest def _fill(text: str, values: Mapping[str, Any]) -> str: diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index d66f086..1d6ac64 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -748,11 +748,15 @@ def start_span( ctx, session_id if isinstance(session_id, str) else None, client.config.api_key ) initial.update(propagated) - initial.update( - prompt_attributes( - client, ctx, attrs.get(ATTR_OPERATION, operation), state.capture_input - ) + prompt = prompt_attributes( + client, ctx, attrs.get(ATTR_OPERATION, operation), state.capture_input ) + if prompt: + # The prompt of with_prompt replaces all td.prompt.* attributes of the app, so that a + # span never mixes two prompts, also when its parent is in with_prompt and + # prompt_attributes_for ran outside. + attrs = {key: value for key, value in attrs.items() if not key.startswith("td.prompt.")} + initial.update(prompt) initial.update(attrs) span = client.tracer.start_span( name, diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 29297e3..423286f 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -3,6 +3,7 @@ import asyncio import contextvars import copy +import dataclasses import gc import itertools import json @@ -22,6 +23,7 @@ from urllib.parse import parse_qsl, urlsplit import pytest +from opentelemetry import context as otel_context from opentelemetry import trace import telemetry_dev @@ -966,26 +968,127 @@ def test_the_newest_compiled_message_wins_also_after_170_messages_of_history( ) -def test_prompt_attributes_for_looks_at_the_last_1000_messages(make: MakeClient) -> None: +def test_a_message_that_a_placeholder_brings_from_an_older_compile_does_not_win( + make: MakeClient, +) -> None: + make() + router = chat_prompt({"role": "user", "content": "{{question}}"}, name="router") + answer = chat_prompt( + {"role": "system", "content": "Answer for {{team}}."}, + {"role": "placeholder", "name": "history"}, + name="answer", + ) + chat = chat_prompt({"role": "user", "content": "{{question}}"}, name="chat") + summary = chat_prompt( + {"role": "system", "content": "Summarize the chat."}, + {"role": "placeholder", "name": "history"}, + name="summary", + ) + + routed = answer.compile(team="billing", history=router.compile(question="Where is #4?")) + # The last 1000 messages are all from chat; the summary message is the first. + conversation = [chat.compile(question=f"question {index}")[0] for index in range(1500)] + summarized = summary.compile(history=conversation) + + assert prompt_attributes_for(routed) == { + "td.prompt.name": "answer", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": ( + '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}' + ), + } + assert prompt_attributes_for(summarized)["td.prompt.name"] == "summary" + + +def test_prompt_attributes_for_looks_at_the_first_and_the_last_1000_messages( + make: MakeClient, +) -> None: make() system = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4")[0] - assert prompt_attributes_for([system, *[{"role": "user"}] * 999]) != {} - assert prompt_attributes_for([system, *[{"role": "user"}] * 1000]) == {} + def around(before: int, after: int) -> list[dict[str, Any]]: + return [*[{"role": "user"}] * before, system, *[{"role": "user"}] * after] + + assert prompt_attributes_for(around(1000, 999)) != {} + assert prompt_attributes_for(around(999, 1001)) != {} + assert prompt_attributes_for(tuple(around(1001, 999))) != {} + assert prompt_attributes_for(around(1000, 1000)) == {} + + +def test_compile_records_its_variables_not_later_changes_of_the_mapping( + make: MakeClient, +) -> None: + memory = make() + prompt = chat_prompt({"role": "user", "content": "Order {{order}}"}) + variables = {"order": "#A"} + + first = prompt.compile(variables) + variables["order"] = "#B" + second = prompt.compile(variables) + variables["order"] = "#C" + with with_prompt(prompt): + start_span("scope", type="generation").end() + + assert prompt_attributes_for(first)["td.prompt.variables"] == '{"order":"#A"}' + assert prompt_attributes_for(second)["td.prompt.variables"] == '{"order":"#B"}' + [span] = memory.span_exporter.get_finished_spans() + assert (span.attributes or {})["td.prompt.variables"] == '{"order":"#B"}' + + +def test_a_compiled_message_rebuilds_like_a_dict_and_the_rebuilt_one_has_no_tag( + make: MakeClient, +) -> None: + make() + compiled = chat_prompt(*DESK_MESSAGES).compile(team="billing", order="#4") + @dataclasses.dataclass + class Conversation: + messages: list[dict[str, Any]] -def test_with_prompt_wins_over_a_compiled_message_of_another_prompt(make: MakeClient) -> None: + rebuilt = dataclasses.asdict(Conversation(messages=compiled))["messages"] + same_type = [type(message)(message) for message in compiled] + plain = [ + {"role": "system", "content": "Answer as the billing desk."}, + {"role": "user", "content": "Order #4: where is it?"}, + ] + + assert rebuilt == plain + assert same_type == plain + assert prompt_attributes_for(rebuilt) == {} + assert prompt_attributes_for(same_type) == {} + + +def test_with_prompt_wins_over_a_compiled_message_of_another_prompt_also_through_a_parent( + make: MakeClient, +) -> None: memory = make() scoped = chat_prompt({"role": "user", "content": "Hi"}, name="scoped/prompt") - compiled = chat_prompt( - {"role": "user", "content": "Cancel order {{order}}."}, name="other/prompt" - ).compile(order="#9") + # Fetched by version, so it has no label that the label of the scope could hide. + other = dataclasses.replace( + chat_prompt({"role": "user", "content": "Cancel order {{order}}."}, name="other/prompt"), + version=3, + label=None, + ) + compiled = other.compile(order="#9") with with_prompt(scoped): - start_span("chat", type="generation", attributes=prompt_attributes_for(compiled)).end() + start_span("in scope", type="generation", attributes=prompt_attributes_for(compiled)).end() + parent = otel_context.get_current() + attributes = prompt_attributes_for(compiled) + start_span("with parent", type="generation", parent=parent, attributes=attributes).end() - [span] = memory.span_exporter.get_finished_spans() - assert (span.attributes or {})["td.prompt.name"] == "scoped/prompt" + assert attributes["td.prompt.name"] == "other/prompt" + for span in memory.span_exporter.get_finished_spans(): + assert { + key: value + for key, value in dict(span.attributes or {}).items() + if key.startswith("td.prompt.") + } == { + "td.prompt.name": "scoped/prompt", + "td.prompt.version": 12, + "td.prompt.label": "production", + } def test_prompt_variables_follow_the_input_capture_of_the_client_and_of_the_span( From 3bc6f53a0afb0ad8b9fe19f8eaf7e5d812a93b0a Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 20:15:11 +0000 Subject: [PATCH 14/32] fix(sdk): keep the variables of recent compiles only A compiled message kept the variables of its compile, with the lists in them. Thus a chat that keeps a compiled message of each turn in a new history list for each turn kept the history list of every turn. Memory increased with the square of the turn count: 49 MB in TypeScript and 38 MB in Python after 2000 turns, also with input capture off. compile() can get a message of an older compile in a variable, or in a list in a variable. If the older compile is 100 or more compiles older, it discards its variables. The newest compile identifies the prompt of a request, thus a call still records the variables of its own compile. A recent compile keeps its variables when a newer compile gets its messages. Thus a judge compile that gets the messages before their call does not remove them. After 2000 turns, the SDKs keep 3.7 MB (TypeScript) and 4.4 MB (Python) with input capture on, and 0.7 MB and 1.3 MB with it off. These numbers include the history of the app. If input capture is off in the client, compile() keeps no variables in the tag, because promptAttributesFor and prompt_attributes_for cannot record them. withPrompt and with_prompt keep the variables of the last compile of each prompt object, as before. compile() also copies each list in the variables. Thus a change to the history of the app after compile() does not change the variables that a call records. compile() does not copy the objects in a list, and the READMEs and docs/sdk-conformance.md tell this. A span that gets its prompt from withPrompt or with_prompt discards the td.prompt.* attributes of the app at start, and also in update, end, updateActiveSpan, and update_current_span. Refs #97 --- docs/sdk-conformance.md | 43 +++--- packages/sdk/README.md | 45 +++--- packages/sdk/src/prompts.ts | 68 +++++++-- packages/sdk/src/span.ts | 23 +++- packages/sdk/tests/prompts.test.ts | 160 ++++++++++++++++++---- sdks/python/README.md | 52 ++++--- sdks/python/src/telemetry_dev/_prompts.py | 57 ++++++-- sdks/python/src/telemetry_dev/_spans.py | 18 ++- sdks/python/tests/test_prompts.py | 140 ++++++++++++++++--- 9 files changed, 459 insertions(+), 147 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index e3e6fec..b49779a 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -183,14 +183,15 @@ Outside a scope, the SDKs find the prompt from the message objects of the reques their text: - `compile()` of a chat prompt tags each message object that it makes with the prompt, a copy of - the top level of the variables of that call, and a serial number that grows with each chat - compile in the process. It does not tag the messages of a placeholder variable: the app passes - those, and they keep the tag of their own compile. TypeScript: a non-enumerable symbol - property; the keys and the JSON of a message do not change. Python: the messages are instances - of a private `dict` subclass that holds the tag in a slot. `isinstance(message, dict)` is true - and the keys and the JSON do not change, but a check of the exact type - (`type(message) is dict`), `yaml.safe_dump`, and `marshal` reject it. Its constructor is the - one of `dict`. + the variables of that call and of each list in them (not of the objects in a list), and a serial + number that grows with each chat compile in the process. With input capture off in the client + at that time, the tag holds no variables. It does not tag the messages of a placeholder + variable: the app passes those, and they keep the tag of their own compile. TypeScript: a + non-enumerable symbol property; the keys and the JSON of a message do not change. Python: the + messages are instances of a private `dict` subclass that holds the tag in a slot. + `isinstance(message, dict)` is true and the keys and the JSON do not change, but a check of the + exact type (`type(message) is dict`), `yaml.safe_dump`, and `marshal` reject it. Its + constructor is the one of `dict`. - A copy has no tag: a spread, `structuredClone`, or a JSON round trip (TypeScript); `dict()`, `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, a JSON round trip, `type(message)(message)`, or `dataclasses.asdict` (Python). A new message with the same text has no tag either, thus a @@ -203,17 +204,21 @@ their text: They do not look inside a message. - In a `withPrompt` / `with_prompt` scope they give nothing. A model-call span whose parent context is in a scope gets the attributes of the scope and drops each `td.prompt.*` attribute that the - app passes, also when the app calls these functions outside the scope and passes an explicit - parent. They never throw: an error goes to `onError` / `on_error`. -- They give `td.prompt.variables` only when input capture is on in the client. Then they - serialize the variables, which calls the `mask` of the client. A span whose effective - captureInput is off drops `td.prompt.variables` from its attributes, also when the app passes - the result of these functions to its own span. -- A tagged message keeps the variables of its compile alive while the message lives, with the - lists of its placeholder variables. Thus a chat history that keeps a compiled message of each - turn keeps the history list of each turn. The SDKs keep no other reference to the variables, - except those of the last compile of each prompt object and of each running scope, for - `withPrompt` / `with_prompt`. + app passes, at start and in each update (`update`, `end`, `updateActiveSpan` / + `update_current_span`), also when the app calls these functions outside the scope and passes an + explicit parent. They never throw: an error goes to `onError` / `on_error`. +- They give `td.prompt.variables` only when input capture is on in the client, at the compile and + at the call. Then they serialize the variables, which calls the `mask` of the client. A span + whose effective captureInput is off drops `td.prompt.variables` from its attributes, also when + the app passes the result of these functions to its own span. +- A tag keeps the variables of its compile alive while a message of the compile lives. When a + chat compile gets a tagged message as a variable, or as an item of a list (Python: a list or + tuple) in a variable, and the tag is at least 100 serial numbers older, the older tag drops its + variables. Thus a chat history that keeps a compiled message of each turn keeps the variables of + about the last 100 compiles, not of each turn, and a request with only such messages records no + `td.prompt.variables`. The SDKs do not look deeper in a variable. They keep no other reference to + the variables, except those of the last compile of each prompt object and of each running scope, + for `withPrompt` / `with_prompt`. - `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. - The provider integrations call them with the request messages before the span starts, also diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 1fe9131..05c4dba 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -182,29 +182,36 @@ and TanStack AI integrations, and from other OpenTelemetry tracers, do not get t Outside `withPrompt`, those provider integrations record the prompt of the message objects that `compile()` returned. `compile()` of a chat prompt tags each message that it makes with its prompt -and a copy of its variables, in a non-enumerable symbol property: the keys and the JSON of the -message do not change. A chat call that sends tagged messages gets `td.prompt.name`, -`td.prompt.version`, and `td.prompt.label` of the newest `compile()` among them, also when input -capture is off, and `td.prompt.variables` when it is on. The SDK looks at the first 1000 and the -last 1000 messages of the call. A message from a placeholder variable keeps the tag of its own, -older `compile()`, so it does not win over the `compile()` that put it in the list. A copy of a -message (a spread, `structuredClone`, or JSON) and a new message with the same text have no tag, -so the SDK does not record them. Bedrock Converse and Google GenAI take another message shape: -change the compiled messages in place, or use `withPrompt`. A text prompt compiles to a string, -which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The integrations do it -for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse, Google GenAI -`generateContent`, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is -bytes), embeddings, images, audio, video, or OpenAI Realtime. - -A tagged message keeps the variables of its `compile()` alive while the message lives, with the -lists of its placeholder variables. If you keep compiled messages in a long chat history, put -copies (`{ ...message }`) in the history: else each turn keeps the history list of its own turn. +and a copy of its variables and of each list in them, in a non-enumerable symbol property: the keys +and the JSON of the message do not change. The objects in a list are not copied, so a change to one +of them after `compile()` changes the variables that a call records. A chat call that sends tagged +messages gets `td.prompt.name`, `td.prompt.version`, and `td.prompt.label` of the newest `compile()` +among them, also when input capture is off, and `td.prompt.variables` when it is on. The SDK looks +at the first 1000 and the last 1000 messages of the call. A message from a placeholder variable +keeps the tag of its own, older `compile()`, so it does not win over the `compile()` that put it in +the list. A copy of a message (a spread, `structuredClone`, or JSON) and a new message with the same +text have no tag, so the SDK does not record them. Bedrock Converse and Google GenAI take another +message shape: change the compiled messages in place, or use `withPrompt`. A text prompt compiles to +a string, which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The +integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse, +Google GenAI `generateContent`, and OpenRouter chat and responses, but not for Bedrock InvokeModel +(its body is bytes), embeddings, images, audio, video, or OpenAI Realtime. + +With input capture on in the client when `compile()` runs, a tagged message keeps the variables of +its `compile()` alive while the message lives. With input capture off, the tag keeps no variables. +When `compile()` gets a message in a variable, or in a list in a variable (for example a chat +history in a placeholder), and the message is from a `compile()` at least 100 compiles older, that +older `compile()` drops its variables. Thus a chat history of compiled messages keeps the variables +of about the last 100 compiles, not of each turn. A call that sends only messages of such an older +`compile()` records its prompt without `td.prompt.variables`. The SDK does not look deeper in a +variable for messages. For your own spans, `promptAttributesFor(messages)` gives the attributes to pass as `attributes`. It gives nothing inside `withPrompt`, and a span with `captureInput: false` drops `td.prompt.variables`. A model call whose parent is in `withPrompt` gets the prompt of the -`withPrompt` and none of the `td.prompt.*` attributes that you pass, also when its `parent` option -leads into a `withPrompt` that the call is not in. +`withPrompt` and none of the `td.prompt.*` attributes that you pass, at start or in `update`, +`end`, or `updateActiveSpan`, also when its `parent` option leads into a `withPrompt` that the call +is not in. ## API diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 8202159..922222a 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -97,9 +97,11 @@ export interface ChatPrompt extends PromptBase { /** * Fills in the variables and replaces each placeholder message with the messages in its * variable. `withPrompt` records the variables of the last call before it starts or inside it. - * It tags each message object that it makes with this prompt and a copy of these variables, so - * that a model call outside `withPrompt` that sends the message records them; see - * `promptAttributesFor`. The message keeps the variables alive while it lives. + * It tags each message object that it makes with this prompt and a copy of these variables and + * of each list in them, so that a model call outside `withPrompt` that sends the message records + * them; see `promptAttributesFor`. The objects in a list are not copied. The tag keeps no + * variables when input capture is off in the client. It drops them when a compile 100 or more + * compiles newer takes one of its messages in a variable, as a chat history does. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -205,7 +207,8 @@ interface PromptScope { /** The `compile()` call that made a chat message. */ interface CompileTag { readonly prompt: Prompt; - readonly variables: PromptVariables; + /** Undefined when input capture was off, or when the messages became a history; see HISTORY_AGE. */ + variables: PromptVariables | undefined; /** Counts the compiles of the process, so a higher one is newer. */ readonly serial: number; } @@ -225,6 +228,13 @@ const PROMPT_KEY = createContextKey("telemetry.dev prompt"); const COMPILED_BY = Symbol("telemetry.dev compiled by"); /** How many messages promptAttributesFor looks at, at each end of the list. */ const TAG_LOOKUP = 1_000; +/** + * A compile drops its variables when a compile at least this many compiles newer takes one of its + * messages in a variable: its messages are then the history of a chat, and the newest compile names + * the request. Without this, a history that keeps a compiled message of each turn would keep the + * history list of every turn. + */ +const HISTORY_AGE = 100; const cache = new Map(); const compiledVariables = new WeakMap(); @@ -375,8 +385,9 @@ export function promptAttributes( * of messages, or one message): those of the newest `compile()` of a chat prompt that made one of * the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the message * object, so a copy of it gives nothing. Gives undefined inside `withPrompt`, whose prompt the span - * gets. `td.prompt.variables` needs input capture on in the client, and the span drops it when its - * own input capture is off. It never throws. + * gets. `td.prompt.variables` needs input capture on in the client, at the compile and now, and + * the span drops it when its own input capture is off. A compile whose messages became the history + * of a newer compile gives no variables; see `compile()`. It never throws. */ export function promptAttributesFor(messages: unknown): Record | undefined { const config = currentClient().core?.config; @@ -622,8 +633,7 @@ function createPrompt(input: { responseFormat: content.responseFormat, isFallback: input.isFallback, compile(variables: PromptVariables = {}) { - // A copy, so that a later change of the app's object does not change what a span records. - const values = { ...variables }; + const values = copyVariables(variables); compiledVariables.set(prompt, values); const scope = activeContext().getValue(PROMPT_KEY) as PromptScope | undefined; @@ -631,20 +641,48 @@ function createPrompt(input: { if (type === "text") return fill(content.text ?? "", values); compiles += 1; - - return compileMessages( - content.messages, - Object.freeze({ prompt, variables: values, serial: compiles }), - ); + dropHistory(values, compiles); + // promptAttributesFor cannot record the variables when input capture is off in the client. + const captureInput = currentClient().core?.config.captureInput ?? true; + + return compileMessages(content.messages, values, { + prompt, + variables: captureInput ? values : undefined, + serial: compiles, + }); }, }) as Prompt; return prompt; } -function compileMessages(messages: readonly PromptMessage[], tag: CompileTag): unknown[] { - const { variables } = tag; +// A copy of each list too, so that a change of the app's history after compile() does not change +// what a span records. +function copyVariables(variables: PromptVariables): PromptVariables { + return Object.fromEntries( + Object.entries(variables).map(([name, value]) => [ + name, + Array.isArray(value) ? [...value] : value, + ]), + ); +} + +/** Drops the variables of each older compile whose message is in `variables`; see HISTORY_AGE. */ +function dropHistory(variables: PromptVariables, serial: number): void { + for (const value of Object.values(variables)) { + for (const item of Array.isArray(value) ? value : [value]) { + const tag = tagOf(item); + + if (tag && serial - tag.serial >= HISTORY_AGE) tag.variables = undefined; + } + } +} +function compileMessages( + messages: readonly PromptMessage[], + variables: PromptVariables, + tag: CompileTag, +): unknown[] { return messages.flatMap((message) => { if (message.role !== "placeholder") { const compiled = { role: message.role, content: fill(message.content, variables) }; diff --git a/packages/sdk/src/span.ts b/packages/sdk/src/span.ts index cf0c374..3ed9cd9 100644 --- a/packages/sdk/src/span.ts +++ b/packages/sdk/src/span.ts @@ -1,4 +1,5 @@ import { + type Attributes, type Context, defaultTextMapGetter, defaultTextMapSetter, @@ -98,6 +99,8 @@ interface SpanMeta { type: SpanType; capture: FieldCaptureConfig; onError?: (cause: Error) => void; + /** The span got its prompt from withPrompt. */ + scopedPrompt?: boolean; } // Remembers each span's type + capture flags so update()/end()/updateActiveSpan map @@ -190,11 +193,22 @@ function applyFields(span: Span, meta: SpanMeta, fields: SpanFields): void { if (fields.name) span.updateName(fields.name); const attrs = fieldsToAttributes(fields, meta.type, meta.capture); + if (meta.scopedPrompt) dropPromptAttributes(attrs); + if (Object.keys(attrs).length > 0) span.setAttributes(attrs); if (fields.error !== undefined) applyError(span, fields.error, meta.capture); } +// The prompt of withPrompt replaces each td.prompt.* attribute that the app passes, at start and in +// each update, so that a span does not mix two prompts when the app calls promptAttributesFor +// outside withPrompt. +function dropPromptAttributes(attrs: Attributes): void { + for (const key of Object.keys(attrs)) { + if (key.startsWith("td.prompt.")) delete attrs[key]; + } +} + function metaFor(core: ClientCore, options?: StartSpanOptions): SpanMeta { const cfg = core.config; @@ -232,14 +246,9 @@ export function createSpanHandle( const attrs = fieldsToAttributes(options, meta.type, meta.capture); const operation = attrs["gen_ai.operation.name"] ?? SPAN_TYPE_OPERATIONS[meta.type]; const prompt = promptAttributes(parentCtx, operation, meta.capture); + meta.scopedPrompt = prompt !== undefined; - // The prompt of withPrompt replaces all td.prompt.* attributes of the app, so that a span never - // mixes two prompts, also when its parent is in withPrompt and promptAttributesFor ran outside. - if (prompt) { - for (const key of Object.keys(attrs)) { - if (key.startsWith("td.prompt.")) delete attrs[key]; - } - } + if (meta.scopedPrompt) dropPromptAttributes(attrs); const span = core.tracer.startSpan( name, diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index a10ebb6..b464c46 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -16,6 +16,7 @@ import { shutdown, startActiveSpan, startSpan, + updateActiveSpan, withPrompt, } from "../src/index.ts"; import { setup } from "./helpers.ts"; @@ -909,10 +910,8 @@ test("a message that a placeholder brings from an older compile does not win", a ], }); - const routed = answer.compile({ - team: "billing", - history: router.compile({ question: "Where is #4?" }) as ChatMessage[], - }); + const question = router.compile({ question: "Where is #4?" }) as ChatMessage[]; + const routed = answer.compile({ team: "billing", history: question }); // The last 1000 messages are all from chat; the summary message is the first. const conversation = Array.from( { length: 1500 }, @@ -928,6 +927,13 @@ test("a message that a placeholder brings from an older compile does not win", a '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}', }); expect(promptAttributesFor(summarized)?.["td.prompt.name"]).toBe("summary"); + // A newer compile that takes a message of a recent compile does not drop its variables. + expect(promptAttributesFor(question)).toEqual({ + "td.prompt.name": "router", + "td.prompt.version": 1, + "td.prompt.label": "production", + "td.prompt.variables": '{"question":"Where is #4?"}', + }); }); test("promptAttributesFor looks at the first and the last 1000 messages", async () => { @@ -942,45 +948,62 @@ test("promptAttributesFor looks at the first and the last 1000 messages", async expect(promptAttributesFor([...filler(1000), system, ...filler(1000)])).toBeUndefined(); }); -test("compile records its variables, not later changes of the object", async () => { +test("compile records its variables, not later changes of the object or of its lists", async () => { const { spans } = setup(); - const prompt = await promptOf({ messages: [{ role: "user", content: "Order {{order}}" }] }); - const variables = { order: "#A" }; + const prompt = await promptOf({ + messages: [ + { role: "placeholder", name: "history" }, + { role: "user", content: "Order {{order}}" }, + ], + }); + const history: ChatMessage[] = [{ role: "user", content: "Hi" }]; + const variables = { order: "#A", history }; const first = prompt.compile(variables); variables.order = "#B"; + history.push({ role: "assistant", content: "Hello" }); const second = prompt.compile(variables); variables.order = "#C"; + history.push({ role: "user", content: "Added after compile" }); withPrompt(prompt, () => startSpan("scope", { type: "generation" }).end()); await flush(); - expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe('{"order":"#A"}'); - expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe('{"order":"#B"}'); - expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBe('{"order":"#B"}'); + const secondVariables = + '{"order":"#B","history":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Hello"}]}'; + expect(promptAttributesFor(first)?.["td.prompt.variables"]).toBe( + '{"order":"#A","history":[{"role":"user","content":"Hi"}]}', + ); + expect(promptAttributesFor(second)?.["td.prompt.variables"]).toBe(secondVariables); + expect(spans.getFinishedSpans()[0]!.attributes["td.prompt.variables"]).toBe(secondVariables); }); -test("the variables of a compile are freed with its messages", async () => { +// A WeakRef keeps its target until the current job ends, so collect in a new one. +async function collect(): Promise { setFlagsFromString("--expose-gc"); const gc = runInNewContext("gc") as () => void; - // A WeakRef keeps its target until the current job ends, so collect in a new one. - const collect = async () => { - await new Promise((resolve) => setTimeout(resolve, 0)); - gc(); - }; - setup(); - const prompt = await promptOf({ - messages: [ - { role: "system", content: "Answer as the {{team}} desk." }, - { role: "placeholder", name: "history" }, - ], - }); + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); +} - let history: ChatMessage[] | undefined = [{ role: "user", content: "Hi" }]; - const freed = new WeakRef(history); - let messages = prompt.compile({ team: "billing", history }) as ChatMessage[] | undefined; - history = undefined; +const RAG_PROMPT = { + messages: [ + { role: "system", content: "Answer from {{documents}}." }, + { role: "user", content: "{{question}}" }, + ], +}; + +test("the variables of a compile are freed with its messages", async () => { + setup(); + const prompt = await promptOf(RAG_PROMPT); + + let document: ChatMessage | undefined = { role: "user", content: "The refund policy" }; + const freed = new WeakRef(document); + let messages = prompt.compile({ documents: [document], question: "q" }) as + | ChatMessage[] + | undefined; + document = undefined; // withPrompt keeps the variables of the last compile of a prompt object. - prompt.compile({ team: "returns" }); + prompt.compile({ documents: [], question: "other" }); await collect(); expect(freed.deref()).toBeDefined(); @@ -990,7 +1013,79 @@ test("the variables of a compile are freed with its messages", async () => { expect(freed.deref()).toBeUndefined(); }); -test("withPrompt wins over a compiled message of another prompt, also through a parent", async () => { +test("with input capture off, compiled messages keep no variables", async () => { + setup({ captureInput: false }); + const prompt = await promptOf(RAG_PROMPT); + const documents: WeakRef[] = []; + const questions: ChatMessage[] = []; + + for (let turn = 0; turn < 200; turn += 1) { + const document: ChatMessage = { role: "user", content: `document ${turn}` }; + documents.push(new WeakRef(document)); + const compiled = prompt.compile({ documents: [document], question: `question ${turn}` }); + questions.push(compiled.at(-1) as ChatMessage); + } + await collect(); + + // withPrompt keeps the variables of the last compile of a prompt object. + expect(documents.filter((document) => document.deref()).length).toBe(1); + + await shutdown(); + setup(); + expect(promptAttributesFor(questions)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); +}); + +test("a chat history of compiled messages keeps the variables of a bounded number of turns", async () => { + setup(); + const prompt = await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], + }); + const turns: WeakRef[] = []; + let history: ChatMessage[] = []; + let last: ChatMessage[] = []; + + // Each turn keeps its compiled question in the history. An unused variable marks the variables + // of the turn. + const chat = async (count: number) => { + for (let turn = 0; turn < count; turn += 1) { + const marker: ChatMessage = { role: "user", content: `turn ${turns.length}` }; + turns.push(new WeakRef(marker)); + last = prompt.compile({ + team: "billing", + question: `question ${turns.length}`, + history, + marker: [marker], + }) as ChatMessage[]; + history = [ + ...history, + last.at(-1)!, + { role: "assistant", content: `answer ${turns.length}` }, + ]; + } + await collect(); + + return turns.filter((turn) => turn.deref()).length; + }; + + const kept = await chat(150); + expect(kept).toBeLessThan(150); + expect(await chat(150)).toBe(kept); + + const variables = JSON.parse(promptAttributesFor(last)!["td.prompt.variables"] as string) as { + history: ChatMessage[]; + }; + expect(variables.history).toHaveLength(598); +}); + +test("withPrompt wins over a compiled message of another prompt, through a parent and in updates", async () => { const { spans } = setup(); const scoped = await promptOf({ name: "scoped/prompt" }); // Fetched by version, so it has no label that the label of the scope could hide. @@ -1018,10 +1113,15 @@ test("withPrompt wins over a compiled message of another prompt, also through a const attributes = promptAttributesFor(compiled); startSpan("with parent", { type: "generation", parent, attributes }).end(); parent.end(); + withPrompt(scoped, () => { + startSpan("updated", { type: "generation" }).update({ attributes }).end(); + startSpan("ended", { type: "generation" }).end({ attributes }); + startActiveSpan("active", { type: "generation" }, () => updateActiveSpan({ attributes })); + }); await flush(); expect(attributes?.["td.prompt.name"]).toBe("other/prompt"); - for (const name of ["in scope", "with parent"]) { + for (const name of ["in scope", "with parent", "updated", "ended", "active"]) { const span = spans.getFinishedSpans().find((finished) => finished.name === name)!; expect( Object.fromEntries( diff --git a/sdks/python/README.md b/sdks/python/README.md index 0345367..67207ed 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -255,35 +255,43 @@ other OpenTelemetry tracers do not get the attributes. Outside `with_prompt`, those provider integrations record the prompt of the message dicts that `compile()` returned. `compile()` of a chat prompt returns instances of a private `dict` subclass -that also hold its prompt and a copy of its variables. A chat call that sends such messages gets -`td.prompt.name`, `td.prompt.version`, and `td.prompt.label` of the newest `compile()` among -them, also when input capture is off, and `td.prompt.variables` when it is on. The SDK looks at -the first 1000 and the last 1000 messages of the call. A message from a placeholder variable keeps -the prompt of its own, older `compile()`, so it does not win over the `compile()` that put it in -the list. A copy of a message (`dict()`, `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, JSON, -`type(message)(message)`, or `dataclasses.asdict`) and a new message with the same text do not -hold the prompt, so the SDK does not record them. Bedrock Converse and Google GenAI take another -message shape: change the compiled dicts in place, or use `with_prompt`. A text prompt compiles to -a string, which cannot hold its prompt: use `with_prompt`. `with_prompt` wins over the message. -The integrations do it for OpenAI chat completions and responses, Anthropic messages (for -`messages.stream()`, at `__enter__`), Bedrock Converse, Google GenAI `generate_content`, LiteLLM -completion and responses, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its -body is bytes), embeddings, images, audio, video, or OpenAI Realtime. With an older -`telemetry-dev` that has no `prompt_attributes_for`, a provider integration records the prompt -only in `with_prompt`. +that also hold its prompt and a copy of its variables and of each list in them. The objects in a +list are not copied, so a change to one of them after `compile()` changes the variables that a call +records. A chat call that sends such messages gets `td.prompt.name`, `td.prompt.version`, and +`td.prompt.label` of the newest `compile()` among them, also when input capture is off, and +`td.prompt.variables` when it is on. The SDK looks at the first 1000 and the last 1000 messages of +the call. A message from a placeholder variable keeps the prompt of its own, older `compile()`, so +it does not win over the `compile()` that put it in the list. A copy of a message (`dict()`, +`.copy()`, `copy.copy`, `copy.deepcopy`, pickle, JSON, `type(message)(message)`, or +`dataclasses.asdict`) and a new message with the same text do not hold the prompt, so the SDK does +not record them. Bedrock Converse and Google GenAI take another message shape: change the compiled +dicts in place, or use `with_prompt`. A text prompt compiles to a string, which cannot hold its +prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations do it for OpenAI +chat completions and responses, Anthropic messages (for `messages.stream()`, at `__enter__`), +Bedrock Converse, Google GenAI `generate_content`, LiteLLM completion and responses, and OpenRouter +chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, +video, or OpenAI Realtime. With an older `telemetry-dev` that has no `prompt_attributes_for`, a +provider integration records the prompt only in `with_prompt`. A compiled message is a `dict` for `isinstance`, and its keys and its JSON (`json`, `orjson`, pydantic) do not change. A check of the exact type (`type(message) is dict`), `yaml.safe_dump`, -and `marshal` reject it: give them `dict(message)`. A compiled message keeps the variables of its -`compile()` alive while the message lives, with the lists of its placeholder variables. If you -keep compiled messages in a long chat history, put copies (`dict(message)`) in the history: else -each turn keeps the history list of its own turn. +and `marshal` reject it: give them `dict(message)`. + +With input capture on in the client when `compile()` runs, a compiled message keeps the variables +of its `compile()` alive while the message lives. With input capture off, it keeps no variables. +When `compile()` gets a message in a variable, or in a list or tuple in a variable (for example a +chat history in a placeholder), and the message is from a `compile()` at least 100 compiles older, +that older `compile()` drops its variables. Thus a chat history of compiled messages keeps the +variables of about the last 100 compiles, not of each turn. A call that sends only messages of such +an older `compile()` records its prompt without `td.prompt.variables`. The SDK does not look deeper +in a variable for messages. For your own spans, `prompt_attributes_for(messages)` gives the attributes to pass as `attributes`. It gives `{}` inside `with_prompt`, and a span with `capture_input=False` drops `td.prompt.variables`. A model call whose parent is in `with_prompt` gets the prompt of the -`with_prompt` and none of the `td.prompt.*` attributes that you pass, also when its `parent` -context comes from a `with_prompt` that the call is not in. +`with_prompt` and none of the `td.prompt.*` attributes that you pass, at start or in `update`, +`end`, or `update_current_span`, also when its `parent` context comes from a `with_prompt` that the +call is not in. ## Auto-metrics diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 28ee83e..59d8a1b 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -14,7 +14,7 @@ import urllib.parse import urllib.request from collections import OrderedDict -from collections.abc import Callable, Generator, Mapping +from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field @@ -53,6 +53,11 @@ _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") # How many messages prompt_attributes_for looks at, at each end of the list. _TAG_LOOKUP = 1000 +# A compile drops its variables when a compile at least this many compiles newer takes one of its +# messages in a variable: its messages are then the history of a chat, and the newest compile names +# the request. Without this, a history that keeps a compiled message of each turn would keep the +# history list of every turn. +_HISTORY_AGE = 100 class PromptModel(TypedDict): @@ -102,12 +107,20 @@ def compile( """Fill in the variables and replace each placeholder message with the list of messages in its variable. ``with_prompt`` records the variables of the last call before it starts or inside it. Each message that it makes is a ``dict`` subclass that also holds this - prompt and a copy of these variables, so that a model call outside ``with_prompt`` that - sends the message records them; see ``prompt_attributes_for``. The message keeps the - variables alive while it lives. A check of the exact type (``type(message) is dict``), - ``yaml.safe_dump``, and ``marshal`` reject it: give them ``dict(message)``.""" + prompt and a copy of these variables and of each list in them, so that a model call + outside ``with_prompt`` that sends the message records them; see + ``prompt_attributes_for``. The objects in a list are not copied. The message holds no + variables when input capture is off in the client. It drops them when a compile 100 or + more compiles newer takes one of its messages in a variable, as a chat history does. A + check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and ``marshal`` + reject it: give them ``dict(message)``.""" values = _record(self, variables, kwargs) - tag = _CompileTag(self, values, next(_compiles)) + serial = next(_compiles) + _drop_history(values, serial) + client = get_client() + # prompt_attributes_for cannot record the variables when input capture is off in the client. + kept = values if client is None or client.capture_input else None + tag = _CompileTag(self, kept, serial) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": @@ -304,7 +317,9 @@ def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: prompt that made one of the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the dict that ``compile()`` made, so a copy of it gives ``{}``. Gives ``{}`` inside ``with_prompt``, whose prompt the span gets. ``td.prompt.variables`` needs input - capture on in the client, and the span drops it when its own input capture is off. + capture on in the client, at the compile and now, and the span drops it when its own input + capture is off. A compile whose messages became the history of a newer compile gives no + variables; see ``ChatPrompt.compile``. It reads only a ``list`` or ``tuple``, not a subclass, so the read runs no code of the app. With input capture on, it serializes the variables, which calls the ``mask`` of the client and @@ -439,7 +454,12 @@ class _PromptScope: def _record( prompt: Prompt, variables: Mapping[str, Any] | None, kwargs: Mapping[str, Any] ) -> dict[str, Any]: - values = {**(variables or {}), **kwargs} + # A copy of each list too, so that a change of the app's history after compile() does not + # change what a span records. + values: dict[str, Any] = { + name: list(cast("list[Any]", value)) if isinstance(value, list) else value + for name, value in {**(variables or {}), **kwargs}.items() + } _compiled_variables[prompt] = values scope = otel_context.get_value(_PROMPT_KEY) if isinstance(scope, _PromptScope) and scope.prompt is prompt: @@ -448,12 +468,13 @@ def _record( return values -@dataclass(frozen=True) +@dataclass(eq=False) class _CompileTag: """The ``compile()`` call that made a chat message.""" prompt: ChatPrompt - variables: dict[str, Any] + variables: dict[str, Any] | None + """None when input capture was off, or when the messages became a history; see _HISTORY_AGE.""" serial: int """Counts the compiles of the process, so a higher one is newer.""" @@ -473,6 +494,22 @@ def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: return dict, (dict(self),) +def _drop_history(values: Mapping[str, Any], serial: int) -> None: + """Drops the variables of each older compile whose message is in ``values``; see + _HISTORY_AGE.""" + for value in values.values(): + items: Sequence[object] = ( + cast("Sequence[object]", value) + if type(value) is list or type(value) is tuple + else (value,) + ) + for item in items: + if type(item) is _CompiledMessage: + tag = getattr(item, "compiled_by", None) + if tag is not None and serial - tag.serial >= _HISTORY_AGE: + tag.variables = None + + def _newest_tag(messages: object) -> _CompileTag | None: """The messages of a placeholder come from an older compile, so the newest compile, not the last tagged message, names the prompt of the request. Its own messages are at the start or the diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 1d6ac64..6328fa3 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -103,6 +103,7 @@ class _SpanState: operation: str capture_input: bool capture_output: bool + prompt_from_scope: bool = False output_chunk_last_ms: float | None = None output_chunk_count: int = 0 output_chunk_sum_s: float = 0 @@ -300,6 +301,8 @@ def _apply_fields(client: Client, span: Span, state: _SpanState, fields: dict[st if name is not None: span.update_name(name) attrs = _fields_to_attributes(client, state, fields) + if state.prompt_from_scope: + attrs = _without_prompt(attrs) if attrs: span.set_attributes(attrs) @@ -308,6 +311,15 @@ def _apply_fields(client: Client, span: Span, state: _SpanState, fields: dict[st _record_error_on_span(client, span, state, error) +def _without_prompt( + attrs: dict[str, otel_types.AttributeValue], +) -> dict[str, otel_types.AttributeValue]: + """The prompt of with_prompt replaces each td.prompt.* attribute that the app passes, at start + and in each update, so that a span does not mix two prompts when the app calls + prompt_attributes_for outside with_prompt.""" + return {key: value for key, value in attrs.items() if not key.startswith("td.prompt.")} + + def _collect_fields( *, input: Any = NOT_GIVEN, @@ -752,10 +764,8 @@ def start_span( client, ctx, attrs.get(ATTR_OPERATION, operation), state.capture_input ) if prompt: - # The prompt of with_prompt replaces all td.prompt.* attributes of the app, so that a - # span never mixes two prompts, also when its parent is in with_prompt and - # prompt_attributes_for ran outside. - attrs = {key: value for key, value in attrs.items() if not key.startswith("td.prompt.")} + state.prompt_from_scope = True + attrs = _without_prompt(attrs) initial.update(prompt) initial.update(attrs) span = client.tracer.start_span( diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 423286f..103c399 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -985,7 +985,8 @@ def test_a_message_that_a_placeholder_brings_from_an_older_compile_does_not_win( name="summary", ) - routed = answer.compile(team="billing", history=router.compile(question="Where is #4?")) + question = router.compile(question="Where is #4?") + routed = answer.compile(team="billing", history=question) # The last 1000 messages are all from chat; the summary message is the first. conversation = [chat.compile(question=f"question {index}")[0] for index in range(1500)] summarized = summary.compile(history=conversation) @@ -999,6 +1000,13 @@ def test_a_message_that_a_placeholder_brings_from_an_older_compile_does_not_win( ), } assert prompt_attributes_for(summarized)["td.prompt.name"] == "summary" + # A newer compile that takes a message of a recent compile does not drop its variables. + assert prompt_attributes_for(question) == { + "td.prompt.name": "router", + "td.prompt.version": 12, + "td.prompt.label": "production", + "td.prompt.variables": '{"question":"Where is #4?"}', + } def test_prompt_attributes_for_looks_at_the_first_and_the_last_1000_messages( @@ -1016,24 +1024,37 @@ def around(before: int, after: int) -> list[dict[str, Any]]: assert prompt_attributes_for(around(1000, 1000)) == {} -def test_compile_records_its_variables_not_later_changes_of_the_mapping( +def test_compile_records_its_variables_not_later_changes_of_the_mapping_or_of_its_lists( make: MakeClient, ) -> None: memory = make() - prompt = chat_prompt({"role": "user", "content": "Order {{order}}"}) - variables = {"order": "#A"} + prompt = chat_prompt( + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "Order {{order}}"}, + ) + history = [{"role": "user", "content": "Hi"}] + variables: dict[str, Any] = {"order": "#A", "history": history} first = prompt.compile(variables) variables["order"] = "#B" + history.append({"role": "assistant", "content": "Hello"}) second = prompt.compile(variables) variables["order"] = "#C" + history.append({"role": "user", "content": "Added after compile"}) with with_prompt(prompt): start_span("scope", type="generation").end() - assert prompt_attributes_for(first)["td.prompt.variables"] == '{"order":"#A"}' - assert prompt_attributes_for(second)["td.prompt.variables"] == '{"order":"#B"}' + second_variables = ( + '{"order":"#B","history":[{"role":"user","content":"Hi"},' + '{"role":"assistant","content":"Hello"}]}' + ) + assert ( + prompt_attributes_for(first)["td.prompt.variables"] + == '{"order":"#A","history":[{"role":"user","content":"Hi"}]}' + ) + assert prompt_attributes_for(second)["td.prompt.variables"] == second_variables [span] = memory.span_exporter.get_finished_spans() - assert (span.attributes or {})["td.prompt.variables"] == '{"order":"#B"}' + assert (span.attributes or {})["td.prompt.variables"] == second_variables def test_a_compiled_message_rebuilds_like_a_dict_and_the_rebuilt_one_has_no_tag( @@ -1059,7 +1080,7 @@ class Conversation: assert prompt_attributes_for(same_type) == {} -def test_with_prompt_wins_over_a_compiled_message_of_another_prompt_also_through_a_parent( +def test_with_prompt_wins_over_a_compiled_message_of_another_prompt_through_a_parent_and_in_updates( make: MakeClient, ) -> None: memory = make() @@ -1077,9 +1098,22 @@ def test_with_prompt_wins_over_a_compiled_message_of_another_prompt_also_through parent = otel_context.get_current() attributes = prompt_attributes_for(compiled) start_span("with parent", type="generation", parent=parent, attributes=attributes).end() + with with_prompt(scoped): + start_span("updated", type="generation").update(attributes=attributes).end() + start_span("ended", type="generation").end(attributes=attributes) + with start_span("current", type="generation"): + telemetry_dev.update_current_span(attributes=attributes) assert attributes["td.prompt.name"] == "other/prompt" - for span in memory.span_exporter.get_finished_spans(): + spans = memory.span_exporter.get_finished_spans() + assert sorted(span.name for span in spans) == [ + "current", + "ended", + "in scope", + "updated", + "with parent", + ] + for span in spans: assert { key: value for key, value in dict(span.attributes or {}).items() @@ -1147,30 +1181,94 @@ def __reversed__(self) -> Any: assert reads == [] +RAG_MESSAGES = [ + {"role": "system", "content": "Answer from {{documents}}."}, + {"role": "user", "content": "{{question}}"}, +] + + +class Document(dict[str, str]): + """A dict that a weak reference can point to.""" + + def test_the_variables_of_a_compile_are_freed_with_its_messages(make: MakeClient) -> None: make() - prompt = chat_prompt( - {"role": "system", "content": "Answer as the {{team}} desk."}, - {"role": "placeholder", "name": "history"}, - ) + prompt = chat_prompt(*RAG_MESSAGES) - class History(list[dict[str, str]]): - """A list that a weak reference can point to.""" - - history = History([{"role": "user", "content": "Hi"}]) - freed = weakref.ref(history) - messages = prompt.compile(team="billing", history=history) - del history + document = Document(content="The refund policy") + freed = weakref.ref(document) + messages = prompt.compile(documents=[document], question="q") + del document # with_prompt keeps the variables of the last compile of a prompt object. - prompt.compile(team="returns") + prompt.compile(documents=[], question="other") gc.collect() assert freed() is not None + assert len(messages) == 2 del messages gc.collect() assert freed() is None +def test_with_input_capture_off_compiled_messages_keep_no_variables(make: MakeClient) -> None: + make(capture_input=False) + prompt = chat_prompt(*RAG_MESSAGES) + documents: list[weakref.ref[Document]] = [] + questions: list[dict[str, Any]] = [] + + for turn in range(200): + document = Document(content=f"document {turn}") + documents.append(weakref.ref(document)) + questions.append(prompt.compile(documents=[document], question=f"question {turn}")[-1]) + gc.collect() + + # with_prompt keeps the variables of the last compile of a prompt object. + assert sum(1 for document in documents if document() is not None) == 1 + + telemetry_dev.shutdown() + make() + assert prompt_attributes_for(questions) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } + + +def test_a_chat_history_of_compiled_messages_keeps_the_variables_of_a_bounded_number_of_turns( + make: MakeClient, +) -> None: + make() + prompt = chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, + ) + turns: list[weakref.ref[Document]] = [] + history: list[dict[str, Any]] = [] + last: list[dict[str, Any]] = [] + + # Each turn keeps its compiled question in the history. An unused variable marks the variables + # of the turn. + def chat(count: int) -> int: + nonlocal history, last + for _ in range(count): + marker = Document(content=f"turn {len(turns)}") + turns.append(weakref.ref(marker)) + last = prompt.compile( + team="billing", question=f"question {len(turns)}", history=history, marker=marker + ) + history = [*history, last[-1], {"role": "assistant", "content": f"answer {len(turns)}"}] + gc.collect() + return sum(1 for turn in turns if turn() is not None) + + kept = chat(150) + assert kept < 150 + assert chat(150) == kept + + variables = json.loads(str(prompt_attributes_for(last)["td.prompt.variables"])) + assert len(variables["history"]) == 598 + + def test_a_fallback_prompt_records_its_name_and_label_but_no_version( server: CaptureServer, make: MakeClient ) -> None: From abb8d10f680d1fa7ace6913a1d53b43a3e0db703 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 21:29:24 +0000 Subject: [PATCH 15/32] fix(sdk): give the base url to getPrompt in the prompt tests The tests of this branch gave apiUrl and api_url to getPrompt and get_prompt. The main branch removed these options in #101. Thus the tests give baseUrl, base_url, and newBaseUrl. Refs #97 --- packages/anthropic/tests/anthropic.test.ts | 2 +- packages/bedrock/tests/bedrock.test.ts | 2 +- packages/google-genai/tests/google-genai.test.ts | 2 +- packages/openai/tests/openai.test.ts | 2 +- packages/openrouter/tests/openrouter.test.ts | 2 +- packages/sdk/tests/prompts.test.ts | 6 +++--- sdks/python/tests/test_prompts.py | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index 12eeae9..6516205 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -298,7 +298,7 @@ test.each([ const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: "https://prompts.anthropic.test", + baseUrl: "https://prompts.anthropic.test", fetch: async () => Response.json({ name: "support/reply", diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index 6098bea..0eb0e7f 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -443,7 +443,7 @@ test.each([ const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: "https://prompts.bedrock.test", + baseUrl: "https://prompts.bedrock.test", fetch: async () => Response.json({ name: "support/reply", diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index e7d5ec6..19d7fbf 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -253,7 +253,7 @@ test.each([ const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: "https://prompts.google-genai.test", + baseUrl: "https://prompts.google-genai.test", fetch: async () => Response.json({ name: "support/reply", diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 4437d75..099ec37 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -2912,7 +2912,7 @@ test.each([ const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: "https://prompts.openai.test", + baseUrl: "https://prompts.openai.test", fetch: async () => Response.json({ name: "support/reply", diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index 3f5cb68..0d50a1b 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -329,7 +329,7 @@ test.each([ const prompt = await getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: "https://prompts.openrouter.test", + baseUrl: "https://prompts.openrouter.test", fetch: async () => Response.json({ name: "support/reply", diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index b464c46..f519e54 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -740,7 +740,7 @@ async function promptOf(overrides: Record): Promise { return getPrompt("support/reply", { apiKey: "td_live_a", - apiUrl: newApiUrl(), + baseUrl: newBaseUrl(), fetch, type: overrides.type as "chat", }); @@ -836,7 +836,7 @@ test("a rollout with two versions of the same messages records the version of ea }, }), ); - const options = { apiKey: "td_live_a", apiUrl: newApiUrl(), fetch }; + const options = { apiKey: "td_live_a", baseUrl: newBaseUrl(), fetch }; vi.spyOn(Math, "random").mockReturnValueOnce(0.159).mockReturnValueOnce(0.16); const rollout = await getPrompt("support/reply", options); @@ -1099,7 +1099,7 @@ test("withPrompt wins over a compiled message of another prompt, through a paren ); const other = await getPrompt("other/prompt", { apiKey: "td_live_a", - apiUrl: newApiUrl(), + baseUrl: newBaseUrl(), version: 3, fetch, }); diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 103c399..324b01e 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -931,8 +931,8 @@ def draw(_stop: int) -> int: return next(draws) monkeypatch.setattr(prompts_module.random, "randrange", draw) - rollout = get_prompt("support/reply", api_key="td_live_a", api_url=server.url) - production = get_prompt("support/reply", api_key="td_live_a", api_url=server.url) + rollout = get_prompt("support/reply", api_key="td_live_a", base_url=server.url) + production = get_prompt("support/reply", api_key="td_live_a", base_url=server.url) from_rollout = rollout.compile(team="billing", order="#4") from_production = production.compile(team="billing", order="#4") From f6aa1c159742123600d6499653916c4dbf7b2ab7 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:19:43 +0000 Subject: [PATCH 16/32] fix(sdk): keep the variables of the last two chat turns A compile removed the variables of a compile 100 or more compiles before it, when it got a message of that compile. A chat that cuts its history to less than approximately 100 turns never gave such a message. Each tag kept a copy of the window, and the messages in the window kept the tags of previous turns. Thus every turn kept its variables: 2000 of 2000 turns in TypeScript and Python. The count also included the compiles of other requests. Thus 99 other compiles before a guard compile, or a guard compile at each step of an agent loop, removed the variables of a request before its model call. With this change, when a compile C gets a message of a compile B in a variable, each compile A whose message B got removes its variables. B keeps its own. The rule does not count compiles, thus the compiles of other requests do not change it. A chain of tags with variables is at most two tags long. The repros of the review keep the variables of 2 of 2000 turns in the two SDKs. This is true for one list that the app cuts in place to 20 messages, for a new list of the last 20 or 198 messages, and for a history with no window. With no window, the SDK keeps two copies of the history list, not approximately 100. The tests do the same check for 300 turns, also for a tuple (Python) and for one message that is not in a list. A guard compile after 150 other compiles, and a guard compile at each of 150 agent steps, keep the variables of the request. In Python, _drop_history reads the variables of a tag one time, because a compile in another thread can remove them. The READMEs and docs/sdk-conformance.md tell the new rule. They again recommend copies for compiled messages that the app keeps but does not give to compile(): a copy keeps no variables. Refs #97 --- docs/sdk-conformance.md | 18 +-- packages/sdk/README.md | 15 ++- packages/sdk/src/prompts.ts | 42 ++++--- packages/sdk/tests/prompts.test.ts | 129 +++++++++++++++------- sdks/python/README.md | 15 ++- sdks/python/src/telemetry_dev/_prompts.py | 48 +++++--- sdks/python/tests/test_prompts.py | 112 ++++++++++++++----- 7 files changed, 256 insertions(+), 123 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index b49779a..49e27fe 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -211,14 +211,16 @@ their text: at the call. Then they serialize the variables, which calls the `mask` of the client. A span whose effective captureInput is off drops `td.prompt.variables` from its attributes, also when the app passes the result of these functions to its own span. -- A tag keeps the variables of its compile alive while a message of the compile lives. When a - chat compile gets a tagged message as a variable, or as an item of a list (Python: a list or - tuple) in a variable, and the tag is at least 100 serial numbers older, the older tag drops its - variables. Thus a chat history that keeps a compiled message of each turn keeps the variables of - about the last 100 compiles, not of each turn, and a request with only such messages records no - `td.prompt.variables`. The SDKs do not look deeper in a variable. They keep no other reference to - the variables, except those of the last compile of each prompt object and of each running scope, - for `withPrompt` / `with_prompt`. +- A tag keeps the variables of its compile alive while a message of the compile lives. A chat + compile C can get a tagged message of a compile B as a variable, or as an item of a list + (Python: a list or tuple) in a variable. Then each compile A whose message B got in the same way + removes its variables, and B keeps its own. Thus a chat history that keeps a compiled message of + each turn keeps the variables of the last two turns, for a history of any length or window, and + a request with only messages of previous turns records no `td.prompt.variables`. A guard prompt + that gets the messages of a request before its call does not remove their variables, also after + other compiles in the process. The SDKs do not look deeper in a variable. They keep no other + reference to the variables, except those of the last compile of each prompt object and of each + running scope, for `withPrompt` / `with_prompt`. - `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. - The provider integrations call them with the request messages before the span starts, also diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 05c4dba..9edf709 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -199,12 +199,15 @@ Google GenAI `generateContent`, and OpenRouter chat and responses, but not for B With input capture on in the client when `compile()` runs, a tagged message keeps the variables of its `compile()` alive while the message lives. With input capture off, the tag keeps no variables. -When `compile()` gets a message in a variable, or in a list in a variable (for example a chat -history in a placeholder), and the message is from a `compile()` at least 100 compiles older, that -older `compile()` drops its variables. Thus a chat history of compiled messages keeps the variables -of about the last 100 compiles, not of each turn. A call that sends only messages of such an older -`compile()` records its prompt without `td.prompt.variables`. The SDK does not look deeper in a -variable for messages. +When `compile()` gets a tagged message in a variable, or in a list in a variable, the `compile()` +that made the message keeps its variables. But each `compile()` whose message that `compile()` got +removes its variables. For a chat history in a placeholder, turn 3 thus removes the variables of +turn 1 and keeps those of turn 2. A chat history of compiled messages keeps the variables of its +last two turns, also when you cut it, and a call that sends only messages of previous turns records +its prompt without `td.prompt.variables`. A guard prompt that gets the messages before their call +does not remove their variables. The SDK does not look deeper in a variable for messages. If you +keep compiled messages and do not give them to `compile()`, each one keeps the variables of its +`compile()`: keep copies (`{ ...message }`) to free them. For your own spans, `promptAttributesFor(messages)` gives the attributes to pass as `attributes`. It gives nothing inside `withPrompt`, and a span with `captureInput: false` drops diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 922222a..55169d5 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -100,8 +100,10 @@ export interface ChatPrompt extends PromptBase { * It tags each message object that it makes with this prompt and a copy of these variables and * of each list in them, so that a model call outside `withPrompt` that sends the message records * them; see `promptAttributesFor`. The objects in a list are not copied. The tag keeps no - * variables when input capture is off in the client. It drops them when a compile 100 or more - * compiles newer takes one of its messages in a variable, as a chat history does. + * variables when input capture is off in the client. When a newer compile gets one of these + * messages in a variable, the compiles whose messages this compile got remove their variables, + * and this compile keeps its own. Thus a chat history keeps the variables of its last two turns, + * and a guard prompt that gets the messages before their call does not remove them. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -207,7 +209,9 @@ interface PromptScope { /** The `compile()` call that made a chat message. */ interface CompileTag { readonly prompt: Prompt; - /** Undefined when input capture was off, or when the messages became a history; see HISTORY_AGE. */ + /** + * Undefined when input capture was off, or when the messages became a history; see dropHistory. + */ variables: PromptVariables | undefined; /** Counts the compiles of the process, so a higher one is newer. */ readonly serial: number; @@ -228,13 +232,6 @@ const PROMPT_KEY = createContextKey("telemetry.dev prompt"); const COMPILED_BY = Symbol("telemetry.dev compiled by"); /** How many messages promptAttributesFor looks at, at each end of the list. */ const TAG_LOOKUP = 1_000; -/** - * A compile drops its variables when a compile at least this many compiles newer takes one of its - * messages in a variable: its messages are then the history of a chat, and the newest compile names - * the request. Without this, a history that keeps a compiled message of each turn would keep the - * history list of every turn. - */ -const HISTORY_AGE = 100; const cache = new Map(); const compiledVariables = new WeakMap(); @@ -386,8 +383,8 @@ export function promptAttributes( * the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the message * object, so a copy of it gives nothing. Gives undefined inside `withPrompt`, whose prompt the span * gets. `td.prompt.variables` needs input capture on in the client, at the compile and now, and - * the span drops it when its own input capture is off. A compile whose messages became the history - * of a newer compile gives no variables; see `compile()`. It never throws. + * the span drops it when its own input capture is off. The compile of a previous turn of a chat + * history gives no variables; see `compile()`. It never throws. */ export function promptAttributesFor(messages: unknown): Record | undefined { const config = currentClient().core?.config; @@ -641,7 +638,7 @@ function createPrompt(input: { if (type === "text") return fill(content.text ?? "", values); compiles += 1; - dropHistory(values, compiles); + dropHistory(values); // promptAttributesFor cannot record the variables when input capture is off in the client. const captureInput = currentClient().core?.config.captureInput ?? true; @@ -667,13 +664,26 @@ function copyVariables(variables: PromptVariables): PromptVariables, serial: number): void { +/** + * When a compile gets a message of a previous compile in `variables`, the compiles whose messages + * that previous compile got are the history of a chat and remove their variables. The previous + * compile keeps its own, because its model call can come after this compile, as with a guard + * prompt. Thus a chain of tags with variables is at most two tags long, for a history of any + * length. + */ +function dropHistory(variables: PromptVariables): void { + for (const tag of tagsIn(variables)) { + if (tag.variables) for (const older of tagsIn(tag.variables)) older.variables = undefined; + } +} + +/** The tags of the messages in `variables`: a value, or an item of a list in a value. */ +function* tagsIn(variables: PromptVariables): Generator { for (const value of Object.values(variables)) { for (const item of Array.isArray(value) ? value : [value]) { const tag = tagOf(item); - if (tag && serial - tag.serial >= HISTORY_AGE) tag.variables = undefined; + if (tag) yield tag; } } } diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index f519e54..381df5a 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -11,6 +11,7 @@ import { getPrompt, type Prompt, promptAttributesFor, + type PromptVariables, propagateAttributes, PromptError, shutdown, @@ -927,13 +928,6 @@ test("a message that a placeholder brings from an older compile does not win", a '{"team":"billing","history":[{"role":"user","content":"Where is #4?"}]}', }); expect(promptAttributesFor(summarized)?.["td.prompt.name"]).toBe("summary"); - // A newer compile that takes a message of a recent compile does not drop its variables. - expect(promptAttributesFor(question)).toEqual({ - "td.prompt.name": "router", - "td.prompt.version": 1, - "td.prompt.label": "production", - "td.prompt.variables": '{"question":"Where is #4?"}', - }); }); test("promptAttributesFor looks at the first and the last 1000 messages", async () => { @@ -1039,50 +1033,101 @@ test("with input capture off, compiled messages keep no variables", async () => }); }); -test("a chat history of compiled messages keeps the variables of a bounded number of turns", async () => { - setup(); - const prompt = await promptOf({ - messages: [ - { role: "system", content: "Answer as the {{team}} desk." }, - { role: "placeholder", name: "history" }, - { role: "user", content: "{{question}}" }, - ], - }); - const turns: WeakRef[] = []; - let history: ChatMessage[] = []; - let last: ChatMessage[] = []; - - // Each turn keeps its compiled question in the history. An unused variable marks the variables - // of the turn. - const chat = async (count: number) => { - for (let turn = 0; turn < count; turn += 1) { - const marker: ChatMessage = { role: "user", content: `turn ${turns.length}` }; +// Each row keeps the compiled question and the answer of a turn for the next turn: the variable +// that gets them, and the value of that variable after the turn. +test.each<[string, string, (kept: unknown, question: object, answer: object) => unknown]>([ + ["a new list each turn", "history", (kept, ...turn) => [...(kept as object[]), ...turn]], + [ + "one list trimmed in place to 20 messages", + "history", + (kept, ...turn) => { + const history = kept as object[]; + history.push(...turn); + history.splice(0, history.length - 20); + + return history; + }, + ], + [ + "a new list of the last 20 messages", + "history", + (kept, ...turn) => [...(kept as object[]), ...turn].slice(-20), + ], + ["the last question, not in a list", "previous", (_kept, question) => question], +])( + "a chat history in %s keeps the variables of its last two turns", + async (_shape, variable, next) => { + setup(); + const prompt = await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], + }); + const turns: WeakRef[] = []; + let kept: unknown = variable === "history" ? [] : undefined; + let last: ChatMessage[] = []; + let older: ChatMessage | undefined; + + // An unused variable marks the variables of each turn. + for (let turn = 0; turn < 300; turn += 1) { + const marker: ChatMessage = { role: "user", content: `turn ${turn}` }; turns.push(new WeakRef(marker)); last = prompt.compile({ team: "billing", - question: `question ${turns.length}`, - history, + question: `question ${turn}`, marker: [marker], - }) as ChatMessage[]; - history = [ - ...history, - last.at(-1)!, - { role: "assistant", content: `answer ${turns.length}` }, - ]; + [variable]: kept, + } as PromptVariables) as ChatMessage[]; + const question = last.at(-1)!; + + if (turn === 297) older = question; + kept = next(kept, question, { role: "assistant", content: `answer ${turn}` }); } await collect(); - return turns.filter((turn) => turn.deref()).length; - }; + expect(turns.filter((turn) => turn.deref()).length).toBe(2); + expect(promptAttributesFor(last)?.["td.prompt.variables"]).toContain( + '"question":"question 299"', + ); + expect(promptAttributesFor(older)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); + }, +); - const kept = await chat(150); - expect(kept).toBeLessThan(150); - expect(await chat(150)).toBe(kept); +test("a guard prompt keeps the variables of its request, also after other compiles", async () => { + setup(); + const answer = await promptOf(RAG_PROMPT); + const guard = await promptOf({ + name: "guard", + messages: [ + { role: "system", content: "Is this safe?" }, + { role: "placeholder", name: "conversation" }, + ], + }); + const other = await promptOf({ name: "other", messages: [{ role: "user", content: "{{q}}" }] }); + const variables = '{"documents":"The refund policy","question":"Where is #4?"}'; - const variables = JSON.parse(promptAttributesFor(last)!["td.prompt.variables"] as string) as { - history: ChatMessage[]; - }; - expect(variables.history).toHaveLength(598); + const messages = answer.compile({ + documents: "The refund policy", + question: "Where is #4?", + }) as ChatMessage[]; + // Other requests of the server compile before this request runs its guard. + for (let index = 0; index < 150; index += 1) other.compile({ q: `${index}` }); + guard.compile({ conversation: messages }); + expect(promptAttributesFor(messages)?.["td.prompt.variables"]).toBe(variables); + + // An agent loop runs the guard before each step. + const conversation = [...messages]; + for (let step = 0; step < 150; step += 1) { + guard.compile({ conversation }); + conversation.push({ role: "assistant", content: `step ${step}` }); + } + expect(promptAttributesFor(conversation)?.["td.prompt.variables"]).toBe(variables); }); test("withPrompt wins over a compiled message of another prompt, through a parent and in updates", async () => { diff --git a/sdks/python/README.md b/sdks/python/README.md index 67207ed..d060322 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -279,12 +279,15 @@ and `marshal` reject it: give them `dict(message)`. With input capture on in the client when `compile()` runs, a compiled message keeps the variables of its `compile()` alive while the message lives. With input capture off, it keeps no variables. -When `compile()` gets a message in a variable, or in a list or tuple in a variable (for example a -chat history in a placeholder), and the message is from a `compile()` at least 100 compiles older, -that older `compile()` drops its variables. Thus a chat history of compiled messages keeps the -variables of about the last 100 compiles, not of each turn. A call that sends only messages of such -an older `compile()` records its prompt without `td.prompt.variables`. The SDK does not look deeper -in a variable for messages. +When `compile()` gets a compiled message in a variable, or in a list or tuple in a variable, the +`compile()` that made the message keeps its variables. But each `compile()` whose message that +`compile()` got removes its variables. For a chat history in a placeholder, turn 3 thus removes the +variables of turn 1 and keeps those of turn 2. A chat history of compiled messages keeps the +variables of its last two turns, also when you cut it, and a call that sends only messages of +previous turns records its prompt without `td.prompt.variables`. A guard prompt that gets the +messages before their call does not remove their variables. The SDK does not look deeper in a +variable for messages. If you keep compiled messages and do not give them to `compile()`, each one +keeps the variables of its `compile()`: keep copies (`dict(message)`) to free them. For your own spans, `prompt_attributes_for(messages)` gives the attributes to pass as `attributes`. It gives `{}` inside `with_prompt`, and a span with `capture_input=False` drops diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 59d8a1b..4811f14 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -53,11 +53,6 @@ _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") # How many messages prompt_attributes_for looks at, at each end of the list. _TAG_LOOKUP = 1000 -# A compile drops its variables when a compile at least this many compiles newer takes one of its -# messages in a variable: its messages are then the history of a chat, and the newest compile names -# the request. Without this, a history that keeps a compiled message of each turn would keep the -# history list of every turn. -_HISTORY_AGE = 100 class PromptModel(TypedDict): @@ -110,13 +105,15 @@ def compile( prompt and a copy of these variables and of each list in them, so that a model call outside ``with_prompt`` that sends the message records them; see ``prompt_attributes_for``. The objects in a list are not copied. The message holds no - variables when input capture is off in the client. It drops them when a compile 100 or - more compiles newer takes one of its messages in a variable, as a chat history does. A - check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and ``marshal`` - reject it: give them ``dict(message)``.""" + variables when input capture is off in the client. When a newer compile gets one of these + messages in a variable, the compiles whose messages this compile got remove their + variables, and this compile keeps its own. Thus a chat history keeps the variables of its + last two turns, and a guard prompt that gets the messages before their call does not + remove them. A check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and + ``marshal`` reject it: give them ``dict(message)``.""" values = _record(self, variables, kwargs) serial = next(_compiles) - _drop_history(values, serial) + _drop_history(values) client = get_client() # prompt_attributes_for cannot record the variables when input capture is off in the client. kept = values if client is None or client.capture_input else None @@ -318,8 +315,8 @@ def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: The tag is on the dict that ``compile()`` made, so a copy of it gives ``{}``. Gives ``{}`` inside ``with_prompt``, whose prompt the span gets. ``td.prompt.variables`` needs input capture on in the client, at the compile and now, and the span drops it when its own input - capture is off. A compile whose messages became the history of a newer compile gives no - variables; see ``ChatPrompt.compile``. + capture is off. The compile of a previous turn of a chat history gives no variables; see + ``ChatPrompt.compile``. It reads only a ``list`` or ``tuple``, not a subclass, so the read runs no code of the app. With input capture on, it serializes the variables, which calls the ``mask`` of the client and @@ -474,7 +471,8 @@ class _CompileTag: prompt: ChatPrompt variables: dict[str, Any] | None - """None when input capture was off, or when the messages became a history; see _HISTORY_AGE.""" + """None when input capture was off, or when the messages became a history; see + _drop_history.""" serial: int """Counts the compiles of the process, so a higher one is newer.""" @@ -494,9 +492,23 @@ def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: return dict, (dict(self),) -def _drop_history(values: Mapping[str, Any], serial: int) -> None: - """Drops the variables of each older compile whose message is in ``values``; see - _HISTORY_AGE.""" +def _drop_history(values: Mapping[str, Any]) -> None: + """When a compile gets a message of a previous compile in ``values``, the compiles whose + messages that previous compile got are the history of a chat and remove their variables. The + previous compile keeps its own, because its model call can come after this compile, as with a + guard prompt. Thus a chain of tags with variables is at most two tags long, for a history of + any length.""" + for tag in _tags_in(values): + # One read: a compile in another thread can remove the variables of the tag. + variables = tag.variables + if variables is not None: + for older in _tags_in(variables): + older.variables = None + + +def _tags_in(values: Mapping[str, Any]) -> Generator[_CompileTag]: + """The tags of the messages in ``values``: a value, or an item of a list or tuple in a value. + Exact types, so the read runs no code of the app.""" for value in values.values(): items: Sequence[object] = ( cast("Sequence[object]", value) @@ -506,8 +518,8 @@ def _drop_history(values: Mapping[str, Any], serial: int) -> None: for item in items: if type(item) is _CompiledMessage: tag = getattr(item, "compiled_by", None) - if tag is not None and serial - tag.serial >= _HISTORY_AGE: - tag.variables = None + if tag is not None: + yield tag def _newest_tag(messages: object) -> _CompileTag | None: diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 324b01e..7b08c51 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -1000,13 +1000,6 @@ def test_a_message_that_a_placeholder_brings_from_an_older_compile_does_not_win( ), } assert prompt_attributes_for(summarized)["td.prompt.name"] == "summary" - # A newer compile that takes a message of a recent compile does not drop its variables. - assert prompt_attributes_for(question) == { - "td.prompt.name": "router", - "td.prompt.version": 12, - "td.prompt.label": "production", - "td.prompt.variables": '{"question":"Where is #4?"}', - } def test_prompt_attributes_for_looks_at_the_first_and_the_last_1000_messages( @@ -1234,8 +1227,45 @@ def test_with_input_capture_off_compiled_messages_keep_no_variables(make: MakeCl } -def test_a_chat_history_of_compiled_messages_keeps_the_variables_of_a_bounded_number_of_turns( +# How a chat keeps the compiled question and the answer of a turn for the next turn. +def _new_list(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: + return [*kept, question, answer] + + +def _list_trimmed_in_place(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: + kept.extend((question, answer)) + del kept[:-20] + return kept + + +def _new_list_of_20(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: + return [*kept, question, answer][-20:] + + +def _tuple_of_20(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: + return (*kept, question, answer)[-20:] + + +def _last_question(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: + return question + + +@pytest.mark.parametrize( + ("variable", "first", "keep"), + [ + ("history", [], _new_list), + ("history", [], _list_trimmed_in_place), + ("history", [], _new_list_of_20), + ("history", (), _tuple_of_20), + ("previous", None, _last_question), + ], + ids=["new-list", "list-trimmed-in-place", "new-list-of-20", "tuple-of-20", "not-in-a-list"], +) +def test_a_chat_history_keeps_the_variables_of_its_last_two_turns( make: MakeClient, + variable: str, + first: Any, + keep: Callable[[Any, dict[str, Any], dict[str, Any]], Any], ) -> None: make() prompt = chat_prompt( @@ -1244,29 +1274,57 @@ def test_a_chat_history_of_compiled_messages_keeps_the_variables_of_a_bounded_nu {"role": "user", "content": "{{question}}"}, ) turns: list[weakref.ref[Document]] = [] - history: list[dict[str, Any]] = [] + kept = copy.copy(first) last: list[dict[str, Any]] = [] + older: dict[str, Any] = {} + + # An unused variable marks the variables of each turn. + for turn in range(300): + marker = Document(content=f"turn {turn}") + turns.append(weakref.ref(marker)) + last = prompt.compile( + {"team": "billing", "question": f"question {turn}", "marker": marker, variable: kept} + ) + if turn == 297: + older = last[-1] + kept = keep(kept, last[-1], {"role": "assistant", "content": f"answer {turn}"}) + gc.collect() - # Each turn keeps its compiled question in the history. An unused variable marks the variables - # of the turn. - def chat(count: int) -> int: - nonlocal history, last - for _ in range(count): - marker = Document(content=f"turn {len(turns)}") - turns.append(weakref.ref(marker)) - last = prompt.compile( - team="billing", question=f"question {len(turns)}", history=history, marker=marker - ) - history = [*history, last[-1], {"role": "assistant", "content": f"answer {len(turns)}"}] - gc.collect() - return sum(1 for turn in turns if turn() is not None) + assert sum(1 for turn in turns if turn() is not None) == 2 + assert '"question":"question 299"' in str(prompt_attributes_for(last)["td.prompt.variables"]) + assert prompt_attributes_for(older) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } - kept = chat(150) - assert kept < 150 - assert chat(150) == kept - variables = json.loads(str(prompt_attributes_for(last)["td.prompt.variables"])) - assert len(variables["history"]) == 598 +def test_a_guard_prompt_keeps_the_variables_of_its_request_also_after_other_compiles( + make: MakeClient, +) -> None: + make() + answer = chat_prompt(*RAG_MESSAGES) + guard = chat_prompt( + {"role": "system", "content": "Is this safe?"}, + {"role": "placeholder", "name": "conversation"}, + name="guard", + ) + other = chat_prompt({"role": "user", "content": "{{q}}"}, name="other") + variables = '{"documents":"The refund policy","question":"Where is #4?"}' + + messages = answer.compile(documents="The refund policy", question="Where is #4?") + # Other requests of the server compile before this request runs its guard. + for index in range(150): + other.compile(q=str(index)) + guard.compile(conversation=messages) + assert prompt_attributes_for(messages)["td.prompt.variables"] == variables + + # An agent loop runs the guard before each step. + conversation = list(messages) + for step in range(150): + guard.compile(conversation=conversation) + conversation.append({"role": "assistant", "content": f"step {step}"}) + assert prompt_attributes_for(conversation)["td.prompt.variables"] == variables def test_a_fallback_prompt_records_its_name_and_label_but_no_version( From 1b099167bbd8acaf5008848ad9ba2a2ab8e35b0d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:25:02 +0000 Subject: [PATCH 17/32] fix(sdk): keep compiled variables for 5 minutes, not two turns With the last commit, a compile C removed the variables of a compile A when C got a message of a compile B, and B got a message of A. An inspection of this rule found four errors: - A chat that keeps the messages of a compile with no compiled messages in its variables kept the variables of each turn. Examples are a question prompt in a placeholder and a reply prompt for each answer. - Other code removed the variables of a shared message. A system message from startup had no variables after two turns of a chat that got it. - Two compiles before a call removed the variables of the request: a guard and then a judge of the guard, or turns of one session that add their question to the history before their calls. - The docs told the reader that a call with only messages of previous turns records no variables. But the call records the variables of the newest of them. A rule that looks at which compile got which message has one of these errors. A shared system message and a message of a chat history have the same shape. Thus a tag keeps its variables for 5 minutes after its compile. A call keeps them for 5 more minutes when the newest compile of the call is that compile. Other compiles do not change this time. After this time, a call records the prompt without td.prompt.variables. Each compile and each call removes the variables of the expired tags, also when the app keeps their messages. The SDKs find these tags by the minute of their time, through a weak reference. Thus the reference does not keep a tag in memory, and a clock change does not stop the minutes after it. In TypeScript, a runtime without WeakRef (Cloudflare Workers before 2025-05-05) keeps no variables in the tag. At one turn in 2 minutes, five chat shapes keep the variables of 3 of 300 turns in the two SDKs. A loop with no time between its compiles keeps all of their variables for 5 minutes. The shared system message, the judge of a guard, and the turns that share a history keep their variables. The tests do these checks, also with threads in Python. The READMEs, docs/sdk-conformance.md, and the docstrings tell the new rule. Refs #97 --- docs/sdk-conformance.md | 25 +- packages/sdk/README.md | 20 +- packages/sdk/src/prompts.ts | 110 +++--- packages/sdk/tests/prompts.test.ts | 410 +++++++++++++++++----- sdks/python/README.md | 20 +- sdks/python/src/telemetry_dev/_prompts.py | 99 +++--- sdks/python/tests/test_prompts.py | 339 ++++++++++++++---- 7 files changed, 759 insertions(+), 264 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 49e27fe..d0ed857 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -211,16 +211,21 @@ their text: at the call. Then they serialize the variables, which calls the `mask` of the client. A span whose effective captureInput is off drops `td.prompt.variables` from its attributes, also when the app passes the result of these functions to its own span. -- A tag keeps the variables of its compile alive while a message of the compile lives. A chat - compile C can get a tagged message of a compile B as a variable, or as an item of a list - (Python: a list or tuple) in a variable. Then each compile A whose message B got in the same way - removes its variables, and B keeps its own. Thus a chat history that keeps a compiled message of - each turn keeps the variables of the last two turns, for a history of any length or window, and - a request with only messages of previous turns records no `td.prompt.variables`. A guard prompt - that gets the messages of a request before its call does not remove their variables, also after - other compiles in the process. The SDKs do not look deeper in a variable. They keep no other - reference to the variables, except those of the last compile of each prompt object and of each - running scope, for `withPrompt` / `with_prompt`. +- A tag keeps the variables of its compile for 5 minutes. A call of these functions keeps them + for 5 more minutes when the newest compile of the call is that compile. After that time, + they give the attributes of the tag without `td.prompt.variables`, and the next compile or call + removes the variables from the tag, at most one minute late, also when the app keeps the + message. Other compiles do not change this time. Thus a chat history keeps the variables of the + compiles of the last 5 minutes, for a history of any length or window. A guard prompt, a queue, + or a compile in other code does not remove the variables of a request before a call in these 5 + minutes. A message that more than one request sends keeps its variables while a call names its + compile at least one time in 5 minutes. A request with only messages of earlier compiles gives + the variables of the newest of them, when they are less than 5 minutes old. +- The SDKs keep a weak reference to each tag with variables, by the minute of its time, thus the + reference does not keep a tag in memory. In a runtime without `WeakRef` (Cloudflare Workers with a + compatibility date before 2025-05-05, or with `disable_weak_ref`), the TypeScript tag keeps no + variables. The SDKs keep no other reference to the variables, except those of the last compile + of each prompt object and of each running scope, for `withPrompt` / `with_prompt`. - `compile()` of a text prompt returns a string, which cannot carry a tag. Outside a scope, the SDKs do not record a text prompt: use `withPrompt` / `with_prompt`. - The provider integrations call them with the request messages before the span starts, also diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 9edf709..7c42fb5 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -198,16 +198,16 @@ Google GenAI `generateContent`, and OpenRouter chat and responses, but not for B (its body is bytes), embeddings, images, audio, video, or OpenAI Realtime. With input capture on in the client when `compile()` runs, a tagged message keeps the variables of -its `compile()` alive while the message lives. With input capture off, the tag keeps no variables. -When `compile()` gets a tagged message in a variable, or in a list in a variable, the `compile()` -that made the message keeps its variables. But each `compile()` whose message that `compile()` got -removes its variables. For a chat history in a placeholder, turn 3 thus removes the variables of -turn 1 and keeps those of turn 2. A chat history of compiled messages keeps the variables of its -last two turns, also when you cut it, and a call that sends only messages of previous turns records -its prompt without `td.prompt.variables`. A guard prompt that gets the messages before their call -does not remove their variables. The SDK does not look deeper in a variable for messages. If you -keep compiled messages and do not give them to `compile()`, each one keeps the variables of its -`compile()`: keep copies (`{ ...message }`) to free them. +its `compile()` for 5 minutes. A call keeps them for 5 more minutes when the newest `compile()` of +the call is that one. After that time, a call records the prompt without `td.prompt.variables`, +and the SDK removes the variables from the tag, also when you keep the message. Other compiles do +not change this time. Thus a chat history keeps the variables of the compiles of its last 5 +minutes, and a guard prompt or a queue before the call does not remove them. A message that you +compile one time and send in many calls keeps its variables while a call names its `compile()` at +least one time in 5 minutes. A call that sends only messages of earlier turns records the prompt of +the newest `compile()` among them, and its variables when they are less than 5 minutes old. With +input capture off, or in a runtime without `WeakRef` (Cloudflare Workers with a compatibility date +before 2025-05-05), the tag keeps no variables. For your own spans, `promptAttributesFor(messages)` gives the attributes to pass as `attributes`. It gives nothing inside `withPrompt`, and a span with `captureInput: false` drops diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 55169d5..6f348ea 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -99,11 +99,10 @@ export interface ChatPrompt extends PromptBase { * variable. `withPrompt` records the variables of the last call before it starts or inside it. * It tags each message object that it makes with this prompt and a copy of these variables and * of each list in them, so that a model call outside `withPrompt` that sends the message records - * them; see `promptAttributesFor`. The objects in a list are not copied. The tag keeps no - * variables when input capture is off in the client. When a newer compile gets one of these - * messages in a variable, the compiles whose messages this compile got remove their variables, - * and this compile keeps its own. Thus a chat history keeps the variables of its last two turns, - * and a guard prompt that gets the messages before their call does not remove them. + * them; see `promptAttributesFor`. The objects in a list are not copied. The tag keeps the + * variables for 5 minutes after this compile and after each call that names it. Other compiles + * do not change this. The tag keeps no variables when input capture is off in the client, or + * when the runtime has no `WeakRef`. */ compile(variables?: PromptVariables): (ChatMessage | M)[]; } @@ -209,12 +208,12 @@ interface PromptScope { /** The `compile()` call that made a chat message. */ interface CompileTag { readonly prompt: Prompt; - /** - * Undefined when input capture was off, or when the messages became a history; see dropHistory. - */ + /** Undefined when input capture was off, or after `expires`; see expire. */ variables: PromptVariables | undefined; /** Counts the compiles of the process, so a higher one is newer. */ readonly serial: number; + /** When the tag removes its variables (Date.now). A call that names the tag moves it. */ + expires: number; } const FETCH_TIMEOUT_MS = 10_000; @@ -232,12 +231,21 @@ const PROMPT_KEY = createContextKey("telemetry.dev prompt"); const COMPILED_BY = Symbol("telemetry.dev compiled by"); /** How many messages promptAttributesFor looks at, at each end of the list. */ const TAG_LOOKUP = 1_000; +/** How long a tag keeps its variables after its compile, or after a call that names it. */ +const VARIABLES_TTL_MS = 300_000; +const MINUTE_MS = 60_000; const cache = new Map(); const compiledVariables = new WeakMap(); /** The Retry-After of the response that failed with this error, in milliseconds. */ const retryAfter = new WeakMap(); let compiles = 0; +/** + * The tags that keep variables, by the minute of their deadline. A weak reference, so that the + * variables of a message that the app dropped do not stay until the deadline. A minute that a + * clock change puts in the future does not stop the minutes after it. + */ +const deadlines = new Map[]>(); /** * Fetches a prompt version from telemetry.dev and keeps it. When the copy is older than @@ -383,8 +391,8 @@ export function promptAttributes( * the messages. It looks at the first 1000 and the last 1000 messages. The tag is on the message * object, so a copy of it gives nothing. Gives undefined inside `withPrompt`, whose prompt the span * gets. `td.prompt.variables` needs input capture on in the client, at the compile and now, and - * the span drops it when its own input capture is off. The compile of a previous turn of a chat - * history gives no variables; see `compile()`. It never throws. + * the span drops it when its own input capture is off. A compile gives no variables 5 minutes + * after it or after the last call that named it; see `compile()`. It never throws. */ export function promptAttributesFor(messages: unknown): Record | undefined { const config = currentClient().core?.config; @@ -392,10 +400,15 @@ export function promptAttributesFor(messages: unknown): Record now) tag.expires = now + VARIABLES_TTL_MS; + else tag.variables = undefined; + return attributesOf(tag.prompt, tag.variables, config) as Record; } catch (error) { reportError(config?.onError, error); @@ -638,21 +651,60 @@ function createPrompt(input: { if (type === "text") return fill(content.text ?? "", values); compiles += 1; - dropHistory(values); - // promptAttributesFor cannot record the variables when input capture is off in the client. - const captureInput = currentClient().core?.config.captureInput ?? true; - - return compileMessages(content.messages, values, { + const now = Date.now(); + expire(now); + const tag: CompileTag = { prompt, - variables: captureInput ? values : undefined, + variables: undefined, serial: compiles, - }); + expires: now + VARIABLES_TTL_MS, + }; + + // promptAttributesFor cannot record the variables when input capture is off in the client. + // Without WeakRef (Cloudflare Workers before 2025-05-05), expire could not remove them. + if ((currentClient().core?.config.captureInput ?? true) && typeof WeakRef === "function") { + tag.variables = values; + addDeadline(new WeakRef(tag), tag.expires); + } + + return compileMessages(content.messages, values, tag); }, }) as Prompt; return prompt; } +/** + * Removes the variables of each expired tag, also when the app keeps its messages. Thus a chat + * history keeps the variables of the last 5 minutes only, and other compiles do not change the time + * of a request. A tag that a call named gets its new deadline. + */ +function expire(now: number): void { + const minute = Math.floor(now / MINUTE_MS); + + for (const [due, refs] of deadlines) { + if (due >= minute) continue; + deadlines.delete(due); + + for (const ref of refs) { + const tag = ref.deref(); + + if (!tag) continue; + + if (tag.expires > now) addDeadline(ref, tag.expires); + else tag.variables = undefined; + } + } +} + +function addDeadline(ref: WeakRef, at: number): void { + const minute = Math.floor(at / MINUTE_MS); + const refs = deadlines.get(minute); + + if (refs) refs.push(ref); + else deadlines.set(minute, [ref]); +} + // A copy of each list too, so that a change of the app's history after compile() does not change // what a span records. function copyVariables(variables: PromptVariables): PromptVariables { @@ -664,30 +716,6 @@ function copyVariables(variables: PromptVariables): PromptVariables): void { - for (const tag of tagsIn(variables)) { - if (tag.variables) for (const older of tagsIn(tag.variables)) older.variables = undefined; - } -} - -/** The tags of the messages in `variables`: a value, or an item of a list in a value. */ -function* tagsIn(variables: PromptVariables): Generator { - for (const value of Object.values(variables)) { - for (const item of Array.isArray(value) ? value : [value]) { - const tag = tagOf(item); - - if (tag) yield tag; - } - } -} - function compileMessages( messages: readonly PromptMessage[], variables: PromptVariables, diff --git a/packages/sdk/tests/prompts.test.ts b/packages/sdk/tests/prompts.test.ts index 381df5a..81f712c 100644 --- a/packages/sdk/tests/prompts.test.ts +++ b/packages/sdk/tests/prompts.test.ts @@ -11,7 +11,6 @@ import { getPrompt, type Prompt, promptAttributesFor, - type PromptVariables, propagateAttributes, PromptError, shutdown, @@ -114,6 +113,7 @@ afterEach(async () => { await shutdown(); vi.useRealTimers(); vi.unstubAllEnvs(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); }); @@ -1007,15 +1007,27 @@ test("the variables of a compile are freed with its messages", async () => { expect(freed.deref()).toBeUndefined(); }); -test("with input capture off, compiled messages keep no variables", async () => { - setup({ captureInput: false }); +// A row removes the global WeakRef, so the test keeps its own. +const TrackedRef = WeakRef; + +test.each<[string, () => void]>([ + ["input capture is off", () => setup({ captureInput: false })], + [ + "the runtime has no WeakRef (Cloudflare Workers before 2025-05-05)", + () => { + setup(); + vi.stubGlobal("WeakRef", undefined); + }, + ], +])("when %s, compiled messages keep no variables", async (_case, start) => { + start(); const prompt = await promptOf(RAG_PROMPT); const documents: WeakRef[] = []; const questions: ChatMessage[] = []; for (let turn = 0; turn < 200; turn += 1) { const document: ChatMessage = { role: "user", content: `document ${turn}` }; - documents.push(new WeakRef(document)); + documents.push(new TrackedRef(document)); const compiled = prompt.compile({ documents: [document], question: `question ${turn}` }); questions.push(compiled.at(-1) as ChatMessage); } @@ -1025,6 +1037,7 @@ test("with input capture off, compiled messages keep no variables", async () => expect(documents.filter((document) => document.deref()).length).toBe(1); await shutdown(); + vi.unstubAllGlobals(); setup(); expect(promptAttributesFor(questions)).toEqual({ "td.prompt.name": "support/reply", @@ -1033,101 +1046,328 @@ test("with input capture off, compiled messages keep no variables", async () => }); }); -// Each row keeps the compiled question and the answer of a turn for the next turn: the variable -// that gets them, and the value of that variable after the turn. -test.each<[string, string, (kept: unknown, question: object, answer: object) => unknown]>([ - ["a new list each turn", "history", (kept, ...turn) => [...(kept as object[]), ...turn]], +const CHAT_PROMPT = { + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], +}; + +interface ChatPrompts { + chat: Prompt; + question: Prompt; + rag: Prompt; + reply: Prompt; +} + +async function chatPrompts(): Promise { + return { + chat: await promptOf(CHAT_PROMPT), + question: await promptOf({ + messages: [{ role: "user", content: "Context: {{documents}}\n\n{{question}}" }], + }), + rag: await promptOf({ + messages: [ + { role: "system", content: "Answer as the {{team}} desk." }, + { role: "placeholder", name: "history" }, + { role: "placeholder", name: "current" }, + ], + }), + reply: await promptOf({ messages: [{ role: "assistant", content: "{{answer}}" }] }), + }; +} + +function answer(turn: number): ChatMessage { + return { role: "assistant", content: `answer ${turn}` }; +} + +// A row runs one turn of a chat. It puts `marker` in the variables of the compile that made the +// message that the chat keeps. It gives the request, the history for the next turn, and that message. +type ChatTurn = ( + prompts: ChatPrompts, + history: ChatMessage[], + turn: number, + marker: ChatMessage, +) => [request: ChatMessage[], history: ChatMessage[], kept: ChatMessage]; + +test.each<[string, ChatTurn]>([ [ - "one list trimmed in place to 20 messages", - "history", - (kept, ...turn) => { - const history = kept as object[]; - history.push(...turn); - history.splice(0, history.length - 20); - - return history; + "a new list each turn", + ({ chat }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)], kept]; }, ], [ "a new list of the last 20 messages", - "history", - (kept, ...turn) => [...(kept as object[]), ...turn].slice(-20), + ({ chat }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)].slice(-20), kept]; + }, ], - ["the last question, not in a list", "previous", (_kept, question) => question], -])( - "a chat history in %s keeps the variables of its last two turns", - async (_shape, variable, next) => { - setup(); - const prompt = await promptOf({ - messages: [ - { role: "system", content: "Answer as the {{team}} desk." }, - { role: "placeholder", name: "history" }, - { role: "user", content: "{{question}}" }, - ], - }); - const turns: WeakRef[] = []; - let kept: unknown = variable === "history" ? [] : undefined; - let last: ChatMessage[] = []; - let older: ChatMessage | undefined; - - // An unused variable marks the variables of each turn. - for (let turn = 0; turn < 300; turn += 1) { - const marker: ChatMessage = { role: "user", content: `turn ${turn}` }; - turns.push(new WeakRef(marker)); - last = prompt.compile({ + [ + "a full list that compile gets as copies", + ({ chat }, history, turn, marker) => { + const request = chat.compile({ team: "billing", + history: history.map((message) => ({ ...message })), question: `question ${turn}`, marker: [marker], - [variable]: kept, - } as PromptVariables) as ChatMessage[]; - const question = last.at(-1)!; - - if (turn === 297) older = question; - kept = next(kept, question, { role: "assistant", content: `answer ${turn}` }); - } - await collect(); - - expect(turns.filter((turn) => turn.deref()).length).toBe(2); - expect(promptAttributesFor(last)?.["td.prompt.variables"]).toContain( - '"question":"question 299"', - ); - expect(promptAttributesFor(older)).toEqual({ - "td.prompt.name": "support/reply", - "td.prompt.version": 12, - "td.prompt.label": "production", - }); - }, -); + }) as ChatMessage[]; + const kept = request.at(-1)!; + + return [request, [...history, kept, answer(turn)], kept]; + }, + ], + [ + "a question prompt for each turn, in a placeholder of the chat prompt", + ({ question, rag }, history, turn, marker) => { + const [kept] = question.compile({ + documents: `document ${turn}`, + question: `question ${turn}`, + marker: [marker], + }) as ChatMessage[]; + const request = rag.compile({ team: "billing", history, current: [kept!] }) as ChatMessage[]; + + return [request, [...history, kept!, answer(turn)], kept!]; + }, + ], + [ + "raw questions and a reply prompt for each answer", + ({ chat, reply }, history, turn, marker) => { + const request = chat.compile({ + team: "billing", + history, + question: `question ${turn}`, + }) as ChatMessage[]; + const [kept] = reply.compile({ answer: `answer ${turn}`, marker: [marker] }) as ChatMessage[]; + + return [request, [...history, { role: "user", content: `question ${turn}` }, kept!], kept!]; + }, + ], +])("a chat history in %s keeps the variables of the last 5 minutes", async (_shape, run) => { + vi.useFakeTimers({ toFake: ["Date"] }); + setup(); + const prompts = await chatPrompts(); + const markers: WeakRef[] = []; + let history: ChatMessage[] = []; + let request: ChatMessage[] = []; + let older: ChatMessage | undefined; + + // One turn in 2 minutes, for 300 turns. An unused variable marks the variables of each turn. + for (let turn = 0; turn < 300; turn += 1) { + if (turn > 0) vi.advanceTimersByTime(120_000); + const marker: ChatMessage = { role: "user", content: `turn ${turn}` }; + markers.push(new WeakRef(marker)); + let kept: ChatMessage; + [request, history, kept] = run(prompts, history, turn, marker); + // The model call of the turn. + promptAttributesFor(request); + + if (turn === 296) older = kept; + } + await collect(); + + // Turns 297, 298, and 299: 4, 2, and 0 minutes old. + expect(markers.filter((marker) => marker.deref()).length).toBe(3); + expect(promptAttributesFor(request)?.["td.prompt.variables"]).toContain("question 299"); + expect(promptAttributesFor(older)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); +}); + +function variablesOf(messages: unknown): unknown { + return promptAttributesFor(messages)?.["td.prompt.variables"]; +} + +const REFUND_VARIABLES = '{"documents":"The refund policy","question":"Where is #4?"}'; + +interface LivePrompts { + answer: Prompt; + guard: Prompt; + judge: Prompt; + other: Prompt; + chat: Prompt; + system: Prompt; + withSystem: Prompt; +} -test("a guard prompt keeps the variables of its request, also after other compiles", async () => { +// A row compiles one or more requests and runs other compiles before their calls. It gives the +// variables that each call records and the variables of the compile of its request. +type LiveRequests = (prompts: LivePrompts) => Promise<[recorded: unknown, compiled: string][]>; + +test.each<[string, LiveRequests]>([ + [ + "150 compiles of other requests, then a guard", + async ({ answer, guard, other }) => { + const request = answer.compile({ documents: "The refund policy", question: "Where is #4?" }); + for (let index = 0; index < 150; index += 1) other.compile({ q: `${index}` }); + guard.compile({ conversation: request }); + + return [[variablesOf(request), REFUND_VARIABLES]]; + }, + ], + [ + "a guard before each of 150 agent steps", + async ({ answer, guard }) => { + const conversation = answer.compile({ + documents: "The refund policy", + question: "Where is #4?", + }) as ChatMessage[]; + + for (let step = 0; step < 150; step += 1) { + guard.compile({ conversation }); + conversation.push({ role: "assistant", content: `step ${step}` }); + } + + return [[variablesOf(conversation), REFUND_VARIABLES]]; + }, + ], + [ + "a guard, then a judge of the messages of the guard", + async ({ answer, guard, judge }) => { + const request = answer.compile({ documents: "The refund policy", question: "Where is #4?" }); + judge.compile({ conversation: guard.compile({ conversation: request }) }); + + return [[variablesOf(request), REFUND_VARIABLES]]; + }, + ], + [ + "three turns of a session that add their question to the history before their calls", + async ({ chat }) => { + const history: ChatMessage[] = []; + let open!: () => void; + const queue = new Promise((resolve) => { + open = resolve; + }); + const turn = async (question: string) => { + const request = chat.compile({ team: "billing", history, question }) as ChatMessage[]; + history.push(request.at(-1)!); + await queue; + + return variablesOf(request); + }; + const turns = ["q1", "q2", "q3"].map(turn); + open(); + const [first, second, third] = await Promise.all(turns); + + return [ + [first, '{"team":"billing","history":[],"question":"q1"}'], + [second, '{"team":"billing","history":[{"role":"user","content":"q1"}],"question":"q2"}'], + [ + third, + '{"team":"billing","history":[{"role":"user","content":"q1"},{"role":"user","content":"q2"}],"question":"q3"}', + ], + ]; + }, + ], + [ + "a system message compiled at startup, after 3 turns of a chat that gets it", + async ({ system, withSystem }) => { + const startup = system.compile({ product: "Acme" }) as ChatMessage[]; + let history: ChatMessage[] = []; + + for (let turn = 0; turn < 3; turn += 1) { + const request = withSystem.compile({ + system: startup, + history, + question: `question ${turn}`, + }) as ChatMessage[]; + promptAttributesFor(request); + history = [...history, request.at(-1)!, answer(turn)]; + } + + return [ + [ + variablesOf([...startup, { role: "user", content: "Where is #4?" }]), + '{"product":"Acme"}', + ], + ]; + }, + ], +])("a request keeps its variables until its call: %s", async (_case, run) => { setup(); - const answer = await promptOf(RAG_PROMPT); - const guard = await promptOf({ - name: "guard", - messages: [ - { role: "system", content: "Is this safe?" }, - { role: "placeholder", name: "conversation" }, - ], + const calls = await run({ + answer: await promptOf(RAG_PROMPT), + guard: await promptOf({ + name: "guard", + messages: [ + { role: "system", content: "Is this safe?" }, + { role: "placeholder", name: "conversation" }, + ], + }), + judge: await promptOf({ + name: "judge", + messages: [ + { role: "system", content: "Check the answer of the guard." }, + { role: "placeholder", name: "conversation" }, + ], + }), + other: await promptOf({ name: "other", messages: [{ role: "user", content: "{{q}}" }] }), + chat: await promptOf(CHAT_PROMPT), + system: await promptOf({ + name: "system", + messages: [{ role: "system", content: "You support {{product}}." }], + }), + withSystem: await promptOf({ + messages: [ + { role: "placeholder", name: "system" }, + { role: "placeholder", name: "history" }, + { role: "user", content: "{{question}}" }, + ], + }), }); - const other = await promptOf({ name: "other", messages: [{ role: "user", content: "{{q}}" }] }); - const variables = '{"documents":"The refund policy","question":"Where is #4?"}'; - const messages = answer.compile({ - documents: "The refund policy", - question: "Where is #4?", - }) as ChatMessage[]; - // Other requests of the server compile before this request runs its guard. - for (let index = 0; index < 150; index += 1) other.compile({ q: `${index}` }); - guard.compile({ conversation: messages }); - expect(promptAttributesFor(messages)?.["td.prompt.variables"]).toBe(variables); - - // An agent loop runs the guard before each step. - const conversation = [...messages]; - for (let step = 0; step < 150; step += 1) { - guard.compile({ conversation }); - conversation.push({ role: "assistant", content: `step ${step}` }); + for (const [recorded, compiled] of calls) expect(recorded).toBe(compiled); +}); + +test("a compile keeps its variables for 5 minutes after it or after the last call that names it", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + setup(); + const chat = await promptOf(CHAT_PROMPT); + let history: ChatMessage[] = []; + + for (let turn = 0; turn < 5; turn += 1) { + const request = chat.compile({ team: "billing", history, question: `q${turn}` }); + promptAttributesFor(request); + history = [...history, request.at(-1) as ChatMessage, answer(turn)]; } - expect(promptAttributesFor(conversation)?.["td.prompt.variables"]).toBe(variables); + // These calls send only messages of the chat, so they name the newest compile among them. + const summary = [...history, { role: "user", content: "Summarize the chat." }]; + const earlier = history.slice(0, -2); + + expect(variablesOf(summary)).toContain('"question":"q4"'); + expect(variablesOf(earlier)).toContain('"question":"q3"'); + + vi.advanceTimersByTime(299_999); + expect(variablesOf(summary)).toContain('"question":"q4"'); + vi.advanceTimersByTime(1); + expect(variablesOf(earlier)).toBeUndefined(); + // The call 1 ms before moved the deadline of q4. + expect(variablesOf(summary)).toContain('"question":"q4"'); + vi.advanceTimersByTime(300_000); + expect(promptAttributesFor(summary)).toEqual({ + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + }); }); test("withPrompt wins over a compiled message of another prompt, through a parent and in updates", async () => { diff --git a/sdks/python/README.md b/sdks/python/README.md index d060322..05b5657 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -278,16 +278,16 @@ pydantic) do not change. A check of the exact type (`type(message) is dict`), `y and `marshal` reject it: give them `dict(message)`. With input capture on in the client when `compile()` runs, a compiled message keeps the variables -of its `compile()` alive while the message lives. With input capture off, it keeps no variables. -When `compile()` gets a compiled message in a variable, or in a list or tuple in a variable, the -`compile()` that made the message keeps its variables. But each `compile()` whose message that -`compile()` got removes its variables. For a chat history in a placeholder, turn 3 thus removes the -variables of turn 1 and keeps those of turn 2. A chat history of compiled messages keeps the -variables of its last two turns, also when you cut it, and a call that sends only messages of -previous turns records its prompt without `td.prompt.variables`. A guard prompt that gets the -messages before their call does not remove their variables. The SDK does not look deeper in a -variable for messages. If you keep compiled messages and do not give them to `compile()`, each one -keeps the variables of its `compile()`: keep copies (`dict(message)`) to free them. +of its `compile()` for 5 minutes. A call keeps them for 5 more minutes when the newest `compile()` +of the call is that one. After that time, a call records the prompt without +`td.prompt.variables`, and the SDK removes the variables from the message, also when you keep the +message. Other compiles do not change this time. Thus a chat history keeps the variables of the +compiles of its last 5 minutes, and a guard prompt, a queue, or a compile in another thread before +the call does not remove them. A message that you compile one time and send in many calls keeps its +variables while a call names its `compile()` at least one time in 5 minutes. A call that sends only +messages of earlier turns records the prompt of the newest `compile()` among them, and its +variables when they are less than 5 minutes old. With input capture off, the message keeps no +variables. For your own spans, `prompt_attributes_for(messages)` gives the attributes to pass as `attributes`. It gives `{}` inside `with_prompt`, and a span with `capture_input=False` drops diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 4811f14..f2177cc 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -14,7 +14,7 @@ import urllib.parse import urllib.request from collections import OrderedDict -from collections.abc import Callable, Generator, Mapping, Sequence +from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field @@ -23,7 +23,7 @@ from email.utils import parsedate_to_datetime from time import monotonic from typing import IO, Any, Literal, TypeAlias, TypedDict, cast, overload -from weakref import WeakKeyDictionary +from weakref import WeakKeyDictionary, ref from opentelemetry import context as otel_context from opentelemetry.context import Context @@ -53,6 +53,9 @@ _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") # How many messages prompt_attributes_for looks at, at each end of the list. _TAG_LOOKUP = 1000 +# How long a tag keeps its variables after its compile, or after a call that names it. +_VARIABLES_TTL_S = 300.0 +_MINUTE_S = 60.0 class PromptModel(TypedDict): @@ -104,20 +107,20 @@ def compile( or inside it. Each message that it makes is a ``dict`` subclass that also holds this prompt and a copy of these variables and of each list in them, so that a model call outside ``with_prompt`` that sends the message records them; see - ``prompt_attributes_for``. The objects in a list are not copied. The message holds no - variables when input capture is off in the client. When a newer compile gets one of these - messages in a variable, the compiles whose messages this compile got remove their - variables, and this compile keeps its own. Thus a chat history keeps the variables of its - last two turns, and a guard prompt that gets the messages before their call does not - remove them. A check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and + ``prompt_attributes_for``. The objects in a list are not copied. The message keeps the + variables for 5 minutes after this compile and after each call that names it. Other + compiles do not change this. The message holds no variables when input capture is off in + the client. A check of the exact type (``type(message) is dict``), ``yaml.safe_dump``, and ``marshal`` reject it: give them ``dict(message)``.""" values = _record(self, variables, kwargs) - serial = next(_compiles) - _drop_history(values) + now = monotonic() + _expire(now) + tag = _CompileTag(self, None, next(_compiles), now + _VARIABLES_TTL_S) client = get_client() # prompt_attributes_for cannot record the variables when input capture is off in the client. - kept = values if client is None or client.capture_input else None - tag = _CompileTag(self, kept, serial) + if client is None or client.capture_input: + tag.variables = values + _add_deadline(ref(tag), tag.expires) compiled: list[dict[str, Any]] = [] for message in self.messages: if message.get("role") != "placeholder": @@ -315,8 +318,8 @@ def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: The tag is on the dict that ``compile()`` made, so a copy of it gives ``{}``. Gives ``{}`` inside ``with_prompt``, whose prompt the span gets. ``td.prompt.variables`` needs input capture on in the client, at the compile and now, and the span drops it when its own input - capture is off. The compile of a previous turn of a chat history gives no variables; see - ``ChatPrompt.compile``. + capture is off. A compile gives no variables 5 minutes after it or after the last call that + named it; see ``ChatPrompt.compile``. It reads only a ``list`` or ``tuple``, not a subclass, so the read runs no code of the app. With input capture on, it serializes the variables, which calls the ``mask`` of the client and @@ -326,11 +329,19 @@ def prompt_attributes_for(messages: object) -> dict[str, AttributeValue]: try: if isinstance(otel_context.get_value(_PROMPT_KEY), _PromptScope): return {} + now = monotonic() + _expire(now) tag = _newest_tag(messages) if tag is None: return {} + # One read: _expire in another thread can remove the variables of the tag. + variables = tag.variables + if tag.expires > now: + tag.expires = now + _VARIABLES_TTL_S + else: + variables = tag.variables = None capture = client if client is not None and client.capture_input else None - return _attributes_of(tag.prompt, tag.variables, capture) + return _attributes_of(tag.prompt, variables, capture) except Exception as exc: if client is not None: client.report("prompt_attributes_for failed", exc) @@ -471,13 +482,17 @@ class _CompileTag: prompt: ChatPrompt variables: dict[str, Any] | None - """None when input capture was off, or when the messages became a history; see - _drop_history.""" + """None when input capture was off, or after ``expires``; see _expire.""" serial: int """Counts the compiles of the process, so a higher one is newer.""" + expires: float + """When the tag removes its variables (monotonic). A call that names the tag moves it.""" _compiles = itertools.count() +# The tags that keep variables, by the minute of their deadline. A weak reference, so that the +# variables of a message that the app dropped do not stay until the deadline. +_deadlines: dict[int, list[ref[_CompileTag]]] = {} class _CompiledMessage(dict[str, Any]): @@ -492,34 +507,28 @@ def __reduce__(self) -> tuple[type[dict[str, Any]], tuple[dict[str, Any]]]: return dict, (dict(self),) -def _drop_history(values: Mapping[str, Any]) -> None: - """When a compile gets a message of a previous compile in ``values``, the compiles whose - messages that previous compile got are the history of a chat and remove their variables. The - previous compile keeps its own, because its model call can come after this compile, as with a - guard prompt. Thus a chain of tags with variables is at most two tags long, for a history of - any length.""" - for tag in _tags_in(values): - # One read: a compile in another thread can remove the variables of the tag. - variables = tag.variables - if variables is not None: - for older in _tags_in(variables): - older.variables = None - - -def _tags_in(values: Mapping[str, Any]) -> Generator[_CompileTag]: - """The tags of the messages in ``values``: a value, or an item of a list or tuple in a value. - Exact types, so the read runs no code of the app.""" - for value in values.values(): - items: Sequence[object] = ( - cast("Sequence[object]", value) - if type(value) is list or type(value) is tuple - else (value,) - ) - for item in items: - if type(item) is _CompiledMessage: - tag = getattr(item, "compiled_by", None) - if tag is not None: - yield tag +def _expire(now: float) -> None: + """Remove the variables of each expired tag, also when the app keeps its messages. Thus a chat + history keeps the variables of the last 5 minutes only, and other compiles do not change the + time of a request. A tag that a call named gets its new deadline. Threads add only to minutes + that are not due, and ``list()`` and ``pop()`` of a dict are atomic, so the threads share the + minutes without a lock.""" + minute = int(now // _MINUTE_S) + for due in list(_deadlines): + if due >= minute: + continue + for tag_ref in _deadlines.pop(due, ()): + tag = tag_ref() + if tag is None: + continue + if tag.expires > now: + _add_deadline(tag_ref, tag.expires) + else: + tag.variables = None + + +def _add_deadline(tag_ref: ref[_CompileTag], at: float) -> None: + _deadlines.setdefault(int(at // _MINUTE_S), []).append(tag_ref) def _newest_tag(messages: object) -> _CompileTag | None: diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 7b08c51..0cd3e31 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -19,7 +19,7 @@ from email.utils import format_datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from types import SimpleNamespace -from typing import Any, NoReturn +from typing import Any, NoReturn, TypeAlias from urllib.parse import parse_qsl, urlsplit import pytest @@ -1227,71 +1227,132 @@ def test_with_input_capture_off_compiled_messages_keep_no_variables(make: MakeCl } -# How a chat keeps the compiled question and the answer of a turn for the next turn. -def _new_list(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: - return [*kept, question, answer] +CHAT_MESSAGES = ( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, +) + + +@dataclasses.dataclass +class ChatPrompts: + chat: ChatPrompt + question: ChatPrompt + rag: ChatPrompt + reply: ChatPrompt -def _list_trimmed_in_place(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: - kept.extend((question, answer)) - del kept[:-20] - return kept +def _answer(turn: int) -> dict[str, Any]: + return {"role": "assistant", "content": f"answer {turn}"} -def _new_list_of_20(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: - return [*kept, question, answer][-20:] +# What a row gives for one turn of a chat: the request, the history for the next turn, and the kept +# message. The row puts `marker` in the variables of the compile that made the kept message. +ChatTurn: TypeAlias = tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]] -def _tuple_of_20(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: - return (*kept, question, answer)[-20:] +def _new_list( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", history=history, question=f"question {turn}", marker=marker + ) + return request, [*history, request[-1], _answer(turn)], request[-1] + +def _new_list_of_20( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", history=history, question=f"question {turn}", marker=marker + ) + return request, [*history, request[-1], _answer(turn)][-20:], request[-1] -def _last_question(kept: Any, question: dict[str, Any], answer: dict[str, Any]) -> Any: - return question + +def _full_list_as_copies( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile( + team="billing", + history=[dict(message) for message in history], + question=f"question {turn}", + marker=marker, + ) + return request, [*history, request[-1], _answer(turn)], request[-1] + + +def _question_prompt_in_a_placeholder( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + [kept] = p.question.compile( + documents=f"document {turn}", question=f"question {turn}", marker=marker + ) + request = p.rag.compile(team="billing", history=history, current=[kept]) + return request, [*history, kept, _answer(turn)], kept + + +def _raw_questions_and_a_reply_prompt( + p: ChatPrompts, history: list[dict[str, Any]], turn: int, marker: Document +) -> ChatTurn: + request = p.chat.compile(team="billing", history=history, question=f"question {turn}") + [kept] = p.reply.compile(answer=f"answer {turn}", marker=marker) + return request, [*history, {"role": "user", "content": f"question {turn}"}, kept], kept @pytest.mark.parametrize( - ("variable", "first", "keep"), + "run", [ - ("history", [], _new_list), - ("history", [], _list_trimmed_in_place), - ("history", [], _new_list_of_20), - ("history", (), _tuple_of_20), - ("previous", None, _last_question), + _new_list, + _new_list_of_20, + _full_list_as_copies, + _question_prompt_in_a_placeholder, + _raw_questions_and_a_reply_prompt, + ], + ids=[ + "new-list", + "new-list-of-20", + "full-list-as-copies", + "question-prompt-in-a-placeholder", + "raw-questions-and-a-reply-prompt", ], - ids=["new-list", "list-trimmed-in-place", "new-list-of-20", "tuple-of-20", "not-in-a-list"], ) -def test_a_chat_history_keeps_the_variables_of_its_last_two_turns( +def test_a_chat_history_keeps_the_variables_of_the_last_5_minutes( make: MakeClient, - variable: str, - first: Any, - keep: Callable[[Any, dict[str, Any], dict[str, Any]], Any], + clock: list[float], + run: Callable[[ChatPrompts, list[dict[str, Any]], int, Document], ChatTurn], ) -> None: make() - prompt = chat_prompt( - {"role": "system", "content": "Answer as the {{team}} desk."}, - {"role": "placeholder", "name": "history"}, - {"role": "user", "content": "{{question}}"}, + prompts = ChatPrompts( + chat=chat_prompt(*CHAT_MESSAGES), + question=chat_prompt({"role": "user", "content": "Context: {{documents}}\n\n{{question}}"}), + rag=chat_prompt( + {"role": "system", "content": "Answer as the {{team}} desk."}, + {"role": "placeholder", "name": "history"}, + {"role": "placeholder", "name": "current"}, + ), + reply=chat_prompt({"role": "assistant", "content": "{{answer}}"}), ) - turns: list[weakref.ref[Document]] = [] - kept = copy.copy(first) - last: list[dict[str, Any]] = [] + markers: list[weakref.ref[Document]] = [] + history: list[dict[str, Any]] = [] + request: list[dict[str, Any]] = [] older: dict[str, Any] = {} - # An unused variable marks the variables of each turn. + # One turn in 2 minutes, for 300 turns. An unused variable marks the variables of each turn. for turn in range(300): + if turn > 0: + clock[0] += 120 marker = Document(content=f"turn {turn}") - turns.append(weakref.ref(marker)) - last = prompt.compile( - {"team": "billing", "question": f"question {turn}", "marker": marker, variable: kept} - ) - if turn == 297: - older = last[-1] - kept = keep(kept, last[-1], {"role": "assistant", "content": f"answer {turn}"}) + markers.append(weakref.ref(marker)) + request, history, kept = run(prompts, history, turn, marker) + # The model call of the turn. + prompt_attributes_for(request) + if turn == 296: + older = kept gc.collect() - assert sum(1 for turn in turns if turn() is not None) == 2 - assert '"question":"question 299"' in str(prompt_attributes_for(last)["td.prompt.variables"]) + # Turns 297, 298, and 299: 4, 2, and 0 minutes old. + assert sum(1 for marker in markers if marker() is not None) == 3 + assert "question 299" in str(prompt_attributes_for(request)["td.prompt.variables"]) assert prompt_attributes_for(older) == { "td.prompt.name": "support/reply", "td.prompt.version": 12, @@ -1299,32 +1360,184 @@ def test_a_chat_history_keeps_the_variables_of_its_last_two_turns( } -def test_a_guard_prompt_keeps_the_variables_of_its_request_also_after_other_compiles( - make: MakeClient, -) -> None: - make() - answer = chat_prompt(*RAG_MESSAGES) - guard = chat_prompt( - {"role": "system", "content": "Is this safe?"}, - {"role": "placeholder", "name": "conversation"}, - name="guard", - ) - other = chat_prompt({"role": "user", "content": "{{q}}"}, name="other") - variables = '{"documents":"The refund policy","question":"Where is #4?"}' +def _variables_of(messages: object) -> object: + return prompt_attributes_for(messages).get("td.prompt.variables") + + +REFUND_VARIABLES = '{"documents":"The refund policy","question":"Where is #4?"}' - messages = answer.compile(documents="The refund policy", question="Where is #4?") - # Other requests of the server compile before this request runs its guard. + +@dataclasses.dataclass +class LivePrompts: + answer: ChatPrompt + guard: ChatPrompt + judge: ChatPrompt + other: ChatPrompt + chat: ChatPrompt + system: ChatPrompt + with_system: ChatPrompt + + +# A row compiles one or more requests and runs other compiles before their calls. It gives the +# variables that each call records and the variables of the compile of its request. +LiveCalls: TypeAlias = list[tuple[object, str]] + + +def _other_compiles_then_a_guard(p: LivePrompts) -> LiveCalls: + request = p.answer.compile(documents="The refund policy", question="Where is #4?") for index in range(150): - other.compile(q=str(index)) - guard.compile(conversation=messages) - assert prompt_attributes_for(messages)["td.prompt.variables"] == variables + p.other.compile(q=str(index)) + p.guard.compile(conversation=request) + return [(_variables_of(request), REFUND_VARIABLES)] - # An agent loop runs the guard before each step. - conversation = list(messages) + +def _a_guard_before_each_agent_step(p: LivePrompts) -> LiveCalls: + conversation = p.answer.compile(documents="The refund policy", question="Where is #4?") for step in range(150): - guard.compile(conversation=conversation) + p.guard.compile(conversation=conversation) conversation.append({"role": "assistant", "content": f"step {step}"}) - assert prompt_attributes_for(conversation)["td.prompt.variables"] == variables + return [(_variables_of(conversation), REFUND_VARIABLES)] + + +def _a_guard_then_a_judge_of_the_guard(p: LivePrompts) -> LiveCalls: + request = p.answer.compile(documents="The refund policy", question="Where is #4?") + p.judge.compile(conversation=p.guard.compile(conversation=request)) + return [(_variables_of(request), REFUND_VARIABLES)] + + +def _turns_in_threads_that_share_a_history(p: LivePrompts) -> LiveCalls: + history: list[dict[str, Any]] = [] + turns = [threading.Event() for _ in range(4)] + compiled = threading.Barrier(3, timeout=5) + recorded: list[object] = [None] * 3 + + def turn(index: int) -> None: + turns[index].wait(5) + request = p.chat.compile(team="billing", history=history, question=f"q{index + 1}") + history.append(request[-1]) + turns[index + 1].set() + # A guard call or a queue: the call comes after the other turns compiled. + compiled.wait() + recorded[index] = _variables_of(request) + + threads = [threading.Thread(target=turn, args=(index,)) for index in range(3)] + for thread in threads: + thread.start() + turns[0].set() + for thread in threads: + thread.join(5) + return list( + zip( + recorded, + [ + '{"team":"billing","history":[],"question":"q1"}', + '{"team":"billing","history":[{"role":"user","content":"q1"}],"question":"q2"}', + '{"team":"billing","history":[{"role":"user","content":"q1"},' + '{"role":"user","content":"q2"}],"question":"q3"}', + ], + strict=True, + ) + ) + + +def _a_system_message_from_startup_after_a_chat_that_gets_it(p: LivePrompts) -> LiveCalls: + startup = p.system.compile(product="Acme") + history: list[dict[str, Any]] = [] + for turn in range(3): + request = p.with_system.compile( + system=startup, history=history, question=f"question {turn}" + ) + prompt_attributes_for(request) + history = [*history, request[-1], _answer(turn)] + return [ + ( + _variables_of([*startup, {"role": "user", "content": "Where is #4?"}]), + '{"product":"Acme"}', + ) + ] + + +@pytest.mark.parametrize( + "run", + [ + _other_compiles_then_a_guard, + _a_guard_before_each_agent_step, + _a_guard_then_a_judge_of_the_guard, + _turns_in_threads_that_share_a_history, + _a_system_message_from_startup_after_a_chat_that_gets_it, + ], + ids=[ + "150-other-compiles-then-a-guard", + "a-guard-before-each-of-150-agent-steps", + "a-guard-then-a-judge-of-the-guard", + "turns-in-threads-that-share-a-history", + "a-system-message-from-startup-after-a-chat-that-gets-it", + ], +) +def test_a_request_keeps_its_variables_until_its_call( + make: MakeClient, run: Callable[[LivePrompts], LiveCalls] +) -> None: + make() + calls = run( + LivePrompts( + answer=chat_prompt(*RAG_MESSAGES), + guard=chat_prompt( + {"role": "system", "content": "Is this safe?"}, + {"role": "placeholder", "name": "conversation"}, + name="guard", + ), + judge=chat_prompt( + {"role": "system", "content": "Check the answer of the guard."}, + {"role": "placeholder", "name": "conversation"}, + name="judge", + ), + other=chat_prompt({"role": "user", "content": "{{q}}"}, name="other"), + chat=chat_prompt(*CHAT_MESSAGES), + system=chat_prompt( + {"role": "system", "content": "You support {{product}}."}, name="system" + ), + with_system=chat_prompt( + {"role": "placeholder", "name": "system"}, + {"role": "placeholder", "name": "history"}, + {"role": "user", "content": "{{question}}"}, + ), + ) + ) + + for recorded, compiled in calls: + assert recorded == compiled + + +def test_a_compile_keeps_its_variables_for_5_minutes_after_it_or_after_the_last_call_that_names_it( + make: MakeClient, clock: list[float] +) -> None: + make() + chat = chat_prompt(*CHAT_MESSAGES) + history: list[dict[str, Any]] = [] + for turn in range(5): + request = chat.compile(team="billing", history=history, question=f"q{turn}") + prompt_attributes_for(request) + history = [*history, request[-1], _answer(turn)] + # These calls send only messages of the chat, so they name the newest compile among them. + summary = [*history, {"role": "user", "content": "Summarize the chat."}] + earlier = history[:-2] + start = clock[0] + + assert '"question":"q4"' in str(_variables_of(summary)) + assert '"question":"q3"' in str(_variables_of(earlier)) + + clock[0] = start + 299.5 + assert '"question":"q4"' in str(_variables_of(summary)) + clock[0] = start + 300 + assert _variables_of(earlier) is None + # The call 0.5 seconds before moved the deadline of q4. + assert '"question":"q4"' in str(_variables_of(summary)) + clock[0] = start + 600 + assert prompt_attributes_for(summary) == { + "td.prompt.name": "support/reply", + "td.prompt.version": 12, + "td.prompt.label": "production", + } def test_a_fallback_prompt_records_its_name_and_label_but_no_version( From b2a0e09e3a2fee5aa8c5d6951a0bac9dac019cfa Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 01:14:04 +0000 Subject: [PATCH 18/32] fix(sdk): measure the variables time of compiled messages in elapsed time In #117, the prompt cache and the Retry-After wait moved to performance.now(), so that a change of the system clock does not move them. But a compile tag used Date.now() for the 5 minutes that it keeps its variables. Thus a clock that moved back an hour kept the variables of each compile for one more hour. A clock that moved forward removed the variables before 5 minutes. The Python tags use monotonic(). With this change, compile() and promptAttributesFor read performance.now(). The test of the 5 minutes moves the system clock back an hour, and the tag continues to remove the variables after 5 minutes. docs/sdk-conformance.md tells that the SDKs measure this time in elapsed time. Refs #97 --- docs/sdk-conformance.md | 3 ++- packages/sdk/src/prompts.ts | 13 ++++++++----- packages/sdk/tests/prompts.test.ts | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index d0ed857..54debda 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -212,7 +212,8 @@ their text: whose effective captureInput is off drops `td.prompt.variables` from its attributes, also when the app passes the result of these functions to its own span. - A tag keeps the variables of its compile for 5 minutes. A call of these functions keeps them - for 5 more minutes when the newest compile of the call is that compile. After that time, + for 5 more minutes when the newest compile of the call is that compile. The SDKs measure this + time in elapsed time, so a change of the system clock does not move it. After that time, they give the attributes of the tag without `td.prompt.variables`, and the next compile or call removes the variables from the tag, at most one minute late, also when the app keeps the message. Other compiles do not change this time. Thus a chat history keeps the variables of the diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 6f348ea..7837933 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -212,7 +212,10 @@ interface CompileTag { variables: PromptVariables | undefined; /** Counts the compiles of the process, so a higher one is newer. */ readonly serial: number; - /** When the tag removes its variables (Date.now). A call that names the tag moves it. */ + /** + * When the tag removes its variables, in `performance.now()` time. A call that names the tag + * moves it. + */ expires: number; } @@ -242,8 +245,8 @@ const retryAfter = new WeakMap(); let compiles = 0; /** * The tags that keep variables, by the minute of their deadline. A weak reference, so that the - * variables of a message that the app dropped do not stay until the deadline. A minute that a - * clock change puts in the future does not stop the minutes after it. + * variables of a message that the app dropped do not stay until the deadline. The minutes are not + * sorted, since a tag that a call named goes back in at its new minute. */ const deadlines = new Map[]>(); @@ -400,7 +403,7 @@ export function promptAttributesFor(messages: unknown): Record([ }, ], ])("a chat history in %s keeps the variables of the last 5 minutes", async (_shape, run) => { - vi.useFakeTimers({ toFake: ["Date"] }); + vi.useFakeTimers({ toFake: ["Date", "performance"] }); setup(); const prompts = await chatPrompts(); const markers: WeakRef[] = []; @@ -1339,7 +1339,7 @@ test.each<[string, LiveRequests]>([ }); test("a compile keeps its variables for 5 minutes after it or after the last call that names it", async () => { - vi.useFakeTimers({ toFake: ["Date"] }); + vi.useFakeTimers({ toFake: ["Date", "performance"] }); setup(); const chat = await promptOf(CHAT_PROMPT); let history: ChatMessage[] = []; @@ -1362,6 +1362,8 @@ test("a compile keeps its variables for 5 minutes after it or after the last cal expect(variablesOf(earlier)).toBeUndefined(); // The call 1 ms before moved the deadline of q4. expect(variablesOf(summary)).toContain('"question":"q4"'); + // The system clock moves back an hour; 5 minutes still pass. + vi.setSystemTime(Date.now() - 3_600_000); vi.advanceTimersByTime(300_000); expect(promptAttributesFor(summary)).toEqual({ "td.prompt.name": "support/reply", From 0c2f2b6e17a534d403179dd277db9136ab17f815 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 01:14:06 +0000 Subject: [PATCH 19/32] docs(sdk): tell that a call with no compiled message records its input In #117, the READMEs tell that a model call outside withPrompt and with_prompt records the compiled messages as its input when input capture is on. This branch replaced that sentence. Its text told that the SDK does not record a copy of a compiled message. The READMEs now tell that such a call gets no td.prompt.* attributes. With input capture on, the call records the messages as its input. Refs #97 --- docs/sdk-conformance.md | 2 +- packages/sdk/README.md | 7 ++++--- sdks/python/README.md | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 54debda..d9790a8 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -195,7 +195,7 @@ their text: - A copy has no tag: a spread, `structuredClone`, or a JSON round trip (TypeScript); `dict()`, `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, a JSON round trip, `type(message)(message)`, or `dataclasses.asdict` (Python). A new message with the same text has no tag either, thus a - message from another call, task, or prompt version is never recorded. + call never records the prompt of a message from another call, task, or prompt version. - `promptAttributesFor(messages)` / `prompt_attributes_for(messages)` take a list of messages or one message (Python: a `list` or `tuple`, not a subclass, so the read runs no code of the app). They look at the first 1000 and the last 1000 items and give the name, version, label, and diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 7c42fb5..ab40381 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -190,9 +190,10 @@ among them, also when input capture is off, and `td.prompt.variables` when it is at the first 1000 and the last 1000 messages of the call. A message from a placeholder variable keeps the tag of its own, older `compile()`, so it does not win over the `compile()` that put it in the list. A copy of a message (a spread, `structuredClone`, or JSON) and a new message with the same -text have no tag, so the SDK does not record them. Bedrock Converse and Google GenAI take another -message shape: change the compiled messages in place, or use `withPrompt`. A text prompt compiles to -a string, which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The +text have no tag, so a call that sends only such messages gets no `td.prompt.*` attributes; with +input capture on, the call still records them as its input. Bedrock Converse and Google GenAI take +another message shape: change the compiled messages in place, or use `withPrompt`. A text prompt +compiles to a string, which cannot carry a tag: use `withPrompt`. `withPrompt` wins over the tag. The integrations do it for OpenAI chat completions and responses, Anthropic messages, Bedrock Converse, Google GenAI `generateContent`, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, video, or OpenAI Realtime. diff --git a/sdks/python/README.md b/sdks/python/README.md index 05b5657..a3010b1 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -263,10 +263,11 @@ records. A chat call that sends such messages gets `td.prompt.name`, `td.prompt. the call. A message from a placeholder variable keeps the prompt of its own, older `compile()`, so it does not win over the `compile()` that put it in the list. A copy of a message (`dict()`, `.copy()`, `copy.copy`, `copy.deepcopy`, pickle, JSON, `type(message)(message)`, or -`dataclasses.asdict`) and a new message with the same text do not hold the prompt, so the SDK does -not record them. Bedrock Converse and Google GenAI take another message shape: change the compiled -dicts in place, or use `with_prompt`. A text prompt compiles to a string, which cannot hold its -prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations do it for OpenAI +`dataclasses.asdict`) and a new message with the same text do not hold the prompt, so a call that +sends only such messages gets no `td.prompt.*` attributes; with input capture on, the call still +records them as its input. Bedrock Converse and Google GenAI take another message shape: change the +compiled dicts in place, or use `with_prompt`. A text prompt compiles to a string, which cannot hold +its prompt: use `with_prompt`. `with_prompt` wins over the message. The integrations do it for OpenAI chat completions and responses, Anthropic messages (for `messages.stream()`, at `__enter__`), Bedrock Converse, Google GenAI `generate_content`, LiteLLM completion and responses, and OpenRouter chat and responses, but not for Bedrock InvokeModel (its body is bytes), embeddings, images, audio, From 789c94b4bbbf7c911f04b24323a941fed996e975 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 02:00:11 +0000 Subject: [PATCH 20/32] fix(cli): break the sign-in lock only when its holder has exited A lock now records its holder's pid and host and is broken on the same machine only once that process has exited. Every removal, a release or a break, claims a marker named after the lock's token first, so one command can't remove a lock another just took, and markers left by a crashed command are cleared the same way. Locks from CLI 0.1.0 or another machine keep the 3 second age rule. --- packages/cli/src/session.ts | 170 +++++++++++++++++++++++++++++---- packages/cli/tests/cli.test.ts | 120 ++++++++++++++++++++++- 2 files changed, 266 insertions(+), 24 deletions(-) diff --git a/packages/cli/src/session.ts b/packages/cli/src/session.ts index 644f9cf..61ffb2e 100644 --- a/packages/cli/src/session.ts +++ b/packages/cli/src/session.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from "node:crypto"; -import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; -import { homedir } from "node:os"; +import { link, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { homedir, hostname } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; @@ -117,38 +117,153 @@ const saveSession = async (session: Session) => { }; const LOCK_RETRY_MS = 25; -// A lock is held only for a read and a write, so one this old was left by a killed process. +// For a lock whose holder can't be checked: one written by CLI 0.1.0, which holds only a +// token, or by another machine sharing the config directory. A lock is held only for a +// read and a write, so one this old was most likely left by a killed process. const LOCK_STALE_MS = 3000; const LOCK_WAIT_MS = 2 * LOCK_STALE_MS; -const readLock = (path: string) => readFile(path, "utf8").catch(() => undefined); +type LockOwner = { readonly token: string; readonly pid?: number; readonly host?: string }; + +// The locks and markers this process holds, by token. +const held = new Set(); + +const readLock = async (path: string): Promise => { + const text = await readFile(path, "utf8").catch(() => undefined); + + if (text === undefined) return undefined; + + try { + const owner = JSON.parse(text) as Partial | null; + + if (typeof owner?.token === "string") return owner as LockOwner; + } catch {} + + return { token: text }; +}; + +/** Creates `path` naming this process as its owner, and returns its token; undefined when it exists. */ +const claim = async (path: string) => { + const owner: LockOwner = { + token: randomBytes(12).toString("hex"), + pid: process.pid, + host: hostname(), + }; + + const record = JSON.stringify(owner); + // Written whole first and linked into place, so where hard links work no command finds + // the file without its owner. Written fresh each time, so its modification time is when + // it was claimed. + const draft = `${path}.${owner.token}.tmp`; + + // Recorded before the file appears: another update in this process can read it before + // the claim resolves, and would take it for one left by an earlier process. + held.add(owner.token); + + try { + await writeFile(draft, record, { flag: "wx", mode: 0o600 }); + await link(draft, path).catch(async (error: NodeJS.ErrnoException) => { + if (error.code === "EEXIST") throw error; + + // A filesystem without hard links; the record is written in place instead. + await writeFile(path, record, { flag: "wx", mode: 0o600 }); + }); + } catch (error) { + held.delete(owner.token); + + if ((error as NodeJS.ErrnoException).code === "EEXIST") return undefined; + + throw error; + } finally { + // Best effort: a leftover draft has a unique name and nothing reads it, and a failure + // here must not turn a claim that succeeded into an error. + await rm(draft, { force: true }).catch(() => {}); + } + + return owner.token; +}; /** - * Takes the lock at `path` and returns its owner token. Only a lock still holding the - * token that was read is broken or released, so a command never removes another's lock. + * Whether the command holding a lock has exited without releasing it. On this machine a + * running holder keeps its lock however long it pauses, such as a laptop asleep mid-write. + * A lock from another machine or CLI 0.1.0, or one caught empty while being written on a + * filesystem without hard links, is judged by its age. */ -const lock = async (path: string) => { - const token = randomBytes(12).toString("hex"); - const deadline = Date.now() + LOCK_WAIT_MS; +const abandoned = async (path: string, owner: LockOwner) => { + const { pid } = owner; + + if ( + typeof pid === "number" && + Number.isSafeInteger(pid) && + pid > 0 && + owner.host === hostname() + ) { + // An earlier process with this one's pid left it, as when every run is pid 1 in a container. + if (pid === process.pid) return !held.has(owner.token); - for (;;) { try { - await writeFile(path, token, { flag: "wx", mode: 0o600 }); + process.kill(pid, 0); - return token; + return false; } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return (error as NodeJS.ErrnoException).code === "ESRCH"; } + } - const held = await stat(path).catch(() => undefined); - const owner = await readLock(path); + const changed = await stat(path).catch(() => undefined); - if (held && owner !== undefined && Date.now() - held.mtimeMs > LOCK_STALE_MS) { - if ((await readLock(path)) === owner) await rm(path, { force: true }); - continue; + return changed !== undefined && Date.now() - changed.mtimeMs > LOCK_STALE_MS; +}; + +/** + * Removes the lock at `path` if it still holds `token`. Every removal, whether a holder + * releasing its lock or a command breaking an abandoned one, first creates a marker named + * after the token, so while holders are judged by their pid, two commands never act on + * one lock, and one can't remove a lock another took after its check. A marker whose + * owner exited partway is removed the same way. Returns false when another command holds + * the marker. + */ +const removeLock = async (path: string, token: string): Promise => { + const marker = `${path}.${createHash("sha256").update(token).digest("hex").slice(0, 16)}`; + const mine = await claim(marker); + + if (!mine) { + const other = await readLock(marker); + + if (other && (await abandoned(marker, other))) await removeLock(marker, other.token); + + return false; + } + + try { + if ((await readLock(path))?.token === token) await rm(path, { force: true }); + + return true; + } finally { + // Held for one read and one removal. Its owner is this running process, so on this + // machine no other command removes it meanwhile. + try { + await rm(marker, { force: true }); + } finally { + held.delete(mine); } + } +}; - if (Date.now() > deadline) { +/** Takes the lock at `path` and returns its owner token. */ +const lock = async (path: string) => { + const deadline = performance.now() + LOCK_WAIT_MS; + + for (;;) { + const token = await claim(path); + + if (token) return token; + + const owner = await readLock(path); + + if (owner && (await abandoned(path, owner))) await removeLock(path, owner.token); + + if (performance.now() > deadline) { throw new CliError( `Another telemetry command is holding the sign-in file. If none is running, delete ${path} and try again.`, ); @@ -158,8 +273,23 @@ const lock = async (path: string) => { } }; +// When another command holds the marker, it is breaking this lock under the age rule, +// which can't see that this command is running. Waits for the lock or the marker to go, +// but not past LOCK_WAIT_MS: once this process exits, its lock is broken by its pid. const unlock = async (path: string, token: string) => { - if ((await readLock(path)) === token) await rm(path, { force: true }); + const deadline = performance.now() + LOCK_WAIT_MS; + + try { + while ( + !(await removeLock(path, token)) && + (await readLock(path))?.token === token && + performance.now() < deadline + ) { + await sleep(LOCK_RETRY_MS); + } + } finally { + held.delete(token); + } }; /** diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 71a80e6..04b9c51 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -1,12 +1,14 @@ -import { execFile } from "node:child_process"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { execFile, spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, beforeEach, expect, test } from "vitest"; -import { sessionFile } from "../src/session.ts"; +import { type Session, sessionFile, updateSession } from "../src/session.ts"; import { beforeRefresh, calls, @@ -367,3 +369,113 @@ test("logout revokes the refresh token and keeps the client registration and oth expect(session.tokens).toBeUndefined(); expect(await readFile(join(home, "session.json"), "utf8")).toBe(production); }); + +const lockPath = () => `${sessionPath()}.lock`; +const markerFor = (path: string, token: string) => + `${path}.${createHash("sha256").update(token).digest("hex").slice(0, 16)}`; +const holdLock = async (path: string, token: string, pid: number, host = hostname()) => + writeFile(path, JSON.stringify({ token, pid, host })); +const backdate = async (path: string) => { + const minuteAgo = new Date(Date.now() - 60_000); + await utimes(path, minuteAgo, minuteAgo); +}; +const exitedPid = () => spawnSync(process.execPath, ["-e", ""]).pid; + +const record = (write: number) => + updateSession( + fixture.url, + (saved) => + ({ + ...saved, + writes: [...((saved as { writes?: number[] }).writes ?? []), write], + }) as Session, + ); +const recorded = async () => + ((await savedSession().catch(() => ({}))) as { writes?: number[] }).writes?.toSorted( + (a, b) => a - b, + ) ?? []; + +// updateSession reads the config directory from the environment on each call. +const inConfigDir = async (run: () => Promise) => { + const configured = process.env.TELEMETRY_DEV_CONFIG_DIR; + process.env.TELEMETRY_DEV_CONFIG_DIR = home; + + try { + await run(); + } finally { + if (configured === undefined) delete process.env.TELEMETRY_DEV_CONFIG_DIR; + else process.env.TELEMETRY_DEV_CONFIG_DIR = configured; + } +}; + +test("the sign-in lock waits for a running holder however long it holds, and for a command already removing an abandoned lock", async () => { + const running = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"]); + + try { + await inConfigDir(async () => { + await holdLock(lockPath(), "running", running.pid!); + await backdate(lockPath()); + + const waiting = Promise.all([0, 1, 2, 3].map(record)); + await sleep(500); + expect(await recorded()).toEqual([]); + + await rm(lockPath()); + await waiting; + expect(await recorded()).toEqual([0, 1, 2, 3]); + + // Another machine's lock is judged by its age, and this one is fresh. + await holdLock(lockPath(), "other-machine", process.ppid, "another-machine"); + const behindOtherMachine = record(5); + await sleep(500); + expect(await recorded()).toEqual([0, 1, 2, 3]); + + await rm(lockPath()); + await behindOtherMachine; + expect(await recorded()).toEqual([0, 1, 2, 3, 5]); + + await holdLock(lockPath(), "abandoned", exitedPid()); + const marker = markerFor(lockPath(), "abandoned"); + await holdLock(marker, "breaking", running.pid!); + + const behindBreaker = record(4); + await sleep(500); + expect(await recorded()).toEqual([0, 1, 2, 3, 5]); + + await rm(marker); + await behindBreaker; + expect(await recorded()).toEqual([0, 1, 2, 3, 4, 5]); + }); + } finally { + running.kill(); + } +}); + +test("the sign-in lock is taken over at once, without lost writes, when its holder or a command breaking it has exited", async () => { + await inConfigDir(async () => { + const takeOver = async (writes: number[]) => { + const started = Date.now(); + await Promise.all(writes.map(record)); + expect(Date.now() - started).toBeLessThan(2000); + }; + + await holdLock(lockPath(), "exited", exitedPid()); + await takeOver([0, 1, 2, 3, 4, 5, 6, 7]); + + await holdLock(lockPath(), "exited-again", exitedPid()); + await holdLock(markerFor(lockPath(), "exited-again"), "exited-breaker", exitedPid()); + await takeOver([8]); + + // Left by an earlier process that had this one's pid. + await holdLock(lockPath(), "same-pid", process.pid); + await takeOver([9]); + + // Another machine's lock can only be judged by its age. + await holdLock(lockPath(), "other-machine", process.ppid, "another-machine"); + await backdate(lockPath()); + await takeOver([10]); + + expect(await recorded()).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(await readdir(home)).toEqual([sessionFile(fixture.url)]); + }); +}); From 1195b11581dc729e346e888cf8406ab3dd917d96 Mon Sep 17 00:00:00 2001 From: Ephraim Duncan <55143799+ephraimduncan@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:37:51 +0000 Subject: [PATCH 21/32] docs(sdk): name the read prompts access of the prompt api key (#103) The TypeScript and Python READMEs said that the key of getPrompt and get_prompt "needs the read scope for prompts". The API keys page of telemetry.dev shows this scope as the Read prompts choice in the Access of a key. The READMEs and the apiKey doc comment now name Read prompts and the prompts:read scope. Refs telemetry-dev/telemetry.dev#512 --- packages/sdk/README.md | 4 ++-- packages/sdk/src/prompts.ts | 5 ++++- sdks/python/README.md | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef1..fa5bf2c 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -161,8 +161,8 @@ const pinned = await getPrompt("support/reply", { version: 13 }); its prompt. - getPrompt uses the API key and base URL of an enabled `init()`, or `TELEMETRY_DEV_API_KEY` and `TELEMETRY_DEV_BASE_URL`, or its own `apiKey`, `baseUrl`, and `fetch` options. It reads the same - host that telemetry goes to, `https://api.telemetry.dev` by default. The key needs the read - scope for prompts. + host that telemetry goes to, `https://api.telemetry.dev` by default. The key must have Read + prompts in its Access on the API keys page (the `prompts:read` scope). - When a label sends part of the traffic to a second version, each session keeps one version: the choice uses the session id of `propagateAttributes` (or of `sessionMode: "process"`). Without a session id, each call chooses at random. diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b17..1f557b2 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -116,7 +116,10 @@ interface GetPromptOptionsBase { version?: number; /** Seconds before a call fetches a kept prompt again, in the background. Default 60. */ cacheTtlSeconds?: number; - /** API key with the prompts:read scope. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + /** + * API key with Read prompts in its Access (the prompts:read scope). Falls back to the initialized + * client, then `TELEMETRY_DEV_API_KEY`. + */ apiKey?: string; /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ baseUrl?: string; diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d10..95cfb6f 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -232,8 +232,8 @@ pinned = telemetry_dev.get_prompt("support/reply", version=13) without its prompt. - `get_prompt` uses the API key and base URL of an enabled `init()`, or `TELEMETRY_DEV_API_KEY` and `TELEMETRY_DEV_BASE_URL`, or its own `api_key` and `base_url`. It reads the same host that - telemetry goes to, `https://api.telemetry.dev` by default. The key needs the read scope for - prompts. + telemetry goes to, `https://api.telemetry.dev` by default. The key must have Read prompts in + its Access on the API keys page (the `prompts:read` scope). - When a label sends part of the traffic to a second version, each session keeps one version: the choice uses the session id of `propagate_attributes` (or of `session_mode="process"`). Without a session id, each call chooses at random. From c6b599ed1d0311ff2c367c0e64e47bd7c2b3b402 Mon Sep 17 00:00:00 2001 From: Ephraim Duncan <55143799+ephraimduncan@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:38:13 +0000 Subject: [PATCH 22/32] fix(sdk-python): fetch prompts again in a forked process (#105) * fix(sdk-python): reset prompt fetches in a forked process A process that os.fork() makes kept the pending fetch events of the parent, but not the fetch threads. In the child, a call with no copy waited 10 s and then got a timeout error. A kept copy never refreshed. A cache lock that a different thread held at the fork stayed locked in the child, and each get_prompt call in the child waited for all time. Fork hooks now hold the cache lock during fork(). In the child, the hooks remove the pending fetch events and release the lock. Thus a call that must fetch starts a new fetch. The child keeps the copies of the parent. Closes #98 * fix(sdk-python): make the prompt cache fork hooks safe A pending fetch keeps the id of the process whose thread runs the fetch. In a different process, a call ignores this fetch and starts a new fetch. Thus a fork without fork hooks also gets a new fetch, for example a fork in C code. The child hook does not reset the entries. The cache lock is reentrant. Thus a fork in the thread that holds the lock does not wait for that thread. The before hook records in a thread local if it got the lock. The after hooks release the lock only in that condition. The module registers the hooks one time in each process, also when importlib.reload runs the module again. The hooks find the lock when they run. The lock test examines the time of the fork and the errors of the holder thread. The fork must return after the holder releases the lock. * fix(sdk-python): replace a held prompt cache lock in a forked child If the before fork hook did not get the cache lock, a thread of the parent can hold the lock in the child. A signal can stop the wait of the before hook. uWSGI with py-call-osafterfork runs only the child hook. In these two conditions, a get_prompt call in the child waited for all time. The child hook replaces the lock in such a child. The code reads the lock from the module each time. The child hook removes the fetches of the parent again. Before this change, the child found these fetches only through the process ID. If the child got the ID of the process that started a fetch, the child waited for that fetch, and its calls got timeout errors. The ID check stays for a fork that runs no hooks. A call keeps its fetch event in a local variable. Thus a fork in the call, as in a signal handler, does not let the call continue with no event and no result. The capture server reads its delay before it records a request. Thus a test that finds the request and sets the delay to 0 does not cancel the wait of that request. The signal test stops its timer. Thus a failure of the test does not stop pytest with SIGALRM. in_fork finds the child through its process ID. Thus a fork function that forks two times does not send a child back into pytest. * fix(sdk-python): use base_url in the prompt fork tests The version_of helper of the fork tests gave the api_url option to get_prompt. #101 replaced api_url with base_url. Thus pyright found an error after the rebase on #101. * fix(sdk-python): correct the prompt fork claims and fork in the test The README and docs/sdk-conformance.md said that a forked process does not wait for a fetch that did not end before the fork. That is not true for one case. If a signal handler forks while a call waits for a fetch, the call continues in the child. No thread of the child completes that fetch. Thus the call waits until 10 seconds after the start of the fetch, and then fails with the timeout code. The two documents now name this case, and the code does not change. The comment at the process ID check in _load, and the message of the commit "replace a held prompt cache lock in a forked child", said that the check handles a fork that runs no hooks. The check removes only the copied fetch. Such a fork can also copy a lock that a thread of the parent holds. Then a call in the child waits for the lock with no limit. The comment now says this, and the comment at the hook registration now says that only a fork that runs the hooks waits for the lock. The signal row of the owner test of the lock replacement called the fork hooks directly. The row now forks with os.fork() through in_fork. A different thread holds the cache lock, and SIGALRM stops the before hook. The child must get the prompt. Thus the row needs os.fork() to run the after hooks when a before hook raises. The row fails when the child hook does not replace the lock. It also fails when an after hook releases a lock that the before hook did not get. If no signal stops the before hook, the holder thread fails the row after 5 s. The fork function of the row stops its timer also when the fork raises. If the child loses the thread local _forking, the child hook replaces the lock and does not release it. The child gets a free lock in the two cases, thus no test can see a difference. --- docs/sdk-conformance.md | 5 + sdks/python/README.md | 7 + sdks/python/src/telemetry_dev/_prompts.py | 64 ++++- sdks/python/tests/otlp_capture.py | 6 +- sdks/python/tests/test_prompts.py | 303 +++++++++++++++++++++- 5 files changed, 377 insertions(+), 8 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dc..ffd2d41 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -142,6 +142,11 @@ Concurrent calls share one fetch. and not to the full response. Only an address lookup can continue after the call stops waiting, until the system resolver gives up. Both SDKs measure the TTL and Retry-After in elapsed time, so a change of the system clock does not move them. +- Fork (Python): a process that `os.fork()` makes gets the copies of the parent, but not the fetch + threads of the parent. A call that starts in the child does not wait for a fetch of the parent. + If the call must fetch, it starts a new fetch. If a signal handler forks while a call waits for a + fetch, that call continues in the child. The call gets `timeout` 10 seconds after the start of + the fetch, because no thread of the child completes the fetch. - Failure: when a fetch fails, the SDKs keep the last copy for another TTL and send the error to `onError` / `on_error`. With no copy, the call gets the fallback. With no fallback, the call gets a `PromptError`. With no copy, the calls of the next 10 seconds (or the TTL, if it is shorter) diff --git a/sdks/python/README.md b/sdks/python/README.md index 95cfb6f..99e9b08 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -219,6 +219,13 @@ pinned = telemetry_dev.get_prompt("support/reply", version=13) call returns it at once and fetches a new copy in a background thread. Only a call with no copy waits for a fetch, for 10 seconds at most, also when the network stops. Calls for the same prompt share one fetch. `get_prompt_async` runs the same function in a worker thread. +- A process that `os.fork()` makes keeps the copies of its parent. This includes a process of + `multiprocessing` with the `fork` start method. A call that starts in the child does not wait + for a fetch of the parent. If the call must fetch, it starts a new fetch. +- If a signal handler forks while a call waits for a fetch, that call continues in the child. No + thread of the child completes that fetch. Thus the call waits until 10 seconds after the start + of the fetch. Then the call returns the `fallback`, or raises `PromptError` with the code + `timeout`. - If a fetch fails, `get_prompt` returns the last copy that it fetched and reports the error through `on_error`. With no copy, it returns the `fallback` (`version` is `None` and `is_fallback` is `True`). With no fallback, it raises `PromptError` whose `code` tells why, for diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 9673740..0ba6b9d 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -5,6 +5,7 @@ import copy import hashlib import json +import os import random import re import socket @@ -455,12 +456,57 @@ class _Entry: retry_at: float = 0.0 # Set when the running fetch ends. pending: threading.Event | None = None + # The process whose thread runs the fetch. A forked process gets the entry, but not the thread. + pending_pid: int = 0 # When calls stop waiting for the running fetch. deadline: float = 0.0 _cache: OrderedDict[tuple[str, str, str, str], _Entry] = OrderedDict() -_cache_lock = threading.Lock() +# Reentrant, so that a fork in a thread that holds the lock, for example in a signal handler, does +# not wait for itself in _lock_for_fork. _reset_in_child can replace it, so read it from the module +# each time. +_cache_lock = threading.RLock() +# Whether _lock_for_fork got the lock in this thread. os.fork() runs the after hooks also when a +# before hook raises, for example in a signal handler. +_forking = threading.local() + + +def _lock_for_fork() -> None: + _cache_lock.acquire() + _forking.locked = True + + +def _unlock_after_fork() -> None: + if getattr(_forking, "locked", False): + _forking.locked = False + _cache_lock.release() + + +def _reset_in_child() -> None: + """Unlock the cache, and drop the fetches of the parent: the child has none of its threads.""" + global _cache_lock + if getattr(_forking, "locked", False): + _forking.locked = False + _cache_lock.release() + else: + # The before hook did not get the lock, because a signal stopped its wait, or did not run, + # as in uWSGI with py-call-osafterfork. A thread of the parent can hold the lock, and only + # this thread exists here. + _cache_lock = threading.RLock() + for entry in _cache.values(): + entry.pending = None + + +# A fork that runs these hooks waits for a thread that holds the lock, so that the child gets no +# entry that a thread changed only in part. os.register_at_fork cannot remove a hook, and +# importlib.reload runs this module again with the same globals. Thus the hooks of the first run +# stay, and they find the lock and the cache when they run. Windows has no fork. +if hasattr(os, "register_at_fork") and "_fork_hooks" not in globals(): + _fork_hooks = True + os.register_at_fork( + before=_lock_for_fork, after_in_parent=_unlock_after_fork, after_in_child=_reset_in_child + ) def _load( @@ -481,15 +527,23 @@ def _load( entry = _cache[key] = _Entry() now = monotonic() ttl = ttl_seconds if entry.response is not None else min(ttl_seconds, _ERROR_TTL_S) - if now >= max(entry.checked_at + ttl, entry.retry_at) and entry.pending is None: - entry.pending = threading.Event() + if entry.pending is not None and entry.pending_pid != os.getpid(): + # A fork in C code that runs no hooks copied the fetch but not its thread. This check + # does not replace the lock. If a thread of the parent held the lock at that fork, a + # call in the child waits for the lock with no limit. + entry.pending = None + # A local, because a fork in this thread, as in a signal handler, can clear entry.pending. + pending = entry.pending + if now >= max(entry.checked_at + ttl, entry.retry_at) and pending is None: + pending = entry.pending = threading.Event() + entry.pending_pid = os.getpid() entry.deadline = now + _FETCH_TIMEOUT_S threading.Thread( - target=_refresh, args=(entry, entry.pending, fetch, on_error), daemon=True + target=_refresh, args=(entry, pending, fetch, on_error), daemon=True ).start() if entry.response is not None: return entry.response - pending, deadline = entry.pending, entry.deadline + deadline = entry.deadline if pending is not None and not pending.wait(deadline - monotonic()): raise _timeout() with _cache_lock: diff --git a/sdks/python/tests/otlp_capture.py b/sdks/python/tests/otlp_capture.py index 0649f72..35cfce9 100644 --- a/sdks/python/tests/otlp_capture.py +++ b/sdks/python/tests/otlp_capture.py @@ -33,7 +33,8 @@ class CaptureServer: status: int = 200 # The (status, JSON body) replies to GET requests, in order; the last one repeats. replies: list[tuple[int, object]] = field(default_factory=lambda: []) - # Seconds that the server waits before it answers a GET request. + # Seconds that the server waits before it answers a GET request. The server reads it before the + # request shows in requests, so that a test that sees the request can change it for the next. delay: float = 0 # The location header of GET replies, for a redirect status. location: str | None = None @@ -66,6 +67,7 @@ def do_POST(self) -> None: self.end_headers() def do_GET(self) -> None: + delay = capture.delay captured.append( CapturedRequest( path=self.requestline.split(" ")[1], @@ -73,7 +75,7 @@ def do_GET(self) -> None: body=b"", ) ) - time.sleep(capture.delay) + time.sleep(delay) status, reply = ( capture.replies[0] if len(capture.replies) == 1 else capture.replies.pop(0) ) diff --git a/sdks/python/tests/test_prompts.py b/sdks/python/tests/test_prompts.py index 892d676..764c4ef 100644 --- a/sdks/python/tests/test_prompts.py +++ b/sdks/python/tests/test_prompts.py @@ -2,17 +2,22 @@ import asyncio import contextvars +import importlib import itertools +import os +import signal import socket +import sys import threading import time +import traceback import urllib.error import urllib.request from collections.abc import Callable, Generator from datetime import datetime, timedelta, timezone from email.utils import format_datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from types import SimpleNamespace +from types import FrameType, SimpleNamespace from typing import Any, NoReturn from urllib.parse import parse_qsl, urlsplit @@ -34,6 +39,12 @@ from .conftest import MakeClient from .otlp_capture import CaptureServer, otlp_capture_server +# Python 3.12+ gives a warning for fork() in a process with threads, and the capture server has one. +pytestmark = pytest.mark.filterwarnings( + "ignore:This process .* is multi-threaded:DeprecationWarning" +) +needs_fork = pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") + def reply(**overrides: Any) -> dict[str, Any]: body: dict[str, Any] = { @@ -93,6 +104,57 @@ def wait_for(check: Callable[[], bool]) -> None: time.sleep(0.01) +def in_fork(child: Callable[[], str], fork: Callable[[], int] | None = None) -> str: + """Run ``child`` in a process that ``fork`` (by default ``os.fork``) makes, and return its + result, or the repr of its exception or of an exception in a fork hook of the prompt module. + The parent kills a child that does not end in 10 s, and the test fails.""" + # Python ignores an exception in a fork hook and only gives it to sys.unraisablehook. The + # hooks of OpenTelemetry raise after an earlier test discards its batch processor. + hook_errors: list[BaseException | None] = [] + + def record(unraisable: sys.UnraisableHookArgs) -> None: + frames = traceback.extract_tb(unraisable.exc_traceback) + if any(frame.filename == prompts_module.__file__ for frame in frames): + hook_errors.append(unraisable.exc_value) + + previous_hook = sys.unraisablehook + parent = os.getpid() + read_end, write_end = os.pipe() + sys.unraisablehook = record + try: + pid = (fork or os.fork)() + finally: + sys.unraisablehook = previous_hook + # Not pid == 0: a fork function that forks more than one time can return a pid in a child. + if os.getpid() != parent: + try: + os.close(read_end) + try: + result = repr(hook_errors[0]) if hook_errors else child() + except BaseException as exc: + result = repr(exc) + os.write(write_end, result.encode()) + finally: + # The child must not return into pytest. + os._exit(0) + os.close(write_end) + with os.fdopen(read_end, "rb") as reader: + deadline = time.monotonic() + 10 + while os.waitpid(pid, os.WNOHANG) == (0, 0): + if time.monotonic() > deadline: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + pytest.fail("The forked process did not end in 10 s") + time.sleep(0.01) + result = reader.read().decode() + assert hook_errors == [] + return result + + +def version_of(server: CaptureServer) -> int | None: + return get_prompt("support/reply", api_key="td_live_a", base_url=server.url).version + + @pytest.mark.parametrize( ("options", "query"), [ @@ -525,6 +587,245 @@ def close(self) -> None: assert timeouts[0] is not None and timeouts[0] <= 0.2 +@needs_fork +@pytest.mark.parametrize("reused_pid", [False, True]) +def test_a_forked_process_fetches_the_prompt_that_the_parent_was_fetching( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch, reused_pid: bool +) -> None: + # This puts a limit on the wait of a child that waits for the fetch of the parent. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + server.delay = 1 + versions: list[int | None] = [] + parent = threading.Thread(target=lambda: versions.append(version_of(server))) + parent.start() + wait_for(lambda: len(server.requests) == 1) + server.delay = 0 + parent_pid = os.getpid() + + def child() -> str: + if reused_pid: + # A reused pid: the child gets the id of the process that started the fetch. + monkeypatch.setattr(os, "getpid", lambda: parent_pid) + return str(version_of(server)) + + assert in_fork(child) == "12" + parent.join() + assert versions == [12] + # The fetch of the parent, and the fetch of the child. + assert len(server.requests) == 2 + + +def test_a_call_does_not_wait_for_a_fetch_that_another_process_started( + server: CaptureServer, +) -> None: + server.delay = 1 + first = threading.Thread(target=version_of, args=(server,)) + first.start() + wait_for(lambda: len(server.requests) == 1) + server.delay = 0 + # A fork that runs no fork hooks, as a fork in C code, copies the fetch of the parent but not + # the thread that runs it. Here this process is the child: the fetch gets the id of its parent. + (entry,) = prompts_module._cache.values() # pyright: ignore[reportPrivateUsage] + entry.pending_pid = os.getppid() + + assert version_of(server) == 12 + assert len(server.requests) == 2 + first.join() + + +@needs_fork +def test_a_forked_process_refreshes_the_copy_that_the_parent_was_refreshing( + server: CaptureServer, clock: list[float] +) -> None: + assert version_of(server) == 12 + clock[0] += 60 + server.replies = [(200, reply(version=13))] + server.delay = 1 + assert version_of(server) == 12 + wait_for(lambda: len(server.requests) == 2) + server.delay = 0 + + def child() -> str: + kept = version_of(server) + wait_for(lambda: version_of(server) == 13) + return str(kept) + + # The child returns the copy of the parent. Then its refresh gets version 13. + assert in_fork(child) == "12" + + +@needs_fork +@pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning") +def test_a_fork_waits_for_a_thread_that_holds_the_cache_lock(server: CaptureServer) -> None: + held = threading.Event() + released_at: list[float] = [] + forked_at: list[float] = [] + errors: list[BaseException] = [] + + def hold() -> None: + try: + with prompts_module._cache_lock: # pyright: ignore[reportPrivateUsage] + held.set() + time.sleep(0.2) + released_at.append(time.monotonic()) + except BaseException as exc: + errors.append(exc) + + def fork() -> int: + pid = os.fork() + if pid != 0: + forked_at.append(time.monotonic()) + return pid + + holder = threading.Thread(target=hold) + holder.start() + held.wait() + # The child does not get a lock that a thread of the parent holds. + assert in_fork(lambda: str(version_of(server)), fork) == "12" + holder.join() + assert errors == [] + assert forked_at[0] > released_at[0] + + +@needs_fork +def test_a_fork_in_the_thread_that_holds_the_cache_lock_returns( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + # A process that still holds the lock after the fork waits for its fetch until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + + def fork_in_the_lock() -> int: + with prompts_module._cache_lock: # pyright: ignore[reportPrivateUsage] + return os.fork() + + def fork_and_fetch() -> str: + child = in_fork(lambda: str(version_of(server)), fork_in_the_lock) + return f"{child} {version_of(server)}" + + # The fork runs in a child, so that a fork that waits for itself stops only that child. + assert in_fork(fork_and_fetch) == "12 12" + + +@needs_fork +def test_a_fork_in_a_call_that_starts_a_fetch_returns_the_prompt( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + # A process that waits for a fetch that it does not have waits until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + pids: list[int] = [] + results: list[str] = [] + + class ForkingThread(threading.Thread): + def start(self) -> None: + # A fork in the call, as in a signal handler, after the call made its fetch and before + # the fetch thread starts. The child hook drops the fetch of the entry. + if not pids: + pids.append(os.fork()) + super().start() + + def fork() -> int: + with monkeypatch.context() as patch: + # Only the threads of the prompt module: a fork hook of OpenTelemetry starts a thread. + patch.setattr( + prompts_module, + "threading", + SimpleNamespace(**{**vars(threading), "Thread": ForkingThread}), + ) + try: + results.append(str(version_of(server))) + except Exception as exc: + # The child must return the pid to in_fork, and not raise into pytest. + results.append(repr(exc)) + return pids[0] + + # The parent and the child each start the fetch thread, and each call returns its result. + assert in_fork(lambda: results[0], fork) == "12" + assert results == ["12"] + + +@needs_fork +def test_a_fork_after_a_reload_of_the_prompt_module_returns( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch +) -> None: + saved = dict(prompts_module.__dict__) + importlib.reload(prompts_module) + try: + # The reload sets the timeout again. A process that still holds the lock after the fork + # waits for its fetch until the timeout. + monkeypatch.setattr(prompts_module, "_FETCH_TIMEOUT_S", 3) + assert in_fork(lambda: str(version_of(server))) == "12" + assert version_of(server) == 12 + finally: + # The reload gave the module new classes and a new cache. + prompts_module.__dict__.update(saved) + + +@needs_fork +@pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="needs signal.setitimer") +@pytest.mark.parametrize("before_hook", ["stopped by a signal", "not run"]) +def test_the_fork_hooks_handle_a_lock_that_the_before_hook_did_not_get( + server: CaptureServer, monkeypatch: pytest.MonkeyPatch, before_hook: str +) -> None: + class Interrupted(Exception): + pass + + lock = prompts_module._cache_lock # pyright: ignore[reportPrivateUsage] + # The child hook replaces the lock. monkeypatch puts it back. + monkeypatch.setattr(prompts_module, "_cache_lock", lock) + held, done = threading.Event(), threading.Event() + errors: list[BaseException] = [] + + def hold() -> None: + try: + with lock: + held.set() + # The fork must end while this thread holds the lock. If no signal stops the + # before hook, the hook waits for this thread, and this assert fails after 5 s. + assert done.wait(5) + except BaseException as exc: + errors.append(exc) + + def interrupt(signum: int, frame: FrameType | None) -> NoReturn: + raise Interrupted + + def fork_in_a_signal() -> int: + # SIGALRM raises while the before hook waits for the lock. os.fork() gives the error to + # sys.unraisablehook, forks, and then runs the after hooks. + in_fork_hook = sys.unraisablehook + + def record(unraisable: sys.UnraisableHookArgs) -> None: + if not isinstance(unraisable.exc_value, Interrupted): + in_fork_hook(unraisable) + + previous = signal.signal(signal.SIGALRM, interrupt) + sys.unraisablehook = record + signal.setitimer(signal.ITIMER_REAL, 0.1) + try: + return os.fork() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + sys.unraisablehook = in_fork_hook + signal.signal(signal.SIGALRM, previous) + + holder = threading.Thread(target=hold) + holder.start() + held.wait() + try: + if before_hook == "stopped by a signal": + # In the child, the holder is a thread of the parent that does not exist there. + assert in_fork(lambda: str(version_of(server)), fork_in_a_signal) == "12" + else: + # uWSGI with py-call-osafterfork forks in C and runs only the child hook. + prompts_module._reset_in_child() # pyright: ignore[reportPrivateUsage] + child_lock = prompts_module._cache_lock # pyright: ignore[reportPrivateUsage] + assert child_lock.acquire(blocking=False) + child_lock.release() + finally: + done.set() + holder.join() + assert errors == [] + + def test_get_prompt_rejects_a_prompt_of_the_other_type_and_bad_options( server: CaptureServer, ) -> None: From ee914849b5d5d27560ea869332758c38490e280b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 07:23:07 +0000 Subject: [PATCH 23/32] chore: release main (#125) --- .release-please-manifest.json | 38 +++++++++---------- packages/anthropic/CHANGELOG.md | 23 +++++++++++ packages/anthropic/package.json | 2 +- packages/bedrock/CHANGELOG.md | 23 +++++++++++ packages/bedrock/package.json | 2 +- packages/cursor/CHANGELOG.md | 9 +++++ packages/cursor/package.json | 2 +- packages/eve/CHANGELOG.md | 9 +++++ packages/eve/package.json | 2 +- packages/google-genai/CHANGELOG.md | 23 +++++++++++ packages/google-genai/package.json | 2 +- packages/mcp/CHANGELOG.md | 11 ++++++ packages/mcp/package.json | 2 +- packages/omp/CHANGELOG.md | 9 +++++ packages/omp/package.json | 2 +- packages/openai/CHANGELOG.md | 23 +++++++++++ packages/openai/package.json | 2 +- packages/opencode/CHANGELOG.md | 9 +++++ packages/opencode/package.json | 2 +- packages/openrouter/CHANGELOG.md | 23 +++++++++++ packages/openrouter/package.json | 2 +- packages/pi/CHANGELOG.md | 9 +++++ packages/pi/package.json | 2 +- packages/sdk/CHANGELOG.md | 20 ++++++++++ packages/sdk/package.json | 2 +- sdks/python-anthropic/CHANGELOG.md | 7 ++++ sdks/python-anthropic/pyproject.toml | 2 +- .../src/telemetry_dev_anthropic/__init__.py | 2 +- sdks/python-anthropic/uv.lock | 4 +- sdks/python-bedrock/CHANGELOG.md | 7 ++++ sdks/python-bedrock/pyproject.toml | 2 +- .../src/telemetry_dev_bedrock/__init__.py | 2 +- sdks/python-bedrock/uv.lock | 4 +- sdks/python-google-genai/CHANGELOG.md | 7 ++++ sdks/python-google-genai/pyproject.toml | 2 +- .../telemetry_dev_google_genai/__init__.py | 2 +- sdks/python-google-genai/uv.lock | 4 +- sdks/python-litellm/CHANGELOG.md | 7 ++++ sdks/python-litellm/pyproject.toml | 2 +- .../src/telemetry_dev_litellm/__init__.py | 2 +- sdks/python-litellm/uv.lock | 4 +- sdks/python-openai/CHANGELOG.md | 7 ++++ sdks/python-openai/pyproject.toml | 2 +- .../src/telemetry_dev_openai/__init__.py | 2 +- sdks/python-openai/uv.lock | 4 +- sdks/python-openrouter/CHANGELOG.md | 7 ++++ sdks/python-openrouter/pyproject.toml | 2 +- .../src/telemetry_dev_openrouter/__init__.py | 2 +- sdks/python-openrouter/uv.lock | 4 +- sdks/python/CHANGELOG.md | 26 +++++++++++++ sdks/python/pyproject.toml | 2 +- sdks/python/uv.lock | 2 +- 52 files changed, 316 insertions(+), 57 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f008dfa..8d8f051 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,25 +1,25 @@ { "packages/otel": "0.2.0", - "packages/sdk": "0.2.1", + "packages/sdk": "0.2.2", "packages/ai": "0.2.0", - "packages/anthropic": "0.2.1", - "packages/bedrock": "0.2.1", - "packages/cursor": "0.2.1", - "packages/eve": "0.2.1", - "packages/google-genai": "0.2.1", - "packages/mcp": "0.2.1", - "packages/omp": "0.2.1", - "packages/openai": "0.2.1", - "packages/opencode": "0.2.1", - "packages/openrouter": "0.2.1", - "packages/pi": "0.2.1", + "packages/anthropic": "0.2.2", + "packages/bedrock": "0.2.2", + "packages/cursor": "0.2.2", + "packages/eve": "0.2.2", + "packages/google-genai": "0.2.2", + "packages/mcp": "0.2.2", + "packages/omp": "0.2.2", + "packages/openai": "0.2.2", + "packages/opencode": "0.2.2", + "packages/openrouter": "0.2.2", + "packages/pi": "0.2.2", "packages/tanstack-ai": "0.2.0", - "sdks/python": "0.3.1", - "sdks/python-anthropic": "0.2.1", - "sdks/python-bedrock": "0.2.1", - "sdks/python-google-genai": "0.2.1", - "sdks/python-litellm": "0.2.1", - "sdks/python-openai": "0.2.1", - "sdks/python-openrouter": "0.2.1", + "sdks/python": "0.3.2", + "sdks/python-anthropic": "0.2.2", + "sdks/python-bedrock": "0.2.2", + "sdks/python-google-genai": "0.2.2", + "sdks/python-litellm": "0.2.2", + "sdks/python-openai": "0.2.2", + "sdks/python-openrouter": "0.2.2", "packages/cli": "0.1.0" } diff --git a/packages/anthropic/CHANGELOG.md b/packages/anthropic/CHANGELOG.md index 8e60fae..4138558 100644 --- a/packages/anthropic/CHANGELOG.md +++ b/packages/anthropic/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/anthropic-v0.2.1...anthropic-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/anthropic-v0.2.0...anthropic-v0.2.1) (2026-10-30) diff --git a/packages/anthropic/package.json b/packages/anthropic/package.json index afa055e..c52455f 100644 --- a/packages/anthropic/package.json +++ b/packages/anthropic/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/anthropic", - "version": "0.2.1", + "version": "0.2.2", "description": "Anthropic SDK instrumentation for telemetry.dev: wraps the Messages API and emits OpenTelemetry GenAI spans.", "keywords": [ "anthropic", diff --git a/packages/bedrock/CHANGELOG.md b/packages/bedrock/CHANGELOG.md index 3c4a5e9..eb08036 100644 --- a/packages/bedrock/CHANGELOG.md +++ b/packages/bedrock/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/bedrock-v0.2.1...bedrock-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/bedrock-v0.2.0...bedrock-v0.2.1) (2026-10-30) diff --git a/packages/bedrock/package.json b/packages/bedrock/package.json index 2efe9cc..c53be42 100644 --- a/packages/bedrock/package.json +++ b/packages/bedrock/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/bedrock", - "version": "0.2.1", + "version": "0.2.2", "description": "AWS Bedrock instrumentation for telemetry.dev: Converse, InvokeModel, and Bedrock Agents spans for AWS SDK v3 clients.", "keywords": [ "aws", diff --git a/packages/cursor/CHANGELOG.md b/packages/cursor/CHANGELOG.md index 1f8628a..f03f5d4 100644 --- a/packages/cursor/CHANGELOG.md +++ b/packages/cursor/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/cursor-v0.2.1...cursor-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/cursor-v0.2.0...cursor-v0.2.1) (2026-10-30) diff --git a/packages/cursor/package.json b/packages/cursor/package.json index 6c6a7c6..7f57bdd 100644 --- a/packages/cursor/package.json +++ b/packages/cursor/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/cursor", - "version": "0.2.1", + "version": "0.2.2", "description": "Cursor coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/eve/CHANGELOG.md b/packages/eve/CHANGELOG.md index 4aaac1a..4b713d4 100644 --- a/packages/eve/CHANGELOG.md +++ b/packages/eve/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/eve-v0.2.1...eve-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/eve-v0.2.0...eve-v0.2.1) (2026-10-30) diff --git a/packages/eve/package.json b/packages/eve/package.json index 3401f18..dcf082e 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/eve", - "version": "0.2.1", + "version": "0.2.2", "description": "Eve (Vercel agent framework) telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/google-genai/CHANGELOG.md b/packages/google-genai/CHANGELOG.md index a44107e..4d8264c 100644 --- a/packages/google-genai/CHANGELOG.md +++ b/packages/google-genai/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/google-genai-v0.2.1...google-genai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/google-genai-v0.2.0...google-genai-v0.2.1) (2026-10-30) diff --git a/packages/google-genai/package.json b/packages/google-genai/package.json index 6d9fe44..9fb7e9b 100644 --- a/packages/google-genai/package.json +++ b/packages/google-genai/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/google-genai", - "version": "0.2.1", + "version": "0.2.2", "description": "Google GenAI (Gemini) SDK instrumentation for telemetry.dev: generation and embedding spans for @google/genai.", "keywords": [ "gemini", diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index cdd7d55..c04a5a7 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/mcp-v0.2.1...mcp-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/mcp-v0.2.0...mcp-v0.2.1) (2026-10-30) diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 0a4b610..108c75b 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/mcp", - "version": "0.2.1", + "version": "0.2.2", "description": "Model Context Protocol SDK instrumentation for telemetry.dev: instruments MCP v2 transports and emits OpenTelemetry spans.", "keywords": [ "mcp", diff --git a/packages/omp/CHANGELOG.md b/packages/omp/CHANGELOG.md index 003d763..5ea2571 100644 --- a/packages/omp/CHANGELOG.md +++ b/packages/omp/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/omp-v0.2.1...omp-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/omp-v0.2.0...omp-v0.2.1) (2026-10-30) diff --git a/packages/omp/package.json b/packages/omp/package.json index 8ac9f27..792b48a 100644 --- a/packages/omp/package.json +++ b/packages/omp/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/omp", - "version": "0.2.1", + "version": "0.2.2", "description": "Oh My Pi (omp) coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/openai/CHANGELOG.md b/packages/openai/CHANGELOG.md index 842e048..a0ca87e 100644 --- a/packages/openai/CHANGELOG.md +++ b/packages/openai/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/openai-v0.2.1...openai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/openai-v0.2.0...openai-v0.2.1) (2026-10-30) diff --git a/packages/openai/package.json b/packages/openai/package.json index c338c47..d1ed5eb 100644 --- a/packages/openai/package.json +++ b/packages/openai/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/openai", - "version": "0.2.1", + "version": "0.2.2", "description": "OpenAI SDK instrumentation for telemetry.dev: generation and embedding spans for the official openai Node SDK.", "keywords": [ "genai", diff --git a/packages/opencode/CHANGELOG.md b/packages/opencode/CHANGELOG.md index e024f86..919db51 100644 --- a/packages/opencode/CHANGELOG.md +++ b/packages/opencode/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/opencode-v0.2.1...opencode-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/opencode-v0.2.0...opencode-v0.2.1) (2026-10-30) diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 1fe8aec..09c1043 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/opencode", - "version": "0.2.1", + "version": "0.2.2", "description": "opencode coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/openrouter/CHANGELOG.md b/packages/openrouter/CHANGELOG.md index 0b8ab81..6242db5 100644 --- a/packages/openrouter/CHANGELOG.md +++ b/packages/openrouter/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/openrouter-v0.2.1...openrouter-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Dependencies + +* The following workspace dependencies were updated + * devDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + * peerDependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/openrouter-v0.2.0...openrouter-v0.2.1) (2026-10-30) diff --git a/packages/openrouter/package.json b/packages/openrouter/package.json index ea040e2..3c75975 100644 --- a/packages/openrouter/package.json +++ b/packages/openrouter/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/openrouter", - "version": "0.2.1", + "version": "0.2.2", "description": "OpenRouter SDK instrumentation for telemetry.dev: generation and embedding spans for the official @openrouter/sdk package.", "keywords": [ "genai", diff --git a/packages/pi/CHANGELOG.md b/packages/pi/CHANGELOG.md index 85e36f7..8f119de 100644 --- a/packages/pi/CHANGELOG.md +++ b/packages/pi/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/pi-v0.2.1...pi-v0.2.2) (2026-10-30) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @telemetry-dev/sdk bumped to 0.2.2 + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/pi-v0.2.0...pi-v0.2.1) (2026-10-30) diff --git a/packages/pi/package.json b/packages/pi/package.json index 028a04f..dd45891 100644 --- a/packages/pi/package.json +++ b/packages/pi/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/pi", - "version": "0.2.1", + "version": "0.2.2", "description": "Pi coding-agent telemetry integration for telemetry.dev.", "keywords": [ "agents", diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 468e95e..fea0a68 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/sdk-v0.2.1...sdk-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep compiled variables for 5 minutes, not two turns ([1b09916](https://github.com/telemetry-dev/telemetry/commit/1b099167bbd8acaf5008848ad9ba2a2ab8e35b0d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of recent compiles only ([3bc6f53](https://github.com/telemetry-dev/telemetry/commit/3bc6f53a0afb0ad8b9fe19f8eaf7e5d812a93b0a)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of the last two chat turns ([f6aa1c1](https://github.com/telemetry-dev/telemetry/commit/f6aa1c159742123600d6499653916c4dbf7b2ab7)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** measure the variables time of compiled messages in elapsed time ([b2a0e09](https://github.com/telemetry-dev/telemetry/commit/b2a0e09e3a2fee5aa8c5d6951a0bac9dac019cfa)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** name the newest compile, not the last tagged message ([a6b98fe](https://github.com/telemetry-dev/telemetry/commit/a6b98fe916d5ed4b37bddd7972b07507182f2d2d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/sdk-v0.2.0...sdk-v0.2.1) (2026-10-30) diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 0248c0d..2158077 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/sdk", - "version": "0.2.1", + "version": "0.2.2", "description": "Pure TypeScript SDK for telemetry.dev: LLM traces, logs, and GenAI metrics on OpenTelemetry, for Node and edge runtimes.", "keywords": [ "genai", diff --git a/sdks/python-anthropic/CHANGELOG.md b/sdks/python-anthropic/CHANGELOG.md index 1b31054..c21b8c1 100644 --- a/sdks/python-anthropic/CHANGELOG.md +++ b/sdks/python-anthropic/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-anthropic-v0.2.1...python-anthropic-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-anthropic-v0.2.0...python-anthropic-v0.2.1) (2026-10-30) diff --git a/sdks/python-anthropic/pyproject.toml b/sdks/python-anthropic/pyproject.toml index 13ac10c..c6298b3 100644 --- a/sdks/python-anthropic/pyproject.toml +++ b/sdks/python-anthropic/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-anthropic" -version = "0.2.1" +version = "0.2.2" description = "Anthropic integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index a0dd315..21fc24b 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -18,7 +18,7 @@ from anthropic.resources.messages import AsyncMessages, Messages from telemetry_dev._serialize import json_dumps -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] diff --git a/sdks/python-anthropic/uv.lock b/sdks/python-anthropic/uv.lock index cb38c4b..4956ec6 100644 --- a/sdks/python-anthropic/uv.lock +++ b/sdks/python-anthropic/uv.lock @@ -745,7 +745,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -772,7 +772,7 @@ dev = [ [[package]] name = "telemetry-dev-anthropic" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "anthropic" }, diff --git a/sdks/python-bedrock/CHANGELOG.md b/sdks/python-bedrock/CHANGELOG.md index 9c23f3a..7fab0e3 100644 --- a/sdks/python-bedrock/CHANGELOG.md +++ b/sdks/python-bedrock/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-bedrock-v0.2.1...python-bedrock-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-bedrock-v0.2.0...python-bedrock-v0.2.1) (2026-10-30) diff --git a/sdks/python-bedrock/pyproject.toml b/sdks/python-bedrock/pyproject.toml index 37fc637..de7e541 100644 --- a/sdks/python-bedrock/pyproject.toml +++ b/sdks/python-bedrock/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-bedrock" -version = "0.2.1" +version = "0.2.2" description = "AWS Bedrock integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py b/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py index 2ea5057..c748493 100644 --- a/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py +++ b/sdks/python-bedrock/src/telemetry_dev_bedrock/__init__.py @@ -2,6 +2,6 @@ from ._instrument import instrument_bedrock, uninstrument_bedrock, wrap_bedrock -__version__ = "0.2.1" +__version__ = "0.2.2" __all__ = ["__version__", "instrument_bedrock", "uninstrument_bedrock", "wrap_bedrock"] diff --git a/sdks/python-bedrock/uv.lock b/sdks/python-bedrock/uv.lock index 6f460cb..92d00ce 100644 --- a/sdks/python-bedrock/uv.lock +++ b/sdks/python-bedrock/uv.lock @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -478,7 +478,7 @@ dev = [ [[package]] name = "telemetry-dev-bedrock" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "boto3" }, diff --git a/sdks/python-google-genai/CHANGELOG.md b/sdks/python-google-genai/CHANGELOG.md index db866b9..d8cf120 100644 --- a/sdks/python-google-genai/CHANGELOG.md +++ b/sdks/python-google-genai/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-google-genai-v0.2.1...python-google-genai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-google-genai-v0.2.0...python-google-genai-v0.2.1) (2026-10-30) diff --git a/sdks/python-google-genai/pyproject.toml b/sdks/python-google-genai/pyproject.toml index b9a2fb3..4c4cde0 100644 --- a/sdks/python-google-genai/pyproject.toml +++ b/sdks/python-google-genai/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-google-genai" -version = "0.2.1" +version = "0.2.2" description = "Google GenAI (Gemini) integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index e094f83..0d4fcb5 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -24,7 +24,7 @@ from google.genai.operations import AsyncOperations, Operations from telemetry_dev._serialize import json_dumps -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] RequestMapper = Callable[[Any, Any, Any], tuple[str, dict[str, Any]]] diff --git a/sdks/python-google-genai/uv.lock b/sdks/python-google-genai/uv.lock index cda5be3..f6f8594 100644 --- a/sdks/python-google-genai/uv.lock +++ b/sdks/python-google-genai/uv.lock @@ -821,7 +821,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -848,7 +848,7 @@ dev = [ [[package]] name = "telemetry-dev-google-genai" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "google-genai" }, diff --git a/sdks/python-litellm/CHANGELOG.md b/sdks/python-litellm/CHANGELOG.md index f7280f4..fce6210 100644 --- a/sdks/python-litellm/CHANGELOG.md +++ b/sdks/python-litellm/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-litellm-v0.2.1...python-litellm-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-litellm-v0.2.0...python-litellm-v0.2.1) (2026-10-30) diff --git a/sdks/python-litellm/pyproject.toml b/sdks/python-litellm/pyproject.toml index fd88c5a..b9d45a5 100644 --- a/sdks/python-litellm/pyproject.toml +++ b/sdks/python-litellm/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-litellm" -version = "0.2.1" +version = "0.2.2" description = "LiteLLM integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index 49ec964..0033de1 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -15,7 +15,7 @@ import telemetry_dev from opentelemetry import trace -__version__ = "0.2.1" +__version__ = "0.2.2" RequestMapper = Callable[[tuple[Any, ...], Mapping[str, Any]], tuple[str, dict[str, Any]]] ResponseMapper = Callable[[Any], dict[str, Any]] diff --git a/sdks/python-litellm/uv.lock b/sdks/python-litellm/uv.lock index 5a9e4ba..acdff9d 100644 --- a/sdks/python-litellm/uv.lock +++ b/sdks/python-litellm/uv.lock @@ -2889,7 +2889,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api", version = "1.43.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -2919,7 +2919,7 @@ dev = [ [[package]] name = "telemetry-dev-litellm" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "litellm" }, diff --git a/sdks/python-openai/CHANGELOG.md b/sdks/python-openai/CHANGELOG.md index 89cb655..f0b94aa 100644 --- a/sdks/python-openai/CHANGELOG.md +++ b/sdks/python-openai/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-openai-v0.2.1...python-openai-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-openai-v0.2.0...python-openai-v0.2.1) (2026-10-30) diff --git a/sdks/python-openai/pyproject.toml b/sdks/python-openai/pyproject.toml index 79c6435..8a3fbf9 100644 --- a/sdks/python-openai/pyproject.toml +++ b/sdks/python-openai/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-openai" -version = "0.2.1" +version = "0.2.2" description = "OpenAI integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index 3cbe610..5475801 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -20,7 +20,7 @@ from openai.resources.images import AsyncImages, Images from openai.resources.responses.responses import AsyncResponses, Responses -__version__ = "0.2.1" +__version__ = "0.2.2" ProviderResolver = Callable[[object | None], str] RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] diff --git a/sdks/python-openai/uv.lock b/sdks/python-openai/uv.lock index 6369336..ffd4d32 100644 --- a/sdks/python-openai/uv.lock +++ b/sdks/python-openai/uv.lock @@ -735,7 +735,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -762,7 +762,7 @@ dev = [ [[package]] name = "telemetry-dev-openai" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "openai" }, diff --git a/sdks/python-openrouter/CHANGELOG.md b/sdks/python-openrouter/CHANGELOG.md index 56ac8c2..d9f6061 100644 --- a/sdks/python-openrouter/CHANGELOG.md +++ b/sdks/python-openrouter/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.2.2](https://github.com/telemetry-dev/telemetry/compare/python-openrouter-v0.2.1...python-openrouter-v0.2.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) + ## [0.2.1](https://github.com/telemetry-dev/telemetry/compare/python-openrouter-v0.2.0...python-openrouter-v0.2.1) (2026-10-30) diff --git a/sdks/python-openrouter/pyproject.toml b/sdks/python-openrouter/pyproject.toml index 98379ec..d17d978 100644 --- a/sdks/python-openrouter/pyproject.toml +++ b/sdks/python-openrouter/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev-openrouter" -version = "0.2.1" +version = "0.2.2" description = "OpenRouter integration for telemetry.dev Python SDK" readme = "README.md" license = "MIT" diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index c15bfb3..0f7c2b4 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -12,7 +12,7 @@ from openrouter.embeddings import Embeddings from openrouter.responses import Responses -__version__ = "0.2.1" +__version__ = "0.2.2" RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] ResponseMapper = Callable[[Any], dict[str, Any]] diff --git a/sdks/python-openrouter/uv.lock b/sdks/python-openrouter/uv.lock index d317aa7..05f8f67 100644 --- a/sdks/python-openrouter/uv.lock +++ b/sdks/python-openrouter/uv.lock @@ -603,7 +603,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "../python" } dependencies = [ { name = "opentelemetry-api" }, @@ -630,7 +630,7 @@ dev = [ [[package]] name = "telemetry-dev-openrouter" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "openrouter" }, diff --git a/sdks/python/CHANGELOG.md b/sdks/python/CHANGELOG.md index c950f82..f72714c 100644 --- a/sdks/python/CHANGELOG.md +++ b/sdks/python/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [0.3.2](https://github.com/telemetry-dev/telemetry/compare/python-v0.3.1...python-v0.3.2) (2026-10-30) + + +### Features + +* **sdk:** record the prompt of compiled messages outside withPrompt ([6435fb6](https://github.com/telemetry-dev/telemetry/commit/6435fb6d2d7488c968a6ce9f9c6437f2cf426456)) +* **sdk:** record the prompt on model calls outside withPrompt ([339eb0c](https://github.com/telemetry-dev/telemetry/commit/339eb0c01c5754191eb7d1b5ec37db21ebd28855)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk-python:** fetch prompts again in a forked process ([#105](https://github.com/telemetry-dev/telemetry/issues/105)) ([c6b599e](https://github.com/telemetry-dev/telemetry/commit/c6b599ed1d0311ff2c367c0e64e47bd7c2b3b402)) +* **sdk:** find the prompt by message identity, not by text ([ae47605](https://github.com/telemetry-dev/telemetry/commit/ae47605555ef9f5dfed1d6f762ac56823e692edb)) +* **sdk:** give the base url to getPrompt in the prompt tests ([abb8d10](https://github.com/telemetry-dev/telemetry/commit/abb8d10f680d1fa7ace6913a1d53b43a3e0db703)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep compiled variables for 5 minutes, not two turns ([1b09916](https://github.com/telemetry-dev/telemetry/commit/1b099167bbd8acaf5008848ad9ba2a2ab8e35b0d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of recent compiles only ([3bc6f53](https://github.com/telemetry-dev/telemetry/commit/3bc6f53a0afb0ad8b9fe19f8eaf7e5d812a93b0a)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** keep the variables of the last two chat turns ([f6aa1c1](https://github.com/telemetry-dev/telemetry/commit/f6aa1c159742123600d6499653916c4dbf7b2ab7)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) +* **sdk:** name the newest compile, not the last tagged message ([a6b98fe](https://github.com/telemetry-dev/telemetry/commit/a6b98fe916d5ed4b37bddd7972b07507182f2d2d)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + + +### Documentation + +* **sdk:** name the read prompts access of the prompt api key ([#103](https://github.com/telemetry-dev/telemetry/issues/103)) ([1195b11](https://github.com/telemetry-dev/telemetry/commit/1195b11581dc729e346e888cf8406ab3dd917d96)) +* **sdk:** tell that a call with no compiled message records its input ([0c2f2b6](https://github.com/telemetry-dev/telemetry/commit/0c2f2b6e17a534d403179dd277db9136ab17f815)), closes [#97](https://github.com/telemetry-dev/telemetry/issues/97) + ## [0.3.1](https://github.com/telemetry-dev/telemetry/compare/python-v0.3.0...python-v0.3.1) (2026-10-30) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 9204b53..1cffcd5 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" description = "telemetry.dev SDK for Python — OpenTelemetry-native GenAI tracing, logs, and metrics" readme = "README.md" license = "MIT" diff --git a/sdks/python/uv.lock b/sdks/python/uv.lock index ee9f2d2..ed63e15 100644 --- a/sdks/python/uv.lock +++ b/sdks/python/uv.lock @@ -404,7 +404,7 @@ wheels = [ [[package]] name = "telemetry-dev" -version = "0.3.1" +version = "0.3.2" source = { editable = "." } dependencies = [ { name = "opentelemetry-api" }, From dcad14b08926862409cfaf94c529ecfab8df26e5 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 08:52:42 +0000 Subject: [PATCH 24/32] feat(cli): report errors as one JSON line with a stable code under --json (#127) With --json, a failure writes one JSON line to stderr with a code from a documented list, a hint, retryable, and the exit code, and nothing to stdout. Color and the spinner are off under --json, and arguments after -- are not options. Network errors are found anywhere in the cause chain, so a refused connection says telemetry.dev couldn't be reached and a connect timeout is a retryable timeout. --- packages/cli/README.md | 31 +++++++++- packages/cli/src/api.ts | 67 ++++++++++++++++----- packages/cli/src/auth.ts | 23 ++++---- packages/cli/src/cli.ts | 72 ++++++++++++++++++----- packages/cli/src/commands.ts | 28 +++++---- packages/cli/src/errors.ts | 53 ++++++++++++++++- packages/cli/tests/cli.test.ts | 103 ++++++++++++++++++++++++++++++++- 7 files changed, 320 insertions(+), 57 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 07df7f7..feb416b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -57,11 +57,11 @@ List commands return 20 results by default. Use `--limit` to return from 1 to 50 ## Scripts and agents -Add `--json` to any command for machine-readable output on stdout. Messages and progress go to stderr, so `telemetry traces --json > traces.json` writes only data. List commands print the MCP tool's result as returned, with the fields that the [MCP server documentation](https://telemetry.dev/docs/platform/mcp-server) describes. +Add `--json` to any command except `login` and `logout` for machine-readable output on stdout. Messages and progress go to stderr, so `telemetry traces --json > traces.json` writes only data. List commands print the MCP tool's result as returned, with the fields that the [MCP server documentation](https://telemetry.dev/docs/platform/mcp-server) describes. `telemetry trace` and `telemetry payload` read every page of a trace or a captured text, up to a limit: 2,000 spans for a trace, and 1,000,000 characters for each payload field. Their JSON output has `complete: false` when the limit cut it short, and human output says so on stderr. -Captured text comes from your users, models, and tools. Human-readable output, including `payload --field`, removes terminal escape sequences and control characters from that text. JSON output escapes them, so use `--json` when you need the exact text. +Captured text comes from your users, models, and tools. telemetry.dev removes ANSI color and cursor sequences, bidirectional overrides, and Unicode tag characters from results before they reach the CLI; it does not clean the text of error messages. Human-readable output, including `payload --field`, removes the remaining escape sequences and control characters. JSON output escapes them instead of removing them, so use `--json` when you need everything the server sent. | Exit code | Meaning | | --------- | --------------------------------------------------------------- | @@ -71,6 +71,33 @@ Captured text comes from your users, models, and tools. Human-readable output, i | 4 | You are not signed in, the sign-in expired, or access is denied | | 130 | Interrupted with Ctrl-C | +With `--json`, an error is one line of JSON on stderr, and stdout stays empty. Ctrl-C exits with 130 and no JSON, and `status --json` reports a missing sign-in as data on stdout with exit code 4. + +```text +{"error":{"code":"not_signed_in","message":"You're not signed in. Run `telemetry login` first.","hint":"telemetry login","retryable":false,"exit_code":4,"cli_version":"0.1.0"}} +``` + +Branch on `code`, not on `message`, whose wording can change. `hint` is a command to run next, when there is one; `telemetry login` needs a person at a browser, so an agent should ask for it rather than run it. `retryable` is true when running the same command again may succeed. New codes may be added, and existing codes keep their meaning. + +| Code | Exit | Meaning | +| ---------------------- | ---- | -------------------------------------------------------------------------------- | +| `invalid_arguments` | 2 | An option or argument is not valid | +| `unknown_command` | 2 | There is no command with that name | +| `project_not_found` | 2 | No project in the workspace has that name or id | +| `project_ambiguous` | 2 | Several projects have that name; pass its id | +| `project_required` | 2 | The workspace has several projects and none was chosen | +| `no_projects` | 1 | The workspace has no projects yet | +| `not_signed_in` | 4 | No sign-in is saved for this deployment | +| `sign_in_expired` | 4 | The sign-in expired or was disconnected | +| `payloads_not_allowed` | 4 | The sign-in doesn't include reading captured prompts and outputs | +| `unreachable` | 1 | telemetry.dev could not be reached | +| `timeout` | 1 | telemetry.dev took too long to answer | +| `tool_error` | 1 | telemetry.dev returned an error; `message` says what it was | +| `result_too_large` | 1 | The result was too large to read; narrow it with filters or `--limit` | +| `request_failed` | 1 | The request failed for another reason | +| `failed` | 1 | The command failed for another reason | +| `internal` | 1 | An unexpected error, such as a configuration file the CLI couldn't read or write | + ## Configuration | Variable | Default | Purpose | diff --git a/packages/cli/src/api.ts b/packages/cli/src/api.ts index b9ee36d..890ff8d 100644 --- a/packages/cli/src/api.ts +++ b/packages/cli/src/api.ts @@ -75,7 +75,9 @@ export class Api { const text = result.content.find((block) => block.type === "text")?.text ?? ""; if (result.isError) - throw new CliError(text || "telemetry.dev could not complete this request."); + throw new CliError(text || "telemetry.dev could not complete this request.", { + code: "tool_error", + }); try { return JSON.parse(text) as T; @@ -83,6 +85,7 @@ export class Api { // The server cuts results over its size cap, which leaves the JSON unterminated. throw new CliError( "The result was too large to read. Narrow it with filters or a smaller --limit.", + { code: "result_too_large" }, ); } } @@ -96,35 +99,71 @@ export class Api { // Each is a refresh or authorization the server refused for good. type AuthErrors = readonly (abstract new (...args: never[]) => Error)[]; +const NETWORK_CODES = new Set([ + "ECONNREFUSED", + "ECONNRESET", + "EAI_AGAIN", + "EHOSTUNREACH", + "ENETUNREACH", + "ENOTFOUND", + "ETIMEDOUT", + "UND_ERR_SOCKET", +]); + +// undici's own timeouts, raised by fetch before any of the CLI's timers fire. +const TIMEOUT_CODES = new Set([ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", +]); + const translateError = (error: unknown, authErrors: AuthErrors): never => { if (error instanceof CliError) throw error; if (authErrors.some((type) => error instanceof type)) { throw new CliError( "Your sign-in has expired or was disconnected. Run `telemetry login` again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_expired", hint: "telemetry login" }, ); } - const cause = error instanceof Error && error.cause instanceof Error ? error.cause : error; - const message = cause instanceof Error ? cause.message : String(cause); - const code = (cause as { code?: unknown } | null)?.code; + // The network error sits at the bottom of a chain: the SDK wraps fetch's + // "fetch failed", which wraps the socket error that has the code. + const chain: unknown[] = [error]; - if ( - typeof code === "string" && - ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ECONNRESET", "ETIMEDOUT"].includes(code) - ) { + for (let link = error; link instanceof Error && link.cause !== undefined && chain.length < 8; ) { + link = link.cause; + chain.push(link); + } + + const codes = chain + .map((link) => (link as { code?: unknown } | null)?.code) + .filter((code): code is string => typeof code === "string"); + const network = codes.find((code) => NETWORK_CODES.has(code)); + // Node's AggregateError for a failed connection has an empty message. + const message = + chain.findLast((link): link is Error => link instanceof Error && link.message !== "") + ?.message ?? String(error); + + if (network) { throw new CliError( - `Couldn't reach telemetry.dev (${code}). Check your connection and try again.`, + `Couldn't reach telemetry.dev (${network}). Check your connection and try again.`, + { code: "unreachable", retryable: true }, ); } if ( - (error as { code?: unknown } | null)?.code === "REQUEST_TIMEOUT" || - (cause instanceof Error && (cause.name === "TimeoutError" || cause.name === "AbortError")) + codes.some((code) => code === "REQUEST_TIMEOUT" || TIMEOUT_CODES.has(code)) || + chain.some( + (link) => + link instanceof Error && (link.name === "TimeoutError" || link.name === "AbortError"), + ) ) { - throw new CliError("telemetry.dev took too long to answer. Try again shortly."); + throw new CliError("telemetry.dev took too long to answer. Try again shortly.", { + code: "timeout", + retryable: true, + }); } - throw new CliError(`telemetry.dev request failed: ${message}`); + throw new CliError(`telemetry.dev request failed: ${message}`, { code: "request_failed" }); }; diff --git a/packages/cli/src/auth.ts b/packages/cli/src/auth.ts index 1e8a14c..7fcdde0 100644 --- a/packages/cli/src/auth.ts +++ b/packages/cli/src/auth.ts @@ -264,10 +264,11 @@ export class SessionAuthProvider implements OAuthClientProvider { } const sessionExpired = () => - new CliError( - "Your sign-in has expired or was disconnected. Run `telemetry login` again.", - EXIT.auth, - ); + new CliError("Your sign-in has expired or was disconnected. Run `telemetry login` again.", { + exitCode: EXIT.auth, + code: "sign_in_expired", + hint: "telemetry login", + }); export const grantedScopes = (session: Session) => new Set((session.tokens?.scope ?? "").split(" ").filter(Boolean)); @@ -441,7 +442,7 @@ const authorizationParams = (address: string, state: string) => { if (params?.get("state") !== state) { throw new CliError( "That isn't the address from this sign-in. Copy the browser's whole address bar after you select Allow.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed" }, ); } @@ -475,14 +476,14 @@ const authorize = async ( error.code === "invalid_grant" ? "That sign-in link was already used or has expired. Run `telemetry login` again." : `Sign-in failed: ${oneLine(error.message).slice(0, 300)}`, - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } if (error instanceof IssuerMismatchError || error instanceof AuthorizationServerMismatchError) { throw new CliError( "The sign-in answer came from a different authorization server than this sign-in started with. Run `telemetry login` again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } @@ -499,7 +500,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => error === "access_denied" ? "Sign-in was cancelled in the browser." : `Sign-in failed: ${oneLine(params.get("error_description") ?? error)}.`, - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } @@ -508,7 +509,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => if (!code) { throw new CliError( "The browser returned without an authorization code. Run `telemetry login` again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ); } @@ -587,7 +588,7 @@ export const login = async ({ if (!pending) { throw new CliError( "No sign-in is waiting to finish. Run `telemetry login --no-browser` to start one.", - EXIT.auth, + { exitCode: EXIT.auth, code: "no_pending_sign_in", hint: "telemetry login --no-browser" }, ); } @@ -661,7 +662,7 @@ export const login = async ({ reject( new CliError( "Sign-in timed out after 10 minutes. Run `telemetry login` to try again.", - EXIT.auth, + { exitCode: EXIT.auth, code: "sign_in_failed", hint: "telemetry login" }, ), ), LOGIN_TIMEOUT_MS, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 19c6769..0369660 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -5,6 +5,7 @@ import { VERSION } from "./api.ts"; import { type Context, home, + jsonText, loginCommand, logoutCommand, logsCommand, @@ -18,7 +19,7 @@ import { usageCommand, useCommand, } from "./commands.ts"; -import { CliError, EXIT, usageError } from "./errors.ts"; +import { CliError, type ErrorCode, EXIT, usageError } from "./errors.ts"; import { clean, disableColor, SHOW_CURSOR, style } from "./terminal.ts"; type Option = { @@ -309,8 +310,18 @@ const closest = (input: string, names: readonly string[]) => { const commandNamed = (name: string | undefined) => name !== undefined && Object.hasOwn(COMMANDS, name) ? COMMANDS[name] : undefined; +// Arguments after `--` are values, never options. +const optionArgs = (argv: readonly string[]) => { + const end = argv.indexOf("--"); + + return end === -1 ? argv : argv.slice(0, end); +}; + +const jsonOutput = optionArgs(process.argv.slice(2)).includes("--json"); + const main = async (argv: readonly string[]) => { - if (argv.includes("--no-color")) disableColor(); + // With --json, stderr carries no spinner or color, so an error line is plain JSON. + if (argv.includes("--no-color") || jsonOutput) disableColor(); // The command comes first; only --no-color may precede it. const first = argv.findIndex((arg) => arg !== "--no-color"); @@ -338,7 +349,12 @@ const main = async (argv: readonly string[]) => { const command = commandNamed(topic); - if (!command) throw usageError(`Unknown command "${topic}". Run telemetry --help.`); + if (!command) { + throw usageError(`Unknown command "${topic}". Run telemetry --help.`, { + code: "unknown_command", + hint: "telemetry --help", + }); + } return void process.stdout.write(`${commandHelp(topic, command)}\n`); } @@ -350,6 +366,7 @@ const main = async (argv: readonly string[]) => { throw usageError( `Unknown command "${name}".${suggestion ? ` Did you mean \`telemetry ${suggestion}\`?` : ""} Run telemetry --help.`, + { code: "unknown_command", hint: "telemetry --help" }, ); } @@ -373,13 +390,17 @@ const main = async (argv: readonly string[]) => { } catch (error) { const message = error instanceof Error ? error.message.split("\n")[0] : String(error); - throw usageError(`${message} Run \`telemetry ${name} --help\`.`); + throw usageError(`${message} Run \`telemetry ${name} --help\`.`, { + hint: `telemetry ${name} --help`, + }); } const extra = parsed.positionals[command.arguments ?? 0]; if (extra !== undefined) { - throw usageError(`Unexpected argument "${extra}". Run \`telemetry ${name} --help\`.`); + throw usageError(`Unexpected argument "${extra}". Run \`telemetry ${name} --help\`.`, { + hint: `telemetry ${name} --help`, + }); } await command.run({ @@ -390,21 +411,40 @@ const main = async (argv: readonly string[]) => { }; const fail = (error: unknown) => { - process.stderr.write(SHOW_CURSOR.repeat(process.stderr.isTTY ? 1 : 0)); - - // Tool errors are written by the server and can quote what the request named. - if (error instanceof CliError) { - process.stderr.write(`${style.red("error", process.stderr)} ${clean(error.message)}\n`); - process.exitCode = error.exitCode; + if (!jsonOutput) process.stderr.write(SHOW_CURSOR.repeat(process.stderr.isTTY ? 1 : 0)); + + const known = error instanceof CliError ? error : undefined; + const exitCode = known?.exitCode ?? EXIT.failure; + process.exitCode = exitCode; + + // With --json, a script or agent reads the error as one JSON line on stderr; stdout + // stays empty, so it never mistakes an error for data. + if (jsonOutput) { + process.stderr.write( + `${jsonText({ + error: { + code: known?.code ?? ("internal" satisfies ErrorCode), + message: known ? known.message : error instanceof Error ? error.message : String(error), + ...(known?.hint ? { hint: known.hint } : {}), + retryable: known?.retryable ?? false, + exit_code: exitCode, + cli_version: VERSION, + }, + })}\n`, + ); return; } - // Unexpected, so the stack is kept for a bug report; it can still quote a server. - process.stderr.write( - `${style.red("error", process.stderr)} ${clean(error instanceof Error ? (error.stack ?? error.message) : String(error))}\n`, - ); - process.exitCode = EXIT.failure; + // Tool errors are written by the server and can quote what the request named. An + // unexpected error keeps its stack for a bug report; it can still quote a server. + const message = known + ? known.message + : error instanceof Error + ? (error.stack ?? error.message) + : String(error); + + process.stderr.write(`${style.red("error", process.stderr)} ${clean(message)}\n`); }; // The MCP SDK warns app developers on the console, for instance when it retries a refresh. diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 5755555..06f401f 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -58,16 +58,17 @@ const note = (text = "") => process.stderr.write(`${text}\n`); /** * JSON for scripts and agents. JSON.stringify escapes C0 controls but not C1 controls - * or bidirectional overrides, which some terminals act on, so those are escaped too. + * or bidirectional overrides, which some terminals act on, or the line and paragraph + * separators some line readers split on, so those are escaped too. */ -const printJson = (value: unknown) => - out( - JSON.stringify(value, null, 2).replace( - /[\u0080-\u009f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, - (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`, - ), +export const jsonText = (value: unknown, indent?: number) => + JSON.stringify(value, null, indent).replace( + /[\u0080-\u009f\u061c\u200e\u200f\u2028\u2029\u202a-\u202e\u2066-\u2069]/g, + (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`, ); +const printJson = (value: unknown) => out(jsonText(value, 2)); + const json = (context: Context) => context.values.json === true; const string = (context: Context, name: string) => { @@ -123,10 +124,14 @@ const matchProject = (projects: ProjectList["projects"], wanted: string): Projec if (byName.length > 1) { throw usageError( `Several projects are named "${wanted}". Pass its id instead: telemetry projects`, + { code: "project_ambiguous", hint: "telemetry projects" }, ); } - throw usageError(`No project "${wanted}" in this workspace. Projects: ${names || "none"}.`); + throw usageError(`No project "${wanted}" in this workspace. Projects: ${names || "none"}.`, { + code: "project_not_found", + hint: "telemetry projects", + }); }; /** @@ -148,10 +153,13 @@ const resolveProject = async (api: Api, context: Context): Promise => { if (projects.length === 1) return toProject(projects[0]!); if (projects.length === 0) - throw new CliError("This workspace has no projects yet. Create one at telemetry.dev."); + throw new CliError("This workspace has no projects yet. Create one at telemetry.dev.", { + code: "no_projects", + }); throw usageError( `Choose a project with --project , or set a default with \`telemetry use \`. Projects: ${projectNames(projects)}.`, + { code: "project_required", hint: "telemetry projects" }, ); }; @@ -1085,7 +1093,7 @@ export async function payloadCommand(context: Context) { if (!grantedScopes(session).has(PAYLOADS_SCOPE)) { throw new CliError( 'This sign-in can\'t read captured prompts and outputs. Run `telemetry login`, check "Also read captured prompts, outputs, and tool arguments", and select Allow.', - EXIT.auth, + { exitCode: EXIT.auth, code: "payloads_not_allowed", hint: "telemetry login" }, ); } diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index 2973b3e..1cb0b3c 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -7,17 +7,64 @@ export const EXIT = { interrupted: 130, } as const; +/** What scripts and agents branch on. Listed in the README; only ever added to. */ +export type ErrorCode = + | "failed" + | "internal" + | "invalid_arguments" + | "unknown_command" + | "project_not_found" + | "project_ambiguous" + | "project_required" + | "no_projects" + | "not_signed_in" + | "sign_in_expired" + | "sign_in_failed" + | "no_pending_sign_in" + | "payloads_not_allowed" + | "unreachable" + | "timeout" + | "tool_error" + | "result_too_large" + | "request_failed"; + +type ErrorDetails = { + readonly exitCode?: number; + readonly code?: ErrorCode; + /** A command to run next. `telemetry login` needs a person at a browser. */ + readonly hint?: string; + /** Whether running the same command again may succeed. */ + readonly retryable?: boolean; +}; + /** A failure with a message written for the person running the command. */ export class CliError extends Error { readonly exitCode: number; + readonly code: ErrorCode; + readonly hint: string | undefined; + readonly retryable: boolean; - constructor(message: string, exitCode: number = EXIT.failure) { + constructor(message: string, details: number | ErrorDetails = {}) { super(message); + const { + exitCode = EXIT.failure, + code, + hint, + retryable = false, + } = typeof details === "number" ? { exitCode: details } : details; this.exitCode = exitCode; + this.code = code ?? (exitCode === EXIT.usage ? "invalid_arguments" : "failed"); + this.hint = hint; + this.retryable = retryable; } } -export const usageError = (message: string) => new CliError(message, EXIT.usage); +export const usageError = (message: string, details: Omit = {}) => + new CliError(message, { ...details, exitCode: EXIT.usage }); export const notSignedIn = () => - new CliError("You're not signed in. Run `telemetry login` first.", EXIT.auth); + new CliError("You're not signed in. Run `telemetry login` first.", { + exitCode: EXIT.auth, + code: "not_signed_in", + hint: "telemetry login", + }); diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 71a80e6..f2c310c 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -1,10 +1,11 @@ import { execFile } from "node:child_process"; import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { type AddressInfo, createServer } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { afterAll, beforeAll, beforeEach, expect, test } from "vitest"; +import { afterAll, beforeAll, beforeEach, expect, test, vi } from "vitest"; import { sessionFile } from "../src/session.ts"; import { @@ -192,6 +193,106 @@ test("input that isn't valid exits 2 before calling telemetry.dev, even from a s expect(calls).toEqual([]); }); +test("with --json, an error is one JSON line on stderr with a code to branch on, and stdout stays empty", async () => { + const errorOf = async (result: { code: number; stdout: string; stderr: string }) => { + expect(result.stdout).toBe(""); + expect(result.stderr.trim().split("\n")).toHaveLength(1); + + return { exit: result.code, ...JSON.parse(result.stderr).error }; + }; + + expect(await errorOf(await run("traces", "--json"))).toMatchObject({ + exit: 4, + code: "not_signed_in", + hint: "telemetry login", + retryable: false, + exit_code: 4, + }); + + await signIn(PROJECTS[0]); + + expect(await errorOf(await run("traces", "--limit", "99", "--json"))).toMatchObject({ + exit: 2, + code: "invalid_arguments", + exit_code: 2, + }); + expect(await errorOf(await run("traces", "--project", "nope", "--json"))).toMatchObject({ + exit: 2, + code: "project_not_found", + hint: "telemetry projects", + }); + expect(await errorOf(await run("trace", "0000", "--json"))).toMatchObject({ + exit: 1, + code: "tool_error", + message: "Trace not found.", + }); + + // A port that was just closed refuses the connection. + const closed = await new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(`http://127.0.0.1:${port}`)); + }); + }); + await writeFile( + join(home, sessionFile(closed)), + JSON.stringify({ server: closed, tokens: { access_token: TOKEN, token_type: "Bearer" } }), + ); + expect( + await errorOf( + await runWith( + { TELEMETRY_DEV_URL: closed }, + "traces", + "--project", + PROJECTS[0]!.id, + "--json", + ), + ), + ).toMatchObject({ exit: 1, code: "unreachable", retryable: true }); + + // After `--`, --json is a value, so the error stays the line a person reads. + const value = await run("use", "--", "--json"); + expect(value.code).toBe(2); + expect(value.stderr).toMatch(/^error No project "--json"/); + + // The same failures without --json keep the line a person reads. + const human = await run("trace", "0000"); + expect(human.code).toBe(1); + expect(human.stderr).toBe("error Trace not found.\n"); +}); + +test("a request that times out inside fetch is a retryable timeout, not a failed request", async () => { + const { Api } = await import("../src/api.ts"); + + for (const code of [ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", + ]) { + vi.stubGlobal("fetch", () => + Promise.reject( + new TypeError("fetch failed", { cause: Object.assign(new Error(""), { code }) }), + ), + ); + + const api = new Api({ + server: fixture.url, + tokens: { access_token: TOKEN, token_type: "Bearer" }, + }); + + try { + await expect(api.call("list_projects")).rejects.toMatchObject({ + code: "timeout", + retryable: true, + exitCode: 1, + }); + } finally { + vi.unstubAllGlobals(); + await api.close(); + } + } +}); + const savedSession = async () => JSON.parse(await readFile(sessionPath(), "utf8")); test("a shell without a terminal signs in with --no-browser, then --finish, keeping the old sign-in until then and revoking it after", async () => { From 02f63f5bb3ac5fc3c3c28d8198c0605961768077 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 08:54:19 +0000 Subject: [PATCH 25/32] feat(cli): summarize a trace's failures and repeated calls, and suggest next commands (#128) trace lists the failed spans where failures began, with their path, and the operations that ran more than once with their wall time counted once for overlapping calls. traces, sessions, and trace suggest the likely next command on stderr in human output, keeping the project and passing option values as --name=value. --- packages/cli/src/commands.ts | 290 +++++++++++++++++++++++++++++++-- packages/cli/tests/cli.test.ts | 108 +++++++++++- packages/cli/tests/fixture.ts | 155 +++++++++++++++--- 3 files changed, 517 insertions(+), 36 deletions(-) diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 06f401f..b70dd7a 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -15,6 +15,7 @@ import { clean, type Column, DASH, + displayWidth, formatAge, formatCost, formatCount, @@ -196,13 +197,59 @@ const choice = ( const DAYS = { "7d": 7, "30d": 30, "90d": 90 } as const; -/** `arg` quoted for a POSIX shell, so a suggested command runs as shown when pasted. */ +/** + * `arg` quoted for a POSIX shell, so a suggested command runs as shown when pasted. A + * leading `=` is quoted too: zsh expands `=name` to a command's path. + */ const shellQuote = (arg: string) => - /^[\w@%+=:,./-]+$/.test(arg) ? arg : `'${arg.replaceAll("'", `'\\''`)}'`; + /^[\w@%+:,./-][\w@%+=:,./-]*$/.test(arg) ? arg : `'${arg.replaceAll("'", `'\\''`)}'`; + +/** + * A telemetry command to suggest, quoted for a POSIX shell, that keeps this command's + * project so it reads the same one when pasted. Option values are joined with `=`, so a + * value starting with `-` isn't read as an option. Undefined when an id starts with `-` + * or a value holds characters the terminal line can't show as they are, since the + * printed command would then name something else. + */ +const suggestion = ( + context: Context, + command: string, + ids: readonly string[], + options: Readonly> = {}, +) => { + const project = string(context, "project"); + const values = { ...options, ...(project ? { project } : {}) }; + const shown = (text: string) => clean(text) === text && !/[\n\t]/.test(text); + + if (ids.some((id) => id.startsWith("-") || !shown(id))) return undefined; + + if (!Object.values(values).every(shown)) return undefined; + + return [ + "telemetry", + command, + ...ids, + ...Object.entries(values).map(([name, value]) => `--${name}=${value}`), + ] + .map(shellQuote) + .join(" "); +}; + +const labelled = (label: string, command: string | undefined) => + command === undefined ? undefined : `${label}: ${command}`; -/** Suggests the same command with the next page's cursor. */ -const suggestNextPage = (context: Context, cursor: string | null) => { - if (!cursor || json(context)) return; +/** Prints likely next commands as one dim block on stderr, never with --json. */ +const suggest = (context: Context, ...lines: (string | undefined)[]) => { + const shown = lines.filter((line): line is string => line !== undefined); + + if (json(context) || shown.length === 0) return; + + note(style.dim(`\n${shown.join("\n")}`, process.stderr)); +}; + +/** The same command with the next page's cursor. */ +const nextPage = (context: Context, cursor: string | null) => { + if (!cursor) return undefined; const kept: string[] = []; @@ -218,9 +265,10 @@ const suggestNextPage = (context: Context, cursor: string | null) => { kept.push(arg); } - const command = [...kept, "--cursor", cursor].map((arg) => shellQuote(clean(arg))).join(" "); + // A cursor is base64url, which can start with `-`. + const command = [...kept, `--cursor=${cursor}`].map((arg) => shellQuote(clean(arg))).join(" "); - note(style.dim(`\nMore: telemetry ${command}`, process.stderr)); + return `More: telemetry ${command}`; }; const statusCell = (status: string): Cell => @@ -517,7 +565,15 @@ export async function tracesCommand(context: Context) { ), ); - suggestNextPage(context, page.next_cursor); + const failure = page.traces.find((trace) => trace.status === "error"); + + suggest( + context, + failure + ? labelled("Inspect the newest failure", suggestion(context, "trace", [failure.id])) + : labelled("Inspect the newest trace", suggestion(context, "trace", [page.traces[0]!.id])), + nextPage(context, page.next_cursor), + ); }); } @@ -579,7 +635,14 @@ export async function sessionsCommand(context: Context) { ), ); - suggestNextPage(context, page.next_cursor); + suggest( + context, + labelled( + "List the most recent session's traces", + suggestion(context, "traces", [], { session: page.sessions[0]!.id }), + ), + nextPage(context, page.next_cursor), + ); }); } @@ -676,7 +739,7 @@ export async function traceCommand(context: Context) { if (!traceId) throw usageError("Name the trace to show: telemetry trace "); - await withApi(async (api) => { + await withApi(async (api, session) => { const project = await resolveProject(api, context); let page = await api.call( "get_trace", @@ -721,9 +784,209 @@ export async function traceCommand(context: Context) { ), ); } + + // The span list carries only the start of an error; payload reads all of it. + const explained = failureCauses(spans).find((span) => span.error?.trim()); + + suggest( + context, + explained && grantedScopes(session).has(PAYLOADS_SCOPE) + ? labelled( + "Read the full error", + suggestion(context, "payload", [page.trace.id, explained.id], { field: "error" }), + ) + : undefined, + ); }); } +/** + * Failed spans with no failed child, in the order they started. Instrumentation marks + * every span an exception passes through as failed, so these are where failures began. + */ +const failureCauses = (spans: readonly Span[]) => { + const failed = spans.filter((span) => span.status === "error"); + const failedParents = new Set(failed.map((span) => span.parent_span_id)); + + return failed + .filter((span) => !failedParents.has(span.id)) + .sort((left, right) => Date.parse(left.started_at) - Date.parse(right.started_at)); +}; + +// The Errors and Repeated sections each show this many rows, then a count of the rest. +const SECTION_ROWS = 5; + +/** A section title like Logs, saying so when it covers only the spans read so far. */ +const sectionTitle = (title: string, count: number, partialOf: number | undefined) => + style.bold( + `${title} ${style.dim(`(${formatInteger(count)}${partialOf === undefined ? "" : ` in the first ${formatInteger(partialOf)} spans`})`)}`, + ); + +const more = (count: number) => + count > SECTION_ROWS ? [style.dim(` and ${formatInteger(count - SECTION_ROWS)} more`)] : []; + +/** The first line of the span's error text, after its type unless the text starts with it. */ +const errorSummary = (span: Span) => { + const type = oneLine(span.error_type ?? ""); + const first = + (span.error ?? "") + .split(/\r\n|\r|\n/) + .map(oneLine) + .find(Boolean) ?? ""; + + if (!first) return type || "error"; + + return !type || first.startsWith(type) ? first : `${type}: ${first}`; +}; + +const PATH = " › "; + +// On a narrow terminal the error text is cut to this many columns before any ancestor +// is dropped from the path, and is left out when fewer than ERROR_MIN columns remain. +const ERROR_ROOM = 30; +const ERROR_MIN = 8; +// The fewest columns worth giving the ancestors before leaving them out. +const ANCESTRY_MIN = 4; + +/** + * A failed span on one line: its ancestors, the span, and its error, fitted to `width`. + * The span's full error text is in the tree below, so the path keeps priority. + */ +const failureLine = (span: Span, byId: ReadonlyMap, width: number) => { + const ancestors: string[] = []; + const seen = new Set([span.id]); + let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined; + + while (parent && !seen.has(parent.id)) { + seen.add(parent.id); + ancestors.push(spanLabel(parent)); + parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined; + } + + ancestors.reverse(); + + const leaf = spanLabel(span); + const error = errorSummary(span); + const room = width - 2; + const errorWidth = displayWidth(error); + const reserve = Math.min(errorWidth, ERROR_ROOM); + const pathWidth = (kept: readonly string[]) => displayWidth([...kept, leaf].join(PATH)); + let kept = ancestors; + + // The root stays, so the path still says which agent or request it was under. + for ( + let dropped = 1; + dropped < ancestors.length && pathWidth(kept) + 2 + reserve > room; + dropped += 1 + ) { + kept = [ancestors[0]!, "…", ...ancestors.slice(dropped + 1)]; + } + + // When even the shortened path is too wide, the ancestors are cut, not the failed span. + const ancestry = kept.join(PATH); + const ancestryRoom = room - displayWidth(leaf) - displayWidth(PATH); + const lead = + kept.length === 0 + ? "" + : displayWidth(ancestry) <= ancestryRoom + ? `${ancestry}${PATH}` + : ancestryRoom >= ANCESTRY_MIN + ? `${fit(ancestry, ancestryRoom)}${PATH}` + : ""; + const path = fit(`${lead}${leaf}`, room); + const painted = path === `${lead}${leaf}` ? `${style.dim(lead)}${leaf}` : path; + const left = room - displayWidth(path) - 2; + const shown = left >= Math.min(errorWidth, ERROR_MIN) ? ` ${style.red(fit(error, left))}` : ""; + + return ` ${painted}${shown}`; +}; + +/** Milliseconds covered by at least one interval, so calls that overlap count once. */ +const wallTime = (intervals: readonly (readonly [number, number])[]) => { + let covered = 0; + let reached = Number.NEGATIVE_INFINITY; + + for (const [start, end] of [...intervals].sort((left, right) => left[0] - right[0])) { + if (end <= reached) continue; + covered += end - Math.max(start, reached); + reached = end; + } + + return covered; +}; + +/** + * Where a long trace went wrong and where its time went, above the span tree: failed + * spans with their path, and operations that ran more than once. + */ +const traceSummary = (spans: readonly Span[], total: number, partialOf: number | undefined) => { + const lines: string[] = []; + const byId = new Map(spans.map((span) => [span.id, span])); + const failed = failureCauses(spans); + const only = failed.length === 1 ? failed[0] : undefined; + // A root that failed with no message says nothing the header doesn't. + const rootOnly = + only !== undefined && + !only.error?.trim() && + !(only.parent_span_id && byId.has(only.parent_span_id)); + + if (failed.length > 0 && !rootOnly) { + lines.push(sectionTitle("Errors", failed.length, partialOf)); + + for (const span of failed.slice(0, SECTION_ROWS)) + lines.push(failureLine(span, byId, outputWidth())); + lines.push(...more(failed.length), ""); + } + + // Grouped by the label the span tree shows: the name with its model, tool, or agent. + const groups = new Map(); + + for (const span of spans) { + const start = Date.parse(span.started_at); + + if (!Number.isFinite(start)) continue; + + const label = spanLabel(span); + const intervals = groups.get(label) ?? []; + + intervals.push([start, start + Math.max(0, span.duration_ms ?? 0)]); + groups.set(label, intervals); + } + + const repeated = [...groups] + .filter(([, intervals]) => intervals.length > 1) + .map(([label, intervals]) => ({ label, count: intervals.length, wall: wallTime(intervals) })) + .sort((left, right) => right.wall - left.wall); + + if (repeated.length > 0) { + lines.push(sectionTitle("Repeated", repeated.length, partialOf)); + + // The empty first column indents the rows; its header line is dropped. + const rows = table( + [ + { header: "" }, + { header: "", flex: true }, + { header: "", align: "right" }, + { header: "", align: "right" }, + { header: "", align: "right" }, + ], + repeated + .slice(0, SECTION_ROWS) + .map(({ label, count, wall }) => [ + "", + label, + `×${formatInteger(count)}`, + formatDuration(wall), + { text: total > 0 ? formatPercent((wall / total) * 100) : DASH, paint: dim }, + ]), + ); + + lines.push(...rows.split("\n").slice(1), ...more(repeated.length), ""); + } + + return lines; +}; + function renderTrace(page: TraceDetailPage, spans: readonly Span[], logs: TraceDetailPage["logs"]) { const { trace } = page; const failed = trace.status === "error"; @@ -765,6 +1028,13 @@ function renderTrace(page: TraceDetailPage, spans: readonly Span[], logs: TraceD const end = Math.max(...spans.map((span, index) => starts[index]! + (span.duration_ms ?? 0))); const total = Math.max(end - origin, trace.duration_ms ?? 0); + for (const line of traceSummary( + spans, + total, + page.next_cursor === null ? undefined : spans.length, + )) + out(line); + const terminal = process.stdout.isTTY === true; const timelineWidth = terminal ? Math.min(60, Math.max(16, outputWidth() - 72)) : 0; diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index f2c310c..1f4743f 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -12,11 +12,14 @@ import { beforeRefresh, calls, EMAIL, + PARALLEL_TRACE_ID, PAYLOADS, PROJECTS, + QUIET_TRACE_ID, REFRESH_TOKEN, startFixture, TOKEN, + TRACE_ID, traces, } from "./fixture.ts"; @@ -35,17 +38,16 @@ afterAll(async () => { const sessionPath = () => join(home, sessionFile(fixture.url)); -const signIn = (project?: (typeof PROJECTS)[number]) => +const signIn = ( + project?: (typeof PROJECTS)[number], + scope = "openid email offline_access telemetry:read telemetry:payloads", +) => writeFile( sessionPath(), JSON.stringify({ server: fixture.url, client: { client_id: "fixture-client" }, - tokens: { - access_token: TOKEN, - token_type: "Bearer", - scope: "openid email offline_access telemetry:read telemetry:payloads", - }, + tokens: { access_token: TOKEN, token_type: "Bearer", scope }, email: "ada@acme.dev", ...(project ? { project: { id: project.id, name: "support-bot" } } : {}), }), @@ -155,6 +157,100 @@ test("payload --field prints the whole text across pages", async () => { expect(calls.filter((call) => call.tool === "get_span_payload").length).toBeGreaterThan(1); }); +test("trace leads with where the run failed and which calls repeated, counting parallel calls once", async () => { + await signIn(PROJECTS[0]); + + const result = await run("trace", PARALLEL_TRACE_ID); + expect(result.code).toBe(0); + + const lines = result.stdout.split("\n"); + // Three 1 s searches started 50 ms apart cover 100 ms to 1,200 ms: 1.1 s of the 3 s + // trace, where adding them up would say 3 s and 100%. + expect(lines).toContain(" execute_tool web_search ×3 1.10s 37%"); + // The request that failed, under the tool that made it, under the agent. + expect(lines).toContain( + " invoke_agent researcher › execute_tool fetch_page › GET /page TimeoutError: Request timed out after 700 ms", + ); + expect(lines.indexOf("Errors (1)")).toBeGreaterThan(0); + expect(lines.indexOf("Repeated (1)")).toBeGreaterThan(lines.indexOf("Errors (1)")); + expect(lines.findIndex((line) => line.startsWith("SPAN"))).toBeGreaterThan( + lines.indexOf("Repeated (1)"), + ); + // Only the request where the failure began is listed and offered, not the tool and the + // agent it passed through. + expect(result.stderr.match(/telemetry payload .*/g)).toEqual([ + `telemetry payload ${PARALLEL_TRACE_ID} t2 --field=error`, + ]); + + // Two chat calls that don't overlap cover 1.18 s + 1.42 s of the 4.21 s trace. Measuring + // from the first start to the last end would say 2.64 s. + const sequential = await run("trace", TRACE_ID); + expect(sequential.stdout.split("\n")).toContain(" chat gpt-5 ×2 2.60s 62%"); + + // A sign-in without the payloads scope isn't pointed at a command that would refuse it. + await signIn(PROJECTS[0], "openid email offline_access telemetry:read"); + const unscoped = await run("trace", PARALLEL_TRACE_ID); + expect(unscoped.code).toBe(0); + expect(unscoped.stderr).not.toContain("telemetry payload"); +}); + +test("a trace with no failures and no repeated calls goes straight to the span tree", async () => { + await signIn(PROJECTS[0]); + + const result = await run("trace", QUIET_TRACE_ID); + expect(result.code).toBe(0); + expect(result.stdout).toMatch(/^SPAN /m); + expect(result.stdout).not.toMatch(/^(Errors|Repeated) /m); + expect(result.stderr).toBe(""); +}); + +/** The words a POSIX shell makes of a pasted command line. */ +const shellWords = (command: string) => + new Promise((resolve, reject) => { + execFile("sh", ["-c", `printf '%s\\n' ${command}`], (error, stdout) => + error ? reject(error) : resolve(stdout.trimEnd().split("\n")), + ); + }); + +test("suggested next commands run as pasted, in the same project, and never print with --json", async () => { + await signIn(); + + // The newest trace succeeded, so the suggestion names the failed one after it. + const listed = await run("traces", "-p", "support-bot"); + expect(listed.code).toBe(0); + const inspect = listed.stderr.match(/Inspect the newest failure: (.*)/)?.[1]; + expect(await shellWords(inspect!)).toEqual([ + "telemetry", + "trace", + TRACE_ID, + "--project=support-bot", + ]); + + const sessions = await run("sessions", "--project=support-bot"); + expect(sessions.code).toBe(0); + const words = await shellWords(sessions.stderr.match(/telemetry traces .*/)![0]); + expect(words).toEqual([ + "telemetry", + "traces", + "--session=Ada's thread 42", + "--project=support-bot", + ]); + + // Run as pasted, it asks for that session in that project. + calls.length = 0; + expect((await run(...words.slice(1))).code).toBe(0); + expect(calls.find((call) => call.tool === "search_traces")?.args).toMatchObject({ + project_id: PROJECTS[0]!.id, + session_id: "Ada's thread 42", + }); + + for (const args of [["traces"], ["sessions"], ["trace", PARALLEL_TRACE_ID]]) { + const result = await run(...args, "-p", "support-bot", "--json"); + expect(result.code).toBe(0); + expect(result.stderr).toBe(""); + } +}); + test("use saves the default project in a file only its owner can read", async () => { await signIn(); diff --git a/packages/cli/tests/fixture.ts b/packages/cli/tests/fixture.ts index f0dc89c..707483c 100644 --- a/packages/cli/tests/fixture.ts +++ b/packages/cli/tests/fixture.ts @@ -24,10 +24,31 @@ export const PROJECTS = [ const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000).toISOString(); -const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +export const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +export const QUIET_TRACE_ID = "9a0b7c6d5e4f30211a2b3c4d5e6f7081"; +export const PARALLEL_TRACE_ID = "3e8a1c5b7d9f40e2a6b4c8d0e2f4a6b8"; export const traces = () => ({ traces: [ + // The newest trace succeeded, so a suggestion to inspect a failure has to look past it. + { + id: QUIET_TRACE_ID, + name: "support.answer", + status: "success", + error_type: null, + started_at: minutesAgo(1), + duration_ms: 1870, + environment: "production", + service_name: "support-api", + session_id: "conv_8f2a91", + user_id: "user_4821", + models: ["gpt-5"], + span_count: 3, + error_span_count: 0, + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }, { id: TRACE_ID, name: "support.answer", @@ -46,24 +67,6 @@ export const traces = () => ({ output_tokens: 373, cost_usd: 0.01234, }, - { - id: "9a0b7c6d5e4f30211a2b3c4d5e6f7081", - name: "support.answer", - status: "success", - error_type: null, - started_at: minutesAgo(9), - duration_ms: 1870, - environment: "production", - service_name: "support-api", - session_id: "conv_8f2a91", - user_id: "user_4821", - models: ["gpt-5"], - span_count: 5, - error_span_count: 0, - input_tokens: 1934, - output_tokens: 211, - cost_usd: 0.00718, - }, { id: "1f2e3d4c5b6a79880a9b8c7d6e5f4a3b", // Captured text can carry terminal escapes; the CLI must never pass them through. @@ -184,8 +187,116 @@ export const traceDetail = () => ({ next_cursor: null, }); +/** A run that succeeded with each operation once: nothing to summarize above the tree. */ +const quietTraceDetail = () => ({ + trace: { + id: QUIET_TRACE_ID, + name: "support.answer", + status: "success", + error_type: null, + started_at: minutesAgo(1), + duration_ms: 1870, + environment: "production", + service_name: "support-api", + framework: "ai-sdk", + session_id: "conv_8f2a91", + user_id: "user_4821", + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }, + span_count: 3, + log_count: 0, + spans: [ + span("q1", null, "invoke_agent support-bot", 0, 1870, { + agent_name: "support-bot", + operation: "invoke_agent", + }), + span("q2", "q1", "execute_tool lookup_order", 30, 240, { tool_name: "lookup_order" }), + span("q3", "q1", "chat gpt-5", 290, 1560, { + model: "gpt-5", + provider: "openai", + input_tokens: 1934, + output_tokens: 211, + cost_usd: 0.00718, + }), + ], + logs: [], + next_cursor: null, +}); + +/** + * An agent that runs three 1 s searches at once, so they cover 1.1 s of a 3 s trace, then + * fails two levels down, in the request a tool made. The failure is marked on the tool + * and the agent too, as instrumentation does when an exception passes through them. + */ +const parallelTraceDetail = () => ({ + trace: { + id: PARALLEL_TRACE_ID, + name: "research.report", + status: "error", + error_type: "TimeoutError", + started_at: minutesAgo(2), + duration_ms: 3000, + environment: "production", + service_name: "research-api", + framework: "ai-sdk", + session_id: null, + user_id: null, + input_tokens: null, + output_tokens: null, + cost_usd: null, + }, + span_count: 6, + log_count: 0, + spans: [ + span("r1", null, "invoke_agent researcher", 0, 3000, { + agent_name: "researcher", + operation: "invoke_agent", + status: "error", + error_type: "TimeoutError", + error: "TimeoutError: tool fetch_page failed", + }), + span("s1", "r1", "execute_tool web_search", 100, 1000, { tool_name: "web_search" }), + span("s2", "r1", "execute_tool web_search", 150, 1000, { tool_name: "web_search" }), + span("s3", "r1", "execute_tool web_search", 200, 1000, { tool_name: "web_search" }), + span("t1", "r1", "execute_tool fetch_page", 1300, 800, { + tool_name: "fetch_page", + status: "error", + error_type: "TimeoutError", + error: "TimeoutError: Request timed out after 700 ms", + }), + span("t2", "t1", "GET /page", 1350, 700, { + status: "error", + error_type: "TimeoutError", + error: "Request timed out after 700 ms\n at fetchPage (tools.ts:41:11)", + }), + ], + logs: [], + next_cursor: null, +}); + +const TRACE_DETAILS: Record unknown> = { + [TRACE_ID]: traceDetail, + [QUIET_TRACE_ID]: quietTraceDetail, + [PARALLEL_TRACE_ID]: parallelTraceDetail, +}; + export const sessions = () => ({ sessions: [ + { + // Session ids come from the user's app and can hold spaces and quotes. + id: "Ada's thread 42", + last_active_at: minutesAgo(1), + user_id: "user_4821", + agent_name: "support-bot", + models: ["gpt-5"], + trace_count: 2, + error_trace_count: 0, + input_tokens: 3120, + output_tokens: 402, + cost_usd: 0.0114, + }, { id: "conv_8f2a91", last_active_at: minutesAgo(2), @@ -337,7 +448,11 @@ const server = () => { tool("list_projects", () => ({ organization_id: "org_fixture", projects: PROJECTS })); tool("search_traces", () => traces()); tool("search_sessions", () => sessions()); - tool("get_trace", (args) => (args.trace_id === TRACE_ID ? traceDetail() : "Trace not found.")); + tool("get_trace", (args) => + Object.hasOwn(TRACE_DETAILS, String(args.trace_id)) + ? TRACE_DETAILS[String(args.trace_id)]!() + : "Trace not found.", + ); tool("list_logs", () => logs()); tool("get_usage_summary", () => usage()); tool("get_span_payload", (args) => { From 56b7ba4e148d8f4fb552e6948782e5345064bb74 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 09:50:08 +0000 Subject: [PATCH 26/32] test(cli): give each cli test 30 seconds, since each one starts the cli as a process (#130) A test that starts the CLI seven times took 5,031 ms on a CI runner, just past vitest's 5 second default, and failed main after #128. --- packages/cli/vite.config.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cli/vite.config.ts b/packages/cli/vite.config.ts index cf8ebff..6a497a8 100644 --- a/packages/cli/vite.config.ts +++ b/packages/cli/vite.config.ts @@ -26,4 +26,9 @@ export default defineConfig({ }, }, fmt: {}, + test: { + // Each test starts the CLI as a process, several times; one start takes up to 700ms + // on a CI runner, which pushes the longer tests past the 5s default. + testTimeout: 30_000, + }, }); From 88a2e0dfe3c04bca6c95d32471ffeade31cd5128 Mon Sep 17 00:00:00 2001 From: Ephraim Duncan <55143799+ephraimduncan@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:27:58 +0000 Subject: [PATCH 27/32] ci(release): release the experiment action with a major tag (#126) A workflow that used the experiment action at main ran each change on the main branch at once. Release Please now gives the action releases such as experiment-action-v1.0.0. After each release, a new job moves experiment-action-v1 to the release commit, but never back to an earlier release. The READMEs tell users to use experiment-action-v1. --- .github/scripts/release_test.sh | 74 +++++++++++++++++++++++++++++++-- .github/workflows/release.yml | 38 +++++++++++++++++ RELEASING.md | 13 ++++++ actions/experiment/README.md | 12 ++++-- packages/sdk/README.md | 3 +- release-please-config.json | 5 +++ sdks/python/README.md | 3 +- 7 files changed, 140 insertions(+), 8 deletions(-) diff --git a/.github/scripts/release_test.sh b/.github/scripts/release_test.sh index 92dc97d..2e672c6 100755 --- a/.github/scripts/release_test.sh +++ b/.github/scripts/release_test.sh @@ -152,6 +152,19 @@ mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" run_push_discovery false '["packages/mcp"]' grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" git -C "$repo" checkout --quiet -- release-please-config.json +jq '.packages["actions/experiment"] = {"component": "experiment-action", "release-type": "simple"}' \ + "$repo/release-please-config.json" > "$repo/release-please-config.json.next" +mv "$repo/release-please-config.json.next" "$repo/release-please-config.json" +jq '.["actions/experiment"] = "1.0.0"' "$repo/.release-please-manifest.json" > "$repo/.release-please-manifest.json.next" +mv "$repo/.release-please-manifest.json.next" "$repo/.release-please-manifest.json" +git -C "$repo" tag experiment-action-v1.0.0 +for released in '["actions/experiment","packages/mcp"]' '[]'; do + run_push_discovery false "$released" + grep -Fxq 'npm=["packages/otel","packages/mcp"]' "$tmp/output" + grep -Fxq 'python=["sdks/python"]' "$tmp/output" +done +git -C "$repo" tag --delete experiment-action-v1.0.0 > /dev/null +git -C "$repo" checkout --quiet -- release-please-config.json .release-please-manifest.json expect_failure "Expected 2 configured packages for released paths, found 1" \ run_push_discovery false '["packages/mcp","packages/unknown"]' expect_failure "Release otel-v0.1.0 is still a draft" run_push_discovery true '["packages/otel"]' @@ -161,6 +174,60 @@ git -C "$repo" commit --quiet --allow-empty -m after-batch expect_failure "Release tag, event, and checkout commits do not match" \ run_push_discovery false '["packages/otel"]' "$(git -C "$repo" rev-parse HEAD)" +action_repo="$tmp/action-repo" +git init --quiet --initial-branch=main "$action_repo" +git -C "$action_repo" config user.name "Release Test" +git -C "$action_repo" config user.email "release-test@example.invalid" +action_commit() { + printf '%s\n' "$1" > "$action_repo/.release-please-manifest.json" + git -C "$action_repo" add .release-please-manifest.json + git -C "$action_repo" commit --quiet -m "$1" + git -C "$action_repo" rev-parse HEAD +} +unreleased_sha=$(action_commit '{}') +v1_0_0_sha=$(action_commit '{"actions/experiment":"1.0.0"}') +git -C "$action_repo" tag experiment-action-v1.0.0 +other_release_sha=$(action_commit '{"actions/experiment":"1.0.0","packages/mcp":"0.1.2"}') +v1_0_1_sha=$(action_commit '{"actions/experiment":"1.0.1"}') +git -C "$action_repo" tag experiment-action-v1.0.1 +v2_0_0_sha=$(action_commit '{"actions/experiment":"2.0.0"}') +git -C "$action_repo" tag experiment-action-v2.0.0 +mkdir -p "$tmp/tag-bin" +cat > "$tmp/tag-bin/gh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +SH +chmod +x "$tmp/tag-bin/gh" +extract_run tag-experiment-action > "$tmp/tag-action.sh" +run_tag_action() { + ( + cd "$action_repo" + git switch --quiet --detach "$1" + rm -f "$tmp/gh-log" + PATH="$tmp/tag-bin:$PATH" GH_LOG="$tmp/gh-log" GITHUB_REPOSITORY=telemetry-dev/telemetry GITHUB_SHA="$1" \ + bash -eo pipefail "$tmp/tag-action.sh" + ) +} +expect_gh_call() { + [[ $(wc -l < "$tmp/gh-log") -eq 1 ]] + grep -Fxq "$1" "$tmp/gh-log" +} +for sha in "$unreleased_sha" "$other_release_sha"; do + run_tag_action "$sha" + [[ ! -e "$tmp/gh-log" ]] +done +run_tag_action "$v1_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v1 -f sha=$v1_0_0_sha" +git -C "$action_repo" tag experiment-action-v1 "$v1_0_0_sha" +run_tag_action "$v1_0_1_sha" +expect_gh_call "api --method PATCH repos/telemetry-dev/telemetry/git/refs/tags/experiment-action-v1 -f sha=$v1_0_1_sha -F force=true" +git -C "$action_repo" tag --force experiment-action-v1 "$v1_0_1_sha" > /dev/null +# A rerun of an earlier release must not move the major tag back. +run_tag_action "$v1_0_0_sha" +[[ ! -e "$tmp/gh-log" ]] +run_tag_action "$v2_0_0_sha" +expect_gh_call "api --method POST repos/telemetry-dev/telemetry/git/refs -f ref=refs/tags/experiment-action-v2 -f sha=$v2_0_0_sha" + release_tree=$(git -C "$repo" show --no-patch --format=%T "$release_sha") unrelated_sha=$(printf 'unrelated\n' | git -C "$repo" commit-tree "$release_tree") git -C "$repo" switch --quiet --detach "$unrelated_sha" @@ -415,7 +482,8 @@ workflow_pnpm=$( ' "$root/.github/workflows/release.yml" ) [[ "$workflow_pnpm" == "$package_pnpm" ]] -[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 2 ]] +[[ $(grep -Ec '^[[:space:]]*concurrency:' "$root/.github/workflows/release.yml") -eq 3 ]] +grep -Fxq " group: tag-experiment-action" "$root/.github/workflows/release.yml" grep -Fxq " group: release-\${{ inputs.release_tag || github.sha }}" \ "$root/.github/workflows/release.yml" grep -Fxq " group: release-please-\${{ github.ref }}" \ @@ -455,13 +523,13 @@ for path, package in packages.items(): manifest = json.loads((root / path / "package.json").read_text()) paths[manifest["name"]] = path dependencies[path] = [*manifest.get("dependencies", {}), *manifest.get("peerDependencies", {})] - else: + elif package["release-type"] == "python": project = tomllib.loads((root / path / "pyproject.toml").read_text())["project"] paths[normalize(project["name"])] = path dependencies[path] = [ normalize(re.match(r"[A-Za-z0-9._-]+", requirement)[0]) for requirement in project.get("dependencies", []) ] -for path in order: +for path in dependencies: for dependency in dependencies[path]: dependency_path = paths.get(dependency) if dependency_path and order.index(dependency_path) > order.index(path): diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac0f9bb..621be50 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,6 +96,8 @@ jobs: case "$release_type" in node) npm+=("$path") ;; python) python+=("$path") ;; + # The experiment action has no package: the tag-experiment-action job moves its major tag. + simple) continue ;; *) echo "Unsupported release type for $path: $release_type" >&2 exit 1 @@ -289,3 +291,39 @@ jobs: .github/scripts/verify-python-release-dependencies.sh "$wheel" uv publish --trusted-publishing always --check-url https://pypi.org/simple "python/$package"/* done < python/publish-order.txt + + tag-experiment-action: + needs: release-please + concurrency: + group: tag-experiment-action + cancel-in-progress: false + # The default queue cancels a waiting job when another push arrives, and that release then keeps the old major tag. + queue: max + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + - env: + GH_TOKEN: ${{ github.token }} + run: | + version=$(jq -r '.["actions/experiment"] // empty' .release-please-manifest.json) + # A rerun after release-please failed partway gets no outputs for the release, so its tag selects it. + if [[ -z "$version" || "$(git rev-parse -q --verify "refs/tags/experiment-action-v$version^{commit}" || true)" != "$GITHUB_SHA" ]]; then + exit 0 + fi + + # Workflows use the major tag, which moves to each release of its major version, but never back. + major="experiment-action-v${version%%.*}" + current=$(git rev-parse -q --verify "refs/tags/$major^{commit}" || true) + if [[ -z "$current" ]]; then + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f "ref=refs/tags/$major" -f "sha=$GITHUB_SHA" + elif git merge-base --is-ancestor "$current" "$GITHUB_SHA"; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/tags/$major" -f "sha=$GITHUB_SHA" -F force=true + else + echo "$major is at a release after experiment-action-v$version" + fi diff --git a/RELEASING.md b/RELEASING.md index 064b46b..ada9052 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -32,3 +32,16 @@ Each registry's publish job stops at the first package that fails, so the packag To retry an existing published release, run the `Release` workflow from `main` and set `release_tag` to that release's full tag. This is a real publication attempt, not a dry run. The tag must be an ancestor of the selected `main` commit. Every configured SDK directory, shared build scripts, lockfiles, release configuration, and root build configuration must be unchanged from the tag; only `packageManager` may differ within root `package.json`. Workflow-only recovery changes are allowed. Source-changing fixes need a new release, not a retry of an old version. Artifact names and versions are checked before upload. npm retries skip an existing version only when its SHA-512 integrity matches the local tarball, and publication waits about 10 minutes for published `@telemetry-dev/*` dependencies and peer dependencies to satisfy their declared ranges. Python publication likewise waits about 10 minutes for each wheel's dependencies to resolve on PyPI. PyPI retries use `uv publish --check-url` to check existing distributions. An integrity mismatch must be investigated, not bypassed or overwritten. + +## Experiment action + +The experiment action in `actions/experiment` has no package. Release Please gives it releases such as `experiment-action-v1.0.0`. The `tag-experiment-action` job of the `Release` workflow then moves the major tag, `experiment-action-v1`, to the release commit. The README of the action and the experiment page of telemetry.dev tell users to use the major tag. The job moves a major tag only forward on `main`, so a rerun of an earlier release does not move it back. + +A change that can break a workflow, for example a removed input or a changed output, must have a `feat!:` or `fix!:` commit. Release Please then gives the action a new major version, and `experiment-action-v1` stays at the last 1.x release. A `release_tag` retry does not apply to the action. If the job fails, re-run the failed job of the release run. + +The job uses `GITHUB_TOKEN`, which cannot have the `workflows` permission. Thus GitHub rejects the tag when no branch head has the same `.github/workflows` files as the release commit. If a merge changed a workflow before the job ran, move the tag with your own credentials: + +```sh +git fetch origin --tags +git push --force origin 'experiment-action-v1.0.1^{commit}:refs/tags/experiment-action-v1' +``` diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 431f2c0..5e7cbbd 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -27,7 +27,7 @@ jobs: node-version: 24 # Installs tsx too, which runs the .ts script. - run: npm ci - - uses: telemetry-dev/telemetry/actions/experiment@main + - uses: telemetry-dev/telemetry/actions/experiment@experiment-action-v1 with: dataset: support-golden script: evals/reply.eval.ts @@ -48,8 +48,14 @@ await runExperiment({ }); ``` -Pin the action to a commit SHA of this repository for a run that does not change when `main` -changes. +## Versions + +The tag `experiment-action-v1` moves to each 1.x release of the action. A change that can break a +workflow starts a new major version, `experiment-action-v2`, and `experiment-action-v1` stays at +the last 1.x release. For a run that does not change, use the tag of one release, for example +`experiment-action-v1.0.0`, or its commit SHA. The +[releases](https://github.com/telemetry-dev/telemetry/releases) of this repository list the +changes of each release. ## Inputs diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ed35293..81f726a 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -301,7 +301,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor the result on the pull request. It gives runExperiment the `dataset` input in `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the -same name. +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. ## Scores diff --git a/release-please-config.json b/release-please-config.json index e8c7e42..79b72d6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -191,6 +191,11 @@ "jsonpath": "$.package[?(@.name.value=='telemetry-dev-openrouter')].version" } ] + }, + "actions/experiment": { + "component": "experiment-action", + "release-type": "simple", + "initial-version": "1.0.0" } } } diff --git a/sdks/python/README.md b/sdks/python/README.md index c7eb87c..bd91e98 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -394,7 +394,8 @@ The [experiment action](../../actions/experiment) runs your eval script in a wor the result on the pull request. It gives `run_experiment` the `dataset` input in `TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the `min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the -same name. +same name. Use the action as `telemetry-dev/telemetry/actions/experiment@experiment-action-v1`, a +tag that moves to each 1.x release of the action. ## Scores From c935591561402b5049b8a537fc639c8e16142d4b Mon Sep 17 00:00:00 2001 From: Ephraim Duncan <55143799+ephraimduncan@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:32:24 +0000 Subject: [PATCH 28/32] chore: ignore the experiment action changelog in the formatter (#132) Release Please writes actions/experiment/CHANGELOG.md in its own format, and the check of the release PR stopped on that file. The formatter already ignores the changelogs of the npm packages for the same reason. --- vite.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vite.config.ts b/vite.config.ts index 6adbb93..b77ab81 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ staged: { "*": "vp check --fix --no-error-on-unmatched-pattern", }, - fmt: { ignorePatterns: ["sdks/**", "packages/*/CHANGELOG.md"] }, + fmt: { ignorePatterns: ["sdks/**", "packages/*/CHANGELOG.md", "actions/*/CHANGELOG.md"] }, lint: { ignorePatterns: ["sdks/**"], options: { typeAware: true, typeCheck: true }, From aa5b7e2c30f2296cead461de04ef805aa9296340 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 13:44:27 +0000 Subject: [PATCH 29/32] chore: release main (#129) Co-authored-by: Ephraim Duncan <55143799+ephraimduncan@users.noreply.github.com> --- .release-please-manifest.json | 3 ++- actions/experiment/CHANGELOG.md | 14 ++++++++++++++ packages/cli/CHANGELOG.md | 8 ++++++++ packages/cli/package.json | 2 +- 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 actions/experiment/CHANGELOG.md diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8d8f051..d727003 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -21,5 +21,6 @@ "sdks/python-litellm": "0.2.2", "sdks/python-openai": "0.2.2", "sdks/python-openrouter": "0.2.2", - "packages/cli": "0.1.0" + "packages/cli": "0.1.1", + "actions/experiment": "1.0.0" } diff --git a/actions/experiment/CHANGELOG.md b/actions/experiment/CHANGELOG.md new file mode 100644 index 0000000..fb5a0a6 --- /dev/null +++ b/actions/experiment/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## 1.0.0 (2026-10-30) + + +### Features + +* **sdk:** add the experiment action for pull requests ([968a45e](https://github.com/telemetry-dev/telemetry/commit/968a45e0eeff183d2409c0336c6328ac07a2e36d)) +* **sdk:** run experiments on datasets and send scores ([fe00efa](https://github.com/telemetry-dev/telemetry/commit/fe00efa1326797f8fda647d665b388621a961d39)) + + +### Bug Fixes + +* **sdk:** use the tsx of the project and change only our comment ([ae0ed55](https://github.com/telemetry-dev/telemetry/commit/ae0ed556f68d1952cbe33ca8f93f29a429fab5a4)) diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index f43ce5e..4056cdd 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.1.1](https://github.com/telemetry-dev/telemetry/compare/cli-v0.1.0...cli-v0.1.1) (2026-10-30) + + +### Features + +* **cli:** report errors as one JSON line with a stable code under --json ([#127](https://github.com/telemetry-dev/telemetry/issues/127)) ([dcad14b](https://github.com/telemetry-dev/telemetry/commit/dcad14b08926862409cfaf94c529ecfab8df26e5)) +* **cli:** summarize a trace's failures and repeated calls, and suggest next commands ([#128](https://github.com/telemetry-dev/telemetry/issues/128)) ([02f63f5](https://github.com/telemetry-dev/telemetry/commit/02f63f5bb3ac5fc3c3c28d8198c0605961768077)) + ## 0.1.0 (2026-10-30) diff --git a/packages/cli/package.json b/packages/cli/package.json index daad4d3..5cf9b7f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/cli", - "version": "0.1.0", + "version": "0.1.1", "description": "The telemetry.dev command line: search traces, sessions, logs, and AI usage from your terminal or an agent's shell.", "keywords": [ "cli", From 505309bf84c00edf3036488348086e10a5870510 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 14:15:26 +0000 Subject: [PATCH 30/32] feat(cli): add --json to login and logout (#131) login --json prints the sign-in link and returns without opening a browser or waiting; login --finish --json prints the sign-in. Sign-in network errors now report unreachable or timeout, and a failed project lookup after sign-in no longer fails login. --- packages/cli/README.md | 20 ++++- packages/cli/src/api.ts | 74 +---------------- packages/cli/src/auth.ts | 29 +++++-- packages/cli/src/cli.ts | 11 ++- packages/cli/src/commands.ts | 142 ++++++++++++++++++++++----------- packages/cli/src/errors.ts | 72 +++++++++++++++++ packages/cli/tests/cli.test.ts | 122 ++++++++++++++++++++++++++-- packages/cli/tests/fixture.ts | 9 ++- 8 files changed, 341 insertions(+), 138 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index feb416b..6088442 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -34,6 +34,18 @@ In a shell with no terminal to paste into, such as an agent's, `telemetry login telemetry login --finish 'http://127.0.0.1:53124/callback?code=…' ``` +A script or agent can run `telemetry login --json` instead. It never opens a browser or waits. It prints the link to pass on to a person, then exits: + +```json +{ + "status": "pending", + "authorization_url": "https://telemetry.dev/api/auth/oauth2/authorize?…", + "next": "telemetry login --finish '
'" +} +``` + +`telemetry login --finish '
' --json` then prints the sign-in, with the same fields as `telemetry status --json` and `"status": "signed_in"`. When the workspace has exactly one project, it becomes the default. `project` is `null` otherwise, and also when the projects couldn't be loaded. + A sign-in covers one workspace. To read another workspace, run `telemetry login` again and choose it. The new sign-in replaces the old one, and the CLI asks telemetry.dev to revoke the old one. ## Use @@ -57,7 +69,7 @@ List commands return 20 results by default. Use `--limit` to return from 1 to 50 ## Scripts and agents -Add `--json` to any command except `login` and `logout` for machine-readable output on stdout. Messages and progress go to stderr, so `telemetry traces --json > traces.json` writes only data. List commands print the MCP tool's result as returned, with the fields that the [MCP server documentation](https://telemetry.dev/docs/platform/mcp-server) describes. +Add `--json` to any command for machine-readable output on stdout. Messages and progress go to stderr, so `telemetry traces --json > traces.json` writes only data. List commands print the MCP tool's result as returned, with the fields that the [MCP server documentation](https://telemetry.dev/docs/platform/mcp-server) describes. `telemetry trace` and `telemetry payload` read every page of a trace or a captured text, up to a limit: 2,000 spans for a trace, and 1,000,000 characters for each payload field. Their JSON output has `complete: false` when the limit cut it short, and human output says so on stderr. @@ -77,7 +89,7 @@ With `--json`, an error is one line of JSON on stderr, and stdout stays empty. C {"error":{"code":"not_signed_in","message":"You're not signed in. Run `telemetry login` first.","hint":"telemetry login","retryable":false,"exit_code":4,"cli_version":"0.1.0"}} ``` -Branch on `code`, not on `message`, whose wording can change. `hint` is a command to run next, when there is one; `telemetry login` needs a person at a browser, so an agent should ask for it rather than run it. `retryable` is true when running the same command again may succeed. New codes may be added, and existing codes keep their meaning. +Branch on `code`, not on `message`, whose wording can change. `hint` is a command to run next, when there is one. `telemetry login` needs a person at a browser, so an agent should run `telemetry login --json` and pass the link on rather than run `telemetry login`. `retryable` is true when running the same command again may succeed. New codes may be added, and existing codes keep their meaning. | Code | Exit | Meaning | | ---------------------- | ---- | -------------------------------------------------------------------------------- | @@ -90,6 +102,8 @@ Branch on `code`, not on `message`, whose wording can change. `hint` is a comman | `not_signed_in` | 4 | No sign-in is saved for this deployment | | `sign_in_expired` | 4 | The sign-in expired or was disconnected | | `payloads_not_allowed` | 4 | The sign-in doesn't include reading captured prompts and outputs | +| `sign_in_failed` | 4 | The sign-in was cancelled, refused, or used an address from another sign-in | +| `no_pending_sign_in` | 4 | `login --finish` found no sign-in waiting to finish | | `unreachable` | 1 | telemetry.dev could not be reached | | `timeout` | 1 | telemetry.dev took too long to answer | | `tool_error` | 1 | telemetry.dev returned an error; `message` says what it was | @@ -107,4 +121,4 @@ Branch on `code`, not on `message`, whose wording can change. `hint` is a comman | `TELEMETRY_DEV_CONFIG_DIR` | `~/.config/telemetry-dev` | Where the sign-in is saved. The default follows `XDG_CONFIG_HOME` when set | | `NO_COLOR` | none | Turns off color. `--no-color` does the same | -The sign-in is saved in `session.json`, which only your user can read. A sign-in to another deployment through `TELEMETRY_DEV_URL` gets its own file next to it. The file holds a refresh token, so keep it private. `telemetry logout` revokes the token at telemetry.dev and removes it from the file, and says so when the server doesn't confirm the revocation. To disconnect the CLI from your account entirely, open account settings at telemetry.dev. +The sign-in is saved in `session.json`, which only your user can read. A sign-in to another deployment through `TELEMETRY_DEV_URL` gets its own file next to it. The file holds a refresh token, so keep it private. `telemetry logout` revokes the token at telemetry.dev and removes it from the file, and says so when the server doesn't confirm the revocation. `telemetry logout --json` prints `{"was_signed_in": true, "revoked": false}` in that case; `revoked` is `null` when there was no sign-in. To disconnect the CLI from your account entirely, open account settings at telemetry.dev. diff --git a/packages/cli/src/api.ts b/packages/cli/src/api.ts index 890ff8d..d448e7e 100644 --- a/packages/cli/src/api.ts +++ b/packages/cli/src/api.ts @@ -1,7 +1,7 @@ import type { Client } from "@modelcontextprotocol/client"; import { SessionAuthProvider } from "./auth.ts"; -import { CliError, EXIT, notSignedIn } from "./errors.ts"; +import { CliError, notSignedIn, translateError } from "./errors.ts"; import { mcpUrl, type Session } from "./session.ts"; import { withSpinner } from "./terminal.ts"; @@ -95,75 +95,3 @@ export class Api { await client?.close().catch(() => {}); } } - -// Each is a refresh or authorization the server refused for good. -type AuthErrors = readonly (abstract new (...args: never[]) => Error)[]; - -const NETWORK_CODES = new Set([ - "ECONNREFUSED", - "ECONNRESET", - "EAI_AGAIN", - "EHOSTUNREACH", - "ENETUNREACH", - "ENOTFOUND", - "ETIMEDOUT", - "UND_ERR_SOCKET", -]); - -// undici's own timeouts, raised by fetch before any of the CLI's timers fire. -const TIMEOUT_CODES = new Set([ - "UND_ERR_CONNECT_TIMEOUT", - "UND_ERR_HEADERS_TIMEOUT", - "UND_ERR_BODY_TIMEOUT", -]); - -const translateError = (error: unknown, authErrors: AuthErrors): never => { - if (error instanceof CliError) throw error; - - if (authErrors.some((type) => error instanceof type)) { - throw new CliError( - "Your sign-in has expired or was disconnected. Run `telemetry login` again.", - { exitCode: EXIT.auth, code: "sign_in_expired", hint: "telemetry login" }, - ); - } - - // The network error sits at the bottom of a chain: the SDK wraps fetch's - // "fetch failed", which wraps the socket error that has the code. - const chain: unknown[] = [error]; - - for (let link = error; link instanceof Error && link.cause !== undefined && chain.length < 8; ) { - link = link.cause; - chain.push(link); - } - - const codes = chain - .map((link) => (link as { code?: unknown } | null)?.code) - .filter((code): code is string => typeof code === "string"); - const network = codes.find((code) => NETWORK_CODES.has(code)); - // Node's AggregateError for a failed connection has an empty message. - const message = - chain.findLast((link): link is Error => link instanceof Error && link.message !== "") - ?.message ?? String(error); - - if (network) { - throw new CliError( - `Couldn't reach telemetry.dev (${network}). Check your connection and try again.`, - { code: "unreachable", retryable: true }, - ); - } - - if ( - codes.some((code) => code === "REQUEST_TIMEOUT" || TIMEOUT_CODES.has(code)) || - chain.some( - (link) => - link instanceof Error && (link.name === "TimeoutError" || link.name === "AbortError"), - ) - ) { - throw new CliError("telemetry.dev took too long to answer. Try again shortly.", { - code: "timeout", - retryable: true, - }); - } - - throw new CliError(`telemetry.dev request failed: ${message}`, { code: "request_failed" }); -}; diff --git a/packages/cli/src/auth.ts b/packages/cli/src/auth.ts index 7fcdde0..a486fe6 100644 --- a/packages/cli/src/auth.ts +++ b/packages/cli/src/auth.ts @@ -12,7 +12,7 @@ import type { StoredOAuthTokens, } from "@modelcontextprotocol/client"; -import { CliError, EXIT } from "./errors.ts"; +import { CliError, EXIT, translateError } from "./errors.ts"; import { loadSession, mcpUrl, type Session, updateSession } from "./session.ts"; import { oneLine } from "./terminal.ts"; @@ -426,6 +426,8 @@ const readPastedCallback = async (state: string, signal: AbortSignal): Promise void; @@ -458,8 +460,13 @@ const authorize = async ( provider: SessionAuthProvider, options: { authorizationCode?: string; iss?: string }, ) => { - const { auth, AuthorizationServerMismatchError, IssuerMismatchError, OAuthError } = - await import("@modelcontextprotocol/client"); + const { + auth, + AuthorizationServerMismatchError, + IssuerMismatchError, + OAuthClientFlowError, + OAuthError, + } = await import("@modelcontextprotocol/client"); try { return await auth(provider, { @@ -487,7 +494,16 @@ const authorize = async ( ); } - throw error; + // Such as a refused client registration or a token endpoint that isn't https. + if (error instanceof OAuthClientFlowError) { + throw new CliError(`Sign-in failed: ${oneLine(error.message).slice(0, 300)}`, { + exitCode: EXIT.auth, + code: "sign_in_failed", + hint: "telemetry login", + }); + } + + return translateError(error, []); } }; @@ -549,6 +565,7 @@ const redeem = async (provider: SessionAuthProvider, params: URLSearchParams) => export const login = async ({ session, browser, + defer = false, finish, onAuthorizationUrl, onWaiting, @@ -612,8 +629,8 @@ export const login = async ({ const state = randomBytes(16).toString("base64url"); const callback = await listenForCallback(state); const abort = new AbortController(); - const useBrowser = browser && canOpenBrowser(); - const deferred = !useBrowser && !process.stdin.isTTY; + const useBrowser = browser && !defer && canOpenBrowser(); + const deferred = defer || (!useBrowser && !process.stdin.isTTY); const provider = new SessionAuthProvider(signedIn, { redirectUrl: callback.redirectUrl, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 0369660..93d877a 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -61,7 +61,7 @@ const cursor: Option = { const COMMANDS: Readonly> = { login: { summary: "Sign in with your browser", - usage: "telemetry login [--no-browser | --finish
]", + usage: "telemetry login [--no-browser | --finish
] [--json]", options: { "no-browser": { type: "boolean", @@ -73,18 +73,23 @@ const COMMANDS: Readonly> = { description: "Finish a sign-in started without a terminal, with the browser's final address", }, + json: { + type: "boolean", + description: "Print JSON: the sign-in link without waiting, or the sign-in after --finish", + }, }, examples: [ "telemetry login", "telemetry login --no-browser # over SSH, in a container, or from an agent's shell", "telemetry login --finish 'http://127.0.0.1:53124/callback?code=…'", + "telemetry login --json # a script passes the link on, then runs --finish", ], run: loginCommand, }, logout: { summary: "Sign out on this machine", - usage: "telemetry logout", - options: {}, + usage: "telemetry logout [--json]", + options: { json }, examples: ["telemetry logout"], run: logoutCommand, }, diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index b70dd7a..b5529ae 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -323,20 +323,29 @@ export async function loginCommand(context: Context) { const server = serverUrl(); const session = await loadSession(server); const stream = process.stderr; + const asJson = json(context); const finish = typeof context.values.finish === "string" ? context.values.finish : undefined; - if (process.stderr.isTTY && finish === undefined) { + if (process.stderr.isTTY && finish === undefined && !asJson) { stream.write("\n"); await printMark(TAGLINE, { animate: true, stream }); stream.write("\n"); } + let authorizationUrl = ""; + const result = await login({ session, browser: context.values["no-browser"] !== true, + // A program reading JSON can't wait for a person, so it gets the link to pass on. + defer: asJson, finish, onAuthorizationUrl: (url, opened) => { + authorizationUrl = url.href; + + if (asJson) return; + note( opened ? "Opening your browser to sign in to telemetry.dev." @@ -350,11 +359,21 @@ export async function loginCommand(context: Context) { note(style.dim("Waiting for you to choose a workspace and select Allow…", stream)); }, }).catch((error: unknown) => { - reportStatus("error"); + if (!asJson) reportStatus("error"); throw error; }); if (result.status === "pending") { + if (asJson) { + printJson({ + status: "pending", + authorization_url: authorizationUrl, + next: "telemetry login --finish '
'", + }); + + return; + } + note( "After you select Allow, the browser opens an http://127.0.0.1 address that doesn't load. Copy the whole address and run:", ); @@ -366,64 +385,94 @@ export async function loginCommand(context: Context) { const signedIn = result.session; - reportStatus("done"); + if (!asJson) reportStatus("done"); + // The sign-in is saved by now, so a failed lookup leaves the project unset rather than + // failing a sign-in that worked. const api = new Api(signedIn); + let projects: ProjectList["projects"] | undefined; + let lookupFailure: CliError | undefined; try { - const { projects } = await api.call("list_projects", {}, "Loading projects"); + ({ projects } = await api.call("list_projects", {}, "Loading projects")); + } catch (error) { + if (!(error instanceof CliError)) throw error; - if (projects.length === 1) { - const project = toProject(projects[0]!); + lookupFailure = error; + } finally { + await api.close(); + } - signedIn.project = project; - await updateSession(signedIn.server, (saved) => ({ ...saved, project })); - } + if (projects?.length === 1) { + const project = toProject(projects[0]!); - note(); + signedIn.project = project; + await updateSession(signedIn.server, (saved) => ({ ...saved, project })); + } + + if (asJson) { + printJson({ status: "signed_in", ...sessionJson(signedIn) }); + + return; + } + + note(); + note( + `${style.green("✓", stream)} Signed in${signedIn.email ? ` as ${style.bold(signedIn.email, stream)}` : ""}`, + ); + + if (!grantedScopes(signedIn).has(PAYLOADS_SCOPE)) { note( - `${style.green("✓", stream)} Signed in${signedIn.email ? ` as ${style.bold(signedIn.email, stream)}` : ""}`, + style.dim( + " Captured prompts and outputs stay hidden. Sign in again and allow them to use `telemetry payload`.", + stream, + ), ); + } - if (!grantedScopes(signedIn).has(PAYLOADS_SCOPE)) { - note( - style.dim( - " Captured prompts and outputs stay hidden. Sign in again and allow them to use `telemetry payload`.", - stream, - ), - ); - } - - if (signedIn.project) { - note(` Project ${style.bold(signedIn.project.name, stream)}`); - note(); - note( - `Try ${style.bold("telemetry traces", stream)} or ${style.bold("telemetry usage", stream)}.`, - ); - } else if (projects.length > 1) { - note(` ${projects.length} projects: ${projectNames(projects)}`); - note(); - note( - `Choose a default: ${style.bold(`telemetry use ${shellQuote(toProject(projects[0]!).name)}`, stream)}`, - ); - } else { - note(" This workspace has no projects yet. Create one at telemetry.dev."); - } - } finally { - await api.close(); + if (lookupFailure) { + note(style.dim(` Couldn't load your projects: ${oneLine(lookupFailure.message)}`, stream)); + note(style.dim(" Run telemetry projects to list them.", stream)); + } else if (signedIn.project) { + note(` Project ${style.bold(signedIn.project.name, stream)}`); + note(); + note( + `Try ${style.bold("telemetry traces", stream)} or ${style.bold("telemetry usage", stream)}.`, + ); + } else if (projects && projects.length > 1) { + note(` ${projects.length} projects: ${projectNames(projects)}`); + note(); + note( + `Choose a default: ${style.bold(`telemetry use ${shellQuote(toProject(projects[0]!).name)}`, stream)}`, + ); + } else { + note(" This workspace has no projects yet. Create one at telemetry.dev."); } } -export async function logoutCommand() { +export async function logoutCommand(context: Context) { const session = await loadSession(serverUrl()); if (!session.tokens) { + if (json(context)) { + printJson({ was_signed_in: false, revoked: null }); + + return; + } + note("You're not signed in."); return; } const { revoked } = await logout(session); + + if (json(context)) { + printJson({ was_signed_in: true, revoked }); + + return; + } + note( `${style.green("✓", process.stderr)} Signed out${session.email ? ` of ${session.email}` : ""}.`, ); @@ -437,19 +486,22 @@ export async function logoutCommand() { ); } +/** What `status --json` and `login --json` say about a sign-in. */ +const sessionJson = (session: Session) => ({ + signed_in: Boolean(session.tokens), + server: session.server, + email: session.email ?? null, + project: session.project ?? null, + payloads_allowed: grantedScopes(session).has(PAYLOADS_SCOPE), +}); + export async function statusCommand(context: Context) { const server = serverUrl(); const session = await loadSession(server); const scopes = grantedScopes(session); if (json(context)) { - printJson({ - signed_in: Boolean(session.tokens), - server, - email: session.email ?? null, - project: session.project ?? null, - payloads_allowed: scopes.has(PAYLOADS_SCOPE), - }); + printJson(sessionJson(session)); } else if (session.tokens) { out( `${style.green("✓")} Signed in${session.email ? ` as ${style.bold(oneLine(session.email))}` : ""}`, diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index 1cb0b3c..30a9d8e 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -68,3 +68,75 @@ export const notSignedIn = () => code: "not_signed_in", hint: "telemetry login", }); + +// Each is a refresh or authorization the server refused for good. +type AuthErrors = readonly (abstract new (...args: never[]) => Error)[]; + +const NETWORK_CODES = new Set([ + "ECONNREFUSED", + "ECONNRESET", + "EAI_AGAIN", + "EHOSTUNREACH", + "ENETUNREACH", + "ENOTFOUND", + "ETIMEDOUT", + "UND_ERR_SOCKET", +]); + +// undici's own timeouts, raised by fetch before any of the CLI's timers fire. +const TIMEOUT_CODES = new Set([ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", +]); + +export const translateError = (error: unknown, authErrors: AuthErrors): never => { + if (error instanceof CliError) throw error; + + if (authErrors.some((type) => error instanceof type)) { + throw new CliError( + "Your sign-in has expired or was disconnected. Run `telemetry login` again.", + { exitCode: EXIT.auth, code: "sign_in_expired", hint: "telemetry login" }, + ); + } + + // The network error sits at the bottom of a chain: the SDK wraps fetch's + // "fetch failed", which wraps the socket error that has the code. + const chain: unknown[] = [error]; + + for (let link = error; link instanceof Error && link.cause !== undefined && chain.length < 8; ) { + link = link.cause; + chain.push(link); + } + + const codes = chain + .map((link) => (link as { code?: unknown } | null)?.code) + .filter((code): code is string => typeof code === "string"); + const network = codes.find((code) => NETWORK_CODES.has(code)); + // Node's AggregateError for a failed connection has an empty message. + const message = + chain.findLast((link): link is Error => link instanceof Error && link.message !== "") + ?.message ?? String(error); + + if (network) { + throw new CliError( + `Couldn't reach telemetry.dev (${network}). Check your connection and try again.`, + { code: "unreachable", retryable: true }, + ); + } + + if ( + codes.some((code) => code === "REQUEST_TIMEOUT" || TIMEOUT_CODES.has(code)) || + chain.some( + (link) => + link instanceof Error && (link.name === "TimeoutError" || link.name === "AbortError"), + ) + ) { + throw new CliError("telemetry.dev took too long to answer. Try again shortly.", { + code: "timeout", + retryable: true, + }); + } + + throw new CliError(`telemetry.dev request failed: ${message}`, { code: "request_failed" }); +}; diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 1f4743f..452b3fa 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -12,6 +12,7 @@ import { beforeRefresh, calls, EMAIL, + failProjects, PARALLEL_TRACE_ID, PAYLOADS, PROJECTS, @@ -58,6 +59,7 @@ beforeEach(async () => { home = await mkdtemp(join(tmpdir(), "telemetry-cli-")); calls.length = 0; delete beforeRefresh.run; + failProjects.value = false; return () => rm(home, { recursive: true, force: true }); }); @@ -83,6 +85,15 @@ const runWith = (env: Record, ...args: string[]) => const run = (...args: string[]) => runWith({}, ...args); +/** The address of a port that was just closed, which refuses connections. */ +const closedServer = () => + new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(`http://127.0.0.1:${port}`)); + }); + }); + const CONTROL = new RegExp(String.raw`[\u0000-\u0008\u000b-\u001f\u007f-\u009f]`); test("human output never passes terminal escapes from captured text, and --json keeps the data", async () => { @@ -323,13 +334,7 @@ test("with --json, an error is one JSON line on stderr with a code to branch on, message: "Trace not found.", }); - // A port that was just closed refuses the connection. - const closed = await new Promise((resolve) => { - const server = createServer().listen(0, "127.0.0.1", () => { - const { port } = server.address() as AddressInfo; - server.close(() => resolve(`http://127.0.0.1:${port}`)); - }); - }); + const closed = await closedServer(); await writeFile( join(home, sessionFile(closed)), JSON.stringify({ server: closed, tokens: { access_token: TOKEN, token_type: "Bearer" } }), @@ -445,6 +450,109 @@ test("a shell without a terminal signs in with --no-browser, then --finish, keep expect(again.code).toBe(4); }); +test("login --json prints the link without opening a browser or waiting, and --finish --json prints the sign-in", async () => { + // A browser that could open: an `open` and `xdg-open` that record being run. + const bin = await mkdtemp(join(tmpdir(), "telemetry-cli-bin-")); + const opened = join(bin, "opened"); + for (const name of ["open", "xdg-open"]) { + await writeFile(join(bin, name), `#!/bin/sh\ntouch '${opened}'\n`, { mode: 0o755 }); + } + const browser = { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":0" }; + + try { + const started = await runWith(browser, "login", "--json"); + expect(started).toMatchObject({ code: 0, stderr: "" }); + + const pending = JSON.parse(started.stdout); + expect(pending).toEqual({ + status: "pending", + authorization_url: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+\/authorize\?/), + next: "telemetry login --finish '
'", + }); + + const approved = await fetch(pending.authorization_url, { redirect: "manual" }); + const finished = await run("login", "--finish", approved.headers.get("location")!, "--json"); + expect(finished).toMatchObject({ code: 0, stderr: "" }); + + const status = JSON.parse((await run("status", "--json")).stdout); + expect(status).toMatchObject({ signed_in: true, email: EMAIL }); + expect(JSON.parse(finished.stdout)).toEqual({ status: "signed_in", ...status }); + + const again = await run("login", "--finish", approved.headers.get("location")!, "--json"); + expect(again).toMatchObject({ code: 4, stdout: "" }); + expect(JSON.parse(again.stderr).error.code).toBe("no_pending_sign_in"); + + expect(JSON.parse((await run("logout", "--json")).stdout)).toEqual({ + was_signed_in: true, + revoked: true, + }); + expect(JSON.parse((await run("logout", "--json")).stdout)).toEqual({ + was_signed_in: false, + revoked: null, + }); + + // `open` runs detached, so by now, several commands later, it would have left its mark. + await expect(stat(opened)).rejects.toThrow(); + } finally { + await rm(bin, { recursive: true }); + } +}); + +test("login --json returns the link at once even when stdin is a terminal", async () => { + const { login } = await import("../src/auth.ts"); + const { loadSession } = await import("../src/session.ts"); + + vi.stubEnv("TELEMETRY_DEV_CONFIG_DIR", home); + const isTTY = Object.getOwnPropertyDescriptor(process.stdin, "isTTY"); + Object.defineProperty(process.stdin, "isTTY", { value: true, configurable: true }); + + try { + const result = await login({ + session: await loadSession(fixture.url), + browser: false, + defer: true, + onAuthorizationUrl: () => {}, + onWaiting: () => { + throw new Error("login waited for a person"); + }, + }); + + expect(result).toEqual({ status: "pending" }); + expect((await loadSession(fixture.url)).pending).toBeDefined(); + } finally { + if (isTTY) Object.defineProperty(process.stdin, "isTTY", isTTY); + else delete (process.stdin as { isTTY?: boolean }).isTTY; + vi.unstubAllEnvs(); + } +}); + +test("a sign-in whose project list fails still signs in, without a default project", async () => { + failProjects.value = true; + + for (const format of [[], ["--json"]]) { + const started = await run("login", "--json"); + const approved = await fetch(JSON.parse(started.stdout).authorization_url, { + redirect: "manual", + }); + const finished = await run("login", "--finish", approved.headers.get("location")!, ...format); + + expect(finished.code).toBe(0); + if (format.length) expect(JSON.parse(finished.stdout)).toMatchObject({ project: null }); + else expect(finished.stderr).toContain("Couldn't load your projects"); + } + + expect(JSON.parse((await run("status", "--json")).stdout)).toMatchObject({ + signed_in: true, + project: null, + }); +}); + +test("a sign-in that can't reach the server says so, like any other command", async () => { + const result = await runWith({ TELEMETRY_DEV_URL: await closedServer() }, "login", "--json"); + expect(result).toMatchObject({ code: 1, stdout: "" }); + expect(JSON.parse(result.stderr).error).toMatchObject({ code: "unreachable", retryable: true }); +}); + test("an expired access token is refreshed and the new one saved", async () => { await writeFile( sessionPath(), diff --git a/packages/cli/tests/fixture.ts b/packages/cli/tests/fixture.ts index 707483c..392bd66 100644 --- a/packages/cli/tests/fixture.ts +++ b/packages/cli/tests/fixture.ts @@ -445,7 +445,11 @@ const server = () => { : result(value); }); - tool("list_projects", () => ({ organization_id: "org_fixture", projects: PROJECTS })); + tool("list_projects", () => + failProjects.value + ? "telemetry.dev could not complete this request. Try again shortly." + : { organization_id: "org_fixture", projects: PROJECTS }, + ); tool("search_traces", () => traces()); tool("search_sessions", () => sessions()); tool("get_trace", (args) => @@ -476,6 +480,9 @@ const server = () => { export const calls: { tool: string; args: Record }[] = []; +/** Makes list_projects answer with a tool error, as a server failure would. */ +export const failProjects = { value: false }; + /** * Runs before the token endpoint answers a refresh, to act as another CLI process or a * server that rotates tokens. "accept" or "refuse" overrides the usual check. From b639de9f388ed3ce42335b82cff8728c61c2d322 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 15:00:01 +0000 Subject: [PATCH 31/32] feat(cli): map telemetry.dev's tool error codes to cli error codes (#134) A tool error's code in _meta now gives project_not_found, access_denied, payloads_not_allowed, invalid_cursor, trace_not_found, span_not_found, or server_error, with their own exit codes and hints. Errors without a known code stay tool_error. --- packages/cli/README.md | 7 ++- packages/cli/src/api.ts | 7 +-- packages/cli/src/commands.ts | 30 ++++++++---- packages/cli/src/errors.ts | 53 ++++++++++++++++++++ packages/cli/tests/cli.test.ts | 88 +++++++++++++++++++++++++++++++--- packages/cli/tests/fixture.ts | 61 ++++++++++++++++++----- 6 files changed, 213 insertions(+), 33 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 6088442..d7deb0c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -96,17 +96,22 @@ Branch on `code`, not on `message`, whose wording can change. `hint` is a comman | `invalid_arguments` | 2 | An option or argument is not valid | | `unknown_command` | 2 | There is no command with that name | | `project_not_found` | 2 | No project in the workspace has that name or id | +| `invalid_cursor` | 2 | `--cursor` isn't from an earlier page of this command | | `project_ambiguous` | 2 | Several projects have that name; pass its id | | `project_required` | 2 | The workspace has several projects and none was chosen | | `no_projects` | 1 | The workspace has no projects yet | | `not_signed_in` | 4 | No sign-in is saved for this deployment | | `sign_in_expired` | 4 | The sign-in expired or was disconnected | | `payloads_not_allowed` | 4 | The sign-in doesn't include reading captured prompts and outputs | +| `access_denied` | 4 | The account no longer has access to the workspace | | `sign_in_failed` | 4 | The sign-in was cancelled, refused, or used an address from another sign-in | | `no_pending_sign_in` | 4 | `login --finish` found no sign-in waiting to finish | +| `trace_not_found` | 1 | No trace has that id, or it is past the project's retention | +| `span_not_found` | 1 | The trace has no span with that id, or it is past the project's retention | | `unreachable` | 1 | telemetry.dev could not be reached | | `timeout` | 1 | telemetry.dev took too long to answer | -| `tool_error` | 1 | telemetry.dev returned an error; `message` says what it was | +| `server_error` | 1 | telemetry.dev failed to complete the request; try again | +| `tool_error` | 1 | telemetry.dev returned another error; `message` says what it was | | `result_too_large` | 1 | The result was too large to read; narrow it with filters or `--limit` | | `request_failed` | 1 | The request failed for another reason | | `failed` | 1 | The command failed for another reason | diff --git a/packages/cli/src/api.ts b/packages/cli/src/api.ts index d448e7e..6a6d962 100644 --- a/packages/cli/src/api.ts +++ b/packages/cli/src/api.ts @@ -1,7 +1,7 @@ import type { Client } from "@modelcontextprotocol/client"; import { SessionAuthProvider } from "./auth.ts"; -import { CliError, notSignedIn, translateError } from "./errors.ts"; +import { CliError, notSignedIn, toolError, translateError } from "./errors.ts"; import { mcpUrl, type Session } from "./session.ts"; import { withSpinner } from "./terminal.ts"; @@ -74,10 +74,7 @@ export class Api { const text = result.content.find((block) => block.type === "text")?.text ?? ""; - if (result.isError) - throw new CliError(text || "telemetry.dev could not complete this request.", { - code: "tool_error", - }); + if (result.isError) throw toolError(text, result._meta); try { return JSON.parse(text) as T; diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index b5529ae..765e57f 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -1,6 +1,6 @@ import { Api } from "./api.ts"; import { grantedScopes, login, logout, PAYLOADS_SCOPE } from "./auth.ts"; -import { CliError, EXIT, notSignedIn, usageError } from "./errors.ts"; +import { CliError, EXIT, notSignedIn, payloadsNotAllowed, usageError } from "./errors.ts"; import { printMark } from "./mark.ts"; import { DEFAULT_SERVER, @@ -803,11 +803,24 @@ export async function traceCommand(context: Context) { let pages = 1; while (page.next_cursor && pages < MAX_TRACE_PAGES) { - page = await api.call( - "get_trace", - { project_id: project.id, trace_id: traceId, cursor: page.next_cursor }, - `Loading spans (${spans.length} of ${page.span_count})`, - ); + page = await api + .call( + "get_trace", + { project_id: project.id, trace_id: traceId, cursor: page.next_cursor }, + `Loading spans (${spans.length} of ${page.span_count})`, + ) + .catch((error: unknown) => { + // The cursor came from the server, so a refused one means the server changed + // between pages, not that the person passed a bad --cursor. + if (error instanceof CliError && error.code === "invalid_cursor") { + throw new CliError( + "telemetry.dev stopped accepting this trace's pages while it loaded. Run the command again.", + { code: "server_error", retryable: true }, + ); + } + + throw error; + }); spans.push(...page.spans); logs.push(...page.logs); pages += 1; @@ -1413,10 +1426,7 @@ export async function payloadCommand(context: Context) { await withApi(async (api, session) => { if (!grantedScopes(session).has(PAYLOADS_SCOPE)) { - throw new CliError( - 'This sign-in can\'t read captured prompts and outputs. Run `telemetry login`, check "Also read captured prompts, outputs, and tool arguments", and select Allow.', - { exitCode: EXIT.auth, code: "payloads_not_allowed", hint: "telemetry login" }, - ); + throw payloadsNotAllowed(); } const project = await resolveProject(api, context); diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index 30a9d8e..4a45f32 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -22,8 +22,13 @@ export type ErrorCode = | "sign_in_failed" | "no_pending_sign_in" | "payloads_not_allowed" + | "access_denied" + | "invalid_cursor" + | "trace_not_found" + | "span_not_found" | "unreachable" | "timeout" + | "server_error" | "tool_error" | "result_too_large" | "request_failed"; @@ -62,6 +67,54 @@ export class CliError extends Error { export const usageError = (message: string, details: Omit = {}) => new CliError(message, { ...details, exitCode: EXIT.usage }); +export const payloadsNotAllowed = () => + new CliError( + 'This sign-in can\'t read captured prompts and outputs. Run `telemetry login`, check "Also read captured prompts, outputs, and tool arguments", and select Allow.', + { exitCode: EXIT.auth, code: "payloads_not_allowed", hint: "telemetry login" }, + ); + +/** + * The CliError for a tool error, from the code telemetry.dev puts in the result's + * `_meta`. A server without codes, or a code this CLI doesn't know, gives `tool_error`. + * The result's text is written for agents, so codes a person acts on get the CLI's own. + */ +export const toolError = (text: string, meta: unknown): CliError => { + const error = (meta as Record | undefined)?.["dev.telemetry/error"] as + | { code?: unknown; retryable?: unknown } + | undefined; + const retryable = error?.retryable === true; + + switch (error?.code) { + case "access_denied": + return new CliError( + "Your account no longer has access to this workspace. Run `telemetry login` and choose a workspace you belong to.", + { exitCode: EXIT.auth, code: "access_denied", hint: "telemetry login" }, + ); + case "payloads_not_allowed": + return payloadsNotAllowed(); + case "project_not_found": + return usageError( + "That project isn't in this workspace; it may have been deleted. Run `telemetry projects` to list the projects, and `telemetry use ` to change the default.", + { code: "project_not_found", hint: "telemetry projects" }, + ); + case "invalid_cursor": + return usageError( + "That cursor isn't from an earlier page of this command. Run it again without --cursor.", + { code: "invalid_cursor" }, + ); + case "trace_not_found": + case "span_not_found": + return new CliError(text, { code: error.code }); + case "server_error": + return new CliError(text, { code: "server_error", retryable }); + default: + return new CliError(text || "telemetry.dev could not complete this request.", { + code: "tool_error", + retryable, + }); + } +}; + export const notSignedIn = () => new CliError("You're not signed in. Run `telemetry login` first.", { exitCode: EXIT.auth, diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 452b3fa..7f1b589 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -12,12 +12,12 @@ import { beforeRefresh, calls, EMAIL, - failProjects, PARALLEL_TRACE_ID, PAYLOADS, PROJECTS, QUIET_TRACE_ID, REFRESH_TOKEN, + serverState, startFixture, TOKEN, TRACE_ID, @@ -59,7 +59,11 @@ beforeEach(async () => { home = await mkdtemp(join(tmpdir(), "telemetry-cli-")); calls.length = 0; delete beforeRefresh.run; - failProjects.value = false; + Object.assign(serverState, { + failProjects: false, + removedFromWorkspace: false, + errorCodes: true, + }); return () => rm(home, { recursive: true, force: true }); }); @@ -330,8 +334,8 @@ test("with --json, an error is one JSON line on stderr with a code to branch on, }); expect(await errorOf(await run("trace", "0000", "--json"))).toMatchObject({ exit: 1, - code: "tool_error", - message: "Trace not found.", + code: "trace_not_found", + message: "Trace not found or outside retention", }); const closed = await closedServer(); @@ -359,7 +363,79 @@ test("with --json, an error is one JSON line on stderr with a code to branch on, // The same failures without --json keep the line a person reads. const human = await run("trace", "0000"); expect(human.code).toBe(1); - expect(human.stderr).toBe("error Trace not found.\n"); + expect(human.stderr).toBe("error Trace not found or outside retention\n"); +}); + +test("the code telemetry.dev gives a tool error becomes the CLI's code, exit code, and hint", async () => { + const errorOf = async (...args: string[]) => { + const result = await run(...args, "--json"); + expect(result.stdout).toBe(""); + + return { exit: result.code, ...JSON.parse(result.stderr).error }; + }; + + // A default project or a project id goes to telemetry.dev without a lookup first. + const deleted = "0f0e0d0c-0b0a-4908-8706-050403020100"; + await signIn({ ...PROJECTS[0]!, id: deleted }); + expect(await errorOf("traces")).toMatchObject({ + exit: 2, + code: "project_not_found", + hint: "telemetry projects", + }); + expect(await errorOf("traces", "--project", deleted)).toMatchObject({ + code: "project_not_found", + }); + // People read the CLI's own words, not the text written for an agent. + expect((await run("traces", "--project", deleted)).stderr).toMatch( + /^error That project isn't in this workspace/, + ); + + await signIn(PROJECTS[0]); + expect(await errorOf("traces", "--cursor", "bm90LWEtY3Vyc29y")).toMatchObject({ + exit: 2, + code: "invalid_cursor", + retryable: false, + }); + + serverState.failProjects = true; + expect(await errorOf("projects")).toMatchObject({ + exit: 1, + code: "server_error", + message: "telemetry.dev could not complete this request. Try again shortly.", + retryable: true, + }); + + serverState.removedFromWorkspace = true; + expect(await errorOf("traces")).toMatchObject({ + exit: 4, + code: "access_denied", + hint: "telemetry login", + }); + + // A server from before the codes, and a code this CLI doesn't know, stay tool_error. + serverState.errorCodes = false; + expect(await errorOf("traces")).toMatchObject({ + exit: 1, + code: "tool_error", + message: "Not a member of this workspace", + retryable: false, + }); + + const { toolError } = await import("../src/errors.ts"); + expect( + toolError("Plan limit reached", { "dev.telemetry/error": { code: "plan_limit" } }), + ).toMatchObject({ + code: "tool_error", + message: "Plan limit reached", + }); + expect( + toolError("…", { "dev.telemetry/error": { code: "payloads_not_allowed", retryable: false } }), + ).toMatchObject({ code: "payloads_not_allowed", exitCode: 4, hint: "telemetry login" }); + expect( + toolError("Span not found", { + "dev.telemetry/error": { code: "span_not_found", retryable: false }, + }), + ).toMatchObject({ code: "span_not_found", exitCode: 1, message: "Span not found" }); }); test("a request that times out inside fetch is a retryable timeout, not a failed request", async () => { @@ -527,7 +603,7 @@ test("login --json returns the link at once even when stdin is a terminal", asyn }); test("a sign-in whose project list fails still signs in, without a default project", async () => { - failProjects.value = true; + serverState.failProjects = true; for (const format of [[], ["--json"]]) { const started = await run("login", "--json"); diff --git a/packages/cli/tests/fixture.ts b/packages/cli/tests/fixture.ts index 392bd66..fdb9bfd 100644 --- a/packages/cli/tests/fixture.ts +++ b/packages/cli/tests/fixture.ts @@ -433,29 +433,61 @@ const result = (value: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(value) }], }); +/** A tool error as telemetry.dev returns it: text for the agent, a code in `_meta`. */ +class ToolFailure { + constructor( + readonly text: string, + readonly code: string, + readonly retryable = false, + ) {} +} + +const SERVER_ERROR = "telemetry.dev could not complete this request. Try again shortly."; + const server = () => { const mcp = new McpServer({ name: "telemetry.dev fixture", version: "1.0.0" }); const tool = (name: string, handler: (args: Record) => unknown) => mcp.registerTool(name, { inputSchema: loose }, (args: Record) => { calls.push({ tool: name, args }); - const value = handler(args); - - return typeof value === "string" - ? { isError: true, content: [{ type: "text" as const, text: value }] } - : result(value); + const value = serverState.removedFromWorkspace + ? new ToolFailure("Not a member of this workspace", "access_denied") + : handler(args); + + if (!(value instanceof ToolFailure)) return result(value); + + return { + isError: true, + content: [{ type: "text" as const, text: value.text }], + ...(serverState.errorCodes + ? { _meta: { "dev.telemetry/error": { code: value.code, retryable: value.retryable } } } + : {}), + }; }); tool("list_projects", () => - failProjects.value - ? "telemetry.dev could not complete this request. Try again shortly." + serverState.failProjects + ? new ToolFailure(SERVER_ERROR, "server_error", true) : { organization_id: "org_fixture", projects: PROJECTS }, ); - tool("search_traces", () => traces()); + tool("search_traces", (args) => { + if (!PROJECTS.some((project) => project.id === args.project_id)) { + return new ToolFailure("Project not found", "project_not_found"); + } + + if (args.cursor !== undefined && args.cursor !== traces().next_cursor) { + return new ToolFailure( + "cursor is not a next_cursor value from search_traces. Omit it to start over.", + "invalid_cursor", + ); + } + + return traces(); + }); tool("search_sessions", () => sessions()); tool("get_trace", (args) => Object.hasOwn(TRACE_DETAILS, String(args.trace_id)) ? TRACE_DETAILS[String(args.trace_id)]!() - : "Trace not found.", + : new ToolFailure("Trace not found or outside retention", "trace_not_found"), ); tool("list_logs", () => logs()); tool("get_usage_summary", () => usage()); @@ -480,8 +512,15 @@ const server = () => { export const calls: { tool: string; args: Record }[] = []; -/** Makes list_projects answer with a tool error, as a server failure would. */ -export const failProjects = { value: false }; +/** How the fixture server misbehaves; tests reset it before each run. */ +export const serverState = { + /** list_projects fails as a server failure would. */ + failProjects: false, + /** Every tool refuses, as for a user removed from the workspace. */ + removedFromWorkspace: false, + /** Tool errors carry a code in `_meta`; false acts as a server from before codes. */ + errorCodes: true, +}; /** * Runs before the token endpoint answers a refresh, to act as another CLI process or a From 511a491fa00d30cea923315b8586e4dd3cff76cd Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Fri, 30 Oct 2026 15:05:47 +0000 Subject: [PATCH 32/32] chore: release main (#133) --- .release-please-manifest.json | 2 +- packages/cli/CHANGELOG.md | 8 ++++++++ packages/cli/package.json | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d727003..3c5dd92 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -21,6 +21,6 @@ "sdks/python-litellm": "0.2.2", "sdks/python-openai": "0.2.2", "sdks/python-openrouter": "0.2.2", - "packages/cli": "0.1.1", + "packages/cli": "0.1.2", "actions/experiment": "1.0.0" } diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 4056cdd..29bd94a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.1.2](https://github.com/telemetry-dev/telemetry/compare/cli-v0.1.1...cli-v0.1.2) (2026-10-30) + + +### Features + +* **cli:** add --json to login and logout ([#131](https://github.com/telemetry-dev/telemetry/issues/131)) ([505309b](https://github.com/telemetry-dev/telemetry/commit/505309bf84c00edf3036488348086e10a5870510)) +* **cli:** map telemetry.dev's tool error codes to cli error codes ([#134](https://github.com/telemetry-dev/telemetry/issues/134)) ([b639de9](https://github.com/telemetry-dev/telemetry/commit/b639de9f388ed3ce42335b82cff8728c61c2d322)) + ## [0.1.1](https://github.com/telemetry-dev/telemetry/compare/cli-v0.1.0...cli-v0.1.1) (2026-10-30) diff --git a/packages/cli/package.json b/packages/cli/package.json index 5cf9b7f..56710fc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@telemetry-dev/cli", - "version": "0.1.1", + "version": "0.1.2", "description": "The telemetry.dev command line: search traces, sessions, logs, and AI usage from your terminal or an agent's shell.", "keywords": [ "cli",