From 4810a4490de9d9276171f4aa318dbd6eee3b5318 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 09:50:59 -0400 Subject: [PATCH 1/5] Defer building the OpenTelemetry tracestate for fresh probability decisions Every root probability decision built an OtelTraceState inside the sampling lock, though only W3C injection and OTLP export read it. Install a ProbabilitySamplingState that keeps the trace id bits and rate, and builds the ot member on first read. Co-Authored-By: Claude Opus 5.5 --- .../core/propagation/PropagationTags.java | 13 ++- .../core/propagation/ptags/PTagsFactory.java | 63 ++++++++++----- .../ptags/ProbabilitySamplingState.java | 51 ++++++++++++ .../ptags/ProbabilitySamplingStateTest.java | 81 +++++++++++++++++++ 4 files changed, 186 insertions(+), 22 deletions(-) create mode 100644 dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java index 3660a6bae81..86552fdf1cf 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java @@ -24,7 +24,11 @@ */ public abstract class PropagationTags { - public static final class SamplingState { + /** + * An immutable snapshot of the trace's sampling decision. Not final only so that a fresh + * probability decision can defer building its OpenTelemetry tracestate until it's read. + */ + public static class SamplingState { private static final byte OTEL_POSITION_UNKNOWN = 0; private static final byte OTEL_POSITION_PRESERVED = 1; private static final byte OTEL_POSITION_REPLACED = 2; @@ -57,6 +61,7 @@ public String getTracestate() { return tracestate; } + /** The {@code ot} tracestate member, which a subclass may build on first read. */ public CharSequence getOtelTraceState() { return otelTraceState; } @@ -70,7 +75,11 @@ public CharSequence getKnuthSamplingRate() { } public boolean preservesInheritedOtelPosition() { - if (tracestate == null || otelTraceState == null) { + if (tracestate == null) { + return false; + } + CharSequence otelTraceState = getOtelTraceState(); + if (otelTraceState == null) { return false; } byte position = otelPosition; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java index d4a01896252..fb197efa226 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java @@ -308,25 +308,37 @@ public boolean tryUpdateProbabilitySamplingDecision( if (!allowOverride && current.getSamplingPriority() != PrioritySampling.UNSET) { return false; } + TagValue nextKnuthSamplingRate = knuthSamplingRateTagValue(sampleRate); + boolean updateDecisionMaker = + canChangeDecisionMaker || samplingMechanism == SamplingMechanism.EXTERNAL_OVERRIDE; OtelTraceState nextOtelTraceState = getOtelTraceState(); if (nextOtelTraceState == null) { if (!rateLimiterRejected) { - nextOtelTraceState = - OtelTraceState.fromProbabilityDecision( - traceIdLowOrderBits, sampleRate, samplingPriority); + // A fresh decision always yields a new otel state, so there's no no-op check to make + TagValue nextDecisionMakerTagValue = + nextDecisionMakerTagValue(samplingPriority, samplingMechanism, updateDecisionMaker); + clearCachedHeader(W3C); + samplingState = + new ProbabilitySamplingState( + samplingPriority, + tracestate, + nextDecisionMakerTagValue, + nextKnuthSamplingRate, + traceIdLowOrderBits, + sampleRate); + return true; } } else if (rateLimiterRejected) { nextOtelTraceState = nextOtelTraceState.withoutThreshold(); } else { nextOtelTraceState = nextOtelTraceState.withoutInheritedThreshold(); } - TagValue nextKnuthSamplingRate = knuthSamplingRateTagValue(sampleRate); installSamplingState( samplingPriority, samplingMechanism, nextOtelTraceState, nextKnuthSamplingRate, - canChangeDecisionMaker || samplingMechanism == SamplingMechanism.EXTERNAL_OVERRIDE); + updateDecisionMaker); return true; } } @@ -364,6 +376,31 @@ private void installSamplingState( TagValue nextKnuthSamplingRateTagValue, boolean updateDecisionMaker) { SamplingState currentState = samplingState; + TagValue nextDecisionMakerTagValue = + nextDecisionMakerTagValue(samplingPriority, samplingMechanism, updateDecisionMaker); + if (currentState.getSamplingPriority() == samplingPriority + && currentState.getTracestate() == tracestate + && currentState.getOtelTraceState() == nextOtelTraceState + && Objects.equals(currentState.getDecisionMaker(), nextDecisionMakerTagValue) + && Objects.equals(currentState.getKnuthSamplingRate(), nextKnuthSamplingRateTagValue)) { + return; + } + clearCachedHeader(W3C); + samplingState = + new SamplingState( + samplingPriority, + tracestate, + nextOtelTraceState, + nextDecisionMakerTagValue, + nextKnuthSamplingRateTagValue); + } + + /** + * The decision maker value after a decision, clearing the cached headers if it changes. Called + * with the sampling state lock held. + */ + private TagValue nextDecisionMakerTagValue( + int samplingPriority, int samplingMechanism, boolean updateDecisionMaker) { TagValue nextDecisionMakerTagValue = getDecisionMakerTagValue(); if (updateDecisionMaker && samplingPriority > 0) { // TODO should try to keep the old sampling mechanism if we override the value? @@ -389,21 +426,7 @@ private void installSamplingState( } nextDecisionMakerTagValue = null; } - if (currentState.getSamplingPriority() == samplingPriority - && currentState.getTracestate() == tracestate - && currentState.getOtelTraceState() == nextOtelTraceState - && Objects.equals(currentState.getDecisionMaker(), nextDecisionMakerTagValue) - && Objects.equals(currentState.getKnuthSamplingRate(), nextKnuthSamplingRateTagValue)) { - return; - } - clearCachedHeader(W3C); - samplingState = - new SamplingState( - samplingPriority, - tracestate, - nextOtelTraceState, - nextDecisionMakerTagValue, - nextKnuthSamplingRateTagValue); + return nextDecisionMakerTagValue; } private static SamplingState initialSamplingState( diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java new file mode 100644 index 00000000000..ee29a0c6bde --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java @@ -0,0 +1,51 @@ +package datadog.trace.core.propagation.ptags; + +import datadog.trace.core.propagation.PropagationTags.SamplingState; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +/** + * The sampling state for a fresh probability decision, one with no inherited OpenTelemetry + * tracestate. Every such root decision used to build its {@link OtelTraceState} up front, though + * only W3C injection and OTLP export ever read it. This keeps the two inputs instead and builds the + * {@code ot} member on first read, so a trace that is never injected or exported over OTLP doesn't + * pay for it. + */ +final class ProbabilitySamplingState extends SamplingState { + private static final AtomicReferenceFieldUpdater + OTEL_TRACE_STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater( + ProbabilitySamplingState.class, OtelTraceState.class, "otelTraceState"); + + private final long traceIdLowOrderBits; + private final double sampleRate; + + private volatile OtelTraceState otelTraceState; + + ProbabilitySamplingState( + int samplingPriority, + String tracestate, + CharSequence decisionMaker, + CharSequence knuthSamplingRate, + long traceIdLowOrderBits, + double sampleRate) { + super(samplingPriority, tracestate, null, decisionMaker, knuthSamplingRate); + this.traceIdLowOrderBits = traceIdLowOrderBits; + this.sampleRate = sampleRate; + } + + /** + * Builds the {@code ot} member on first read. Racing readers agree on one instance, because + * callers compare otel states by identity to detect changes. + */ + @Override + public CharSequence getOtelTraceState() { + OtelTraceState current = otelTraceState; + if (current != null) { + return current; + } + OtelTraceState built = + OtelTraceState.fromProbabilityDecision( + traceIdLowOrderBits, sampleRate, getSamplingPriority()); + return OTEL_TRACE_STATE_UPDATER.compareAndSet(this, null, built) ? built : otelTraceState; + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java new file mode 100644 index 00000000000..85bef8be246 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java @@ -0,0 +1,81 @@ +package datadog.trace.core.propagation.ptags; + +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.api.sampling.SamplingMechanism.AGENT_RATE; +import static datadog.trace.api.sampling.SamplingMechanism.MANUAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.core.propagation.PropagationTags; +import datadog.trace.core.propagation.PropagationTags.SamplingState; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class ProbabilitySamplingStateTest { + private static final long TRACE_ID_LOW_ORDER_BITS = 0xfff972474538efffL; + + private final PropagationTags.Factory factory = PropagationTags.factory(); + + @TableTest({ + "scenario | priority | rate", + "keep | SAMPLER_KEEP | 0.1 ", + "drop | SAMPLER_DROP | 0.1 ", + "always | SAMPLER_KEEP | 1.0 ", + "never | SAMPLER_DROP | 0.0 " + }) + void freshDecisionBuildsSameOtelStateAsEagerConversion(String priority, double rate) { + int samplingPriority = "SAMPLER_KEEP".equals(priority) ? SAMPLER_KEEP : SAMPLER_DROP; + PropagationTags ptags = factory.empty(); + + ptags.tryUpdateProbabilitySamplingDecision( + samplingPriority, AGENT_RATE, rate, false, TRACE_ID_LOW_ORDER_BITS, false); + + SamplingState state = ptags.samplingState(); + assertInstanceOf(ProbabilitySamplingState.class, state); + assertEquals( + OtelTraceState.fromProbabilityDecision(TRACE_ID_LOW_ORDER_BITS, rate, samplingPriority) + .toString(), + state.getOtelTraceState().toString()); + } + + @Test + void repeatedReadsReturnOneInstance() { + PropagationTags ptags = factory.empty(); + ptags.tryUpdateProbabilitySamplingDecision( + SAMPLER_KEEP, AGENT_RATE, 0.1, false, TRACE_ID_LOW_ORDER_BITS, false); + + SamplingState state = ptags.samplingState(); + + assertSame(state.getOtelTraceState(), state.getOtelTraceState()); + } + + @Test + void rateLimiterRejectionRecordsNoOtelState() { + PropagationTags ptags = factory.empty(); + ptags.tryUpdateProbabilitySamplingDecision( + SAMPLER_DROP, AGENT_RATE, 0.1, true, TRACE_ID_LOW_ORDER_BITS, false); + + SamplingState state = ptags.samplingState(); + + assertEquals(SamplingState.class, state.getClass()); + assertNull(state.getOtelTraceState()); + } + + @Test + void laterDecisionStartsFromDeferredOtelState() { + PropagationTags ptags = factory.empty(); + ptags.tryUpdateProbabilitySamplingDecision( + SAMPLER_KEEP, AGENT_RATE, 0.1, false, TRACE_ID_LOW_ORDER_BITS, false); + + ptags.forceKeep(MANUAL); + + // A locally made probability state has nothing inherited to keep, so a manual keep drops it + assertNull( + OtelTraceState.fromProbabilityDecision(TRACE_ID_LOW_ORDER_BITS, 0.1, SAMPLER_KEEP) + .forNonProbabilityDecision()); + assertNull(ptags.samplingState().getOtelTraceState()); + } +} From b9308b8c53511ac8c8c698350573aa21521d1a3c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 10:02:57 -0400 Subject: [PATCH 2/5] Clarify why lazy otel state readers agree on one instance Co-Authored-By: Claude Opus 5.5 --- .../core/propagation/ptags/ProbabilitySamplingState.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java index ee29a0c6bde..92c7d55709c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingState.java @@ -34,8 +34,9 @@ final class ProbabilitySamplingState extends SamplingState { } /** - * Builds the {@code ot} member on first read. Racing readers agree on one instance, because - * callers compare otel states by identity to detect changes. + * Builds the {@code ot} member on first read. Racing readers agree on one instance, so identity + * checks on otel states keep skipping redundant work, and publication doesn't depend on {@link + * OtelTraceState} keeping its fields final. */ @Override public CharSequence getOtelTraceState() { From 8d2ef88a7c854f191aeb2b2f5d4081a05a6384ba Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 15:13:21 -0400 Subject: [PATCH 3/5] Add benchmarks for the lazy otel state's inject, export and service-shape cost SamplingDecisionBenchmark gains arms that follow a probability decision with the Datadog header, the W3C tracestate header and the OTLP tracestate, so the deferred OtelTraceState is measured where it's built or skipped. ServiceShapePropagationBenchmark measures head, leaf and intermediate services with a default-configured tracer. It only uses API unchanged since v1.66.0, so it can be built onto release tags to put a cost on #12397. Co-Authored-By: Claude Opus 5.5 --- .../trace/core/SamplingDecisionBenchmark.java | 24 ++ .../ServiceShapePropagationBenchmark.java | 236 ++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/SamplingDecisionBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/SamplingDecisionBenchmark.java index 05f8421c91b..8fa9e42982e 100644 --- a/dd-trace-core/src/jmh/java/datadog/trace/core/SamplingDecisionBenchmark.java +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/SamplingDecisionBenchmark.java @@ -127,6 +127,30 @@ public PropagationTags ptagsProbabilityKeep(ThreadState thread) { return ptags; } + /** + * A probability decision followed by the Datadog header, which doesn't read the OTel tracestate. + */ + @Benchmark + public String ptagsProbabilityKeepDatadogHeader(ThreadState thread) { + return ptagsProbabilityKeep(thread).headerValue(PropagationTags.HeaderType.DATADOG); + } + + /** + * A probability decision followed by the W3C {@code tracestate} header, which includes the OTel + * {@code ot} member. + */ + @Benchmark + public String ptagsProbabilityKeepW3CHeader(ThreadState thread) { + return ptagsProbabilityKeep(thread).headerValue(PropagationTags.HeaderType.W3C); + } + + /** A probability decision followed by the tracestate OTLP export writes, OTel member included. */ + @Benchmark + public String ptagsProbabilityKeepOtlpTracestate(ThreadState thread) { + PropagationTags ptags = ptagsProbabilityKeep(thread); + return ptags.getW3CTracestate(ptags.samplingState()); + } + /** A priority-only decision (manual keep), on fresh trace-level propagation tags. */ @Benchmark public PropagationTags ptagsPriorityKeep() { diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java new file mode 100644 index 00000000000..00d7e97650d --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java @@ -0,0 +1,236 @@ +package datadog.trace.core; + +import static datadog.trace.api.config.TracerConfig.TRACE_RATE_LIMIT; +import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import datadog.context.Context; +import datadog.context.propagation.CarrierSetter; +import datadog.context.propagation.CarrierVisitor; +import datadog.context.propagation.Propagators; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.common.sampling.Sampler; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Per-request tracing cost of the standard propagation shapes, with a default-configured tracer + * (default propagation styles: Datadog, W3C trace context and baggage). It puts a cost on the + * per-trace sampling and propagation bookkeeping, such as the OpenTelemetry {@code ot} tracestate + * member that v1.67.0 started generating, as a service actually sees it: + * + *
    + *
  • {@link #headNoPropagation}: a root service span that makes the sampling decision (at + * finish) and calls nothing downstream. + *
  • {@link #headPropagation}: a root service span plus an outbound client span that injects, so + * the decision is made at injection and written into the headers. + *
  • {@link #leaf}: extracts an upstream decision and calls nothing downstream. + *
  • {@link #intermediate}: extracts an upstream decision and relays it through an outbound + * client span. + *
+ * + *

{@code upstream} selects what the extracting shapes receive: {@code sameVersion} is what this + * tracer itself injects (a fleet on one version), and {@code withOtelMember} is a fixed current + * format carrying an {@code ot} member (an upgraded upstream, whatever this tracer's version). + * Upstream headers cycle through 1024 distinct trace ids so caches keyed by header content see + * realistic variety. + * + *

Read the allocation columns. Run with {@code -prof gc}; {@code gc.alloc.rate.norm} + * (B/op) is the primary signal. Throughput is directional only. + * + *

Drift-stable so it can be built onto past release tags (e.g. v1.66.0, the last release + * before the {@code ot} member was generated). It uses only the {@code datadog.context} propagation + * API, {@link CoreTracer} builder/span API, {@link Sampler.Builder#forConfig} and {@link + * DropWriter}, all unchanged from v1.66.0 to master. Keep it inside that surface. + * + * 

+ *   ./gradlew :dd-trace-core:jmhJar
+ *   java -jar dd-trace-core/build/libs/dd-trace-core-*-jmh.jar ServiceShapePropagationBenchmark -prof gc
+ *
+ */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@Threads(8) +@OutputTimeUnit(MICROSECONDS) +@Fork(value = 3, jvmArgsAppend = "-DTEST_LOG_LEVEL=warn") +public class ServiceShapePropagationBenchmark { + private static final String INSTRUMENTATION_NAME = "bench"; + private static final String SERVER_OPERATION_NAME = "servlet.request"; + private static final String CLIENT_OPERATION_NAME = "http.request"; + private static final int UPSTREAM_TRACE_COUNT = 1024; + // 128-bit trace ids carry their creation time in the top 32 bits, the rest of the high half zero + private static final long TRACE_ID_EPOCH_SECONDS = 1_760_000_000L; + + /** Injects into the benchmark's blackhole, so header values are consumed without a map. */ + private static final CarrierSetter BLACKHOLE_SETTER = + (blackhole, key, value) -> blackhole.consume(value); + + private static final CarrierVisitor> MAP_VISITOR = Map::forEach; + + /** + * {@code default}: the default sampler with no agent rates, as most services run. {@code rule}: a + * configured sample rate (1.0, limit raised so the benchmark isn't rate limited). + */ + @Param({"default", "rule"}) + String sampler; + + CoreTracer tracer; + + @Setup + public void setup(Blackhole blackhole) { + CoreTracer.CoreTracerBuilder builder = CoreTracer.builder().writer(new DropWriter(blackhole)); + if ("rule".equals(sampler)) { + Properties ruleConfig = new Properties(); + ruleConfig.setProperty(TRACE_SAMPLE_RATE, "1.0"); + ruleConfig.setProperty(TRACE_RATE_LIMIT, String.valueOf(Integer.MAX_VALUE)); + builder.sampler(Sampler.Builder.forConfig(ruleConfig)); + } + // Building the tracer registers its propagators as the default ones. + tracer = builder.build(); + } + + @TearDown + public void tearDown() { + tracer.close(); + } + + @Benchmark + public void headNoPropagation() { + AgentSpan server = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + server.finish(); + } + + @Benchmark + public void headPropagation(Blackhole blackhole) { + AgentSpan server = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + callDownstream(server, blackhole); + server.finish(); + } + + @Benchmark + public void leaf(Upstream upstream, Cursor cursor) { + AgentSpan server = + tracer + .buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME) + .asChildOf(extract(upstream.headers[cursor.next()])) + .start(); + server.finish(); + } + + @Benchmark + public void intermediate(Upstream upstream, Cursor cursor, Blackhole blackhole) { + AgentSpan server = + tracer + .buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME) + .asChildOf(extract(upstream.headers[cursor.next()])) + .start(); + callDownstream(server, blackhole); + server.finish(); + } + + private void callDownstream(AgentSpan server, Blackhole blackhole) { + AgentSpan client = + tracer + .buildSpan(INSTRUMENTATION_NAME, CLIENT_OPERATION_NAME) + .asChildOf(server.spanContext()) + .start(); + Propagators.defaultPropagator().inject(client, blackhole, BLACKHOLE_SETTER); + client.finish(); + } + + private static AgentSpanContext extract(Map headers) { + Context extracted = + Propagators.defaultPropagator().extract(Context.root(), headers, MAP_VISITOR); + return AgentSpan.fromContext(extracted).spanContext(); + } + + /** The incoming headers the extracting shapes cycle through. */ + @State(Scope.Benchmark) + public static class Upstream { + @Param({"sameVersion", "withOtelMember"}) + String upstream; + + @SuppressWarnings("unchecked") + final Map[] headers = new Map[UPSTREAM_TRACE_COUNT]; + + @Setup + public void setup(ServiceShapePropagationBenchmark benchmark) { + Random random = new Random(42); + for (int i = 0; i < UPSTREAM_TRACE_COUNT; ++i) { + headers[i] = + "sameVersion".equals(upstream) + ? injectedBy(benchmark.tracer) + : currentFormat( + random.nextLong() | 1, + (TRACE_ID_EPOCH_SECONDS + i) << 32, + random.nextLong() | 1); + } + } + + private static Map injectedBy(CoreTracer tracer) { + Map headers = new HashMap<>(); + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, CLIENT_OPERATION_NAME).start(); + Propagators.defaultPropagator().inject(span, headers, Map::put); + span.finish(); + return headers; + } + + /** What a current Datadog tracer sends after a default-sampler keep. */ + private static Map currentFormat( + long traceIdLow, long traceIdHigh, long parentId) { + String tid = hex16(traceIdHigh); + String parent = hex16(parentId); + Map headers = new HashMap<>(); + headers.put("x-datadog-trace-id", Long.toUnsignedString(traceIdLow)); + headers.put("x-datadog-parent-id", Long.toUnsignedString(parentId)); + headers.put("x-datadog-sampling-priority", "1"); + headers.put("x-datadog-tags", "_dd.p.dm=-1,_dd.p.tid=" + tid + ",_dd.p.ksr=1"); + headers.put("traceparent", "00-" + tid + hex16(traceIdLow) + "-" + parent + "-01"); + headers.put( + "tracestate", + "dd=s:1;p:" + + parent + + ";t.dm:-1;t.tid:" + + tid + + ";t.ksr:1,ot=rv:" + + hex16(traceIdLow).substring(2) + + ";th:0"); + return headers; + } + + private static String hex16(long value) { + String hex = Long.toHexString(value); + return "0000000000000000".substring(hex.length()) + hex; + } + } + + /** Per-thread position in the upstream headers. */ + @State(Scope.Thread) + public static class Cursor { + int index; + + int next() { + index = (index + 1) & (UPSTREAM_TRACE_COUNT - 1); + return index; + } + } +} From 1d4880bbb45013ba92cf0708df030bee67983a12 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 15:32:31 -0400 Subject: [PATCH 4/5] Rename ServiceShapePropagationBenchmark to PropagationScenariosBenchmark Describe it as a scenario benchmark: each arm is a whole request driven through the tracer's public API, rather than one method in isolation. Co-Authored-By: Claude Opus 5.5 --- ...ava => PropagationScenariosBenchmark.java} | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) rename dd-trace-core/src/jmh/java/datadog/trace/core/{ServiceShapePropagationBenchmark.java => PropagationScenariosBenchmark.java} (88%) diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/PropagationScenariosBenchmark.java similarity index 88% rename from dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java rename to dd-trace-core/src/jmh/java/datadog/trace/core/PropagationScenariosBenchmark.java index 00d7e97650d..746357ebadc 100644 --- a/dd-trace-core/src/jmh/java/datadog/trace/core/ServiceShapePropagationBenchmark.java +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/PropagationScenariosBenchmark.java @@ -31,10 +31,15 @@ import org.openjdk.jmh.infra.Blackhole; /** - * Per-request tracing cost of the standard propagation shapes, with a default-configured tracer + * Per-request tracing cost of the standard propagation scenarios, with a default-configured tracer * (default propagation styles: Datadog, W3C trace context and baggage). It puts a cost on the * per-trace sampling and propagation bookkeeping, such as the OpenTelemetry {@code ot} tracestate - * member that v1.67.0 started generating, as a service actually sees it: + * member that v1.67.0 started generating, as a service actually sees it. + * + *

This is a scenario benchmark: each arm is a whole request as a service handles it, + * driven through the tracer's public API, rather than one method in isolation. A change to any + * piece shows up as what it costs a real request, and the result doesn't depend on which internals + * the benchmark happens to call. The scenarios: * *

    *
  • {@link #headNoPropagation}: a root service span that makes the sampling decision (at @@ -46,11 +51,11 @@ * client span. *
* - *

{@code upstream} selects what the extracting shapes receive: {@code sameVersion} is what this - * tracer itself injects (a fleet on one version), and {@code withOtelMember} is a fixed current - * format carrying an {@code ot} member (an upgraded upstream, whatever this tracer's version). - * Upstream headers cycle through 1024 distinct trace ids so caches keyed by header content see - * realistic variety. + *

{@code upstream} selects what the extracting scenarios receive: {@code sameVersion} is what + * this tracer itself injects (a fleet on one version), and {@code withOtelMember} is a fixed + * current format carrying an {@code ot} member (an upgraded upstream, whatever this tracer's + * version). Upstream headers cycle through 1024 distinct trace ids so caches keyed by header + * content see realistic variety. * *

Read the allocation columns. Run with {@code -prof gc}; {@code gc.alloc.rate.norm} * (B/op) is the primary signal. Throughput is directional only. @@ -62,7 +67,7 @@ * * 

  *   ./gradlew :dd-trace-core:jmhJar
- *   java -jar dd-trace-core/build/libs/dd-trace-core-*-jmh.jar ServiceShapePropagationBenchmark -prof gc
+ *   java -jar dd-trace-core/build/libs/dd-trace-core-*-jmh.jar PropagationScenariosBenchmark -prof gc
  *
*/ @State(Scope.Benchmark) @@ -72,7 +77,7 @@ @Threads(8) @OutputTimeUnit(MICROSECONDS) @Fork(value = 3, jvmArgsAppend = "-DTEST_LOG_LEVEL=warn") -public class ServiceShapePropagationBenchmark { +public class PropagationScenariosBenchmark { private static final String INSTRUMENTATION_NAME = "bench"; private static final String SERVER_OPERATION_NAME = "servlet.request"; private static final String CLIENT_OPERATION_NAME = "http.request"; @@ -163,7 +168,7 @@ private static AgentSpanContext extract(Map headers) { return AgentSpan.fromContext(extracted).spanContext(); } - /** The incoming headers the extracting shapes cycle through. */ + /** The incoming headers the extracting scenarios cycle through. */ @State(Scope.Benchmark) public static class Upstream { @Param({"sameVersion", "withOtelMember"}) @@ -173,7 +178,7 @@ public static class Upstream { final Map[] headers = new Map[UPSTREAM_TRACE_COUNT]; @Setup - public void setup(ServiceShapePropagationBenchmark benchmark) { + public void setup(PropagationScenariosBenchmark benchmark) { Random random = new Random(42); for (int i = 0; i < UPSTREAM_TRACE_COUNT; ++i) { headers[i] = From 74c29818e7c183a3c727b4fd6a1af22fe0f7e410 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 22 Oct 2026 09:16:31 -0400 Subject: [PATCH 5/5] Test that racing readers get one OpenTelemetry state instance Co-Authored-By: Claude Opus 5.5 --- .../ptags/ProbabilitySamplingStateTest.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java index 85bef8be246..b5e0e81f381 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/ProbabilitySamplingStateTest.java @@ -11,6 +11,7 @@ import datadog.trace.core.propagation.PropagationTags; import datadog.trace.core.propagation.PropagationTags.SamplingState; +import java.util.concurrent.CountDownLatch; import org.junit.jupiter.api.Test; import org.tabletest.junit.TableTest; @@ -52,6 +53,49 @@ void repeatedReadsReturnOneInstance() { assertSame(state.getOtelTraceState(), state.getOtelTraceState()); } + /** + * Readers racing to build the state agree on one instance. A stress test can't force the race, so + * this is a smoke test of the compare-and-set: it can't fail while that holds. + */ + @Test + void racingReadsReturnOneInstance() throws InterruptedException { + int readers = 8; + for (int trial = 0; trial < 100; trial++) { + PropagationTags ptags = factory.empty(); + ptags.tryUpdateProbabilitySamplingDecision( + SAMPLER_KEEP, AGENT_RATE, 0.1, false, TRACE_ID_LOW_ORDER_BITS, false); + SamplingState state = ptags.samplingState(); + + CharSequence[] seen = new CharSequence[readers]; + CountDownLatch start = new CountDownLatch(1); + Thread[] threads = new Thread[readers]; + for (int i = 0; i < readers; i++) { + int reader = i; + threads[i] = + new Thread( + () -> { + try { + start.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + seen[reader] = state.getOtelTraceState(); + }); + threads[i].start(); + } + start.countDown(); + for (Thread thread : threads) { + thread.join(); + } + + for (CharSequence otelTraceState : seen) { + assertSame(seen[0], otelTraceState); + } + assertSame(seen[0], state.getOtelTraceState()); + } + } + @Test void rateLimiterRejectionRecordsNoOtelState() { PropagationTags ptags = factory.empty();