From 6f54e3e69685dc694aae8bc5ef4384fa735e0263 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 19 Oct 2026 13:22:19 -0400 Subject: [PATCH 1/6] Add test for remote config tracing tags overriding DD_TAGS Remote config tracing_tags replaces the global tags (DD_TAGS), and removing the override restores them. This pins the intent stated in system-tests' TestDynamicConfigV2.test_tracing_client_tracing_tags, which only asserts that the RC tags are present. Co-Authored-By: Claude Opus 5.5 --- .../datadog/trace/core/CoreTracerTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java index 666431d8f1f..92abbb92e07 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java @@ -50,6 +50,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; @@ -597,6 +598,56 @@ void verifyConfigurationPollingWithCustomTags( } } + /** + * Remote config {@code tracing_tags} overrides the global tags ({@code DD_TAGS}), and removing + * the override restores them, matching system-tests' {@code + * TestDynamicConfigV2.test_tracing_client_tracing_tags}. + */ + @Test + void remoteConfigTracingTagsOverrideGlobalTags() throws Exception { + ParsedConfigKey key = ParsedConfigKey.parse("datadog/2/APM_TRACING/config_overrides/config"); + ConfigurationPoller poller = mock(ConfigurationPoller.class); + SharedCommunicationObjects sco = createScoWithPoller(poller); + + ProductListener[] capturedUpdater = {null}; + doAnswer( + inv -> { + capturedUpdater[0] = inv.getArgument(1, ProductListener.class); + return null; + }) + .when(poller) + .addListener(eq(Product.APM_TRACING), any(ProductListener.class)); + + Properties properties = new Properties(); + properties.setProperty(GeneralConfig.TAGS, "local_key:local_val"); + CoreTracer tracer = + CoreTracer.builder() + .withProperties(properties) + .sharedCommunicationObjects(sco) + .pollForTracingConfiguration() + .build(); + unclosedTracers.add(tracer); + + try { + Map localTags = buildStringMap("local_key", "local_val"); + assertEquals(localTags, tracer.captureTraceConfig().mergedTracerTags); + + String json = "{\"lib_config\":{\"tracing_tags\": [\"rc_key:rc_val\"]}}"; + capturedUpdater[0].accept(key, json.getBytes(StandardCharsets.UTF_8), null); + capturedUpdater[0].commit(null); + + assertEquals( + buildStringMap("rc_key", "rc_val"), tracer.captureTraceConfig().mergedTracerTags); + + capturedUpdater[0].remove(key, null); + capturedUpdater[0].commit(null); + + assertEquals(localTags, tracer.captureTraceConfig().mergedTracerTags); + } finally { + tracer.close(); + } + } + static final String ACTION_JSON = "'{\"action\": \"enable\", \"lib_config\":" + "{\"tracing_sampling_rate\": null," From 1b8134eb87f604b124c1cfc62312fd3cc241e761 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 19 Oct 2026 14:21:32 -0400 Subject: [PATCH 2/6] Keep global tags apart from span defaults in the config snapshot Remote config tracing_tags now replaces only the global tags (DD_TAGS). Previously the dynamic config held DD_TAGS and DD_SPAN_TAGS merged, so an RC tag override also dropped DD_SPAN_TAGS until it was removed. ConfigSnapshot now keeps the global tags and the span defaults as separate frozen TagMaps, and builds mergedTracerTags from them (span defaults win, as before). Span defaults are DD_SPAN_TAGS, or the tags given to the builder's defaultSpanTags, and are no longer lost on the first RC tag change. Co-Authored-By: Claude Opus 5.5 --- .../java/datadog/trace/core/CoreTracer.java | 44 ++++++++++++++----- .../main/java/datadog/trace/api/Config.java | 5 +++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 8f52b49052c..801feac5357 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -221,11 +221,12 @@ private static final class LocalRootSpanTags { */ private final boolean apmTracingEnabled; - /** A set of tags that are added to every span */ + /** + * Span defaults added to every span: {@code DD_SPAN_TAGS}, or the tags given to the builder. + * Unlike the global tags ({@code DD_TAGS}), remote config doesn't replace these. + */ private final TagMap defaultSpanTags; - private final boolean defaultSpanTagsNeedsIntercept; - /** number of spans in a pending trace before they get flushed */ private final int partialFlushMinSpans; @@ -543,7 +544,7 @@ public CoreTracerBuilder config(final Config config) { invertMap(config.getBaggageMapping()))); // Explicitly skip setting scope manager because it depends on statsDClient localRootSpanTags(config.getLocalRootSpanTags()); - defaultSpanTags(withTracerTags(config.getMergedSpanTags(), config, null)); + defaultSpanTags(config.getSpanTags()); serviceNameMappings(config.getServiceMapping()); taggedHeaders(config.getRequestHeaderTags()); baggageMapping(config.getBaggageMapping()); @@ -737,7 +738,6 @@ private CoreTracer( null == tagInterceptor ? new TagInterceptor(new RuleFlags(config)) : tagInterceptor; this.defaultSpanTags = defaultSpanTags; - this.defaultSpanTagsNeedsIntercept = this.tagInterceptor.needsIntercept(this.defaultSpanTags); this.dynamicConfig = DynamicConfig.create(ConfigSnapshot::new) @@ -751,7 +751,7 @@ private CoreTracer( .setTraceSampleRate(config.getTraceSampleRate()) .setSpanSamplingRules(spanSamplingRules.getRules()) .setTraceSamplingRules(traceSamplingRules.getRules(), traceSamplingRulesJson) - .setTracingTags(config.getMergedSpanTags()) + .setTracingTags(config.getGlobalTags()) .setDataStreamsTransactionExtractors(dataStreamsTransactionExtractors.getExtractors()) .apply(); @@ -2590,7 +2590,21 @@ public void run() { protected class ConfigSnapshot extends DynamicConfig.Snapshot { final Sampler sampler; + /** + * The global tags ({@code DD_TAGS}, or their remote-config override), as given. These are what + * an OTLP export carries as resource attributes. + */ + final TagMap globalTags; + + /** + * The span defaults with the tracer-defined tags ({@code env}, {@code _dd.dsm.enabled}, ...), + * but without the global tags. + */ + final TagMap spanDefaultTags; + + /** The global tags overlaid by {@link #spanDefaultTags}: the tags every span starts with. */ final TagMap mergedTracerTags; + final boolean mergedTracerTagsNeedsIntercept; protected ConfigSnapshot( @@ -2607,14 +2621,22 @@ protected ConfigSnapshot( sampler = Sampler.Builder.forConfig(CoreTracer.this.initialConfig, this, agentSampler); } - if (null == oldSnapshot) { - mergedTracerTags = CoreTracer.this.defaultSpanTags.immutableCopy(); - this.mergedTracerTagsNeedsIntercept = CoreTracer.this.defaultSpanTagsNeedsIntercept; - } else if (getTracingTags().equals(oldSnapshot.getTracingTags())) { + if (null != oldSnapshot && getTracingTags().equals(oldSnapshot.getTracingTags())) { + globalTags = oldSnapshot.globalTags; + spanDefaultTags = oldSnapshot.spanDefaultTags; mergedTracerTags = oldSnapshot.mergedTracerTags; mergedTracerTagsNeedsIntercept = oldSnapshot.mergedTracerTagsNeedsIntercept; } else { - mergedTracerTags = withTracerTags(getTracingTags(), CoreTracer.this.initialConfig, this); + Config config = CoreTracer.this.initialConfig; + TagMap defaultSpanTags = CoreTracer.this.defaultSpanTags; + globalTags = TagMap.fromMapImmutable(getTracingTags()); + spanDefaultTags = withTracerTags(defaultSpanTags, config, this); + + // span defaults take precedence over global tags + TagMap userSpanTags = TagMap.create(globalTags.size() + defaultSpanTags.size()); + userSpanTags.putAll(globalTags); + userSpanTags.putAll(defaultSpanTags); + mergedTracerTags = withTracerTags(userSpanTags, config, this); mergedTracerTagsNeedsIntercept = CoreTracer.this.tagInterceptor.needsIntercept(mergedTracerTags); } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 3d81035b6a8..6a67188cbf9 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -5555,6 +5555,11 @@ public String getVersion() { return version; } + /** Tags set on every span ({@code DD_SPAN_TAGS}), without the global tags. */ + public Map getSpanTags() { + return spanTags; + } + public Map getMergedSpanTags() { // Do not include runtimeId into span tags: we only want that added to the root span final Map result = newHashMap(getGlobalTags().size() + spanTags.size()); From 367ddbbca7b3a6b6fe911de2b55f545a006f0410 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 19 Oct 2026 14:21:34 -0400 Subject: [PATCH 3/6] Test that remote config tracing tags keep span defaults Co-Authored-By: Claude Opus 5.5 --- .../datadog/trace/core/CoreTracerTest.java | 86 +++++++++++++++++-- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java index 92abbb92e07..1143fa11a97 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java @@ -601,10 +601,17 @@ void verifyConfigurationPollingWithCustomTags( /** * Remote config {@code tracing_tags} overrides the global tags ({@code DD_TAGS}), and removing * the override restores them, matching system-tests' {@code - * TestDynamicConfigV2.test_tracing_client_tracing_tags}. + * TestDynamicConfigV2.test_tracing_client_tracing_tags}. Span tags ({@code DD_SPAN_TAGS}) are + * separate local config, so they're kept alongside the override. */ - @Test - void remoteConfigTracingTagsOverrideGlobalTags() throws Exception { + @TableTest({ + "scenario | localTagsKey | expectedWithOverride ", + "DD_TAGS | tags | [rc_key: rc_val] ", + "DD_SPAN_TAGS | trace.span.tags | [local_key: local_val, rc_key: rc_val]" + }) + void remoteConfigTracingTagsOverrideGlobalTags( + String scenario, String localTagsKey, Map expectedWithOverride) + throws Exception { ParsedConfigKey key = ParsedConfigKey.parse("datadog/2/APM_TRACING/config_overrides/config"); ConfigurationPoller poller = mock(ConfigurationPoller.class); SharedCommunicationObjects sco = createScoWithPoller(poller); @@ -619,7 +626,7 @@ void remoteConfigTracingTagsOverrideGlobalTags() throws Exception { .addListener(eq(Product.APM_TRACING), any(ProductListener.class)); Properties properties = new Properties(); - properties.setProperty(GeneralConfig.TAGS, "local_key:local_val"); + properties.setProperty(localTagsKey, "local_key:local_val"); CoreTracer tracer = CoreTracer.builder() .withProperties(properties) @@ -636,8 +643,7 @@ void remoteConfigTracingTagsOverrideGlobalTags() throws Exception { capturedUpdater[0].accept(key, json.getBytes(StandardCharsets.UTF_8), null); capturedUpdater[0].commit(null); - assertEquals( - buildStringMap("rc_key", "rc_val"), tracer.captureTraceConfig().mergedTracerTags); + assertEquals(expectedWithOverride, tracer.captureTraceConfig().mergedTracerTags); capturedUpdater[0].remove(key, null); capturedUpdater[0].commit(null); @@ -648,6 +654,74 @@ void remoteConfigTracingTagsOverrideGlobalTags() throws Exception { } } + /** + * The config snapshot keeps the global tags apart from the span defaults, so an OTLP export can + * carry the global tags as resource attributes; remote config replaces only the global tags. + */ + @Test + void configSnapshotSeparatesGlobalTagsFromSpanDefaults() throws Exception { + ParsedConfigKey key = ParsedConfigKey.parse("datadog/2/APM_TRACING/config_overrides/config"); + ConfigurationPoller poller = mock(ConfigurationPoller.class); + SharedCommunicationObjects sco = createScoWithPoller(poller); + + ProductListener[] capturedUpdater = {null}; + doAnswer( + inv -> { + capturedUpdater[0] = inv.getArgument(1, ProductListener.class); + return null; + }) + .when(poller) + .addListener(eq(Product.APM_TRACING), any(ProductListener.class)); + + Properties properties = new Properties(); + properties.setProperty(GeneralConfig.TAGS, "global_key:global_val,shared_key:global_val"); + properties.setProperty(TracerConfig.SPAN_TAGS, "span_key:span_val,shared_key:span_val"); + properties.setProperty(GeneralConfig.ENV, "prod"); + CoreTracer tracer = + CoreTracer.builder() + .withProperties(properties) + .sharedCommunicationObjects(sco) + .pollForTracingConfiguration() + .build(); + unclosedTracers.add(tracer); + + try { + Map spanDefaults = + buildStringMap("span_key", "span_val", "shared_key", "span_val", "env", "prod"); + + ConfigSnapshot config = tracer.captureTraceConfig(); + // Config adds DD_ENV to the global tags + assertEquals( + buildStringMap("global_key", "global_val", "shared_key", "global_val", "env", "prod"), + config.globalTags); + assertEquals(spanDefaults, config.spanDefaultTags); + assertEquals( + buildStringMap( + "global_key", "global_val", + "span_key", "span_val", + "shared_key", "span_val", + "env", "prod"), + config.mergedTracerTags); + + String json = "{\"lib_config\":{\"tracing_tags\": [\"rc_key:rc_val\"]}}"; + capturedUpdater[0].accept(key, json.getBytes(StandardCharsets.UTF_8), null); + capturedUpdater[0].commit(null); + + config = tracer.captureTraceConfig(); + assertEquals(buildStringMap("rc_key", "rc_val"), config.globalTags); + assertEquals(spanDefaults, config.spanDefaultTags); + assertEquals( + buildStringMap( + "rc_key", "rc_val", + "span_key", "span_val", + "shared_key", "span_val", + "env", "prod"), + config.mergedTracerTags); + } finally { + tracer.close(); + } + } + static final String ACTION_JSON = "'{\"action\": \"enable\", \"lib_config\":" + "{\"tracing_sampling_rate\": null," From 3d144d9774a8bb72875142319313fc6e7aa72bf9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 19 Oct 2026 14:23:56 -0400 Subject: [PATCH 4/6] Fix mergedTracerTags Javadoc to match how it's built Co-Authored-By: Claude Opus 5.5 --- .../src/main/java/datadog/trace/core/CoreTracer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 801feac5357..e2f327f97b3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -2602,7 +2602,11 @@ protected class ConfigSnapshot extends DynamicConfig.Snapshot { */ final TagMap spanDefaultTags; - /** The global tags overlaid by {@link #spanDefaultTags}: the tags every span starts with. */ + /** + * The tags every span starts with: the global tags merged with the span defaults (span defaults + * win), then given the tracer-defined tags. {@code version} is dropped here even when it's a + * global tag, unless it's a split-by tag. + */ final TagMap mergedTracerTags; final boolean mergedTracerTagsNeedsIntercept; From 734aa302ef3f511307f5e7ca0052f0aba00e8c0a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 09:00:00 -0400 Subject: [PATCH 5/6] Keep remote config tags overrides replacing span defaults, and keep env and version Keeps the split of global tags and span defaults in the config snapshot, but restores the existing remote config behavior: while a tracing_tags override is active, it replaces DD_TRACE_SPAN_TAGS too. Whether span defaults should survive an override is left for a separate change. The override's global tags keep the configured env and version, so a consumer of the global tags (the OTLP resource) still sees them. Also names the span tags setting DD_TRACE_SPAN_TAGS, and switches the remote config tests to injectSysConfig. Co-Authored-By: Claude Opus 5.5 --- .../java/datadog/trace/core/CoreTracer.java | 40 +++++++++--- .../datadog/trace/core/CoreTracerTest.java | 62 +++++++++---------- .../main/java/datadog/trace/api/Config.java | 2 +- 3 files changed, 63 insertions(+), 41 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index e2f327f97b3..23e7f7b9e6b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -222,8 +222,9 @@ private static final class LocalRootSpanTags { private final boolean apmTracingEnabled; /** - * Span defaults added to every span: {@code DD_SPAN_TAGS}, or the tags given to the builder. - * Unlike the global tags ({@code DD_TAGS}), remote config doesn't replace these. + * Span defaults added to every span: {@code DD_TRACE_SPAN_TAGS}, or the tags given to the + * builder. A remote config {@code tracing_tags} override replaces these along with the global + * tags while it's active. */ private final TagMap defaultSpanTags; @@ -2591,14 +2592,16 @@ protected class ConfigSnapshot extends DynamicConfig.Snapshot { final Sampler sampler; /** - * The global tags ({@code DD_TAGS}, or their remote-config override), as given. These are what - * an OTLP export carries as resource attributes. + * The global tags ({@code DD_TAGS}, or their remote-config override, which keeps the configured + * {@code env} and {@code version}). These are what an OTLP export carries as resource + * attributes. */ final TagMap globalTags; /** * The span defaults with the tracer-defined tags ({@code env}, {@code _dd.dsm.enabled}, ...), - * but without the global tags. + * but without the global tags. Only the tracer-defined tags while a remote config {@code + * tracing_tags} override is active. */ final TagMap spanDefaultTags; @@ -2632,8 +2635,15 @@ protected ConfigSnapshot( mergedTracerTagsNeedsIntercept = oldSnapshot.mergedTracerTagsNeedsIntercept; } else { Config config = CoreTracer.this.initialConfig; - TagMap defaultSpanTags = CoreTracer.this.defaultSpanTags; - globalTags = TagMap.fromMapImmutable(getTracingTags()); + Map tracingTags = getTracingTags(); + boolean overridden = !tracingTags.equals(config.getGlobalTags()); + // A remote config tracing_tags override stands in for all the user's tags, span defaults + // included, until it's removed. It doesn't change the service's env or version. + TagMap defaultSpanTags = overridden ? TagMap.EMPTY : CoreTracer.this.defaultSpanTags; + globalTags = + overridden + ? withIdentityTags(tracingTags, config.getGlobalTags()) + : TagMap.fromMapImmutable(tracingTags); spanDefaultTags = withTracerTags(defaultSpanTags, config, this); // span defaults take precedence over global tags @@ -2647,6 +2657,22 @@ protected ConfigSnapshot( } } + /** {@code overrideTags} with the {@code env} and {@code version} from {@code configTags}. */ + private static TagMap withIdentityTags( + Map overrideTags, Map configTags) { + TagMap result = TagMap.create(overrideTags.size() + 2); + result.putAll(overrideTags); + String env = configTags.get(Tags.ENV); + if (env != null) { + result.set(Tags.ENV, env); + } + String version = configTags.get(Tags.VERSION); + if (version != null) { + result.set(Tags.VERSION, version); + } + return result.freeze(); + } + /** * Tags added by the tracer to all spans; combines user-supplied tags with tracer-defined tags. */ diff --git a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java index 1143fa11a97..ad2c3163c62 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java @@ -50,7 +50,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; @@ -601,13 +600,13 @@ void verifyConfigurationPollingWithCustomTags( /** * Remote config {@code tracing_tags} overrides the global tags ({@code DD_TAGS}), and removing * the override restores them, matching system-tests' {@code - * TestDynamicConfigV2.test_tracing_client_tracing_tags}. Span tags ({@code DD_SPAN_TAGS}) are - * separate local config, so they're kept alongside the override. + * TestDynamicConfigV2.test_tracing_client_tracing_tags}. The override replaces the span tags + * ({@code DD_TRACE_SPAN_TAGS}) too. */ @TableTest({ - "scenario | localTagsKey | expectedWithOverride ", - "DD_TAGS | tags | [rc_key: rc_val] ", - "DD_SPAN_TAGS | trace.span.tags | [local_key: local_val, rc_key: rc_val]" + "scenario | localTagsKey | expectedWithOverride", + "DD_TAGS | tags | [rc_key: rc_val] ", + "DD_TRACE_SPAN_TAGS | trace.span.tags | [rc_key: rc_val] " }) void remoteConfigTracingTagsOverrideGlobalTags( String scenario, String localTagsKey, Map expectedWithOverride) @@ -625,14 +624,9 @@ void remoteConfigTracingTagsOverrideGlobalTags( .when(poller) .addListener(eq(Product.APM_TRACING), any(ProductListener.class)); - Properties properties = new Properties(); - properties.setProperty(localTagsKey, "local_key:local_val"); + injectSysConfig(localTagsKey, "local_key:local_val"); CoreTracer tracer = - CoreTracer.builder() - .withProperties(properties) - .sharedCommunicationObjects(sco) - .pollForTracingConfiguration() - .build(); + CoreTracer.builder().sharedCommunicationObjects(sco).pollForTracingConfiguration().build(); unclosedTracers.add(tracer); try { @@ -656,7 +650,7 @@ void remoteConfigTracingTagsOverrideGlobalTags( /** * The config snapshot keeps the global tags apart from the span defaults, so an OTLP export can - * carry the global tags as resource attributes; remote config replaces only the global tags. + * carry the global tags as resource attributes. A remote config override replaces both. */ @Test void configSnapshotSeparatesGlobalTagsFromSpanDefaults() throws Exception { @@ -673,16 +667,12 @@ void configSnapshotSeparatesGlobalTagsFromSpanDefaults() throws Exception { .when(poller) .addListener(eq(Product.APM_TRACING), any(ProductListener.class)); - Properties properties = new Properties(); - properties.setProperty(GeneralConfig.TAGS, "global_key:global_val,shared_key:global_val"); - properties.setProperty(TracerConfig.SPAN_TAGS, "span_key:span_val,shared_key:span_val"); - properties.setProperty(GeneralConfig.ENV, "prod"); + injectSysConfig(GeneralConfig.TAGS, "global_key:global_val,shared_key:global_val"); + injectSysConfig(TracerConfig.SPAN_TAGS, "span_key:span_val,shared_key:span_val"); + injectSysConfig(GeneralConfig.ENV, "prod"); + injectSysConfig(GeneralConfig.VERSION, "1.2.3"); CoreTracer tracer = - CoreTracer.builder() - .withProperties(properties) - .sharedCommunicationObjects(sco) - .pollForTracingConfiguration() - .build(); + CoreTracer.builder().sharedCommunicationObjects(sco).pollForTracingConfiguration().build(); unclosedTracers.add(tracer); try { @@ -690,9 +680,13 @@ void configSnapshotSeparatesGlobalTagsFromSpanDefaults() throws Exception { buildStringMap("span_key", "span_val", "shared_key", "span_val", "env", "prod"); ConfigSnapshot config = tracer.captureTraceConfig(); - // Config adds DD_ENV to the global tags + // Config adds DD_ENV and DD_VERSION to the global tags assertEquals( - buildStringMap("global_key", "global_val", "shared_key", "global_val", "env", "prod"), + buildStringMap( + "global_key", "global_val", + "shared_key", "global_val", + "env", "prod", + "version", "1.2.3"), config.globalTags); assertEquals(spanDefaults, config.spanDefaultTags); assertEquals( @@ -708,15 +702,17 @@ void configSnapshotSeparatesGlobalTagsFromSpanDefaults() throws Exception { capturedUpdater[0].commit(null); config = tracer.captureTraceConfig(); - assertEquals(buildStringMap("rc_key", "rc_val"), config.globalTags); - assertEquals(spanDefaults, config.spanDefaultTags); + // the override keeps the configured env and version assertEquals( - buildStringMap( - "rc_key", "rc_val", - "span_key", "span_val", - "shared_key", "span_val", - "env", "prod"), - config.mergedTracerTags); + buildStringMap("rc_key", "rc_val", "env", "prod", "version", "1.2.3"), config.globalTags); + assertEquals(buildStringMap("env", "prod"), config.spanDefaultTags); + assertEquals(buildStringMap("rc_key", "rc_val", "env", "prod"), config.mergedTracerTags); + + capturedUpdater[0].remove(key, null); + capturedUpdater[0].commit(null); + + config = tracer.captureTraceConfig(); + assertEquals(spanDefaults, config.spanDefaultTags); } finally { tracer.close(); } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 6a67188cbf9..0b388f0c1b0 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -5555,7 +5555,7 @@ public String getVersion() { return version; } - /** Tags set on every span ({@code DD_SPAN_TAGS}), without the global tags. */ + /** Tags set on every span ({@code DD_TRACE_SPAN_TAGS}), without the global tags. */ public Map getSpanTags() { return spanTags; } From 1e46ce6f2ed801036fb62acf2d28dd37daf81ed2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 21 Oct 2026 13:56:04 -0400 Subject: [PATCH 6/6] Track a remote config tracing tags override explicitly Deciding whether an override was active by comparing the tags to DD_TAGS missed an override equal to them, including an empty override when no global tags are set. Mark tags set from remote config in DynamicConfig and use that, also when deciding whether to reuse a snapshot. Co-Authored-By: Claude Opus 5.5 --- .../java/datadog/trace/core/CoreTracer.java | 6 ++++-- .../datadog/trace/core/TracingConfigPoller.java | 2 +- .../java/datadog/trace/core/CoreTracerTest.java | 16 ++++++++++------ .../java/datadog/trace/api/DynamicConfig.java | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 23e7f7b9e6b..3f841a1aa31 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -2628,7 +2628,9 @@ protected ConfigSnapshot( sampler = Sampler.Builder.forConfig(CoreTracer.this.initialConfig, this, agentSampler); } - if (null != oldSnapshot && getTracingTags().equals(oldSnapshot.getTracingTags())) { + if (null != oldSnapshot + && isTracingTagsOverridden() == oldSnapshot.isTracingTagsOverridden() + && getTracingTags().equals(oldSnapshot.getTracingTags())) { globalTags = oldSnapshot.globalTags; spanDefaultTags = oldSnapshot.spanDefaultTags; mergedTracerTags = oldSnapshot.mergedTracerTags; @@ -2636,7 +2638,7 @@ protected ConfigSnapshot( } else { Config config = CoreTracer.this.initialConfig; Map tracingTags = getTracingTags(); - boolean overridden = !tracingTags.equals(config.getGlobalTags()); + boolean overridden = isTracingTagsOverridden(); // A remote config tracing_tags override stands in for all the user's tags, span defaults // included, until it's removed. It doesn't change the service's env or version. TagMap defaultSpanTags = overridden ? TagMap.EMPTY : CoreTracer.this.defaultSpanTags; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/TracingConfigPoller.java b/dd-trace-core/src/main/java/datadog/trace/core/TracingConfigPoller.java index daa85debc6c..15b648786e2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/TracingConfigPoller.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/TracingConfigPoller.java @@ -255,7 +255,7 @@ void applyConfigOverrides(LibConfig libConfig) { } maybeOverride(builder::setTraceSampleRate, libConfig.traceSampleRate); - maybeOverride(builder::setTracingTags, parseTagListToMap(libConfig.tracingTags)); + maybeOverride(builder::overrideTracingTags, parseTagListToMap(libConfig.tracingTags)); DebuggerConfigBridge.updateConfig( new DebuggerConfigUpdate( libConfig.dynamicInstrumentationEnabled, diff --git a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java index ad2c3163c62..6e313d17bc4 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/CoreTracerTest.java @@ -601,15 +601,19 @@ void verifyConfigurationPollingWithCustomTags( * Remote config {@code tracing_tags} overrides the global tags ({@code DD_TAGS}), and removing * the override restores them, matching system-tests' {@code * TestDynamicConfigV2.test_tracing_client_tracing_tags}. The override replaces the span tags - * ({@code DD_TRACE_SPAN_TAGS}) too. + * ({@code DD_TRACE_SPAN_TAGS}) too, even when it's empty. */ @TableTest({ - "scenario | localTagsKey | expectedWithOverride", - "DD_TAGS | tags | [rc_key: rc_val] ", - "DD_TRACE_SPAN_TAGS | trace.span.tags | [rc_key: rc_val] " + "scenario | localTagsKey | overrideTags | expectedWithOverride", + "DD_TAGS | tags | '[\"rc_key:rc_val\"]' | [rc_key: rc_val] ", + "DD_TRACE_SPAN_TAGS | trace.span.tags | '[\"rc_key:rc_val\"]' | [rc_key: rc_val] ", + "empty, DD_TRACE_SPAN_TAGS | trace.span.tags | '[]' | [:] " }) void remoteConfigTracingTagsOverrideGlobalTags( - String scenario, String localTagsKey, Map expectedWithOverride) + String scenario, + String localTagsKey, + String overrideTags, + Map expectedWithOverride) throws Exception { ParsedConfigKey key = ParsedConfigKey.parse("datadog/2/APM_TRACING/config_overrides/config"); ConfigurationPoller poller = mock(ConfigurationPoller.class); @@ -633,7 +637,7 @@ void remoteConfigTracingTagsOverrideGlobalTags( Map localTags = buildStringMap("local_key", "local_val"); assertEquals(localTags, tracer.captureTraceConfig().mergedTracerTags); - String json = "{\"lib_config\":{\"tracing_tags\": [\"rc_key:rc_val\"]}}"; + String json = "{\"lib_config\":{\"tracing_tags\": " + overrideTags + "}}"; capturedUpdater[0].accept(key, json.getBytes(StandardCharsets.UTF_8), null); capturedUpdater[0].commit(null); diff --git a/internal-api/src/main/java/datadog/trace/api/DynamicConfig.java b/internal-api/src/main/java/datadog/trace/api/DynamicConfig.java index 19a1ca84abf..0f3b58c6b7d 100644 --- a/internal-api/src/main/java/datadog/trace/api/DynamicConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/DynamicConfig.java @@ -104,6 +104,7 @@ public final class Builder { Map requestHeaderTags; Map responseHeaderTags; Map tracingTags; + boolean tracingTagsOverridden; Map baggageMapping; List spanSamplingRules; @@ -134,6 +135,7 @@ public final class Builder { this.traceSampleRate = snapshot.traceSampleRate; this.tracingTags = snapshot.tracingTags; + this.tracingTagsOverridden = snapshot.tracingTagsOverridden; this.preferredServiceNameAndSource = snapshot.preferredServiceNameAndSource; this.dataStreamsTransactionExtractors = snapshot.dataStreamsTransactionExtractors; @@ -218,6 +220,14 @@ public Builder setTraceSamplingRules( public Builder setTracingTags(Map tracingTags) { this.tracingTags = tracingTags; + this.tracingTagsOverridden = false; + return this; + } + + /** Sets the tracing tags from a remote config override, even an empty one. */ + public Builder overrideTracingTags(Map tracingTags) { + this.tracingTags = tracingTags; + this.tracingTagsOverridden = true; return this; } @@ -332,6 +342,7 @@ public static class Snapshot implements TraceConfig { final Double traceSampleRate; final Map tracingTags; + final boolean tracingTagsOverridden; final Pair preferredServiceNameAndSource; final List dataStreamsTransactionExtractors; @@ -354,6 +365,7 @@ protected Snapshot(DynamicConfig.Builder builder, Snapshot oldSnapshot) { this.traceSampleRate = builder.traceSampleRate; this.tracingTags = nullToEmpty(builder.tracingTags); + this.tracingTagsOverridden = builder.tracingTagsOverridden; this.preferredServiceNameAndSource = builder.preferredServiceNameAndSource; this.dataStreamsTransactionExtractors = builder.dataStreamsTransactionExtractors; @@ -437,6 +449,11 @@ public Map getTracingTags() { return tracingTags; } + /** True while the tracing tags come from a remote config override. */ + public boolean isTracingTagsOverridden() { + return tracingTagsOverridden; + } + @Override public String toString() { return "DynamicConfig{"