Split Micrometer observation into separate operation and command types

nhachicha · nhachicha · commit 4912105093d4 · 2026-04-13T11:49:21.000+01:00
The driver used a single observation name ("mongodb") for both operation-level and command-level spans, which have different sets of low-cardinality tag keys. Prometheus requires all meters sharing a name to have identical tag key sets, causing the second observation type to be silently dropped.
Split MongodbObservation.MONGODB_OBSERVATION into MONGODB_OPERATION (name "mongodb.operation") and MONGODB_COMMAND (name "mongodb.command"), each declaring its own low-cardinality key set. Updated Tracer and TracingManager to pass the observation type through span creation.
diff --git a/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java b/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java
@@ -95,7 +95,7 @@
 import static com.mongodb.internal.connection.ProtocolHelper.isCommandOk;
 import static com.mongodb.internal.logging.LogMessage.Level.DEBUG;
 import static com.mongodb.internal.observability.micrometer.MongodbObservation.HighCardinalityKeyNames.QUERY_TEXT;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.RESPONSE_STATUS_CODE;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.CommandLowCardinalityKeyNames.RESPONSE_STATUS_CODE;
 import static com.mongodb.internal.thread.InterruptionUtil.translateInterruptedException;
 import static java.util.Arrays.asList;
 
diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java
@@ -33,10 +33,9 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.EXCEPTION_MESSAGE;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.EXCEPTION_STACKTRACE;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.EXCEPTION_TYPE;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.MONGODB_OBSERVATION;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.CommandLowCardinalityKeyNames.EXCEPTION_MESSAGE;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.CommandLowCardinalityKeyNames.EXCEPTION_STACKTRACE;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.CommandLowCardinalityKeyNames.EXCEPTION_TYPE;
 import static com.mongodb.internal.observability.micrometer.TracingManager.ENV_OBSERVABILITY_QUERY_TEXT_MAX_LENGTH;
 import static java.lang.System.getenv;
 import static java.util.Optional.ofNullable;
@@ -81,8 +80,9 @@ public MicrometerTracer(final ObservationRegistry observationRegistry, final boo
     }
 
     @Override
-    public Span nextSpan(final String name, @Nullable final TraceContext parent, @Nullable final MongoNamespace namespace) {
-        Observation observation = getObservation(name);
+    public Span nextSpan(final MongodbObservation observationType, final String name,
+            @Nullable final TraceContext parent, @Nullable final MongoNamespace namespace) {
+        Observation observation = getObservation(observationType, name);
 
         if (parent instanceof MicrometerTraceContext) {
             Observation parentObservation = ((MicrometerTraceContext) parent).observation;
@@ -104,8 +104,8 @@ public boolean includeCommandPayload() {
         return allowCommandPayload;
     }
 
-    private Observation getObservation(final String name) {
-        Observation observation = MONGODB_OBSERVATION.observation(observationRegistry,
+    private Observation getObservation(final MongodbObservation observationType, final String name) {
+        Observation observation = observationType.observation(observationRegistry,
                         () -> new SenderContext<>((carrier, key, value) -> {}, Kind.CLIENT))
                 .contextualName(name);
         observation.getContext().put(QUERY_TEXT_LENGTH_CONTEXT_KEY, textMaxLength);
diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/MongodbObservation.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/MongodbObservation.java
@@ -21,34 +21,58 @@
 import io.micrometer.observation.docs.ObservationDocumentation;
 
 /**
- * A MongoDB-based {@link Observation}.
+ * MongoDB {@link ObservationDocumentation} definitions for operation-level and command-level observations.
+ * <p>
+ * These are split into two separate observation types so that each has a distinct name and a fixed set
+ * of low-cardinality tag keys. This is required by Prometheus which rejects meters that share a name
+ * but have different tag key sets.
+ * </p>
  *
  * @since 5.7
  */
 public enum MongodbObservation implements ObservationDocumentation {
 
-    MONGODB_OBSERVATION {
+    /**
+     * Observation for high-level MongoDB operations (e.g. find, insert, update).
+     * Created per user-initiated operation, may contain multiple command spans.
+     */
+    MONGODB_OPERATION {
         @Override
         public String getName() {
-            return "mongodb";
+            return "mongodb.operation";
         }
 
         @Override
         public KeyName[] getLowCardinalityKeyNames() {
-            return LowCardinalityKeyNames.values();
+            return OperationLowCardinalityKeyNames.values();
+        }
+    },
+
+    /**
+     * Observation for wire-protocol MongoDB commands sent to the server.
+     * Created per actual command (nested under an operation span).
+     */
+    MONGODB_COMMAND {
+        @Override
+        public String getName() {
+            return "mongodb.command";
+        }
+
+        @Override
+        public KeyName[] getLowCardinalityKeyNames() {
+            return CommandLowCardinalityKeyNames.values();
         }
 
         @Override
         public KeyName[] getHighCardinalityKeyNames() {
             return HighCardinalityKeyNames.values();
         }
-
     };
 
     /**
-     * Enums related to low cardinality key names for MongoDB tags.
+     * Low cardinality key names for operation-level observations.
      */
-    public enum LowCardinalityKeyNames implements KeyName {
+    public enum OperationLowCardinalityKeyNames implements KeyName {
 
         SYSTEM {
             @Override
@@ -74,22 +98,41 @@ public String asString() {
                 return "db.operation.name";
             }
         },
-        COMMAND_NAME {
+        OPERATION_SUMMARY {
             @Override
             public String asString() {
-                return "db.command.name";
+                return "db.operation.summary";
+            }
+        }
+    }
+
+    /**
+     * Low cardinality key names for command-level observations.
+     */
+    public enum CommandLowCardinalityKeyNames implements KeyName {
+
+        SYSTEM {
+            @Override
+            public String asString() {
+                return "db.system";
             }
         },
-        NETWORK_TRANSPORT {
+        NAMESPACE {
             @Override
             public String asString() {
-                return "network.transport";
+                return "db.namespace";
             }
         },
-        OPERATION_SUMMARY {
+        COLLECTION {
             @Override
             public String asString() {
-                return "db.operation.summary";
+                return "db.collection.name";
+            }
+        },
+        COMMAND_NAME {
+            @Override
+            public String asString() {
+                return "db.command.name";
             }
         },
         QUERY_SUMMARY {
@@ -98,10 +141,10 @@ public String asString() {
                 return "db.query.summary";
             }
         },
-        CURSOR_ID {
+        NETWORK_TRANSPORT {
             @Override
             public String asString() {
-                return "db.mongodb.cursor_id";
+                return "network.transport";
             }
         },
         SERVER_ADDRESS {
@@ -128,6 +171,12 @@ public String asString() {
                 return "db.mongodb.server_connection_id";
             }
         },
+        CURSOR_ID {
+            @Override
+            public String asString() {
+                return "db.mongodb.cursor_id";
+            }
+        },
         TRANSACTION_NUMBER {
             @Override
             public String asString() {
@@ -167,7 +216,7 @@ public String asString() {
     }
 
     /**
-     * Enums related to high cardinality (highly variable values) key names for MongoDB tags.
+     * High cardinality (highly variable values) key names for command-level observations.
      */
     public enum HighCardinalityKeyNames implements KeyName {
 
diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/Tracer.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/Tracer.java
@@ -30,7 +30,8 @@
 public interface Tracer {
     Tracer NO_OP = new Tracer() {
         @Override
-        public Span nextSpan(final String name, @Nullable final TraceContext parent, @Nullable final MongoNamespace namespace) {
+        public Span nextSpan(final MongodbObservation observationType, final String name,
+                @Nullable final TraceContext parent, @Nullable final MongoNamespace namespace) {
             return Span.EMPTY;
         }
 
@@ -46,14 +47,15 @@ public boolean includeCommandPayload() {
     };
 
     /**
-     * Creates a new span with the specified name and optional parent trace context.
+     * Creates a new span with the specified observation type, name and optional parent trace context.
      *
+     * @param observationType The {@link MongodbObservation} type (operation or command).
      * @param name   The name of the span.
      * @param parent The parent {@link TraceContext}, or null if no parent context is provided.
      * @param namespace The {@link MongoNamespace} associated with the span, or null if none is provided.
      * @return A {@link Span} representing the newly created span.
      */
-    Span nextSpan(String name, @Nullable TraceContext parent, @Nullable MongoNamespace namespace);
+    Span nextSpan(MongodbObservation observationType, String name, @Nullable TraceContext parent, @Nullable MongoNamespace namespace);
 
     /**
      * Indicates whether tracing is enabled.
diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java
@@ -34,21 +34,10 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.CLIENT_CONNECTION_ID;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.COLLECTION;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.COMMAND_NAME;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.NAMESPACE;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.OPERATION_NAME;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.OPERATION_SUMMARY;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.NETWORK_TRANSPORT;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.QUERY_SUMMARY;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.SERVER_ADDRESS;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.SERVER_CONNECTION_ID;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.SERVER_PORT;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.SESSION_ID;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.SYSTEM;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.TRANSACTION_NUMBER;
-import static com.mongodb.internal.observability.micrometer.MongodbObservation.LowCardinalityKeyNames.CURSOR_ID;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.MONGODB_COMMAND;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.MONGODB_OPERATION;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.CommandLowCardinalityKeyNames;
+import static com.mongodb.internal.observability.micrometer.MongodbObservation.OperationLowCardinalityKeyNames;
 import static java.lang.System.getenv;
 
 /**
@@ -109,35 +98,31 @@ public TracingManager(@Nullable final ObservabilitySettings observabilitySetting
     }
 
     /**
-     * Creates a new span with the specified name and parent trace context.
-     * <p>
-     * This method is used to create a span that is linked to a parent context,
-     * enabling hierarchical tracing of operations.
-     * </p>
+     * Creates a new span with the specified observation type, name and parent trace context.
      *
-     * @param name          The name of the span.
-     * @param parentContext The parent trace context to associate with the span.
+     * @param observationType The observation type (operation or command).
+     * @param name            The name of the span.
+     * @param parentContext   The parent trace context to associate with the span.
      * @return The created span.
      */
-    public Span addSpan(final String name, @Nullable final TraceContext parentContext) {
-        return tracer.nextSpan(name, parentContext, null);
+    public Span addSpan(final MongodbObservation observationType, final String name,
+            @Nullable final TraceContext parentContext) {
+        return tracer.nextSpan(observationType, name, parentContext, null);
     }
 
     /**
-     * Creates a new span with the specified name, parent trace context, and MongoDB namespace.
-     * <p>
-     * This method is used to create a span that is linked to a parent context,
-     * enabling hierarchical tracing of operations. The MongoDB namespace can be used
-     * by nested spans to access the database and collection name (which might not be easily accessible at connection layer).
-     * </p>
+     * Creates a new span with the specified observation type, name, parent trace context,
+     * and MongoDB namespace.
      *
-     * @param name          The name of the span.
-     * @param parentContext The parent trace context to associate with the span.
-     * @param namespace     The MongoDB namespace associated with the operation.
+     * @param observationType The observation type (operation or command).
+     * @param name            The name of the span.
+     * @param parentContext   The parent trace context to associate with the span.
+     * @param namespace       The MongoDB namespace associated with the operation.
      * @return The created span.
      */
-    public Span addSpan(final String name, @Nullable final TraceContext parentContext, final MongoNamespace namespace) {
-        return tracer.nextSpan(name, parentContext, namespace);
+    public Span addSpan(final MongodbObservation observationType, final String name,
+            @Nullable final TraceContext parentContext, final MongoNamespace namespace) {
+        return tracer.nextSpan(observationType, name, parentContext, namespace);
     }
 
     /**
@@ -146,8 +131,8 @@ public Span addSpan(final String name, @Nullable final TraceContext parentContex
      * @return The created transaction span.
      */
     public Span addTransactionSpan() {
-        Span span = tracer.nextSpan("transaction", null, null);
-        span.tagLowCardinality(SYSTEM.withValue("mongodb"));
+        Span span = tracer.nextSpan(MONGODB_OPERATION, "transaction", null, null);
+        span.tagLowCardinality(OperationLowCardinalityKeyNames.SYSTEM.withValue("mongodb"));
         return span;
     }
 
@@ -205,13 +190,13 @@ public Span createTracingSpan(final CommandMessage message,
         }
 
         Span operationSpan = operationContext.getTracingSpan();
-        Span span = addSpan(commandName,  operationSpan != null ? operationSpan.context() : null);
+        Span span = addSpan(MONGODB_COMMAND, commandName, operationSpan != null ? operationSpan.context() : null);
 
         if (command.containsKey("getMore")) {
             long cursorId = command.getInt64("getMore").longValue();
-            span.tagLowCardinality(CURSOR_ID.withValue(String.valueOf(cursorId)));
+            span.tagLowCardinality(CommandLowCardinalityKeyNames.CURSOR_ID.withValue(String.valueOf(cursorId)));
             if (operationSpan != null) {
-                operationSpan.tagLowCardinality(CURSOR_ID.withValue(String.valueOf(cursorId)));
+                operationSpan.tagLowCardinality(CommandLowCardinalityKeyNames.CURSOR_ID.withValue(String.valueOf(cursorId)));
             }
         }
 
@@ -234,33 +219,33 @@ public Span createTracingSpan(final CommandMessage message,
         String summary = commandName + " " + namespace + (collection.isEmpty() ? "" : "." + collection);
 
         KeyValues keyValues = KeyValues.of(
-                SYSTEM.withValue("mongodb"),
-                NAMESPACE.withValue(namespace),
-                QUERY_SUMMARY.withValue(summary),
-                COMMAND_NAME.withValue(commandName));
+                CommandLowCardinalityKeyNames.SYSTEM.withValue("mongodb"),
+                CommandLowCardinalityKeyNames.NAMESPACE.withValue(namespace),
+                CommandLowCardinalityKeyNames.QUERY_SUMMARY.withValue(summary),
+                CommandLowCardinalityKeyNames.COMMAND_NAME.withValue(commandName));
 
         if (!collection.isEmpty()) {
-            keyValues = keyValues.and(COLLECTION.withValue(collection));
+            keyValues = keyValues.and(CommandLowCardinalityKeyNames.COLLECTION.withValue(collection));
         }
         span.tagLowCardinality(keyValues);
 
         // tag server and connection info
         ServerAddress serverAddress = serverAddressSupplier.get();
         ConnectionId connectionId = connectionIdSupplier.get();
         span.tagLowCardinality(KeyValues.of(
-                SERVER_ADDRESS.withValue(serverAddress.getHost()),
-                SERVER_PORT.withValue(String.valueOf(serverAddress.getPort())),
-                CLIENT_CONNECTION_ID.withValue(String.valueOf(connectionId.getLocalValue())),
-                SERVER_CONNECTION_ID.withValue(String.valueOf(connectionId.getServerValue())),
-                NETWORK_TRANSPORT.withValue(serverAddress instanceof UnixServerAddress ? "unix" : "tcp")
+                CommandLowCardinalityKeyNames.SERVER_ADDRESS.withValue(serverAddress.getHost()),
+                CommandLowCardinalityKeyNames.SERVER_PORT.withValue(String.valueOf(serverAddress.getPort())),
+                CommandLowCardinalityKeyNames.CLIENT_CONNECTION_ID.withValue(String.valueOf(connectionId.getLocalValue())),
+                CommandLowCardinalityKeyNames.SERVER_CONNECTION_ID.withValue(String.valueOf(connectionId.getServerValue())),
+                CommandLowCardinalityKeyNames.NETWORK_TRANSPORT.withValue(serverAddress instanceof UnixServerAddress ? "unix" : "tcp")
         ));
 
         // tag session and transaction info
         SessionContext sessionContext = operationContext.getSessionContext();
         if (sessionContext.hasSession() && !sessionContext.isImplicitSession()) {
             span.tagLowCardinality(KeyValues.of(
-                    TRANSACTION_NUMBER.withValue(String.valueOf(sessionContext.getTransactionNumber())),
-                    SESSION_ID.withValue(String.valueOf(sessionContext.getSessionId()
+                    CommandLowCardinalityKeyNames.TRANSACTION_NUMBER.withValue(String.valueOf(sessionContext.getTransactionNumber())),
+                    CommandLowCardinalityKeyNames.SESSION_ID.withValue(String.valueOf(sessionContext.getSessionId()
                             .get(sessionContext.getSessionId().getFirstKey())
                             .asBinary().asUuid()))
             ));
@@ -298,15 +283,15 @@ public Span createOperationSpan(@Nullable final TransactionSpan transactionSpan,
                 : "." + namespace.getCollectionName());
 
         KeyValues keyValues = KeyValues.of(
-                SYSTEM.withValue("mongodb"),
-                NAMESPACE.withValue(namespace.getDatabaseName()));
+                OperationLowCardinalityKeyNames.SYSTEM.withValue("mongodb"),
+                OperationLowCardinalityKeyNames.NAMESPACE.withValue(namespace.getDatabaseName()));
         if (!MongoNamespaceHelper.COMMAND_COLLECTION_NAME.equalsIgnoreCase(namespace.getCollectionName())) {
-            keyValues = keyValues.and(COLLECTION.withValue(namespace.getCollectionName()));
+            keyValues = keyValues.and(OperationLowCardinalityKeyNames.COLLECTION.withValue(namespace.getCollectionName()));
         }
-        keyValues = keyValues.and(OPERATION_NAME.withValue(commandName),
-                OPERATION_SUMMARY.withValue(name));
+        keyValues = keyValues.and(OperationLowCardinalityKeyNames.OPERATION_NAME.withValue(commandName),
+                OperationLowCardinalityKeyNames.OPERATION_SUMMARY.withValue(name));
 
-        Span span = addSpan(name, parentContext, namespace);
+        Span span = addSpan(MONGODB_OPERATION, name, parentContext, namespace);
         span.tagLowCardinality(keyValues);
         operationContext.setTracingSpan(span);
         return span;
diff --git a/driver-sync/src/test/functional/com/mongodb/client/observability/SpanTree.java b/driver-sync/src/test/functional/com/mongodb/client/observability/SpanTree.java
