From 2e2e08d01a38e300b7e09dd8e2946fa6e3a3d8e9 Mon Sep 17 00:00:00 2001 From: kinsaurralde Date: Wed, 4 Mar 2026 21:59:11 +0000 Subject: [PATCH 1/2] Switch EEF metrics to using built in OpenTelemetry --- .../cloud/spanner/BuiltInMetricsConstant.java | 54 +++++++++++++++++++ .../SpannerCloudMonitoringExporterUtils.java | 4 +- .../google/cloud/spanner/SpannerOptions.java | 7 +++ .../cloud/spanner/spi/v1/GapicSpannerRpc.java | 14 ++++- .../spanner/spi/v1/GapicSpannerRpcTest.java | 11 ++++ 5 files changed, 88 insertions(+), 2 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 56fd6d28eb8..dff490832c8 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -41,6 +41,7 @@ public class BuiltInMetricsConstant { public static final String METER_NAME = "spanner.googleapis.com/internal/client"; public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; + public static final String GRPC_GCP_METER_NAME = "grpc-gcp"; static final String SPANNER_METER_NAME = "spanner-java"; static final String GRPC_METER_NAME = "grpc-java"; static final String GFE_LATENCIES_NAME = "gfe_latencies"; @@ -53,6 +54,8 @@ public class BuiltInMetricsConstant { static final String ATTEMPT_LATENCY_NAME = "attempt_latency"; static final String OPERATION_COUNT_NAME = "operation_count"; static final String ATTEMPT_COUNT_NAME = "attempt_count"; + static final String EEF_FALLBACK_COUNT_NAME = "eef.fallback_count"; + static final String EEF_CALL_STATUS_NAME = "eef.call_status"; public static final Set SPANNER_METRICS = ImmutableSet.of( @@ -117,6 +120,29 @@ public class BuiltInMetricsConstant { "grpc.xds_client.resource_updates_invalid", "grpc.xds_client.resource_updates_valid"); + public static final AttributeKey CHANNEL_NAME_KEY = + AttributeKey.stringKey("channel_name"); + public static final AttributeKey FROM_CHANNEL_NAME_KEY = + AttributeKey.stringKey("from_channel_name"); + public static final AttributeKey TO_CHANNEL_NAME_KEY = + AttributeKey.stringKey("to_channel_name"); + public static final AttributeKey STATUS_CODE_KEY = AttributeKey.stringKey("status_code"); + + static final Set GRPC_GCP_EEF_FALLBACK_COUNT_ATTRIBUTES = + ImmutableSet.of(FROM_CHANNEL_NAME_KEY.getKey(), TO_CHANNEL_NAME_KEY.getKey()); + + static final Set GRPC_GCP_EEF_CALL_STATUS_ATTRIBUTES = + ImmutableSet.of(CHANNEL_NAME_KEY.getKey(), STATUS_CODE_KEY.getKey()); + + static final Map> GRPC_GCP_METRIC_ADDITIONAL_ATTRIBUTES = + ImmutableMap.>builder() + .put(EEF_FALLBACK_COUNT_NAME, GRPC_GCP_EEF_FALLBACK_COUNT_ATTRIBUTES) + .put(EEF_CALL_STATUS_NAME, GRPC_GCP_EEF_CALL_STATUS_ATTRIBUTES) + .build(); + + static final Collection GRPC_GCP_METRICS_TO_ENABLE = + ImmutableList.of(EEF_FALLBACK_COUNT_NAME, EEF_CALL_STATUS_NAME); + public static final String SPANNER_RESOURCE_TYPE = "spanner_instance_client"; public static final AttributeKey PROJECT_ID_KEY = AttributeKey.stringKey("project_id"); @@ -215,6 +241,7 @@ static Map getAllViews() { "1"); defineSpannerView(views); defineGRPCView(views); + defineGrpcGcpView(views); return views.build(); } @@ -281,4 +308,31 @@ private static void defineGRPCView(ImmutableMap.Builder viewMap) { + for (String metric : GRPC_GCP_METRICS_TO_ENABLE) { + InstrumentSelector selector = + InstrumentSelector.builder() + .setName(metric) + .setMeterName(BuiltInMetricsConstant.GRPC_GCP_METER_NAME) + .build(); + + Set attributesFilter = + BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet()); + + attributesFilter.addAll( + GRPC_GCP_METRIC_ADDITIONAL_ATTRIBUTES.getOrDefault(metric, ImmutableSet.of())); + + View view = + View.builder() + .setName(BuiltInMetricsConstant.METER_NAME + '/' + metric.replace(".", "/")) + .setAggregation(Aggregation.sum()) + .setAttributeFilter(attributesFilter) + .build(); + + viewMap.put(selector, view); + } + } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java index e6afa86749f..0f6d8006866 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java @@ -24,6 +24,7 @@ import static com.google.api.MetricDescriptor.ValueType.INT64; import static com.google.cloud.spanner.BuiltInMetricsConstant.ALLOWED_EXEMPLARS_ATTRIBUTES; import static com.google.cloud.spanner.BuiltInMetricsConstant.GAX_METER_NAME; +import static com.google.cloud.spanner.BuiltInMetricsConstant.GRPC_GCP_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.GRPC_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METER_NAME; @@ -86,7 +87,8 @@ static List convertToSpannerTimeSeries( // Get metrics data from GAX library, GRPC library and Spanner library if (!(metricData.getInstrumentationScopeInfo().getName().equals(GAX_METER_NAME) || metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME) - || metricData.getInstrumentationScopeInfo().getName().equals(GRPC_METER_NAME))) { + || metricData.getInstrumentationScopeInfo().getName().equals(GRPC_METER_NAME) + || metricData.getInstrumentationScopeInfo().getName().equals(GRPC_GCP_METER_NAME))) { // Filter out metric data for instruments that are not part of the spanner metrics list continue; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 9eea86ab599..c84bca407be 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -2380,6 +2380,13 @@ public ApiTracerFactory getApiTracerFactory() { return createApiTracerFactory(false, false); } + /** Returns the internal OpenTelemetry instance used for built-in metrics. */ + @InternalApi + public OpenTelemetry getBuiltInOpenTelemetry() { + return this.builtInMetricsProvider.getOrCreateOpenTelemetry( + this.getProjectId(), getCredentials(), this.monitoringHost, getUniverseDomain()); + } + public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) { if (SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { this.builtInMetricsProvider.enableGrpcMetrics( diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index ee90b72ff4b..03ead86bb12 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -232,6 +232,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; +import io.opentelemetry.api.OpenTelemetry; /** Implementation of Cloud Spanner remote calls using Gapic libraries. */ @InternalApi @@ -567,6 +568,17 @@ GcpFallbackChannelOptions createFallbackChannelOptions( .build(); } + @VisibleForTesting + OpenTelemetry getFallbackOpenTelemetry(SpannerOptions options) { + if (options.isEnableBuiltInMetrics()) { + OpenTelemetry builtInOtel = options.getBuiltInOpenTelemetry(); + if (builtInOtel != null) { + return builtInOtel; + } + } + return OpenTelemetry.noop(); + } + private static KeyAwareChannel extractKeyAwareChannel(TransportChannel transportChannel) { if (transportChannel instanceof GrpcTransportChannel) { Channel channel = ((GrpcTransportChannel) transportChannel).getChannel(); @@ -671,7 +683,7 @@ public ClientCall interceptCall( GcpFallbackOpenTelemetry fallbackTelemetry = GcpFallbackOpenTelemetry.newBuilder() - .withSdk(options.getOpenTelemetry()) + .withSdk(getFallbackOpenTelemetry(options)) .disableAllMetrics() .enableMetrics(Arrays.asList("fallback_count", "call_status")) .build(); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java index dada645a8c7..99602b92903 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java @@ -116,6 +116,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; +import java.util.Arrays; @RunWith(Parameterized.class) public class GapicSpannerRpcTest { @@ -1138,6 +1139,11 @@ public TestableGapicSpannerRpc(SpannerOptions options) { super(options); } + @Override + OpenTelemetry getFallbackOpenTelemetry(SpannerOptions options) { + return options.getOpenTelemetry(); + } + @Override GcpFallbackChannelOptions createFallbackChannelOptions( GcpFallbackOpenTelemetry fallbackTelemetry, int minFailedCalls) { @@ -1217,6 +1223,11 @@ public TestableGapicSpannerRpcWithLowerMinFailedCalls(SpannerOptions options) { super(options); } + @Override + OpenTelemetry getFallbackOpenTelemetry(SpannerOptions options) { + return options.getOpenTelemetry(); + } + @Override GcpFallbackChannelOptions createFallbackChannelOptions( GcpFallbackOpenTelemetry fallbackTelemetry, int minFailedCalls) { From e2b8d7da4c1e74cd4b561818c6ae1d3ba8931012 Mon Sep 17 00:00:00 2001 From: kinsaurralde Date: Wed, 4 Mar 2026 22:35:46 +0000 Subject: [PATCH 2/2] formatting --- .../java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java | 4 ++-- .../com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index 03ead86bb12..2faa3a62fb9 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -201,6 +201,7 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.MethodDescriptor; import io.grpc.auth.MoreCallCredentials; +import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; @@ -232,7 +233,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; -import io.opentelemetry.api.OpenTelemetry; /** Implementation of Cloud Spanner remote calls using Gapic libraries. */ @InternalApi @@ -576,7 +576,7 @@ OpenTelemetry getFallbackOpenTelemetry(SpannerOptions options) { return builtInOtel; } } - return OpenTelemetry.noop(); + return OpenTelemetry.noop(); } private static KeyAwareChannel extractKeyAwareChannel(TransportChannel transportChannel) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java index 99602b92903..98be2b8d156 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java @@ -116,7 +116,6 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import java.util.Arrays; @RunWith(Parameterized.class) public class GapicSpannerRpcTest {