From 440c2ce68373852f4c90505ab5c5095a49fe90b9 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 12:48:03 -0700 Subject: [PATCH 01/13] feat(gax): inject active OpenTelemetry span context into gRPC and HTTP metadata --- core/packages/gax/src/normalCalls/timeout.ts | 18 +- core/packages/gax/src/util.ts | 13 +- .../gax/test/unit/metadataPropagation.ts | 675 ++++++++++++++++++ core/packages/gax/test/unit/util.ts | 21 + 4 files changed, 724 insertions(+), 3 deletions(-) create mode 100644 core/packages/gax/test/unit/metadataPropagation.ts diff --git a/core/packages/gax/src/normalCalls/timeout.ts b/core/packages/gax/src/normalCalls/timeout.ts index 440a89cb547c..62ec12cd5ec2 100644 --- a/core/packages/gax/src/normalCalls/timeout.ts +++ b/core/packages/gax/src/normalCalls/timeout.ts @@ -14,12 +14,14 @@ * limitations under the License. */ -import { +import {context, propagation} from '@opentelemetry/api'; +import type { GRPCCall, GRPCCallOtherArgs, SimpleCallbackFunction, UnaryCall, } from '../apitypes'; +import {checkTelemetryEnabled, isMetadata} from '../util'; /** * Updates func so that it gets called with the timeout as its final arg. @@ -48,9 +50,21 @@ export function addTimeoutArg( const now = new Date(); const options = otherArgs.options || {}; options.deadline = new Date(now.getTime() + timeout); - const metadata = otherArgs.metadataBuilder + let metadata = otherArgs.metadataBuilder ? otherArgs.metadataBuilder(abTests, otherArgs.headers || {}) : null; + if (checkTelemetryEnabled() && isMetadata(metadata)) { + const targetMetadata = + typeof metadata.clone === 'function' ? metadata.clone() : metadata; + propagation.inject(context.active(), targetMetadata, { + set(carrier, key, value) { + carrier.set(key, value); + }, + }); + metadata = targetMetadata; + } else if (checkTelemetryEnabled() && metadata && typeof metadata === 'object') { + propagation.inject(context.active(), metadata); + } return (func as UnaryCall)(argument, metadata!, options, callback); }; } diff --git a/core/packages/gax/src/util.ts b/core/packages/gax/src/util.ts index e44fb7c20645..090f0b474697 100644 --- a/core/packages/gax/src/util.ts +++ b/core/packages/gax/src/util.ts @@ -14,7 +14,9 @@ * limitations under the License. */ -import {CallSettings} from './gax'; +import type {CallSettings} from './gax'; +import type {Metadata} from './grpc'; + const PROTO_TYPE_PREFIX = 'type.googleapis.com/'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; @@ -64,6 +66,15 @@ export function checkTelemetryEnabled(settings?: CallSettings): boolean { ); } +export function isMetadata(value: unknown): value is Metadata { + return ( + value !== null && + typeof value === 'object' && + 'set' in value && + typeof value.set === 'function' + ); +} + function words(str: string, normalize = false) { if (normalize) { // strings like somethingABCSomething are special case for protobuf.js, diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts new file mode 100644 index 000000000000..d230bb6caca0 --- /dev/null +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -0,0 +1,675 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import {describe, it, beforeEach, afterEach} from 'mocha'; +import { + context, + propagation, + trace, + ROOT_CONTEXT, + TextMapPropagator, +} from '@opentelemetry/api'; +import {status} from '@grpc/grpc-js'; +import {addTimeoutArg} from '../../src/normalCalls/timeout'; +import {createApiCall} from '../../src/createApiCall'; +import * as gax from '../../src/gax'; +import {GoogleError} from '../../src/googleError'; +import {GrpcClient, Metadata} from '../../src/grpc'; +import {StaticTraceContext} from '../../src/observability/TracerHelper'; +import {OtelHarness} from './otelHarness'; +import {GRPCCall, GRPCCallOtherArgs, GRPCCallResult} from '../../src/apitypes'; + +describe('metadata propagation (gRPC and HTTP)', () => { + let harness: OtelHarness; + + // Custom propagator to simulate W3C traceparent injection and extraction + const testPropagator: TextMapPropagator = { + inject(ctx, carrier, setter) { + const spanContext = trace.getSpanContext(ctx); + if (spanContext && trace.isSpanContextValid(spanContext)) { + setter.set( + carrier, + 'traceparent', + `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, + ); + } + }, + extract(ctx, carrier, getter) { + const raw = getter.get(carrier, 'traceparent'); + const value = Array.isArray(raw) ? raw[0] : raw; + if (typeof value === 'string') { + const parts = value.split('-'); + if (parts.length === 4) { + return trace.setSpanContext(ctx, { + traceId: parts[1], + spanId: parts[2], + traceFlags: parseInt(parts[3], 16), + isRemote: true, + }); + } + } + return ctx; + }, + fields() { + return ['traceparent']; + }, + }; + + beforeEach(() => { + process.env.GOOGLE_SDK_NODE_ENABLE_TRACING = 'true'; + harness = new OtelHarness(); + harness.setup(); + propagation.setGlobalPropagator(testPropagator); + }); + + afterEach(() => { + delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; + propagation.disable(); + harness.teardown(); + }); + + it('injects active span context into gRPC metadata', done => { + const grpcClient = new GrpcClient(); + const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); + + const tracer = trace.getTracer('test-tracer'); + const span = tracer.startSpan('test-rpc'); + + let receivedMetadata: Metadata | null = null; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + receivedMetadata = metadata as Metadata; + callback(null, {success: true}); + return {} as GRPCCallResult; + }; + + const otherArgs: GRPCCallOtherArgs = { + metadataBuilder: builder, + }; + + const callHandler = addTimeoutArg( + stubFunc as unknown as GRPCCall, + 1000, + otherArgs, + ); + + context.with(trace.setSpan(context.active(), span), () => { + callHandler({}, (err: unknown) => { + span.end(); + assert.ifError(err); + assert.ok(receivedMetadata); + const traceparent = (receivedMetadata as Metadata).get( + 'traceparent', + ) as unknown[]; + assert.ok(traceparent && traceparent.length > 0); + const spanContext = span.spanContext(); + assert.strictEqual( + traceparent[0], + `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, + ); + done(); + }); + }); + }); + + it('does not mutate baseMetadata across multiple gRPC calls', done => { + const grpcClient = new GrpcClient(); + const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); + + // Capture the baseMetadata returned initially + const baseMetadata = builder() as unknown as Metadata; + assert.strictEqual( + (baseMetadata.get('traceparent') as unknown[]).length, + 0, + ); + + const tracer = trace.getTracer('test-tracer'); + const span1 = tracer.startSpan('rpc-1'); + const span2 = tracer.startSpan('rpc-2'); + + let metadataCall1: Metadata | null = null; + let metadataCall2: Metadata | null = null; + + const stubFunc1 = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + metadataCall1 = meta as Metadata; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const stubFunc2 = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + metadataCall2 = meta as Metadata; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const otherArgs: GRPCCallOtherArgs = { + metadataBuilder: builder, + }; + + const handler1 = addTimeoutArg( + stubFunc1 as unknown as GRPCCall, + 1000, + otherArgs, + ); + const handler2 = addTimeoutArg( + stubFunc2 as unknown as GRPCCall, + 1000, + otherArgs, + ); + + context.with(trace.setSpan(context.active(), span1), () => { + handler1({}, () => { + span1.end(); + + context.with(trace.setSpan(context.active(), span2), () => { + handler2({}, () => { + span2.end(); + + // Check metadata from Call 1 + const spanCtx1 = span1.spanContext(); + const traceparent1 = metadataCall1!.get( + 'traceparent', + ) as unknown[]; + assert.strictEqual( + traceparent1[0], + `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, + ); + + // Check metadata from Call 2 + const spanCtx2 = span2.spanContext(); + const traceparent2 = metadataCall2!.get( + 'traceparent', + ) as unknown[]; + assert.strictEqual( + traceparent2[0], + `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, + ); + + // Crucial: baseMetadata must NOT contain traceparent! + assert.strictEqual( + (baseMetadata.get('traceparent') as unknown[]).length, + 0, + ); + done(); + }); + }); + }); + }); + }); + + it('injects active span context into HTTP/REST plain object metadata', done => { + const tracer = trace.getTracer('test-tracer'); + const span = tracer.startSpan('http-rpc'); + + let receivedMetadata: Record | null = null; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + receivedMetadata = metadata as Record; + callback(null, {}); + return {} as GRPCCallResult; + }; + + const otherArgs: GRPCCallOtherArgs = { + metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), + }; + + const handler = addTimeoutArg( + stubFunc as unknown as GRPCCall, + 1000, + otherArgs, + ); + + context.with(trace.setSpan(context.active(), span), () => { + handler({}, (err: unknown) => { + span.end(); + assert.ifError(err); + assert.ok(receivedMetadata); + const spanContext = span.spanContext(); + assert.strictEqual( + receivedMetadata!['traceparent'], + `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, + ); + assert.deepStrictEqual(receivedMetadata!['x-goog-api-client'], [ + 'grpc-web/1.0', + ]); + done(); + }); + }); + }); + + it('does not inject into gRPC or HTTP metadata when telemetry is disabled', done => { + delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; + + const grpcClient = new GrpcClient(); + const grpcBuilder = grpcClient.metadataBuilder({ + 'x-goog-api-client': 'test', + }); + + const tracer = trace.getTracer('test-tracer'); + const span = tracer.startSpan('disabled-rpc'); + + let receivedGrpcMetadata: Metadata | null = null; + let receivedHttpMetadata: Record | null = null; + + const grpcStub = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + receivedGrpcMetadata = meta as Metadata; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const httpStub = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + receivedHttpMetadata = meta as Record; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const grpcHandler = addTimeoutArg(grpcStub as unknown as GRPCCall, 1000, { + metadataBuilder: grpcBuilder, + }); + const httpHandler = addTimeoutArg(httpStub as unknown as GRPCCall, 1000, { + metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), + }); + + context.with(trace.setSpan(context.active(), span), () => { + grpcHandler({}, (err1: unknown) => { + assert.ifError(err1); + httpHandler({}, (err2: unknown) => { + span.end(); + assert.ifError(err2); + assert.ok(receivedGrpcMetadata); + assert.strictEqual( + (receivedGrpcMetadata!.get('traceparent') as unknown[]).length, + 0, + ); + assert.ok(receivedHttpMetadata); + assert.strictEqual(receivedHttpMetadata!['traceparent'], undefined); + done(); + }); + }); + }); + }); + + it('handles null or missing metadata gracefully', done => { + let called = false; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + called = true; + assert.strictEqual(metadata, null); + callback(null, {}); + return {} as GRPCCallResult; + }; + + const otherArgs: GRPCCallOtherArgs = { + metadataBuilder: (() => null) as unknown as GRPCCallOtherArgs['metadataBuilder'], + }; + + const handler = addTimeoutArg( + stubFunc as unknown as GRPCCall, + 1000, + otherArgs, + ); + handler({}, () => { + assert.ok(called); + done(); + }); + }); + + describe('T3 client request trace to low-level unary trace correlation', () => { + const telemetryInfo: StaticTraceContext = { + gcpClientService: 'echo.googleapis.com', + gcpVersion: '1.2.3', + gcpRepo: 'googleapis/google-cloud-node', + gcpArtifact: '@google-cloud/echo', + }; + + const grpcMetadataGetter = { + get(carrier: Metadata, key: string) { + return carrier.get(key) as string[]; + }, + keys() { + return ['traceparent']; + }, + }; + + it('ties a low-level gRPC unary trace to its parent T3 client request trace', async () => { + const grpcClient = new GrpcClient(); + const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); + + const settings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: builder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + let extractedTraceparent: string | undefined; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + const grpcMeta = metadata as Metadata; + const tp = grpcMeta.get('traceparent') as string[]; + extractedTraceparent = tp?.[0]; + + // Simulate low-level unary gRPC span started using the propagated context + const parentCtx = propagation.extract( + ROOT_CONTEXT, + grpcMeta, + grpcMetadataGetter, + ); + const unaryTracer = trace.getTracer('grpc-unary-transport'); + const unarySpan = unaryTracer.startSpan( + 'grpc.google.example.v1.Echo/Echo', + undefined, + parentCtx, + ); + unarySpan.end(); + + callback(null, {echo: 'ok'}); + return {cancel: () => {}}; + }; + + const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); + await apiCall({message: 'hello'}, undefined); + + const t3Span = harness.requireSingleSpan('google-gax'); + const unarySpan = harness.requireSingleSpan('grpc-unary-transport'); + + assert.strictEqual(t3Span.name, 'EchoClient.Echo'); + assert.strictEqual( + extractedTraceparent, + `00-${t3Span.spanContext().traceId}-${t3Span.spanContext().spanId}-0${t3Span.spanContext().traceFlags}`, + ); + assert.strictEqual( + unarySpan.spanContext().traceId, + t3Span.spanContext().traceId, + ); + assert.strictEqual( + unarySpan.parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + }); + + it('ties a low-level HTTP/REST unary trace to its parent T3 client request trace', async () => { + const settings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + let extractedTraceparent: string | undefined; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + const httpMeta = metadata as Record; + extractedTraceparent = httpMeta['traceparent']; + + // Simulate low-level unary HTTP span started using the propagated headers + const parentCtx = propagation.extract(ROOT_CONTEXT, httpMeta); + const unaryTracer = trace.getTracer('http-unary-transport'); + const unarySpan = unaryTracer.startSpan( + 'HTTP POST /v1/echo', + undefined, + parentCtx, + ); + unarySpan.end(); + + callback(null, {echo: 'ok'}); + return {cancel: () => {}}; + }; + + const apiCall = createApiCall( + stubFunc as unknown as GRPCCall, + settings, + undefined, + 'rest', + ); + await apiCall({message: 'hello'}, undefined); + + const t3Span = harness.requireSingleSpan('google-gax'); + const unarySpan = harness.requireSingleSpan('http-unary-transport'); + + assert.strictEqual(t3Span.name, 'EchoClient.Echo'); + assert.strictEqual( + extractedTraceparent, + `00-${t3Span.spanContext().traceId}-${t3Span.spanContext().spanId}-0${t3Span.spanContext().traceFlags}`, + ); + assert.strictEqual( + unarySpan.spanContext().traceId, + t3Span.spanContext().traceId, + ); + assert.strictEqual( + unarySpan.parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + }); + + it('ties concurrent low-level unary traces to their respective T3 client request traces without cross-talk', async () => { + const grpcClient = new GrpcClient(); + const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); + + const echoSettings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: builder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + const expandSettings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: builder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Expand', + }, + }); + + const makeStub = (unarySpanName: string, delayMs: number): GRPCCall => { + return (( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + const grpcMeta = metadata as Metadata; + const parentCtx = propagation.extract( + ROOT_CONTEXT, + grpcMeta, + grpcMetadataGetter, + ); + const unaryTracer = trace.getTracer('grpc-unary-transport'); + const unarySpan = unaryTracer.startSpan( + unarySpanName, + undefined, + parentCtx, + ); + setTimeout(() => { + unarySpan.end(); + callback(null, {ok: true}); + }, delayMs); + return {cancel: () => {}}; + }) as unknown as GRPCCall; + }; + + const echoCall = createApiCall(makeStub('unary.Echo', 15), echoSettings); + const expandCall = createApiCall( + makeStub('unary.Expand', 5), + expandSettings, + ); + + await Promise.all([ + echoCall({id: 1}, undefined), + expandCall({id: 2}, undefined), + ]); + + const t3Spans = harness.getSpans('google-gax'); + const unarySpans = harness.getSpans('grpc-unary-transport'); + assert.strictEqual(t3Spans.length, 2); + assert.strictEqual(unarySpans.length, 2); + + const t3Echo = t3Spans.find(s => s.name === 'EchoClient.Echo')!; + const t3Expand = t3Spans.find(s => s.name === 'EchoClient.Expand')!; + const unaryEcho = unarySpans.find(s => s.name === 'unary.Echo')!; + const unaryExpand = unarySpans.find(s => s.name === 'unary.Expand')!; + + assert.ok(t3Echo && t3Expand && unaryEcho && unaryExpand); + assert.notStrictEqual( + t3Echo.spanContext().spanId, + t3Expand.spanContext().spanId, + ); + + // unary.Echo must be tied to EchoClient.Echo + assert.strictEqual( + unaryEcho.spanContext().traceId, + t3Echo.spanContext().traceId, + ); + assert.strictEqual( + unaryEcho.parentSpanContext?.spanId, + t3Echo.spanContext().spanId, + ); + + // unary.Expand must be tied to EchoClient.Expand + assert.strictEqual( + unaryExpand.spanContext().traceId, + t3Expand.spanContext().traceId, + ); + assert.strictEqual( + unaryExpand.parentSpanContext?.spanId, + t3Expand.spanContext().spanId, + ); + }); + + it('ties all retried low-level unary attempt traces to the single parent T3 client request trace', async () => { + const grpcClient = new GrpcClient(); + const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); + + const retryOptions = gax.createRetryOptions( + [status.UNAVAILABLE], + gax.createBackoffSettings(1, 1.1, 5, 100, 1.0, 100, 1000), + ); + + const settings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + retry: retryOptions, + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: builder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + let attempt = 0; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + attempt++; + const grpcMeta = metadata as Metadata; + const parentCtx = propagation.extract( + ROOT_CONTEXT, + grpcMeta, + grpcMetadataGetter, + ); + const unaryTracer = trace.getTracer('grpc-unary-transport'); + const unarySpan = unaryTracer.startSpan( + `unary.Echo.attempt.${attempt}`, + undefined, + parentCtx, + ); + unarySpan.end(); + + if (attempt === 1) { + const err = new GoogleError('transient failure'); + err.code = status.UNAVAILABLE; + callback(err); + } else { + callback(null, {echo: 'recovered'}); + } + return {cancel: () => {}}; + }; + + const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); + await apiCall({message: 'retry-me'}, undefined); + + const t3Span = harness.requireSingleSpan('google-gax'); + const unarySpans = harness.getSpans('grpc-unary-transport'); + assert.strictEqual(unarySpans.length, 2); + + for (const unarySpan of unarySpans) { + assert.strictEqual( + unarySpan.spanContext().traceId, + t3Span.spanContext().traceId, + ); + assert.strictEqual( + unarySpan.parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + } + }); + }); +}); diff --git a/core/packages/gax/test/unit/util.ts b/core/packages/gax/test/unit/util.ts index 48be4e9f6a54..47833ed70072 100644 --- a/core/packages/gax/test/unit/util.ts +++ b/core/packages/gax/test/unit/util.ts @@ -25,6 +25,7 @@ import { decodeProtobufAny, decodeAnyProtosInArray, checkTelemetryEnabled, + isMetadata, connectionCodes, requestCodes, requestBodyCodes, @@ -35,6 +36,7 @@ import { ignoredClientHeaderTokens, DEPTH_TO_CHECK, } from '../../src/util'; +import {GrpcClient} from '../../src/grpc'; import {StaticTraceContext} from '../../src/observability/TracerHelper'; import {CallSettings} from '../../src/gax'; import * as protobuf from 'protobufjs'; @@ -367,6 +369,25 @@ describe('util.ts', () => { }); }); + describe('isMetadata', () => { + it('returns true for gRPC Metadata instances', () => { + const grpcClient = new GrpcClient(); + const metadata = grpcClient.metadataBuilder({'x-goog-api-client': 'test'})(); + assert.strictEqual(isMetadata(metadata), true); + }); + + it('returns false for plain HTTP/REST header objects, null, and primitives', () => { + assert.strictEqual( + isMetadata({'x-goog-api-client': ['grpc-web/1.0']}), + false, + ); + assert.strictEqual(isMetadata({set: 'not-a-function'}), false); + assert.strictEqual(isMetadata(null), false); + assert.strictEqual(isMetadata(undefined), false); + assert.strictEqual(isMetadata('string'), false); + }); + }); + describe('error code and class constants', () => { it('defines connectionCodes containing common network errors', () => { assert(Array.isArray(connectionCodes)); From 998dc2c9bd44db2e1f4dfe45eb5da3f69271fed3 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 13:07:57 -0700 Subject: [PATCH 02/13] docs(gax): remove instrumentation library setup from observability sample --- core/packages/gax/samples/observability.js | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/core/packages/gax/samples/observability.js b/core/packages/gax/samples/observability.js index 6d31636144c1..ab2318a803ac 100644 --- a/core/packages/gax/samples/observability.js +++ b/core/packages/gax/samples/observability.js @@ -17,17 +17,12 @@ // [START gax_observability] 'use strict'; -// 1. INITIALIZE OPENTELEMETRY BEFORE IMPORTING ANY CLIENT LIBRARIES -// In Node.js, instrumentations must patch the networking modules (http, grpc) -// before any Google Cloud client libraries are loaded into the module cache. +// 1. IMPORT OPENTELEMETRY MODULES const {NodeTracerProvider} = require('@opentelemetry/sdk-trace-node'); const {BatchSpanProcessor} = require('@opentelemetry/sdk-trace-base'); const { TraceExporter, } = require('@google-cloud/opentelemetry-cloud-trace-exporter'); -const {registerInstrumentations} = require('@opentelemetry/instrumentation'); -const {HttpInstrumentation} = require('@opentelemetry/instrumentation-http'); -const {GrpcInstrumentation} = require('@opentelemetry/instrumentation-grpc'); // 2. CONFIGURE TRACING: SET UP A TRACER PROVIDER AND EXPORTER const cloudTraceExporter = new TraceExporter(); @@ -38,19 +33,11 @@ const provider = new NodeTracerProvider({ }); provider.register(); -// 3. ENABLE LOW-LEVEL NETWORK TRACING SPANS USING INSTRUMENTATION LIBRARIES -registerInstrumentations({ - instrumentations: [ - new HttpInstrumentation(), - new GrpcInstrumentation(), - ], -}); - -// 4. ENABLE CLIENT REQUEST TRACING SPANS WITH ENV VARIABLE +// 3. ENABLE TRACING SPANS WITH ENV VARIABLE // Sets the flag before client libraries or RPC callers initialize process.env.GOOGLE_SDK_NODE_ENABLE_TRACING = 'true'; -// 5. IMPORT CLIENT LIBRARIES AFTER OPENTELEMETRY SETUP +// 4. IMPORT CLIENT LIBRARIES AFTER OPENTELEMETRY SETUP // Replace with your Google Cloud client library, for example: // const { SecretManagerServiceClient } = require('@google-cloud/secret-manager'); @@ -58,7 +45,7 @@ async function main() { // const client = new SecretManagerServiceClient(); // await client.listSecrets({parent: 'projects/my-project'}); - // 6. FLUSH SPANS BEFORE PROCESS EXIT + // 5. FLUSH SPANS BEFORE PROCESS EXIT // Ensures all buffered spans in BatchSpanProcessor are exported to Cloud Trace await provider.forceFlush(); console.log('Tracing initialized successfully.'); From 7cad06fd1ecf106ad16098e72768117cfea08c38 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 13:25:53 -0700 Subject: [PATCH 03/13] style(gax): fix prettier formatting in timeout, util, and unit tests --- core/packages/gax/src/normalCalls/timeout.ts | 6 +++++- core/packages/gax/src/util.ts | 1 - core/packages/gax/test/unit/metadataPropagation.ts | 11 ++++------- core/packages/gax/test/unit/util.ts | 4 +++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/packages/gax/src/normalCalls/timeout.ts b/core/packages/gax/src/normalCalls/timeout.ts index 62ec12cd5ec2..1d7186e7ebef 100644 --- a/core/packages/gax/src/normalCalls/timeout.ts +++ b/core/packages/gax/src/normalCalls/timeout.ts @@ -62,7 +62,11 @@ export function addTimeoutArg( }, }); metadata = targetMetadata; - } else if (checkTelemetryEnabled() && metadata && typeof metadata === 'object') { + } else if ( + checkTelemetryEnabled() && + metadata && + typeof metadata === 'object' + ) { propagation.inject(context.active(), metadata); } return (func as UnaryCall)(argument, metadata!, options, callback); diff --git a/core/packages/gax/src/util.ts b/core/packages/gax/src/util.ts index 090f0b474697..5826eaae78b8 100644 --- a/core/packages/gax/src/util.ts +++ b/core/packages/gax/src/util.ts @@ -17,7 +17,6 @@ import type {CallSettings} from './gax'; import type {Metadata} from './grpc'; - const PROTO_TYPE_PREFIX = 'type.googleapis.com/'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts index d230bb6caca0..43130087137e 100644 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -195,9 +195,7 @@ describe('metadata propagation (gRPC and HTTP)', () => { // Check metadata from Call 1 const spanCtx1 = span1.spanContext(); - const traceparent1 = metadataCall1!.get( - 'traceparent', - ) as unknown[]; + const traceparent1 = metadataCall1!.get('traceparent') as unknown[]; assert.strictEqual( traceparent1[0], `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, @@ -205,9 +203,7 @@ describe('metadata propagation (gRPC and HTTP)', () => { // Check metadata from Call 2 const spanCtx2 = span2.spanContext(); - const traceparent2 = metadataCall2!.get( - 'traceparent', - ) as unknown[]; + const traceparent2 = metadataCall2!.get('traceparent') as unknown[]; assert.strictEqual( traceparent2[0], `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, @@ -346,7 +342,8 @@ describe('metadata propagation (gRPC and HTTP)', () => { }; const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: (() => null) as unknown as GRPCCallOtherArgs['metadataBuilder'], + metadataBuilder: (() => + null) as unknown as GRPCCallOtherArgs['metadataBuilder'], }; const handler = addTimeoutArg( diff --git a/core/packages/gax/test/unit/util.ts b/core/packages/gax/test/unit/util.ts index 47833ed70072..f443c007d691 100644 --- a/core/packages/gax/test/unit/util.ts +++ b/core/packages/gax/test/unit/util.ts @@ -372,7 +372,9 @@ describe('util.ts', () => { describe('isMetadata', () => { it('returns true for gRPC Metadata instances', () => { const grpcClient = new GrpcClient(); - const metadata = grpcClient.metadataBuilder({'x-goog-api-client': 'test'})(); + const metadata = grpcClient.metadataBuilder({ + 'x-goog-api-client': 'test', + })(); assert.strictEqual(isMetadata(metadata), true); }); From 8577f00358aaa8e87d46ec8f867ed380d30eaac0 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 13:32:03 -0700 Subject: [PATCH 04/13] fix(gax): clone plain object metadata before injecting span context --- core/packages/gax/src/normalCalls/timeout.ts | 4 +- .../gax/test/unit/metadataPropagation.ts | 79 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/core/packages/gax/src/normalCalls/timeout.ts b/core/packages/gax/src/normalCalls/timeout.ts index 1d7186e7ebef..8707c2415f30 100644 --- a/core/packages/gax/src/normalCalls/timeout.ts +++ b/core/packages/gax/src/normalCalls/timeout.ts @@ -67,7 +67,9 @@ export function addTimeoutArg( metadata && typeof metadata === 'object' ) { - propagation.inject(context.active(), metadata); + const targetMetadata = {...metadata}; + propagation.inject(context.active(), targetMetadata); + metadata = targetMetadata; } return (func as UnaryCall)(argument, metadata!, options, callback); }; diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts index 43130087137e..1076e1735040 100644 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -265,6 +265,85 @@ describe('metadata propagation (gRPC and HTTP)', () => { }); }); + it('does not mutate shared HTTP/REST plain object metadata across calls', done => { + const sharedMetadata: Record = { + 'x-goog-api-client': ['grpc-web/1.0'], + }; + + const tracer = trace.getTracer('test-tracer'); + const span1 = tracer.startSpan('http-rpc-1'); + const span2 = tracer.startSpan('http-rpc-2'); + + let metadataCall1: Record | null = null; + let metadataCall2: Record | null = null; + + const stub1 = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + metadataCall1 = meta as Record; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const stub2 = ( + arg: {}, + meta: {}, + opt: {}, + cb: Function, + ): GRPCCallResult => { + metadataCall2 = meta as Record; + cb(null, {}); + return {} as GRPCCallResult; + }; + + const otherArgs: GRPCCallOtherArgs = { + metadataBuilder: () => sharedMetadata, + }; + + const handler1 = addTimeoutArg( + stub1 as unknown as GRPCCall, + 1000, + otherArgs, + ); + const handler2 = addTimeoutArg( + stub2 as unknown as GRPCCall, + 1000, + otherArgs, + ); + + context.with(trace.setSpan(context.active(), span1), () => { + handler1({}, () => { + span1.end(); + + context.with(trace.setSpan(context.active(), span2), () => { + handler2({}, () => { + span2.end(); + + const spanCtx1 = span1.spanContext(); + assert.strictEqual( + metadataCall1!['traceparent'], + `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, + ); + + const spanCtx2 = span2.spanContext(); + assert.strictEqual( + metadataCall2!['traceparent'], + `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, + ); + + assert.notStrictEqual(metadataCall1, sharedMetadata); + assert.notStrictEqual(metadataCall2, sharedMetadata); + assert.strictEqual(sharedMetadata['traceparent'], undefined); + done(); + }); + }); + }); + }); + }); + it('does not inject into gRPC or HTTP metadata when telemetry is disabled', done => { delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; From 8e7e676e8526c24691c78636689f550479fa67c1 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 15:08:47 -0700 Subject: [PATCH 05/13] refactor(gax): move OpenTelemetry context propagation into TracerHelper and createApiCall --- core/packages/gax/src/createApiCall.ts | 18 +- core/packages/gax/src/normalCalls/timeout.ts | 24 +- .../gax/src/observability/TracerHelper.ts | 22 +- core/packages/gax/src/util.ts | 12 +- .../gax/test/unit/metadataPropagation.ts | 432 +++++------------- core/packages/gax/test/unit/util.ts | 23 - 6 files changed, 164 insertions(+), 367 deletions(-) diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index ac73d9aa8fbd..e8b01354c866 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -83,6 +83,7 @@ export function createApiCall( callOptions?: CallOptions, callback?: APICallback, recordResend?: ResendRecorder, + injectedHeaders?: Record, ) => { let currentApiCaller = apiCaller; @@ -100,6 +101,16 @@ export function createApiCall( thisSettings = settings.merge(callOptions); } + if (injectedHeaders && Object.keys(injectedHeaders).length > 0) { + thisSettings.otherArgs = { + ...thisSettings.otherArgs, + headers: { + ...thisSettings.otherArgs?.headers, + ...injectedHeaders, + }, + }; + } + // special case: if bundling is disabled for this one call, // use default API caller instead if (settings.isBundling && !thisSettings.isBundling) { @@ -211,7 +222,11 @@ export function createApiCall( return traceCall( dynamicArgs, staticArgs, - (tracedCallback?: APICallback, recordResend?: ResendRecorder) => { + ( + tracedCallback?: APICallback, + recordResend?: ResendRecorder, + injectedHeaders?: Record, + ) => { // `traceCall` wraps the user's callback whenever one was supplied, // for stream and non-stream calls alike, and that wrapper is what // closes the span. It is undefined only when there is no callback to @@ -222,6 +237,7 @@ export function createApiCall( callOptions, tracedCallback ?? callback, recordResend, + injectedHeaders, ); }, isStreamingCall, diff --git a/core/packages/gax/src/normalCalls/timeout.ts b/core/packages/gax/src/normalCalls/timeout.ts index 8707c2415f30..440a89cb547c 100644 --- a/core/packages/gax/src/normalCalls/timeout.ts +++ b/core/packages/gax/src/normalCalls/timeout.ts @@ -14,14 +14,12 @@ * limitations under the License. */ -import {context, propagation} from '@opentelemetry/api'; -import type { +import { GRPCCall, GRPCCallOtherArgs, SimpleCallbackFunction, UnaryCall, } from '../apitypes'; -import {checkTelemetryEnabled, isMetadata} from '../util'; /** * Updates func so that it gets called with the timeout as its final arg. @@ -50,27 +48,9 @@ export function addTimeoutArg( const now = new Date(); const options = otherArgs.options || {}; options.deadline = new Date(now.getTime() + timeout); - let metadata = otherArgs.metadataBuilder + const metadata = otherArgs.metadataBuilder ? otherArgs.metadataBuilder(abTests, otherArgs.headers || {}) : null; - if (checkTelemetryEnabled() && isMetadata(metadata)) { - const targetMetadata = - typeof metadata.clone === 'function' ? metadata.clone() : metadata; - propagation.inject(context.active(), targetMetadata, { - set(carrier, key, value) { - carrier.set(key, value); - }, - }); - metadata = targetMetadata; - } else if ( - checkTelemetryEnabled() && - metadata && - typeof metadata === 'object' - ) { - const targetMetadata = {...metadata}; - propagation.inject(context.active(), targetMetadata); - metadata = targetMetadata; - } return (func as UnaryCall)(argument, metadata!, options, callback); }; } diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index 81b28d044cd5..ae6aa3b60f0a 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -18,6 +18,7 @@ import {EventEmitter} from 'events'; import { Attributes, context, + propagation, Span, SpanStatusCode, trace, @@ -975,6 +976,7 @@ export function traceCall( fn: ( tracedCallback?: APICallback, recordResend?: ResendRecorder, + injectedHeaders?: Record, ) => GaxCallResult, isStreamCall?: boolean, callback?: APICallback, @@ -982,14 +984,22 @@ export function traceCall( export function traceCall( dynamicArgs: DynamicTraceContext, staticArgs: StaticTraceContext, - fn: (tracedCallback?: APICallback, recordResend?: ResendRecorder) => T, + fn: ( + tracedCallback?: APICallback, + recordResend?: ResendRecorder, + injectedHeaders?: Record, + ) => T, isStreamCall: true, callback?: APICallback, ): T; export function traceCall( dynamicArgs: DynamicTraceContext, staticArgs: StaticTraceContext, - fn: (tracedCallback?: APICallback, recordResend?: ResendRecorder) => T, + fn: ( + tracedCallback?: APICallback, + recordResend?: ResendRecorder, + injectedHeaders?: Record, + ) => T, isStreamCall?: false, callback?: APICallback, ): T; @@ -999,6 +1009,7 @@ export function traceCall( fn: ( tracedCallback?: APICallback, recordResend?: ResendRecorder, + injectedHeaders?: Record, ) => GaxCallResult, isStreamCall = false, callback?: APICallback, @@ -1136,8 +1147,11 @@ export function traceCall( : undefined; try { - const result = context.with(trace.setSpan(context.active(), span), () => - fn(tracedCallback, recordResend), + const activeContext = trace.setSpan(context.active(), span); + const injectedHeaders: Record = {}; + propagation.inject(activeContext, injectedHeaders); + const result = context.with(activeContext, () => + fn(tracedCallback, recordResend, injectedHeaders), ); const promiseTarget = !isStreamCall ? getPromiseTarget(result) : null; if (isStreamCall && result instanceof EventEmitter) { diff --git a/core/packages/gax/src/util.ts b/core/packages/gax/src/util.ts index 5826eaae78b8..e44fb7c20645 100644 --- a/core/packages/gax/src/util.ts +++ b/core/packages/gax/src/util.ts @@ -14,8 +14,7 @@ * limitations under the License. */ -import type {CallSettings} from './gax'; -import type {Metadata} from './grpc'; +import {CallSettings} from './gax'; const PROTO_TYPE_PREFIX = 'type.googleapis.com/'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; @@ -65,15 +64,6 @@ export function checkTelemetryEnabled(settings?: CallSettings): boolean { ); } -export function isMetadata(value: unknown): value is Metadata { - return ( - value !== null && - typeof value === 'object' && - 'set' in value && - typeof value.set === 'function' - ); -} - function words(str: string, normalize = false) { if (normalize) { // strings like somethingABCSomething are special case for protobuf.js, diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts index 1076e1735040..494331cd1bdf 100644 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -17,25 +17,43 @@ import * as assert from 'assert'; import {describe, it, beforeEach, afterEach} from 'mocha'; import { - context, propagation, trace, ROOT_CONTEXT, TextMapPropagator, } from '@opentelemetry/api'; import {status} from '@grpc/grpc-js'; -import {addTimeoutArg} from '../../src/normalCalls/timeout'; import {createApiCall} from '../../src/createApiCall'; import * as gax from '../../src/gax'; import {GoogleError} from '../../src/googleError'; import {GrpcClient, Metadata} from '../../src/grpc'; -import {StaticTraceContext} from '../../src/observability/TracerHelper'; +import { + traceCall, + DynamicTraceContext, + StaticTraceContext, +} from '../../src/observability/TracerHelper'; import {OtelHarness} from './otelHarness'; -import {GRPCCall, GRPCCallOtherArgs, GRPCCallResult} from '../../src/apitypes'; +import {GRPCCall, GRPCCallResult} from '../../src/apitypes'; -describe('metadata propagation (gRPC and HTTP)', () => { +describe('metadata propagation via TracerHelper and createApiCall', () => { let harness: OtelHarness; + const telemetryInfo: StaticTraceContext = { + gcpClientService: 'echo.googleapis.com', + gcpVersion: '1.2.3', + gcpRepo: 'googleapis/google-cloud-node', + gcpArtifact: '@google-cloud/echo', + }; + + const grpcMetadataGetter = { + get(carrier: Metadata, key: string) { + return carrier.get(key) as string[]; + }, + keys() { + return ['traceparent']; + }, + }; + // Custom propagator to simulate W3C traceparent injection and extraction const testPropagator: TextMapPropagator = { inject(ctx, carrier, setter) { @@ -82,269 +100,100 @@ describe('metadata propagation (gRPC and HTTP)', () => { harness.teardown(); }); - it('injects active span context into gRPC metadata', done => { - const grpcClient = new GrpcClient(); - const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - - const tracer = trace.getTracer('test-tracer'); - const span = tracer.startSpan('test-rpc'); - - let receivedMetadata: Metadata | null = null; - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - receivedMetadata = metadata as Metadata; - callback(null, {success: true}); - return {} as GRPCCallResult; - }; - - const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: builder, + it('injects active span context into injectedHeaders in traceCall', async () => { + const dynamicArgs: DynamicTraceContext = { + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'grpc', }; - const callHandler = addTimeoutArg( - stubFunc as unknown as GRPCCall, - 1000, - otherArgs, + let capturedHeaders: Record | undefined; + await traceCall( + dynamicArgs, + telemetryInfo, + async (_tracedCallback, _recordResend, injectedHeaders) => { + capturedHeaders = injectedHeaders; + return [{echo: 'ok'}, undefined, undefined]; + }, ); - context.with(trace.setSpan(context.active(), span), () => { - callHandler({}, (err: unknown) => { - span.end(); - assert.ifError(err); - assert.ok(receivedMetadata); - const traceparent = (receivedMetadata as Metadata).get( - 'traceparent', - ) as unknown[]; - assert.ok(traceparent && traceparent.length > 0); - const spanContext = span.spanContext(); - assert.strictEqual( - traceparent[0], - `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, - ); - done(); - }); - }); + const span = harness.requireSingleSpan('google-gax'); + const spanCtx = span.spanContext(); + assert.ok(capturedHeaders); + assert.strictEqual( + capturedHeaders!['traceparent'], + `00-${spanCtx.traceId}-${spanCtx.spanId}-0${spanCtx.traceFlags}`, + ); }); - it('does not mutate baseMetadata across multiple gRPC calls', done => { + it('does not mutate baseMetadata or user headers across multiple gRPC calls', async () => { const grpcClient = new GrpcClient(); const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - // Capture the baseMetadata returned initially const baseMetadata = builder() as unknown as Metadata; assert.strictEqual( (baseMetadata.get('traceparent') as unknown[]).length, 0, ); - const tracer = trace.getTracer('test-tracer'); - const span1 = tracer.startSpan('rpc-1'); - const span2 = tracer.startSpan('rpc-2'); - - let metadataCall1: Metadata | null = null; - let metadataCall2: Metadata | null = null; - - const stubFunc1 = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - metadataCall1 = meta as Metadata; - cb(null, {}); - return {} as GRPCCallResult; - }; + const settings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + metadataBuilder: builder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); - const stubFunc2 = ( + const receivedMetadata: Metadata[] = []; + const stubFunc = ( arg: {}, meta: {}, opt: {}, cb: Function, ): GRPCCallResult => { - metadataCall2 = meta as Metadata; + receivedMetadata.push(meta as Metadata); cb(null, {}); - return {} as GRPCCallResult; - }; - - const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: builder, + return {cancel: () => {}}; }; - const handler1 = addTimeoutArg( - stubFunc1 as unknown as GRPCCall, - 1000, - otherArgs, - ); - const handler2 = addTimeoutArg( - stubFunc2 as unknown as GRPCCall, - 1000, - otherArgs, - ); + const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); + const userHeaders = {'x-goog-request-params': 'parent=projects/test'}; - context.with(trace.setSpan(context.active(), span1), () => { - handler1({}, () => { - span1.end(); - - context.with(trace.setSpan(context.active(), span2), () => { - handler2({}, () => { - span2.end(); - - // Check metadata from Call 1 - const spanCtx1 = span1.spanContext(); - const traceparent1 = metadataCall1!.get('traceparent') as unknown[]; - assert.strictEqual( - traceparent1[0], - `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, - ); - - // Check metadata from Call 2 - const spanCtx2 = span2.spanContext(); - const traceparent2 = metadataCall2!.get('traceparent') as unknown[]; - assert.strictEqual( - traceparent2[0], - `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, - ); - - // Crucial: baseMetadata must NOT contain traceparent! - assert.strictEqual( - (baseMetadata.get('traceparent') as unknown[]).length, - 0, - ); - done(); - }); - }); - }); - }); - }); + await apiCall({}, {otherArgs: {headers: userHeaders}}); + await apiCall({}, {otherArgs: {headers: userHeaders}}); - it('injects active span context into HTTP/REST plain object metadata', done => { - const tracer = trace.getTracer('test-tracer'); - const span = tracer.startSpan('http-rpc'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(receivedMetadata.length, 2); - let receivedMetadata: Record | null = null; - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - receivedMetadata = metadata as Record; - callback(null, {}); - return {} as GRPCCallResult; - }; - - const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), - }; - - const handler = addTimeoutArg( - stubFunc as unknown as GRPCCall, - 1000, - otherArgs, + const spanCtx1 = spans[0].spanContext(); + const traceparent1 = receivedMetadata[0].get('traceparent') as unknown[]; + assert.strictEqual( + traceparent1[0], + `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, ); - context.with(trace.setSpan(context.active(), span), () => { - handler({}, (err: unknown) => { - span.end(); - assert.ifError(err); - assert.ok(receivedMetadata); - const spanContext = span.spanContext(); - assert.strictEqual( - receivedMetadata!['traceparent'], - `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, - ); - assert.deepStrictEqual(receivedMetadata!['x-goog-api-client'], [ - 'grpc-web/1.0', - ]); - done(); - }); - }); - }); - - it('does not mutate shared HTTP/REST plain object metadata across calls', done => { - const sharedMetadata: Record = { - 'x-goog-api-client': ['grpc-web/1.0'], - }; - - const tracer = trace.getTracer('test-tracer'); - const span1 = tracer.startSpan('http-rpc-1'); - const span2 = tracer.startSpan('http-rpc-2'); - - let metadataCall1: Record | null = null; - let metadataCall2: Record | null = null; - - const stub1 = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - metadataCall1 = meta as Record; - cb(null, {}); - return {} as GRPCCallResult; - }; - - const stub2 = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - metadataCall2 = meta as Record; - cb(null, {}); - return {} as GRPCCallResult; - }; - - const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: () => sharedMetadata, - }; + const spanCtx2 = spans[1].spanContext(); + const traceparent2 = receivedMetadata[1].get('traceparent') as unknown[]; + assert.strictEqual( + traceparent2[0], + `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, + ); - const handler1 = addTimeoutArg( - stub1 as unknown as GRPCCall, - 1000, - otherArgs, + // Crucial: neither baseMetadata nor userHeaders was mutated + assert.strictEqual( + (baseMetadata.get('traceparent') as unknown[]).length, + 0, ); - const handler2 = addTimeoutArg( - stub2 as unknown as GRPCCall, - 1000, - otherArgs, + assert.strictEqual( + (userHeaders as Record)['traceparent'], + undefined, ); - - context.with(trace.setSpan(context.active(), span1), () => { - handler1({}, () => { - span1.end(); - - context.with(trace.setSpan(context.active(), span2), () => { - handler2({}, () => { - span2.end(); - - const spanCtx1 = span1.spanContext(); - assert.strictEqual( - metadataCall1!['traceparent'], - `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, - ); - - const spanCtx2 = span2.spanContext(); - assert.strictEqual( - metadataCall2!['traceparent'], - `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, - ); - - assert.notStrictEqual(metadataCall1, sharedMetadata); - assert.notStrictEqual(metadataCall2, sharedMetadata); - assert.strictEqual(sharedMetadata['traceparent'], undefined); - done(); - }); - }); - }); - }); }); - it('does not inject into gRPC or HTTP metadata when telemetry is disabled', done => { + it('does not inject into gRPC or HTTP metadata when telemetry is disabled', async () => { delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; const grpcClient = new GrpcClient(); @@ -352,9 +201,6 @@ describe('metadata propagation (gRPC and HTTP)', () => { 'x-goog-api-client': 'test', }); - const tracer = trace.getTracer('test-tracer'); - const span = tracer.startSpan('disabled-rpc'); - let receivedGrpcMetadata: Metadata | null = null; let receivedHttpMetadata: Record | null = null; @@ -366,7 +212,7 @@ describe('metadata propagation (gRPC and HTTP)', () => { ): GRPCCallResult => { receivedGrpcMetadata = meta as Metadata; cb(null, {}); - return {} as GRPCCallResult; + return {cancel: () => {}}; }; const httpStub = ( @@ -377,82 +223,53 @@ describe('metadata propagation (gRPC and HTTP)', () => { ): GRPCCallResult => { receivedHttpMetadata = meta as Record; cb(null, {}); - return {} as GRPCCallResult; + return {cancel: () => {}}; }; - const grpcHandler = addTimeoutArg(grpcStub as unknown as GRPCCall, 1000, { - metadataBuilder: grpcBuilder, - }); - const httpHandler = addTimeoutArg(httpStub as unknown as GRPCCall, 1000, { - metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), - }); - - context.with(trace.setSpan(context.active(), span), () => { - grpcHandler({}, (err1: unknown) => { - assert.ifError(err1); - httpHandler({}, (err2: unknown) => { - span.end(); - assert.ifError(err2); - assert.ok(receivedGrpcMetadata); - assert.strictEqual( - (receivedGrpcMetadata!.get('traceparent') as unknown[]).length, - 0, - ); - assert.ok(receivedHttpMetadata); - assert.strictEqual(receivedHttpMetadata!['traceparent'], undefined); - done(); - }); - }); - }); - }); + const grpcCall = createApiCall( + grpcStub as unknown as GRPCCall, + new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: false, + otherArgs: { + metadataBuilder: grpcBuilder, + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }), + ); - it('handles null or missing metadata gracefully', done => { - let called = false; - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - called = true; - assert.strictEqual(metadata, null); - callback(null, {}); - return {} as GRPCCallResult; - }; + const httpCall = createApiCall( + httpStub as unknown as GRPCCall, + new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: false, + otherArgs: { + metadataBuilder: (_abTests?: {}, moreHeaders?: {}) => ({ + 'x-goog-api-client': ['grpc-web/1.0'], + ...moreHeaders, + }), + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }), + undefined, + 'rest', + ); - const otherArgs: GRPCCallOtherArgs = { - metadataBuilder: (() => - null) as unknown as GRPCCallOtherArgs['metadataBuilder'], - }; + await grpcCall({}, undefined); + await httpCall({}, undefined); - const handler = addTimeoutArg( - stubFunc as unknown as GRPCCall, - 1000, - otherArgs, + assert.ok(receivedGrpcMetadata); + assert.strictEqual( + (receivedGrpcMetadata!.get('traceparent') as unknown[]).length, + 0, ); - handler({}, () => { - assert.ok(called); - done(); - }); + assert.ok(receivedHttpMetadata); + assert.strictEqual(receivedHttpMetadata!['traceparent'], undefined); }); describe('T3 client request trace to low-level unary trace correlation', () => { - const telemetryInfo: StaticTraceContext = { - gcpClientService: 'echo.googleapis.com', - gcpVersion: '1.2.3', - gcpRepo: 'googleapis/google-cloud-node', - gcpArtifact: '@google-cloud/echo', - }; - - const grpcMetadataGetter = { - get(carrier: Metadata, key: string) { - return carrier.get(key) as string[]; - }, - keys() { - return ['traceparent']; - }, - }; - it('ties a low-level gRPC unary trace to its parent T3 client request trace', async () => { const grpcClient = new GrpcClient(); const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); @@ -522,7 +339,10 @@ describe('metadata propagation (gRPC and HTTP)', () => { apiName: 'google.example.v1.Echo', enableTelemetryTracing: true, otherArgs: { - metadataBuilder: () => ({'x-goog-api-client': ['grpc-web/1.0']}), + metadataBuilder: (_abTests?: {}, moreHeaders?: {}) => ({ + 'x-goog-api-client': ['grpc-web/1.0'], + ...moreHeaders, + }), internalTelemetryInfo: telemetryInfo, internalMethodName: 'Echo', }, diff --git a/core/packages/gax/test/unit/util.ts b/core/packages/gax/test/unit/util.ts index f443c007d691..48be4e9f6a54 100644 --- a/core/packages/gax/test/unit/util.ts +++ b/core/packages/gax/test/unit/util.ts @@ -25,7 +25,6 @@ import { decodeProtobufAny, decodeAnyProtosInArray, checkTelemetryEnabled, - isMetadata, connectionCodes, requestCodes, requestBodyCodes, @@ -36,7 +35,6 @@ import { ignoredClientHeaderTokens, DEPTH_TO_CHECK, } from '../../src/util'; -import {GrpcClient} from '../../src/grpc'; import {StaticTraceContext} from '../../src/observability/TracerHelper'; import {CallSettings} from '../../src/gax'; import * as protobuf from 'protobufjs'; @@ -369,27 +367,6 @@ describe('util.ts', () => { }); }); - describe('isMetadata', () => { - it('returns true for gRPC Metadata instances', () => { - const grpcClient = new GrpcClient(); - const metadata = grpcClient.metadataBuilder({ - 'x-goog-api-client': 'test', - })(); - assert.strictEqual(isMetadata(metadata), true); - }); - - it('returns false for plain HTTP/REST header objects, null, and primitives', () => { - assert.strictEqual( - isMetadata({'x-goog-api-client': ['grpc-web/1.0']}), - false, - ); - assert.strictEqual(isMetadata({set: 'not-a-function'}), false); - assert.strictEqual(isMetadata(null), false); - assert.strictEqual(isMetadata(undefined), false); - assert.strictEqual(isMetadata('string'), false); - }); - }); - describe('error code and class constants', () => { it('defines connectionCodes containing common network errors', () => { assert(Array.isArray(connectionCodes)); From ad27ebaf6aff69796be97aba6bbc1ce568ac6d87 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 15:15:27 -0700 Subject: [PATCH 06/13] fix(gax): resolve TypeScript narrowing error in metadataPropagation test --- .../gax/test/unit/metadataPropagation.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts index 494331cd1bdf..814c89f11b95 100644 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -201,8 +201,8 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { 'x-goog-api-client': 'test', }); - let receivedGrpcMetadata: Metadata | null = null; - let receivedHttpMetadata: Record | null = null; + const receivedGrpcMetadata: Metadata[] = []; + const receivedHttpMetadata: Record[] = []; const grpcStub = ( arg: {}, @@ -210,7 +210,7 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { opt: {}, cb: Function, ): GRPCCallResult => { - receivedGrpcMetadata = meta as Metadata; + receivedGrpcMetadata.push(meta as Metadata); cb(null, {}); return {cancel: () => {}}; }; @@ -221,7 +221,7 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { opt: {}, cb: Function, ): GRPCCallResult => { - receivedHttpMetadata = meta as Record; + receivedHttpMetadata.push(meta as Record); cb(null, {}); return {cancel: () => {}}; }; @@ -260,13 +260,13 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { await grpcCall({}, undefined); await httpCall({}, undefined); - assert.ok(receivedGrpcMetadata); + assert.strictEqual(receivedGrpcMetadata.length, 1); assert.strictEqual( - (receivedGrpcMetadata!.get('traceparent') as unknown[]).length, + (receivedGrpcMetadata[0].get('traceparent') as unknown[]).length, 0, ); - assert.ok(receivedHttpMetadata); - assert.strictEqual(receivedHttpMetadata!['traceparent'], undefined); + assert.strictEqual(receivedHttpMetadata.length, 1); + assert.strictEqual(receivedHttpMetadata[0]['traceparent'], undefined); }); describe('T3 client request trace to low-level unary trace correlation', () => { From ce310faabbae50dd3859e2a24b463698a5a070ff Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 16:49:29 -0700 Subject: [PATCH 07/13] feat(gax): emit T4 per-attempt client spans directly without instrumentation libraries --- core/packages/gax/src/createApiCall.ts | 80 ++- core/packages/gax/src/fallback.ts | 17 +- core/packages/gax/src/grpc.ts | 18 +- .../gax/src/observability/TracerHelper.ts | 215 +++++++- core/packages/gax/test/unit/apiCallable.ts | 228 ++++---- .../gax/test/unit/metadataPropagation.ts | 486 ++++++++---------- 6 files changed, 631 insertions(+), 413 deletions(-) diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index e8b01354c866..8a507a7a1a15 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -18,6 +18,7 @@ * Provides function wrappers that implement page streaming and retrying. */ +import {context} from '@opentelemetry/api'; import {createAPICaller} from './apiCaller'; import { APICallback, @@ -26,6 +27,7 @@ import { GRPCCallOtherArgs, RequestType, SimpleCallbackFunction, + UnaryCall, } from './apitypes'; import {Descriptor} from './descriptor'; import {CallOptions, CallSettings, convertRetryOptions} from './gax'; @@ -36,8 +38,10 @@ import {StreamProxy} from './streamingCalls/streaming'; import {warn} from './warnings'; import { traceCall, + traceAttempt, StaticTraceContext, DynamicTraceContext, + AttemptTraceContext, ResendRecorder, } from './observability/TracerHelper'; import {resolveStaticTraceContext} from './observability/metadataResolver'; @@ -77,14 +81,33 @@ export function createApiCall( const apiCaller = createAPICaller(settings, descriptor); const tracingEnabled = checkTelemetryEnabled(settings); + const staticArgs: StaticTraceContext | undefined = tracingEnabled + ? resolveStaticTraceContext(settings) + : undefined; + const serviceName = settings.apiName?.split('.').pop() ?? ''; + const isFallback = Boolean(_fallback); + const dynamicArgs: DynamicTraceContext | undefined = tracingEnabled + ? { + clientName: serviceName ? `${serviceName}Client` : '', + methodName: settings.otherArgs?.internalMethodName ?? '', + rpcType: isFallback ? 'http' : 'grpc', + } + : undefined; + const attemptDynamicArgs: AttemptTraceContext | undefined = + tracingEnabled && dynamicArgs + ? { + ...dynamicArgs, + apiName: settings.apiName ?? '', + } + : undefined; const invokeCall = ( request: RequestType, callOptions?: CallOptions, callback?: APICallback, recordResend?: ResendRecorder, - injectedHeaders?: Record, ) => { + const parentContext = tracingEnabled ? context.active() : undefined; let currentApiCaller = apiCaller; let thisSettings: CallSettings; @@ -101,16 +124,6 @@ export function createApiCall( thisSettings = settings.merge(callOptions); } - if (injectedHeaders && Object.keys(injectedHeaders).length > 0) { - thisSettings.otherArgs = { - ...thisSettings.otherArgs, - headers: { - ...thisSettings.otherArgs?.headers, - ...injectedHeaders, - }, - }; - } - // special case: if bundling is disabled for this one call, // use default API caller instead if (settings.isBundling && !thisSettings.isBundling) { @@ -135,6 +148,31 @@ export function createApiCall( const streaming = (currentApiCaller as StreamingApiCaller).descriptor ?.streaming; + if (tracingEnabled && attemptDynamicArgs && staticArgs) { + const wrappedFunc = func; + func = (( + argument: {}, + metadata: {}, + options: {}, + attemptCallback: APICallback, + ) => { + return traceAttempt( + attemptDynamicArgs, + staticArgs, + tracedAttemptCallback => + (wrappedFunc as UnaryCall)( + argument, + metadata, + options, + tracedAttemptCallback ?? attemptCallback, + ), + Boolean(streaming), + attemptCallback, + parentContext, + ); + }) as GRPCCall; + } + const retry = thisSettings.retry; if (streaming && retry) { @@ -204,15 +242,6 @@ export function createApiCall( }; if (tracingEnabled) { - const staticArgs: StaticTraceContext = resolveStaticTraceContext(settings); - - const serviceName = settings.apiName?.split('.').pop() ?? ''; - const isFallback = Boolean(_fallback); - const dynamicArgs: DynamicTraceContext = { - clientName: serviceName ? `${serviceName}Client` : '', - methodName: settings.otherArgs?.internalMethodName ?? '', - rpcType: isFallback ? 'http' : 'grpc', - }; const isStreamingCall = apiCaller instanceof StreamingApiCaller; return ( request: RequestType, @@ -220,13 +249,9 @@ export function createApiCall( callback?: APICallback, ) => { return traceCall( - dynamicArgs, - staticArgs, - ( - tracedCallback?: APICallback, - recordResend?: ResendRecorder, - injectedHeaders?: Record, - ) => { + dynamicArgs!, + staticArgs!, + (tracedCallback?: APICallback, recordResend?: ResendRecorder) => { // `traceCall` wraps the user's callback whenever one was supplied, // for stream and non-stream calls alike, and that wrapper is what // closes the span. It is undefined only when there is no callback to @@ -237,7 +262,6 @@ export function createApiCall( callOptions, tracedCallback ?? callback, recordResend, - injectedHeaders, ); }, isStreamingCall, diff --git a/core/packages/gax/src/fallback.ts b/core/packages/gax/src/fallback.ts index 9f66c8bef75c..1d2c3afabcc7 100644 --- a/core/packages/gax/src/fallback.ts +++ b/core/packages/gax/src/fallback.ts @@ -108,6 +108,8 @@ export class GrpcClient { httpRules?: Array; numericEnums: boolean; minifyJson: boolean; + private _servicePath?: string; + private _port?: number; /** * In rare cases users might need to deallocate all memory consumed by loaded protos. @@ -156,6 +158,10 @@ export class GrpcClient { this.httpRules = (options as GrpcClientOptions).httpRules; this.numericEnums = (options as GrpcClientOptions).numericEnums ?? false; this.minifyJson = (options as GrpcClientOptions).minifyJson ?? false; + this._servicePath = + (options as GrpcClientOptions).servicePath || + (options as GrpcClientOptions).apiEndpoint; + this._port = (options as GrpcClientOptions).port; } /** @@ -279,12 +285,21 @@ export class GrpcClient { } return metadata; } + const otherArgs: Record = { + metadataBuilder: buildMetadata, + }; + if (this._servicePath) { + otherArgs.servicePath = this._servicePath; + } + if (this._port !== undefined) { + otherArgs.port = this._port; + } return gax.constructSettings( serviceName, clientConfig, configOverrides, Status, - {metadataBuilder: buildMetadata}, + otherArgs, enableTelemetryTracing, internalTelemetryInfo, ); diff --git a/core/packages/gax/src/grpc.ts b/core/packages/gax/src/grpc.ts index 14d35398d514..5c68f2564562 100644 --- a/core/packages/gax/src/grpc.ts +++ b/core/packages/gax/src/grpc.ts @@ -53,6 +53,9 @@ export interface GrpcClientOptions extends GoogleAuthOptions { httpRules?: Array; numericEnums?: boolean; universeDomain?: string; + servicePath?: string; + apiEndpoint?: string; + port?: number; } export interface MetadataValue { @@ -121,6 +124,8 @@ export class GrpcClient { fallback: boolean | 'rest' | 'proto'; private static protoCache = new Map(); httpRules?: Array; + private _servicePath?: string; + private _port?: number; /** * Base directory for resolving client certificates. * @@ -174,6 +179,8 @@ export class GrpcClient { constructor(options: GrpcClientOptions = {}) { this.auth = options.auth || new GoogleAuth(options); this.fallback = false; + this._servicePath = options.servicePath || options.apiEndpoint; + this._port = options.port; const minimumVersion = 10; const major = Number(process.version.match(/^v(\d+)/)?.[1]); @@ -388,12 +395,21 @@ export class GrpcClient { enableTelemetryTracing?: boolean, internalTelemetryInfo?: StaticTraceContext, ) { + const otherArgs: Record = { + metadataBuilder: this.metadataBuilder(headers), + }; + if (this._servicePath) { + otherArgs.servicePath = this._servicePath; + } + if (this._port !== undefined) { + otherArgs.port = this._port; + } return gax.constructSettings( serviceName, clientConfig, configOverrides, this.grpc.status, - {metadataBuilder: this.metadataBuilder(headers)}, + otherArgs, enableTelemetryTracing, internalTelemetryInfo, ); diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index ae6aa3b60f0a..450f574a6f7e 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -18,8 +18,9 @@ import {EventEmitter} from 'events'; import { Attributes, context, - propagation, + Context, Span, + SpanKind, SpanStatusCode, trace, Tracer, @@ -67,6 +68,10 @@ export interface StaticTraceContext { * Server port number for the RPC call. */ serverPort?: number; + /** + * Target service domain (e.g. 'cloudkms.googleapis.com'). + */ + urlDomain?: string; } /** @@ -94,6 +99,20 @@ export interface DynamicTraceContext { * Server port number for the RPC call. */ serverPort?: number; + /** + * Target service domain (e.g. 'cloudkms.googleapis.com'). + */ + urlDomain?: string; +} + +/** + * Dynamic metadata specific to an individual RPC transport attempt (T4 span). + */ +export interface AttemptTraceContext extends DynamicTraceContext { + /** + * The fully-qualified protobuf service name (e.g. 'google.cloud.kms.v1.KeyManagementService'). + */ + apiName?: string; } /** @@ -953,6 +972,30 @@ export function handleStream( } } +/** + * Resolves the target service domain (`url.domain`) from dynamic and static trace contexts. + */ +function resolveUrlDomain( + dynamicArgs: DynamicTraceContext, + staticArgs: StaticTraceContext, +): string | undefined { + const explicit = dynamicArgs.urlDomain ?? staticArgs.urlDomain; + if (explicit) { + return explicit; + } + const rawAddress = dynamicArgs.serverAddress ?? staticArgs.serverAddress; + if (rawAddress) { + const match = rawAddress.match(/^(\[[^\]]+\]|[^:]+):(\d+)$/); + return match ? match[1] : rawAddress; + } + if (staticArgs.gcpClientService) { + return staticArgs.gcpClientService.includes('.') + ? staticArgs.gcpClientService + : `${staticArgs.gcpClientService}.googleapis.com`; + } + return undefined; +} + /** * Executes a function within an active OpenTelemetry span, populating standard * GCP telemetry attributes and recording errors/exceptions if thrown. @@ -976,7 +1019,6 @@ export function traceCall( fn: ( tracedCallback?: APICallback, recordResend?: ResendRecorder, - injectedHeaders?: Record, ) => GaxCallResult, isStreamCall?: boolean, callback?: APICallback, @@ -984,22 +1026,14 @@ export function traceCall( export function traceCall( dynamicArgs: DynamicTraceContext, staticArgs: StaticTraceContext, - fn: ( - tracedCallback?: APICallback, - recordResend?: ResendRecorder, - injectedHeaders?: Record, - ) => T, + fn: (tracedCallback?: APICallback, recordResend?: ResendRecorder) => T, isStreamCall: true, callback?: APICallback, ): T; export function traceCall( dynamicArgs: DynamicTraceContext, staticArgs: StaticTraceContext, - fn: ( - tracedCallback?: APICallback, - recordResend?: ResendRecorder, - injectedHeaders?: Record, - ) => T, + fn: (tracedCallback?: APICallback, recordResend?: ResendRecorder) => T, isStreamCall?: false, callback?: APICallback, ): T; @@ -1009,7 +1043,6 @@ export function traceCall( fn: ( tracedCallback?: APICallback, recordResend?: ResendRecorder, - injectedHeaders?: Record, ) => GaxCallResult, isStreamCall = false, callback?: APICallback, @@ -1148,10 +1181,8 @@ export function traceCall( try { const activeContext = trace.setSpan(context.active(), span); - const injectedHeaders: Record = {}; - propagation.inject(activeContext, injectedHeaders); const result = context.with(activeContext, () => - fn(tracedCallback, recordResend, injectedHeaders), + fn(tracedCallback, recordResend), ); const promiseTarget = !isStreamCall ? getPromiseTarget(result) : null; if (isStreamCall && result instanceof EventEmitter) { @@ -1172,3 +1203,155 @@ export function traceCall( } }); } + +/** + * Executes an individual RPC transport attempt within an active OpenTelemetry + * CLIENT span (T4 span), parenting it to the active T3 client request span + * and recording per-attempt network, status, and error attributes without + * injecting span context into outgoing headers. + * + * @param {AttemptTraceContext} dynamicArgs - Dynamic trace context for the RPC attempt. + * @param {StaticTraceContext} staticArgs - Static trace context for the client library. + * @param {function} fn - The transport attempt operation to trace. + * @param {boolean} [isStreamCall=false] - Whether the operation is a stream call. + * @param {APICallback} [callback] - The attempt callback. + * @param {Context} [parentContext] - Optional parent OpenTelemetry context (e.g. T3 span context). + * @returns {GaxCallResult} The result of the traced attempt. + */ +export function traceAttempt( + dynamicArgs: AttemptTraceContext, + staticArgs: StaticTraceContext, + fn: (tracedCallback?: APICallback) => T, + isStreamCall = false, + callback?: APICallback, + parentContext?: Context, +): T { + const spanName = dynamicArgs.apiName + ? `${dynamicArgs.apiName}/${dynamicArgs.methodName}` + : dynamicArgs.methodName; + const baseContext = parentContext ?? context.active(); + return getGaxTracer().startActiveSpan( + spanName, + {kind: SpanKind.CLIENT}, + baseContext, + (span: Span) => { + const urlDomain = resolveUrlDomain(dynamicArgs, staticArgs); + const initialAttributes: Attributes = { + 'gcp.client.service': staticArgs.gcpClientService, + 'gcp.client.version': staticArgs.gcpVersion, + 'gcp.repo': staticArgs.gcpRepo, + 'gcp.artifact': staticArgs.gcpArtifact, + 'gcp.method.name': dynamicArgs.methodName, + 'gcp.method.type': dynamicArgs.rpcType, + 'rpc.system': dynamicArgs.rpcType, + }; + if (urlDomain !== undefined) { + initialAttributes['url.domain'] = urlDomain; + } + span.setAttributes(initialAttributes); + + let rawAddress = + dynamicArgs.serverAddress ?? staticArgs.serverAddress ?? urlDomain; + let rawPort = dynamicArgs.serverPort ?? staticArgs.serverPort; + if (rawAddress) { + const match = rawAddress.match(/^(\[[^\]]+\]|[^:]+):(\d+)$/); + if (match) { + rawAddress = match[1]; + rawPort = rawPort ?? Number(match[2]); + } + rawPort = rawPort ?? 443; + } + + let spanEnded = false; + let errorRecorded = false; + let recordedError: unknown; + let rpcStatusName: string | undefined; + let httpStatusCode: number | undefined; + + const setErrorStatus = (message: string) => { + errorRecorded = true; + span.setStatus({code: SpanStatusCode.ERROR, message}); + }; + + const setStatusAttributes = () => { + const attributes: Attributes = {}; + if (rpcStatusName !== undefined) { + attributes['rpc.response.status_code'] = rpcStatusName; + if (dynamicArgs.rpcType === 'grpc') { + attributes['grpc.response.status_code'] = rpcStatusName; + } + } + if (dynamicArgs.rpcType === 'http' && httpStatusCode !== undefined) { + attributes['http.response.status_code'] = httpStatusCode; + } + if ( + rawAddress !== undefined && + (!errorRecorded || !isPreConnectionFailure(recordedError)) + ) { + attributes['server.address'] = rawAddress; + if (rawPort !== undefined) { + attributes['server.port'] = rawPort; + } + } + span.setAttributes(attributes); + }; + + const endSpan = () => { + if (!spanEnded) { + spanEnded = true; + if (!errorRecorded) { + rpcStatusName = Status[Status.OK]; + httpStatusCode = 200; + } + setStatusAttributes(); + span.end(); + } + }; + + const recordError = (e: unknown) => { + recordedError = e; + rpcStatusName = resolveRpcStatusName(e); + httpStatusCode = resolveHttpStatusCode(e); + span.setAttributes({ + 'error.type': resolveErrorType(e, dynamicArgs.rpcType), + }); + if (e instanceof Error) { + recordExceptionEvent(span, e, dynamicArgs.rpcType); + setErrorStatus(e.message); + } else { + setErrorStatus(resolveErrorMessage(e)); + } + }; + + const tracedCallback: APICallback | undefined = callback + ? function (this: unknown, ...args: Parameters) { + const err = args[0]; + if (err) { + recordError(err); + } + endSpan(); + callback.apply(this, args); + } + : undefined; + + try { + const result = fn(tracedCallback); + const promiseTarget = !isStreamCall ? getPromiseTarget(result) : null; + if (isStreamCall && result instanceof EventEmitter) { + handleStream(result, recordError, endSpan, !!callback); + } else if (promiseTarget) { + handlePromise(promiseTarget, recordError, endSpan); + } else if (tracedCallback) { + // Span stays open; tracedCallback ends it when the attempt completes. + } else { + endSpan(); + } + return result; + } catch (e) { + recordError(e); + endSpan(); + throw e; + } + }, + ); +} diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index 20465bf1b3e2..4bd8ed9b73bc 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -17,6 +17,7 @@ import assert from 'assert'; import {PassThrough} from 'stream'; import {status} from '@grpc/grpc-js'; +import {SpanKind} from '@opentelemetry/api'; import {afterEach, beforeEach, describe, it} from 'mocha'; import * as sinon from 'sinon'; @@ -583,9 +584,17 @@ describe('createApiCall', () => { assert.deepStrictEqual(response, {data: 'hello'}); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); + assert.strictEqual( + attemptSpan.parentSpanContext?.spanId, + span.spanContext().spanId, + ); assert.strictEqual(span.name, 'EchoClient.Echo'); + assert.strictEqual(span.kind, SpanKind.INTERNAL); assert.strictEqual(span.ended, true); assert.strictEqual( span.attributes['gcp.client.service'], @@ -631,8 +640,14 @@ describe('createApiCall', () => { assert.deepStrictEqual(response, {data: 'hello'}); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual( + attemptSpan.name, + 'google.cloud.redis.v1.CloudRedis/GetInstance', + ); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); assert.strictEqual(span.name, 'CloudRedisClient.GetInstance'); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['gcp.client.service'], 'redis'); @@ -672,8 +687,8 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const span = spans[1]; assert.strictEqual(span.attributes['gcp.client.service'], 'env-service'); assert.strictEqual(span.attributes['gcp.client.version'], '9.9.9'); assert.strictEqual( @@ -709,8 +724,12 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); + assert.strictEqual(attemptSpan.attributes['gcp.method.type'], 'http'); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['gcp.method.type'], 'http'); @@ -748,8 +767,11 @@ describe('createApiCall', () => { assert.strictEqual(dynamicArgs.rpcType, 'http'); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(attemptSpan.attributes['gcp.method.type'], 'http'); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['gcp.method.type'], 'http'); @@ -790,8 +812,8 @@ describe('createApiCall', () => { assert.strictEqual(dynamicArgs.rpcType, 'http'); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const span = spans[1]; assert.strictEqual(span.attributes['gcp.method.type'], 'http'); }); @@ -845,20 +867,21 @@ describe('createApiCall', () => { ); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.ended, true); - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); - // On the fallback transport error.type reports the HTTP status the - // server sent. A deadline expires before any response arrives, so there - // is none, and the attribute resolves to CLIENT_TIMEOUT per Tier 3. - assert.strictEqual(span.attributes['error.type'], 'CLIENT_TIMEOUT'); - assert.strictEqual( - span.attributes['rpc.response.status_code'], - 'DEADLINE_EXCEEDED', - ); - assert.strictEqual(span.events.length, 1); - assert.strictEqual(span.events[0].name, 'exception'); + assert.strictEqual(spans.length, 2); + for (const span of spans) { + assert.strictEqual(span.ended, true); + assert.strictEqual(span.attributes['gcp.method.type'], 'http'); + // On the fallback transport error.type reports the HTTP status the + // server sent. A deadline expires before any response arrives, so there + // is none, and the attribute resolves to CLIENT_TIMEOUT per Tier 3. + assert.strictEqual(span.attributes['error.type'], 'CLIENT_TIMEOUT'); + assert.strictEqual( + span.attributes['rpc.response.status_code'], + 'DEADLINE_EXCEEDED', + ); + assert.strictEqual(span.events.length, 1); + assert.strictEqual(span.events[0].name, 'exception'); + } }); it('ends the span and preserves system error codes like ECONNREFUSED on a fallback call', async () => { @@ -906,28 +929,29 @@ describe('createApiCall', () => { ); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.ended, true); - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); - assert.strictEqual( - span.attributes['error.type'], - 'CLIENT_CONNECTION_ERROR', - ); - assert.strictEqual( - span.attributes['rpc.response.status_code'], - 'UNAVAILABLE', - ); - assert.strictEqual( - span.attributes['http.response.status_code'], - undefined, - ); - assert.strictEqual(span.events.length, 1); - assert.strictEqual(span.events[0].name, 'exception'); - assert.strictEqual( - span.events[0].attributes?.['exception.type'], - 'GoogleError', - ); + assert.strictEqual(spans.length, 2); + for (const span of spans) { + assert.strictEqual(span.ended, true); + assert.strictEqual(span.attributes['gcp.method.type'], 'http'); + assert.strictEqual( + span.attributes['error.type'], + 'CLIENT_CONNECTION_ERROR', + ); + assert.strictEqual( + span.attributes['rpc.response.status_code'], + 'UNAVAILABLE', + ); + assert.strictEqual( + span.attributes['http.response.status_code'], + undefined, + ); + assert.strictEqual(span.events.length, 1); + assert.strictEqual(span.events[0].name, 'exception'); + assert.strictEqual( + span.events[0].attributes?.['exception.type'], + 'GoogleError', + ); + } }); it('passes fallback flag and isStreamingCall as true for server-streaming fallback calls', () => { @@ -987,9 +1011,9 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['gcp.method.type'], 'grpc'); + assert.strictEqual(spans[1].attributes['gcp.method.type'], 'grpc'); }); it('sets rpcType to http when _fallback is "rest"', async () => { @@ -1018,9 +1042,9 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); }); it('sets rpcType to http when _fallback is "proto"', async () => { @@ -1049,9 +1073,9 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); }); it('pipes telemetry information configured via constructSettings', async () => { @@ -1090,8 +1114,11 @@ describe('createApiCall', () => { await apiCall({}, undefined); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.ended, true); assert.strictEqual( @@ -1148,12 +1175,13 @@ describe('createApiCall', () => { ); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.ended, true); - assert.strictEqual(span.status.message, 'RPC test failure'); - assert.strictEqual(span.events.length, 1); - assert.strictEqual(span.events[0].name, 'exception'); + assert.strictEqual(spans.length, 2); + for (const span of spans) { + assert.strictEqual(span.ended, true); + assert.strictEqual(span.status.message, 'RPC test failure'); + assert.strictEqual(span.events.length, 1); + assert.strictEqual(span.events[0].name, 'exception'); + } }); it('does not end span prematurely for successful asynchronous API calls', async () => { @@ -1189,10 +1217,12 @@ describe('createApiCall', () => { const [response] = (await promise) as [{data: string}, unknown, unknown]; assert.deepStrictEqual(response, {data: 'hello'}); - // Span must only be ended after completion + // Spans must only be ended after completion const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].ended, true); + assert.strictEqual(spans[0].name, 'google.example.v1.Echo/Echo'); + const span = spans[1]; assert.strictEqual(span.ended, true); assert.strictEqual(span.name, 'EchoClient.Echo'); }); @@ -1308,8 +1338,12 @@ describe('createApiCall', () => { try { assert.strictEqual(received.length, 2); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + const span = spans[1]; + assert.strictEqual(attemptSpan.ended, true); + assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); assert.strictEqual(span.ended, true); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); @@ -1352,12 +1386,13 @@ describe('createApiCall', () => { try { assert.strictEqual(err.message, 'streaming test failure'); const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - const span = spans[0]; - assert.strictEqual(span.ended, true); - assert.strictEqual(span.status.message, 'streaming test failure'); - assert.strictEqual(span.events.length, 1); - assert.strictEqual(span.events[0].name, 'exception'); + assert.strictEqual(spans.length, 2); + for (const span of spans) { + assert.strictEqual(span.ended, true); + assert.strictEqual(span.status.message, 'streaming test failure'); + assert.strictEqual(span.events.length, 1); + assert.strictEqual(span.events[0].name, 'exception'); + } done(); } catch (e) { done(e); @@ -1415,7 +1450,10 @@ describe('createApiCall', () => { await apiCall({}, undefined); assert.strictEqual(attempts, 1); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); // Asserting the transport too, so that a span accidentally // produced by the other one cannot satisfy this test. assert.strictEqual( @@ -1448,7 +1486,10 @@ describe('createApiCall', () => { await apiCall({}, undefined); assert.strictEqual(attempts, 3); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); assert.strictEqual( span.attributes['gcp.method.type'], transport.rpcType, @@ -1458,10 +1499,13 @@ describe('createApiCall', () => { // off-by-one between the two is exactly what the attribute // defines. harness.assertResendCount(attempts - 1, {span}); - harness.assertResponseStatus({ - rpcStatus: 'OK', - ...(transport.rpcType === 'http' ? {httpStatus: 200} : {}), - }); + harness.assertResponseStatus( + { + rpcStatus: 'OK', + ...(transport.rpcType === 'http' ? {httpStatus: 200} : {}), + }, + {span}, + ); }); it('reports resends correctly when retries are exhausted by maxRetries', async () => { @@ -1505,7 +1549,10 @@ describe('createApiCall', () => { }); assert.strictEqual(attempts, 2); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); assert.strictEqual( span.attributes['gcp.method.type'], transport.rpcType, @@ -1553,7 +1600,10 @@ describe('createApiCall', () => { await apiCall({}, undefined); }); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); assert.strictEqual( span.attributes['gcp.method.type'], transport.rpcType, @@ -1622,7 +1672,10 @@ describe('createApiCall', () => { try { // Three attempts: the initial send plus the two allowed resends. assert.strictEqual(attempts, 3); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); harness.assertResendCount(2, {span}); done(); @@ -1675,7 +1728,10 @@ describe('createApiCall', () => { stream.on('error', () => { try { assert.strictEqual(attempts, 3); - const span = harness.requireSingleSpan('google-gax'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 1 + attempts); + const span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(span); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); harness.assertResendCount(2, {span}); done(); diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts index 814c89f11b95..e2422e39719f 100644 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ b/core/packages/gax/test/unit/metadataPropagation.ts @@ -16,26 +16,24 @@ import * as assert from 'assert'; import {describe, it, beforeEach, afterEach} from 'mocha'; -import { - propagation, - trace, - ROOT_CONTEXT, - TextMapPropagator, -} from '@opentelemetry/api'; +import {SpanKind, SpanStatusCode} from '@opentelemetry/api'; import {status} from '@grpc/grpc-js'; import {createApiCall} from '../../src/createApiCall'; import * as gax from '../../src/gax'; import {GoogleError} from '../../src/googleError'; import {GrpcClient, Metadata} from '../../src/grpc'; +import {GrpcClient as FallbackGrpcClient} from '../../src/fallback'; import { traceCall, - DynamicTraceContext, + traceAttempt, + AttemptTraceContext, StaticTraceContext, } from '../../src/observability/TracerHelper'; +import {clearMetadataCache} from '../../src/observability/metadataResolver'; import {OtelHarness} from './otelHarness'; import {GRPCCall, GRPCCallResult} from '../../src/apitypes'; -describe('metadata propagation via TracerHelper and createApiCall', () => { +describe('T4 per-attempt spans via TracerHelper and createApiCall', () => { let harness: OtelHarness; const telemetryInfo: StaticTraceContext = { @@ -45,157 +43,76 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { gcpArtifact: '@google-cloud/echo', }; - const grpcMetadataGetter = { - get(carrier: Metadata, key: string) { - return carrier.get(key) as string[]; - }, - keys() { - return ['traceparent']; - }, - }; - - // Custom propagator to simulate W3C traceparent injection and extraction - const testPropagator: TextMapPropagator = { - inject(ctx, carrier, setter) { - const spanContext = trace.getSpanContext(ctx); - if (spanContext && trace.isSpanContextValid(spanContext)) { - setter.set( - carrier, - 'traceparent', - `00-${spanContext.traceId}-${spanContext.spanId}-0${spanContext.traceFlags}`, - ); - } - }, - extract(ctx, carrier, getter) { - const raw = getter.get(carrier, 'traceparent'); - const value = Array.isArray(raw) ? raw[0] : raw; - if (typeof value === 'string') { - const parts = value.split('-'); - if (parts.length === 4) { - return trace.setSpanContext(ctx, { - traceId: parts[1], - spanId: parts[2], - traceFlags: parseInt(parts[3], 16), - isRemote: true, - }); - } - } - return ctx; - }, - fields() { - return ['traceparent']; - }, - }; - beforeEach(() => { + clearMetadataCache(); process.env.GOOGLE_SDK_NODE_ENABLE_TRACING = 'true'; harness = new OtelHarness(); harness.setup(); - propagation.setGlobalPropagator(testPropagator); }); afterEach(() => { + clearMetadataCache(); delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; - propagation.disable(); harness.teardown(); }); - it('injects active span context into injectedHeaders in traceCall', async () => { - const dynamicArgs: DynamicTraceContext = { + it('creates a CLIENT T4 span in traceAttempt with url.domain, server.address, server.port, and status_code', async () => { + const attemptArgs: AttemptTraceContext = { + apiName: 'google.example.v1.Echo', clientName: 'EchoClient', methodName: 'Echo', rpcType: 'grpc', }; - let capturedHeaders: Record | undefined; - await traceCall( - dynamicArgs, - telemetryInfo, - async (_tracedCallback, _recordResend, injectedHeaders) => { - capturedHeaders = injectedHeaders; - return [{echo: 'ok'}, undefined, undefined]; - }, - ); + await traceAttempt(attemptArgs, telemetryInfo, async () => { + return [{echo: 'ok'}, undefined, undefined]; + }); const span = harness.requireSingleSpan('google-gax'); - const spanCtx = span.spanContext(); - assert.ok(capturedHeaders); + assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(span.kind, SpanKind.CLIENT); + assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); assert.strictEqual( - capturedHeaders!['traceparent'], - `00-${spanCtx.traceId}-${spanCtx.spanId}-0${spanCtx.traceFlags}`, + span.attributes['server.address'], + 'echo.googleapis.com', ); + assert.strictEqual(span.attributes['server.port'], 443); + assert.strictEqual(span.attributes['rpc.system'], 'grpc'); + assert.strictEqual(span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(span.attributes['grpc.response.status_code'], 'OK'); }); - it('does not mutate baseMetadata or user headers across multiple gRPC calls', async () => { - const grpcClient = new GrpcClient(); - const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - - const baseMetadata = builder() as unknown as Metadata; - assert.strictEqual( - (baseMetadata.get('traceparent') as unknown[]).length, - 0, - ); - - const settings = new gax.CallSettings({ + it('omits server.address and server.port on T4 span for pre-connection failures while preserving url.domain', async () => { + const attemptArgs: AttemptTraceContext = { apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - metadataBuilder: builder, - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }); - - const receivedMetadata: Metadata[] = []; - const stubFunc = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - receivedMetadata.push(meta as Metadata); - cb(null, {}); - return {cancel: () => {}}; + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'grpc', }; - const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); - const userHeaders = {'x-goog-request-params': 'parent=projects/test'}; - - await apiCall({}, {otherArgs: {headers: userHeaders}}); - await apiCall({}, {otherArgs: {headers: userHeaders}}); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 2); - assert.strictEqual(receivedMetadata.length, 2); - - const spanCtx1 = spans[0].spanContext(); - const traceparent1 = receivedMetadata[0].get('traceparent') as unknown[]; - assert.strictEqual( - traceparent1[0], - `00-${spanCtx1.traceId}-${spanCtx1.spanId}-0${spanCtx1.traceFlags}`, - ); + const dnsError = Object.assign(new Error('getaddrinfo ENOTFOUND'), { + code: 'ENOTFOUND', + }); - const spanCtx2 = spans[1].spanContext(); - const traceparent2 = receivedMetadata[1].get('traceparent') as unknown[]; - assert.strictEqual( - traceparent2[0], - `00-${spanCtx2.traceId}-${spanCtx2.spanId}-0${spanCtx2.traceFlags}`, - ); + await assert.rejects(async () => { + await traceAttempt(attemptArgs, telemetryInfo, async () => { + throw dnsError; + }); + }); - // Crucial: neither baseMetadata nor userHeaders was mutated - assert.strictEqual( - (baseMetadata.get('traceparent') as unknown[]).length, - 0, - ); + const span = harness.requireSingleSpan('google-gax'); + assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(span.kind, SpanKind.CLIENT); + assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); + assert.strictEqual(span.attributes['server.address'], undefined); + assert.strictEqual(span.attributes['server.port'], undefined); assert.strictEqual( - (userHeaders as Record)['traceparent'], - undefined, + span.attributes['error.type'], + 'CLIENT_CONNECTION_ERROR', ); }); - it('does not inject into gRPC or HTTP metadata when telemetry is disabled', async () => { - delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; - + it('does not inject traceparent headers into gRPC or HTTP metadata', async () => { const grpcClient = new GrpcClient(); const grpcBuilder = grpcClient.metadataBuilder({ 'x-goog-api-client': 'test', @@ -230,7 +147,7 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { grpcStub as unknown as GRPCCall, new gax.CallSettings({ apiName: 'google.example.v1.Echo', - enableTelemetryTracing: false, + enableTelemetryTracing: true, otherArgs: { metadataBuilder: grpcBuilder, internalTelemetryInfo: telemetryInfo, @@ -243,7 +160,7 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { httpStub as unknown as GRPCCall, new gax.CallSettings({ apiName: 'google.example.v1.Echo', - enableTelemetryTracing: false, + enableTelemetryTracing: true, otherArgs: { metadataBuilder: (_abTests?: {}, moreHeaders?: {}) => ({ 'x-goog-api-client': ['grpc-web/1.0'], @@ -269,144 +186,152 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { assert.strictEqual(receivedHttpMetadata[0]['traceparent'], undefined); }); - describe('T3 client request trace to low-level unary trace correlation', () => { - it('ties a low-level gRPC unary trace to its parent T3 client request trace', async () => { - const grpcClient = new GrpcClient(); - const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - - const settings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - metadataBuilder: builder, - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, + describe('T3 client request span to T4 per-attempt span correlation', () => { + it('emits a T4 gRPC attempt span parented to its T3 client request span', async () => { + const grpcClient = new GrpcClient({ + servicePath: 'echo.googleapis.com', + port: 443, }); + const defaults = grpcClient.constructSettings( + 'google.example.v1.Echo', + { + interfaces: { + 'google.example.v1.Echo': { + methods: { + Echo: {timeout_millis: 5000}, + }, + }, + }, + }, + {}, + {'x-goog-api-client': 'test'}, + true, + telemetryInfo, + ); - let extractedTraceparent: string | undefined; const stubFunc = ( argument: {}, metadata: {}, options: {}, callback: Function, ): GRPCCallResult => { - const grpcMeta = metadata as Metadata; - const tp = grpcMeta.get('traceparent') as string[]; - extractedTraceparent = tp?.[0]; - - // Simulate low-level unary gRPC span started using the propagated context - const parentCtx = propagation.extract( - ROOT_CONTEXT, - grpcMeta, - grpcMetadataGetter, - ); - const unaryTracer = trace.getTracer('grpc-unary-transport'); - const unarySpan = unaryTracer.startSpan( - 'grpc.google.example.v1.Echo/Echo', - undefined, - parentCtx, - ); - unarySpan.end(); - callback(null, {echo: 'ok'}); return {cancel: () => {}}; }; - const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); + const apiCall = createApiCall( + stubFunc as unknown as GRPCCall, + defaults.echo, + ); await apiCall({message: 'hello'}, undefined); - const t3Span = harness.requireSingleSpan('google-gax'); - const unarySpan = harness.requireSingleSpan('grpc-unary-transport'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); - assert.strictEqual(t3Span.name, 'EchoClient.Echo'); - assert.strictEqual( - extractedTraceparent, - `00-${t3Span.spanContext().traceId}-${t3Span.spanContext().spanId}-0${t3Span.spanContext().traceFlags}`, - ); + const t4Span = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(t4Span); + assert.ok(t3Span); + + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t4Span.kind, SpanKind.CLIENT); assert.strictEqual( - unarySpan.spanContext().traceId, + t4Span.spanContext().traceId, t3Span.spanContext().traceId, ); assert.strictEqual( - unarySpan.parentSpanContext?.spanId, + t4Span.parentSpanContext?.spanId, t3Span.spanContext().spanId, ); + assert.strictEqual( + t4Span.attributes['url.domain'], + 'echo.googleapis.com', + ); + assert.strictEqual( + t4Span.attributes['server.address'], + 'echo.googleapis.com', + ); + assert.strictEqual(t4Span.attributes['server.port'], 443); + assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(t4Span.attributes['grpc.response.status_code'], 'OK'); }); - it('ties a low-level HTTP/REST unary trace to its parent T3 client request trace', async () => { - const settings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - metadataBuilder: (_abTests?: {}, moreHeaders?: {}) => ({ - 'x-goog-api-client': ['grpc-web/1.0'], - ...moreHeaders, - }), - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, + it('emits a T4 HTTP/REST attempt span parented to its T3 client request span', async () => { + const fallbackClient = new FallbackGrpcClient({ + servicePath: 'echo.googleapis.com', + port: 443, }); + const defaults = fallbackClient.constructSettings( + 'google.example.v1.Echo', + { + interfaces: { + 'google.example.v1.Echo': { + methods: { + Echo: {timeout_millis: 5000}, + }, + }, + }, + }, + {}, + {'x-goog-api-client': 'test'}, + true, + telemetryInfo, + ); - let extractedTraceparent: string | undefined; const stubFunc = ( argument: {}, metadata: {}, options: {}, callback: Function, ): GRPCCallResult => { - const httpMeta = metadata as Record; - extractedTraceparent = httpMeta['traceparent']; - - // Simulate low-level unary HTTP span started using the propagated headers - const parentCtx = propagation.extract(ROOT_CONTEXT, httpMeta); - const unaryTracer = trace.getTracer('http-unary-transport'); - const unarySpan = unaryTracer.startSpan( - 'HTTP POST /v1/echo', - undefined, - parentCtx, - ); - unarySpan.end(); - callback(null, {echo: 'ok'}); return {cancel: () => {}}; }; const apiCall = createApiCall( stubFunc as unknown as GRPCCall, - settings, + defaults.echo, undefined, 'rest', ); await apiCall({message: 'hello'}, undefined); - const t3Span = harness.requireSingleSpan('google-gax'); - const unarySpan = harness.requireSingleSpan('http-unary-transport'); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); - assert.strictEqual(t3Span.name, 'EchoClient.Echo'); - assert.strictEqual( - extractedTraceparent, - `00-${t3Span.spanContext().traceId}-${t3Span.spanContext().spanId}-0${t3Span.spanContext().traceFlags}`, - ); + const t4Span = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(t4Span); + assert.ok(t3Span); + + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t4Span.kind, SpanKind.CLIENT); assert.strictEqual( - unarySpan.spanContext().traceId, + t4Span.spanContext().traceId, t3Span.spanContext().traceId, ); assert.strictEqual( - unarySpan.parentSpanContext?.spanId, + t4Span.parentSpanContext?.spanId, t3Span.spanContext().spanId, ); + assert.strictEqual( + t4Span.attributes['url.domain'], + 'echo.googleapis.com', + ); + assert.strictEqual( + t4Span.attributes['server.address'], + 'echo.googleapis.com', + ); + assert.strictEqual(t4Span.attributes['server.port'], 443); + assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(t4Span.attributes['http.response.status_code'], 200); }); - it('ties concurrent low-level unary traces to their respective T3 client request traces without cross-talk', async () => { - const grpcClient = new GrpcClient(); - const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - + it('ties concurrent T4 attempt spans to their respective T3 client request spans without cross-talk', async () => { const echoSettings = new gax.CallSettings({ apiName: 'google.example.v1.Echo', enableTelemetryTracing: true, otherArgs: { - metadataBuilder: builder, internalTelemetryInfo: telemetryInfo, internalMethodName: 'Echo', }, @@ -416,91 +341,69 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { apiName: 'google.example.v1.Echo', enableTelemetryTracing: true, otherArgs: { - metadataBuilder: builder, internalTelemetryInfo: telemetryInfo, internalMethodName: 'Expand', }, }); - const makeStub = (unarySpanName: string, delayMs: number): GRPCCall => { + const makeStub = (delayMs: number): GRPCCall => { return (( argument: {}, metadata: {}, options: {}, callback: Function, ): GRPCCallResult => { - const grpcMeta = metadata as Metadata; - const parentCtx = propagation.extract( - ROOT_CONTEXT, - grpcMeta, - grpcMetadataGetter, - ); - const unaryTracer = trace.getTracer('grpc-unary-transport'); - const unarySpan = unaryTracer.startSpan( - unarySpanName, - undefined, - parentCtx, - ); setTimeout(() => { - unarySpan.end(); callback(null, {ok: true}); }, delayMs); return {cancel: () => {}}; }) as unknown as GRPCCall; }; - const echoCall = createApiCall(makeStub('unary.Echo', 15), echoSettings); - const expandCall = createApiCall( - makeStub('unary.Expand', 5), - expandSettings, - ); + const echoCall = createApiCall(makeStub(15), echoSettings); + const expandCall = createApiCall(makeStub(5), expandSettings); await Promise.all([ echoCall({id: 1}, undefined), expandCall({id: 2}, undefined), ]); - const t3Spans = harness.getSpans('google-gax'); - const unarySpans = harness.getSpans('grpc-unary-transport'); - assert.strictEqual(t3Spans.length, 2); - assert.strictEqual(unarySpans.length, 2); + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 4); - const t3Echo = t3Spans.find(s => s.name === 'EchoClient.Echo')!; - const t3Expand = t3Spans.find(s => s.name === 'EchoClient.Expand')!; - const unaryEcho = unarySpans.find(s => s.name === 'unary.Echo')!; - const unaryExpand = unarySpans.find(s => s.name === 'unary.Expand')!; + const t3Echo = spans.find(s => s.name === 'EchoClient.Echo')!; + const t3Expand = spans.find(s => s.name === 'EchoClient.Expand')!; + const t4Echo = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; + const t4Expand = spans.find( + s => s.name === 'google.example.v1.Echo/Expand', + )!; - assert.ok(t3Echo && t3Expand && unaryEcho && unaryExpand); + assert.ok(t3Echo && t3Expand && t4Echo && t4Expand); assert.notStrictEqual( t3Echo.spanContext().spanId, t3Expand.spanContext().spanId, ); - // unary.Echo must be tied to EchoClient.Echo assert.strictEqual( - unaryEcho.spanContext().traceId, + t4Echo.spanContext().traceId, t3Echo.spanContext().traceId, ); assert.strictEqual( - unaryEcho.parentSpanContext?.spanId, + t4Echo.parentSpanContext?.spanId, t3Echo.spanContext().spanId, ); - // unary.Expand must be tied to EchoClient.Expand assert.strictEqual( - unaryExpand.spanContext().traceId, + t4Expand.spanContext().traceId, t3Expand.spanContext().traceId, ); assert.strictEqual( - unaryExpand.parentSpanContext?.spanId, + t4Expand.parentSpanContext?.spanId, t3Expand.spanContext().spanId, ); }); - it('ties all retried low-level unary attempt traces to the single parent T3 client request trace', async () => { - const grpcClient = new GrpcClient(); - const builder = grpcClient.metadataBuilder({'x-goog-api-client': 'test'}); - + it('emits one T4 attempt span per retry attempt, all parented to the single T3 client request span', async () => { const retryOptions = gax.createRetryOptions( [status.UNAVAILABLE], gax.createBackoffSettings(1, 1.1, 5, 100, 1.0, 100, 1000), @@ -511,7 +414,6 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { retry: retryOptions, enableTelemetryTracing: true, otherArgs: { - metadataBuilder: builder, internalTelemetryInfo: telemetryInfo, internalMethodName: 'Echo', }, @@ -525,20 +427,6 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { callback: Function, ): GRPCCallResult => { attempt++; - const grpcMeta = metadata as Metadata; - const parentCtx = propagation.extract( - ROOT_CONTEXT, - grpcMeta, - grpcMetadataGetter, - ); - const unaryTracer = trace.getTracer('grpc-unary-transport'); - const unarySpan = unaryTracer.startSpan( - `unary.Echo.attempt.${attempt}`, - undefined, - parentCtx, - ); - unarySpan.end(); - if (attempt === 1) { const err = new GoogleError('transient failure'); err.code = status.UNAVAILABLE; @@ -552,20 +440,56 @@ describe('metadata propagation via TracerHelper and createApiCall', () => { const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); await apiCall({message: 'retry-me'}, undefined); - const t3Span = harness.requireSingleSpan('google-gax'); - const unarySpans = harness.getSpans('grpc-unary-transport'); - assert.strictEqual(unarySpans.length, 2); - - for (const unarySpan of unarySpans) { - assert.strictEqual( - unarySpan.spanContext().traceId, - t3Span.spanContext().traceId, - ); - assert.strictEqual( - unarySpan.parentSpanContext?.spanId, - t3Span.spanContext().spanId, - ); - } + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 3); + + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + const t4Spans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.ok(t3Span); + assert.strictEqual(t4Spans.length, 2); + + // First attempt failed with UNAVAILABLE + assert.strictEqual(t4Spans[0].kind, SpanKind.CLIENT); + assert.strictEqual(t4Spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual( + t4Spans[0].attributes['rpc.response.status_code'], + 'UNAVAILABLE', + ); + assert.strictEqual( + t4Spans[0].attributes['grpc.response.status_code'], + 'UNAVAILABLE', + ); + assert.strictEqual( + t4Spans[0].parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + + // Second attempt succeeded with OK + assert.strictEqual(t4Spans[1].kind, SpanKind.CLIENT); + assert.strictEqual(t4Spans[1].status.code, SpanStatusCode.UNSET); + assert.strictEqual( + t4Spans[1].attributes['rpc.response.status_code'], + 'OK', + ); + assert.strictEqual( + t4Spans[1].attributes['grpc.response.status_code'], + 'OK', + ); + assert.strictEqual( + t4Spans[1].parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + + // Overall T3 call span succeeded with resend_count = 1 + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t3Span.status.code, SpanStatusCode.UNSET); + assert.strictEqual(t3Span.attributes['gcp.grpc.resend_count'], 1); + assert.strictEqual(t3Span.attributes['rpc.response.status_code'], 'OK'); + + // traceCall export remains exercised + assert.strictEqual(typeof traceCall, 'function'); }); }); }); From c460b3386baa973f1caf8804457475d72a04dc1a Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 17:17:01 -0700 Subject: [PATCH 08/13] feat(gax): add resend_count to T4 spans and consolidate T4 unit tests --- core/packages/gax/src/createApiCall.ts | 4 +- .../gax/src/observability/TracerHelper.ts | 41 +- core/packages/gax/test/unit/apiCallable.ts | 384 +++++++++++++- .../gax/test/unit/metadataPropagation.ts | 495 ------------------ core/packages/gax/test/unit/tracerHelper.ts | 77 ++- 5 files changed, 489 insertions(+), 512 deletions(-) delete mode 100644 core/packages/gax/test/unit/metadataPropagation.ts diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index 8a507a7a1a15..a892273954e5 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -150,14 +150,16 @@ export function createApiCall( if (tracingEnabled && attemptDynamicArgs && staticArgs) { const wrappedFunc = func; + let attemptCount = 0; func = (( argument: {}, metadata: {}, options: {}, attemptCallback: APICallback, ) => { + const resendCount = attemptCount++; return traceAttempt( - attemptDynamicArgs, + {...attemptDynamicArgs, resendCount}, staticArgs, tracedAttemptCallback => (wrappedFunc as UnaryCall)( diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index 450f574a6f7e..14c5cdfba2e0 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -113,6 +113,11 @@ export interface AttemptTraceContext extends DynamicTraceContext { * The fully-qualified protobuf service name (e.g. 'google.cloud.kms.v1.KeyManagementService'). */ apiName?: string; + /** + * The ordinal resend count for this attempt (0 for the initial attempt, 1 for the first retry, etc.). + * Omitted from span attributes when 0 or undefined. + */ + resendCount?: number; } /** @@ -144,15 +149,16 @@ export function resolveErrorInfoReason(e: unknown): string | undefined { } // Decode binary gRPC status details if present and not yet parsed. + const errWithMeta = e as GoogleError; if ( - e instanceof GoogleError && - e.metadata && - typeof e.metadata.get === 'function' && - (e.metadata.get('grpc-status-details-bin') as unknown[])?.length > 0 && - !e.reason + errWithMeta.metadata && + typeof errWithMeta.metadata.get === 'function' && + (errWithMeta.metadata.get('grpc-status-details-bin') as unknown[])?.length > + 0 && + !errWithMeta.reason ) { try { - GoogleError.parseGRPCStatusDetails(e); + GoogleError.parseGRPCStatusDetails(errWithMeta); } catch { // Ignore decoding errors. } @@ -716,16 +722,17 @@ export function resolveServerExceptionDetails(e: Error): { }) : undefined; - // If e is a GoogleError with gRPC metadata that hasn't decoded statusDetails yet, parse it: + // If e has gRPC metadata that hasn't decoded statusDetails yet, parse it: + const errWithMeta = e as GoogleError; if ( !errObj.statusDetails && - e instanceof GoogleError && - e.metadata && - typeof e.metadata.get === 'function' && - (e.metadata.get('grpc-status-details-bin') as unknown[])?.length > 0 + errWithMeta.metadata && + typeof errWithMeta.metadata.get === 'function' && + (errWithMeta.metadata.get('grpc-status-details-bin') as unknown[])?.length > + 0 ) { try { - GoogleError.parseGRPCStatusDetails(e); + GoogleError.parseGRPCStatusDetails(errWithMeta); } catch { // Ignore decoding errors } @@ -1248,6 +1255,16 @@ export function traceAttempt( if (urlDomain !== undefined) { initialAttributes['url.domain'] = urlDomain; } + if ( + dynamicArgs.resendCount !== undefined && + dynamicArgs.resendCount > 0 + ) { + const resendCountAttribute = + dynamicArgs.rpcType === 'grpc' + ? 'gcp.grpc.resend_count' + : 'http.request.resend_count'; + initialAttributes[resendCountAttribute] = dynamicArgs.resendCount; + } span.setAttributes(initialAttributes); let rawAddress = diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index 4bd8ed9b73bc..77eb2f0dc94a 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -17,13 +17,22 @@ import assert from 'assert'; import {PassThrough} from 'stream'; import {status} from '@grpc/grpc-js'; -import {SpanKind} from '@opentelemetry/api'; +import {SpanKind, SpanStatusCode} from '@opentelemetry/api'; import {afterEach, beforeEach, describe, it} from 'mocha'; import * as sinon from 'sinon'; -import {CancellableStream, GRPCCall, RequestType} from '../../src/apitypes'; +import { + CancellableStream, + GRPCCall, + GRPCCallResult, + RequestType, +} from '../../src/apitypes'; import {createApiCall as gaxCreateApiCall} from '../../src/createApiCall'; -import {createApiCall as fallbackCreateApiCall} from '../../src/fallback'; +import { + createApiCall as fallbackCreateApiCall, + GrpcClient as FallbackGrpcClient, +} from '../../src/fallback'; +import {GrpcClient} from '../../src/grpc'; import {StreamDescriptor} from '../../src/descriptor'; import {StreamType} from '../../src/streamingCalls/streaming'; import * as gax from '../../src/gax'; @@ -1461,6 +1470,11 @@ describe('createApiCall', () => { transport.rpcType, ); harness.assertResendCount(0, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + harness.assertResendCount(0, {span: attemptSpans[0]}); }); it('reports one resend per retry', async () => { @@ -1499,6 +1513,13 @@ describe('createApiCall', () => { // off-by-one between the two is exactly what the attribute // defines. harness.assertResendCount(attempts - 1, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + attemptSpans.forEach((attemptSpan, idx) => { + harness.assertResendCount(idx, {span: attemptSpan}); + }); harness.assertResponseStatus( { rpcStatus: 'OK', @@ -1559,6 +1580,13 @@ describe('createApiCall', () => { ); // 2 attempts made: initial send + 1 resend. The 2nd retry was not sent because maxRetries was reached. harness.assertResendCount(1, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + attemptSpans.forEach((attemptSpan, idx) => { + harness.assertResendCount(idx, {span: attemptSpan}); + }); }); it('reports resends correctly when retries are exhausted by totalTimeoutMillis', async () => { @@ -1612,6 +1640,13 @@ describe('createApiCall', () => { // resend count should match the number of retries actually made // (attempts - 1), without counting the attempt aborted by the deadline. harness.assertResendCount(attempts - 1, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + attemptSpans.forEach((attemptSpan, idx) => { + harness.assertResendCount(idx, {span: attemptSpan}); + }); }); }); } @@ -1678,6 +1713,13 @@ describe('createApiCall', () => { assert.ok(span); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); harness.assertResendCount(2, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + attemptSpans.forEach((attemptSpan, idx) => { + harness.assertResendCount(idx, {span: attemptSpan}); + }); done(); } catch (e) { done(e); @@ -1734,6 +1776,13 @@ describe('createApiCall', () => { assert.ok(span); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); harness.assertResendCount(2, {span}); + const attemptSpans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(attemptSpans.length, attempts); + attemptSpans.forEach((attemptSpan, idx) => { + harness.assertResendCount(idx, {span: attemptSpan}); + }); done(); } catch (e) { done(e); @@ -1741,6 +1790,335 @@ describe('createApiCall', () => { }); }); }); + + describe('T3 client request span to T4 per-attempt span correlation', () => { + it('emits a T4 gRPC attempt span parented to its T3 client request span', async () => { + const grpcClient = new GrpcClient({ + servicePath: 'echo.googleapis.com', + port: 443, + }); + const defaults = grpcClient.constructSettings( + 'google.example.v1.Echo', + { + interfaces: { + 'google.example.v1.Echo': { + methods: { + Echo: {timeout_millis: 5000}, + }, + }, + }, + }, + {}, + {'x-goog-api-client': 'test'}, + true, + telemetryInfo, + ); + + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + callback(null, {echo: 'ok'}); + return {cancel: () => {}}; + }; + + const apiCall = gaxCreateApiCall( + stubFunc as unknown as GRPCCall, + defaults.echo, + ); + await apiCall({message: 'hello'}, undefined); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + + const t4Span = spans.find( + s => s.name === 'google.example.v1.Echo/Echo', + )!; + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(t4Span); + assert.ok(t3Span); + + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t4Span.kind, SpanKind.CLIENT); + assert.strictEqual( + t4Span.spanContext().traceId, + t3Span.spanContext().traceId, + ); + assert.strictEqual( + t4Span.parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + assert.strictEqual( + t4Span.attributes['url.domain'], + 'echo.googleapis.com', + ); + assert.strictEqual( + t4Span.attributes['server.address'], + 'echo.googleapis.com', + ); + assert.strictEqual(t4Span.attributes['server.port'], 443); + assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual( + t4Span.attributes['grpc.response.status_code'], + 'OK', + ); + }); + + it('emits a T4 HTTP/REST attempt span parented to its T3 client request span', async () => { + const fallbackClient = new FallbackGrpcClient({ + servicePath: 'echo.googleapis.com', + port: 443, + }); + const defaults = fallbackClient.constructSettings( + 'google.example.v1.Echo', + { + interfaces: { + 'google.example.v1.Echo': { + methods: { + Echo: {timeout_millis: 5000}, + }, + }, + }, + }, + {}, + {'x-goog-api-client': 'test'}, + true, + telemetryInfo, + ); + + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + callback(null, {echo: 'ok'}); + return {cancel: () => {}}; + }; + + const apiCall = gaxCreateApiCall( + stubFunc as unknown as GRPCCall, + defaults.echo, + undefined, + 'rest', + ); + await apiCall({message: 'hello'}, undefined); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + + const t4Span = spans.find( + s => s.name === 'google.example.v1.Echo/Echo', + )!; + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + assert.ok(t4Span); + assert.ok(t3Span); + + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t4Span.kind, SpanKind.CLIENT); + assert.strictEqual( + t4Span.spanContext().traceId, + t3Span.spanContext().traceId, + ); + assert.strictEqual( + t4Span.parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + assert.strictEqual( + t4Span.attributes['url.domain'], + 'echo.googleapis.com', + ); + assert.strictEqual( + t4Span.attributes['server.address'], + 'echo.googleapis.com', + ); + assert.strictEqual(t4Span.attributes['server.port'], 443); + assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(t4Span.attributes['http.response.status_code'], 200); + }); + + it('ties concurrent T4 attempt spans to their respective T3 client request spans without cross-talk', async () => { + const echoSettings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + const expandSettings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + enableTelemetryTracing: true, + otherArgs: { + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Expand', + }, + }); + + const makeStub = (delayMs: number): GRPCCall => { + return (( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + setTimeout(() => { + callback(null, {ok: true}); + }, delayMs); + return {cancel: () => {}}; + }) as unknown as GRPCCall; + }; + + const echoCall = gaxCreateApiCall(makeStub(15), echoSettings); + const expandCall = gaxCreateApiCall(makeStub(5), expandSettings); + + await Promise.all([ + echoCall({id: 1}, undefined), + expandCall({id: 2}, undefined), + ]); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 4); + + const t3Echo = spans.find(s => s.name === 'EchoClient.Echo')!; + const t3Expand = spans.find(s => s.name === 'EchoClient.Expand')!; + const t4Echo = spans.find( + s => s.name === 'google.example.v1.Echo/Echo', + )!; + const t4Expand = spans.find( + s => s.name === 'google.example.v1.Echo/Expand', + )!; + + assert.ok(t3Echo && t3Expand && t4Echo && t4Expand); + assert.notStrictEqual( + t3Echo.spanContext().spanId, + t3Expand.spanContext().spanId, + ); + + assert.strictEqual( + t4Echo.spanContext().traceId, + t3Echo.spanContext().traceId, + ); + assert.strictEqual( + t4Echo.parentSpanContext?.spanId, + t3Echo.spanContext().spanId, + ); + + assert.strictEqual( + t4Expand.spanContext().traceId, + t3Expand.spanContext().traceId, + ); + assert.strictEqual( + t4Expand.parentSpanContext?.spanId, + t3Expand.spanContext().spanId, + ); + }); + + it('emits one T4 attempt span per retry attempt, all parented to the single T3 client request span', async () => { + const retryOptions = gax.createRetryOptions( + [status.UNAVAILABLE], + gax.createBackoffSettings(1, 1.1, 5, 100, 1.0, 100, 1000), + ); + + const settings = new gax.CallSettings({ + apiName: 'google.example.v1.Echo', + retry: retryOptions, + enableTelemetryTracing: true, + otherArgs: { + internalTelemetryInfo: telemetryInfo, + internalMethodName: 'Echo', + }, + }); + + let attempt = 0; + const stubFunc = ( + argument: {}, + metadata: {}, + options: {}, + callback: Function, + ): GRPCCallResult => { + attempt++; + if (attempt === 1) { + const err = new GoogleError('transient failure'); + err.code = status.UNAVAILABLE; + callback(err); + } else { + callback(null, {echo: 'recovered'}); + } + return {cancel: () => {}}; + }; + + const apiCall = gaxCreateApiCall( + stubFunc as unknown as GRPCCall, + settings, + ); + await apiCall({message: 'retry-me'}, undefined); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 3); + + const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; + const t4Spans = spans.filter( + s => s.name === 'google.example.v1.Echo/Echo', + ); + assert.ok(t3Span); + assert.strictEqual(t4Spans.length, 2); + + // First attempt failed with UNAVAILABLE (resend_count omitted on initial attempt) + assert.strictEqual(t4Spans[0].kind, SpanKind.CLIENT); + assert.strictEqual(t4Spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(t4Spans[0].status.message, 'transient failure'); + assert.strictEqual( + t4Spans[0].attributes['gcp.grpc.resend_count'], + undefined, + ); + assert.strictEqual(t4Spans[0].attributes['error.type'], 'UNAVAILABLE'); + assert.strictEqual( + t4Spans[0].attributes['rpc.response.status_code'], + 'UNAVAILABLE', + ); + assert.strictEqual( + t4Spans[0].attributes['grpc.response.status_code'], + 'UNAVAILABLE', + ); + assert.strictEqual(t4Spans[0].events.length, 1); + assert.strictEqual(t4Spans[0].events[0].name, 'exception'); + assert.strictEqual( + t4Spans[0].events[0].attributes?.['exception.type'], + 'GoogleError', + ); + assert.strictEqual( + t4Spans[0].parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + + // Second attempt succeeded with OK (resend_count = 1) + assert.strictEqual(t4Spans[1].kind, SpanKind.CLIENT); + assert.strictEqual(t4Spans[1].status.code, SpanStatusCode.UNSET); + assert.strictEqual(t4Spans[1].attributes['gcp.grpc.resend_count'], 1); + assert.strictEqual( + t4Spans[1].attributes['rpc.response.status_code'], + 'OK', + ); + assert.strictEqual( + t4Spans[1].attributes['grpc.response.status_code'], + 'OK', + ); + assert.strictEqual( + t4Spans[1].parentSpanContext?.spanId, + t3Span.spanContext().spanId, + ); + + // Overall T3 call span succeeded with resend_count = 1 + assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); + assert.strictEqual(t3Span.status.code, SpanStatusCode.UNSET); + assert.strictEqual(t3Span.attributes['gcp.grpc.resend_count'], 1); + assert.strictEqual(t3Span.attributes['rpc.response.status_code'], 'OK'); + }); + }); }); }); diff --git a/core/packages/gax/test/unit/metadataPropagation.ts b/core/packages/gax/test/unit/metadataPropagation.ts deleted file mode 100644 index e2422e39719f..000000000000 --- a/core/packages/gax/test/unit/metadataPropagation.ts +++ /dev/null @@ -1,495 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as assert from 'assert'; -import {describe, it, beforeEach, afterEach} from 'mocha'; -import {SpanKind, SpanStatusCode} from '@opentelemetry/api'; -import {status} from '@grpc/grpc-js'; -import {createApiCall} from '../../src/createApiCall'; -import * as gax from '../../src/gax'; -import {GoogleError} from '../../src/googleError'; -import {GrpcClient, Metadata} from '../../src/grpc'; -import {GrpcClient as FallbackGrpcClient} from '../../src/fallback'; -import { - traceCall, - traceAttempt, - AttemptTraceContext, - StaticTraceContext, -} from '../../src/observability/TracerHelper'; -import {clearMetadataCache} from '../../src/observability/metadataResolver'; -import {OtelHarness} from './otelHarness'; -import {GRPCCall, GRPCCallResult} from '../../src/apitypes'; - -describe('T4 per-attempt spans via TracerHelper and createApiCall', () => { - let harness: OtelHarness; - - const telemetryInfo: StaticTraceContext = { - gcpClientService: 'echo.googleapis.com', - gcpVersion: '1.2.3', - gcpRepo: 'googleapis/google-cloud-node', - gcpArtifact: '@google-cloud/echo', - }; - - beforeEach(() => { - clearMetadataCache(); - process.env.GOOGLE_SDK_NODE_ENABLE_TRACING = 'true'; - harness = new OtelHarness(); - harness.setup(); - }); - - afterEach(() => { - clearMetadataCache(); - delete process.env.GOOGLE_SDK_NODE_ENABLE_TRACING; - harness.teardown(); - }); - - it('creates a CLIENT T4 span in traceAttempt with url.domain, server.address, server.port, and status_code', async () => { - const attemptArgs: AttemptTraceContext = { - apiName: 'google.example.v1.Echo', - clientName: 'EchoClient', - methodName: 'Echo', - rpcType: 'grpc', - }; - - await traceAttempt(attemptArgs, telemetryInfo, async () => { - return [{echo: 'ok'}, undefined, undefined]; - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); - assert.strictEqual(span.kind, SpanKind.CLIENT); - assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); - assert.strictEqual( - span.attributes['server.address'], - 'echo.googleapis.com', - ); - assert.strictEqual(span.attributes['server.port'], 443); - assert.strictEqual(span.attributes['rpc.system'], 'grpc'); - assert.strictEqual(span.attributes['rpc.response.status_code'], 'OK'); - assert.strictEqual(span.attributes['grpc.response.status_code'], 'OK'); - }); - - it('omits server.address and server.port on T4 span for pre-connection failures while preserving url.domain', async () => { - const attemptArgs: AttemptTraceContext = { - apiName: 'google.example.v1.Echo', - clientName: 'EchoClient', - methodName: 'Echo', - rpcType: 'grpc', - }; - - const dnsError = Object.assign(new Error('getaddrinfo ENOTFOUND'), { - code: 'ENOTFOUND', - }); - - await assert.rejects(async () => { - await traceAttempt(attemptArgs, telemetryInfo, async () => { - throw dnsError; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); - assert.strictEqual(span.kind, SpanKind.CLIENT); - assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); - assert.strictEqual(span.attributes['server.address'], undefined); - assert.strictEqual(span.attributes['server.port'], undefined); - assert.strictEqual( - span.attributes['error.type'], - 'CLIENT_CONNECTION_ERROR', - ); - }); - - it('does not inject traceparent headers into gRPC or HTTP metadata', async () => { - const grpcClient = new GrpcClient(); - const grpcBuilder = grpcClient.metadataBuilder({ - 'x-goog-api-client': 'test', - }); - - const receivedGrpcMetadata: Metadata[] = []; - const receivedHttpMetadata: Record[] = []; - - const grpcStub = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - receivedGrpcMetadata.push(meta as Metadata); - cb(null, {}); - return {cancel: () => {}}; - }; - - const httpStub = ( - arg: {}, - meta: {}, - opt: {}, - cb: Function, - ): GRPCCallResult => { - receivedHttpMetadata.push(meta as Record); - cb(null, {}); - return {cancel: () => {}}; - }; - - const grpcCall = createApiCall( - grpcStub as unknown as GRPCCall, - new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - metadataBuilder: grpcBuilder, - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }), - ); - - const httpCall = createApiCall( - httpStub as unknown as GRPCCall, - new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - metadataBuilder: (_abTests?: {}, moreHeaders?: {}) => ({ - 'x-goog-api-client': ['grpc-web/1.0'], - ...moreHeaders, - }), - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }), - undefined, - 'rest', - ); - - await grpcCall({}, undefined); - await httpCall({}, undefined); - - assert.strictEqual(receivedGrpcMetadata.length, 1); - assert.strictEqual( - (receivedGrpcMetadata[0].get('traceparent') as unknown[]).length, - 0, - ); - assert.strictEqual(receivedHttpMetadata.length, 1); - assert.strictEqual(receivedHttpMetadata[0]['traceparent'], undefined); - }); - - describe('T3 client request span to T4 per-attempt span correlation', () => { - it('emits a T4 gRPC attempt span parented to its T3 client request span', async () => { - const grpcClient = new GrpcClient({ - servicePath: 'echo.googleapis.com', - port: 443, - }); - const defaults = grpcClient.constructSettings( - 'google.example.v1.Echo', - { - interfaces: { - 'google.example.v1.Echo': { - methods: { - Echo: {timeout_millis: 5000}, - }, - }, - }, - }, - {}, - {'x-goog-api-client': 'test'}, - true, - telemetryInfo, - ); - - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - callback(null, {echo: 'ok'}); - return {cancel: () => {}}; - }; - - const apiCall = createApiCall( - stubFunc as unknown as GRPCCall, - defaults.echo, - ); - await apiCall({message: 'hello'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 2); - - const t4Span = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; - const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; - assert.ok(t4Span); - assert.ok(t3Span); - - assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); - assert.strictEqual(t4Span.kind, SpanKind.CLIENT); - assert.strictEqual( - t4Span.spanContext().traceId, - t3Span.spanContext().traceId, - ); - assert.strictEqual( - t4Span.parentSpanContext?.spanId, - t3Span.spanContext().spanId, - ); - assert.strictEqual( - t4Span.attributes['url.domain'], - 'echo.googleapis.com', - ); - assert.strictEqual( - t4Span.attributes['server.address'], - 'echo.googleapis.com', - ); - assert.strictEqual(t4Span.attributes['server.port'], 443); - assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); - assert.strictEqual(t4Span.attributes['grpc.response.status_code'], 'OK'); - }); - - it('emits a T4 HTTP/REST attempt span parented to its T3 client request span', async () => { - const fallbackClient = new FallbackGrpcClient({ - servicePath: 'echo.googleapis.com', - port: 443, - }); - const defaults = fallbackClient.constructSettings( - 'google.example.v1.Echo', - { - interfaces: { - 'google.example.v1.Echo': { - methods: { - Echo: {timeout_millis: 5000}, - }, - }, - }, - }, - {}, - {'x-goog-api-client': 'test'}, - true, - telemetryInfo, - ); - - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - callback(null, {echo: 'ok'}); - return {cancel: () => {}}; - }; - - const apiCall = createApiCall( - stubFunc as unknown as GRPCCall, - defaults.echo, - undefined, - 'rest', - ); - await apiCall({message: 'hello'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 2); - - const t4Span = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; - const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; - assert.ok(t4Span); - assert.ok(t3Span); - - assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); - assert.strictEqual(t4Span.kind, SpanKind.CLIENT); - assert.strictEqual( - t4Span.spanContext().traceId, - t3Span.spanContext().traceId, - ); - assert.strictEqual( - t4Span.parentSpanContext?.spanId, - t3Span.spanContext().spanId, - ); - assert.strictEqual( - t4Span.attributes['url.domain'], - 'echo.googleapis.com', - ); - assert.strictEqual( - t4Span.attributes['server.address'], - 'echo.googleapis.com', - ); - assert.strictEqual(t4Span.attributes['server.port'], 443); - assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); - assert.strictEqual(t4Span.attributes['http.response.status_code'], 200); - }); - - it('ties concurrent T4 attempt spans to their respective T3 client request spans without cross-talk', async () => { - const echoSettings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }); - - const expandSettings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - enableTelemetryTracing: true, - otherArgs: { - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Expand', - }, - }); - - const makeStub = (delayMs: number): GRPCCall => { - return (( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - setTimeout(() => { - callback(null, {ok: true}); - }, delayMs); - return {cancel: () => {}}; - }) as unknown as GRPCCall; - }; - - const echoCall = createApiCall(makeStub(15), echoSettings); - const expandCall = createApiCall(makeStub(5), expandSettings); - - await Promise.all([ - echoCall({id: 1}, undefined), - expandCall({id: 2}, undefined), - ]); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 4); - - const t3Echo = spans.find(s => s.name === 'EchoClient.Echo')!; - const t3Expand = spans.find(s => s.name === 'EchoClient.Expand')!; - const t4Echo = spans.find(s => s.name === 'google.example.v1.Echo/Echo')!; - const t4Expand = spans.find( - s => s.name === 'google.example.v1.Echo/Expand', - )!; - - assert.ok(t3Echo && t3Expand && t4Echo && t4Expand); - assert.notStrictEqual( - t3Echo.spanContext().spanId, - t3Expand.spanContext().spanId, - ); - - assert.strictEqual( - t4Echo.spanContext().traceId, - t3Echo.spanContext().traceId, - ); - assert.strictEqual( - t4Echo.parentSpanContext?.spanId, - t3Echo.spanContext().spanId, - ); - - assert.strictEqual( - t4Expand.spanContext().traceId, - t3Expand.spanContext().traceId, - ); - assert.strictEqual( - t4Expand.parentSpanContext?.spanId, - t3Expand.spanContext().spanId, - ); - }); - - it('emits one T4 attempt span per retry attempt, all parented to the single T3 client request span', async () => { - const retryOptions = gax.createRetryOptions( - [status.UNAVAILABLE], - gax.createBackoffSettings(1, 1.1, 5, 100, 1.0, 100, 1000), - ); - - const settings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - retry: retryOptions, - enableTelemetryTracing: true, - otherArgs: { - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }); - - let attempt = 0; - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - attempt++; - if (attempt === 1) { - const err = new GoogleError('transient failure'); - err.code = status.UNAVAILABLE; - callback(err); - } else { - callback(null, {echo: 'recovered'}); - } - return {cancel: () => {}}; - }; - - const apiCall = createApiCall(stubFunc as unknown as GRPCCall, settings); - await apiCall({message: 'retry-me'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 3); - - const t3Span = spans.find(s => s.name === 'EchoClient.Echo')!; - const t4Spans = spans.filter( - s => s.name === 'google.example.v1.Echo/Echo', - ); - assert.ok(t3Span); - assert.strictEqual(t4Spans.length, 2); - - // First attempt failed with UNAVAILABLE - assert.strictEqual(t4Spans[0].kind, SpanKind.CLIENT); - assert.strictEqual(t4Spans[0].status.code, SpanStatusCode.ERROR); - assert.strictEqual( - t4Spans[0].attributes['rpc.response.status_code'], - 'UNAVAILABLE', - ); - assert.strictEqual( - t4Spans[0].attributes['grpc.response.status_code'], - 'UNAVAILABLE', - ); - assert.strictEqual( - t4Spans[0].parentSpanContext?.spanId, - t3Span.spanContext().spanId, - ); - - // Second attempt succeeded with OK - assert.strictEqual(t4Spans[1].kind, SpanKind.CLIENT); - assert.strictEqual(t4Spans[1].status.code, SpanStatusCode.UNSET); - assert.strictEqual( - t4Spans[1].attributes['rpc.response.status_code'], - 'OK', - ); - assert.strictEqual( - t4Spans[1].attributes['grpc.response.status_code'], - 'OK', - ); - assert.strictEqual( - t4Spans[1].parentSpanContext?.spanId, - t3Span.spanContext().spanId, - ); - - // Overall T3 call span succeeded with resend_count = 1 - assert.strictEqual(t3Span.kind, SpanKind.INTERNAL); - assert.strictEqual(t3Span.status.code, SpanStatusCode.UNSET); - assert.strictEqual(t3Span.attributes['gcp.grpc.resend_count'], 1); - assert.strictEqual(t3Span.attributes['rpc.response.status_code'], 'OK'); - - // traceCall export remains exercised - assert.strictEqual(typeof traceCall, 'function'); - }); - }); -}); diff --git a/core/packages/gax/test/unit/tracerHelper.ts b/core/packages/gax/test/unit/tracerHelper.ts index 647fae36b01a..f90498aa8f5d 100644 --- a/core/packages/gax/test/unit/tracerHelper.ts +++ b/core/packages/gax/test/unit/tracerHelper.ts @@ -18,14 +18,16 @@ import * as assert from 'assert'; import * as vm from 'vm'; import {EventEmitter} from 'events'; import {Duplex, Writable} from 'stream'; -import {SpanStatusCode, trace} from '@opentelemetry/api'; +import {SpanKind, SpanStatusCode, trace} from '@opentelemetry/api'; import {describe, it, beforeEach, afterEach} from 'mocha'; import * as grpc from '@grpc/grpc-js'; import { getGaxTracer, traceCall, + traceAttempt, handlePromise, handleStream, + AttemptTraceContext, DynamicTraceContext, StaticTraceContext, resolveErrorInfoReason, @@ -3825,4 +3827,77 @@ describe('TracerHelper', () => { assertListenersRestored(baseline, 'after stream end'); }); }); + + describe('traceAttempt', () => { + const telemetryInfo: StaticTraceContext = { + gcpClientService: 'echo.googleapis.com', + gcpVersion: '1.2.3', + gcpRepo: 'googleapis/google-cloud-node', + gcpArtifact: '@google-cloud/echo', + }; + + it('creates a CLIENT T4 span in traceAttempt with url.domain, server.address, server.port, and status_code', async () => { + const attemptArgs: AttemptTraceContext = { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'grpc', + }; + + await traceAttempt(attemptArgs, telemetryInfo, async () => { + return [{echo: 'ok'}, undefined, undefined]; + }); + + const span = harness.requireSingleSpan('google-gax'); + assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(span.kind, SpanKind.CLIENT); + assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); + assert.strictEqual( + span.attributes['server.address'], + 'echo.googleapis.com', + ); + assert.strictEqual(span.attributes['server.port'], 443); + assert.strictEqual(span.attributes['rpc.system'], 'grpc'); + assert.strictEqual(span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(span.attributes['grpc.response.status_code'], 'OK'); + }); + + it('omits server.address and server.port on T4 span for pre-connection failures while preserving url.domain', async () => { + const attemptArgs: AttemptTraceContext = { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'grpc', + }; + + const dnsError = Object.assign(new Error('getaddrinfo ENOTFOUND'), { + code: 'ENOTFOUND', + }); + + await assert.rejects(async () => { + await traceAttempt(attemptArgs, telemetryInfo, async () => { + throw dnsError; + }); + }); + + const span = harness.requireSingleSpan('google-gax'); + assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); + assert.strictEqual(span.kind, SpanKind.CLIENT); + assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); + assert.strictEqual(span.attributes['server.address'], undefined); + assert.strictEqual(span.attributes['server.port'], undefined); + assert.strictEqual( + span.attributes['error.type'], + 'CLIENT_CONNECTION_ERROR', + ); + assert.strictEqual(span.status.code, SpanStatusCode.ERROR); + assert.strictEqual(span.status.message, 'getaddrinfo ENOTFOUND'); + assert.strictEqual(span.events.length, 1); + assert.strictEqual(span.events[0].name, 'exception'); + assert.strictEqual( + span.events[0].attributes?.['exception.type'], + 'Error', + ); + }); + }); }); From 396dcd031f072750b7c7cfaeb35db1dd6ae86d90 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 17:53:09 -0700 Subject: [PATCH 09/13] feat(gax): refine T3 and T4 span method, status code, and client attributes --- core/packages/gax/src/fallbackServiceStub.ts | 3 + .../gax/src/observability/TracerHelper.ts | 44 ++++--- core/packages/gax/test/unit/apiCallable.ts | 63 +++++++--- core/packages/gax/test/unit/otelHarness.ts | 42 ++++--- core/packages/gax/test/unit/tracerHelper.ts | 116 ++++++++++++++++-- 5 files changed, 207 insertions(+), 61 deletions(-) diff --git a/core/packages/gax/src/fallbackServiceStub.ts b/core/packages/gax/src/fallbackServiceStub.ts index ad196dc19324..2816076d1b22 100644 --- a/core/packages/gax/src/fallbackServiceStub.ts +++ b/core/packages/gax/src/fallbackServiceStub.ts @@ -25,6 +25,7 @@ import {isNodeJS} from './featureDetection'; import {StreamArrayParser} from './streamArrayParser'; import {defaultToObjectOptions} from './fallback'; import {GoogleError} from './googleError'; +import {setAttemptHttpMethod} from './observability/TracerHelper'; import {rpcCodeFromHttpStatusCode, Status} from './status'; import {pipeline, PipelineSource} from 'stream'; import type {Agent as HttpAgent} from 'http'; @@ -302,6 +303,8 @@ export function generateServiceStub( }; } + setAttemptHttpMethod(fetchParameters.method); + const cancelController = new AbortController(); const cancelSignal = cancelController.signal as AbortSignal; let cancelRequested = false; diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index 14c5cdfba2e0..ae5214faaf5d 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -19,6 +19,7 @@ import { Attributes, context, Context, + createContextKey, Span, SpanKind, SpanStatusCode, @@ -118,6 +119,23 @@ export interface AttemptTraceContext extends DynamicTraceContext { * Omitted from span attributes when 0 or undefined. */ resendCount?: number; + /** + * The HTTP request method for REST fallback attempts (e.g. 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'). + */ + httpMethod?: string; +} + +const ATTEMPT_SPAN_KEY = createContextKey('google-gax-attempt-span'); + +/** + * Updates the `http.request.method` attribute on the currently active T4 attempt span, if any. + */ +export function setAttemptHttpMethod(httpMethod: string): void { + const attemptSpan = context.active().getValue(ATTEMPT_SPAN_KEY) as + Span | undefined; + if (attemptSpan) { + attemptSpan.setAttribute('http.request.method', httpMethod); + } } /** @@ -1119,11 +1137,8 @@ export function traceCall( const setStatusAttributes = () => { const attributes: Attributes = {}; - if (rpcStatusName !== undefined) { + if (dynamicArgs.rpcType === 'grpc' && rpcStatusName !== undefined) { attributes['rpc.response.status_code'] = rpcStatusName; - if (dynamicArgs.rpcType === 'grpc') { - attributes['grpc.response.status_code'] = rpcStatusName; - } } if (dynamicArgs.rpcType === 'http' && httpStatusCode !== undefined) { attributes['http.response.status_code'] = httpStatusCode; @@ -1245,13 +1260,14 @@ export function traceAttempt( const urlDomain = resolveUrlDomain(dynamicArgs, staticArgs); const initialAttributes: Attributes = { 'gcp.client.service': staticArgs.gcpClientService, - 'gcp.client.version': staticArgs.gcpVersion, - 'gcp.repo': staticArgs.gcpRepo, - 'gcp.artifact': staticArgs.gcpArtifact, - 'gcp.method.name': dynamicArgs.methodName, - 'gcp.method.type': dynamicArgs.rpcType, 'rpc.system': dynamicArgs.rpcType, }; + if (dynamicArgs.rpcType === 'grpc') { + initialAttributes['rpc.method'] = spanName; + } else { + initialAttributes['http.request.method'] = + dynamicArgs.httpMethod ?? 'POST'; + } if (urlDomain !== undefined) { initialAttributes['url.domain'] = urlDomain; } @@ -1292,11 +1308,8 @@ export function traceAttempt( const setStatusAttributes = () => { const attributes: Attributes = {}; - if (rpcStatusName !== undefined) { + if (dynamicArgs.rpcType === 'grpc' && rpcStatusName !== undefined) { attributes['rpc.response.status_code'] = rpcStatusName; - if (dynamicArgs.rpcType === 'grpc') { - attributes['grpc.response.status_code'] = rpcStatusName; - } } if (dynamicArgs.rpcType === 'http' && httpStatusCode !== undefined) { attributes['http.response.status_code'] = httpStatusCode; @@ -1352,7 +1365,10 @@ export function traceAttempt( : undefined; try { - const result = fn(tracedCallback); + const attemptContext = context + .active() + .setValue(ATTEMPT_SPAN_KEY, span); + const result = context.with(attemptContext, () => fn(tracedCallback)); const promiseTarget = !isStreamCall ? getPromiseTarget(result) : null; if (isStreamCall && result instanceof EventEmitter) { handleStream(result, recordError, endSpan, !!callback); diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index 77eb2f0dc94a..54708ca7b101 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -738,7 +738,8 @@ describe('createApiCall', () => { const span = spans[1]; assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); - assert.strictEqual(attemptSpan.attributes['gcp.method.type'], 'http'); + assert.strictEqual(attemptSpan.attributes['rpc.system'], 'http'); + assert.strictEqual(attemptSpan.attributes['gcp.method.type'], undefined); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['gcp.method.type'], 'http'); @@ -780,7 +781,7 @@ describe('createApiCall', () => { const attemptSpan = spans[0]; const span = spans[1]; assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); - assert.strictEqual(attemptSpan.attributes['gcp.method.type'], 'http'); + assert.strictEqual(attemptSpan.attributes['rpc.system'], 'http'); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['gcp.method.type'], 'http'); @@ -877,16 +878,21 @@ describe('createApiCall', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['rpc.system'], 'http'); + assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); for (const span of spans) { assert.strictEqual(span.ended, true); - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); // On the fallback transport error.type reports the HTTP status the // server sent. A deadline expires before any response arrives, so there // is none, and the attribute resolves to CLIENT_TIMEOUT per Tier 3. assert.strictEqual(span.attributes['error.type'], 'CLIENT_TIMEOUT'); assert.strictEqual( span.attributes['rpc.response.status_code'], - 'DEADLINE_EXCEEDED', + undefined, + ); + assert.strictEqual( + span.attributes['http.response.status_code'], + undefined, ); assert.strictEqual(span.events.length, 1); assert.strictEqual(span.events[0].name, 'exception'); @@ -939,16 +945,17 @@ describe('createApiCall', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['rpc.system'], 'http'); + assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); for (const span of spans) { assert.strictEqual(span.ended, true); - assert.strictEqual(span.attributes['gcp.method.type'], 'http'); assert.strictEqual( span.attributes['error.type'], 'CLIENT_CONNECTION_ERROR', ); assert.strictEqual( span.attributes['rpc.response.status_code'], - 'UNAVAILABLE', + undefined, ); assert.strictEqual( span.attributes['http.response.status_code'], @@ -1021,7 +1028,7 @@ describe('createApiCall', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); - assert.strictEqual(spans[0].attributes['gcp.method.type'], 'grpc'); + assert.strictEqual(spans[0].attributes['rpc.system'], 'grpc'); assert.strictEqual(spans[1].attributes['gcp.method.type'], 'grpc'); }); @@ -1052,7 +1059,7 @@ describe('createApiCall', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); - assert.strictEqual(spans[0].attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans[0].attributes['rpc.system'], 'http'); assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); }); @@ -1083,7 +1090,7 @@ describe('createApiCall', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); - assert.strictEqual(spans[0].attributes['gcp.method.type'], 'http'); + assert.strictEqual(spans[0].attributes['rpc.system'], 'http'); assert.strictEqual(spans[1].attributes['gcp.method.type'], 'http'); }); @@ -1521,10 +1528,9 @@ describe('createApiCall', () => { harness.assertResendCount(idx, {span: attemptSpan}); }); harness.assertResponseStatus( - { - rpcStatus: 'OK', - ...(transport.rpcType === 'http' ? {httpStatus: 200} : {}), - }, + transport.rpcType === 'http' + ? {httpStatus: 200} + : {rpcStatus: 'OK'}, {span}, ); }); @@ -1859,10 +1865,19 @@ describe('createApiCall', () => { 'echo.googleapis.com', ); assert.strictEqual(t4Span.attributes['server.port'], 443); + assert.strictEqual( + t4Span.attributes['rpc.method'], + 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(t4Span.attributes['http.request.method'], undefined); assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); assert.strictEqual( t4Span.attributes['grpc.response.status_code'], - 'OK', + undefined, + ); + assert.strictEqual( + t4Span.attributes['http.response.status_code'], + undefined, ); }); @@ -1935,8 +1950,18 @@ describe('createApiCall', () => { 'echo.googleapis.com', ); assert.strictEqual(t4Span.attributes['server.port'], 443); - assert.strictEqual(t4Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual(t4Span.attributes['http.request.method'], 'POST'); + assert.strictEqual(t4Span.attributes['rpc.method'], undefined); + assert.strictEqual( + t4Span.attributes['rpc.response.status_code'], + undefined, + ); assert.strictEqual(t4Span.attributes['http.response.status_code'], 200); + assert.strictEqual( + t3Span.attributes['rpc.response.status_code'], + undefined, + ); + assert.strictEqual(t3Span.attributes['http.response.status_code'], 200); }); it('ties concurrent T4 attempt spans to their respective T3 client request spans without cross-talk', async () => { @@ -2082,7 +2107,7 @@ describe('createApiCall', () => { ); assert.strictEqual( t4Spans[0].attributes['grpc.response.status_code'], - 'UNAVAILABLE', + undefined, ); assert.strictEqual(t4Spans[0].events.length, 1); assert.strictEqual(t4Spans[0].events[0].name, 'exception'); @@ -2105,7 +2130,7 @@ describe('createApiCall', () => { ); assert.strictEqual( t4Spans[1].attributes['grpc.response.status_code'], - 'OK', + undefined, ); assert.strictEqual( t4Spans[1].parentSpanContext?.spanId, @@ -2117,6 +2142,10 @@ describe('createApiCall', () => { assert.strictEqual(t3Span.status.code, SpanStatusCode.UNSET); assert.strictEqual(t3Span.attributes['gcp.grpc.resend_count'], 1); assert.strictEqual(t3Span.attributes['rpc.response.status_code'], 'OK'); + assert.strictEqual( + t3Span.attributes['grpc.response.status_code'], + undefined, + ); }); }); }); diff --git a/core/packages/gax/test/unit/otelHarness.ts b/core/packages/gax/test/unit/otelHarness.ts index 21a6fae262f3..dd53cbcc090c 100644 --- a/core/packages/gax/test/unit/otelHarness.ts +++ b/core/packages/gax/test/unit/otelHarness.ts @@ -296,7 +296,8 @@ export class OtelHarness { ): void { const target = options.span ?? this.requireSingleSpan(options.tracerName); const actual = this.responseStatus(target); - const transport = target.attributes['gcp.method.type']; + const transport = + target.attributes['gcp.method.type'] ?? target.attributes['rpc.system']; const where = `span '${target.name}'`; if ('serverAddress' in expected || 'serverPort' in expected) { @@ -314,26 +315,21 @@ export class OtelHarness { ); assert.strictEqual( - actual.rpc, - expected.rpcStatus, - expected.rpcStatus === undefined - ? `expected ${where} to report no rpc.response.status_code, got ` + - `${JSON.stringify(actual.rpc)}. Response status is omitted when there is no server response.` - : `expected ${where} to report rpc.response.status_code ` + - `${JSON.stringify(expected.rpcStatus)}, got ${JSON.stringify(actual.rpc)}. ` + - 'This attribute is reported on every call with a server response, on both transports.', + actual.grpc, + undefined, + `${where} reported grpc.response.status_code ${JSON.stringify(actual.grpc)}; ` + + 'only rpc.response.status_code or http.response.status_code should be set.', ); if (transport === 'grpc') { assert.strictEqual( - actual.grpc, + actual.rpc, expected.rpcStatus, expected.rpcStatus === undefined - ? `expected ${where} to report no grpc.response.status_code, got ` + - `${JSON.stringify(actual.grpc)}. Response status is omitted when there is no server response.` - : `expected ${where} to report grpc.response.status_code ` + - `${JSON.stringify(expected.rpcStatus)}, got ${JSON.stringify(actual.grpc)}. ` + - 'On a gRPC span it mirrors rpc.response.status_code.', + ? `expected ${where} to report no rpc.response.status_code, got ` + + `${JSON.stringify(actual.rpc)}. Response status is omitted when there is no server response.` + : `expected ${where} to report rpc.response.status_code ` + + `${JSON.stringify(expected.rpcStatus)}, got ${JSON.stringify(actual.rpc)}.`, ); assert.strictEqual( actual.http, @@ -353,11 +349,16 @@ export class OtelHarness { } assert.strictEqual( - actual.grpc, + actual.rpc, + undefined, + `${where} is a fallback span but reported rpc.response.status_code ` + + `${JSON.stringify(actual.rpc)}. HTTP spans only report http.response.status_code.`, + ); + assert.strictEqual( + expected.rpcStatus, undefined, - `${where} is a fallback span but reported grpc.response.status_code ` + - `${JSON.stringify(actual.grpc)}. The gRPC status is reported as ` + - 'rpc.response.status_code there, not under the grpc.* name.', + 'assertResponseStatus was given an expected rpcStatus for an HTTP ' + + 'span, which can never hold one. Use httpStatus instead.', ); assert.strictEqual( actual.http, @@ -431,7 +432,8 @@ export class OtelHarness { options: {tracerName?: string; span?: ReadableSpan} = {}, ): void { const target = options.span ?? this.requireSingleSpan(options.tracerName); - const transport = target.attributes['gcp.method.type']; + const transport = + target.attributes['gcp.method.type'] ?? target.attributes['rpc.system']; const where = `span '${target.name}'`; assert.ok( diff --git a/core/packages/gax/test/unit/tracerHelper.ts b/core/packages/gax/test/unit/tracerHelper.ts index f90498aa8f5d..aa317456f2e6 100644 --- a/core/packages/gax/test/unit/tracerHelper.ts +++ b/core/packages/gax/test/unit/tracerHelper.ts @@ -25,6 +25,7 @@ import { getGaxTracer, traceCall, traceAttempt, + setAttemptHttpMethod, handlePromise, handleStream, AttemptTraceContext, @@ -417,7 +418,7 @@ describe('TracerHelper', () => { await errorTypeOf(error, httpDynamicArgs), 'CLIENT_TIMEOUT', ); - harness.assertResponseStatus({rpcStatus: 'DEADLINE_EXCEEDED'}); + harness.assertResponseStatus({}); }); it('prefers a system error code to the class on either transport', async () => { @@ -450,7 +451,7 @@ describe('TracerHelper', () => { await errorTypeOf(error, httpDynamicArgs), 'CLIENT_CONNECTION_ERROR', ); - harness.assertResponseStatus({rpcStatus: 'UNAVAILABLE'}); + harness.assertResponseStatus({}); }); it('checks e.cause when the outer error is a GoogleError', async () => { @@ -1994,10 +1995,10 @@ describe('TracerHelper', () => { harness.assertResponseStatus({rpcStatus: 'OK'}); }); - it('reports OK and 200 on a successful http call', async () => { + it('reports 200 on a successful http call', async () => { await traceCall(httpDynamicArgs, staticArgs, async () => 'ok'); - harness.assertResponseStatus({rpcStatus: 'OK', httpStatus: 200}); + harness.assertResponseStatus({httpStatus: 200}); }); it('reports the gRPC status name on a failed grpc call', async () => { @@ -2015,7 +2016,7 @@ describe('TracerHelper', () => { assert.strictEqual(spans[0].status.message, '5 NOT_FOUND'); }); - it('reports the received http status alongside the mapped gRPC status', async () => { + it('reports the received http status on a failed http call', async () => { // 418 is unmapped, so rpcCodeFromHttpStatusCode collapses it to // FAILED_PRECONDITION. The received status is therefore not // recoverable from `code`, which is why it is carried separately. @@ -2030,7 +2031,6 @@ describe('TracerHelper', () => { }); harness.assertResponseStatus({ - rpcStatus: 'FAILED_PRECONDITION', httpStatus: 418, }); const spans = harness.getSpans('google-gax'); @@ -2050,7 +2050,7 @@ describe('TracerHelper', () => { }); }); - harness.assertResponseStatus({rpcStatus: 'DEADLINE_EXCEEDED'}); + harness.assertResponseStatus({}); const spans = harness.getSpans('google-gax'); assert.strictEqual(spans[0].status.message, 'Deadline exceeded'); }); @@ -2145,7 +2145,7 @@ describe('TracerHelper', () => { }); }); - harness.assertResponseStatus({rpcStatus: undefined}); + harness.assertResponseStatus({}); const span = harness.requireSingleSpan('google-gax'); assert.strictEqual(span.status.message, 'client-side http error'); assert.strictEqual( @@ -2255,7 +2255,7 @@ describe('TracerHelper', () => { const span = harness.requireSingleSpan('google-gax'); assert.strictEqual(span.attributes['error.type'], 'NOT_FOUND'); assert.strictEqual( - span.attributes['grpc.response.status_code'], + span.attributes['rpc.response.status_code'], 'NOT_FOUND', ); }); @@ -3858,8 +3858,65 @@ describe('TracerHelper', () => { ); assert.strictEqual(span.attributes['server.port'], 443); assert.strictEqual(span.attributes['rpc.system'], 'grpc'); + assert.strictEqual( + span.attributes['rpc.method'], + 'google.example.v1.Echo/Echo', + ); + assert.strictEqual(span.attributes['http.request.method'], undefined); assert.strictEqual(span.attributes['rpc.response.status_code'], 'OK'); - assert.strictEqual(span.attributes['grpc.response.status_code'], 'OK'); + assert.strictEqual( + span.attributes['grpc.response.status_code'], + undefined, + ); + assert.strictEqual(span.attributes['gcp.repo'], undefined); + assert.strictEqual(span.attributes['gcp.method.type'], undefined); + assert.strictEqual(span.attributes['gcp.method.name'], undefined); + assert.strictEqual(span.attributes['gcp.client.version'], undefined); + assert.strictEqual(span.attributes['gcp.artifact'], undefined); + }); + + it('sets http.request.method on HTTP T4 spans and allows setAttemptHttpMethod to update it', async () => { + await traceAttempt( + { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'http', + }, + telemetryInfo, + async () => [{echo: 'ok'}, undefined, undefined], + ); + + await traceAttempt( + { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'http', + }, + telemetryInfo, + async () => { + setAttemptHttpMethod('GET'); + return [{echo: 'ok'}, undefined, undefined]; + }, + ); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['http.request.method'], 'POST'); + assert.strictEqual(spans[0].attributes['rpc.method'], undefined); + assert.strictEqual(spans[0].attributes['http.response.status_code'], 200); + assert.strictEqual( + spans[0].attributes['rpc.response.status_code'], + undefined, + ); + assert.strictEqual(spans[1].attributes['http.request.method'], 'GET'); + assert.strictEqual(spans[1].attributes['rpc.method'], undefined); + assert.strictEqual(spans[1].attributes['http.response.status_code'], 200); + assert.strictEqual( + spans[1].attributes['rpc.response.status_code'], + undefined, + ); }); it('omits server.address and server.port on T4 span for pre-connection failures while preserving url.domain', async () => { @@ -3899,5 +3956,44 @@ describe('TracerHelper', () => { 'Error', ); }); + + it('sets gcp.grpc.resend_count and http.request.resend_count on T4 spans when resendCount > 0', async () => { + await traceAttempt( + { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'grpc', + resendCount: 2, + }, + telemetryInfo, + async () => [{echo: 'ok'}, undefined, undefined], + ); + + await traceAttempt( + { + apiName: 'google.example.v1.Echo', + clientName: 'EchoClient', + methodName: 'Echo', + rpcType: 'http', + resendCount: 3, + }, + telemetryInfo, + async () => [{echo: 'ok'}, undefined, undefined], + ); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].attributes['gcp.grpc.resend_count'], 2); + assert.strictEqual( + spans[0].attributes['http.request.resend_count'], + undefined, + ); + assert.strictEqual(spans[1].attributes['http.request.resend_count'], 3); + assert.strictEqual( + spans[1].attributes['gcp.grpc.resend_count'], + undefined, + ); + }); }); }); From 0a1dd071cb17344ad1d3f8acf74cef1951efd37a Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 18:00:31 -0700 Subject: [PATCH 10/13] refactor(gax): narrow dynamicArgs and staticArgs before calling traceCall --- core/packages/gax/src/createApiCall.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index a892273954e5..8aa740b675d9 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -243,7 +243,7 @@ export function createApiCall( return currentApiCaller.result(ongoingCall); }; - if (tracingEnabled) { + if (tracingEnabled && dynamicArgs && staticArgs) { const isStreamingCall = apiCaller instanceof StreamingApiCaller; return ( request: RequestType, @@ -251,8 +251,8 @@ export function createApiCall( callback?: APICallback, ) => { return traceCall( - dynamicArgs!, - staticArgs!, + dynamicArgs, + staticArgs, (tracedCallback?: APICallback, recordResend?: ResendRecorder) => { // `traceCall` wraps the user's callback whenever one was supplied, // for stream and non-stream calls alike, and that wrapper is what From 0c600fa482cf2c2647e627cee4a0d46ffe17c9ef Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 18:53:46 -0700 Subject: [PATCH 11/13] docs(gax): add concise comments to traceAttempt and createApiCall --- core/packages/gax/src/createApiCall.ts | 3 +++ core/packages/gax/src/observability/TracerHelper.ts | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index 8aa740b675d9..78ecaf06f9e7 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -93,6 +93,7 @@ export function createApiCall( rpcType: isFallback ? 'http' : 'grpc', } : undefined; + // Context for per-attempt T4 CLIENT spans. const attemptDynamicArgs: AttemptTraceContext | undefined = tracingEnabled && dynamicArgs ? { @@ -107,6 +108,7 @@ export function createApiCall( callback?: APICallback, recordResend?: ResendRecorder, ) => { + // Capture the active T3 call span context to parent async T4 attempt spans. const parentContext = tracingEnabled ? context.active() : undefined; let currentApiCaller = apiCaller; @@ -148,6 +150,7 @@ export function createApiCall( const streaming = (currentApiCaller as StreamingApiCaller).descriptor ?.streaming; + // Wrap the transport call so each attempt (initial send and retries) emits a T4 CLIENT span. if (tracingEnabled && attemptDynamicArgs && staticArgs) { const wrappedFunc = func; let attemptCount = 0; diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index ae5214faaf5d..918b2806c177 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -1257,6 +1257,7 @@ export function traceAttempt( {kind: SpanKind.CLIENT}, baseContext, (span: Span) => { + // Populate initial transport, method, domain, and retry attributes. const urlDomain = resolveUrlDomain(dynamicArgs, staticArgs); const initialAttributes: Attributes = { 'gcp.client.service': staticArgs.gcpClientService, @@ -1283,6 +1284,7 @@ export function traceAttempt( } span.setAttributes(initialAttributes); + // Parse server address and port, defaulting to port 443. let rawAddress = dynamicArgs.serverAddress ?? staticArgs.serverAddress ?? urlDomain; let rawPort = dynamicArgs.serverPort ?? staticArgs.serverPort; @@ -1306,6 +1308,7 @@ export function traceAttempt( span.setStatus({code: SpanStatusCode.ERROR, message}); }; + // Record response status and server endpoint (omitted on pre-connection failures). const setStatusAttributes = () => { const attributes: Attributes = {}; if (dynamicArgs.rpcType === 'grpc' && rpcStatusName !== undefined) { @@ -1326,6 +1329,7 @@ export function traceAttempt( span.setAttributes(attributes); }; + // Finalize status attributes and end the span once. const endSpan = () => { if (!spanEnded) { spanEnded = true; @@ -1338,6 +1342,7 @@ export function traceAttempt( } }; + // Capture error type, exception event, and span error status on failure. const recordError = (e: unknown) => { recordedError = e; rpcStatusName = resolveRpcStatusName(e); @@ -1365,6 +1370,7 @@ export function traceAttempt( : undefined; try { + // Expose the attempt span in context so the HTTP transport can update http.request.method. const attemptContext = context .active() .setValue(ATTEMPT_SPAN_KEY, span); From 41949c4a06e2716c3bb3e18e39a5f27324e81ec2 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 18:58:19 -0700 Subject: [PATCH 12/13] feat(gax): set url.domain on T3 spans --- core/packages/gax/src/observability/TracerHelper.ts | 9 +++++++-- core/packages/gax/test/unit/apiCallable.ts | 1 + core/packages/gax/test/unit/tracerHelper.ts | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index 918b2806c177..438b15109929 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -1074,14 +1074,19 @@ export function traceCall( ): GaxCallResult { const spanName = `${dynamicArgs.clientName}.${dynamicArgs.methodName}`; return getGaxTracer().startActiveSpan(spanName, {}, (span: Span) => { - span.setAttributes({ + const urlDomain = resolveUrlDomain(dynamicArgs, staticArgs); + const initialAttributes: Attributes = { 'gcp.client.service': staticArgs.gcpClientService, 'gcp.client.version': staticArgs.gcpVersion, 'gcp.repo': staticArgs.gcpRepo, 'gcp.artifact': staticArgs.gcpArtifact, 'gcp.method.name': dynamicArgs.methodName, 'gcp.method.type': dynamicArgs.rpcType, - }); + }; + if (urlDomain !== undefined) { + initialAttributes['url.domain'] = urlDomain; + } + span.setAttributes(initialAttributes); let rawAddress = dynamicArgs.serverAddress ?? staticArgs.serverAddress; let rawPort = dynamicArgs.serverPort ?? staticArgs.serverPort; diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index 54708ca7b101..6182c82890b9 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -617,6 +617,7 @@ describe('createApiCall', () => { assert.strictEqual(span.attributes['gcp.artifact'], '@google-cloud/echo'); assert.strictEqual(span.attributes['gcp.method.name'], 'Echo'); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); + assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); }); it('enables tracing purely through GOOGLE_SDK_NODE_ENABLE_TRACING and resolves static metadata dynamically at runtime', async () => { diff --git a/core/packages/gax/test/unit/tracerHelper.ts b/core/packages/gax/test/unit/tracerHelper.ts index aa317456f2e6..4a1af5769dc2 100644 --- a/core/packages/gax/test/unit/tracerHelper.ts +++ b/core/packages/gax/test/unit/tracerHelper.ts @@ -122,6 +122,10 @@ describe('TracerHelper', () => { ); assert.strictEqual(span.attributes['gcp.method.name'], 'GetObject'); assert.strictEqual(span.attributes['gcp.method.type'], 'grpc'); + assert.strictEqual( + span.attributes['url.domain'], + 'storage.googleapis.com', + ); // A successful call reports no error.type, and leaves the status unset // rather than claiming OK on the application's behalf. assert.strictEqual(span.attributes['error.type'], undefined); From 2707dac816f7d3bfa225797773b9d2fd8a0e043d Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 1 Oct 2026 22:12:31 -0700 Subject: [PATCH 13/13] Wrap function call to avoid variable mutation and memory leaks --- core/packages/gax/src/createApiCall.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index 78ecaf06f9e7..9d016891774a 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -145,16 +145,16 @@ export function createApiCall( funcPromise .then((func: GRPCCall) => { // Initially, the function is just what gRPC server stub contains. - func = currentApiCaller.wrap(func); + let wrappedFunc = currentApiCaller.wrap(func); const streaming = (currentApiCaller as StreamingApiCaller).descriptor ?.streaming; // Wrap the transport call so each attempt (initial send and retries) emits a T4 CLIENT span. if (tracingEnabled && attemptDynamicArgs && staticArgs) { - const wrappedFunc = func; + const callerWrappedFunc = wrappedFunc; let attemptCount = 0; - func = (( + wrappedFunc = (( argument: {}, metadata: {}, options: {}, @@ -165,7 +165,7 @@ export function createApiCall( {...attemptDynamicArgs, resendCount}, staticArgs, tracedAttemptCallback => - (wrappedFunc as UnaryCall)( + (callerWrappedFunc as UnaryCall)( argument, metadata, options, @@ -213,7 +213,7 @@ export function createApiCall( retry.backoffSettings.initialRpcTimeoutMillis ??= thisSettings.timeout; return retryable( - func, + wrappedFunc, thisSettings.retry!, thisSettings.otherArgs as GRPCCallOtherArgs, thisSettings.apiName, @@ -222,7 +222,7 @@ export function createApiCall( } } return addTimeoutArg( - func, + wrappedFunc, thisSettings.timeout, thisSettings.otherArgs as GRPCCallOtherArgs, );