diff --git a/helm/polaris/templates/deployment.yaml b/helm/polaris/templates/deployment.yaml index 7b3a6cd9515..e85934f68c8 100644 --- a/helm/polaris/templates/deployment.yaml +++ b/helm/polaris/templates/deployment.yaml @@ -80,7 +80,7 @@ spec: {{- end }} image: "{{ tpl .Values.image.repository . }}:{{ tpl .Values.image.tag . | default .Chart.Version }}" imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} - {{ if or .Values.storage.secret.name .Values.persistence.relationalJdbc.secret.name .Values.persistence.nosql.secret.name .Values.oidc.client.secret.name .Values.extraEnv -}} + {{ if or .Values.storage.secret.name .Values.persistence.relationalJdbc.secret.name .Values.persistence.nosql.secret.name .Values.oidc.client.secret.name (not .Values.tracing.enabled) .Values.tracing.disabledResourceProviders .Values.extraEnv -}} env: {{- include "polaris.secretToEnv" (list .Values.storage.secret "awsAccessKeyId" "polaris.storage.aws.access-key") | indent 12 -}} {{- include "polaris.secretToEnv" (list .Values.storage.secret "awsSecretAccessKey" "polaris.storage.aws.secret-key") | indent 12 -}} @@ -90,6 +90,10 @@ spec: {{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "jdbcUrl" "quarkus.datasource.jdbc.url") | indent 12 -}} {{- include "polaris.secretToEnv" (list .Values.persistence.nosql.secret "connectionString" "quarkus.mongodb.connection-string") | indent 12 -}} {{- include "polaris.secretToEnv" (list .Values.oidc.client.secret "key" "quarkus.oidc.credentials.secret") | indent 12 -}} + {{- if or (not .Values.tracing.enabled) .Values.tracing.disabledResourceProviders }} + - name: OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS + value: {{ default "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" .Values.tracing.disabledResourceProviders | quote }} + {{- end }} {{- if .Values.extraEnv -}} {{- tpl (toYaml .Values.extraEnv) . | nindent 12 -}} {{- end -}} diff --git a/helm/polaris/tests/deployment_test.yaml b/helm/polaris/tests/deployment_test.yaml index d2afd89f216..5db2116d543 100644 --- a/helm/polaris/tests/deployment_test.yaml +++ b/helm/polaris/tests/deployment_test.yaml @@ -382,11 +382,36 @@ tests: value: Always # spec.template.spec.containers[0].env - - it: should not set container env by default + - it: should set OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS by default when tracing is disabled template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS + value: "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" + - it: should not set OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS when tracing is enabled + template: deployment.yaml + set: + tracing: + enabled: true + endpoint: "http://otlp-collector:4317" asserts: - notExists: path: spec.template.spec.containers[0].env + - it: should set OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS when tracing is enabled and disabledResourceProviders is set + template: deployment.yaml + set: + tracing: + enabled: true + endpoint: "http://otlp-collector:4317" + disabledResourceProviders: "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS + value: "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" - it: should set container env template: deployment.yaml set: @@ -1144,11 +1169,17 @@ tests: configMapRef: name: polaris-env-configmap - - it: should not set any environment variables by default + - it: should set OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS as the only env var by default template: deployment.yaml asserts: - - notExists: + - lengthEqual: + path: spec.template.spec.containers[0].env + count: 1 + - contains: path: spec.template.spec.containers[0].env + content: + name: OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS + value: "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" - it: should not set envFrom by default template: deployment.yaml diff --git a/helm/polaris/values.yaml b/helm/polaris/values.yaml index 2428c2b996b..d6a438b5a6a 100644 --- a/helm/polaris/values.yaml +++ b/helm/polaris/values.yaml @@ -717,6 +717,12 @@ tracing: attributes: # service.name: my-polaris {} + # -- Comma-separated OTel resource provider class names to disable. + # When tracing is disabled (the default), the GCP resource detector is disabled + # automatically to prevent a ~135s startup stall on non-GCP Kubernetes clusters. + # Set this explicitly to override when tracing is enabled but the cluster is non-GCP. + # @section -- Observability + disabledResourceProviders: "" metrics: # -- Specifies whether metrics for the polaris server should be enabled. See [Metrics](https://polaris.apache.org/in-dev/unreleased/telemetry/#metrics) for details. diff --git a/site/content/in-dev/unreleased/telemetry.md b/site/content/in-dev/unreleased/telemetry.md index 59a9b763ee6..5ad2924ab67 100644 --- a/site/content/in-dev/unreleased/telemetry.md +++ b/site/content/in-dev/unreleased/telemetry.md @@ -117,6 +117,34 @@ Finally, two additional span attributes are added to all request parent spans: - `polaris.realm`: The unique identifier of the realm. Always set (unless the request failed because of a realm resolution error). +### Slow startup in non-GCP Kubernetes clusters + +Polaris carries the GCP resource detector (`GCPResourceProvider`) on its classpath via a transitive +dependency on GCS support. OpenTelemetry's autoconfigure runs resource providers during Quarkus +startup even when tracing is disabled, so if `metadata.google.internal` resolves but silently drops +packets (common in kind, k3d, minikube, and OrbStack Kubernetes), the `main` thread blocks for +~135s waiting for a TCP connect timeout. Under the Helm chart's default liveness probe the pod +crash-loops before startup completes. + +**Symptom:** no `started in` log line for ~135s, with no error or stack trace. A `SIGQUIT` thread +dump shows `GCPMetadataConfig.fetchAttribute` on the `main` thread. + +**Helm chart:** when `tracing.enabled: false` (the default), the chart automatically sets +`OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS` to skip the GCP detector. No action is needed. + +If you enable tracing but run on a non-GCP cluster, add the following to your Helm values: + +```yaml +tracing: + disabledResourceProviders: "io.opentelemetry.contrib.gcp.resource.GCPResourceProvider" +``` + +Or set the environment variable directly: + +```shell +OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS=io.opentelemetry.contrib.gcp.resource.GCPResourceProvider +``` + ### Troubleshooting Traces If the server is unable to publish traces, check first for a log warning message like the following: