diff --git a/CHANGELOG.md b/CHANGELOG.md index 3287db55cad..c49e031e5e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638 * [BUGFIX] Storage: Default the Azure `endpoint_suffix` to `blob.core.windows.net` instead of empty. Cortex builds the Thanos Azure config directly and bypasses Thanos' default, so an unset suffix produced an invalid FQDN (`.`) and components hung on startup with DNS errors. #5449 * [BUGFIX] Store Gateway: Fix misleading "no index cache backend addresses" validation error being reported for chunks-cache, metadata-cache, and parquet caches when their memcached or redis backend is configured without addresses. The message is now the cache-type-agnostic "no cache backend addresses". #7675 +* [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698 * [BUGFIX] Ring: Fix DynamoDB KV CAS not retrying on transactional conditional check failures. `TransactWriteItems` reports condition failures as `TransactionCanceledException` with a `ConditionalCheckFailed` cancellation reason, which was not recognized as retryable, so any concurrent ring update conflict (e.g. many ingesters joining during a rolling update) failed immediately instead of re-reading and retrying. `TransactionConflict` cancellation reasons are also treated as retryable. #7706 ## 1.21.0 2026-04-24 diff --git a/pkg/util/grpcutil/dns_resolver.go b/pkg/util/grpcutil/dns_resolver.go index ef9c6398944..b99f0a0e98d 100644 --- a/pkg/util/grpcutil/dns_resolver.go +++ b/pkg/util/grpcutil/dns_resolver.go @@ -244,6 +244,12 @@ func (w *dnsWatcher) lookup() []*Update { // return any A record info available. newAddrs = w.lookupHost() } + if newAddrs == nil { + // Both the SRV and A record lookups failed. Keep the previously + // resolved addresses cached instead of removing them, so a transient + // DNS failure doesn't drop all currently known endpoints. + return nil + } result := w.compileUpdate(newAddrs) w.curAddrs = newAddrs return result diff --git a/pkg/util/grpcutil/dns_resolver_test.go b/pkg/util/grpcutil/dns_resolver_test.go new file mode 100644 index 00000000000..9098c067ebd --- /dev/null +++ b/pkg/util/grpcutil/dns_resolver_test.go @@ -0,0 +1,80 @@ +package grpcutil + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func stubLookups(t *testing.T, host func(ctx context.Context, host string) ([]string, error)) { + t.Helper() + orig := lookupHost + lookupHost = host + t.Cleanup(func() { lookupHost = orig }) +} + +func newTestDNSWatcher() *dnsWatcher { + ctx, cancel := context.WithCancel(context.Background()) + return &dnsWatcher{ + r: &Resolver{freq: time.Hour}, + logger: log.NewNopLogger(), + host: "myhost", + port: "80", + ctx: ctx, + cancel: cancel, + t: time.NewTimer(0), + } +} + +func TestDNSWatcher_Next(t *testing.T) { + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"1.2.3.4"}, nil + }) + + w := newTestDNSWatcher() + updates, err := w.Next() + require.NoError(t, err) + assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, updates) + assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) + + // Lookup returns only 5.6.7.8, removing 1.2.3.4 + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"5.6.7.8"}, nil + }) + + // Fire the timer again so Next() performs another lookup immediately. + w.t.Reset(0) + updates, err = w.Next() + require.NoError(t, err) + assert.ElementsMatch(t, []*Update{ + {Op: Delete, Addr: "1.2.3.4:80"}, + {Op: Add, Addr: "5.6.7.8:80"}, + }, updates) + assert.Equal(t, map[string]*Update{"5.6.7.8:80": {Addr: "5.6.7.8:80"}}, w.curAddrs) +} + +func TestDNSWatcher_Lookup_TransientFailureRetainsCache(t *testing.T) { + // First resolution succeeds and populates curAddrs. + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"1.2.3.4"}, nil + }) + + w := newTestDNSWatcher() + result := w.lookup() + assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, result) + require.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) + + // Fail next lookup + stubLookups(t, func(context.Context, string) ([]string, error) { + return nil, errors.New("unable to resolve address") + }) + + result = w.lookup() + assert.Nil(t, result) + assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) +}