From 8dbe59d5d2a13820eaef57a6efb6d32b64ac1c9e Mon Sep 17 00:00:00 2001 From: Wilson La Date: Fri, 17 Jul 2026 12:29:07 -0700 Subject: [PATCH 1/4] Prevent DNS watcher from clearing endpoints on lookup failure Signed-off-by: Wilson La --- pkg/util/grpcutil/dns_resolver.go | 8 ++- pkg/util/grpcutil/dns_resolver_test.go | 80 ++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 pkg/util/grpcutil/dns_resolver_test.go diff --git a/pkg/util/grpcutil/dns_resolver.go b/pkg/util/grpcutil/dns_resolver.go index ef9c6398944..1dff5a9b422 100644 --- a/pkg/util/grpcutil/dns_resolver.go +++ b/pkg/util/grpcutil/dns_resolver.go @@ -244,6 +244,12 @@ func (w *dnsWatcher) lookup() []*Update { // return any A record info available. newAddrs = w.lookupHost() } + if newAddrs == nil { + // Both the SRV and A record lookups failed. Keep the previously + // resolved addresses cached instead of removing them, so a transient + // DNS failure doesn't drop all currently known endpoints. + return nil + } result := w.compileUpdate(newAddrs) w.curAddrs = newAddrs return result @@ -261,7 +267,7 @@ func (w *dnsWatcher) Next() ([]*Update, error) { result := w.lookup() // Next lookup should happen after an interval defined by w.r.freq. w.t.Reset(w.r.freq) - if len(result) > 0 { + if result != nil && len(result) > 0 { return result, nil } } diff --git a/pkg/util/grpcutil/dns_resolver_test.go b/pkg/util/grpcutil/dns_resolver_test.go new file mode 100644 index 00000000000..9098c067ebd --- /dev/null +++ b/pkg/util/grpcutil/dns_resolver_test.go @@ -0,0 +1,80 @@ +package grpcutil + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func stubLookups(t *testing.T, host func(ctx context.Context, host string) ([]string, error)) { + t.Helper() + orig := lookupHost + lookupHost = host + t.Cleanup(func() { lookupHost = orig }) +} + +func newTestDNSWatcher() *dnsWatcher { + ctx, cancel := context.WithCancel(context.Background()) + return &dnsWatcher{ + r: &Resolver{freq: time.Hour}, + logger: log.NewNopLogger(), + host: "myhost", + port: "80", + ctx: ctx, + cancel: cancel, + t: time.NewTimer(0), + } +} + +func TestDNSWatcher_Next(t *testing.T) { + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"1.2.3.4"}, nil + }) + + w := newTestDNSWatcher() + updates, err := w.Next() + require.NoError(t, err) + assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, updates) + assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) + + // Lookup returns only 5.6.7.8, removing 1.2.3.4 + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"5.6.7.8"}, nil + }) + + // Fire the timer again so Next() performs another lookup immediately. + w.t.Reset(0) + updates, err = w.Next() + require.NoError(t, err) + assert.ElementsMatch(t, []*Update{ + {Op: Delete, Addr: "1.2.3.4:80"}, + {Op: Add, Addr: "5.6.7.8:80"}, + }, updates) + assert.Equal(t, map[string]*Update{"5.6.7.8:80": {Addr: "5.6.7.8:80"}}, w.curAddrs) +} + +func TestDNSWatcher_Lookup_TransientFailureRetainsCache(t *testing.T) { + // First resolution succeeds and populates curAddrs. + stubLookups(t, func(context.Context, string) ([]string, error) { + return []string{"1.2.3.4"}, nil + }) + + w := newTestDNSWatcher() + result := w.lookup() + assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, result) + require.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) + + // Fail next lookup + stubLookups(t, func(context.Context, string) ([]string, error) { + return nil, errors.New("unable to resolve address") + }) + + result = w.lookup() + assert.Nil(t, result) + assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs) +} From a7aefb8ee77a99770b4bf08f7ee33807d5244388 Mon Sep 17 00:00:00 2001 From: Wilson La Date: Fri, 17 Jul 2026 13:03:07 -0700 Subject: [PATCH 2/4] Update CHANGELOG Signed-off-by: Wilson La --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da309a0d683..7b54845d18c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638 * [BUGFIX] Storage: Default the Azure `endpoint_suffix` to `blob.core.windows.net` instead of empty. Cortex builds the Thanos Azure config directly and bypasses Thanos' default, so an unset suffix produced an invalid FQDN (`.`) and components hung on startup with DNS errors. #5449 * [BUGFIX] Store Gateway: Fix misleading "no index cache backend addresses" validation error being reported for chunks-cache, metadata-cache, and parquet caches when their memcached or redis backend is configured without addresses. The message is now the cache-type-agnostic "no cache backend addresses". #7675 +* [BUGFIX] DNS: Fix DNS watcher from dropping resolved addresses on transient lookup failures. #7698 ## 1.21.0 2026-04-24 From b0d30aaf751bf93be55e61c0b3544f4d5d71ade4 Mon Sep 17 00:00:00 2001 From: Wilson La Date: Fri, 17 Jul 2026 13:12:07 -0700 Subject: [PATCH 3/4] Fix lint Signed-off-by: Wilson La --- pkg/util/grpcutil/dns_resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/grpcutil/dns_resolver.go b/pkg/util/grpcutil/dns_resolver.go index 1dff5a9b422..b99f0a0e98d 100644 --- a/pkg/util/grpcutil/dns_resolver.go +++ b/pkg/util/grpcutil/dns_resolver.go @@ -267,7 +267,7 @@ func (w *dnsWatcher) Next() ([]*Update, error) { result := w.lookup() // Next lookup should happen after an interval defined by w.r.freq. w.t.Reset(w.r.freq) - if result != nil && len(result) > 0 { + if len(result) > 0 { return result, nil } } From 5d954fa7ffe3e3b086c7f9a425b8cbaa498ed6b8 Mon Sep 17 00:00:00 2001 From: Wilson La Date: Mon, 20 Jul 2026 08:33:54 -0700 Subject: [PATCH 4/4] Update CHANGELOG.md Co-authored-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Signed-off-by: Wilson La --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b54845d18c..5580f21b037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,7 @@ * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638 * [BUGFIX] Storage: Default the Azure `endpoint_suffix` to `blob.core.windows.net` instead of empty. Cortex builds the Thanos Azure config directly and bypasses Thanos' default, so an unset suffix produced an invalid FQDN (`.`) and components hung on startup with DNS errors. #5449 * [BUGFIX] Store Gateway: Fix misleading "no index cache backend addresses" validation error being reported for chunks-cache, metadata-cache, and parquet caches when their memcached or redis backend is configured without addresses. The message is now the cache-type-agnostic "no cache backend addresses". #7675 -* [BUGFIX] DNS: Fix DNS watcher from dropping resolved addresses on transient lookup failures. #7698 +* [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698 ## 1.21.0 2026-04-24