Fix DNS watcher from clearing known endpoints on lookup failure

CHANGELOG.md

@@ -82,6 +82,7 @@
 * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638
 * [BUGFIX] Storage: Default the Azure `endpoint_suffix` to `blob.core.windows.net` instead of empty. Cortex builds the Thanos Azure config directly and bypasses Thanos' default, so an unset suffix produced an invalid FQDN (`<account>.`) and components hung on startup with DNS errors. #5449
 * [BUGFIX] Store Gateway: Fix misleading "no index cache backend addresses" validation error being reported for chunks-cache, metadata-cache, and parquet caches when their memcached or redis backend is configured without addresses. The message is now the cache-type-agnostic "no cache backend addresses". #7675
+* [BUGFIX] DNS: Fix DNS watcher from dropping resolved addresses on transient lookup failures. #7698
 
 ## 1.21.0 2026-04-24
 

pkg/util/grpcutil/dns_resolver.go

@@ -244,6 +244,12 @@ func (w *dnsWatcher) lookup() []*Update {
 		// return any A record info available.
 		newAddrs = w.lookupHost()
 	}
+	if newAddrs == nil {
+		// Both the SRV and A record lookups failed. Keep the previously
+		// resolved addresses cached instead of removing them, so a transient
+		// DNS failure doesn't drop all currently known endpoints.
+		return nil
+	}
 	result := w.compileUpdate(newAddrs)
 	w.curAddrs = newAddrs
 	return result

pkg/util/grpcutil/dns_resolver_test.go

@@ -0,0 +1,80 @@
+package grpcutil
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stubLookups(t *testing.T, host func(ctx context.Context, host string) ([]string, error)) {
+	t.Helper()
+	orig := lookupHost
+	lookupHost = host
+	t.Cleanup(func() { lookupHost = orig })
+}
+
+func newTestDNSWatcher() *dnsWatcher {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &dnsWatcher{
+		r:      &Resolver{freq: time.Hour},
+		logger: log.NewNopLogger(),
+		host:   "myhost",
+		port:   "80",
+		ctx:    ctx,
+		cancel: cancel,
+		t:      time.NewTimer(0),
+	}
+}
+
+func TestDNSWatcher_Next(t *testing.T) {
+	stubLookups(t, func(context.Context, string) ([]string, error) {
+		return []string{"1.2.3.4"}, nil
+	})
+
+	w := newTestDNSWatcher()
+	updates, err := w.Next()
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, updates)
+	assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs)
+
+	// Lookup returns only 5.6.7.8, removing 1.2.3.4
+	stubLookups(t, func(context.Context, string) ([]string, error) {
+		return []string{"5.6.7.8"}, nil
+	})
+
+	// Fire the timer again so Next() performs another lookup immediately.
+	w.t.Reset(0)
+	updates, err = w.Next()
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []*Update{
+		{Op: Delete, Addr: "1.2.3.4:80"},
+		{Op: Add, Addr: "5.6.7.8:80"},
+	}, updates)
+	assert.Equal(t, map[string]*Update{"5.6.7.8:80": {Addr: "5.6.7.8:80"}}, w.curAddrs)
+}
+
+func TestDNSWatcher_Lookup_TransientFailureRetainsCache(t *testing.T) {
+	// First resolution succeeds and populates curAddrs.
+	stubLookups(t, func(context.Context, string) ([]string, error) {
+		return []string{"1.2.3.4"}, nil
+	})
+
+	w := newTestDNSWatcher()
+	result := w.lookup()
+	assert.ElementsMatch(t, []*Update{{Op: Add, Addr: "1.2.3.4:80"}}, result)
+	require.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs)
+
+	// Fail next lookup
+	stubLookups(t, func(context.Context, string) ([]string, error) {
+		return nil, errors.New("unable to resolve address")
+	})
+
+	result = w.lookup()
+	assert.Nil(t, result)
+	assert.Equal(t, map[string]*Update{"1.2.3.4:80": {Addr: "1.2.3.4:80"}}, w.curAddrs)
+}