From c32649c232f624cb5f1e6b22dc6b488ef3e1838b Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 17 Jul 2026 06:14:06 +0000 Subject: [PATCH] Add cortex_ingester_tsdb_head_max_timestamp metric Re-export the TSDB head max timestamp (prometheus_tsdb_head_max_time) from the ingester per user. This helps investigate ingestion issues such as out-of-bounds (sample too old) errors by making the per-tenant TSDB head max time visible. Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + pkg/ingester/metrics.go | 7 +++++++ pkg/ingester/metrics_test.go | 17 +++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 563d7aabf9b..d0a4646d64b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ * [ENHANCEMENT] Query Frontend: Rename `time_taken` field to `time_taken_ms` and make it return millisecond count. #7649 * [ENHANCEMENT] Update prometheus alertmanager version to v0.33.0. #7647 * [ENHANCEMENT] Querier/Ingester: Detach ingester series from gRPC buffers to reduce heap. #7670 +* [ENHANCEMENT] Ingester: Add `cortex_ingester_tsdb_head_max_timestamp` metric that re-exports the TSDB head max timestamp (`prometheus_tsdb_head_max_time`) per user, to help investigate ingestion issues like out-of-bounds (too old sample) errors. #7694 * [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370 * [BUGFIX] Metrics Helper: Fix non-deterministic bucket order in merged histograms by sorting buckets after map iteration, matching Prometheus client library behavior. #7380 * [BUGFIX] Distributor: Return HTTP 401 Unauthorized when tenant ID resolution fails in the Prometheus Remote Write 2.0 path. #7389 diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 31c165986fd..3ad21faad6d 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -472,6 +472,7 @@ type tsdbMetrics struct { tsdbHeadTruncateTotal *prometheus.Desc tsdbHeadGcDuration *prometheus.Desc tsdbHeadStaleSeries *prometheus.Desc + tsdbHeadMaxTimestamp *prometheus.Desc tsdbActiveAppenders *prometheus.Desc tsdbSeriesNotFound *prometheus.Desc tsdbChunks *prometheus.Desc @@ -609,6 +610,10 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics { "cortex_ingester_tsdb_head_stale_series", "Total number of stale series in the head block.", []string{"user"}, nil), + tsdbHeadMaxTimestamp: prometheus.NewDesc( + "cortex_ingester_tsdb_head_max_timestamp", + "Maximum timestamp of the head block, in milliseconds since epoch.", + []string{"user"}, nil), tsdbActiveAppenders: prometheus.NewDesc( "cortex_ingester_tsdb_head_active_appenders", "Number of currently active TSDB appender transactions.", @@ -775,6 +780,7 @@ func (sm *tsdbMetrics) Describe(out chan<- *prometheus.Desc) { out <- sm.tsdbHeadTruncateFail out <- sm.tsdbHeadTruncateTotal out <- sm.tsdbHeadStaleSeries + out <- sm.tsdbHeadMaxTimestamp out <- sm.tsdbHeadGcDuration out <- sm.tsdbActiveAppenders out <- sm.tsdbSeriesNotFound @@ -841,6 +847,7 @@ func (sm *tsdbMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCounters(out, sm.tsdbHeadTruncateTotal, "prometheus_tsdb_head_truncations_total") data.SendSumOfSummaries(out, sm.tsdbHeadGcDuration, "prometheus_tsdb_head_gc_duration_seconds") data.SendSumOfGaugesPerUser(out, sm.tsdbHeadStaleSeries, "prometheus_tsdb_head_stale_series") + data.SendMaxOfGaugesPerUser(out, sm.tsdbHeadMaxTimestamp, "prometheus_tsdb_head_max_time") data.SendSumOfGauges(out, sm.tsdbActiveAppenders, "prometheus_tsdb_head_active_appenders") data.SendSumOfCounters(out, sm.tsdbSeriesNotFound, "prometheus_tsdb_head_series_not_found_total") data.SendSumOfGauges(out, sm.tsdbChunks, "prometheus_tsdb_head_chunks") diff --git a/pkg/ingester/metrics_test.go b/pkg/ingester/metrics_test.go index ad10efd41a0..af09d07e2a3 100644 --- a/pkg/ingester/metrics_test.go +++ b/pkg/ingester/metrics_test.go @@ -631,6 +631,12 @@ func TestTSDBMetrics(t *testing.T) { cortex_ingester_tsdb_head_stale_series{user="user1"} 382695 cortex_ingester_tsdb_head_stale_series{user="user2"} 2659397 cortex_ingester_tsdb_head_stale_series{user="user3"} 30969 + # HELP cortex_ingester_tsdb_head_max_timestamp Maximum timestamp of the head block, in milliseconds since epoch. + # TYPE cortex_ingester_tsdb_head_max_timestamp gauge + # 36 * base + cortex_ingester_tsdb_head_max_timestamp{user="user1"} 444420 + cortex_ingester_tsdb_head_max_timestamp{user="user2"} 3088332 + cortex_ingester_tsdb_head_max_timestamp{user="user3"} 35964 `)) require.NoError(t, err) } @@ -925,6 +931,11 @@ func TestTSDBMetricsWithRemoval(t *testing.T) { # TYPE cortex_ingester_tsdb_head_stale_series gauge cortex_ingester_tsdb_head_stale_series{user="user1"} 382695 cortex_ingester_tsdb_head_stale_series{user="user2"} 2659397 + # HELP cortex_ingester_tsdb_head_max_timestamp Maximum timestamp of the head block, in milliseconds since epoch. + # TYPE cortex_ingester_tsdb_head_max_timestamp gauge + # 36 * base + cortex_ingester_tsdb_head_max_timestamp{user="user1"} 444420 + cortex_ingester_tsdb_head_max_timestamp{user="user2"} 3088332 `)) require.NoError(t, err) } @@ -1258,6 +1269,12 @@ func populateTSDBMetrics(base float64) *prometheus.Registry { }) headStaleSeries.Set(31 * base) + headMaxTime := promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_tsdb_head_max_time", + Help: "Maximum timestamp of the head block. The unit is decided by the library consumer.", + }) + headMaxTime.Set(36 * base) + recordPartWrites := promauto.With(r).NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_wal_record_part_writes_total", Help: "Total number of record parts written before flushing.",