From 6411791fa32fceb854b5e249665041755bbf1df9 Mon Sep 17 00:00:00 2001 From: kkh Date: Thu, 22 Jan 2026 01:19:30 +0900 Subject: [PATCH] tsdb: fix negative head chunk count during WAL replay During WAL replay, resetSeriesWithMMappedChunks could cause the prometheus_tsdb_head_chunks gauge to go negative. This happened when processing duplicate series records where the old chunks had not yet been added to the gauge. The fix checks whether we're processing the same series ref before subtracting old chunk counts. For duplicate series with different refs, we only add the new chunk count without subtracting. Fixes #10884 Signed-off-by: kkh --- tsdb/head_wal.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index 3c9aa7980e..2072386a46 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -506,12 +506,22 @@ func (h *Head) resetSeriesWithMMappedChunks(mSeries *memSeries, mmc, oooMmc []*m } h.metrics.chunksCreated.Add(float64(len(mmc) + len(oooMmc))) - h.metrics.chunksRemoved.Add(float64(len(mSeries.mmappedChunks))) - h.metrics.chunks.Add(float64(len(mmc) + len(oooMmc) - len(mSeries.mmappedChunks))) - if mSeries.ooo != nil { - h.metrics.chunksRemoved.Add(float64(len(mSeries.ooo.oooMmappedChunks))) - h.metrics.chunks.Sub(float64(len(mSeries.ooo.oooMmappedChunks))) + // During WAL replay with duplicate series records (mSeries.ref != walSeriesRef), + // the old chunks in mSeries may not have been added to the gauge yet, + // so subtracting them would cause the gauge to go negative. + // Only count chunks as removed when we're certain they were previously added. + if mSeries.ref == walSeriesRef { + // Same series ref - old chunks were definitely counted, safe to subtract + h.metrics.chunksRemoved.Add(float64(len(mSeries.mmappedChunks))) + h.metrics.chunks.Add(float64(len(mmc) + len(oooMmc) - len(mSeries.mmappedChunks))) + if mSeries.ooo != nil { + h.metrics.chunksRemoved.Add(float64(len(mSeries.ooo.oooMmappedChunks))) + h.metrics.chunks.Sub(float64(len(mSeries.ooo.oooMmappedChunks))) + } + } else { + // Duplicate series with different ref - old chunks may not have been counted + h.metrics.chunks.Add(float64(len(mmc) + len(oooMmc))) } mSeries.mmappedChunks = mmc