upstream/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2026-02-03 20:39:32 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

test: Add benchmark without storage + fix skipRecording mock feature (#17987)

Browse source 
* test: Add benchmark without storage

Signed-off-by: bwplotka <bwplotka@gmail.com>

make bench fair

Signed-off-by: bwplotka <bwplotka@gmail.com>

tmp

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Apply suggestions from code review

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

---------

Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Bartlomiej Plotka 2026-02-02 12:44:11 +00:00 committed by GitHub
parent 717d37bbca
commit 848b16d686
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 114 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
scrape/scrape_test.go
View file

@ -1725,48 +1725,67 @@ func TestScrapeLoopAppend_WithStorage(t *testing.T) {
// BenchmarkScrapeLoopAppend benchmarks scrape appends for typical cases. // BenchmarkScrapeLoopAppend benchmarks scrape appends for typical cases.
// //
// Benchmark compares append function run across 4 dimensions: // Benchmark compares append function run across 5 dimensions:
// * `appV2`: appender V1 or V2 // * `withStorage`: without storage isolates the benchmark to the scrape loop append code. With storage is an
// * `appendMetadataToWAL`: metadata-wal-records feature enabled or not // integration benchmark with the TSDB head appender code. For acceptance criteria run with storage, without for debugging.
// *`data`: different sizes of metrics scraped e.g. one big gauge metric family // * `appV2`: appender V1 or V2.
// * `appendMetadataToWAL`: metadata-wal-records feature enabled or not (problematic feature we might need to change
// soon, see https://github.com/prometheus/prometheus/issues/15911.
// * `data`: different sizes of metrics scraped e.g. one big gauge metric family
// with a thousand series and more realistic scenario with common types. // with a thousand series and more realistic scenario with common types.
// *`fmt`: different scrape formats which will benchmark different parsers e.g. // * `fmt`: different scrape formats which will benchmark different parsers e.g.
// promtext, omtext and promproto. // promtext, omtext and promproto.
// //
// Recommended CLI invocation: // NOTE: withStorage=true uses sync.Pool buffers which is heavily non-deterministic and shared across go routines.
// As a result, it's recommended to run dimensions you want to compare with in e.g. separate go tool invocations.
// Recommended CLI invocation(s):
/* /*
export bench=append && go test ./scrape/... \ # Acceptance: With storage with V1 and V2 in separate process:
-run '^$' -bench '^BenchmarkScrapeLoopAppend$' \ export bench=appendV1 && go test ./scrape/... \
-run '^$' -bench '^BenchmarkScrapeLoopAppend/withStorage=true/appV2=false/$' \
-benchtime 2s -count 6 -cpu 2 -timeout 999m \
| tee ${bench}.txt
export bench=appendV2 && go test ./scrape/... \
-run '^$' -bench '^BenchmarkScrapeLoopAppend/withStorage=true/appV2=true/$' \
-benchtime 2s -count 6 -cpu 2 -timeout 999m \
| tee ${bench}.txt
# For debugging scrape overheads:
export bench=appendNoStorage && go test ./scrape/... \
-run '^$' -bench '^BenchmarkScrapeLoopAppend/withStorage=false/$' \
-benchtime 2s -count 6 -cpu 2 -timeout 999m \ -benchtime 2s -count 6 -cpu 2 -timeout 999m \
| tee ${bench}.txt | tee ${bench}.txt
*/ */
func BenchmarkScrapeLoopAppend(b *testing.B) { func BenchmarkScrapeLoopAppend(b *testing.B) {
for _, appV2 := range []bool{false, true} { for _, withStorage := range []bool{false, true} {
for _, appendMetadataToWAL := range []bool{false, true} { for _, appV2 := range []bool{false, true} {
for _, data := range []struct { for _, appendMetadataToWAL := range []bool{false, true} {
name string for _, data := range []struct {
parsableText []byte name string
}{ parsableText []byte
{name: "1Fam2000Gauges", parsableText: makeTestGauges(2000)}, // ~68.1 KB, ~77.9 KB in proto. }{
{name: "237FamsAllTypes", parsableText: readTextParseTestMetrics(b)}, // ~185.7 KB, ~70.6 KB in proto. {name: "1Fam2000Gauges", parsableText: makeTestGauges(2000)}, // ~68.1 KB, ~77.9 KB in proto.
} { {name: "237FamsAllTypes", parsableText: readTextParseTestMetrics(b)}, // ~185.7 KB, ~70.6 KB in proto.
b.Run(fmt.Sprintf("appV2=%v/appendMetadataToWAL=%v/data=%v", appV2, appendMetadataToWAL, data.name), func(b *testing.B) { } {
metricsProto := promTextToProto(b, data.parsableText) b.Run(fmt.Sprintf("withStorage=%v/appV2=%v/appendMetadataToWAL=%v/data=%v", withStorage, appV2, appendMetadataToWAL, data.name), func(b *testing.B) {
metricsProto := promTextToProto(b, data.parsableText)
for _, bcase := range []struct { for _, bcase := range []struct {
name string name string
contentType string contentType string
parsable []byte parsable []byte
}{ }{
{name: "PromText", contentType: "text/plain", parsable: data.parsableText}, {name: "PromText", contentType: "text/plain", parsable: data.parsableText},
{name: "OMText", contentType: "application/openmetrics-text", parsable: data.parsableText}, {name: "OMText", contentType: "application/openmetrics-text", parsable: data.parsableText},
{name: "PromProto", contentType: "application/vnd.google.protobuf", parsable: metricsProto}, {name: "PromProto", contentType: "application/vnd.google.protobuf", parsable: metricsProto},
} { } {
b.Run(fmt.Sprintf("fmt=%v", bcase.name), func(b *testing.B) { b.Run(fmt.Sprintf("fmt=%v", bcase.name), func(b *testing.B) {
benchScrapeLoopAppend(b, appV2, bcase.parsable, bcase.contentType, appendMetadataToWAL, false) benchScrapeLoopAppend(b, withStorage, appV2, bcase.parsable, bcase.contentType, appendMetadataToWAL, false)
}) })
} }
}) })
}
} }
} }
} }
@ -1774,30 +1793,32 @@ func BenchmarkScrapeLoopAppend(b *testing.B) {
func benchScrapeLoopAppend( func benchScrapeLoopAppend(
b *testing.B, b *testing.B,
withStorage bool,
appV2 bool, appV2 bool,
parsable []byte, parsable []byte,
contentType string, contentType string,
appendMetadataToWAL bool, appendMetadataToWAL bool,
enableExemplarStorage bool, enableExemplarStorage bool,
) { ) {
// Need a full storage for correct Add/AddFast semantics. var a compatAppendable = teststorage.NewAppendable().SkipRecording(true) // Make it noop for benchmark purposes.
s := teststorage.New(b, func(opt *tsdb.Options) { if withStorage {
opt.EnableMetadataWALRecords = appendMetadataToWAL a = teststorage.New(b, func(opt *tsdb.Options) {
if enableExemplarStorage { opt.EnableMetadataWALRecords = appendMetadataToWAL
opt.EnableExemplarStorage = true if enableExemplarStorage {
opt.MaxExemplars = 1e5 opt.EnableExemplarStorage = true
} opt.MaxExemplars = 1e5
}) }
})
sl, _ := newTestScrapeLoop(b, withAppendable(s, appV2), func(sl *scrapeLoop) { }
sl, _ := newTestScrapeLoop(b, withAppendable(a, appV2), func(sl *scrapeLoop) {
sl.appendMetadataToWAL = appendMetadataToWAL sl.appendMetadataToWAL = appendMetadataToWAL
}) })
app := sl.appender()
ts := time.Time{} ts := time.Time{}
b.ReportAllocs() b.ReportAllocs()
b.ResetTimer() b.ResetTimer()
for b.Loop() { for b.Loop() {
app := sl.appender()
ts = ts.Add(time.Second) ts = ts.Add(time.Second)
_, _, _, err := app.append(parsable, contentType, ts) _, _, _, err := app.append(parsable, contentType, ts)
if err != nil { if err != nil {
@ -1808,7 +1829,6 @@ func benchScrapeLoopAppend(
if err := app.Rollback(); err != nil { if err := app.Rollback(); err != nil {
b.Fatal(err) b.Fatal(err)
} }
app = sl.appender()
} }
} }
@ -1827,7 +1847,7 @@ func BenchmarkScrapeLoopAppend_HistogramsWithExemplars(b *testing.B) {
for _, appV2 := range []bool{false, true} { for _, appV2 := range []bool{false, true} {
b.Run(fmt.Sprintf("appV2=%v", appV2), func(b *testing.B) { b.Run(fmt.Sprintf("appV2=%v", appV2), func(b *testing.B) {
parsable := makeTestHistogramsWithExemplars(100) // ~255.8 KB in OM text. parsable := makeTestHistogramsWithExemplars(100) // ~255.8 KB in OM text.
benchScrapeLoopAppend(b, appV2, parsable, "application/openmetrics-text", false, true) benchScrapeLoopAppend(b, true, appV2, parsable, "application/openmetrics-text", false, true)
}) })
} }
} }
@ -3398,7 +3418,9 @@ metric: <
} }
sl.alwaysScrapeClassicHist = test.alwaysScrapeClassicHist sl.alwaysScrapeClassicHist = test.alwaysScrapeClassicHist
// This test does not care about metadata. // This test does not care about metadata.
// TODO(bwplotka): Add metadata expectations and turn it on. // Having this true would mean we need to add metadata to sample
// expectations.
// TODO(bwplotka): Add cases for append metadata to WAL and pass metadata
sl.appendMetadataToWAL = false sl.appendMetadataToWAL = false
}) })
app := sl.appender() app := sl.appender()

81
util/teststorage/appender.go
View file

@ -409,9 +409,11 @@ func (a *appender) Append(ref storage.SeriesRef, ls labels.Labels, t int64, v fl
} }
} }
a.a.mtx.Lock() if !a.a.skipRecording {
a.a.pendingSamples = append(a.a.pendingSamples, Sample{L: ls, T: t, V: v}) a.a.mtx.Lock()
a.a.mtx.Unlock() a.a.pendingSamples = append(a.a.pendingSamples, Sample{L: ls, T: t, V: v})
a.a.mtx.Unlock()
}
if a.next != nil { if a.next != nil {
return a.next.Append(ref, ls, t, v) return a.next.Append(ref, ls, t, v)
@ -445,9 +447,11 @@ func (a *appender) AppendHistogram(ref storage.SeriesRef, ls labels.Labels, t in
} }
} }
a.a.mtx.Lock() if !a.a.skipRecording {
a.a.pendingSamples = append(a.a.pendingSamples, Sample{L: ls, T: t, H: h, FH: fh}) a.a.mtx.Lock()
a.a.mtx.Unlock() a.a.pendingSamples = append(a.a.pendingSamples, Sample{L: ls, T: t, H: h, FH: fh})
a.a.mtx.Unlock()
}
if a.next != nil { if a.next != nil {
return a.next.AppendHistogram(ref, ls, t, h, fh) return a.next.AppendHistogram(ref, ls, t, h, fh)
@ -463,23 +467,26 @@ func (a *appender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exem
if a.a.appendExemplarsError != nil { if a.a.appendExemplarsError != nil {
return 0, a.a.appendExemplarsError return 0, a.a.appendExemplarsError
} }
var appended bool
a.a.mtx.Lock() if !a.a.skipRecording {
// NOTE(bwplotka): Eventually exemplar has to be attached to a series and soon var appended bool
// the AppenderV2 will guarantee that for TSDB. Assume this from the mock perspective
// with the naive attaching. See: https://github.com/prometheus/prometheus/issues/17632 a.a.mtx.Lock()
i := len(a.a.pendingSamples) - 1 // NOTE(bwplotka): Eventually exemplar has to be attached to a series and soon
for ; i >= 0; i-- { // Attach exemplars to the last matching sample. // the AppenderV2 will guarantee that for TSDB. Assume this from the mock perspective
if labels.Equal(l, a.a.pendingSamples[i].L) { // with the naive attaching. See: https://github.com/prometheus/prometheus/issues/17632
a.a.pendingSamples[i].ES = append(a.a.pendingSamples[i].ES, e) i := len(a.a.pendingSamples) - 1
appended = true for ; i >= 0; i-- { // Attach exemplars to the last matching sample.
break if labels.Equal(l, a.a.pendingSamples[i].L) {
a.a.pendingSamples[i].ES = append(a.a.pendingSamples[i].ES, e)
appended = true
break
}
}
a.a.mtx.Unlock()
if !appended {
return 0, fmt.Errorf("teststorage.appender: exemplar appender without series; ref %v; l %v; exemplar: %v", ref, l, e)
} }
}
a.a.mtx.Unlock()
if !appended {
return 0, fmt.Errorf("teststorage.appender: exemplar appender without series; ref %v; l %v; exemplar: %v", ref, l, e)
} }
if a.next != nil { if a.next != nil {
@ -504,23 +511,25 @@ func (a *appender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m meta
return 0, err return 0, err
} }
var updated bool if !a.a.skipRecording {
var updated bool
a.a.mtx.Lock() a.a.mtx.Lock()
// NOTE(bwplotka): Eventually metadata has to be attached to a series and soon // NOTE(bwplotka): Eventually metadata has to be attached to a series and soon
// the AppenderV2 will guarantee that for TSDB. Assume this from the mock perspective // the AppenderV2 will guarantee that for TSDB. Assume this from the mock perspective
// with the naive attaching. See: https://github.com/prometheus/prometheus/issues/17632 // with the naive attaching. See: https://github.com/prometheus/prometheus/issues/17632
i := len(a.a.pendingSamples) - 1 i := len(a.a.pendingSamples) - 1
for ; i >= 0; i-- { // Attach metadata to the last matching sample. for ; i >= 0; i-- { // Attach metadata to the last matching sample.
if labels.Equal(l, a.a.pendingSamples[i].L) { if labels.Equal(l, a.a.pendingSamples[i].L) {
a.a.pendingSamples[i].M = m a.a.pendingSamples[i].M = m
updated = true updated = true
break break
}
}
a.a.mtx.Unlock()
if !updated {
return 0, fmt.Errorf("teststorage.appender: metadata update without series; ref %v; l %v; m: %v", ref, l, m)
} }
}
a.a.mtx.Unlock()
if !updated {
return 0, fmt.Errorf("teststorage.appender: metadata update without series; ref %v; l %v; m: %v", ref, l, m)
} }
if a.next != nil { if a.next != nil {