From 04696703fe253001571b4dbcb012a575f3eec18b Mon Sep 17 00:00:00 2001 From: bwplotka Date: Fri, 28 Nov 2025 12:13:38 +0000 Subject: [PATCH] refactor(appenderV2): add AppenderV2 interface Signed-off-by: bwplotka --- storage/interface.go | 47 ++++++---- storage/interface_append.go | 169 ++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 storage/interface_append.go diff --git a/storage/interface.go b/storage/interface.go index 19b4db4210..fe9b3fa6e8 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -58,11 +58,14 @@ var ( // their own reference types. type SeriesRef uint64 -// Appendable allows creating appenders. +// Appendable allows creating Appender. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type Appendable interface { - // Appender returns a new appender for the storage. The implementation - // can choose whether or not to use the context, for deadlines or to check - // for errors. + // Appender returns a new appender for the storage. + // + // Implementations CAN choose whether to use the context e.g. for deadlines, + // but it's not mandatory. Appender(ctx context.Context) Appender } @@ -255,7 +258,13 @@ func (f QueryableFunc) Querier(mint, maxt int64) (Querier, error) { return f(mint, maxt) } +// AppendOptions provides options for implementations of the Appender interface. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type AppendOptions struct { + // DiscardOutOfOrder tells implementation that this append should not be out + // of order. An OOO append MUST be rejected with storage.ErrOutOfOrderSample + // error. DiscardOutOfOrder bool } @@ -267,7 +276,11 @@ type AppendOptions struct { // The order of samples appended via the Appender is preserved within each // series. I.e. samples are not reordered per timestamp, or by float/histogram // type. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type Appender interface { + AppenderTransaction + // Append adds a sample pair for the given series. // An optional series reference can be provided to accelerate calls. // A series reference number is returned which can be used to add further @@ -278,16 +291,6 @@ type Appender interface { // If the reference is 0 it must not be used for caching. Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) - // Commit submits the collected samples and purges the batch. If Commit - // returns a non-nil error, it also rolls back all modifications made in - // the appender so far, as Rollback would do. In any case, an Appender - // must not be used anymore after Commit has been called. - Commit() error - - // Rollback rolls back all modifications made in the appender so far. - // Appender has to be discarded after rollback. - Rollback() error - // SetOptions configures the appender with specific append options such as // discarding out-of-order samples even if out-of-order is enabled in the TSDB. SetOptions(opts *AppendOptions) @@ -301,8 +304,8 @@ type Appender interface { // GetRef is an extra interface on Appenders used by downstream projects // (e.g. Cortex) to avoid maintaining a parallel set of references. type GetRef interface { - // Returns reference number that can be used to pass to Appender.Append(), - // and a set of labels that will not cause another copy when passed to Appender.Append(). + // GetRef returns a reference number that can be used to pass to AppenderV2.Append(), + // and a set of labels that will not cause another copy when passed to AppenderV2.Append(). // 0 means the appender does not have a reference to this series. // hash should be a hash of lset. GetRef(lset labels.Labels, hash uint64) (SeriesRef, labels.Labels) @@ -310,6 +313,8 @@ type GetRef interface { // ExemplarAppender provides an interface for adding samples to exemplar storage, which // within Prometheus is in-memory only. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type ExemplarAppender interface { // AppendExemplar adds an exemplar for the given series labels. // An optional reference number can be provided to accelerate calls. @@ -326,6 +331,8 @@ type ExemplarAppender interface { } // HistogramAppender provides an interface for appending histograms to the storage. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type HistogramAppender interface { // AppendHistogram adds a histogram for the given series labels. An // optional reference number can be provided to accelerate calls. A @@ -356,6 +363,8 @@ type HistogramAppender interface { } // MetadataUpdater provides an interface for associating metadata to stored series. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type MetadataUpdater interface { // UpdateMetadata updates a metadata entry for the given series and labels. // A series reference number is returned which can be used to modify the @@ -368,6 +377,8 @@ type MetadataUpdater interface { } // StartTimestampAppender provides an interface for appending ST to storage. +// +// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026). type StartTimestampAppender interface { // AppendSTZeroSample adds synthetic zero sample for the given st timestamp, // which will be associated with given series, labels and the incoming @@ -390,10 +401,10 @@ type SeriesSet interface { Next() bool // At returns full series. Returned series should be iterable even after Next is called. At() Series - // The error that iteration has failed with. + // Err returns the error that iteration has failed with. // When an error occurs, set cannot continue to iterate. Err() error - // A collection of warnings for the whole set. + // Warnings returns a collection of warnings for the whole set. // Warnings could be return even iteration has not failed with error. Warnings() annotations.Annotations } diff --git a/storage/interface_append.go b/storage/interface_append.go new file mode 100644 index 0000000000..c8d1b46ce8 --- /dev/null +++ b/storage/interface_append.go @@ -0,0 +1,169 @@ +package storage + +import ( + "context" + "errors" + + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" +) + +// AppendableV2 allows creating AppenderV2. +type AppendableV2 interface { + // AppenderV2 returns a new appender for the storage. + // + // Implementations CAN choose whether to use the context e.g. for deadlines, + // but it's not mandatory. + AppenderV2(ctx context.Context) AppenderV2 +} + +// AOptions is a shorthand for AppendV2Options. +// NOTE: AppendOption is used already. +type AOptions = AppendV2Options + +// AppendV2Options provides optional, auxiliary data and configuration for AppenderV2.Append. +type AppendV2Options struct { + // MetricFamilyName (optional) provides metric family name for the appended sample's + // series. If the client of the AppenderV2 has this information + // (e.g. from scrape) it's recommended to pass it to the appender. + // + // Provided string bytes are unsafe to reuse, it only lives for the duration of the Append call. + // + // Some implementations use this to avoid slow and prone to error metric family detection for: + // * Metadata per metric family storages (e.g. Prometheus metadata WAL/API/RW1) + // * Strictly complex types storages (e.g. OpenTelemetry Collector). + // + // NOTE(krajorama): Example purpose is highlighted in OTLP ingestion: OTLP calculates the + // metric family name for all metrics and uses it for generating summary, + // histogram series by adding the magic suffixes. The metric family name is + // passed down to the appender in case the storage needs it for metadata updates. + // Known user of this is Mimir that implements /api/v1/metadata and uses + // Remote-Write 1.0 for this. Might be removed later if no longer + // needed by any downstream project. + // NOTE(bwplotka): Long term, once Prometheus uses complex types on storage level + // the MetricFamilyName can be removed as MetricFamilyName will equal to __name__ always. + MetricFamilyName string + + // Metadata (optional) attached to the appended sample. + // Metadata strings are safe for reuse. + // IMPORTANT: Appender v1 was only providing update. This field MUST be + // set (if known) even if it didn't change since the last iteration. + // This moves the responsibility for metadata storage options to TSDB. + Metadata metadata.Metadata + + // Exemplars (optional) attached to the appended sample. + // Exemplar slice MUST be sorted by Exemplar.TS. + // Exemplar slice is unsafe for reuse. + Exemplars []exemplar.Exemplar + + // RejectOutOfOrder tells implementation that this append should not be out + // of order. An OOO append MUST be rejected with storage.ErrOutOfOrderSample + // error. + RejectOutOfOrder bool +} + +// AppendPartialError represents an AppenderV2.Append error that tells +// callers sample was written but some auxiliary optional data (e.g. exemplars) +// was not (or partially written) +// +// It's up to the caller to decide if it's an ignorable error or not, plus +// it allows extra reporting (e.g. for Remote Write 2.0 X-Remote-Write-Written headers). +type AppendPartialError struct { + ExemplarErrors []error +} + +// Error returns combined error string. +func (e *AppendPartialError) Error() string { + errs := errors.Join(e.ExemplarErrors...) + if errs == nil { + return "" + } + return errs.Error() +} + +var _ error = &AppendPartialError{} + +// AppenderV2 provides appends against a storage for all types of samples. +// It must be completed with a call to Commit or Rollback and must not be reused afterwards. +// +// Operations on the AppenderV2 interface are not goroutine-safe. +// +// The order of samples appended via the AppenderV2 is preserved within each +// series. I.e. samples are not reordered per timestamp, or by float/histogram +// type. +type AppenderV2 interface { + AppenderTransaction + + // Append appends a sample and related exemplars, metadata, and start timestamp (st) to the storage. + // + // ref (optional) represents the stable ID for the given series identified by ls (excluding metadata). + // Callers MAY provide the ref to help implementation avoid ls -> ref computation, otherwise ref MUST be 0 (unknown). + // + // ls represents labels for the sample's series. + // + // st (optional) represents sample start timestamp. 0 means unknown. Implementations + // are responsible for any potential ST storage logic (e.g. ST zero injections). + // + // t represents sample timestamp. + // + // v, h, fh represents sample value for each sample type. + // Callers MUST only provide one of the sample types (either v, h or fh). + // Implementations can detect the type of the sample with the following switch: + // + // switch { + // case fh != nil: It's a float histogram append. + // case h != nil: It's a histogram append. + // default: It's a float append. + // } + // TODO(bwplotka): We plan to experiment on using generics for complex sampleType, but do it after we unify interface (derisk) and before we add native summaries. + // + // Implementations MUST attempt to append sample even if metadata, exemplar or (st) start timestamp appends fail. + // Implementations MAY return AppendPartialError as an error. Use errors.As to detect. + // For the successful Append, Implementations MUST return valid SeriesRef that represents ls. + // NOTE(bwplotka): Given OTLP and native histograms and the relaxation of the requirement for + // type and unit suffixes in metric names we start to hit cases of ls being not enough for id + // of the series (metadata matters). Current solution is to enable 'type-and-unit-label' features for those cases, but we may + // start to extend the id with metadata one day. + Append(ref SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts AppendV2Options) (SeriesRef, error) +} + +// AppenderTransaction allows transactional appends. +type AppenderTransaction interface { + // Commit submits the collected samples and purges the batch. If Commit + // returns a non-nil error, it also rolls back all modifications made in + // the appender so far, as Rollback would do. In any case, an Appender + // must not be used anymore after Commit has been called. + Commit() error + + // Rollback rolls back all modifications made in the appender so far. + // Appender has to be discarded after rollback. + Rollback() error +} + +// LimitedAppenderV1 is an Appender that only supports appending float and histogram samples. +// This is to support migration to AppenderV2. +// TODO(bwplotka): Remove once migration to AppenderV2 is fully complete. +type LimitedAppenderV1 interface { + Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) + AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) +} + +// AppenderV2AsLimitedV1 returns appender that exposes AppenderV2 as LimitedAppenderV1 +// TODO(bwplotka): Remove once migration to AppenderV2 is fully complete. +func AppenderV2AsLimitedV1(app AppenderV2) LimitedAppenderV1 { + return &limitedAppenderV1{AppenderV2: app} +} + +type limitedAppenderV1 struct { + AppenderV2 +} + +func (a *limitedAppenderV1) Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) { + return a.AppenderV2.Append(ref, l, 0, t, v, nil, nil, AppendV2Options{}) +} + +func (a *limitedAppenderV1) AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) { + return a.AppenderV2.Append(ref, l, 0, t, 0, h, fh, AppendV2Options{}) +}