diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 8b82049f50..dffe65e3fb 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -281,6 +281,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { case "promql-extended-range-selectors": parser.EnableExtendedRangeSelectors = true logger.Info("Experimental PromQL extended range selectors enabled.") + case "promql-binop-fill-modifiers": + parser.EnableBinopFillModifiers = true + logger.Info("Experimental PromQL binary operator fill modifiers enabled.") case "": continue case "old-ui": @@ -578,7 +581,7 @@ func main() { a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode) diff --git a/cmd/prometheus/testdata/features.json b/cmd/prometheus/testdata/features.json index 145bb04d77..4c893daae2 100644 --- a/cmd/prometheus/testdata/features.json +++ b/cmd/prometheus/testdata/features.json @@ -28,6 +28,9 @@ "by": true, "delayed_name_removal": false, "duration_expr": false, + "fill": false, + "fill_left": false, + "fill_right": false, "group_left": true, "group_right": true, "ignoring": true, diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md index d4a8cd4f20..251fdfd6a4 100644 --- a/docs/command-line/prometheus.md +++ b/docs/command-line/prometheus.md @@ -59,7 +59,7 @@ The Prometheus monitoring server | --query.timeout | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | --query.max-concurrency | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | --query.max-samples | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | -| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | | --agent | Run Prometheus in 'Agent mode'. | | | --log.level | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | --log.format | Output format of log messages. One of: [logfmt, json] | `logfmt` | diff --git a/docs/feature_flags.md b/docs/feature_flags.md index af08eebb45..247941c5ce 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -67,12 +67,12 @@ Currently, Prometheus supports start timestamps on the * `PrometheusProto` * `OpenMetrics1.0.0` - + From the above, Prometheus recommends `PrometheusProto`. This is because OpenMetrics 1.0 Start Timestamp information is shared as a `_created` metric and parsing those are prone to errors and expensive (thus, adding an overhead). You also need to be careful to not pollute your Prometheus with extra `_created` metrics. - -Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to + +Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to `[ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ]`, resulting in negotiating the Prometheus Protobuf protocol first (unless the `scrape_protocols` option is set to a different value explicitly). Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. @@ -288,8 +288,8 @@ when wrong types are used on wrong functions, automatic renames, delta types and ### Behavior with metadata records -When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, -the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, +When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, +the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, if the metadata record somehow (e.g. due to bug) says "counter", but `__type__="gauge"` the remote time series will be set to a gauge. ## Use Uncached IO @@ -338,9 +338,25 @@ Example query: > **Note for alerting and recording rules:** > The `smoothed` modifier requires samples after the evaluation interval, so using it directly in alerting or recording rules will typically *under-estimate* the result, as future samples are not available at evaluation time. -> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available. +> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available. > For critical alerting, set the offset to at least one scrape interval; for less critical or more resilient use cases, consider a larger offset (multiple scrape intervals) to tolerate missed scrapes. For more details, see the [design doc](https://github.com/prometheus/proposals/blob/main/proposals/2025-04-04_extended-range-selectors-semantics.md). **Note**: Extended Range Selectors are not supported for subqueries. + +## Binary operator fill modifiers + +`--enable-feature=promql-binop-fill-modifiers` + +Enables experimental `fill()`, `fill_left()`, and `fill_right()` modifiers for PromQL binary operators. These modifiers allow filling in missing matches on either side of a binary operation with a provided default sample value. + +Example query: + +``` + rate(successful_requests[5m]) ++ fill(0) + rate(failed_requests[5m]) +``` + +See [the fill modifiers documentation](querying/operators.md#filling-in-missing-matches) for more details and examples. diff --git a/docs/querying/operators.md b/docs/querying/operators.md index b320d8e86e..b15c02aedc 100644 --- a/docs/querying/operators.md +++ b/docs/querying/operators.md @@ -47,9 +47,9 @@ special values like `NaN`, `+Inf`, and `-Inf`. scalar that is the result of the operator applied to both scalar operands. **Between an instant vector and a scalar**, the operator is applied to the -value of every data sample in the vector. +value of every data sample in the vector. -If the data sample is a float, the operation is performed between that float and the scalar. +If the data sample is a float, the operation is performed between that float and the scalar. For example, if an instant vector of float samples is multiplied by 2, the result is another vector of float samples in which every sample value of the original vector is multiplied by 2. @@ -81,8 +81,9 @@ following: **Between two instant vectors**, a binary arithmetic operator is applied to each entry in the LHS vector and its [matching element](#vector-matching) in the RHS vector. The result is propagated into the result vector with the -grouping labels becoming the output label set. Entries for which no matching -entry in the right-hand vector can be found are not part of the result. +grouping labels becoming the output label set. By default, series for which +no matching entry in the opposite vector can be found are not part of the +result. This behavior can be adjusted using [fill modifiers](#filling-in-missing-matches). If two float samples are matched, the arithmetic operator is applied to the two input values. @@ -97,7 +98,7 @@ If two histogram samples are matched, only `+` and `-` are valid operations, each adding or subtracting all matching bucket populations and the count and the sum of observations. All other operations result in the removal of the corresponding element from the output vector, flagged by an info-level -annotation. The `+` and -` operations should generally only be applied to gauge +annotation. The `+` and `-` operations should generally only be applied to gauge histograms, but PromQL allows them for counter histograms, too, to cover specific use cases, for which special attention is required to avoid problems with unaligned counter resets. (Certain incompatibilities of counter resets can @@ -106,7 +107,7 @@ two counter histograms results in a counter histogram. All other combination of operands and all subtractions result in a gauge histogram. **In any arithmetic binary operation involving vectors**, the metric name is -dropped. This occurs even if `__name__` is explicitly mentioned in `on` +dropped. This occurs even if `__name__` is explicitly mentioned in `on` (see https://github.com/prometheus/prometheus/issues/16631 for further discussion). **For any arithmetic binary operation that may result in a negative @@ -156,9 +157,9 @@ info-level annotation. applied to matching entries. Vector elements for which the expression is not true or which do not find a match on the other side of the expression get dropped from the result, while the others are propagated into a result vector -with the grouping labels becoming the output label set. +with the grouping labels becoming the output label set. -Matches between two float samples work as usual. +Matches between two float samples work as usual. Matches between a float sample and a histogram sample are invalid, and the corresponding element is removed from the result vector, flagged by an info-level @@ -171,8 +172,8 @@ comparison binary operations are again invalid. modifier changes the behavior in the following ways: * Vector elements which find a match on the other side of the expression but for - which the expression is false instead have the value `0` and vector elements - that do find a match and for which the expression is true have the value `1`. + which the expression is false instead have the value `0`, and vector elements + that do find a match and for which the expression is true have the value `1`. (Note that elements with no match or invalid operations involving histogram samples still return no result rather than the value `0`.) * The metric name is dropped. @@ -216,11 +217,10 @@ matching behavior: One-to-one and many-to-one/one-to-many. ### Vector matching keywords -These vector matching keywords allow for matching between series with different label sets -providing: +These vector matching keywords allow for matching between series with different label sets: -* `on` -* `ignoring` +* `on(