mirror of
https://github.com/kubernetes/kubernetes.git
synced 2026-02-03 20:40:26 -05:00
Merge 1f7f235df3 into 1861c1a072
This commit is contained in:
commit
dbbca19ae9
18 changed files with 861 additions and 533 deletions
|
|
@ -58,6 +58,7 @@ allowed_prometheus_importers=(
|
|||
./staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go
|
||||
./staging/src/k8s.io/component-base/metrics/metric.go
|
||||
./staging/src/k8s.io/component-base/metrics/opts.go
|
||||
./staging/src/k8s.io/component-base/metrics/options_test.go
|
||||
./staging/src/k8s.io/component-base/metrics/processstarttime_others.go
|
||||
./staging/src/k8s.io/component-base/metrics/registry.go
|
||||
./staging/src/k8s.io/component-base/metrics/registry_test.go
|
||||
|
|
|
|||
124
staging/src/k8s.io/component-base/metrics/api/v1/config.go
Normal file
124
staging/src/k8s.io/component-base/metrics/api/v1/config.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"github.com/blang/semver/v4"
|
||||
"go.yaml.in/yaml/v2"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
)
|
||||
|
||||
var (
|
||||
labelExpr = `[a-zA-Z_][a-zA-Z0-9_]*`
|
||||
metricNameExpr = `[a-zA-Z_:][a-zA-Z0-9_:]*`
|
||||
)
|
||||
|
||||
// Validate validates a MetricsConfiguration.
|
||||
func Validate(c *MetricsConfiguration, currentVersion semver.Version, fldPath *field.Path) field.ErrorList {
|
||||
errs := field.ErrorList{}
|
||||
if c == nil {
|
||||
return errs
|
||||
}
|
||||
errs = append(errs, validateShowHiddenMetricsVersion(currentVersion, c.ShowHiddenMetricsForVersion, fldPath.Child("showHiddenMetricsForVersion"))...)
|
||||
errs = append(errs, validateDisabledMetrics(c.DisabledMetrics, fldPath.Child("disabledMetrics"))...)
|
||||
errs = append(errs, validateAllowListMapping(c.AllowListMapping, fldPath.Child("allowListMapping"))...)
|
||||
errs = append(errs, validateAllowListMappingManifest(c.AllowListMappingManifest, fldPath.Child("allowListMappingManifest"))...)
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateAllowListMapping(allowListMapping map[string]string, fldPath *field.Path) field.ErrorList {
|
||||
errs := field.ErrorList{}
|
||||
allowListMappingKeyRegex := regexp.MustCompile(metricNameExpr + `,` + labelExpr)
|
||||
for k := range allowListMapping {
|
||||
if allowListMappingKeyRegex.FindString(k) != k {
|
||||
return append(errs, field.Invalid(fldPath, allowListMapping, fmt.Sprintf("must have keys with format `metricName,labelName` where metricName matches %q and labelName matches %q", metricNameExpr, labelExpr)))
|
||||
}
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// validateAllowListMappingManifest validates the allow list mapping manifest file.
|
||||
// This function is used to validate the manifest file provided via the flag --allow-metric-labels-manifest, or the configuration file.
|
||||
// In the former case, the path resolution is relative to the current working directory.
|
||||
// In the latter case, the path resolution is relative to the configuration file's location, and components are required to pass in the resolved absolute path.
|
||||
// NOTE: If its the latter case, components are expected to pass in the *absolute* path to the manifest file.
|
||||
func validateAllowListMappingManifest(allowListMappingManifestPath string, fldPath *field.Path) field.ErrorList {
|
||||
errs := field.ErrorList{}
|
||||
if allowListMappingManifestPath == "" {
|
||||
return errs
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Clean(allowListMappingManifestPath))
|
||||
if err != nil {
|
||||
return append(errs, field.Invalid(fldPath, allowListMappingManifestPath, fmt.Errorf("failed to read allow list manifest: %w", err).Error()))
|
||||
}
|
||||
allowListMapping := make(map[string]string)
|
||||
err = yaml.Unmarshal(data, &allowListMapping)
|
||||
if err != nil {
|
||||
return append(errs, field.Invalid(fldPath, allowListMappingManifestPath, fmt.Errorf("failed to parse allow list manifest: %w", err).Error()))
|
||||
}
|
||||
allowListMappingErrs := validateAllowListMapping(allowListMapping, fldPath)
|
||||
if len(allowListMappingErrs) > 0 {
|
||||
return append(errs, field.Invalid(fldPath, allowListMappingManifestPath, fmt.Sprintf("invalid allow list mapping in manifest: %v", allowListMappingErrs)))
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateDisabledMetrics(names []string, fldPath *field.Path) field.ErrorList {
|
||||
errs := field.ErrorList{}
|
||||
metricNameRegex := regexp.MustCompile(`^` + metricNameExpr + `$`)
|
||||
for _, name := range names {
|
||||
if !metricNameRegex.MatchString(name) {
|
||||
return append(errs, field.Invalid(fldPath, names, fmt.Sprintf("must be fully qualified metric names matching %q, got %q", metricNameRegex.String(), name)))
|
||||
}
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateShowHiddenMetricsVersion(currentVersion semver.Version, targetVersionStr string, fldPath *field.Path) field.ErrorList {
|
||||
errs := field.ErrorList{}
|
||||
if targetVersionStr == "" {
|
||||
return errs
|
||||
}
|
||||
|
||||
validVersionStr := fmt.Sprintf("%d.%d", currentVersion.Major, currentVersion.Minor-1)
|
||||
if targetVersionStr != validVersionStr {
|
||||
return append(errs, field.Invalid(fldPath, targetVersionStr, fmt.Sprintf("must be omitted or have the value '%v'; only the previous minor version is allowed", validVersionStr)))
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// ValidateShowHiddenMetricsVersionForKubeletBackwardCompatOnly validates the ShowHiddenMetricsForVersion field.
|
||||
// TODO: This is kept here for backward compatibility in Kubelet (as metrics configuration fields were exposed on an individual basis earlier).
|
||||
// TODO: Revisit this after Kubelet supports the new metrics configuration API: https://github.com/kubernetes/kubernetes/pull/123426
|
||||
func ValidateShowHiddenMetricsVersionForKubeletBackwardCompatOnly(currentVersion semver.Version, targetVersionStr string) error {
|
||||
errs := validateShowHiddenMetricsVersion(currentVersion, targetVersionStr, field.NewPath("showHiddenMetricsForVersion"))
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("invalid showHiddenMetricsForVersion: %v", errs.ToAggregate().Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
162
staging/src/k8s.io/component-base/metrics/api/v1/config_test.go
Normal file
162
staging/src/k8s.io/component-base/metrics/api/v1/config_test.go
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/*
|
||||
Copyright 2021 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/blang/semver/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
)
|
||||
|
||||
func TestValidateShowHiddenMetricsVersion(t *testing.T) {
|
||||
currentVersion := semver.MustParse("1.17.0")
|
||||
|
||||
var tests = []struct {
|
||||
desc string
|
||||
targetVersion string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
desc: "invalid version is not allowed",
|
||||
targetVersion: "1.invalid",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "patch version is not allowed",
|
||||
targetVersion: "1.16.0",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "old version is not allowed",
|
||||
targetVersion: "1.15",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "new version is not allowed",
|
||||
targetVersion: "1.17",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "valid version is allowed",
|
||||
targetVersion: "1.16",
|
||||
expectedError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
errs := validateShowHiddenMetricsVersion(currentVersion, tc.targetVersion, field.NewPath("showHiddenMetricsForVersion"))
|
||||
|
||||
if tc.expectedError {
|
||||
assert.Errorf(t, errs.ToAggregate(), "Failed to test: %s", tc.desc)
|
||||
} else {
|
||||
assert.NoErrorf(t, errs.ToAggregate(), "Failed to test: %s", tc.desc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDisabledMetrics(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
input []string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
"validated",
|
||||
[]string{"metric_name", "another_metric"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"empty input",
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
name: "empty metric name",
|
||||
input: []string{"", "another_metric"},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
"invalid metric name",
|
||||
[]string{"metric_.name", "another_metric"},
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
errs := validateDisabledMetrics(tt.input, field.NewPath("disabledMetrics"))
|
||||
if len(errs) == 0 && tt.expectedError {
|
||||
t.Error("Got no error, wanted error(s)")
|
||||
}
|
||||
if len(errs) != 0 && !tt.expectedError {
|
||||
t.Errorf("Got error(s): %v, wanted no error", errs.ToAggregate().Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAllowListMapping(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
input map[string]string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
"validated",
|
||||
map[string]string{
|
||||
"metric_name,label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"metric name is not valid",
|
||||
map[string]string{
|
||||
"-metric_name,label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"label name is not valid",
|
||||
map[string]string{
|
||||
"metric_name,:label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"no label name",
|
||||
map[string]string{
|
||||
"metric_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
errs := validateAllowListMapping(tt.input, field.NewPath("allowListMapping"))
|
||||
if len(errs) == 0 && tt.expectedError {
|
||||
t.Error("Got no error, wanted error(s)")
|
||||
}
|
||||
if len(errs) != 0 && !tt.expectedError {
|
||||
t.Errorf("Got error: %v, wanted no error(s)", errs.ToAggregate().Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
34
staging/src/k8s.io/component-base/metrics/api/v1/doc.go
Normal file
34
staging/src/k8s.io/component-base/metrics/api/v1/doc.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// +k8s:deepcopy-gen=package
|
||||
|
||||
// Package v1 contains the configuration API for metrics.
|
||||
//
|
||||
// The intention is to only have a single version of this API, potentially with
|
||||
// new fields added over time in a backwards-compatible manner. Fields for
|
||||
// alpha or beta features are allowed as long as they are defined so that not
|
||||
// changing the defaults leaves those features disabled.
|
||||
//
|
||||
// The "v1" package name is just a reminder that API compatibility rules apply,
|
||||
// not an indication of the stability of all features covered by it.
|
||||
//
|
||||
// NOTE: Component owners are advised to rely on `k8s.io/component-base/metrics` to operate upon
|
||||
// `k8s.io/component-base/metrics/api/v1.MetricsConfiguration` as the former contains functions to apply and validate
|
||||
// the configuration, which in turn rely on members of the same package, which cannot be moved,
|
||||
// or imported (cyclic dependency) here.
|
||||
|
||||
package v1 // import "k8s.io/component-base/metrics/api/v1"
|
||||
39
staging/src/k8s.io/component-base/metrics/api/v1/types.go
Normal file
39
staging/src/k8s.io/component-base/metrics/api/v1/types.go
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1
|
||||
|
||||
// MetricsConfiguration contains all metrics options.
|
||||
type MetricsConfiguration struct {
|
||||
// ShowHiddenMetricsForVersion is the previous version for which you want to show hidden metrics.
|
||||
// Only the previous minor version is meaningful, other values will not be allowed.
|
||||
// The format is <major>.<minor>, e.g.: '1.16'.
|
||||
// The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics,
|
||||
// rather than being surprised when they are permanently removed in the release after that.
|
||||
ShowHiddenMetricsForVersion string `json:"showHiddenMetricsForVersion,omitempty"`
|
||||
// DisabledMetrics is a list of fully qualified metric names that should be disabled.
|
||||
// Disabling metrics is higher in precedence than showing hidden metrics.
|
||||
DisabledMetrics []string `json:"disabledMetrics,omitempty"`
|
||||
// AllowListMapping is the map from metric-label to value allow-list of this label.
|
||||
// The key's format is <MetricName>,<LabelName>, while its value is a list of allowed values for that label of that metric, i.e., <allowed_value>,<allowed_value>,...
|
||||
// For e.g., metric1,label1='v1,v2,v3', metric1,label2='v1,v2,v3' metric2,label1='v1,v2,v3'."
|
||||
AllowListMapping map[string]string `json:"allowListMapping,omitempty"`
|
||||
// The path to the manifest file that contains the allow-list mapping. Provided for convenience over AllowListMapping.
|
||||
// The file contents must represent a map of string keys and values, i.e.,
|
||||
// "metric1,label1": "value11,value12"
|
||||
// "metric2,label2": ""
|
||||
AllowListMappingManifest string `json:"allowListMappingManifest,omitempty"`
|
||||
}
|
||||
50
staging/src/k8s.io/component-base/metrics/api/v1/zz_generated.deepcopy.go
generated
Normal file
50
staging/src/k8s.io/component-base/metrics/api/v1/zz_generated.deepcopy.go
generated
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
//go:build !ignore_autogenerated
|
||||
// +build !ignore_autogenerated
|
||||
|
||||
/*
|
||||
Copyright The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by deepcopy-gen. DO NOT EDIT.
|
||||
|
||||
package v1
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *MetricsConfiguration) DeepCopyInto(out *MetricsConfiguration) {
|
||||
*out = *in
|
||||
if in.DisabledMetrics != nil {
|
||||
in, out := &in.DisabledMetrics, &out.DisabledMetrics
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.AllowListMapping != nil {
|
||||
in, out := &in.AllowListMapping, &out.AllowListMapping
|
||||
*out = make(map[string]string, len(*in))
|
||||
for key, val := range *in {
|
||||
(*out)[key] = val
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsConfiguration.
|
||||
func (in *MetricsConfiguration) DeepCopy() *MetricsConfiguration {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(MetricsConfiguration)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
|
@ -415,7 +415,7 @@ func TestCounterWithLabelValueAllowList(t *testing.T) {
|
|||
})
|
||||
c := NewCounterVec(opts, labels)
|
||||
registry.MustRegister(c)
|
||||
SetLabelAllowListFromCLI(labelAllowValues)
|
||||
SetLabelAllowList(labelAllowValues)
|
||||
for _, lv := range test.labelValues {
|
||||
c.WithLabelValues(lv...).Inc()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ func (d *Desc) determineDeprecationStatus(currentVersion semver.Version) {
|
|||
d.markDeprecationOnce.Do(func() {
|
||||
d.isDeprecated = isDeprecated(currentVersion, *deprecatedVersion)
|
||||
if shouldHide(d.stabilityLevel, ¤tVersion, deprecatedVersion) {
|
||||
if shouldShowHidden() {
|
||||
if ShouldShowHidden() {
|
||||
klog.Warningf("Hidden metrics(%s) have been manually overridden, showing this very deprecated metric.", d.fqName)
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -560,7 +560,7 @@ func TestGaugeWithLabelValueAllowList(t *testing.T) {
|
|||
})
|
||||
g := NewGaugeVec(opts, labels)
|
||||
registry.MustRegister(g)
|
||||
SetLabelAllowListFromCLI(labelAllowValues)
|
||||
SetLabelAllowList(labelAllowValues)
|
||||
for _, lv := range test.labelValues {
|
||||
g.WithLabelValues(lv...).Set(100.0)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -447,7 +447,7 @@ func TestHistogramWithLabelValueAllowList(t *testing.T) {
|
|||
})
|
||||
c := NewHistogramVec(opts, labels)
|
||||
registry.MustRegister(c)
|
||||
SetLabelAllowListFromCLI(labelAllowValues)
|
||||
SetLabelAllowList(labelAllowValues)
|
||||
|
||||
for _, lv := range test.labelValues {
|
||||
c.WithLabelValues(lv...).Observe(1.0)
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ func (r *lazyMetric) preprocessMetric(currentVersion semver.Version) {
|
|||
r.isDeprecated.Store(isDeprecated(currentVersion, *deprecatedVersion))
|
||||
|
||||
if shouldHide(r.stabilityLevel, ¤tVersion, deprecatedVersion) {
|
||||
if shouldShowHidden() {
|
||||
if ShouldShowHidden() {
|
||||
klog.Warningf("Hidden metrics (%s) have been manually overridden, showing this very deprecated metric.", r.fqName)
|
||||
return
|
||||
}
|
||||
|
|
@ -189,7 +189,7 @@ func (r *lazyMetric) FQName() string {
|
|||
|
||||
/*
|
||||
This code is directly lifted from the prometheus codebase. It's a convenience struct which
|
||||
allows you satisfy the Collector interface automatically if you already satisfy the Metric interface.
|
||||
allows you to satisfy the Collector interface automatically if you already satisfy the Metric interface.
|
||||
|
||||
For reference: https://github.com/prometheus/client_golang/blob/v0.9.2/prometheus/collector.go#L98-L120
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -17,21 +17,59 @@ limitations under the License.
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/blang/semver/v4"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
"k8s.io/component-base/version"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
"go.yaml.in/yaml/v2"
|
||||
"k8s.io/component-base/metrics/api/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
disabledMetricsLock sync.RWMutex
|
||||
disabledMetrics = map[string]struct{}{}
|
||||
showHiddenOnce sync.Once
|
||||
showHidden atomic.Bool
|
||||
)
|
||||
|
||||
var (
|
||||
disabledMetricsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "disabled_metrics_total",
|
||||
Help: "The count of disabled metrics.",
|
||||
StabilityLevel: BETA,
|
||||
},
|
||||
)
|
||||
|
||||
hiddenMetricsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "hidden_metrics_total",
|
||||
Help: "The count of hidden metrics.",
|
||||
StabilityLevel: BETA,
|
||||
},
|
||||
)
|
||||
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "cardinality_enforcement_unexpected_categorizations_total",
|
||||
Help: "The count of unexpected categorizations during cardinality enforcement.",
|
||||
StabilityLevel: ALPHA,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
// Options has all parameters needed for exposing metrics from components
|
||||
type Options struct {
|
||||
ShowHiddenMetricsForVersion string
|
||||
DisabledMetrics []string
|
||||
AllowListMapping map[string]string
|
||||
AllowListMappingManifest string
|
||||
// Configuration serialization is omitted here since the parent is never expected to be embedded.
|
||||
v1.MetricsConfiguration `json:"-"`
|
||||
}
|
||||
|
||||
// NewOptions returns default metrics options
|
||||
|
|
@ -39,29 +77,8 @@ func NewOptions() *Options {
|
|||
return &Options{}
|
||||
}
|
||||
|
||||
// Validate validates metrics flags options.
|
||||
func (o *Options) Validate() []error {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errs []error
|
||||
err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), o.ShowHiddenMetricsForVersion)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
if err := validateAllowMetricLabel(o.AllowListMapping); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// AddFlags adds flags for exposing component metrics.
|
||||
// This won't be called in embedded instances within component configurations.
|
||||
func (o *Options) AddFlags(fs *pflag.FlagSet) {
|
||||
if o == nil {
|
||||
return
|
||||
|
|
@ -84,53 +101,175 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
|
|||
"e.g. metric1,label1='v1,v2,v3', metric1,label2='v1,v2,v3' metric2,label1='v1,v2,v3'.")
|
||||
fs.StringVar(&o.AllowListMappingManifest, "allow-metric-labels-manifest", o.AllowListMappingManifest,
|
||||
"The path to the manifest file that contains the allow-list mapping. "+
|
||||
"The format of the file is the same as the flag --allow-metric-labels. "+
|
||||
"The format of the file is the same as the flag --allow-metric-labels, i.e., \n"+
|
||||
"allowListMapping:\n \"metric1,label1\": \"value11,value12\"\n \"metric2,label2\": \"\"\n"+
|
||||
"Note that the flag --allow-metric-labels will override the manifest file.")
|
||||
}
|
||||
|
||||
// SetShowHidden will enable showing hidden metrics. This will no-opt
|
||||
// after the initial call
|
||||
func SetShowHidden() {
|
||||
showHiddenOnce.Do(func() {
|
||||
showHidden.Store(true)
|
||||
|
||||
// re-register collectors that has been hidden in phase of last registry.
|
||||
for _, r := range registries {
|
||||
r.enableHiddenCollectors()
|
||||
r.enableHiddenStableCollectors()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ShouldShowHidden returns whether showing hidden deprecated metrics is enabled.
|
||||
// While the primary use case for this is internal (to determine registration behavior) this can also be used to introspect.
|
||||
func ShouldShowHidden() bool {
|
||||
return showHidden.Load()
|
||||
}
|
||||
|
||||
// SetDisabledMetric will disable a metric by name.
|
||||
// This will also increment the disabled metrics counter.
|
||||
// Note that this is a no-op if the metric is already disabled.
|
||||
func SetDisabledMetrics(names []string) {
|
||||
for _, name := range names {
|
||||
func(name string) {
|
||||
// An empty metric name is not a valid Prometheus metric.
|
||||
if name == "" {
|
||||
klog.Warningf("Attempted to disable an empty metric name, ignoring.")
|
||||
return
|
||||
}
|
||||
disabledMetricsLock.Lock()
|
||||
defer disabledMetricsLock.Unlock()
|
||||
if _, ok := disabledMetrics[name]; !ok {
|
||||
disabledMetrics[name] = struct{}{}
|
||||
disabledMetricsTotal.Inc()
|
||||
}
|
||||
}(name)
|
||||
}
|
||||
}
|
||||
|
||||
type MetricLabelAllowList struct {
|
||||
labelToAllowList map[string]sets.Set[string]
|
||||
}
|
||||
|
||||
func (allowList *MetricLabelAllowList) ConstrainToAllowedList(labelNameList, labelValueList []string) {
|
||||
for index, value := range labelValueList {
|
||||
name := labelNameList[index]
|
||||
if allowValues, ok := allowList.labelToAllowList[name]; ok {
|
||||
if !allowValues.Has(value) {
|
||||
labelValueList[index] = "unexpected"
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (allowList *MetricLabelAllowList) ConstrainLabelMap(labels map[string]string) {
|
||||
for name, value := range labels {
|
||||
if allowValues, ok := allowList.labelToAllowList[name]; ok {
|
||||
if !allowValues.Has(value) {
|
||||
labels[name] = "unexpected"
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SetLabelAllowList(allowListMapping map[string]string) {
|
||||
if len(allowListMapping) == 0 {
|
||||
klog.Errorf("empty allow-list mapping supplied, ignoring.")
|
||||
return
|
||||
}
|
||||
|
||||
allowListLock.Lock()
|
||||
defer allowListLock.Unlock()
|
||||
for metricLabelName, labelValues := range allowListMapping {
|
||||
metricName := strings.Split(metricLabelName, ",")[0]
|
||||
labelName := strings.Split(metricLabelName, ",")[1]
|
||||
valueSet := sets.New[string](strings.Split(labelValues, ",")...)
|
||||
|
||||
allowList, ok := labelValueAllowLists[metricName]
|
||||
if ok {
|
||||
allowList.labelToAllowList[labelName] = valueSet
|
||||
} else {
|
||||
labelToAllowList := make(map[string]sets.Set[string])
|
||||
labelToAllowList[labelName] = valueSet
|
||||
labelValueAllowLists[metricName] = &MetricLabelAllowList{
|
||||
labelToAllowList,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SetLabelAllowListFromManifest(manifest string) {
|
||||
if manifest == "" {
|
||||
klog.Errorf("The manifest file is empty, ignoring.")
|
||||
return
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Clean(manifest))
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to read allow list manifest: %v", err)
|
||||
return
|
||||
}
|
||||
allowListMapping := make(map[string]string)
|
||||
err = yaml.Unmarshal(data, &allowListMapping)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to parse allow list manifest: %v", err)
|
||||
return
|
||||
}
|
||||
SetLabelAllowList(allowListMapping)
|
||||
}
|
||||
|
||||
// Apply applies a MetricsConfiguration into global configuration of metrics.
|
||||
func Apply(c *v1.MetricsConfiguration) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if len(c.ShowHiddenMetricsForVersion) > 0 {
|
||||
SetShowHidden()
|
||||
}
|
||||
SetDisabledMetrics(c.DisabledMetrics)
|
||||
if c.AllowListMapping != nil {
|
||||
SetLabelAllowList(c.AllowListMapping)
|
||||
} else {
|
||||
SetLabelAllowListFromManifest(c.AllowListMappingManifest)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply applies parameters into global configuration of metrics.
|
||||
func (o *Options) Apply() {
|
||||
if o == nil {
|
||||
return
|
||||
}
|
||||
if len(o.ShowHiddenMetricsForVersion) > 0 {
|
||||
SetShowHidden()
|
||||
}
|
||||
// set disabled metrics
|
||||
for _, metricName := range o.DisabledMetrics {
|
||||
SetDisabledMetric(metricName)
|
||||
}
|
||||
if o.AllowListMapping != nil {
|
||||
SetLabelAllowListFromCLI(o.AllowListMapping)
|
||||
} else if len(o.AllowListMappingManifest) > 0 {
|
||||
SetLabelAllowListFromManifest(o.AllowListMappingManifest)
|
||||
}
|
||||
|
||||
Apply(&o.MetricsConfiguration)
|
||||
}
|
||||
|
||||
func validateShowHiddenMetricsVersion(currentVersion semver.Version, targetVersionStr string) error {
|
||||
if targetVersionStr == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
validVersionStr := fmt.Sprintf("%d.%d", currentVersion.Major, currentVersion.Minor-1)
|
||||
if targetVersionStr != validVersionStr {
|
||||
return fmt.Errorf("--show-hidden-metrics-for-version must be omitted or have the value '%v'. Only the previous minor version is allowed", validVersionStr)
|
||||
// ValidateShowHiddenMetricsVersion checks invalid version for which show hidden metrics.
|
||||
// TODO: This is kept here for backward compatibility in Kubelet (as metrics configuration fields were exposed on an individual basis earlier).
|
||||
// TODO: Revisit this after Kubelet supports the new metrics configuration API.
|
||||
func ValidateShowHiddenMetricsVersion(v string) []error {
|
||||
err := v1.ValidateShowHiddenMetricsVersionForKubeletBackwardCompatOnly(parseVersion(version.Get()), v)
|
||||
if err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAllowMetricLabel(allowListMapping map[string]string) error {
|
||||
if allowListMapping == nil {
|
||||
// Validate validates metrics flags options.
|
||||
func (o *Options) Validate() []error {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
metricNameRegex := `[a-zA-Z_:][a-zA-Z0-9_:]*`
|
||||
labelRegex := `[a-zA-Z_][a-zA-Z0-9_]*`
|
||||
for k := range allowListMapping {
|
||||
reg := regexp.MustCompile(metricNameRegex + `,` + labelRegex)
|
||||
if reg.FindString(k) != k {
|
||||
return fmt.Errorf("--allow-metric-labels must have a list of kv pair with format `metricName,labelName=labelValue, labelValue,...`")
|
||||
}
|
||||
currentVersion := parseVersion(version.Get())
|
||||
fldPath := field.NewPath("metrics")
|
||||
fldErrs := v1.Validate(&o.MetricsConfiguration, currentVersion, fldPath)
|
||||
|
||||
if len(fldErrs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
|
||||
return fldErrs.ToAggregate().Errors()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,52 +16,256 @@ limitations under the License.
|
|||
|
||||
package metrics
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
apimachineryversion "k8s.io/apimachinery/pkg/version"
|
||||
)
|
||||
|
||||
func TestEnableHiddenMetrics(t *testing.T) {
|
||||
currentVersion := apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.1-alpha-1.12345",
|
||||
}
|
||||
|
||||
func TestValidateAllowMetricLabel(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
input map[string]string
|
||||
expectedError bool
|
||||
name string
|
||||
fqName string
|
||||
counter *Counter
|
||||
mustRegister bool
|
||||
expectedMetric string
|
||||
}{
|
||||
{
|
||||
"validated",
|
||||
map[string]string{
|
||||
"metric_name,label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
false,
|
||||
name: "hide by register",
|
||||
fqName: "hidden_metric_register",
|
||||
counter: NewCounter(&CounterOpts{
|
||||
Name: "hidden_metric_register",
|
||||
Help: "counter help",
|
||||
StabilityLevel: STABLE,
|
||||
DeprecatedVersion: "1.14.0",
|
||||
}),
|
||||
mustRegister: false,
|
||||
expectedMetric: `
|
||||
# HELP hidden_metrics_total [BETA] The count of hidden metrics.
|
||||
# TYPE hidden_metrics_total counter
|
||||
hidden_metrics_total 1
|
||||
# HELP hidden_metric_register [STABLE] (Deprecated since 1.14.0) counter help
|
||||
# TYPE hidden_metric_register counter
|
||||
hidden_metric_register 1
|
||||
`,
|
||||
},
|
||||
{
|
||||
"metric name is not valid",
|
||||
map[string]string{
|
||||
"-metric_name,label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"label name is not valid",
|
||||
map[string]string{
|
||||
"metric_name,:label_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"no label name",
|
||||
map[string]string{
|
||||
"metric_name": "labelValue1,labelValue2",
|
||||
},
|
||||
true,
|
||||
name: "hide by must register",
|
||||
fqName: "hidden_metric_must_register",
|
||||
counter: NewCounter(&CounterOpts{
|
||||
Name: "hidden_metric_must_register",
|
||||
Help: "counter help",
|
||||
StabilityLevel: STABLE,
|
||||
DeprecatedVersion: "1.14.0",
|
||||
}),
|
||||
mustRegister: true,
|
||||
expectedMetric: `
|
||||
# HELP hidden_metric_must_register [STABLE] (Deprecated since 1.14.0) counter help
|
||||
# TYPE hidden_metric_must_register counter
|
||||
hidden_metric_must_register 1
|
||||
# HELP hidden_metrics_total [BETA] The count of hidden metrics.
|
||||
# TYPE hidden_metrics_total counter
|
||||
hidden_metrics_total 2
|
||||
`,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateAllowMetricLabel(tt.input)
|
||||
if err == nil && tt.expectedError {
|
||||
t.Error("Got error is nil, wanted error is not nil")
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
registry.MustRegister(hiddenMetricsTotal)
|
||||
if tc.mustRegister {
|
||||
registry.MustRegister(tc.counter)
|
||||
} else {
|
||||
_ = registry.Register(tc.counter)
|
||||
}
|
||||
if err != nil && !tt.expectedError {
|
||||
t.Errorf("Got error is %v, wanted no error", err)
|
||||
|
||||
tc.counter.Inc() // no-ops, because counter hasn't been initialized
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(""), tc.fqName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
SetShowHidden()
|
||||
defer func() {
|
||||
showHiddenOnce = sync.Once{}
|
||||
showHidden.Store(false)
|
||||
}()
|
||||
|
||||
tc.counter.Inc()
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectedMetric), tc.fqName, hiddenMetricsTotal.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnableHiddenStableCollector(t *testing.T) {
|
||||
var currentVersion = apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.0-alpha-1.12345",
|
||||
}
|
||||
var normal = NewDesc("test_enable_hidden_custom_metric_normal", "this is a normal metric", []string{"name"}, nil, STABLE, "")
|
||||
var hiddenA = NewDesc("test_enable_hidden_custom_metric_hidden_a", "this is the hidden metric A", []string{"name"}, nil, STABLE, "1.14.0")
|
||||
var hiddenB = NewDesc("test_enable_hidden_custom_metric_hidden_b", "this is the hidden metric B", []string{"name"}, nil, STABLE, "1.14.0")
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
descriptors []*Desc
|
||||
metricNames []string
|
||||
expectMetricsBeforeEnable string
|
||||
expectMetricsAfterEnable string
|
||||
}{
|
||||
{
|
||||
name: "all hidden",
|
||||
descriptors: []*Desc{hiddenA, hiddenB},
|
||||
metricNames: []string{"test_enable_hidden_custom_metric_hidden_a",
|
||||
"test_enable_hidden_custom_metric_hidden_b"},
|
||||
expectMetricsBeforeEnable: "",
|
||||
expectMetricsAfterEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_a gauge
|
||||
test_enable_hidden_custom_metric_hidden_a{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_b gauge
|
||||
test_enable_hidden_custom_metric_hidden_b{name="value"} 1
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "partial hidden",
|
||||
descriptors: []*Desc{normal, hiddenA, hiddenB},
|
||||
metricNames: []string{"test_enable_hidden_custom_metric_normal",
|
||||
"test_enable_hidden_custom_metric_hidden_a",
|
||||
"test_enable_hidden_custom_metric_hidden_b"},
|
||||
expectMetricsBeforeEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_normal [STABLE] this is a normal metric
|
||||
# TYPE test_enable_hidden_custom_metric_normal gauge
|
||||
test_enable_hidden_custom_metric_normal{name="value"} 1
|
||||
`,
|
||||
expectMetricsAfterEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_normal [STABLE] this is a normal metric
|
||||
# TYPE test_enable_hidden_custom_metric_normal gauge
|
||||
test_enable_hidden_custom_metric_normal{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_a gauge
|
||||
test_enable_hidden_custom_metric_hidden_a{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_b gauge
|
||||
test_enable_hidden_custom_metric_hidden_b{name="value"} 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
customCollector := newTestCustomCollector(tc.descriptors...)
|
||||
registry.CustomMustRegister(customCollector)
|
||||
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetricsBeforeEnable), tc.metricNames...); err != nil {
|
||||
t.Fatalf("before enable test failed: %v", err)
|
||||
}
|
||||
|
||||
SetShowHidden()
|
||||
defer func() {
|
||||
showHiddenOnce = sync.Once{}
|
||||
showHidden.Store(false)
|
||||
}()
|
||||
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetricsAfterEnable), tc.metricNames...); err != nil {
|
||||
t.Fatalf("after enable test failed: %v", err)
|
||||
}
|
||||
|
||||
// refresh descriptors to share with cases.
|
||||
for _, d := range tc.descriptors {
|
||||
d.ClearState()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShowHiddenMetric(t *testing.T) {
|
||||
registry := newKubeRegistry(apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "15",
|
||||
GitVersion: "v1.15.0-alpha-1.12345",
|
||||
})
|
||||
|
||||
expectedMetricCount := 0
|
||||
registry.MustRegister(alphaHiddenCounter)
|
||||
|
||||
ms, err := registry.Gather()
|
||||
require.NoError(t, err, "Gather failed %v", err)
|
||||
assert.Lenf(t, ms, expectedMetricCount, "Got %v metrics, Want: %v metrics", len(ms), expectedMetricCount)
|
||||
|
||||
showHidden.Store(true)
|
||||
defer showHidden.Store(false)
|
||||
registry.MustRegister(NewCounter(
|
||||
&CounterOpts{
|
||||
Namespace: "some_namespace",
|
||||
Name: "test_alpha_show_hidden_counter",
|
||||
Subsystem: "subsystem",
|
||||
StabilityLevel: ALPHA,
|
||||
Help: "counter help",
|
||||
DeprecatedVersion: "1.14.0",
|
||||
},
|
||||
))
|
||||
expectedMetricCount = 1
|
||||
|
||||
ms, err = registry.Gather()
|
||||
require.NoError(t, err, "Gather failed %v", err)
|
||||
assert.Lenf(t, ms, expectedMetricCount, "Got %v metrics, Want: %v metrics", len(ms), expectedMetricCount)
|
||||
}
|
||||
|
||||
func TestDisabledMetrics(t *testing.T) {
|
||||
o := NewOptions()
|
||||
o.DisabledMetrics = []string{"should_be_disabled", "should_be_disabled"} // should be deduplicated (disabled_metrics_total == 1)
|
||||
currentVersion := apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.1-alpha-1.12345",
|
||||
}
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
registry.MustRegister(disabledMetricsTotal)
|
||||
o.Apply()
|
||||
disabledMetric := NewCounterVec(&CounterOpts{
|
||||
Name: "should_be_disabled",
|
||||
Help: "this metric should be disabled",
|
||||
}, []string{"label"})
|
||||
// gauges cannot be reset
|
||||
enabledMetric := NewGauge(&GaugeOpts{
|
||||
Name: "should_be_enabled",
|
||||
Help: "this metric should not be disabled",
|
||||
})
|
||||
|
||||
registry.MustRegister(disabledMetric)
|
||||
registry.MustRegister(enabledMetric)
|
||||
disabledMetric.WithLabelValues("one").Inc()
|
||||
disabledMetric.WithLabelValues("two").Inc()
|
||||
disabledMetric.WithLabelValues("two").Inc()
|
||||
enabledMetric.Inc()
|
||||
|
||||
enabledMetricOutput := `# HELP disabled_metrics_total [BETA] The count of disabled metrics.
|
||||
# TYPE disabled_metrics_total counter
|
||||
disabled_metrics_total 1
|
||||
# HELP should_be_enabled [ALPHA] this metric should not be disabled
|
||||
# TYPE should_be_enabled gauge
|
||||
should_be_enabled 1
|
||||
`
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(enabledMetricOutput), "should_be_disabled", "should_be_enabled", disabledMetricsTotal.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,18 +18,12 @@ package metrics
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
yaml "go.yaml.in/yaml/v2"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
promext "k8s.io/component-base/metrics/prometheusextension"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -325,66 +319,3 @@ func (o *SummaryOpts) toPromSummaryOpts() prometheus.SummaryOpts {
|
|||
BufCap: o.BufCap,
|
||||
}
|
||||
}
|
||||
|
||||
type MetricLabelAllowList struct {
|
||||
labelToAllowList map[string]sets.Set[string]
|
||||
}
|
||||
|
||||
func (allowList *MetricLabelAllowList) ConstrainToAllowedList(labelNameList, labelValueList []string) {
|
||||
for index, value := range labelValueList {
|
||||
name := labelNameList[index]
|
||||
if allowValues, ok := allowList.labelToAllowList[name]; ok {
|
||||
if !allowValues.Has(value) {
|
||||
labelValueList[index] = "unexpected"
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (allowList *MetricLabelAllowList) ConstrainLabelMap(labels map[string]string) {
|
||||
for name, value := range labels {
|
||||
if allowValues, ok := allowList.labelToAllowList[name]; ok {
|
||||
if !allowValues.Has(value) {
|
||||
labels[name] = "unexpected"
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SetLabelAllowListFromCLI(allowListMapping map[string]string) {
|
||||
allowListLock.Lock()
|
||||
defer allowListLock.Unlock()
|
||||
for metricLabelName, labelValues := range allowListMapping {
|
||||
metricName := strings.Split(metricLabelName, ",")[0]
|
||||
labelName := strings.Split(metricLabelName, ",")[1]
|
||||
valueSet := sets.New[string](strings.Split(labelValues, ",")...)
|
||||
|
||||
allowList, ok := labelValueAllowLists[metricName]
|
||||
if ok {
|
||||
allowList.labelToAllowList[labelName] = valueSet
|
||||
} else {
|
||||
labelToAllowList := make(map[string]sets.Set[string])
|
||||
labelToAllowList[labelName] = valueSet
|
||||
labelValueAllowLists[metricName] = &MetricLabelAllowList{
|
||||
labelToAllowList,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SetLabelAllowListFromManifest(manifest string) {
|
||||
allowListMapping := make(map[string]string)
|
||||
data, err := os.ReadFile(filepath.Clean(manifest))
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to read allow list manifest: %v", err)
|
||||
return
|
||||
}
|
||||
err = yaml.Unmarshal(data, &allowListMapping)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to parse allow list manifest: %v", err)
|
||||
return
|
||||
}
|
||||
SetLabelAllowListFromCLI(allowListMapping)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package metrics
|
|||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/blang/semver/v4"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
|
@ -30,12 +29,8 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
showHiddenOnce sync.Once
|
||||
disabledMetricsLock sync.RWMutex
|
||||
showHidden atomic.Bool
|
||||
registries []*kubeRegistry // stores all registries created by NewKubeRegistry()
|
||||
registriesLock sync.RWMutex
|
||||
disabledMetrics = map[string]struct{}{}
|
||||
registries []*kubeRegistry // stores all registries created by NewKubeRegistry()
|
||||
registriesLock sync.RWMutex
|
||||
|
||||
registeredMetricsTotal = NewCounterVec(
|
||||
&CounterOpts{
|
||||
|
|
@ -45,30 +40,6 @@ var (
|
|||
},
|
||||
[]string{"stability_level", "deprecated_version"},
|
||||
)
|
||||
|
||||
disabledMetricsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "disabled_metrics_total",
|
||||
Help: "The count of disabled metrics.",
|
||||
StabilityLevel: BETA,
|
||||
},
|
||||
)
|
||||
|
||||
hiddenMetricsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "hidden_metrics_total",
|
||||
Help: "The count of hidden metrics.",
|
||||
StabilityLevel: BETA,
|
||||
},
|
||||
)
|
||||
|
||||
cardinalityEnforcementUnexpectedCategorizationsTotal = NewCounter(
|
||||
&CounterOpts{
|
||||
Name: "cardinality_enforcement_unexpected_categorizations_total",
|
||||
Help: "The count of unexpected categorizations during cardinality enforcement.",
|
||||
StabilityLevel: ALPHA,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
// shouldHide is used to check if a specific metric with deprecated version should be hidden
|
||||
|
|
@ -125,44 +96,6 @@ func isDeprecated(currentVersion, deprecatedVersion semver.Version) bool {
|
|||
return currentVersion.Minor >= deprecatedVersion.Minor
|
||||
}
|
||||
|
||||
// ValidateShowHiddenMetricsVersion checks invalid version for which show hidden metrics.
|
||||
func ValidateShowHiddenMetricsVersion(v string) []error {
|
||||
err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), v)
|
||||
if err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetDisabledMetric(name string) {
|
||||
disabledMetricsLock.Lock()
|
||||
defer disabledMetricsLock.Unlock()
|
||||
disabledMetrics[name] = struct{}{}
|
||||
disabledMetricsTotal.Inc()
|
||||
}
|
||||
|
||||
// SetShowHidden will enable showing hidden metrics. This will no-opt
|
||||
// after the initial call
|
||||
func SetShowHidden() {
|
||||
showHiddenOnce.Do(func() {
|
||||
showHidden.Store(true)
|
||||
|
||||
// re-register collectors that has been hidden in phase of last registry.
|
||||
for _, r := range registries {
|
||||
r.enableHiddenCollectors()
|
||||
r.enableHiddenStableCollectors()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// shouldShowHidden returns whether showing hidden deprecated metrics
|
||||
// is enabled. While the primary usecase for this is internal (to determine
|
||||
// registration behavior) this can also be used to introspect
|
||||
func shouldShowHidden() bool {
|
||||
return showHidden.Load()
|
||||
}
|
||||
|
||||
// Registerable is an interface for a collector metric which we
|
||||
// will register with KubeRegistry.
|
||||
type Registerable interface {
|
||||
|
|
@ -350,7 +283,7 @@ func (kr *kubeRegistry) trackStableCollectors(cs ...StableCollector) {
|
|||
kr.stableCollectors = append(kr.stableCollectors, cs...)
|
||||
}
|
||||
|
||||
// enableHiddenCollectors will re-register all of the hidden collectors.
|
||||
// enableHiddenCollectors will re-register all the hidden collectors.
|
||||
func (kr *kubeRegistry) enableHiddenCollectors() {
|
||||
if len(kr.hiddenCollectors) == 0 {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -18,15 +18,12 @@ package metrics
|
|||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/blang/semver/v4"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
apimachineryversion "k8s.io/apimachinery/pkg/version"
|
||||
)
|
||||
|
||||
|
|
@ -395,254 +392,6 @@ func TestMustRegister(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
func TestShowHiddenMetric(t *testing.T) {
|
||||
registry := newKubeRegistry(apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "15",
|
||||
GitVersion: "v1.15.0-alpha-1.12345",
|
||||
})
|
||||
|
||||
expectedMetricCount := 0
|
||||
registry.MustRegister(alphaHiddenCounter)
|
||||
|
||||
ms, err := registry.Gather()
|
||||
require.NoError(t, err, "Gather failed %v", err)
|
||||
assert.Lenf(t, ms, expectedMetricCount, "Got %v metrics, Want: %v metrics", len(ms), expectedMetricCount)
|
||||
|
||||
showHidden.Store(true)
|
||||
defer showHidden.Store(false)
|
||||
registry.MustRegister(NewCounter(
|
||||
&CounterOpts{
|
||||
Namespace: "some_namespace",
|
||||
Name: "test_alpha_show_hidden_counter",
|
||||
Subsystem: "subsystem",
|
||||
StabilityLevel: ALPHA,
|
||||
Help: "counter help",
|
||||
DeprecatedVersion: "1.14.0",
|
||||
},
|
||||
))
|
||||
expectedMetricCount = 1
|
||||
|
||||
ms, err = registry.Gather()
|
||||
require.NoError(t, err, "Gather failed %v", err)
|
||||
assert.Lenf(t, ms, expectedMetricCount, "Got %v metrics, Want: %v metrics", len(ms), expectedMetricCount)
|
||||
}
|
||||
|
||||
func TestValidateShowHiddenMetricsVersion(t *testing.T) {
|
||||
currentVersion := parseVersion(apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.1-alpha-1.12345",
|
||||
})
|
||||
|
||||
var tests = []struct {
|
||||
desc string
|
||||
targetVersion string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
desc: "invalid version is not allowed",
|
||||
targetVersion: "1.invalid",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "patch version is not allowed",
|
||||
targetVersion: "1.16.0",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "old version is not allowed",
|
||||
targetVersion: "1.15",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "new version is not allowed",
|
||||
targetVersion: "1.17",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
desc: "valid version is allowed",
|
||||
targetVersion: "1.16",
|
||||
expectedError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
err := validateShowHiddenMetricsVersion(currentVersion, tc.targetVersion)
|
||||
|
||||
if tc.expectedError {
|
||||
assert.Errorf(t, err, "Failed to test: %s", tc.desc)
|
||||
} else {
|
||||
assert.NoErrorf(t, err, "Failed to test: %s", tc.desc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnableHiddenMetrics(t *testing.T) {
|
||||
currentVersion := apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.1-alpha-1.12345",
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
fqName string
|
||||
counter *Counter
|
||||
mustRegister bool
|
||||
expectedMetric string
|
||||
}{
|
||||
{
|
||||
name: "hide by register",
|
||||
fqName: "hidden_metric_register",
|
||||
counter: NewCounter(&CounterOpts{
|
||||
Name: "hidden_metric_register",
|
||||
Help: "counter help",
|
||||
StabilityLevel: STABLE,
|
||||
DeprecatedVersion: "1.14.0",
|
||||
}),
|
||||
mustRegister: false,
|
||||
expectedMetric: `
|
||||
# HELP hidden_metric_register [STABLE] (Deprecated since 1.14.0) counter help
|
||||
# TYPE hidden_metric_register counter
|
||||
hidden_metric_register 1
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "hide by must register",
|
||||
fqName: "hidden_metric_must_register",
|
||||
counter: NewCounter(&CounterOpts{
|
||||
Name: "hidden_metric_must_register",
|
||||
Help: "counter help",
|
||||
StabilityLevel: STABLE,
|
||||
DeprecatedVersion: "1.14.0",
|
||||
}),
|
||||
mustRegister: true,
|
||||
expectedMetric: `
|
||||
# HELP hidden_metric_must_register [STABLE] (Deprecated since 1.14.0) counter help
|
||||
# TYPE hidden_metric_must_register counter
|
||||
hidden_metric_must_register 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
if tc.mustRegister {
|
||||
registry.MustRegister(tc.counter)
|
||||
} else {
|
||||
_ = registry.Register(tc.counter)
|
||||
}
|
||||
|
||||
tc.counter.Inc() // no-ops, because counter hasn't been initialized
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(""), tc.fqName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
SetShowHidden()
|
||||
defer func() {
|
||||
showHiddenOnce = *new(sync.Once)
|
||||
showHidden.Store(false)
|
||||
}()
|
||||
|
||||
tc.counter.Inc()
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectedMetric), tc.fqName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnableHiddenStableCollector(t *testing.T) {
|
||||
var currentVersion = apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.0-alpha-1.12345",
|
||||
}
|
||||
var normal = NewDesc("test_enable_hidden_custom_metric_normal", "this is a normal metric", []string{"name"}, nil, STABLE, "")
|
||||
var hiddenA = NewDesc("test_enable_hidden_custom_metric_hidden_a", "this is the hidden metric A", []string{"name"}, nil, STABLE, "1.14.0")
|
||||
var hiddenB = NewDesc("test_enable_hidden_custom_metric_hidden_b", "this is the hidden metric B", []string{"name"}, nil, STABLE, "1.14.0")
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
descriptors []*Desc
|
||||
metricNames []string
|
||||
expectMetricsBeforeEnable string
|
||||
expectMetricsAfterEnable string
|
||||
}{
|
||||
{
|
||||
name: "all hidden",
|
||||
descriptors: []*Desc{hiddenA, hiddenB},
|
||||
metricNames: []string{"test_enable_hidden_custom_metric_hidden_a",
|
||||
"test_enable_hidden_custom_metric_hidden_b"},
|
||||
expectMetricsBeforeEnable: "",
|
||||
expectMetricsAfterEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_a gauge
|
||||
test_enable_hidden_custom_metric_hidden_a{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_b gauge
|
||||
test_enable_hidden_custom_metric_hidden_b{name="value"} 1
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "partial hidden",
|
||||
descriptors: []*Desc{normal, hiddenA, hiddenB},
|
||||
metricNames: []string{"test_enable_hidden_custom_metric_normal",
|
||||
"test_enable_hidden_custom_metric_hidden_a",
|
||||
"test_enable_hidden_custom_metric_hidden_b"},
|
||||
expectMetricsBeforeEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_normal [STABLE] this is a normal metric
|
||||
# TYPE test_enable_hidden_custom_metric_normal gauge
|
||||
test_enable_hidden_custom_metric_normal{name="value"} 1
|
||||
`,
|
||||
expectMetricsAfterEnable: `
|
||||
# HELP test_enable_hidden_custom_metric_normal [STABLE] this is a normal metric
|
||||
# TYPE test_enable_hidden_custom_metric_normal gauge
|
||||
test_enable_hidden_custom_metric_normal{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_a gauge
|
||||
test_enable_hidden_custom_metric_hidden_a{name="value"} 1
|
||||
# HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B
|
||||
# TYPE test_enable_hidden_custom_metric_hidden_b gauge
|
||||
test_enable_hidden_custom_metric_hidden_b{name="value"} 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
tc := test
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
customCollector := newTestCustomCollector(tc.descriptors...)
|
||||
registry.CustomMustRegister(customCollector)
|
||||
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetricsBeforeEnable), tc.metricNames...); err != nil {
|
||||
t.Fatalf("before enable test failed: %v", err)
|
||||
}
|
||||
|
||||
SetShowHidden()
|
||||
defer func() {
|
||||
showHiddenOnce = *new(sync.Once)
|
||||
showHidden.Store(false)
|
||||
}()
|
||||
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetricsAfterEnable), tc.metricNames...); err != nil {
|
||||
t.Fatalf("after enable test failed: %v", err)
|
||||
}
|
||||
|
||||
// refresh descriptors so as to share with cases.
|
||||
for _, d := range tc.descriptors {
|
||||
d.ClearState()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryReset(t *testing.T) {
|
||||
currentVersion := apimachineryversion.Info{
|
||||
|
|
@ -687,41 +436,3 @@ func TestRegistryReset(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledMetrics(t *testing.T) {
|
||||
o := NewOptions()
|
||||
o.DisabledMetrics = []string{"should_be_disabled"}
|
||||
o.Apply()
|
||||
currentVersion := apimachineryversion.Info{
|
||||
Major: "1",
|
||||
Minor: "17",
|
||||
GitVersion: "v1.17.1-alpha-1.12345",
|
||||
}
|
||||
registry := newKubeRegistry(currentVersion)
|
||||
disabledMetric := NewCounterVec(&CounterOpts{
|
||||
Name: "should_be_disabled",
|
||||
Help: "this metric should be disabled",
|
||||
}, []string{"label"})
|
||||
// gauges cannot be reset
|
||||
enabledMetric := NewGauge(&GaugeOpts{
|
||||
Name: "should_be_enabled",
|
||||
Help: "this metric should not be disabled",
|
||||
})
|
||||
|
||||
registry.MustRegister(disabledMetric)
|
||||
registry.MustRegister(enabledMetric)
|
||||
disabledMetric.WithLabelValues("one").Inc()
|
||||
disabledMetric.WithLabelValues("two").Inc()
|
||||
disabledMetric.WithLabelValues("two").Inc()
|
||||
enabledMetric.Inc()
|
||||
|
||||
enabledMetricOutput := `
|
||||
# HELP should_be_enabled [ALPHA] this metric should not be disabled
|
||||
# TYPE should_be_enabled gauge
|
||||
should_be_enabled 1
|
||||
`
|
||||
|
||||
if err := testutil.GatherAndCompare(registry, strings.NewReader(enabledMetricOutput), "should_be_disabled", "should_be_enabled"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -403,7 +403,7 @@ func TestSummaryWithLabelValueAllowList(t *testing.T) {
|
|||
})
|
||||
c := NewSummaryVec(opts, labels)
|
||||
registry.MustRegister(c)
|
||||
SetLabelAllowListFromCLI(labelAllowValues)
|
||||
SetLabelAllowList(labelAllowValues)
|
||||
|
||||
for _, lv := range test.labelValues {
|
||||
c.WithLabelValues(lv...).Observe(1.0)
|
||||
|
|
|
|||
|
|
@ -512,7 +512,7 @@ func TestTimingHistogramWithLabelValueAllowList(t *testing.T) {
|
|||
clk := testclock.NewFakePassiveClock(t0)
|
||||
c := NewTestableTimingHistogramVec(clk.Now, opts, labels)
|
||||
registry.MustRegister(c)
|
||||
SetLabelAllowListFromCLI(labelAllowValues)
|
||||
SetLabelAllowList(labelAllowValues)
|
||||
var v0 float64 = 13
|
||||
for _, lv := range test.labelValues {
|
||||
c.WithLabelValues(lv...).Set(v0)
|
||||
|
|
|
|||
Loading…
Reference in a new issue