diff --git a/discovery/kubernetes/endpoints_test.go b/discovery/kubernetes/endpoints_test.go index 0ac472324d..0427ad43c1 100644 --- a/discovery/kubernetes/endpoints_test.go +++ b/discovery/kubernetes/endpoints_test.go @@ -550,8 +550,8 @@ func TestEndpointsDiscoveryWithNodeMetadata(t *testing.T) { metadataConfig := AttachMetadataConfig{Node: true} nodeLabels1 := map[string]string{"az": "us-east1"} nodeLabels2 := map[string]string{"az": "us-west2"} - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", @@ -623,8 +623,8 @@ func TestEndpointsDiscoveryWithUpdatedNodeMetadata(t *testing.T) { t.Parallel() nodeLabels1 := map[string]string{"az": "us-east1"} nodeLabels2 := map[string]string{"az": "us-west2"} - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) metadataConfig := AttachMetadataConfig{Node: true} svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/discovery/kubernetes/endpointslice_test.go b/discovery/kubernetes/endpointslice_test.go index b4dc0c36ce..c4d838e223 100644 --- a/discovery/kubernetes/endpointslice_test.go +++ b/discovery/kubernetes/endpointslice_test.go @@ -654,7 +654,7 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) { }, }, } - objs := []runtime.Object{makeEndpointSliceV1("default"), makeNode("foobar", "", "", nodeLabels1, nil), makeNode("barbaz", "", "", nodeLabels2, nil), svc} + objs := []runtime.Object{makeEndpointSliceV1("default"), makeNode("foobar", "", "", nodeLabels1, nil, nil), makeNode("barbaz", "", "", nodeLabels2, nil, nil), svc} n, _ := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, objs...) k8sDiscoveryTest{ @@ -754,8 +754,8 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) { }, }, } - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) objs := []runtime.Object{makeEndpointSliceV1("default"), node1, node2, svc} n, c := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, objs...) diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go index cbc69dd0ca..08e2c604cf 100644 --- a/discovery/kubernetes/node.go +++ b/discovery/kubernetes/node.go @@ -20,6 +20,7 @@ import ( "log/slog" "net" "strconv" + "strings" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -160,6 +161,7 @@ func nodeSourceFromName(name string) string { const ( nodeProviderIDLabel = metaLabelPrefix + "node_provider_id" + nodeConditionPrefix = metaLabelPrefix + "node_condition_" nodeAddressPrefix = metaLabelPrefix + "node_address_" ) @@ -169,6 +171,13 @@ func nodeLabels(n *apiv1.Node) model.LabelSet { ls[nodeProviderIDLabel] = lv(n.Spec.ProviderID) + // Export all node conditions as individual meta labels + for _, condition := range n.Status.Conditions { + conditionType := strings.ToLower(string(condition.Type)) + labelName := nodeConditionPrefix + strutil.SanitizeLabelName(conditionType) + ls[model.LabelName(labelName)] = lv(strings.ToLower(string(condition.Status))) + } + addObjectMetaLabels(ls, n.ObjectMeta, RoleNode) return ls diff --git a/discovery/kubernetes/node_test.go b/discovery/kubernetes/node_test.go index 9e56b95bb9..4bee0a2e69 100644 --- a/discovery/kubernetes/node_test.go +++ b/discovery/kubernetes/node_test.go @@ -25,7 +25,7 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" ) -func makeNode(name, address, providerID string, labels, annotations map[string]string) *v1.Node { +func makeNode(name, address, providerID string, labels, annotations map[string]string, conditions []v1.NodeCondition) *v1.Node { return &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -47,12 +47,13 @@ func makeNode(name, address, providerID string, labels, annotations map[string]s Port: 10250, }, }, + Conditions: conditions, }, } } func makeEnumeratedNode(i int) *v1.Node { - return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", fmt.Sprintf("aws:///de-west-3a/i-%d", i), map[string]string{}, map[string]string{}) + return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", fmt.Sprintf("aws:///de-west-3a/i-%d", i), map[string]string{}, map[string]string{}, nil) } func TestNodeDiscoveryBeforeStart(t *testing.T) { @@ -68,6 +69,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) { "aws:///nl-north-7b/i-03149834983492827", map[string]string{"test-label": "testvalue"}, map[string]string{"test-annotation": "testannotationvalue"}, + nil, ) c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{}) }, @@ -159,6 +161,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) { "aws:///fr-south-1c/i-49508290343823952", map[string]string{"Unschedulable": "true"}, map[string]string{}, + nil, ) c.CoreV1().Nodes().Update(context.Background(), obj2, metav1.UpdateOptions{}) }, @@ -183,3 +186,122 @@ func TestNodeDiscoveryUpdate(t *testing.T) { }, }.Run(t) } + +func TestNodeDiscoveryConditions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + conditions []v1.NodeCondition + expected model.LabelSet + }{ + { + name: "node ready true", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "true", + }, + }, + { + name: "node ready false", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionFalse, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "false", + }, + }, + { + name: "node ready unknown", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "unknown", + }, + }, + { + name: "node no conditions", + conditions: nil, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + }, + }, + { + name: "node multiple conditions", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeMemoryPressure, + Status: v1.ConditionFalse, + }, + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + { + Type: v1.NodeDiskPressure, + Status: v1.ConditionFalse, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_memorypressure": "false", + "__meta_kubernetes_node_condition_ready": "true", + "__meta_kubernetes_node_condition_diskpressure": "false", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}) + + k8sDiscoveryTest{ + discovery: n, + beforeRun: func() { + obj := makeNode( + "test", + "1.2.3.4", + "aws:///test-zone/i-test", + map[string]string{}, + map[string]string{}, + tt.conditions, + ) + c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{}) + }, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "node/test": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:10250", + "instance": "test", + "__meta_kubernetes_node_address_InternalIP": "1.2.3.4", + }, + }, + Labels: tt.expected, + Source: "node/test", + }, + }, + }.Run(t) + }) + } +} diff --git a/discovery/kubernetes/pod_test.go b/discovery/kubernetes/pod_test.go index db5db546d0..6a7d335194 100644 --- a/discovery/kubernetes/pod_test.go +++ b/discovery/kubernetes/pod_test.go @@ -481,7 +481,7 @@ func TestPodDiscoveryWithNodeMetadata(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { - nodes := makeNode("testnode", "", "", nodeLbls, nil) + nodes := makeNode("testnode", "", "", nodeLbls, nil, nil) c.CoreV1().Nodes().Create(context.Background(), nodes, metav1.CreateOptions{}) pods := makePods("default") @@ -502,14 +502,14 @@ func TestPodDiscoveryWithNodeMetadataUpdateNode(t *testing.T) { discovery: n, beforeRun: func() { oldNodeLbls := map[string]string{"l1": "v1"} - nodes := makeNode("testnode", "", "", oldNodeLbls, nil) + nodes := makeNode("testnode", "", "", oldNodeLbls, nil, nil) c.CoreV1().Nodes().Create(context.Background(), nodes, metav1.CreateOptions{}) }, afterStart: func() { pods := makePods("default") c.CoreV1().Pods(pods.Namespace).Create(context.Background(), pods, metav1.CreateOptions{}) - nodes := makeNode("testnode", "", "", nodeLbls, nil) + nodes := makeNode("testnode", "", "", nodeLbls, nil, nil) c.CoreV1().Nodes().Update(context.Background(), nodes, metav1.UpdateOptions{}) }, expectedMaxItems: 2, diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 1f2f9931e8..06be6b91e8 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -2070,6 +2070,7 @@ Available meta labels: * `__meta_kubernetes_node_name`: The name of the node object. * `__meta_kubernetes_node_provider_id`: The cloud provider's name for the node object. +* `__meta_kubernetes_node_condition_`: For every entry in node.Status.Conditions, a label with the condition type in lowercase. Possible values are `true`, `false`, or `unknown`. Examples: `__meta_kubernetes_node_condition_ready`, `__meta_kubernetes_node_condition_memorypressure`, `__meta_kubernetes_node_condition_diskpressure`. * `__meta_kubernetes_node_label_`: Each label from the node object, with any unsupported characters converted to an underscore. * `__meta_kubernetes_node_labelpresent_`: `true` for each label from the node object, with any unsupported characters converted to an underscore. * `__meta_kubernetes_node_annotation_`: Each annotation from the node object.