test(hpa): add sample-external-metrics-server (#136251)

Signed-off-by: Omer Aplatony <omerap12@gmail.com>
2026-02-03 20:40:26 -05:00 · 2026-01-29 16:19:49 +02:00 · 2026-01-29 16:19:49 +02:00 · 10f1b28712
commit 10f1b28712
parent c99976db2a
7 changed files with 666 additions and 0 deletions
--- a/test/images/sample-external-metrics-server/BASEIMAGE
+++ b/test/images/sample-external-metrics-server/BASEIMAGE
@ -0,0 +1,5 @@
+linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6
+linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6
+linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6
+windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809
+windows/amd64/ltsc2022=mcr.microsoft.com/windows/nanoserver:ltsc2022
--- a/test/images/sample-external-metrics-server/Dockerfile
+++ b/test/images/sample-external-metrics-server/Dockerfile
@ -0,0 +1,20 @@
+# Copyright The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASEIMAGE
+FROM $BASEIMAGE
+
+ADD sample-external-metrics-server /sample-external-metrics-server
+EXPOSE 6443
+ENTRYPOINT ["/sample-external-metrics-server"]
--- a/test/images/sample-external-metrics-server/Makefile
+++ b/test/images/sample-external-metrics-server/Makefile
@ -0,0 +1,26 @@
+# Copyright The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SRCS = sample-external-metrics-server
+OS ?= linux
+ARCH ?= amd64
+TARGET ?= $(CURDIR)
+GOLANG_VERSION ?= latest
+SRC_DIR = $(notdir $(shell pwd))
+export
+
+bin:
+	../image-util.sh bin $(SRCS)
+
+.PHONY: bin
--- a/test/images/sample-external-metrics-server/README.md
+++ b/test/images/sample-external-metrics-server/README.md
@ -0,0 +1,118 @@
+# Test Metrics Server
+
+A Kubernetes external metrics API server for testing and development purposes. This server implements the `external.metrics.k8s.io` API and provides endpoints to dynamically create, update, and configure metrics for testing HPA (Horizontal Pod Autoscaler) and other metric-based scenarios.
+
+## API Endpoints
+
+### Health Checks
+
+#### Check Server Health
+```bash
+curl -k https://localhost:6443/healthz
+curl -k https://localhost:6443/readyz
+```
+
+### Kubernetes External Metrics API
+
+#### List API Groups
+```bash
+curl -k https://localhost:6443/apis/external.metrics.k8s.io
+```
+
+#### List Available Resources
+```bash
+curl -k https://localhost:6443/apis/external.metrics.k8s.io/v1beta1
+```
+
+#### Get Metric Value
+```bash
+# Query a specific metric
+curl -k https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/default/queue_messages_ready
+
+# Query a metric with label selector
+curl -k "https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/production/error_rate?labelSelector=env=prod,region=us-west"
+
+# Or via kubectl
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/queue_messages_ready"
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/error_rate?labelSelector=env=prod"
+```
+
+**Error Response (Metric Not Found):**
+```bash
+curl -k https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/default/nonexistent_metric
+# HTTP 404: no metric name called nonexistent_metric
+```
+
+**Error Response (Metric Configured to Fail):**
+```bash
+# HTTP 500: metric queue_messages_ready is configured to fail
+```
+
+---
+
+### Metric Management
+
+#### Create a New Metric
+```bash
+# Create a metric with value 250
+curl -k "https://localhost:6443/create/my_custom_metric?value=250"
+
+# Create a metric with labels
+curl -k "https://localhost:6443/create/error_rate?value=25&labels=env=prod,region=us-west"
+
+# Create a metric that will fail
+curl -k "https://localhost:6443/create/failing_metric?value=100&fail=true"
+
+# Create a metric with labels that will fail
+curl -k "https://localhost:6443/create/error_rate?value=50&labels=env=staging&fail=true"
+```
+
+#### Update Metric Value
+```bash
+# Update the value of an existing metric
+curl -k "https://localhost:6443/set/queue_messages_ready?value=500"
+```
+
+```bash
+# Update the value of an existing labels metric
+curl -k "https://localhost:6443/set/queue_messages_ready?value=500&labels=env=staging"
+```
+
+**Error Response (Metric Not Found):**
+```bash
+curl -k "https://localhost:6443/set/nonexistent_metric?value=100"
+# HTTP 404: metric nonexistent_metric not found
+```
+
+#### Configure Metric Failure State
+```bash
+# Make a metric fail (return HTTP 500)
+curl -k "https://localhost:6443/fail/queue_messages_ready?fail=true"
+
+# Make a metric succeed again
+curl -k "https://localhost:6443/fail/queue_messages_ready?fail=false"
+
+# Make a metric with specific labels fail
+curl -k "https://localhost:6443/fail/error_rate?fail=true&labels=env=prod,region=us-west"
+
+# Make a labeled metric succeed again
+curl -k "https://localhost:6443/fail/error_rate?fail=false&labels=env=prod,region=us-west"
+```
+### Querying Metrics with Labels
+```bash
+# Get all response_time metrics (returns all label variants)
+curl -k "https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/default/response_time"
+
+# Get only prod environment metrics
+curl -k "https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/default/response_time?labelSelector=env=prod"
+
+# Get specific metric with exact label match
+curl -k "https://localhost:6443/apis/external.metrics.k8s.io/v1beta1/namespaces/default/response_time?labelSelector=env=prod,service=api"
+```
+
+## Default Metrics
+
+The server starts with two pre-configured metrics:
+
+1. **queue_messages_ready** - Initial value: 100
+2. **http_requests_total** - Initial value: 500
--- a/test/images/sample-external-metrics-server/VERSION
+++ b/test/images/sample-external-metrics-server/VERSION
@ -0,0 +1 @@
+1.0.0
--- a/test/images/sample-external-metrics-server/main.go
+++ b/test/images/sample-external-metrics-server/main.go
@ -0,0 +1,327 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+)
+
+var provider *metricProvider
+
+func main() {
+	flags := pflag.NewFlagSet("sample-external-metrics", pflag.ExitOnError)
+	klog.InitFlags(nil)
+
+	// Initialize the metric provider
+	provider = NewConfigurableProvider()
+	// Create some default metrics
+	provider.createMetric("queue_messages_ready", nil, 100, false)
+	provider.createMetric("http_requests_total", nil, 500, false)
+
+	secureServing := options.NewSecureServingOptions()
+	secureServing.BindPort = 6443
+	secureServing.ServerCert.CertDirectory = "/tmp/cert"
+	secureServing.ServerCert.PairName = "apiserver"
+	secureServing.AddFlags(flags)
+
+	if err := flags.Parse(os.Args[1:]); err != nil {
+		klog.Fatalf("Error parsing flags: %v", err)
+	}
+
+	// Generate self-signed TLS certificates if none exist. This allows the server to run with HTTPS
+	// without requiring manually provisioned certificates. The certs are valid for "localhost" and
+	// the loopback IP 127.0.0.1. The second parameter (nil) means no additional alternate names.
+	if err := secureServing.MaybeDefaultWithSelfSignedCerts(
+		"localhost",
+		nil,
+		[]net.IP{netutils.ParseIPSloppy("127.0.0.1")},
+	); err != nil {
+		klog.Fatalf("Error creating self-signed certs: %v", err)
+	}
+
+	var servingInfo *genericapiserver.SecureServingInfo
+	if err := secureServing.ApplyTo(&servingInfo); err != nil {
+		klog.Fatalf("Error applying secure serving: %v", err)
+	}
+
+	if servingInfo == nil {
+		klog.Fatal("SecureServingInfo is nil")
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/apis/external.metrics.k8s.io/", handleMetrics)
+	mux.HandleFunc("/apis/external.metrics.k8s.io", handleMetrics)
+	mux.HandleFunc("/healthz", healthz)
+	mux.HandleFunc("/readyz", healthz)
+	mux.HandleFunc("/fail/", failMetric)
+	mux.HandleFunc("/set/", setMetric)
+	mux.HandleFunc("/create/", createMetric)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	klog.InfoS("Starting server on", "address", servingInfo.Listener.Addr().String())
+
+	stoppedCh, listenerStoppedCh, err := servingInfo.Serve(mux, 30*time.Second, ctx.Done())
+	if err != nil {
+		klog.Fatalf("Error starting server: %v", err)
+	}
+
+	<-listenerStoppedCh
+	<-stoppedCh
+}
+
+func healthz(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	if _, err := w.Write([]byte("ok")); err != nil {
+		klog.ErrorS(err, "failed to write healthz response")
+	}
+}
+
+func handleMetrics(w http.ResponseWriter, r *http.Request) {
+	klog.InfoS("", "method", r.Method, "path", r.URL.Path)
+	w.Header().Set("Content-Type", "application/json")
+
+	path := strings.TrimPrefix(r.URL.Path, "/apis/external.metrics.k8s.io")
+	path = strings.TrimPrefix(path, "/")
+
+	if path == "" {
+		if err := json.NewEncoder(w).Encode(map[string]interface{}{
+			"kind":       "APIGroup",
+			"apiVersion": "v1",
+			"name":       "external.metrics.k8s.io",
+			"versions": []map[string]string{
+				{"groupVersion": "external.metrics.k8s.io/v1beta1", "version": "v1beta1"},
+			},
+			"preferredVersion": map[string]string{
+				"groupVersion": "external.metrics.k8s.io/v1beta1",
+				"version":      "v1beta1",
+			},
+		}); err != nil {
+			klog.ErrorS(err, "failed to encode APIGroup response")
+		}
+		return
+	}
+
+	if path == "v1beta1" || path == "v1beta1/" {
+		if err := json.NewEncoder(w).Encode(map[string]interface{}{
+			"kind":         "APIResourceList",
+			"apiVersion":   "v1",
+			"groupVersion": "external.metrics.k8s.io/v1beta1",
+			"resources": []map[string]interface{}{
+				{"name": "*", "namespaced": true, "kind": "ExternalMetricValueList", "verbs": []string{"get"}},
+			},
+		}); err != nil {
+			klog.ErrorS(err, "failed to encode APIResourceList response")
+		}
+		return
+	}
+
+	parts := strings.Split(path, "/")
+	if len(parts) >= 4 && parts[1] == "namespaces" {
+		metricName := parts[3]
+
+		// Parse label selector from query parameter
+		labelSelector := r.URL.Query().Get("labelSelector")
+		labels := parseLabels(labelSelector)
+
+		// Get all metrics matching the name and label selector
+		metrics := provider.getMetrics(metricName, labels)
+
+		if len(metrics) == 0 {
+			errorMessage := fmt.Sprintf("no metric name called %s", metricName)
+			if len(labels) > 0 {
+				errorMessage = fmt.Sprintf("no metric name called %s with labels %v", metricName, labels)
+			}
+			http.Error(w, errorMessage, http.StatusNotFound)
+			return
+		}
+
+		// Build response items
+		var items []map[string]interface{}
+		for _, metric := range metrics {
+			if metric.shouldFail {
+				errorMessage := fmt.Sprintf("metric %s is configured to fail", metricName)
+				http.Error(w, errorMessage, http.StatusInternalServerError)
+				return
+			}
+
+			items = append(items, map[string]interface{}{
+				"metricName":   metric.metricName,
+				"metricLabels": metric.labels,
+				"timestamp":    time.Now().UTC().Format(time.RFC3339), // TODO: maybe we need to make this configurable?
+				"value":        fmt.Sprintf("%d", metric.value),
+			})
+		}
+
+		if err := json.NewEncoder(w).Encode(map[string]interface{}{
+			"kind":       "ExternalMetricValueList",
+			"apiVersion": "external.metrics.k8s.io/v1beta1",
+			"metadata":   map[string]interface{}{},
+			"items":      items,
+		}); err != nil {
+			klog.ErrorS(err, "failed to encode ExternalMetricValueList response")
+		}
+		return
+	}
+
+	http.NotFound(w, r)
+}
+
+// failMetric handles requests to mark a metric as failing or not failing.
+// Supports label selectors via the "labels" query parameter (format: key1=value1,key2=value2).
+func failMetric(w http.ResponseWriter, r *http.Request) {
+	klog.InfoS("", "method", r.Method, "path", r.URL.Path)
+	w.Header().Set("Content-Type", "application/json")
+
+	path := strings.TrimPrefix(r.URL.Path, "/fail/")
+	parts := strings.Split(path, "/")
+	if len(parts) == 0 || parts[0] == "" {
+		http.Error(w, "metric name required", http.StatusBadRequest)
+		return
+	}
+
+	metricName := parts[0]
+	failParam := r.URL.Query().Get("fail")
+
+	if failParam != "true" && failParam != "false" {
+		http.Error(w, "fail param should be true or false", http.StatusBadRequest)
+		return
+	}
+
+	labelSelector := r.URL.Query().Get("labels")
+	labels := parseLabels(labelSelector)
+	metricKey := buildMetricKey(metricName, labels)
+
+	if err := provider.setMetricFailure(metricKey, failParam); err != nil {
+		klog.ErrorS(err, "failed to set metric failure", "metric", metricName, "labels", labels)
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	if err := json.NewEncoder(w).Encode(map[string]interface{}{
+		"status":  "success",
+		"metric":  metricName,
+		"labels":  labels,
+		"failing": failParam,
+	}); err != nil {
+		klog.ErrorS(err, "failed to encode failMetric response")
+	}
+}
+
+func setMetric(w http.ResponseWriter, r *http.Request) {
+	klog.InfoS("", "method", r.Method, "path", r.URL.Path)
+	w.Header().Set("Content-Type", "application/json")
+
+	path := strings.TrimPrefix(r.URL.Path, "/set/")
+	parts := strings.Split(path, "/")
+	if len(parts) == 0 || parts[0] == "" {
+		http.Error(w, "metric name required", http.StatusBadRequest)
+		return
+	}
+
+	metricName := parts[0]
+	valueParam := r.URL.Query().Get("value")
+	if valueParam == "" {
+		http.Error(w, "value param required", http.StatusBadRequest)
+		return
+	}
+
+	// Parse labels from query parameter
+	labelSelector := r.URL.Query().Get("labels")
+	labels := parseLabels(labelSelector)
+	metricKey := buildMetricKey(metricName, labels)
+
+	var value int
+	if _, err := fmt.Sscanf(valueParam, "%d", &value); err != nil {
+		http.Error(w, "value param must be an integer", http.StatusBadRequest)
+		return
+	}
+
+	if err := provider.setMetricValue(metricKey, value); err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	if err := json.NewEncoder(w).Encode(map[string]interface{}{
+		"status": "success",
+		"metric": metricName,
+		"value":  value,
+	}); err != nil {
+		klog.ErrorS(err, "failed to encode setMetric response")
+	}
+}
+
+func createMetric(w http.ResponseWriter, r *http.Request) {
+	klog.InfoS("", "method", r.Method, "path", r.URL.Path)
+	w.Header().Set("Content-Type", "application/json")
+
+	path := strings.TrimPrefix(r.URL.Path, "/create/")
+	parts := strings.Split(path, "/")
+	if len(parts) == 0 || parts[0] == "" {
+		http.Error(w, "metric name required", http.StatusBadRequest)
+		return
+	}
+
+	metricName := parts[0]
+	valueParam := r.URL.Query().Get("value")
+	if valueParam == "" {
+		http.Error(w, "value param required", http.StatusBadRequest)
+		return
+	}
+
+	var value int
+	if _, err := fmt.Sscanf(valueParam, "%d", &value); err != nil {
+		http.Error(w, "value param must be an integer", http.StatusBadRequest)
+		return
+	}
+
+	failParam := r.URL.Query().Get("fail")
+	shouldFail := failParam == "true"
+
+	// Parse labels from query parameter
+	labelSelector := r.URL.Query().Get("labels")
+	labels := parseLabels(labelSelector)
+
+	provider.createMetric(metricName, labels, value, shouldFail)
+
+	w.WriteHeader(http.StatusCreated)
+	if err := json.NewEncoder(w).Encode(map[string]interface{}{
+		"status":  "created",
+		"metric":  metricName,
+		"labels":  labels,
+		"value":   value,
+		"failing": shouldFail,
+	}); err != nil {
+		klog.ErrorS(err, "failed to encode createMetric response")
+	}
+}
--- a/test/images/sample-external-metrics-server/provider.go
+++ b/test/images/sample-external-metrics-server/provider.go
@ -0,0 +1,169 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type externalMetric struct {
+	metricName string
+	labels     map[string]string
+	value      int
+	shouldFail bool        // If true, this metric returns errors
+	timestamp  metav1.Time // for mocking we are returning only the current time
+}
+
+type metricProvider struct {
+	externalMetricsLock sync.RWMutex
+	externalMetrics     map[string]*externalMetric
+}
+
+func NewConfigurableProvider() *metricProvider {
+	p := &metricProvider{
+		externalMetrics:     make(map[string]*externalMetric),
+		externalMetricsLock: sync.RWMutex{},
+	}
+	return p
+}
+
+// buildMetricKey creates a unique key combining metric name and labels
+// This allows multiple instances of the same metric with different labels
+func buildMetricKey(metricName string, metricLabels map[string]string) string {
+	key := metricName
+	if len(metricLabels) > 0 {
+		var labelPairs []string
+		for k, v := range metricLabels {
+			labelPairs = append(labelPairs, k+"="+v)
+		}
+		key = metricName + ":" + strings.Join(labelPairs, ",")
+	}
+	return key
+}
+
+// parseLabels parses label selector from query string format
+// Example: "env=prod,region=us-west"
+func parseLabels(labelSelector string) map[string]string {
+	labels := make(map[string]string)
+	if labelSelector == "" {
+		return labels
+	}
+
+	pairs := strings.Split(labelSelector, ",")
+	for _, pair := range pairs {
+		kv := strings.SplitN(pair, "=", 2)
+		if len(kv) == 2 {
+			labels[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
+		}
+	}
+	return labels
+}
+
+// setMetricFailure controls whether a specific metric should fail
+func (p *metricProvider) setMetricFailure(metricKey string, failStr string) error {
+	shouldFail := failStr == "true"
+
+	p.externalMetricsLock.Lock()
+	defer p.externalMetricsLock.Unlock()
+
+	metric, exists := p.externalMetrics[metricKey]
+	if !exists {
+		return fmt.Errorf("metric %s not found", metricKey)
+	}
+	metric.shouldFail = shouldFail
+	return nil
+}
+
+// setMetricFailure controls whether a specific metric should fail
+func (p *metricProvider) setMetricValue(metricKey string, metricValue int) error {
+	p.externalMetricsLock.Lock()
+	defer p.externalMetricsLock.Unlock()
+	metric, exists := p.externalMetrics[metricKey]
+	if !exists {
+		return fmt.Errorf("metric %s not found", metricKey)
+	}
+	metric.value = metricValue
+	return nil
+}
+
+func (p *metricProvider) createMetric(metricName string, metricLabels map[string]string, metricValue int, shouldFail bool) {
+	p.externalMetricsLock.Lock()
+	defer p.externalMetricsLock.Unlock()
+	key := buildMetricKey(metricName, metricLabels)
+	p.externalMetrics[key] = &externalMetric{
+		metricName: metricName,
+		labels:     metricLabels,
+		value:      metricValue,
+		shouldFail: shouldFail,
+		timestamp:  metav1.Now(),
+	}
+}
+
+// getMetrics returns all metrics matching the given name and label selector
+func (p *metricProvider) getMetrics(metricName string, labelSelector map[string]string) []*externalMetric {
+	p.externalMetricsLock.RLock()
+	defer p.externalMetricsLock.RUnlock()
+
+	var results []*externalMetric
+
+	// If label selector is provided, try exact match first
+	if len(labelSelector) > 0 {
+		key := buildMetricKey(metricName, labelSelector)
+		if metric, exists := p.externalMetrics[key]; exists {
+			results = append(results, metric)
+			return results
+		}
+	}
+
+	// Otherwise, find all metrics with the given name
+	for _, metric := range p.externalMetrics {
+		if metric.metricName != metricName {
+			continue
+		}
+
+		// If no label selector, include all metrics with this name
+		if len(labelSelector) == 0 {
+			results = append(results, metric)
+			continue
+		}
+
+		// Check if metric labels match the selector
+		matches := true
+		for selectorKey, selectorValue := range labelSelector {
+			if metricValue, exists := metric.labels[selectorKey]; !exists || metricValue != selectorValue {
+				matches = false
+				break
+			}
+		}
+		if matches {
+			results = append(results, metric)
+		}
+	}
+
+	// For backward compatibility, also check if metricName exists as a key without labels
+	if len(results) == 0 && len(labelSelector) == 0 {
+		if metric, exists := p.externalMetrics[metricName]; exists {
+			results = append(results, metric)
+		}
+	}
+
+	return results
+}