Index selectable fields in all known types (#117023)

* Include all known manifests. Initialize SelectableFields in existing builders.
* Move AppManifests to separate file.
* Add code-generation and CI check for app_manifests.go
* Extract common code from custom builders into NewIndexableDocumentFromValue.
* Return error from BuildSelectableFields when object and kind don't match.
This commit is contained in:
Peter Štibraný 2026-01-29 15:18:05 +01:00 committed by GitHub
parent ee8520cec6
commit 0b3ba76434
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 302 additions and 63 deletions

View file

@ -56,6 +56,7 @@ jobs:
CODEGEN_VERIFY=1 make gen-cue
CODEGEN_VERIFY=1 make gen-jsonnet
CODEGEN_VERIFY=1 make gen-apps
CODEGEN_VERIFY=1 make gen-app-manifests-unistore
- name: Validate go.mod
run: go run scripts/modowners/modowners.go check go.mod

View file

@ -217,6 +217,20 @@ gen-go: gen-enterprise-go ## Generate Wire graph
@echo "generating Wire graph"
$(GO) run ./pkg/build/wire/cmd/wire/main.go gen -tags "oss" -gen_tags "(!enterprise && !pro)" ./pkg/server
.PHONY: gen-app-manifests-unistore
gen-app-manifests-unistore: ## Generate unified storage app manifests list
@echo "generating unified storage app manifests"
$(GO) generate ./pkg/storage/unified/resource/app_manifests.go
@if [ -n "$$CODEGEN_VERIFY" ]; then \
echo "Verifying generated code is up to date..."; \
if ! git diff --quiet pkg/storage/unified/resource/app_manifests.go; then \
echo "Error: pkg/storage/unified/resource/app_manifests.go is not up to date. Please run 'make gen-app-manifests-unistore' to regenerate."; \
git diff pkg/storage/unified/resource/app_manifests.go; \
exit 1; \
fi; \
echo "Generated app manifests code is up to date."; \
fi
.PHONY: fix-cue
fix-cue:
@echo "formatting cue files"

View file

@ -234,9 +234,25 @@ require (
github.com/grafana/grafana-aws-sdk v1.4.2 // indirect
github.com/grafana/grafana-azure-sdk-go/v2 v2.3.1 // indirect
github.com/grafana/grafana-plugin-sdk-go v0.286.0 // indirect
github.com/grafana/grafana/apps/advisor v0.0.0 // indirect
github.com/grafana/grafana/apps/alerting/historian v0.0.0 // indirect
github.com/grafana/grafana/apps/alerting/notifications v0.0.0 // indirect
github.com/grafana/grafana/apps/alerting/rules v0.0.0 // indirect
github.com/grafana/grafana/apps/annotation v0.0.0 // indirect
github.com/grafana/grafana/apps/collections v0.0.0 // indirect
github.com/grafana/grafana/apps/correlations v0.0.0 // indirect
github.com/grafana/grafana/apps/dashboard v0.0.0 // indirect
github.com/grafana/grafana/apps/dashvalidator v0.0.0-20260127080522-461c3f3f9fb6 // indirect
github.com/grafana/grafana/apps/example v0.0.0-20260119093047-426e55f358f5 // indirect
github.com/grafana/grafana/apps/live v0.0.0 // indirect
github.com/grafana/grafana/apps/logsdrilldown v0.0.0 // indirect
github.com/grafana/grafana/apps/playlist v0.0.0 // indirect
github.com/grafana/grafana/apps/plugins v0.0.0 // indirect
github.com/grafana/grafana/apps/preferences v0.0.0 // indirect
github.com/grafana/grafana/apps/provisioning v0.0.0 // indirect
github.com/grafana/grafana/apps/quotas v0.0.0-20251209183543-1013d74f13f2 // indirect
github.com/grafana/grafana/apps/secret v0.0.0 // indirect
github.com/grafana/grafana/apps/shorturl v0.0.0 // indirect
github.com/grafana/grafana/pkg/aggregator v0.0.0 // indirect
github.com/grafana/grafana/pkg/apiserver v0.0.0 // indirect
github.com/grafana/grafana/pkg/plugins v0.0.0 // indirect
@ -432,7 +448,6 @@ require (
golang.org/x/text v0.33.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.41.0 // indirect
golang.org/x/tools/godoc v0.1.0-deprecated // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
gonum.org/v1/gonum v0.16.0 // indirect

View file

@ -0,0 +1,58 @@
package resource
//go:generate sh -c "cd ../../../.. && bash pkg/storage/unified/resource/generate_manifests.sh"
import (
"github.com/grafana/grafana-app-sdk/app"
advisor "github.com/grafana/grafana/apps/advisor/pkg/apis"
alerting_historian "github.com/grafana/grafana/apps/alerting/historian/pkg/apis"
alerting_notifications "github.com/grafana/grafana/apps/alerting/notifications/pkg/apis"
alerting_rules "github.com/grafana/grafana/apps/alerting/rules/pkg/apis"
annotation "github.com/grafana/grafana/apps/annotation/pkg/apis"
collections "github.com/grafana/grafana/apps/collections/pkg/apis/manifestdata"
correlations "github.com/grafana/grafana/apps/correlations/pkg/apis"
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis"
dashvalidator "github.com/grafana/grafana/apps/dashvalidator/pkg/apis/manifestdata"
dashvalidator1 "github.com/grafana/grafana/apps/dashvalidator/pkg/generated/manifestdata"
example "github.com/grafana/grafana/apps/example/pkg/apis/manifestdata"
folder "github.com/grafana/grafana/apps/folder/pkg/apis/manifestdata"
iam "github.com/grafana/grafana/apps/iam/pkg/apis"
live "github.com/grafana/grafana/apps/live/pkg/apis/manifestdata"
logsdrilldown "github.com/grafana/grafana/apps/logsdrilldown/pkg/apis"
playlist "github.com/grafana/grafana/apps/playlist/pkg/apis/manifestdata"
plugins "github.com/grafana/grafana/apps/plugins/pkg/apis"
preferences "github.com/grafana/grafana/apps/preferences/pkg/apis/manifestdata"
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/manifestdata"
quotas "github.com/grafana/grafana/apps/quotas/pkg/apis"
secret "github.com/grafana/grafana/apps/secret/pkg/apis"
shorturl "github.com/grafana/grafana/apps/shorturl/pkg/apis"
)
func AppManifests() []app.Manifest {
// TODO: don't use hardcoded list of manifests when possible.
return []app.Manifest{
advisor.LocalManifest(),
alerting_historian.LocalManifest(),
alerting_notifications.LocalManifest(),
alerting_rules.LocalManifest(),
annotation.LocalManifest(),
collections.LocalManifest(),
correlations.LocalManifest(),
dashboard.LocalManifest(),
dashvalidator.LocalManifest(),
dashvalidator1.LocalManifest(),
example.LocalManifest(),
folder.LocalManifest(),
iam.LocalManifest(),
live.LocalManifest(),
logsdrilldown.LocalManifest(),
playlist.LocalManifest(),
plugins.LocalManifest(),
preferences.LocalManifest(),
provisioning.LocalManifest(),
quotas.LocalManifest(),
secret.LocalManifest(),
shorturl.LocalManifest(),
}
}

View file

@ -3,9 +3,11 @@ package resource
import (
"context"
"fmt"
"strconv"
"strings"
"sync"
"github.com/grafana/grafana-app-sdk/app"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -235,11 +237,14 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa
return doc.UpdateCopyFields()
}
func StandardDocumentBuilder() DocumentBuilder {
return &standardDocumentBuilder{}
func StandardDocumentBuilder(manifests []app.Manifest) DocumentBuilder {
return &standardDocumentBuilder{selectableFields: SelectableFieldsForManifests(manifests)}
}
type standardDocumentBuilder struct{}
type standardDocumentBuilder struct {
// Maps "group/resource" (in lowercase) to list of selectable fields.
selectableFields map[string][]string
}
func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*IndexableDocument, error) {
tmp := &unstructured.Unstructured{}
@ -254,9 +259,37 @@ func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resour
}
doc := NewIndexableDocument(key, rv, obj)
sfKey := strings.ToLower(key.GetGroup() + "/" + key.GetResource())
doc.SelectableFields = getSelectableFieldsFromObject(tmp, s.selectableFields[sfKey])
return doc, nil
}
func getSelectableFieldsFromObject(tmp *unstructured.Unstructured, fields []string) map[string]string {
result := map[string]string{}
for _, field := range fields {
path := strings.Split(field, ".")
val, ok, err := unstructured.NestedFieldNoCopy(tmp.Object, path...)
if err != nil || !ok {
continue
}
switch v := val.(type) {
case string:
result[field] = v
case bool:
result[field] = strconv.FormatBool(v)
default:
// In practice there should only be strings, bools and int/float selectable fields.
result[field] = fmt.Sprintf("%v", v)
}
}
return result
}
type searchableDocumentFields struct {
names []string
fields map[string]*resourceTableColumn

View file

@ -14,7 +14,7 @@ import (
func TestStandardDocumentBuilder(t *testing.T) {
ctx := context.Background()
builder := StandardDocumentBuilder()
builder := StandardDocumentBuilder(nil)
body, err := os.ReadFile("testdata/playlist-resource.json")
require.NoError(t, err)

View file

@ -0,0 +1,101 @@
#!/bin/bash
# Generate app_manifests.go file
set -e
# Use gawk if available, otherwise fall back to awk
if command -v gawk &> /dev/null; then
AWK=gawk
else
AWK=awk
fi
OUTPUT_FILE="pkg/storage/unified/resource/app_manifests.go"
TEMP_FILE=$(mktemp)
# Find all paths and store them
find apps -name '*.go' 2>/dev/null | \
xargs grep -l 'func LocalManifest() app.Manifest' 2>/dev/null | \
$AWK '{
path = $1
sub(/\/[^\/]+\.go$/, "", path)
if (match(path, /^apps\/(.+)\/pkg/, arr)) {
print path
}
}' | sort > "$TEMP_FILE"
# Start generating the file
cat > "$OUTPUT_FILE" << 'HEADER'
package resource
//go:generate sh -c "cd ../../../.. && bash pkg/storage/unified/resource/generate_manifests.sh"
import (
"github.com/grafana/grafana-app-sdk/app"
HEADER
# Generate imports with duplicate handling
$AWK '{
path = $1
if (match(path, /^apps\/(.+)\/pkg/, arr)) {
app = arr[1]
pkg = app
gsub("/", "_", pkg)
# Handle duplicates by adding numeric suffix
if (pkg in seen) {
suffix = seen[pkg]
seen[pkg] = suffix + 1
pkg = pkg suffix
} else {
seen[pkg] = 1
}
# Store the mapping for later use in function calls
pkg_map[path] = pkg
print "\t" pkg " \"github.com/grafana/grafana/" path "\""
}
}' "$TEMP_FILE" >> "$OUTPUT_FILE"
# Close imports and start function
cat >> "$OUTPUT_FILE" << 'MIDDLE'
)
func AppManifests() []app.Manifest {
// TODO: don't use hardcoded list of manifests when possible.
return []app.Manifest{
MIDDLE
# Generate manifest calls with same duplicate handling
$AWK '{
path = $1
if (match(path, /^apps\/(.+)\/pkg/, arr)) {
app = arr[1]
pkg = app
gsub("/", "_", pkg)
# Handle duplicates by adding numeric suffix (same logic as above)
if (pkg in seen) {
suffix = seen[pkg]
seen[pkg] = suffix + 1
pkg = pkg suffix
} else {
seen[pkg] = 1
}
print "\t\t" pkg ".LocalManifest(),"
}
}' "$TEMP_FILE" >> "$OUTPUT_FILE"
# Close function
cat >> "$OUTPUT_FILE" << 'FOOTER'
}
}
FOOTER
rm -f "$TEMP_FILE"
echo "Generated $OUTPUT_FILE"

View file

@ -5,18 +5,8 @@ import (
"strings"
"github.com/grafana/grafana-app-sdk/app"
iam "github.com/grafana/grafana/apps/iam/pkg/apis"
)
func AppManifests() []app.Manifest {
return []app.Manifest{
// TODO: don't use hardcoded list of manifests.
// We include iam manifests because they actually have some selectable fields defined.
iam.LocalManifest(),
}
}
// SelectableFields returns map of <group>/<Kind> to list of selectable fields for known manifests.
func SelectableFields() map[string][]string {
return SelectableFieldsForManifests(AppManifests())

View file

@ -62,7 +62,6 @@ type kvStorageBackend struct {
dataStore *dataStore
eventStore *eventStore
notifier notifier
builder DocumentBuilder
log log.Logger
withPruner bool
eventRetentionPeriod time.Duration
@ -137,7 +136,6 @@ func NewKVStorageBackend(opts KVBackendOptions) (KVBackend, error) {
eventStore: eventStore,
notifier: newNotifier(eventStore, notifierOptions{log: logger, useChannelNotifier: opts.UseChannelNotifier}),
snowflake: s,
builder: StandardDocumentBuilder(), // For now we use the standard document builder.
log: logger,
eventRetentionPeriod: eventRetentionPeriod,
eventPruningInterval: eventPruningInterval,

View file

@ -277,6 +277,7 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
summary.ID = obj.GetDeprecatedInternalID() // nolint:staticcheck
doc := resource.NewIndexableDocument(key, rv, obj)
// TODO: add selectable fields
doc.Title = summary.Title
doc.Description = summary.Description
doc.Tags = summary.Tags

View file

@ -1,13 +1,19 @@
package builders
import (
"bytes"
"context"
"encoding/json"
claims "github.com/grafana/authlib/types"
sdkResource "github.com/grafana/grafana-app-sdk/resource"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
// All returns all document builders from this package.
@ -79,3 +85,40 @@ func All(sql db.DB, sprinkles DashboardStats) ([]resource.DocumentBuilderInfo, e
return []resource.DocumentBuilderInfo{dashboards, users, extGroupMappings, teams, teamBindings}, nil
}
// NewIndexableDocumentFromValue parses provided bytes value into object, and initializes IndexableDocument from it.
func NewIndexableDocumentFromValue(key *resourcepb.ResourceKey, rv int64, value []byte, resObj sdkResource.Object, kind sdkResource.Kind) (*resource.IndexableDocument, error) {
err := json.NewDecoder(bytes.NewReader(value)).Decode(resObj)
if err != nil {
return nil, err
}
obj, err := utils.MetaAccessor(resObj)
if err != nil {
return nil, err
}
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Fields = make(map[string]any)
doc.SelectableFields, err = BuildSelectableFields(resObj, kind)
return doc, err
}
// BuildSelectableFields returns a map of non-empty selectable fields and their values based on selectable fields defined for the kind.
func BuildSelectableFields(obj sdkResource.Object, kind sdkResource.Kind) (map[string]string, error) {
if len(kind.SelectableFields()) == 0 {
return nil, nil
}
result := make(map[string]string, len(kind.SelectableFields()))
for _, sf := range kind.SelectableFields() {
val, err := sf.FieldValueFunc(obj)
if err != nil {
return nil, err
}
if val != "" {
result[sf.FieldSelector] = val
}
}
return result, nil
}

View file

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
iamv0 "github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
@ -128,7 +129,7 @@ func TestDashboardDocumentBuilder(t *testing.T) {
"aaa",
})
builder = resource.StandardDocumentBuilder()
builder = resource.StandardDocumentBuilder(nil)
doSnapshotTests(t, builder, "folder", &resourcepb.ResourceKey{
Namespace: "default",
Group: "folder.grafana.app",
@ -152,3 +153,23 @@ func TestDashboardDocumentBuilder(t *testing.T) {
"aaa",
})
}
func TestBuildSelectableFields(t *testing.T) {
tb := &iamv0.TeamBinding{}
tb.Spec.Subject.Name = "subject name"
tb.Spec.TeamRef.Name = "teamref name"
expected := map[string]string{
"spec.subject.name": "subject name",
"spec.teamRef.name": "teamref name",
}
res, err := BuildSelectableFields(tb, iamv0.TeamBindingKind())
require.NoError(t, err)
require.Equal(t, expected, res)
// Test passing mixed type and kind.
user := &iamv0.User{}
_, err = BuildSelectableFields(user, iamv0.TeamBindingKind())
require.Error(t, err)
}

View file

@ -1,12 +1,9 @@
package builders
import (
"bytes"
"context"
"encoding/json"
iamv0 "github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@ -56,19 +53,11 @@ type extGroupMappingDocumentBuilder struct{}
// BuildDocument implements resource.DocumentBuilder.
func (u *extGroupMappingDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) {
extGroupMapping := &iamv0.ExternalGroupMapping{}
err := json.NewDecoder(bytes.NewReader(value)).Decode(extGroupMapping)
doc, err := NewIndexableDocumentFromValue(key, rv, value, extGroupMapping, iamv0.ExternalGroupMappingKind())
if err != nil {
return nil, err
}
obj, err := utils.MetaAccessor(extGroupMapping)
if err != nil {
return nil, err
}
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Fields = make(map[string]any)
if extGroupMapping.Spec.TeamRef.Name != "" {
doc.Fields[EXTERNAL_GROUP_MAPPING_TEAM] = extGroupMapping.Spec.TeamRef.Name
}

View file

@ -1,14 +1,11 @@
package builders
import (
"bytes"
"context"
"encoding/json"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@ -60,19 +57,11 @@ type teamSearchBuilder struct{}
func (t *teamSearchBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) {
team := &v0alpha1.Team{}
err := json.NewDecoder(bytes.NewReader(value)).Decode(team)
doc, err := NewIndexableDocumentFromValue(key, rv, value, team, v0alpha1.TeamKind())
if err != nil {
return nil, err
}
obj, err := utils.MetaAccessor(team)
if err != nil {
return nil, err
}
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Fields = make(map[string]any)
if team.Spec.Email != "" {
doc.Fields[TEAM_SEARCH_EMAIL] = team.Spec.Email
}

View file

@ -1,12 +1,9 @@
package builders
import (
"bytes"
"context"
"encoding/json"
iamv0 "github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@ -55,15 +52,11 @@ type teamBindingDocumentBuilder struct{}
func (b *teamBindingDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) {
tb := &iamv0.TeamBinding{}
if err := json.NewDecoder(bytes.NewReader(value)).Decode(tb); err != nil {
return nil, err
}
obj, err := utils.MetaAccessor(tb)
doc, err := NewIndexableDocumentFromValue(key, rv, value, tb, iamv0.TeamBindingKind())
if err != nil {
return nil, err
}
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Fields = map[string]any{
TEAM_BINDING_SUBJECT_NAME: tb.Spec.Subject.Name,
TEAM_BINDING_TEAM_REF: tb.Spec.TeamRef.Name,

View file

@ -15,5 +15,9 @@
"permission": "Viewer",
"subject_name": "user.one",
"team_ref": "team.one"
},
"selectableFields": {
"spec.subject.name": "user.one",
"spec.teamRef.name": "team.one"
}
}

View file

@ -1,12 +1,9 @@
package builders
import (
"bytes"
"context"
"encoding/json"
iamv0 "github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@ -82,19 +79,11 @@ type userDocumentBuilder struct{}
func (u *userDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) {
user := &iamv0.User{}
err := json.NewDecoder(bytes.NewReader(value)).Decode(user)
doc, err := NewIndexableDocumentFromValue(key, rv, value, user, iamv0.UserKind())
if err != nil {
return nil, err
}
obj, err := utils.MetaAccessor(user)
if err != nil {
return nil, err
}
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Fields = make(map[string]any)
if user.Spec.Email != "" {
doc.Fields[USER_EMAIL] = user.Spec.Email
}

View file

@ -25,7 +25,7 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui
result := []resource.DocumentBuilderInfo{
{
Builder: resource.StandardDocumentBuilder(),
Builder: resource.StandardDocumentBuilder(resource.AppManifests()),
},
}
return append(result, all...), nil