VAULT-39876 Add sys/reporting/scan to Vault, allowing an output of files with paths and names of Vault secrets (#10068) (#10323)

* VAULT-39876 sys/reporting/scan for KV secrets

* make fmt

* changelog

* stray t.log

* typo

* fix race probably

* Bug fix, add local mount

* remove comment

* bolster external tests

Co-authored-by: Violet Hynes <violet.hynes@hashicorp.com>

api/sudo_paths.go

@@ -56,6 +56,7 @@ var sudoPaths = map[string]*regexp.Regexp{
 	"/sys/replication/reindex":                             regexp.MustCompile(`^/sys/replication/reindex$`),
 	"/sys/storage/raft/snapshot-auto/config":               regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/?$`),
 	"/sys/storage/raft/snapshot-auto/config/{name}":        regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/[^/]+$`),
+	"/sys/reporting/scan":                                  regexp.MustCompile(`^/sys/reporting/scan$`),
 }
 
 func SudoPaths() map[string]*regexp.Regexp {

api/sys_reporting_scan.go

@@ -0,0 +1,49 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+func (c *Sys) ReportingScan() (*ReportingScanOutput, error) {
+	return c.ReportingScanWithContext(context.Background())
+}
+
+func (c *Sys) ReportingScanWithContext(ctx context.Context) (*ReportingScanOutput, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest(http.MethodPost, "/v1/sys/reporting/scan")
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	secret, err := ParseSecret(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if secret == nil || secret.Data == nil {
+		return nil, errors.New("data from server response is empty")
+	}
+
+	var result ReportingScanOutput
+	err = mapstructure.Decode(secret.Data, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return &result, err
+}
+
+type ReportingScanOutput struct {
+	Timestamp string `json:"timestamp" structs:"timestamp" mapstructure:"timestamp"`
+}

changelog/_10068.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+api: Added sudo-permissioned `sys/reporting/scan` endpoint which will output a set of files containing information about Vault state to the location specified by the `reporting_scan_directory` config item.
+```

command/server.go

@@ -2989,6 +2989,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.
 		Experiments:                    config.Experiments,
 		AdministrativeNamespacePath:    config.AdministrativeNamespacePath,
 		ObservationSystemConfig:        config.Observations,
+		ReportingScanDirectory:         config.ReportingScanDirectory,
 	}
 
 	if c.flagDev {

command/server/config.go

@@ -130,6 +130,8 @@ type Config struct {
 
 	EnablePostUnsealTrace bool   `hcl:"enable_post_unseal_trace"`
 	PostUnsealTraceDir    string `hcl:"post_unseal_trace_directory"`
+
+	ReportingScanDirectory string `hcl:"reporting_scan_directory"`
 }
 
 const (
@@ -479,6 +481,11 @@ func (c *Config) Merge(c2 *Config) *Config {
 		result.PostUnsealTraceDir = c2.PostUnsealTraceDir
 	}
 
+	result.ReportingScanDirectory = c.ReportingScanDirectory
+	if c2.ReportingScanDirectory != "" {
+		result.ReportingScanDirectory = c2.ReportingScanDirectory
+	}
+
 	// Use values from top-level configuration for storage if set
 	if storage := result.Storage; storage != nil {
 		if result.APIAddr != "" {

command/server/config_test.go

@@ -95,6 +95,15 @@ func TestUnknownFieldValidationListenerAndStorage(t *testing.T) {
 	testUnknownFieldValidationStorageAndListener(t)
 }
 
+// Test_ReportingScanDirectory makes sure that the reporting scan directory is correctly parsed
+func Test_ReportingScanDirectory(t *testing.T) {
+	config, err := LoadConfigFile("./test-fixtures/reporting_directory.hcl")
+	require.NoError(t, err)
+	require.NotNil(t, config)
+	require.NotEmpty(t, config.ReportingScanDirectory)
+	require.Equal(t, "/foo/bar/", config.ReportingScanDirectory)
+}
+
 // Test_ObservationSystemConfig makes sure that the observation system config
 // is properly loaded.
 func Test_ObservationSystemConfig(t *testing.T) {

command/server/test-fixtures/reporting_directory.hcl

@@ -0,0 +1,4 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+reporting_scan_directory = "/foo/bar/"

vault/core.go

@@ -754,6 +754,9 @@ type Core struct {
 	// Activation flags for enterprise features that require a one-time activation
 	FeatureActivationFlags *activationflags.FeatureActivationFlags
 	licenseReloadCh        chan error
+
+	// reportingScanDirectory is where the files emitted by /sys/reporting/scan go.
+	reportingScanDirectory string
 }
 
 func (c *Core) ActiveNodeClockSkewMillis() int64 {
@@ -943,6 +946,9 @@ type CoreConfig struct {
 
 	ClusterAddrBridge *raft.ClusterAddrBridge
 	LicenseReload     chan error
+
+	// ReportingScanDirectory is where files generated by /sys/reporting/scan will go.
+	ReportingScanDirectory string
 }
 
 // GetServiceRegistration returns the config's ServiceRegistration, or nil if it does
@@ -1125,6 +1131,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
 		activeNodeClockSkewMillis:      uberAtomic.NewInt64(0),
 		periodicLeaderRefreshInterval:  conf.PeriodicLeaderRefreshInterval,
 		rpcLastSuccessfulHeartbeat:     new(atomic.Value),
+		reportingScanDirectory:         conf.ReportingScanDirectory,
 	}
 
 	c.standbyStopCh.Store(make(chan struct{}))

vault/core_metrics.go

@@ -410,10 +410,12 @@ func (c *Core) emitMetricsActiveNode(stopCh chan struct{}) {
 }
 
 type kvMount struct {
-	Namespace  *namespace.Namespace
-	MountPoint string
-	Version    string
-	NumSecrets int
+	Namespace     *namespace.Namespace
+	MountPoint    string
+	MountAccessor string
+	Version       string
+	Local         bool
+	NumSecrets    int
 }
 
 func (c *Core) findKvMounts() []*kvMount {
@@ -436,10 +438,12 @@ func (c *Core) findKvMounts() []*kvMount {
 				version = "1"
 			}
 			mounts = append(mounts, &kvMount{
-				Namespace:  entry.namespace,
-				MountPoint: entry.Path,
-				Version:    version,
-				NumSecrets: 0,
+				Namespace:     entry.namespace,
+				MountPoint:    entry.Path,
+				MountAccessor: entry.Accessor,
+				Version:       version,
+				NumSecrets:    0,
+				Local:         entry.Local,
 			})
 		}
 	}

vault/external_tests/api/sudo_paths_test.go

@@ -74,7 +74,8 @@ func TestSudoPaths(t *testing.T) {
 				t.Fatalf(
 					"A path in the static list of sudo paths in the api module "+
 						"is not marked as a sudo path in the OpenAPI spec (%s). Please reconcile the two "+
-						"accordingly.", path)
+						"accordingly. This involves adding to the Root string array in the PathsSpecial declaration "+
+						"for the backend in question. For example, for sys/, this would be in NewSystemBackend.", path)
 			}
 		}
 	}

vault/logical_system.go

@@ -135,6 +135,7 @@ func NewSystemBackend(core *Core, logger log.Logger, config *logical.BackendConf
 				"leases/lookup/*",
 				"storage/raft/snapshot-auto/config/*",
 				"leases",
+				"reporting/scan",
 				"internal/inspect/*",
 				"internal/counters/activity/export",
 				// sys/seal and sys/step-down actually have their sudo requirement enforced through hardcoding

vault/logical_system_helpers.go

@@ -133,6 +133,11 @@ var (
 			"config/group-policy-application$": {operations: []logical.Operation{logical.ReadOperation, logical.UpdateOperation}},
 		})...)
 
+		// reporting paths
+		paths = append(paths, buildEnterpriseOnlyPaths(map[string]enterprisePathStub{
+			"reporting/scan$": {operations: []logical.Operation{logical.UpdateOperation}},
+		})...)
+
 		// namespaces paths
 		paths = append(paths, buildEnterpriseOnlyPaths(map[string]enterprisePathStub{
 			"namespaces/?$": {operations: []logical.Operation{logical.ListOperation}},

vault/testing.go

@@ -1494,6 +1494,7 @@ func NewTestCluster(t testing.TB, base *CoreConfig, opts *TestClusterOptions) *T
 		coreConfig.AdministrativeNamespacePath = base.AdministrativeNamespacePath
 		coreConfig.ServiceRegistration = base.ServiceRegistration
 		coreConfig.ImpreciseLeaseRoleTracking = base.ImpreciseLeaseRoleTracking
+		coreConfig.ReportingScanDirectory = base.ReportingScanDirectory
 
 		if base.BuiltinRegistry != nil {
 			coreConfig.BuiltinRegistry = base.BuiltinRegistry