Vault CLI: show detailed information with ListResponseWithInfo (#15417)

* CLI: Add ability to display ListResponseWithInfos

The Vault Server API includes a ListResponseWithInfo call, allowing LIST
responses to contain additional information about their keys. This is in
a key=value mapping format (both for each key, to get the additional
metadata, as well as within each metadata).

Expand the `vault list` CLI command with a `-detailed` flag (and env var
VAULT_DETAILED_LISTS) to print this additional metadata. This looks
roughly like the following:

    $ vault list -detailed pki/issuers
    Keys                                    issuer_name
    ----                                    -----------
    0cba84d7-bbbe-836a-4ff6-a11b31dc0fb7    n/a
    35dfb02d-0cdb-3d35-ee64-d0cd6568c6b0    n/a
    382fad1e-e99c-9c54-e147-bb1faa8033d3    n/a
    8bb4a793-2ad9-460c-9fa8-574c84a981f7    n/a
    8bd231d7-20e2-f21f-ae1a-7aa3319715e7    n/a
    9425d51f-cb81-426d-d6ad-5147d092094e    n/a
    ae679732-b497-ab0d-3220-806a2b9d81ed    n/a
    c5a44a1f-2ae4-2140-3acf-74b2609448cc    utf8
    d41d2419-efce-0e36-c96b-e91179a24dc1    something

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Allow detailed printing of LIST responses in JSON

When using the JSON formatter, only the absolute list of keys were
returned. Reuse the `-detailed` flag value for the `-format=json` list
response printer, allowing us to show the complete API response returned
by Vault.

This returns something like the following:

    {
      "request_id": "e9a25dcd-b67a-97d7-0f08-3670918ef3ff",
      "lease_id": "",
      "lease_duration": 0,
      "renewable": false,
      "data": {
        "key_info": {
          "0cba84d7-bbbe-836a-4ff6-a11b31dc0fb7": {
            "issuer_name": ""
          },
          "35dfb02d-0cdb-3d35-ee64-d0cd6568c6b0": {
            "issuer_name": ""
          },
          "382fad1e-e99c-9c54-e147-bb1faa8033d3": {
            "issuer_name": ""
          },
          "8bb4a793-2ad9-460c-9fa8-574c84a981f7": {
            "issuer_name": ""
          },
          "8bd231d7-20e2-f21f-ae1a-7aa3319715e7": {
            "issuer_name": ""
          },
          "9425d51f-cb81-426d-d6ad-5147d092094e": {
            "issuer_name": ""
          },
          "ae679732-b497-ab0d-3220-806a2b9d81ed": {
            "issuer_name": ""
          },
          "c5a44a1f-2ae4-2140-3acf-74b2609448cc": {
            "issuer_name": "utf8"
          },
          "d41d2419-efce-0e36-c96b-e91179a24dc1": {
            "issuer_name": "something"
          }
        },
        "keys": [
          "0cba84d7-bbbe-836a-4ff6-a11b31dc0fb7",
          "35dfb02d-0cdb-3d35-ee64-d0cd6568c6b0",
          "382fad1e-e99c-9c54-e147-bb1faa8033d3",
          "8bb4a793-2ad9-460c-9fa8-574c84a981f7",
          "8bd231d7-20e2-f21f-ae1a-7aa3319715e7",
          "9425d51f-cb81-426d-d6ad-5147d092094e",
          "ae679732-b497-ab0d-3220-806a2b9d81ed",
          "c5a44a1f-2ae4-2140-3acf-74b2609448cc",
          "d41d2419-efce-0e36-c96b-e91179a24dc1"
        ]
      },
      "warnings": null
    }

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add changelog

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Use field on UI rather than secret.Data

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Only include headers from visitable key_infos

Certain API endpoints return data from non-visitable key_infos, by
virtue of using a hand-rolled response. Limit our headers to those
from visitable key_infos. This means we won't return entire columns with
n/a entries, if no key matches the key_info key that includes that
header.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Use setupEnv sourced detailed info

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Fix changelog environment variable

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Fix broken tests using setupEnv

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

changelog/15417.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+command: Support the optional '-detailed' flag to be passed to 'vault list' command to show ListResponseWithInfo data. Also supports the VAULT_DETAILED env var.
+```

command/base.go

@@ -56,6 +56,7 @@ type BaseCommand struct {
 
 	flagFormat           string
 	flagField            string
+	flagDetailed         bool
 	flagOutputCurlString bool
 	flagOutputPolicy     bool
 	flagNonInteractive   bool
@@ -304,6 +305,7 @@ const (
 	FlagSetHTTP
 	FlagSetOutputField
 	FlagSetOutputFormat
+	FlagSetOutputDetailed
 )
 
 // flagSet creates the flags for this command. The result is cached on the
@@ -496,11 +498,11 @@ func (c *BaseCommand) flagSet(bit FlagSetBit) *FlagSets {
 
 		}
 
-		if bit&(FlagSetOutputField|FlagSetOutputFormat) != 0 {
-			f := set.NewFlagSet("Output Options")
+		if bit&(FlagSetOutputField|FlagSetOutputFormat|FlagSetOutputDetailed) != 0 {
+			outputSet := set.NewFlagSet("Output Options")
 
 			if bit&FlagSetOutputField != 0 {
-				f.StringVar(&StringVar{
+				outputSet.StringVar(&StringVar{
 					Name:       "field",
 					Target:     &c.flagField,
 					Default:    "",
@@ -513,7 +515,7 @@ func (c *BaseCommand) flagSet(bit FlagSetBit) *FlagSets {
 			}
 
 			if bit&FlagSetOutputFormat != 0 {
-				f.StringVar(&StringVar{
+				outputSet.StringVar(&StringVar{
 					Name:       "format",
 					Target:     &c.flagFormat,
 					Default:    "table",
@@ -523,6 +525,16 @@ func (c *BaseCommand) flagSet(bit FlagSetBit) *FlagSets {
 						are "table", "json", "yaml", or "pretty".`,
 				})
 			}
+
+			if bit&FlagSetOutputDetailed != 0 {
+				outputSet.BoolVar(&BoolVar{
+					Name:    "detailed",
+					Target:  &c.flagDetailed,
+					Default: false,
+					EnvVar:  EnvVaultDetailed,
+					Usage:   "Enables additional metadata during some operations",
+				})
+			}
 		}
 
 		c.flags = set

command/commands.go

@@ -80,6 +80,8 @@ const (
 	// EnvVaultLicensePath is an env var used in Vault Enterprise to provide a
 	// path to a license file on disk
 	EnvVaultLicensePath = "VAULT_LICENSE_PATH"
+	// EnvVaultDetailed is to output detailed information (e.g., ListResponseWithInfo).
+	EnvVaultDetailed = `VAULT_DETAILED`
 
 	// DisableSSCTokens is an env var used to disable index bearing
 	// token functionality

command/format.go

@@ -86,6 +86,15 @@ func Format(ui cli.Ui) string {
 	return format
 }
 
+func Detailed(ui cli.Ui) bool {
+	switch ui := ui.(type) {
+	case *VaultUI:
+		return ui.detailed
+	}
+
+	return false
+}
+
 // An output formatter for json output of an object
 type JsonFormatter struct{}
 
@@ -98,6 +107,20 @@ func (j JsonFormatter) Output(ui cli.Ui, secret *api.Secret, data interface{}) e
 	if err != nil {
 		return err
 	}
+
+	if secret != nil {
+		shouldListWithInfo := Detailed(ui)
+
+		// Show the raw JSON of the LIST call, rather than only the
+		// list of keys.
+		if shouldListWithInfo {
+			b, err = j.Format(secret)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	ui.Output(string(b))
 	return nil
 }
@@ -320,6 +343,15 @@ func (t TableFormatter) OutputSealStatusStruct(ui cli.Ui, secret *api.Secret, da
 func (t TableFormatter) OutputList(ui cli.Ui, secret *api.Secret, data interface{}) error {
 	t.printWarnings(ui, secret)
 
+	// Determine if we have additional information from a ListResponseWithInfo endpoint.
+	var additionalInfo map[string]interface{}
+	if secret != nil {
+		shouldListWithInfo := Detailed(ui)
+		if additional, ok := secret.Data["key_info"]; shouldListWithInfo && ok && len(additional.(map[string]interface{})) > 0 {
+			additionalInfo = additional.(map[string]interface{})
+		}
+	}
+
 	switch data := data.(type) {
 	case []interface{}:
 	case []string:
@@ -342,10 +374,71 @@ func (t TableFormatter) OutputList(ui cli.Ui, secret *api.Secret, data interface
 		}
 		sort.Strings(keys)
 
-		// Prepend the header
-		keys = append([]string{"Keys"}, keys...)
+		// If we have a ListResponseWithInfo endpoint, we'll need to show
+		// additional headers. To satisfy the table outputter, we'll need
+		// to concat them with the deliminator.
+		var headers []string
+		header := "Keys"
+		if len(additionalInfo) > 0 {
+			seenHeaders := make(map[string]bool)
+			for key, rawValues := range additionalInfo {
+				// Most endpoints use the well-behaved ListResponseWithInfo.
+				// However, some use a hand-rolled equivalent, where the
+				// returned "keys" doesn't match the key of the "key_info"
+				// member (namely, /sys/policies/egp). We seek to exclude
+				// headers only visible from "non-visitable" key_info rows,
+				// to make table output less confusing. These non-visitable
+				// rows will still be visible in the JSON output.
+				index := sort.SearchStrings(keys, key)
+				if index < len(keys) && keys[index] != key {
+					continue
+				}
 
-		ui.Output(tableOutput(keys, &columnize.Config{
+				values := rawValues.(map[string]interface{})
+				for key := range values {
+					seenHeaders[key] = true
+				}
+			}
+
+			for key := range seenHeaders {
+				headers = append(headers, key)
+			}
+			sort.Strings(headers)
+
+			header = header + hopeDelim + strings.Join(headers, hopeDelim)
+		}
+
+		// Finally, if we have a ListResponseWithInfo, we'll need to update
+		// the returned rows to not just have the keys (in the sorted order),
+		// but also have the values for each header (in their sorted order).
+		rows := keys
+		if len(additionalInfo) > 0 && len(headers) > 0 {
+			for index, row := range rows {
+				formatted := []string{row}
+				if rawValues, ok := additionalInfo[row]; ok {
+					values := rawValues.(map[string]interface{})
+					for _, header := range headers {
+						if rawValue, ok := values[header]; ok {
+							if looksLikeDuration(header) {
+								rawValue = humanDurationInt(rawValue)
+							}
+
+							formatted = append(formatted, fmt.Sprintf("%v", rawValue))
+						} else {
+							// Show a default empty n/a when this field is
+							// missing from the additional information.
+							formatted = append(formatted, "n/a")
+						}
+					}
+				}
+
+				rows[index] = strings.Join(formatted, hopeDelim)
+			}
+		}
+
+		// Prepend the header to the formatted rows.
+		output := append([]string{header}, rows...)
+		ui.Output(tableOutput(output, &columnize.Config{
 			Delim: hopeDelim,
 		}))
 	}

command/list.go

@@ -42,7 +42,8 @@ Usage: vault list [options] PATH
 }
 
 func (c *ListCommand) Flags() *FlagSets {
-	return c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat | FlagSetOutputDetailed)
+	return set
 }
 
 func (c *ListCommand) AutocompleteArgs() complete.Predictor {

command/main.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"text/tabwriter"
 
@@ -19,13 +20,16 @@ import (
 
 type VaultUI struct {
 	cli.Ui
-	format string
+	format   string
+	detailed bool
 }
 
 // setupEnv parses args and may replace them and sets some env vars to known
 // values based on format options
-func setupEnv(args []string) (retArgs []string, format string, outputCurlString bool, outputPolicy bool) {
+func setupEnv(args []string) (retArgs []string, format string, detailed bool, outputCurlString bool, outputPolicy bool) {
+	var err error
 	var nextArgFormat bool
+	var haveDetailed bool
 
 	for _, arg := range args {
 		if nextArgFormat {
@@ -64,6 +68,28 @@ func setupEnv(args []string) (retArgs []string, format string, outputCurlString
 		if arg == "-format" || arg == "--format" {
 			nextArgFormat = true
 		}
+
+		// Parse a given flag here, which overrides the env var
+		if strings.HasPrefix(arg, "--detailed=") {
+			detailed, err = strconv.ParseBool(strings.TrimPrefix(arg, "--detailed="))
+			if err != nil {
+				detailed = false
+			}
+			haveDetailed = true
+		}
+		if strings.HasPrefix(arg, "-detailed=") {
+			detailed, err = strconv.ParseBool(strings.TrimPrefix(arg, "-detailed="))
+			if err != nil {
+				detailed = false
+			}
+			haveDetailed = true
+		}
+		// For backwards compat, it could be specified without an equal sign to enable
+		// detailed output.
+		if arg == "-detailed" || arg == "--detailed" {
+			detailed = true
+			haveDetailed = true
+		}
 	}
 
 	envVaultFormat := os.Getenv(EnvVaultFormat)
@@ -77,7 +103,16 @@ func setupEnv(args []string) (retArgs []string, format string, outputCurlString
 		format = "table"
 	}
 
-	return args, format, outputCurlString, outputPolicy
+	envVaultDetailed := os.Getenv(EnvVaultDetailed)
+	// If we did not parse a value, fetch the env var
+	if !haveDetailed && envVaultDetailed != "" {
+		detailed, err = strconv.ParseBool(envVaultDetailed)
+		if err != nil {
+			detailed = false
+		}
+	}
+
+	return args, format, detailed, outputCurlString, outputPolicy
 }
 
 type RunOptions struct {
@@ -100,9 +135,10 @@ func RunCustom(args []string, runOpts *RunOptions) int {
 	}
 
 	var format string
+	var detailed bool
 	var outputCurlString bool
 	var outputPolicy bool
-	args, format, outputCurlString, outputPolicy = setupEnv(args)
+	args, format, detailed, outputCurlString, outputPolicy = setupEnv(args)
 
 	// Don't use color if disabled
 	useColor := true
@@ -145,7 +181,8 @@ func RunCustom(args []string, runOpts *RunOptions) int {
 				ErrorWriter: uiErrWriter,
 			},
 		},
-		format: format,
+		format:   format,
+		detailed: detailed,
 	}
 
 	serverCmdUi := &VaultUI{

command/operator_unseal_test.go

@@ -167,7 +167,7 @@ func TestOperatorUnsealCommand_Format(t *testing.T) {
 		Client: client,
 	}
 
-	args, format, _, _ := setupEnv([]string{"operator", "unseal", "-format", "json"})
+	args, format, _, _, _ := setupEnv([]string{"operator", "unseal", "-format", "json"})
 	if format != "json" {
 		t.Fatalf("expected %q, got %q", "json", format)
 	}

command/version_history_test.go

@@ -57,7 +57,7 @@ func TestVersionHistoryCommand_JsonOutput(t *testing.T) {
 		Client: client,
 	}
 
-	args, format, _, _ := setupEnv([]string{"version-history", "-format", "json"})
+	args, format, _, _, _ := setupEnv([]string{"version-history", "-format", "json"})
 	if format != "json" {
 		t.Fatalf("expected format to be %q, actual %q", "json", format)
 	}