diff --git a/builtin/logical/pkiext/pkiext_binary/pki_cluster.go b/builtin/logical/pkiext/pkiext_binary/pki_cluster.go index 4462f51038..0c2c3d96d6 100644 --- a/builtin/logical/pkiext/pkiext_binary/pki_cluster.go +++ b/builtin/logical/pkiext/pkiext_binary/pki_cluster.go @@ -40,6 +40,7 @@ func NewVaultPkiCluster(t *testing.T) *VaultPkiCluster { }, NumCores: 3, }, + DisableMlock: true, } cluster := docker.NewTestDockerCluster(t, opts) diff --git a/changelog/29974.txt b/changelog/29974.txt new file mode 100644 index 0000000000..313e3c5740 --- /dev/null +++ b/changelog/29974.txt @@ -0,0 +1,3 @@ +```release-note:change +server: disable_mlock configuration option is now required for integrated storage and no longer has a default. If you are using the default value with integrated storage, you must now explicitly set disable_mlock to true or false or Vault server will fail to start. +``` diff --git a/command/server.go b/command/server.go index a1c41de1d0..1823e689f9 100644 --- a/command/server.go +++ b/command/server.go @@ -446,7 +446,6 @@ func (c *ServerCommand) parseConfig() (*server.Config, []configutil.ConfigError, c.UI.Warn("WARNING: Entropy Augmentation is not supported in FIPS 140-2 Inside mode; disabling from server configuration!\n") config.Entropy = nil } - entCheckRequestLimiter(c, config) return config, configErrors, nil @@ -1132,7 +1131,8 @@ func (c *ServerCommand) Run(args []string) int { logProxyEnvironmentVariables(c.logger) - if envMlock := os.Getenv("VAULT_DISABLE_MLOCK"); envMlock != "" { + envMlock := os.Getenv("VAULT_DISABLE_MLOCK") + if envMlock != "" { var err error config.DisableMlock, err = strconv.ParseBool(envMlock) if err != nil { @@ -1141,6 +1141,30 @@ func (c *ServerCommand) Run(args []string) int { } } + isMlockSet := func() bool { + // DisableMlock key has been found and thus explicitly set + return strutil.StrListContainsCaseInsensitive(config.SharedConfig.FoundKeys, "DisableMlock") || + // envvar set + envMlock != "" + } + + // ensure that the DisableMlock key is explicitly set if using integrated storage + if config.Storage != nil && config.Storage.Type == storageTypeRaft && !isMlockSet() { + + c.UI.Error(wrapAtLength( + "ERROR: disable_mlock must be configured 'true' or 'false': Mlock " + + "prevents memory from being swapped to disk for security reasons, " + + "but can cause Vault on Integrated Storage to run out of memory " + + "if the machine was not provisioned with enough RAM. Disabling " + + "mlock can prevent this issue, but can result in the reveal of " + + "plaintext secrets when memory is swapped to disk, so it is only " + + "recommended on systems with encrypted swap or where swap is disabled." + + "Prior to Vault 1.20, disable_mlock defaulted to 'false' when the" + + "value was not set explicitly.", + )) + return 1 + } + if envLicensePath := os.Getenv(EnvVaultLicensePath); envLicensePath != "" { config.LicensePath = envLicensePath } diff --git a/command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl b/command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl index eaf3660d59..7e9032c04e 100644 --- a/command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl +++ b/command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl @@ -1,6 +1,7 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 +disable_mlock = true storage "raft" { path = "/path/to/raft/data" node_id = "raft_node_1" diff --git a/command/server_test.go b/command/server_test.go index 9a1328ad7e..23342d926d 100644 --- a/command/server_test.go +++ b/command/server_test.go @@ -17,6 +17,7 @@ import ( "fmt" "io/ioutil" "os" + "regexp" "strings" "sync" "testing" @@ -32,16 +33,26 @@ import ( "github.com/stretchr/testify/require" ) +// Modifier is a function that modifies a string +type Modifier func(string) string + +// regexReplacer returns a Modifier that replaces all occurrences of re with repl +func regexReplacer(re, repl string) Modifier { + return func(s string) string { + return regexp.MustCompile(re).ReplaceAllString(s, repl) + } +} + func init() { if signed := os.Getenv("VAULT_LICENSE_CI"); signed != "" { os.Setenv(EnvVaultLicense, signed) } } -func testBaseHCL(tb testing.TB, listenerExtras string) string { +func testBaseHCL(tb testing.TB, listenerExtras string, modifiers ...Modifier) string { tb.Helper() - return strings.TrimSpace(fmt.Sprintf(` + base := strings.TrimSpace(fmt.Sprintf(` disable_mlock = true listener "tcp" { address = "127.0.0.1:%d" @@ -49,6 +60,11 @@ func testBaseHCL(tb testing.TB, listenerExtras string) string { %s } `, 0, listenerExtras)) + + for _, mod := range modifiers { + base = mod(base) + } + return base } const ( @@ -62,6 +78,12 @@ const ( badListenerWriteTimeout = `http_write_timeout = "56lbs"` badListenerIdleTimeout = `http_idle_timeout = "78gophers"` + raftHCL = ` +storage "raft" { + path = "/path/to/raft/data" + node_id = "raft_node_1" +} + ` inmemHCL = ` backend "inmem_ha" { advertise_addr = "http://127.0.0.1:8200" @@ -72,7 +94,6 @@ ha_backend "inmem_ha" { redirect_addr = "http://127.0.0.1:8200" } ` - badHAInmemHCL = ` ha_backend "inmem" {} ` @@ -275,6 +296,13 @@ func TestServer(t *testing.T) { 0, []string{"-test-verify-only", "-recovery"}, }, + { + "missing_disable_mlock_value_with_integrated_storage", + testBaseHCL(t, goodListenerTimeouts, regexReplacer(`\s*disable_mlock\s*=\s*.+`, "")) + raftHCL, + "disable_mlock must be configured", + 1, + []string{"-test-server-config"}, + }, } for _, tc := range cases { diff --git a/command/test-fixtures/config.hcl b/command/test-fixtures/config.hcl index e15c243e41..520a1d6a95 100644 --- a/command/test-fixtures/config.hcl +++ b/command/test-fixtures/config.hcl @@ -1,4 +1,4 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 - -token_helper = "foo" \ No newline at end of file +disable_mlock = true +token_helper = "foo" diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index 89540ca3ba..8bf937c4a7 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -156,6 +156,7 @@ globals { bundle = "/opt/vault/bin" package = "/usr/bin" } - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_tag_key = "vault-cluster" + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) + vault_tag_key = "vault-cluster" + vault_disable_mlock = false } diff --git a/enos/modules/start_vault/main.tf b/enos/modules/start_vault/main.tf index 9e386e01e4..e2eec5fbd3 100644 --- a/enos/modules/start_vault/main.tf +++ b/enos/modules/start_vault/main.tf @@ -188,7 +188,9 @@ resource "enos_vault_start" "leader" { bin_path = local.bin_path config_dir = var.config_dir config_mode = var.config_mode - environment = var.environment + environment = merge(var.environment, { + VAULT_DISABLE_MLOCK = var.disable_mlock + }) config = { api_addr = local.api_addrs_internal[tonumber(each.value)][var.ip_version] cluster_addr = local.cluster_addrs_internal[tonumber(each.value)][var.ip_version] @@ -230,7 +232,9 @@ resource "enos_vault_start" "followers" { bin_path = local.bin_path config_dir = var.config_dir config_mode = var.config_mode - environment = var.environment + environment = merge(var.environment, { + VAULT_DISABLE_MLOCK = var.disable_mlock + }) config = { api_addr = local.api_addrs_internal[tonumber(each.value)][var.ip_version] cluster_addr = local.cluster_addrs_internal[tonumber(each.value)][var.ip_version] diff --git a/enos/modules/start_vault/variables.tf b/enos/modules/start_vault/variables.tf index 2571b0c2dd..21d4a4e358 100644 --- a/enos/modules/start_vault/variables.tf +++ b/enos/modules/start_vault/variables.tf @@ -34,6 +34,12 @@ variable "config_mode" { } } +variable "disable_mlock" { + type = bool + description = "Disable mlock for Vault process." + default = false +} + variable "environment" { description = "Optional Vault configuration environment variables to set starting Vault" type = map(string) diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf index 1fbe31547a..a70ab698c4 100644 --- a/enos/modules/vault_cluster/main.tf +++ b/enos/modules/vault_cluster/main.tf @@ -20,6 +20,7 @@ locals { bin_path = "${var.install_dir}/vault" consul_bin_path = "${var.consul_install_dir}/consul" enable_audit_devices = var.enable_audit_devices && var.initialize_cluster + disable_mlock = false // In order to get Terraform to plan we have to use collections with keys // that are known at plan time. In order for our module to work our var.hosts // must be a map with known keys at plan time. Here we're creating locals @@ -156,6 +157,7 @@ module "start_vault" { cluster_tag_key = var.cluster_tag_key config_dir = var.config_dir config_mode = var.config_mode + disable_mlock = local.disable_mlock external_storage_port = var.external_storage_port hosts = var.hosts install_dir = var.install_dir diff --git a/sdk/helper/testcluster/docker/environment.go b/sdk/helper/testcluster/docker/environment.go index 8dd40904f7..15248c723e 100644 --- a/sdk/helper/testcluster/docker/environment.go +++ b/sdk/helper/testcluster/docker/environment.go @@ -77,7 +77,8 @@ type DockerCluster struct { Logger log.Logger builtTags map[string]struct{} - storage testcluster.ClusterStorage + storage testcluster.ClusterStorage + disableMlock bool } func (dc *DockerCluster) NamedLogger(s string) log.Logger { @@ -401,7 +402,7 @@ func (n *DockerClusterNode) setupCert(ip string) error { func NewTestDockerCluster(t *testing.T, opts *DockerClusterOptions) *DockerCluster { if opts == nil { - opts = &DockerClusterOptions{} + opts = &DockerClusterOptions{DisableMlock: true} } if opts.ClusterName == "" { opts.ClusterName = strings.ReplaceAll(t.Name(), "/", "-") @@ -431,7 +432,7 @@ func NewDockerCluster(ctx context.Context, opts *DockerClusterOptions) (*DockerC } if opts == nil { - opts = &DockerClusterOptions{} + opts = &DockerClusterOptions{DisableMlock: true} } if opts.Logger == nil { opts.Logger = log.NewNullLogger() @@ -441,12 +442,13 @@ func NewDockerCluster(ctx context.Context, opts *DockerClusterOptions) (*DockerC } dc := &DockerCluster{ - DockerAPI: api, - ClusterName: opts.ClusterName, - Logger: opts.Logger, - builtTags: map[string]struct{}{}, - CA: opts.CA, - storage: opts.Storage, + DockerAPI: api, + ClusterName: opts.ClusterName, + Logger: opts.Logger, + builtTags: map[string]struct{}{}, + CA: opts.CA, + storage: opts.Storage, + disableMlock: opts.DisableMlock, } if err := dc.setupDockerCluster(ctx, opts); err != nil { @@ -706,7 +708,7 @@ func (n *DockerClusterNode) Start(ctx context.Context, opts *DockerClusterOption //// disable_mlock is required for working in the Docker environment with //// custom plugins - vaultCfg["disable_mlock"] = true + vaultCfg["disable_mlock"] = opts.DisableMlock protocol := "https" if opts.DisableTLS { @@ -811,6 +813,7 @@ func (n *DockerClusterNode) Start(ctx context.Context, opts *DockerClusterOption "SKIP_SETCAP=true", "VAULT_LOG_FORMAT=json", "VAULT_LICENSE=" + opts.VaultLicense, + "VAULT_DISABLE_MLOCK=" + strconv.FormatBool(opts.DisableMlock), } envs = append(envs, opts.Envs...) @@ -1085,17 +1088,18 @@ func (l LogConsumerWriter) Write(p []byte) (n int, err error) { // DockerClusterOptions has options for setting up the docker cluster type DockerClusterOptions struct { testcluster.ClusterOptions - CAKey *ecdsa.PrivateKey - NetworkName string - ImageRepo string - ImageTag string - CA *testcluster.CA - VaultBinary string - Args []string - Envs []string - StartProbe func(*api.Client) error - Storage testcluster.ClusterStorage - DisableTLS bool + CAKey *ecdsa.PrivateKey + NetworkName string + ImageRepo string + ImageTag string + CA *testcluster.CA + VaultBinary string + Args []string + Envs []string + StartProbe func(*api.Client) error + Storage testcluster.ClusterStorage + DisableTLS bool + DisableMlock bool } func ensureLeaderMatches(ctx context.Context, client *api.Client, ready func(response *api.LeaderResponse) error) error { @@ -1259,7 +1263,7 @@ func (dc *DockerCluster) joinNode(ctx context.Context, nodeIdx int, leaderIdx in func (dc *DockerCluster) setupImage(ctx context.Context, opts *DockerClusterOptions) (string, error) { if opts == nil { - opts = &DockerClusterOptions{} + opts = &DockerClusterOptions{DisableMlock: true} } sourceTag := opts.ImageTag if sourceTag == "" { diff --git a/sdk/helper/testcluster/docker/environment_test.go b/sdk/helper/testcluster/docker/environment_test.go index bb23764052..caa51cd76b 100644 --- a/sdk/helper/testcluster/docker/environment_test.go +++ b/sdk/helper/testcluster/docker/environment_test.go @@ -11,9 +11,10 @@ func TestSettingEnvsToContainer(t *testing.T) { expectedEnv := "TEST_ENV=value1" expectedEnv2 := "TEST_ENV2=value2" opts := &DockerClusterOptions{ - ImageRepo: "hashicorp/vault", - ImageTag: "latest", - Envs: []string{expectedEnv, expectedEnv2}, + ImageRepo: "hashicorp/vault", + ImageTag: "latest", + Envs: []string{expectedEnv, expectedEnv2}, + DisableMlock: true, } cluster := NewTestDockerCluster(t, opts) defer cluster.Cleanup() diff --git a/sdk/helper/testcluster/docker/replication.go b/sdk/helper/testcluster/docker/replication.go index c313e7af4d..0ec897ad0e 100644 --- a/sdk/helper/testcluster/docker/replication.go +++ b/sdk/helper/testcluster/docker/replication.go @@ -27,6 +27,7 @@ func DefaultOptions(t *testing.T) *DockerClusterOptions { LogLevel: "TRACE", }, }, + DisableMlock: true, } } diff --git a/sdk/helper/testcluster/types.go b/sdk/helper/testcluster/types.go index 0c04c224c1..58872e854a 100644 --- a/sdk/helper/testcluster/types.go +++ b/sdk/helper/testcluster/types.go @@ -41,8 +41,8 @@ type VaultNodeConfig struct { // ClusterAddr string `hcl:"cluster_addr"` // Storage *Storage `hcl:"-"` // HAStorage *Storage `hcl:"-"` - // DisableMlock bool `hcl:"disable_mlock"` // ClusterName string `hcl:"cluster_name"` + // DisableMlock bool `hcl:"disable_mlock"` // Not configurable yet: // Listeners []*Listener `hcl:"-"` diff --git a/vault/external_tests/misc/misc_binary/recovery_test.go b/vault/external_tests/misc/misc_binary/recovery_test.go index 00e5570c30..6a5685b13a 100644 --- a/vault/external_tests/misc/misc_binary/recovery_test.go +++ b/vault/external_tests/misc/misc_binary/recovery_test.go @@ -31,7 +31,8 @@ func TestRecovery_Docker(t *testing.T) { t.Skip("only running docker test when $VAULT_BINARY present") } opts := &docker.DockerClusterOptions{ - ImageRepo: "hashicorp/vault", + ImageRepo: "hashicorp/vault", + DisableMlock: true, // We're replacing the binary anyway, so we're not too particular about // the docker image version tag. ImageTag: "latest", diff --git a/vault/external_tests/misc/recover_from_panic_test.go b/vault/external_tests/misc/recover_from_panic_test.go index 9452ad8409..9e850fde7b 100644 --- a/vault/external_tests/misc/recover_from_panic_test.go +++ b/vault/external_tests/misc/recover_from_panic_test.go @@ -19,7 +19,8 @@ func TestRecoverFromPanic(t *testing.T) { LogicalBackends: map[string]logical.Factory{ "noop": vault.NoopBackendFactory, }, - EnableRaw: true, + EnableRaw: true, + DisableMlock: true, } cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ HandlerFunc: vaulthttp.Handler, diff --git a/vault/external_tests/misc/recovery_test.go b/vault/external_tests/misc/recovery_test.go index 5939f282de..836d6e84cd 100644 --- a/vault/external_tests/misc/recovery_test.go +++ b/vault/external_tests/misc/recovery_test.go @@ -27,7 +27,8 @@ func TestRecovery(t *testing.T) { var rootToken string { conf := vault.CoreConfig{ - Physical: inm, + Physical: inm, + DisableMlock: true, } opts := vault.TestClusterOptions{ HandlerFunc: http.Handler, diff --git a/vault/external_tests/raft/raft_binary/raft_test.go b/vault/external_tests/raft/raft_binary/raft_test.go index 135ff4fb28..3173a438fe 100644 --- a/vault/external_tests/raft/raft_binary/raft_test.go +++ b/vault/external_tests/raft/raft_binary/raft_test.go @@ -32,7 +32,8 @@ func TestRaft_Configuration_Docker(t *testing.T) { t.Skip("only running docker test when $VAULT_BINARY present") } opts := &docker.DockerClusterOptions{ - ImageRepo: "hashicorp/vault", + ImageRepo: "hashicorp/vault", + DisableMlock: true, // We're replacing the binary anyway, so we're not too particular about // the docker image version tag. ImageTag: "latest", @@ -138,7 +139,8 @@ func TestDocker_LogStore_Boltdb_To_Raftwal_And_Back(t *testing.T) { t.Skip("only running docker test when $VAULT_BINARY present") } opts := &docker.DockerClusterOptions{ - ImageRepo: "hashicorp/vault", + ImageRepo: "hashicorp/vault", + DisableMlock: true, // We're replacing the binary anyway, so we're not too particular about // the docker image version tag. ImageTag: "latest", @@ -332,7 +334,8 @@ func TestRaft_LogStore_Migration_Snapshot(t *testing.T) { t.Skip("only running docker test when $VAULT_BINARY present") } opts := &docker.DockerClusterOptions{ - ImageRepo: "hashicorp/vault", + ImageRepo: "hashicorp/vault", + DisableMlock: true, // We're replacing the binary anyway, so we're not too particular about // the docker image version tag. ImageTag: "latest", diff --git a/website/content/docs/concepts/production-hardening.mdx b/website/content/docs/concepts/production-hardening.mdx index 532bd2b720..cd656aa5de 100644 --- a/website/content/docs/concepts/production-hardening.mdx +++ b/website/content/docs/concepts/production-hardening.mdx @@ -123,7 +123,7 @@ not be suitable for every deployment. When using platform-specific identity solutions, you should be mindful of auth method and secret engine configuration within namespaces. You can share - platform identity across Vault namespaces, as these provider features + platform identity across Vault namespaces, as these provider features generally offer host-based identity solutions. If that is not applicable, set the credentials as environment variables @@ -154,7 +154,7 @@ not be suitable for every deployment. give secret values from either standard input or with command-line arguments. Command-line arguments can persisted in shell history, and are readable by other unprivileged users on the same host. -- **Develop an off-boarding process**. Removing accounts in Vault or associated +- **Develop an off-boarding process**. Removing accounts in Vault or associated identity providers may not immediately revoke [token-based access](/vault/docs/concepts/tokens#user-management-considerations). Depending on how you manage access to Vault, operators should consider: @@ -213,3 +213,13 @@ not be suitable for every deployment. - **Be aware of special container considerations**. To use memory locking (mlock) inside a Vault container, you need to use the `overlayfs2` or another supporting driver. + +- **Consider memory usage when configuring `disable_mlock` with integrated storage**. + When you leave `mlock` enabled, Vault disallows data swaps from memory to disk. + Disallowing swaps ensures that sensitive data remains in RAM, but also + increases the risk of out-of-memory (OOM) errors in under-provisioned systems. + Conversely, disabling `mlock` lets the operating system swap memory to disk + under pressure, which can reduce the risk of OOM errors but increase the + exposure risk for secrets that end up in the swap file. If you choose to + disable `mlock`, we recommend encrypting your swap file to limit the exposure + risk. diff --git a/website/content/docs/configuration/index.mdx b/website/content/docs/configuration/index.mdx index 590742bec7..eaf89de39b 100644 --- a/website/content/docs/configuration/index.mdx +++ b/website/content/docs/configuration/index.mdx @@ -97,9 +97,12 @@ to specify where the configuration is. the read cache used by the physical storage subsystem. This will very significantly impact performance. -- `disable_mlock` `(bool: false)` – Disables the server from executing the - `mlock` syscall. `mlock` prevents memory from being swapped to disk. Disabling - `mlock` is not recommended unless using [integrated storage](/vault/docs/internals/integrated-storage). +- `disable_mlock` `(bool: )` – Stops Vault from executing the `mlock` + syscall. `mlock` prevents data swaps from memory to disk. You must set an + explicit value for `disable_mlock` if you use + [integrated storage](/vault/docs/internals/integrated-storage). We do not + recommend disabling `mlock` for deployments that do not use integrated storage. + Follow the additional security precautions outlined below when disabling `mlock`. This can also be provided via the environment variable `VAULT_DISABLE_MLOCK`. @@ -264,7 +267,7 @@ can have a negative effect on performance due to the tracking of each lock attem - `enable_post_unseal_trace` `(bool: false)` - Enables the server to generate a Go trace during the execution of the `core.postUnseal` function for debug purposes. The resulting trace can be viewed with the `go tool trace` command. The output directory can be specified with the `post_unseal_trace_directory` parameter. This should only be enabled temporarily for - debugging purposes as it can have a significant performance impact. This can be updated on a running Vault process with a + debugging purposes as it can have a significant performance impact. This can be updated on a running Vault process with a SIGHUP signal. - `post_unseal_trace_directory` `(string: "")` - Specifies the directory where the trace file will be written, which must exist