disable_mlock must now be explicitly included in config (#29974)

* require explicit value for disable_mlock

* set disable_mlock back to true for all docker tests

* fix build error

* update test config files

* change explicit mlock check to apply to integrated storage only.

* formatting and typo fixes

* added test for raft

* remove erroneous test

* remove unecessary doc line

* remove unecessary var

* pr suggestions

* test compile fix

* add mlock config value to enos tests

* enos lint

* update enos tests to pass disable_mlock value

* move mlock error to runtime to check for env var

* fixed mlock config detection logic

* call out mlock on/off tradeoffs to docs

* rewording production hardening section on mlock for clarity

* update error message when missing disable_mlock value to help customers with the previous default

* fix config doc error and update production-hardening doc to align with existing recommendations.

* remove extra check for mlock config value

* fix docker recovery test

* Update changelog/29974.txt

Explicitly call out that Vault will not start without disable_mlock included in the config.

Co-authored-by: Kuba Wieczorek <kuba.wieczorek@hashicorp.com>

* more docker test experimentation.

* passing disable_mlock into test cluster

* add VAULT_DISABLE_MLOCK envvar to docker tests and pass through the value

* add missing envvar for docker env test

* upate additional docker test disable_mlock values

* Apply suggestions from code review

Use active voice.

Co-authored-by: Sarah Chavis <62406755+schavis@users.noreply.github.com>

---------

Co-authored-by: Kuba Wieczorek <kuba.wieczorek@hashicorp.com>
Co-authored-by: Sarah Chavis <62406755+schavis@users.noreply.github.com>

builtin/logical/pkiext/pkiext_binary/pki_cluster.go

@@ -40,6 +40,7 @@ func NewVaultPkiCluster(t *testing.T) *VaultPkiCluster {
 			},
 			NumCores: 3,
 		},
+		DisableMlock: true,
 	}
 
 	cluster := docker.NewTestDockerCluster(t, opts)

changelog/29974.txt

@@ -0,0 +1,3 @@
+```release-note:change
+server: disable_mlock configuration option is now required for integrated storage and no longer has a default. If you are using the default value with integrated storage, you must now explicitly set disable_mlock to true or false or Vault server will fail to start.
+```

command/server.go

@@ -446,7 +446,6 @@ func (c *ServerCommand) parseConfig() (*server.Config, []configutil.ConfigError,
 		c.UI.Warn("WARNING: Entropy Augmentation is not supported in FIPS 140-2 Inside mode; disabling from server configuration!\n")
 		config.Entropy = nil
 	}
-
 	entCheckRequestLimiter(c, config)
 
 	return config, configErrors, nil
@@ -1132,7 +1131,8 @@ func (c *ServerCommand) Run(args []string) int {
 
 	logProxyEnvironmentVariables(c.logger)
 
-	if envMlock := os.Getenv("VAULT_DISABLE_MLOCK"); envMlock != "" {
+	envMlock := os.Getenv("VAULT_DISABLE_MLOCK")
+	if envMlock != "" {
 		var err error
 		config.DisableMlock, err = strconv.ParseBool(envMlock)
 		if err != nil {
@@ -1141,6 +1141,30 @@ func (c *ServerCommand) Run(args []string) int {
 		}
 	}
 
+	isMlockSet := func() bool {
+		// DisableMlock key has been found and thus explicitly set
+		return strutil.StrListContainsCaseInsensitive(config.SharedConfig.FoundKeys, "DisableMlock") ||
+			// envvar set
+			envMlock != ""
+	}
+
+	// ensure that the DisableMlock key is explicitly set if using integrated storage
+	if config.Storage != nil && config.Storage.Type == storageTypeRaft && !isMlockSet() {
+
+		c.UI.Error(wrapAtLength(
+			"ERROR: disable_mlock must be configured 'true' or 'false': Mlock " +
+				"prevents memory from being swapped to disk for security reasons, " +
+				"but can cause Vault on Integrated Storage to run out of memory " +
+				"if the machine was not provisioned with enough RAM. Disabling " +
+				"mlock can prevent this issue, but can result in the reveal of " +
+				"plaintext secrets when memory is swapped to disk, so it is only " +
+				"recommended on systems with encrypted swap or where swap is disabled." +
+				"Prior to Vault 1.20, disable_mlock defaulted to 'false' when the" +
+				"value was not set explicitly.",
+		))
+		return 1
+	}
+
 	if envLicensePath := os.Getenv(EnvVaultLicensePath); envLicensePath != "" {
 		config.LicensePath = envLicensePath
 	}

command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl

@@ -1,6 +1,7 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+disable_mlock = true
 storage "raft" {
   path = "/path/to/raft/data"
   node_id = "raft_node_1"

command/server_test.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 	"sync"
 	"testing"
@@ -32,16 +33,26 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// Modifier is a function that modifies a string
+type Modifier func(string) string
+
+// regexReplacer returns a Modifier that replaces all occurrences of re with repl
+func regexReplacer(re, repl string) Modifier {
+	return func(s string) string {
+		return regexp.MustCompile(re).ReplaceAllString(s, repl)
+	}
+}
+
 func init() {
 	if signed := os.Getenv("VAULT_LICENSE_CI"); signed != "" {
 		os.Setenv(EnvVaultLicense, signed)
 	}
 }
 
-func testBaseHCL(tb testing.TB, listenerExtras string) string {
+func testBaseHCL(tb testing.TB, listenerExtras string, modifiers ...Modifier) string {
 	tb.Helper()
 
-	return strings.TrimSpace(fmt.Sprintf(`
+	base := strings.TrimSpace(fmt.Sprintf(`
 		disable_mlock = true
 		listener "tcp" {
 			address     = "127.0.0.1:%d"
@@ -49,6 +60,11 @@ func testBaseHCL(tb testing.TB, listenerExtras string) string {
 			%s
 		}
 	`, 0, listenerExtras))
+
+	for _, mod := range modifiers {
+		base = mod(base)
+	}
+	return base
 }
 
 const (
@@ -62,6 +78,12 @@ const (
 	badListenerWriteTimeout      = `http_write_timeout = "56lbs"`
 	badListenerIdleTimeout       = `http_idle_timeout = "78gophers"`
 
+	raftHCL = `
+storage "raft" {
+  path = "/path/to/raft/data"
+  node_id = "raft_node_1"
+}
+	`
 	inmemHCL = `
 backend "inmem_ha" {
   advertise_addr       = "http://127.0.0.1:8200"
@@ -72,7 +94,6 @@ ha_backend "inmem_ha" {
   redirect_addr        = "http://127.0.0.1:8200"
 }
 `
-
 	badHAInmemHCL = `
 ha_backend "inmem" {}
 `
@@ -275,6 +296,13 @@ func TestServer(t *testing.T) {
 			0,
 			[]string{"-test-verify-only", "-recovery"},
 		},
+		{
+			"missing_disable_mlock_value_with_integrated_storage",
+			testBaseHCL(t, goodListenerTimeouts, regexReplacer(`\s*disable_mlock\s*=\s*.+`, "")) + raftHCL,
+			"disable_mlock must be configured",
+			1,
+			[]string{"-test-server-config"},
+		},
 	}
 
 	for _, tc := range cases {

command/test-fixtures/config.hcl

@@ -1,4 +1,4 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
-
-token_helper = "foo"
+disable_mlock = true
+token_helper = "foo"

enos/enos-globals.hcl

@@ -156,6 +156,7 @@ globals {
     bundle  = "/opt/vault/bin"
     package = "/usr/bin"
   }
-  vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic"))
-  vault_tag_key      = "vault-cluster"
+  vault_license_path  = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic"))
+  vault_tag_key       = "vault-cluster"
+  vault_disable_mlock = false
 }

enos/modules/start_vault/main.tf

@@ -188,7 +188,9 @@ resource "enos_vault_start" "leader" {
   bin_path    = local.bin_path
   config_dir  = var.config_dir
   config_mode = var.config_mode
-  environment = var.environment
+  environment = merge(var.environment, {
+    VAULT_DISABLE_MLOCK = var.disable_mlock
+  })
   config = {
     api_addr     = local.api_addrs_internal[tonumber(each.value)][var.ip_version]
     cluster_addr = local.cluster_addrs_internal[tonumber(each.value)][var.ip_version]
@@ -230,7 +232,9 @@ resource "enos_vault_start" "followers" {
   bin_path    = local.bin_path
   config_dir  = var.config_dir
   config_mode = var.config_mode
-  environment = var.environment
+  environment = merge(var.environment, {
+    VAULT_DISABLE_MLOCK = var.disable_mlock
+  })
   config = {
     api_addr     = local.api_addrs_internal[tonumber(each.value)][var.ip_version]
     cluster_addr = local.cluster_addrs_internal[tonumber(each.value)][var.ip_version]

enos/modules/start_vault/variables.tf

@@ -34,6 +34,12 @@ variable "config_mode" {
   }
 }
 
+variable "disable_mlock" {
+  type        = bool
+  description = "Disable mlock for Vault process."
+  default     = false
+}
+
 variable "environment" {
   description = "Optional Vault configuration environment variables to set starting Vault"
   type        = map(string)

enos/modules/vault_cluster/main.tf

@@ -20,6 +20,7 @@ locals {
   bin_path               = "${var.install_dir}/vault"
   consul_bin_path        = "${var.consul_install_dir}/consul"
   enable_audit_devices   = var.enable_audit_devices && var.initialize_cluster
+  disable_mlock          = false
   // In order to get Terraform to plan we have to use collections with keys
   // that are known at plan time. In order for our module to work our var.hosts
   // must be a map with known keys at plan time. Here we're creating locals
@@ -156,6 +157,7 @@ module "start_vault" {
   cluster_tag_key           = var.cluster_tag_key
   config_dir                = var.config_dir
   config_mode               = var.config_mode
+  disable_mlock             = local.disable_mlock
   external_storage_port     = var.external_storage_port
   hosts                     = var.hosts
   install_dir               = var.install_dir

sdk/helper/testcluster/docker/environment.go

@@ -77,7 +77,8 @@ type DockerCluster struct {
 	Logger    log.Logger
 	builtTags map[string]struct{}
 
-	storage testcluster.ClusterStorage
+	storage      testcluster.ClusterStorage
+	disableMlock bool
 }
 
 func (dc *DockerCluster) NamedLogger(s string) log.Logger {
@@ -401,7 +402,7 @@ func (n *DockerClusterNode) setupCert(ip string) error {
 
 func NewTestDockerCluster(t *testing.T, opts *DockerClusterOptions) *DockerCluster {
 	if opts == nil {
-		opts = &DockerClusterOptions{}
+		opts = &DockerClusterOptions{DisableMlock: true}
 	}
 	if opts.ClusterName == "" {
 		opts.ClusterName = strings.ReplaceAll(t.Name(), "/", "-")
@@ -431,7 +432,7 @@ func NewDockerCluster(ctx context.Context, opts *DockerClusterOptions) (*DockerC
 	}
 
 	if opts == nil {
-		opts = &DockerClusterOptions{}
+		opts = &DockerClusterOptions{DisableMlock: true}
 	}
 	if opts.Logger == nil {
 		opts.Logger = log.NewNullLogger()
@@ -441,12 +442,13 @@ func NewDockerCluster(ctx context.Context, opts *DockerClusterOptions) (*DockerC
 	}
 
 	dc := &DockerCluster{
-		DockerAPI:   api,
-		ClusterName: opts.ClusterName,
-		Logger:      opts.Logger,
-		builtTags:   map[string]struct{}{},
-		CA:          opts.CA,
-		storage:     opts.Storage,
+		DockerAPI:    api,
+		ClusterName:  opts.ClusterName,
+		Logger:       opts.Logger,
+		builtTags:    map[string]struct{}{},
+		CA:           opts.CA,
+		storage:      opts.Storage,
+		disableMlock: opts.DisableMlock,
 	}
 
 	if err := dc.setupDockerCluster(ctx, opts); err != nil {
@@ -706,7 +708,7 @@ func (n *DockerClusterNode) Start(ctx context.Context, opts *DockerClusterOption
 
 	//// disable_mlock is required for working in the Docker environment with
 	//// custom plugins
-	vaultCfg["disable_mlock"] = true
+	vaultCfg["disable_mlock"] = opts.DisableMlock
 
 	protocol := "https"
 	if opts.DisableTLS {
@@ -811,6 +813,7 @@ func (n *DockerClusterNode) Start(ctx context.Context, opts *DockerClusterOption
 		"SKIP_SETCAP=true",
 		"VAULT_LOG_FORMAT=json",
 		"VAULT_LICENSE=" + opts.VaultLicense,
+		"VAULT_DISABLE_MLOCK=" + strconv.FormatBool(opts.DisableMlock),
 	}
 	envs = append(envs, opts.Envs...)
 
@@ -1085,17 +1088,18 @@ func (l LogConsumerWriter) Write(p []byte) (n int, err error) {
 // DockerClusterOptions has options for setting up the docker cluster
 type DockerClusterOptions struct {
 	testcluster.ClusterOptions
-	CAKey       *ecdsa.PrivateKey
-	NetworkName string
-	ImageRepo   string
-	ImageTag    string
-	CA          *testcluster.CA
-	VaultBinary string
-	Args        []string
-	Envs        []string
-	StartProbe  func(*api.Client) error
-	Storage     testcluster.ClusterStorage
-	DisableTLS  bool
+	CAKey        *ecdsa.PrivateKey
+	NetworkName  string
+	ImageRepo    string
+	ImageTag     string
+	CA           *testcluster.CA
+	VaultBinary  string
+	Args         []string
+	Envs         []string
+	StartProbe   func(*api.Client) error
+	Storage      testcluster.ClusterStorage
+	DisableTLS   bool
+	DisableMlock bool
 }
 
 func ensureLeaderMatches(ctx context.Context, client *api.Client, ready func(response *api.LeaderResponse) error) error {
@@ -1259,7 +1263,7 @@ func (dc *DockerCluster) joinNode(ctx context.Context, nodeIdx int, leaderIdx in
 
 func (dc *DockerCluster) setupImage(ctx context.Context, opts *DockerClusterOptions) (string, error) {
 	if opts == nil {
-		opts = &DockerClusterOptions{}
+		opts = &DockerClusterOptions{DisableMlock: true}
 	}
 	sourceTag := opts.ImageTag
 	if sourceTag == "" {

sdk/helper/testcluster/docker/environment_test.go

@@ -11,9 +11,10 @@ func TestSettingEnvsToContainer(t *testing.T) {
 	expectedEnv := "TEST_ENV=value1"
 	expectedEnv2 := "TEST_ENV2=value2"
 	opts := &DockerClusterOptions{
-		ImageRepo: "hashicorp/vault",
-		ImageTag:  "latest",
-		Envs:      []string{expectedEnv, expectedEnv2},
+		ImageRepo:    "hashicorp/vault",
+		ImageTag:     "latest",
+		Envs:         []string{expectedEnv, expectedEnv2},
+		DisableMlock: true,
 	}
 	cluster := NewTestDockerCluster(t, opts)
 	defer cluster.Cleanup()

sdk/helper/testcluster/docker/replication.go

@@ -27,6 +27,7 @@ func DefaultOptions(t *testing.T) *DockerClusterOptions {
 				LogLevel: "TRACE",
 			},
 		},
+		DisableMlock: true,
 	}
 }
 

sdk/helper/testcluster/types.go

@@ -41,8 +41,8 @@ type VaultNodeConfig struct {
 	//   ClusterAddr          string      `hcl:"cluster_addr"`
 	//   Storage   *Storage `hcl:"-"`
 	//   HAStorage *Storage `hcl:"-"`
-	//   DisableMlock bool `hcl:"disable_mlock"`
 	//   ClusterName string `hcl:"cluster_name"`
+	// 	 DisableMlock bool  `hcl:"disable_mlock"`
 
 	// Not configurable yet:
 	//   Listeners []*Listener `hcl:"-"`

vault/external_tests/misc/misc_binary/recovery_test.go

@@ -31,7 +31,8 @@ func TestRecovery_Docker(t *testing.T) {
 		t.Skip("only running docker test when $VAULT_BINARY present")
 	}
 	opts := &docker.DockerClusterOptions{
-		ImageRepo: "hashicorp/vault",
+		ImageRepo:    "hashicorp/vault",
+		DisableMlock: true,
 		// We're replacing the binary anyway, so we're not too particular about
 		// the docker image version tag.
 		ImageTag:    "latest",

vault/external_tests/misc/recover_from_panic_test.go

@@ -19,7 +19,8 @@ func TestRecoverFromPanic(t *testing.T) {
 		LogicalBackends: map[string]logical.Factory{
 			"noop": vault.NoopBackendFactory,
 		},
-		EnableRaw: true,
+		EnableRaw:    true,
+		DisableMlock: true,
 	}
 	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
 		HandlerFunc: vaulthttp.Handler,

vault/external_tests/misc/recovery_test.go

@@ -27,7 +27,8 @@ func TestRecovery(t *testing.T) {
 	var rootToken string
 	{
 		conf := vault.CoreConfig{
-			Physical: inm,
+			Physical:     inm,
+			DisableMlock: true,
 		}
 		opts := vault.TestClusterOptions{
 			HandlerFunc: http.Handler,

vault/external_tests/raft/raft_binary/raft_test.go

@@ -32,7 +32,8 @@ func TestRaft_Configuration_Docker(t *testing.T) {
 		t.Skip("only running docker test when $VAULT_BINARY present")
 	}
 	opts := &docker.DockerClusterOptions{
-		ImageRepo: "hashicorp/vault",
+		ImageRepo:    "hashicorp/vault",
+		DisableMlock: true,
 		// We're replacing the binary anyway, so we're not too particular about
 		// the docker image version tag.
 		ImageTag:    "latest",
@@ -138,7 +139,8 @@ func TestDocker_LogStore_Boltdb_To_Raftwal_And_Back(t *testing.T) {
 		t.Skip("only running docker test when $VAULT_BINARY present")
 	}
 	opts := &docker.DockerClusterOptions{
-		ImageRepo: "hashicorp/vault",
+		ImageRepo:    "hashicorp/vault",
+		DisableMlock: true,
 		// We're replacing the binary anyway, so we're not too particular about
 		// the docker image version tag.
 		ImageTag:    "latest",
@@ -332,7 +334,8 @@ func TestRaft_LogStore_Migration_Snapshot(t *testing.T) {
 		t.Skip("only running docker test when $VAULT_BINARY present")
 	}
 	opts := &docker.DockerClusterOptions{
-		ImageRepo: "hashicorp/vault",
+		ImageRepo:    "hashicorp/vault",
+		DisableMlock: true,
 		// We're replacing the binary anyway, so we're not too particular about
 		// the docker image version tag.
 		ImageTag:    "latest",

website/content/docs/concepts/production-hardening.mdx

@@ -123,7 +123,7 @@ not be suitable for every deployment.
 
   When using platform-specific identity solutions, you should be mindful of auth
   method and secret engine configuration within namespaces. You can share
-  platform identity across Vault namespaces, as these provider features 
+  platform identity across Vault namespaces, as these provider features
   generally offer host-based identity solutions.
 
   If that is not applicable, set the credentials as environment variables
@@ -154,7 +154,7 @@ not be suitable for every deployment.
   give secret values from either standard input or with command-line arguments.
   Command-line arguments can persisted in shell history, and are readable by other unprivileged users on the same host.
 
-- **Develop an off-boarding process**. Removing accounts in Vault or associated 
+- **Develop an off-boarding process**. Removing accounts in Vault or associated
   identity providers may not immediately revoke [token-based access](/vault/docs/concepts/tokens#user-management-considerations).
   Depending on how you manage access to Vault, operators should consider:
 
@@ -213,3 +213,13 @@ not be suitable for every deployment.
 - **Be aware of special container considerations**. To use memory locking
   (mlock) inside a Vault container, you need to use the `overlayfs2` or another
   supporting driver.
+
+-	**Consider memory usage when configuring `disable_mlock` with integrated storage**.
+  When you leave `mlock` enabled, Vault disallows data swaps from memory to disk.
+  Disallowing swaps ensures that sensitive data remains in RAM, but also
+  increases the risk of out-of-memory (OOM) errors in under-provisioned systems.
+  Conversely, disabling `mlock` lets the operating system swap memory to disk
+  under pressure, which can reduce the risk of OOM errors but increase the
+  exposure risk for secrets that end up in the swap file. If you choose to
+  disable `mlock`, we recommend encrypting your swap file to limit the exposure
+  risk.

website/content/docs/configuration/index.mdx

@@ -97,9 +97,12 @@ to specify where the configuration is.
   the read cache used by the physical storage subsystem. This will very
   significantly impact performance.
 
-- `disable_mlock` `(bool: false)` – Disables the server from executing the
-  `mlock` syscall. `mlock` prevents memory from being swapped to disk. Disabling
-  `mlock` is not recommended unless using [integrated storage](/vault/docs/internals/integrated-storage).
+- `disable_mlock` `(bool: <required>)` – Stops Vault from executing the `mlock`
+  syscall. `mlock` prevents data swaps from memory to disk. You must set an
+  explicit value for `disable_mlock` if you use
+  [integrated storage](/vault/docs/internals/integrated-storage). We do not
+  recommend disabling `mlock` for deployments that do not use integrated storage.
+
   Follow the additional security precautions outlined below when disabling `mlock`.
   This can also be provided via the environment variable `VAULT_DISABLE_MLOCK`.
 
@@ -264,7 +267,7 @@ can have a negative effect on performance due to the tracking of each lock attem
 - `enable_post_unseal_trace` `(bool: false)` - Enables the server to generate a Go trace during the execution of the
   `core.postUnseal` function for debug purposes. The resulting trace can be viewed with the `go tool trace` command. The output
   directory can be specified with the `post_unseal_trace_directory` parameter. This should only be enabled temporarily for
-  debugging purposes as it can have a significant performance impact. This can be updated on a running Vault process with a 
+  debugging purposes as it can have a significant performance impact. This can be updated on a running Vault process with a
   SIGHUP signal.
 
 - `post_unseal_trace_directory` `(string: "")` - Specifies the directory where the trace file will be written, which must exist