enos: Add Default LCQ validation to autopilot upgrade scenario (#24602)

* enos: Add default lcq validation to autopilot upgrade scenario

* Add timeout/retries to default lcq autopilot test

enos/enos-modules.hcl

@@ -226,6 +226,13 @@ module "vault_verify_undo_logs" {
   vault_instance_count = var.vault_instance_count
 }
 
+module "vault_verify_default_lcq" {
+  source = "./modules/vault_verify_default_lcq"
+
+  vault_autopilot_default_max_leases = "300000"
+  vault_instance_count               = var.vault_instance_count
+}
+
 module "vault_verify_replication" {
   source = "./modules/vault_verify_replication"
 

enos/enos-scenario-autopilot.hcl

@@ -50,8 +50,9 @@ scenario "autopilot" {
       rhel   = provider.enos.rhel
       ubuntu = provider.enos.ubuntu
     }
-    manage_service    = matrix.artifact_type == "bundle"
-    vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro]
+    manage_service                     = matrix.artifact_type == "bundle"
+    vault_install_dir                  = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro]
+    vault_autopilot_default_max_leases = semverconstraint(matrix.initial_version, ">=1.16.0-0") ? "300000" : ""
   }
 
   step "build_vault" {
@@ -524,6 +525,27 @@ scenario "autopilot" {
     }
   }
 
+  # Verify that upgrading from a version <1.16.0 does not introduce Default LCQ
+  step "verify_default_lcq" {
+    module = module.vault_verify_default_lcq
+    depends_on = [
+      step.create_vault_cluster_upgrade_targets,
+      step.remove_old_nodes,
+      step.upgrade_vault_cluster_with_autopilot,
+      step.verify_autopilot_idle_state
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_instances                    = step.upgrade_vault_cluster_with_autopilot.target_hosts
+      vault_root_token                   = step.create_vault_cluster.root_token
+      vault_autopilot_default_max_leases = local.vault_autopilot_default_max_leases
+    }
+  }
+
   output "audit_device_file_path" {
     description = "The file path for the file audit device, if enabled"
     value       = step.create_vault_cluster.audit_device_file_path

enos/modules/vault_verify_default_lcq/main.tf

@@ -0,0 +1,74 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_instances" {
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster instances that were created"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "vault_autopilot_default_max_leases" {
+  type        = string
+  description = "The autopilot upgrade expected max_leases"
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+locals {
+  public_ips = {
+    for idx in range(var.vault_instance_count) : idx => {
+      public_ip  = values(var.vault_instances)[idx].public_ip
+      private_ip = values(var.vault_instances)[idx].private_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "smoke_verify_default_lcq" {
+  for_each = local.public_ips
+
+  environment = {
+    RETRY_INTERVAL  = var.retry_interval
+    TIMEOUT_SECONDS = var.timeout
+    VAULT_ADDR      = "http://localhost:8200"
+    VAULT_TOKEN     = var.vault_root_token
+    DEFAULT_LCQ     = var.vault_autopilot_default_max_leases
+  }
+
+  scripts = [abspath("${path.module}/scripts/smoke-verify-default-lcq.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}

enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+function fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+
+getMaxLeases() {
+  curl --request GET --header "X-Vault-Token: $VAULT_TOKEN" \
+    "$VAULT_ADDR/v1/sys/quotas/lease-count/default" | jq '.data.max_leases // empty'
+}
+
+waitForMaxLeases() {
+  local max_leases
+  if ! max_leases=$(getMaxLeases); then
+    echo "failed getting /v1/sys/quotas/lease-count/default data" 1>&2
+    return 1
+  fi
+
+  if [[ "$max_leases" == "$DEFAULT_LCQ" ]]; then
+    echo "$max_leases"
+    return 0
+  else
+    echo "Expected Default LCQ $DEFAULT_LCQ but got $max_leases"
+    return 1
+  fi
+}
+
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+while [ "$(date +%s)" -lt "$end_time" ]; do
+  if waitForMaxLeases; then
+    exit 0
+  fi
+
+  sleep "$RETRY_INTERVAL"
+done
+
+fail "Timed out waiting for Default LCQ verification to complete. Data:\n\t$(getMaxLeases)"