From 3389a572b97e0cf618b2f7cb5e39504a0967ad36 Mon Sep 17 00:00:00 2001 From: Mike Palmiotto Date: Wed, 20 Dec 2023 17:25:20 -0500 Subject: [PATCH] enos: Add Default LCQ validation to autopilot upgrade scenario (#24602) * enos: Add default lcq validation to autopilot upgrade scenario * Add timeout/retries to default lcq autopilot test --- enos/enos-modules.hcl | 7 ++ enos/enos-scenario-autopilot.hcl | 26 ++++++- enos/modules/vault_verify_default_lcq/main.tf | 74 +++++++++++++++++++ .../scripts/smoke-verify-default-lcq.sh | 46 ++++++++++++ 4 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 enos/modules/vault_verify_default_lcq/main.tf create mode 100755 enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 4b9eb8000e..8772a9e140 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -226,6 +226,13 @@ module "vault_verify_undo_logs" { vault_instance_count = var.vault_instance_count } +module "vault_verify_default_lcq" { + source = "./modules/vault_verify_default_lcq" + + vault_autopilot_default_max_leases = "300000" + vault_instance_count = var.vault_instance_count +} + module "vault_verify_replication" { source = "./modules/vault_verify_replication" diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 0e3c1c73ac..dfa9c646da 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -50,8 +50,9 @@ scenario "autopilot" { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - manage_service = matrix.artifact_type == "bundle" - vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] + vault_autopilot_default_max_leases = semverconstraint(matrix.initial_version, ">=1.16.0-0") ? "300000" : "" } step "build_vault" { @@ -524,6 +525,27 @@ scenario "autopilot" { } } + # Verify that upgrading from a version <1.16.0 does not introduce Default LCQ + step "verify_default_lcq" { + module = module.vault_verify_default_lcq + depends_on = [ + step.create_vault_cluster_upgrade_targets, + step.remove_old_nodes, + step.upgrade_vault_cluster_with_autopilot, + step.verify_autopilot_idle_state + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_instances = step.upgrade_vault_cluster_with_autopilot.target_hosts + vault_root_token = step.create_vault_cluster.root_token + vault_autopilot_default_max_leases = local.vault_autopilot_default_max_leases + } + } + output "audit_device_file_path" { description = "The file path for the file audit device, if enabled" value = step.create_vault_cluster.audit_device_file_path diff --git a/enos/modules/vault_verify_default_lcq/main.tf b/enos/modules/vault_verify_default_lcq/main.tf new file mode 100644 index 0000000000..d79fd63d5e --- /dev/null +++ b/enos/modules/vault_verify_default_lcq/main.tf @@ -0,0 +1,74 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "vault_instance_count" { + type = number + description = "How many vault instances are in the cluster" +} + +variable "vault_instances" { + type = map(object({ + private_ip = string + public_ip = string + })) + description = "The vault cluster instances that were created" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "vault_autopilot_default_max_leases" { + type = string + description = "The autopilot upgrade expected max_leases" +} + +variable "timeout" { + type = number + description = "The max number of seconds to wait before timing out" + default = 60 +} + +variable "retry_interval" { + type = number + description = "How many seconds to wait between each retry" + default = 2 +} + +locals { + public_ips = { + for idx in range(var.vault_instance_count) : idx => { + public_ip = values(var.vault_instances)[idx].public_ip + private_ip = values(var.vault_instances)[idx].private_ip + } + } +} + +resource "enos_remote_exec" "smoke_verify_default_lcq" { + for_each = local.public_ips + + environment = { + RETRY_INTERVAL = var.retry_interval + TIMEOUT_SECONDS = var.timeout + VAULT_ADDR = "http://localhost:8200" + VAULT_TOKEN = var.vault_root_token + DEFAULT_LCQ = var.vault_autopilot_default_max_leases + } + + scripts = [abspath("${path.module}/scripts/smoke-verify-default-lcq.sh")] + + transport = { + ssh = { + host = each.value.public_ip + } + } +} diff --git a/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh b/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh new file mode 100755 index 0000000000..493d8b8ba9 --- /dev/null +++ b/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +function fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" + +getMaxLeases() { + curl --request GET --header "X-Vault-Token: $VAULT_TOKEN" \ + "$VAULT_ADDR/v1/sys/quotas/lease-count/default" | jq '.data.max_leases // empty' +} + +waitForMaxLeases() { + local max_leases + if ! max_leases=$(getMaxLeases); then + echo "failed getting /v1/sys/quotas/lease-count/default data" 1>&2 + return 1 + fi + + if [[ "$max_leases" == "$DEFAULT_LCQ" ]]; then + echo "$max_leases" + return 0 + else + echo "Expected Default LCQ $DEFAULT_LCQ but got $max_leases" + return 1 + fi +} + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "$end_time" ]; do + if waitForMaxLeases; then + exit 0 + fi + + sleep "$RETRY_INTERVAL" +done + +fail "Timed out waiting for Default LCQ verification to complete. Data:\n\t$(getMaxLeases)" \ No newline at end of file