VAULT-33008: Enos tests for removed raft nodes (#29214)

* add test

* add as module

* more debugging of scenario

* fixes

* smoke test working

* autopilot test working

* revert local autopilot changes, cleanup comments and raft remove peer changes

* enos fmt

* modules fmt

* add vault_install_dir

* skip removal correctly for consul

* lint

* pr fixes

* passed run

* pr comments

* change step name everywhere

* fix

* check correct field

* remove cluster_name

enos/enos-modules.hcl

@@ -48,6 +48,10 @@ module "create_vpc" {
   common_tags = var.tags
 }
 
+module "choose_follower_host" {
+  source = "./modules/choose_follower_host"
+}
+
 module "ec2_info" {
   source = "./modules/ec2_info"
 }
@@ -92,6 +96,11 @@ module "replication_data" {
   source = "./modules/replication_data"
 }
 
+module "restart_vault" {
+  source            = "./modules/restart_vault"
+  vault_install_dir = var.vault_install_dir
+}
+
 module "seal_awskms" {
   source = "./modules/seal_awskms"
 
@@ -221,6 +230,11 @@ module "vault_failover_update_dr_primary" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "vault_raft_remove_node_and_verify" {
+  source            = "./modules/vault_raft_remove_node_and_verify"
+  vault_install_dir = var.vault_install_dir
+}
+
 module "vault_raft_remove_peer" {
   source            = "./modules/vault_raft_remove_peer"
   vault_install_dir = var.vault_install_dir
@@ -281,6 +295,18 @@ module "vault_verify_dr_replication" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "vault_verify_removed_node" {
+  source = "./modules/vault_verify_raft_removed"
+
+  vault_install_dir = var.vault_install_dir
+}
+
+module "vault_verify_removed_node_shim" {
+  source            = "./modules/vault_verify_removed_node_shim"
+  vault_install_dir = var.vault_install_dir
+}
+
+
 module "vault_verify_secrets_engines_create" {
   source = "./modules/verify_secrets_engines/modules/create"
 

enos/enos-qualities.hcl

@@ -552,6 +552,22 @@ quality "vault_raft_voters" {
   description = global.description.verify_raft_cluster_all_nodes_are_voters
 }
 
+quality "vault_raft_removed_after_restart" {
+  description = "A removed raft node will continue reporting as removed after the process is restarted"
+}
+
+quality "vault_raft_removed_statuses" {
+  description = "A removed raft node reports itself as removed in the status endpoints"
+}
+
+quality "vault_raft_removed_cant_rejoin" {
+  description = "A removed raft node cannot rejoin a cluster while it still has old vault/raft data"
+}
+
+quality "vault_raft_removed_rejoin_after_deletion" {
+  description = "A removed raft node can rejoin a cluster if it has deleted its old vault/raft data"
+}
+
 quality "vault_replication_ce_disabled" {
   description = "Replication is not enabled for CE editions"
 }

enos/enos-scenario-autopilot.hcl

@@ -656,12 +656,50 @@ scenario "autopilot" {
     }
   }
 
+  step "verify_raft_node_removed" {
+    description = <<-EOF
+      Verify that the removed nodes are marked as such
+    EOF
+    module      = semverconstraint(var.vault_upgrade_initial_version, ">=1.19.0-0") ? "vault_verify_removed_node" : "vault_verify_removed_node_shim"
+    depends_on = [
+      step.create_vault_cluster,
+      step.create_vault_cluster_targets,
+      step.get_updated_vault_cluster_ips,
+      step.raft_remove_peers,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_raft_removed_after_restart,
+      quality.vault_raft_removed_statuses,
+      quality.vault_raft_removed_cant_rejoin,
+    ]
+
+    variables {
+      add_back_nodes    = false
+      cluster_port      = step.create_vault_cluster.cluster_port
+      hosts             = step.create_vault_cluster.hosts
+      ip_version        = matrix.ip_version
+      listener_port     = step.create_vault_cluster.listener_port
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_leader_host = step.get_updated_vault_cluster_ips.leader_host
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_root_token  = step.create_vault_cluster.root_token
+      vault_seal_type   = matrix.seal
+      vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
+    }
+  }
+
   step "remove_old_nodes" {
     description = global.description.shutdown_nodes
     module      = module.shutdown_multiple_nodes
     depends_on = [
       step.create_vault_cluster,
-      step.raft_remove_peers
+      step.raft_remove_peers,
+      step.verify_raft_node_removed,
     ]
 
     providers = {

enos/enos-scenario-smoke.hcl

@@ -495,10 +495,71 @@ scenario "smoke" {
     }
   }
 
+
+  step "verify_raft_auto_join_voter" {
+    description = global.description.verify_raft_cluster_all_nodes_are_voters
+    skip_step   = matrix.backend != "raft"
+    module      = module.vault_verify_raft_auto_join_voter
+    depends_on  = [step.verify_vault_unsealed]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = quality.vault_raft_voters
+
+    variables {
+      hosts             = step.create_vault_cluster_targets.hosts
+      ip_version        = matrix.ip_version
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "vault_remove_node_and_verify" {
+    description = <<-EOF
+      Remove a follower and ensure that it's marked as removed and can be added back once its data has been deleted
+    EOF
+    module      = semverconstraint(var.vault_product_version, ">=1.19.0-0") && matrix.backend == "raft" ? "vault_raft_remove_node_and_verify" : "vault_verify_removed_node_shim"
+    depends_on = [
+      step.create_vault_cluster,
+      step.get_vault_cluster_ips,
+      step.verify_vault_unsealed,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_sys_storage_raft_remove_peer_write_removes_peer,
+      quality.vault_cli_operator_raft_remove_peer,
+      quality.vault_raft_removed_after_restart,
+      quality.vault_raft_removed_statuses,
+      quality.vault_raft_removed_cant_rejoin,
+      quality.vault_raft_removed_rejoin_after_deletion,
+    ]
+
+    variables {
+      add_back_nodes    = true
+      cluster_port      = step.create_vault_cluster.cluster_port
+      hosts             = step.get_vault_cluster_ips.follower_hosts
+      ip_version        = matrix.ip_version
+      listener_port     = step.create_vault_cluster.listener_port
+      vault_addr        = step.create_vault_cluster.api_addr_localhost
+      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_leader_host = step.get_vault_cluster_ips.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+      vault_seal_type   = matrix.seal
+      vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
+    }
+  }
+
   step "verify_secrets_engines_create" {
     description = global.description.verify_secrets_engines_create
     module      = module.vault_verify_secrets_engines_create
-    depends_on  = [step.verify_vault_unsealed]
+    depends_on  = [step.vault_remove_node_and_verify]
 
     providers = {
       enos = local.enos_provider[matrix.distro]
@@ -532,31 +593,10 @@ scenario "smoke" {
     }
   }
 
-  step "verify_raft_auto_join_voter" {
-    description = global.description.verify_raft_cluster_all_nodes_are_voters
-    skip_step   = matrix.backend != "raft"
-    module      = module.vault_verify_raft_auto_join_voter
-    depends_on  = [step.verify_vault_unsealed]
-
-    providers = {
-      enos = local.enos_provider[matrix.distro]
-    }
-
-    verifies = quality.vault_raft_voters
-
-    variables {
-      hosts             = step.create_vault_cluster_targets.hosts
-      ip_version        = matrix.ip_version
-      vault_addr        = step.create_vault_cluster.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
-      vault_root_token  = step.create_vault_cluster.root_token
-    }
-  }
-
   step "verify_replication" {
     description = global.description.verify_replication_status
     module      = module.vault_verify_replication
-    depends_on  = [step.verify_vault_unsealed]
+    depends_on  = [step.vault_remove_node_and_verify]
 
     providers = {
       enos = local.enos_provider[matrix.distro]
@@ -636,7 +676,7 @@ scenario "smoke" {
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui
-    depends_on  = [step.verify_vault_unsealed]
+    depends_on  = [step.vault_remove_node_and_verify]
 
     providers = {
       enos = local.enos_provider[matrix.distro]

enos/modules/choose_follower_host/main.tf

@@ -0,0 +1,17 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "followers" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault follower hosts"
+}
+
+output "chosen_follower" {
+  value = {
+    0 : try(var.followers[0], null)
+  }
+}

enos/modules/restart_vault/main.tf

@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "hosts" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault hosts"
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The local vault api address"
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the vault binary is installed"
+}
+
+
+resource "enos_remote_exec" "restart" {
+  for_each = var.hosts
+
+  environment = {
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  scripts = [abspath("${path.module}/scripts/restart-vault.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+

enos/modules/vault_raft_remove_node_and_verify/main.tf

@@ -0,0 +1,125 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "hosts" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster followers"
+}
+
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}
+
+variable "listener_port" {
+  type        = number
+  description = "The listener port for vault"
+}
+variable "vault_leader_host" {
+  type = object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  })
+  description = "The leader's host information"
+}
+variable "vault_addr" {
+  type        = string
+  description = "The local address to use to query vault"
+}
+variable "cluster_port" {
+  type        = number
+  description = "The cluster port for vault"
+}
+variable "ip_version" {
+  type        = number
+  description = "The IP version to use for the Vault TCP listeners"
+
+  validation {
+    condition     = contains([4, 6], var.ip_version)
+    error_message = "The ip_version must be either 4 or 6"
+  }
+}
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+variable "vault_seal_type" {
+  type        = string
+  description = "The Vault seal type"
+}
+
+variable "add_back_nodes" {
+  type        = bool
+  description = "whether to add the nodes back"
+}
+
+variable "vault_unseal_keys" {}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the vault binary is installed"
+}
+
+
+module "choose_follower_to_remove" {
+  source    = "../choose_follower_host"
+  followers = var.hosts
+}
+
+module "remove_raft_node" {
+  source     = "../vault_raft_remove_peer"
+  depends_on = [module.choose_follower_to_remove]
+
+
+  hosts                   = module.choose_follower_to_remove.chosen_follower
+  ip_version              = var.ip_version
+  is_voter                = true
+  operator_instance       = var.vault_leader_host.public_ip
+  vault_addr              = var.vault_addr
+  vault_cluster_addr_port = var.cluster_port
+  vault_install_dir       = var.vault_install_dir
+  vault_root_token        = var.vault_root_token
+}
+
+module "verify_removed" {
+  source = "../vault_verify_removed_node"
+  depends_on = [
+    module.remove_raft_node
+  ]
+
+  add_back_nodes    = true
+  cluster_port      = var.cluster_port
+  hosts             = module.choose_follower_to_remove.chosen_follower
+  ip_version        = var.ip_version
+  listener_port     = var.listener_port
+  vault_addr        = var.vault_addr
+  vault_install_dir = var.vault_install_dir
+  vault_leader_host = var.vault_leader_host
+  vault_root_token  = var.vault_root_token
+  vault_seal_type   = var.vault_seal_type
+  vault_unseal_keys = var.vault_seal_type == "shamir" ? var.vault_unseal_keys : null
+}

enos/modules/vault_raft_remove_peer/main.tf

@@ -53,6 +53,12 @@ variable "vault_root_token" {
   description = "The vault root token"
 }
 
+variable "is_voter" {
+  type        = bool
+  default     = false
+  description = "Whether the nodes that are going to be removed are voters"
+}
+
 resource "enos_remote_exec" "vault_raft_remove_peer" {
   for_each = var.hosts
 
@@ -61,6 +67,7 @@ resource "enos_remote_exec" "vault_raft_remove_peer" {
     VAULT_TOKEN               = var.vault_root_token
     VAULT_ADDR                = var.vault_addr
     VAULT_INSTALL_DIR         = var.vault_install_dir
+    REMOVE_NODE_IS_VOTER      = var.is_voter
   }
 
   scripts = [abspath("${path.module}/scripts/raft-remove-peer.sh")]

enos/modules/vault_raft_remove_peer/scripts/raft-remove-peer.sh

@@ -35,8 +35,8 @@ retry() {
 }
 
 remove_peer() {
-  if ! node_id=$("$binpath" operator raft list-peers -format json | jq -Mr --argjson expected "false" '.data.config.servers[] | select(.address=='\""$node_addr"\"') | select(.voter==$expected) | .node_id'); then
-    fail "failed to get node id of a non-voter node"
+  if ! node_id=$("$binpath" operator raft list-peers -format json | jq -Mr --argjson expected "${REMOVE_NODE_IS_VOTER}" '.data.config.servers[] | select(.address=='\""$node_addr"\"') | select(.voter==$expected) | .node_id'); then
+    fail "failed to get node id of a node with voter status ${REMOVE_NODE_IS_VOTER}"
   fi
 
   $binpath operator raft remove-peer "$node_id"

enos/modules/vault_upgrade/main.tf

@@ -132,21 +132,11 @@ module "get_ip_addresses" {
   vault_root_token  = var.vault_root_token
 }
 
-resource "enos_remote_exec" "restart_followers" {
-  for_each = module.get_ip_addresses.follower_hosts
-
-  environment = {
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-  }
-
-  scripts = [abspath("${path.module}/scripts/restart-vault.sh")]
-
-  transport = {
-    ssh = {
-      host = each.value.public_ip
-    }
-  }
+module "restart_followers" {
+  source            = "../restart_vault"
+  hosts             = module.get_ip_addresses.follower_hosts
+  vault_addr        = var.vault_addr
+  vault_install_dir = var.vault_install_dir
 }
 
 resource "enos_vault_unseal" "followers" {
@@ -154,7 +144,7 @@ resource "enos_vault_unseal" "followers" {
     for idx, host in module.get_ip_addresses.follower_hosts : idx => host
     if var.vault_seal_type == "shamir"
   }
-  depends_on = [enos_remote_exec.restart_followers]
+  depends_on = [module.restart_followers]
 
   bin_path    = local.vault_bin_path
   vault_addr  = var.vault_addr
@@ -171,7 +161,7 @@ resource "enos_vault_unseal" "followers" {
 module "wait_for_followers_unsealed" {
   source = "../vault_wait_for_cluster_unsealed"
   depends_on = [
-    enos_remote_exec.restart_followers,
+    module.restart_followers,
     enos_vault_unseal.followers,
   ]
 
@@ -180,26 +170,17 @@ module "wait_for_followers_unsealed" {
   vault_install_dir = var.vault_install_dir
 }
 
-resource "enos_remote_exec" "restart_leader" {
-  depends_on = [module.wait_for_followers_unsealed]
-
-  environment = {
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-  }
-
-  scripts = [abspath("${path.module}/scripts/restart-vault.sh")]
-
-  transport = {
-    ssh = {
-      host = module.get_ip_addresses.leader_public_ip
-    }
-  }
+module "restart_leader" {
+  depends_on        = [module.wait_for_followers_unsealed]
+  source            = "../restart_vault"
+  hosts             = module.get_ip_addresses.leader_hosts
+  vault_addr        = var.vault_addr
+  vault_install_dir = var.vault_install_dir
 }
 
 resource "enos_vault_unseal" "leader" {
   count      = var.vault_seal_type == "shamir" ? 1 : 0
-  depends_on = [enos_remote_exec.restart_leader]
+  depends_on = [module.restart_leader]
 
   bin_path    = local.vault_bin_path
   vault_addr  = var.vault_addr

enos/modules/vault_verify_removed_node/main.tf

@@ -0,0 +1,246 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "hosts" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster instances that were removed"
+}
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}
+
+variable "listener_port" {
+  type        = number
+  description = "The listener port for vault"
+}
+variable "vault_leader_host" {
+  type = object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  })
+  description = "The leader's host information"
+}
+variable "vault_addr" {
+  type        = string
+  description = "The local address to use to query vault"
+}
+variable "cluster_port" {
+  type        = number
+  description = "The cluster port for vault"
+}
+
+
+variable "ip_version" {
+  type        = number
+  description = "The IP version to use for the Vault TCP listeners"
+
+  validation {
+    condition     = contains([4, 6], var.ip_version)
+    error_message = "The ip_version must be either 4 or 6"
+  }
+}
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+variable "vault_seal_type" {
+  type        = string
+  description = "The Vault seal type"
+}
+
+variable "add_back_nodes" {
+  type        = bool
+  description = "whether to add the nodes back"
+}
+
+variable "vault_unseal_keys" {}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the vault binary is installed"
+}
+
+resource "enos_remote_exec" "verify_raft_peer_removed" {
+  for_each = var.hosts
+
+  environment = {
+    RETRY_INTERVAL    = var.retry_interval
+    TIMEOUT_SECONDS   = var.timeout
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  scripts = [abspath("${path.module}/scripts/verify_raft_remove_peer.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "verify_unseal_fails" {
+  for_each = {
+    for idx, host in var.hosts : idx => host
+    if var.vault_seal_type == "shamir"
+  }
+
+  environment = {
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    UNSEAL_KEYS       = join(",", var.vault_unseal_keys)
+  }
+
+  scripts = [abspath("${path.module}/scripts/verify_unseal_fails.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "verify_rejoin_fails" {
+  for_each = var.hosts
+
+  environment = {
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    RETRY_INTERVAL    = var.retry_interval
+    TIMEOUT_SECONDS   = var.timeout
+    VAULT_LEADER_ADDR = "${var.ip_version == 4 ? "${var.vault_leader_host.private_ip}" : "[${var.vault_leader_host.ipv6}]"}:${var.listener_port}"
+  }
+
+  scripts = [abspath("${path.module}/scripts/verify_manual_rejoin_fails.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+module "restart" {
+  depends_on        = [enos_remote_exec.verify_rejoin_fails, enos_remote_exec.verify_raft_peer_removed]
+  source            = "../restart_vault"
+  hosts             = var.hosts
+  vault_addr        = var.vault_addr
+  vault_install_dir = var.vault_install_dir
+}
+
+resource "enos_remote_exec" "verify_removed_after_restart" {
+  depends_on = [module.restart]
+  for_each   = var.hosts
+
+  environment = {
+    RETRY_INTERVAL    = var.retry_interval
+    TIMEOUT_SECONDS   = var.timeout
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+  }
+
+  scripts = [abspath("${path.module}/scripts/verify_raft_remove_peer.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+module "stop" {
+  depends_on = [enos_remote_exec.verify_removed_after_restart]
+  source     = "../stop_vault"
+  count      = var.add_back_nodes ? 1 : 0
+
+  hosts = var.hosts
+}
+
+resource "enos_remote_exec" "delete_data" {
+  depends_on = [module.stop]
+  for_each = {
+    for idx, host in var.hosts : idx => host
+    if var.add_back_nodes
+  }
+
+  inline = ["sudo rm -rf /opt/raft/data/*"]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+
+}
+
+resource "enos_remote_exec" "start" {
+  depends_on = [enos_remote_exec.delete_data]
+  for_each = {
+    for idx, host in var.hosts : idx => host
+    if var.add_back_nodes
+  }
+  inline = ["sudo systemctl start vault; sleep 5"]
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+resource "enos_vault_unseal" "unseal" {
+  depends_on = [
+    enos_remote_exec.start
+  ]
+  for_each = {
+    for idx, host in var.hosts : idx => host
+    if var.vault_seal_type == "shamir" && var.add_back_nodes
+  }
+
+  bin_path    = "${var.vault_install_dir}/vault"
+  vault_addr  = var.vault_addr
+  seal_type   = var.vault_seal_type
+  unseal_keys = var.vault_seal_type != "shamir" ? null : var.vault_unseal_keys
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+module "verify_rejoin_succeeds" {
+  source                  = "../vault_verify_raft_auto_join_voter"
+  depends_on              = [enos_vault_unseal.unseal]
+  count                   = var.add_back_nodes ? 1 : 0
+  hosts                   = var.hosts
+  ip_version              = var.ip_version
+  vault_root_token        = var.vault_root_token
+  vault_install_dir       = var.vault_install_dir
+  vault_addr              = var.vault_addr
+  vault_cluster_addr_port = var.cluster_port
+}

enos/modules/vault_verify_removed_node/scripts/verify_manual_rejoin_fails.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+function fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+[[ -z "$VAULT_LEADER_ADDR" ]] && fail "VAULT_LEADER_ADDR env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+result=$($binpath operator raft join "$VAULT_LEADER_ADDR")
+output=$?
+if [ $output -ne 2 ]; then
+  fail "Joining did not return code 2, instead $output: $result"
+fi

enos/modules/vault_verify_removed_node/scripts/verify_raft_remove_peer.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+function fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+getSysHealth() {
+  $binpath read -format=json sys/health sealedcode=299 haunhealthycode=299 removedcode=299 | jq -eMc '.data.removed_from_cluster'
+}
+
+getStatus() {
+  $binpath status --format=json | jq -eMc '.removed_from_cluster'
+}
+
+expectRemoved() {
+  local status
+  if ! status=$(getStatus); then
+    echo "failed to get vault status: $status"
+    return 1
+  fi
+  if [[ "$status" != "true" ]]; then
+    echo "unexpected status $status"
+    return 1
+  fi
+
+  local health
+  health=$(getSysHealth)
+  if ! health=$(getSysHealth); then
+    echo "failed to get health: $health"
+    return 1
+  fi
+  if [[ "$health" != "true" ]]; then
+    echo "unexpected health $health"
+  fi
+
+  return 0
+}
+
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+while [ "$(date +%s)" -lt "$end_time" ]; do
+  if expectRemoved; then
+    exit 0
+  fi
+
+  sleep "$RETRY_INTERVAL"
+done
+
+fail "Timed out waiting for raft removed status"

enos/modules/vault_verify_removed_node/scripts/verify_unseal_fails.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+function fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+
+IFS="," read -r -a keys <<< "${UNSEAL_KEYS}"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+result=$($binpath operator unseal "${keys[0]}")
+code=$?
+if [ $code -eq 0 ]; then
+  fail "expected unseal to fail but got exit code $code: $result"
+fi

enos/modules/vault_verify_removed_node_shim/main.tf

@@ -0,0 +1,89 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "hosts" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster followers"
+}
+
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}
+
+variable "listener_port" {
+  type        = number
+  description = "The listener port for vault"
+}
+
+variable "vault_leader_host" {
+  type = object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  })
+  description = "The leader's host information"
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The local address to use to query vault"
+}
+
+variable "cluster_port" {
+  type        = number
+  description = "The cluster port for vault"
+}
+
+variable "ip_version" {
+  type        = number
+  description = "The IP version to use for the Vault TCP listeners"
+
+  validation {
+    condition     = contains([4, 6], var.ip_version)
+    error_message = "The ip_version must be either 4 or 6"
+  }
+}
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+variable "vault_seal_type" {
+  type        = string
+  description = "The Vault seal type"
+}
+
+variable "add_back_nodes" {
+  type        = bool
+  description = "whether to add the nodes back"
+}
+
+variable "vault_unseal_keys" {}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the vault binary is installed"
+}