VAULT-39598: Fixing Enos Dev Scenario KMIP Error (#9375) (#9396)

* testing kmip enos dev fix

* updating kmip

* updating kmip

* updating kmip

* updating kmip

* updating description

Co-authored-by: Tin Vo <tintvo08@gmail.com>

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -197,6 +197,7 @@ jobs:
             echo 'ENOS_VAR_vault_revision=${{ inputs.vault-revision }}'
             echo 'ENOS_VAR_vault_upgrade_initial_version=${{ matrix.attributes.upgrade_initial_version }}'
             echo 'ENOS_VAR_verify_aws_secrets_engine=false'
+            echo 'ENOS_VAR_verify_kmip_secrets_engine=true'
             echo 'ENOS_VAR_verify_ldap_secrets_engine=false'
             echo 'ENOS_VAR_verify_log_secrets=true'
           } | tee -a "$GITHUB_ENV"

enos/enos-modules.hcl

@@ -333,6 +333,7 @@ module "vault_verify_secrets_engines_create" {
 
   aws_enabled       = var.verify_aws_secrets_engine
   ldap_enabled      = var.verify_ldap_secrets_engine
+  kmip_enabled      = var.verify_kmip_secrets_engine
   vault_install_dir = var.vault_install_dir
 }
 
@@ -341,6 +342,7 @@ module "vault_verify_secrets_engines_read" {
 
   aws_enabled       = var.verify_aws_secrets_engine
   ldap_enabled      = var.verify_ldap_secrets_engine
+  kmip_enabled      = var.verify_kmip_secrets_engine
   vault_install_dir = var.vault_install_dir
 }
 

enos/enos-variables.hcl

@@ -205,6 +205,12 @@ variable "verify_aws_secrets_engine" {
   default     = false
 }
 
+variable "verify_kmip_secrets_engine" {
+  description = "If true we'll verify KMIP secrets engines behavior"
+  type        = bool
+  default     = false
+}
+
 variable "verify_ldap_secrets_engine" {
   description = "If true we'll verify LDAP secrets engines behavior"
   type        = bool

enos/modules/verify_secrets_engines/modules/create/kmip.tf

@@ -1,182 +1,27 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
-variable "kmip_listen_address" {
-  type        = string
-  description = "The KMIP listen address for the Vault server"
-  default     = "0.0.0.0"
+module "create_kmip_secret_engine" {
+  depends_on = [
+    enos_remote_exec.policy_write_kv_writer,
+  ]
+  count  = var.kmip_enabled ? 1 : 0
+  source = "./kmip"
+
+  integration_host_state = var.integration_host_state
+  ip_version             = var.ip_version
+  leader_host            = var.leader_host
+  ports                  = var.ports
+  vault_addr             = var.vault_addr
+  vault_edition          = var.vault_edition
+  vault_root_token       = var.vault_root_token
+  vault_install_dir      = var.vault_install_dir
 }
 
 locals {
-  kmip_scope_name  = "kmip_scope"
-  kmip_role_name   = "kmip_role"
-  kmip_cert_format = "pem"
-  kmip_mount_path  = "kmip"
-
-  // Response data - only access if Vault Enterprise (count > 0)
-  server_ca   = var.vault_edition == "ce" ? "" : enos_remote_exec.kmip_configure[0].stdout
-  client_cert = var.vault_edition == "ce" ? "" : enos_remote_exec.kmip_generate_certificate[0].stdout
-
-  kmip_output = {
-    server_ca      = local.server_ca
-    client_cert    = local.client_cert
-    test_server_ip = var.integration_host_state.kmip.host.public_ip
-    port           = var.ports.kmip.port
-  }
+  kmip_output = var.kmip_enabled ? module.create_kmip_secret_engine[0].kmip : null
 }
 
 output "kmip" {
   value = local.kmip_output
 }
-
-resource "enos_remote_exec" "secrets_enable_kmip_secret" {
-  environment = {
-    ENGINE            = "kmip"
-    MOUNT             = local.kmip_mount_path
-    VAULT_ADDR        = var.vault_addr
-    VAULT_TOKEN       = var.vault_root_token
-    VAULT_INSTALL_DIR = var.vault_install_dir
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/secrets-enable.sh")]
-
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}
-
-resource "enos_remote_exec" "kmip_configure" {
-  depends_on = [enos_remote_exec.secrets_enable_kmip_secret]
-  environment = {
-    MOUNT             = local.kmip_mount_path
-    VAULT_ADDR        = var.vault_addr
-    VAULT_TOKEN       = var.vault_root_token
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    KMIP_MOUNT        = local.kmip_mount_path
-    KMIP_LISTEN_ADDR  = var.kmip_listen_address
-    KMIP_PORT         = var.ports.kmip.port
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/kmip/kmip-configure.sh")]
-
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}
-
-# Creating KMIP Scope
-resource "enos_remote_exec" "kmip_create_scope" {
-  depends_on = [enos_remote_exec.kmip_configure]
-
-  environment = {
-    REQPATH           = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}"
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    VAULT_TOKEN       = var.vault_root_token
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/write.sh")]
-
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}
-
-# Creating KMIP Role
-resource "enos_remote_exec" "kmip_create_role" {
-  depends_on = [enos_remote_exec.kmip_create_scope]
-
-  environment = {
-    REQPATH = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}/role/${local.kmip_role_name}"
-    PAYLOAD = jsonencode({
-      operation_all = true,
-    })
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    VAULT_TOKEN       = var.vault_root_token
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/write.sh")]
-
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}
-
-# Generating KMIP Certificate
-resource "enos_remote_exec" "kmip_generate_certificate" {
-  depends_on = [enos_remote_exec.kmip_create_role]
-
-  environment = {
-    MOUNT             = local.kmip_mount_path
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    VAULT_TOKEN       = var.vault_root_token
-    SCOPE_NAME        = local.kmip_scope_name
-    ROLE_NAME         = local.kmip_role_name
-    CERT_FORMAT       = local.kmip_cert_format
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/kmip/kmip-generate-cert.sh")]
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}
-
-# Managing KMIP Roles
-resource "enos_remote_exec" "kmip_manage_roles" {
-  depends_on = [enos_remote_exec.kmip_generate_certificate]
-  environment = {
-    REQPATH = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}/role/${local.kmip_role_name}"
-    PAYLOAD = jsonencode({
-      operation_activate = true,
-      operation_create   = true,
-      operation_get      = true
-    })
-    VAULT_ADDR        = var.vault_addr
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    VAULT_TOKEN       = var.vault_root_token
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/write.sh")]
-
-  transport = {
-    ssh = {
-      host = var.leader_host.public_ip
-    }
-  }
-}

enos/modules/verify_secrets_engines/modules/create/kmip/kmip.tf

@@ -0,0 +1,240 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "kmip_listen_address" {
+  type        = string
+  description = "The KMIP listen address for the Vault server"
+  default     = "0.0.0.0"
+}
+
+variable "leader_host" {
+  type = object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  })
+
+  description = "Vault cluster leader host"
+}
+
+variable "ports" {
+  description = "Port configuration for services"
+  type = map(object({
+    port        = string
+    description = string
+  }))
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The local vault API listen address"
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The Vault root token"
+  default     = null
+}
+
+variable "integration_host_state" {
+  description = "The state of the test server from the 'set_up_external_integration' module"
+}
+
+variable "ip_version" {
+  type        = string
+  description = "IP Version (4 or 6)"
+  default     = "4"
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "IP Version (4 or 6)"
+  default     = "4"
+}
+
+locals {
+  kmip_scope_name  = "kmip_scope"
+  kmip_role_name   = "kmip_role"
+  kmip_cert_format = "pem"
+  kmip_mount_path  = "kmip"
+
+  // Response data - only access if Vault Enterprise (count > 0)
+  server_ca   = var.vault_edition == "ce" ? "" : enos_remote_exec.kmip_configure[0].stdout
+  client_cert = var.vault_edition == "ce" ? "" : enos_remote_exec.kmip_generate_certificate[0].stdout
+
+  kmip_output = {
+    server_ca      = local.server_ca
+    client_cert    = local.client_cert
+    test_server_ip = var.integration_host_state.kmip.host.public_ip
+    port           = var.ports.kmip.port
+  }
+}
+
+output "kmip" {
+  value = local.kmip_output
+}
+
+resource "enos_remote_exec" "secrets_enable_kmip_secret" {
+  environment = {
+    ENGINE            = "kmip"
+    MOUNT             = local.kmip_mount_path
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/secrets-enable.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "kmip_configure" {
+  depends_on = [enos_remote_exec.secrets_enable_kmip_secret]
+  environment = {
+    MOUNT             = local.kmip_mount_path
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    KMIP_MOUNT        = local.kmip_mount_path
+    KMIP_LISTEN_ADDR  = var.kmip_listen_address
+    KMIP_PORT         = var.ports.kmip.port
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/kmip/kmip-configure.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+# Creating KMIP Scope
+resource "enos_remote_exec" "kmip_create_scope" {
+  depends_on = [enos_remote_exec.kmip_configure]
+
+  environment = {
+    REQPATH           = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}"
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/write.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+# Creating KMIP Role
+resource "enos_remote_exec" "kmip_create_role" {
+  depends_on = [enos_remote_exec.kmip_create_scope]
+
+  environment = {
+    REQPATH = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}/role/${local.kmip_role_name}"
+    PAYLOAD = jsonencode({
+      operation_all = true,
+    })
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/write.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+# Generating KMIP Certificate
+resource "enos_remote_exec" "kmip_generate_certificate" {
+  depends_on = [enos_remote_exec.kmip_create_role]
+
+  environment = {
+    MOUNT             = local.kmip_mount_path
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+    SCOPE_NAME        = local.kmip_scope_name
+    ROLE_NAME         = local.kmip_role_name
+    CERT_FORMAT       = local.kmip_cert_format
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/kmip/kmip-generate-cert.sh")]
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+# Managing KMIP Roles
+resource "enos_remote_exec" "kmip_manage_roles" {
+  depends_on = [enos_remote_exec.kmip_generate_certificate]
+  environment = {
+    REQPATH = "${local.kmip_mount_path}/scope/${local.kmip_scope_name}/role/${local.kmip_role_name}"
+    PAYLOAD = jsonencode({
+      operation_activate = true,
+      operation_create   = true,
+      operation_get      = true
+    })
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/write.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}

enos/modules/verify_secrets_engines/modules/create/main.tf

@@ -28,6 +28,12 @@ variable "ldap_enabled" {
   default     = false
 }
 
+variable "kmip_enabled" {
+  type        = bool
+  description = "Whether or not we'll verify the KMIP secrets engine"
+  default     = false
+}
+
 variable "ipv4_cidr" {
   type        = string
   description = "The CIDR block for the VPC when using IPv4 mode"

enos/modules/verify_secrets_engines/modules/read/kmip.tf

@@ -1,33 +1,15 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
-locals {
-  kmip_output = {
-    mount      = "kmip"
-    ip_address = var.ip_version == "6" ? var.hosts[0].ipv6 : var.hosts[0].public_ip
-  }
+# Verifying Vault LDAP Configurations
+module "verify_kmip_secret_engine" {
+  count  = var.kmip_enabled ? 1 : 0
+  source = "./kmip"
+
+  create_state  = var.create_state
+  hosts         = var.hosts
+  ip_version    = var.ip_version
+  vault_addr    = var.vault_addr
+  vault_edition = var.vault_edition
 }
 
-# KMIP Client Configuration
-resource "enos_remote_exec" "kmip_client_configure" {
-
-  environment = {
-    VAULT_ADDR = var.vault_addr
-    SERVER_CA  = var.create_state.kmip.server_ca
-    CLIENT_CA  = var.create_state.kmip.client_cert
-    KMIP_PORT  = var.create_state.kmip.port
-  }
-
-  // Only perform KMIP operations for Vault Enterprise
-  // The KMIP secrets engine is not available in Vault CE
-  count = var.vault_edition == "ce" ? 0 : 1
-
-  scripts = [abspath("${path.module}/../../scripts/kmip/kmip-client-configure.sh")]
-
-  transport = {
-    ssh = {
-      host = var.create_state.kmip.test_server_ip
-      user = "ubuntu" # Assuming Ubuntu for the test server
-    }
-  }
-}

enos/modules/verify_secrets_engines/modules/read/kmip/kmip.tf

@@ -0,0 +1,71 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "create_state" {
+  description = "The state of the secrets engines from the 'create' module"
+}
+
+variable "hosts" {
+  type = map(object({
+    ipv6       = string
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The Vault cluster instances that were created"
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The local vault API listen address"
+}
+
+variable "ip_version" {
+  type        = string
+  description = "IP Version (4 or 6)"
+  default     = "4"
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "IP Version (4 or 6)"
+  default     = "4"
+}
+
+locals {
+  kmip_output = {
+    mount      = "kmip"
+    ip_address = var.ip_version == "6" ? var.hosts[0].ipv6 : var.hosts[0].public_ip
+  }
+}
+
+# KMIP Client Configuration
+resource "enos_remote_exec" "kmip_client_configure" {
+
+  environment = {
+    VAULT_ADDR = var.vault_addr
+    SERVER_CA  = var.create_state.kmip.server_ca
+    CLIENT_CA  = var.create_state.kmip.client_cert
+    KMIP_PORT  = var.create_state.kmip.port
+  }
+
+  // Only perform KMIP operations for Vault Enterprise
+  // The KMIP secrets engine is not available in Vault CE
+  count = var.vault_edition == "ce" ? 0 : 1
+
+  scripts = [abspath("${path.module}/../../../scripts/kmip/kmip-client-configure.sh")]
+
+  transport = {
+    ssh = {
+      host = var.create_state.kmip.test_server_ip
+      user = "ubuntu" # Assuming Ubuntu for the test server
+    }
+  }
+}

enos/modules/verify_secrets_engines/modules/read/main.tf

@@ -63,6 +63,12 @@ variable "aws_enabled" {
   default     = false
 }
 
+variable "kmip_enabled" {
+  type        = bool
+  description = "Whether or not we'll verify the KMIP secrets engine"
+  default     = false
+}
+
 variable "ldap_enabled" {
   type        = bool
   description = "Whether or not we'll verify the LDAP secrets engine"