Fix AWS EKS tests (#2277)

This commit is contained in:
Aleksandr Rybolovlev 2023-09-05 21:53:17 +02:00 committed by GitHub
parent 12d9cae69d
commit c5bbf6463f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 181 additions and 82 deletions

View file

@ -11,22 +11,30 @@ on:
default: 2
clusterVersion:
description: The EKS cluster version
default: 1.25
default: 1.27
nodesPerAz:
description: The number of cluster nodes in each AZ
default: 2
default: 1
instanceType:
description: The type of EC2 instance to use for cluster nodes
default: m5.large
default: m7g.large
parallelRuns:
description: The maximum number of tests to run simultaneously
default: 8
runTests:
description: The regex passed to the -run option of `go test`
default: ".*"
terraformVersion:
description: Terraform version
default: 1.4.0
default: 1.5.6
schedule:
- cron: '0 20 * * *'
env:
AWS_REGION: ${{ github.event.inputs.region }}
KUBE_CONFIG_PATH: ${{ github.workspace }}/kubernetes/test-infra/eks/kubeconfig
PARALLEL_RUNS: ${{ github.event.inputs.parallelRuns || vars.PARALLEL_RUNS }}
TERRAFORM_VERSION: ${{ github.event.inputs.terraformVersion || vars.TERRAFORM_VERSION }}
TF_VAR_az_span: ${{ github.event.inputs.azSpan }}
TF_VAR_cluster_version: ${{ github.event.inputs.clusterVersion }}
TF_VAR_nodes_per_az: ${{ github.event.inputs.nodesPerAz }}
@ -34,7 +42,7 @@ env:
jobs:
acceptance_tests:
runs-on: ubuntu-latest
runs-on: [custom, linux, medium]
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Set up Go
@ -53,17 +61,25 @@ jobs:
- name: Install Terraform
uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
with:
terraform_version: ${{ github.event.inputs.terraformVersion }}
terraform_version: ${{ env.TERRAFORM_VERSION }}
terraform_wrapper: false
- name: Provision EKS Cluster
working-directory: ${{ github.workspace }}/kubernetes/test-infra/eks
run: |
terraform init
terraform apply --auto-approve
- name: Install AWS EBS CSI Driver
working-directory: ${{ github.workspace }}/kubernetes/test-infra/aws-ebs-csi-driver
run: |
terraform init
terraform apply --auto-approve
- name: Run Acceptance Test Suite
env:
TF_ACC_TERRAFORM_VERSION: ${{ github.event.inputs.terraformVersion }}
TESTARGS: -run '${{ github.event.inputs.runTests }}'
KUBE_CONFIG_PATH: ${{ github.workspace }}/kubernetes/test-infra/eks/kubeconfig
# Do not set TF_ACC_TERRAFORM_PATH or TF_ACC_TERRAFORM_VERSION.
# In this case, the framework will search for the Terraform CLI binary based on the operating system PATH.
# Eventually, it will use the one we set up.
# More information: https://developer.hashicorp.com/terraform/plugin/sdkv2/testing/acceptance-tests#terraform-cli-installation-behaviors
run: |
make testacc
- name: Destroy EKS cluster

1
.gitignore vendored
View file

@ -5,6 +5,7 @@ example.tf
kubeconfig
terraform.tfplan
terraform.tfstate
.terraform.tfstate.lock.info
bin/
modules-dev/
/pkg/

View file

@ -25,6 +25,7 @@ func TestAccKubernetesCertificateSigningRequestV1_basic(t *testing.T) {
PreCheck: func() {
testAccPreCheck(t)
skipIfClusterVersionLessThan(t, "1.22.0")
skipIfNotRunningInKind(t)
},
IDRefreshName: resourceName,
IDRefreshIgnore: []string{"metadata.0.resource_version"},
@ -52,6 +53,7 @@ func TestAccKubernetesCertificateSigningRequestV1_generateName(t *testing.T) {
PreCheck: func() {
testAccPreCheck(t)
skipIfClusterVersionLessThan(t, "1.22.0")
skipIfNotRunningInKind(t)
},
IDRefreshName: resourceName,
IDRefreshIgnore: []string{"metadata.0.resource_version"},
@ -69,6 +71,39 @@ func TestAccKubernetesCertificateSigningRequestV1_generateName(t *testing.T) {
})
}
func TestAccKubernetesCertificateSigningRequestV1_awsBasic(t *testing.T) {
name := fmt.Sprintf("tf-acc-test-%s", acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum))
usages := []string{"digital signature"}
// More information about the signer name:
// - https://docs.aws.amazon.com/eks/latest/userguide/cert-signing.html
// - https://github.com/aws/containers-roadmap/issues/1604
signerName := "beta.eks.amazonaws.com/app-serving"
resourceName := "kubernetes_certificate_signing_request_v1.test"
resource.ParallelTest(t, resource.TestCase{
PreCheck: func() {
testAccPreCheck(t)
skipIfClusterVersionLessThan(t, "1.22.0")
skipIfNotRunningInEks(t)
},
IDRefreshName: resourceName,
IDRefreshIgnore: []string{"metadata.0.resource_version"},
ProviderFactories: testAccProviderFactories,
CheckDestroy: testAccCheckKubernetesCertificateSigningRequestV1Destroy,
Steps: []resource.TestStep{
{
Config: testAccKubernetesCertificateSigningRequestV1Config_basic(name, signerName, usages, true),
Check: resource.ComposeAggregateTestCheckFunc(
testAccCheckKubernetesCertificateSigningRequestV1Valid,
resource.TestCheckResourceAttrSet(resourceName, "certificate"),
resource.TestCheckResourceAttr(resourceName, "spec.0.signer_name", signerName),
resource.TestCheckResourceAttr(resourceName, "spec.0.usages.0", usages[0]),
),
},
},
})
}
// testAccCheckKubernetesCertificateSigningRequestV1Valid checks to see that the locally-stored certificate
// contains a valid PEM preamble. It also checks that the CSR resource has been deleted from Kubernetes, since
// the CSR is only supposed to exist momentarily as the certificate is generated. (CSR resources are ephemeral

View file

@ -224,9 +224,10 @@ func TestAccKubernetesClusterRoleBindingV1_group_subject(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
},
})

View file

@ -181,9 +181,10 @@ func TestAccKubernetesConfigMap_generatedName(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
},
})

View file

@ -42,9 +42,10 @@ func TestAccKubernetesCSIDriverV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
},
})

View file

@ -41,9 +41,10 @@ func TestAccKubernetesCSIDriverV1Beta1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
},
})

View file

@ -50,9 +50,10 @@ func TestAccKubernetesHorizontalPodAutoscaler_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesHorizontalPodAutoscalerConfig_metaModified(name),

View file

@ -50,9 +50,10 @@ func TestAccKubernetesHorizontalPodAutoscalerV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesHorizontalPodAutoscalerV1Config_metaModified(name),

View file

@ -131,9 +131,10 @@ func TestAccKubernetesHorizontalPodAutoscalerV2_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesHorizontalPodAutoscalerV2Config_modified(name),

View file

@ -133,9 +133,10 @@ func TestAccKubernetesHorizontalPodAutoscalerV2Beta2_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesHorizontalPodAutoscalerV2Beta2Config_modified(name),

View file

@ -40,9 +40,10 @@ func TestAccKubernetesIngressClassV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
},
})
@ -75,9 +76,10 @@ func TestAccKubernetesIngressClassV1_parameters(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesIngressClassV1ConfigParameters(rName, rNameUpdated),
@ -124,9 +126,10 @@ func TestAccKubernetesIngressClassV1_parameters_apiGroup(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesIngressClassV1ConfigParametersApiGroup(rName, rNameUpdated),

View file

@ -65,9 +65,10 @@ func TestAccKubernetesMutatingWebhookConfiguration_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesMutatingWebhookConfigurationConfig_modified(name),

View file

@ -65,9 +65,10 @@ func TestAccKubernetesMutatingWebhookConfigurationV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesMutatingWebhookConfigurationV1Config_modified(name),

View file

@ -49,9 +49,10 @@ func TestAccKubernetesNetworkPolicyV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesNetworkPolicyV1Config_metaModified(name),

View file

@ -410,9 +410,10 @@ func TestAccKubernetesPersistentVolumeV1_googleCloud_volumeSource(t *testing.T)
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesPersistentVolumeV1Config_hostPath_volumeSource(name, "/custom/testing/path", ""),
@ -775,9 +776,10 @@ func TestAccKubernetesPersistentVolumeV1_csi_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesPersistentVolumeV1Config_csi_modified(name),

View file

@ -54,9 +54,10 @@ func TestAccKubernetesPodDisruptionBudget_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesPodDisruptionBudgetConfig_minAvailable(name),

View file

@ -52,9 +52,10 @@ func TestAccKubernetesPodDisruptionBudgetV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesPodDisruptionBudgetV1Config_minAvailable(name),

View file

@ -74,9 +74,10 @@ func TestAccKubernetesPodSecurityPolicyV1Beta1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesPodSecurityPolicyV1Beta1Config_metaModified(name),

View file

@ -47,9 +47,10 @@ func TestAccKubernetesRoleBindingV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesRoleBindingConfigV1_modified(name),

View file

@ -504,7 +504,6 @@ func testAccKubernetesStatefulSetV1ConfigBasic(name, imageName string) string {
spec {
access_modes = ["ReadWriteOnce"]
resources {
requests = {
storage = "1Gi"

View file

@ -64,9 +64,10 @@ func TestAccKubernetesValidatingWebhookConfigurationV1_basic(t *testing.T) {
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesValidatingWebhookConfigurationV1Config_modified(name),

View file

@ -68,9 +68,10 @@ func TestAccKubernetesValidatingWebhookConfigurationV1Beta1_basic(t *testing.T)
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"metadata.0.resource_version"},
},
{
Config: testAccKubernetesValidatingWebhookConfigurationV1Beta1Config_modified(name),

View file

@ -0,0 +1,3 @@
# Amazon Elastic Block Store (EBS) CSI driver
The Amazon Elastic Block Store Container Storage Interface (CSI) Driver provides a CSI interface used by Container Orchestrators to manage the lifecycle of Amazon EBS volumes. More information [here](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master).

View file

@ -0,0 +1,12 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
resource "helm_release" "aws_ebs_csi_driver" {
name = "aws-ebs-csi-driver"
repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
chart = "aws-ebs-csi-driver"
version = var.chart_version
namespace = "kube-system"
}

View file

@ -0,0 +1,6 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
variable "chart_version" {
default = "2.22.0"
}

View file

@ -40,12 +40,18 @@ module "eks" {
eks_managed_node_groups = {
default_node_group = {
desired_size = local.node_count
min_size = 1
max_size = local.node_count
instance_types = [var.instance_type]
ami_type = "AL2_ARM_64"
desired_size = local.node_count
min_size = 1
max_size = local.node_count
instance_types = [var.instance_type]
use_custom_launch_template = false
iam_role_additional_policies = {
AmazonEBSCSIDriverPolicy = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
}
}
}
tags = local.tags

View file

@ -11,19 +11,18 @@ variable "nodes_per_az" {
}
variable "instance_type" {
default = "m5.large"
default = "m7g.xlarge"
}
variable "az_span" {
type = number
default = 3
default = 2
validation {
condition = var.az_span > 1
error_message = "Cluster must span at least 2 AZs"
}
}
variable "cluster_name" {
default = ""
}