fix(misc): contrib/terraform/aws (#7818)

* fix(misc): terraform/aws - handles deployment with a single availability zone - handles deployment with more than two availability zone - handles etcd collocation with control-plane nodes (`aws_etcd_num=0`) - allows to set a bastion instances count (`aws_bastion_num`) - allows to set bastion/etcd/control-plane/workers rootfs volume size - removes variables from terraform.tfvars that were not re-used - adds .terraform.lock.hcl to .gitignore - changes/updates base image from ubuntu-18.03 to debian-10 tested by a few coworkers of mine, and myself: thanks for the outstanding work, on both those terraform samples and kubespray playbooks. I did not test ubuntu deployments, I could still swap from buster to focal. LMK. * fix(gitlab-ci) AFAIU, terraform.tfvars indentation should be fixed for / no diff returned running `terraform fmt -check -diff` https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/jobs/1445622114
3 years ago · bfebcfa2c5
8 changed files with 84 additions and 41 deletions
--- a/contrib/terraform/aws/.gitignore
+++ b/contrib/terraform/aws/.gitignore
@ -1,2 +1,3 @@
 *.tfstate*
+.terraform.lock.hcl
 .terraform
--- a/contrib/terraform/aws/create-infrastructure.tf
+++ b/contrib/terraform/aws/create-infrastructure.tf
@ -20,7 +20,7 @@ module "aws-vpc" {

  aws_cluster_name         = var.aws_cluster_name
  aws_vpc_cidr_block       = var.aws_vpc_cidr_block
-  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
  aws_cidr_subnets_private = var.aws_cidr_subnets_private
  aws_cidr_subnets_public  = var.aws_cidr_subnets_public
  default_tags             = var.default_tags
@ -31,7 +31,7 @@ module "aws-elb" {

  aws_cluster_name      = var.aws_cluster_name
  aws_vpc_id            = module.aws-vpc.aws_vpc_id
-  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
  aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
  aws_elb_api_port      = var.aws_elb_api_port
  k8s_secure_api_port   = var.k8s_secure_api_port
@ -52,9 +52,9 @@ module "aws-iam" {
 resource "aws_instance" "bastion-server" {
  ami                         = data.aws_ami.distro.id
  instance_type               = var.aws_bastion_size
-  count                       = length(var.aws_cidr_subnets_public)
+  count                       = var.aws_bastion_num
  associate_public_ip_address = true
-  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
  subnet_id                   = element(module.aws-vpc.aws_subnet_ids_public, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group
@ -79,11 +79,15 @@ resource "aws_instance" "k8s-master" {

  count = var.aws_kube_master_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

+  root_block_device {
+    volume_size = var.aws_kube_master_disk_size
+  }
+
  iam_instance_profile = module.aws-iam.kube_control_plane-profile
  key_name             = var.AWS_SSH_KEY_NAME

@ -106,11 +110,15 @@ resource "aws_instance" "k8s-etcd" {

  count = var.aws_etcd_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

+  root_block_device {
+    volume_size = var.aws_etcd_disk_size
+  }
+
  key_name = var.AWS_SSH_KEY_NAME

  tags = merge(var.default_tags, tomap({
@ -126,11 +134,15 @@ resource "aws_instance" "k8s-worker" {

  count = var.aws_kube_worker_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

+  root_block_device {
+    volume_size = var.aws_kube_worker_disk_size
+  }
+
  iam_instance_profile = module.aws-iam.kube-worker-profile
  key_name             = var.AWS_SSH_KEY_NAME

@ -152,10 +164,10 @@ data "template_file" "inventory" {
    public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
    connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
    connection_strings_node   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
-    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
    list_master               = join("\n", aws_instance.k8s-master.*.private_dns)
    list_node                 = join("\n", aws_instance.k8s-worker.*.private_dns)
-    list_etcd                 = join("\n", aws_instance.k8s-etcd.*.private_dns)
+    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)), ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip))))
+    list_etcd                 = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
    elb_api_fqdn              = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
  }
 }
--- a/contrib/terraform/aws/output.tf
+++ b/contrib/terraform/aws/output.tf
@ -11,7 +11,7 @@ output "workers" {
 }

 output "etcd" {
-  value = join("\n", aws_instance.k8s-etcd.*.private_ip)
+  value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
 }

 output "aws_elb_api_fqdn" {
--- a/contrib/terraform/aws/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/aws/sample-inventory/cluster.tfvars
@ -9,6 +9,8 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]

 #Bastion Host
+aws_bastion_num = 1
+
 aws_bastion_size = "t2.medium"

 #Kubernetes Cluster
@ -17,22 +19,26 @@ aws_kube_master_num = 3

 aws_kube_master_size = "t2.medium"

+aws_kube_master_disk_size = 50
+
 aws_etcd_num = 3

 aws_etcd_size = "t2.medium"

+aws_etcd_disk_size = 50
+
 aws_kube_worker_num = 4

 aws_kube_worker_size = "t2.medium"

+aws_kube_worker_disk_size = 50
+
 #Settings AWS ELB

 aws_elb_api_port = 6443

 k8s_secure_api_port = 6443

-kube_insecure_apiserver_address = "0.0.0.0"
-
 default_tags = {
  #  Env = "devtest"  #  Product = "kubernetes"
 }
--- a/contrib/terraform/aws/templates/inventory.tpl
+++ b/contrib/terraform/aws/templates/inventory.tpl
@ -10,19 +10,18 @@ ${public_ip_address_bastion}
 [kube_control_plane]
 ${list_master}

-
 [kube_node]
 ${list_node}

-
 [etcd]
 ${list_etcd}

+[calico_rr]

 [k8s_cluster:children]
 kube_node
 kube_control_plane
-
+calico_rr

 [k8s_cluster:vars]
 ${elb_api_fqdn}
--- a/contrib/terraform/aws/terraform.tfvars
+++ b/contrib/terraform/aws/terraform.tfvars
@ -6,26 +6,34 @@ aws_vpc_cidr_block       = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public  = ["10.250.224.0/20", "10.250.240.0/20"]

-#Bastion Host
-aws_bastion_size = "t2.medium"
+# single AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/20"]
+#aws_cidr_subnets_public  = ["10.250.224.0/20"]

+# 3+ AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
+#aws_cidr_subnets_public  = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]

-#Kubernetes Cluster
+#Bastion Host
+aws_bastion_num  = 1
+aws_bastion_size = "t3.small"

-aws_kube_master_num  = 3
-aws_kube_master_size = "t2.medium"
+#Kubernetes Cluster
+aws_kube_master_num       = 3
+aws_kube_master_size      = "t3.medium"
+aws_kube_master_disk_size = 50

-aws_etcd_num  = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_num       = 0
+aws_etcd_size      = "t3.medium"
+aws_etcd_disk_size = 50

-aws_kube_worker_num  = 4
-aws_kube_worker_size = "t2.medium"
+aws_kube_worker_num       = 4
+aws_kube_worker_size      = "t3.medium"
+aws_kube_worker_disk_size = 50

 #Settings AWS ELB
-
-aws_elb_api_port                = 6443
-k8s_secure_api_port             = 6443
-kube_insecure_apiserver_address = "0.0.0.0"
+aws_elb_api_port    = 6443
+k8s_secure_api_port = 6443

 default_tags = {
  #  Env = "devtest"
--- a/contrib/terraform/aws/terraform.tfvars.example
+++ b/contrib/terraform/aws/terraform.tfvars.example
@ -8,25 +8,26 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
 aws_avail_zones = ["eu-central-1a","eu-central-1b"]

 #Bastion Host
-aws_bastion_ami = "ami-5900cc36"
-aws_bastion_size = "t2.small"
-
+aws_bastion_num = 1
+aws_bastion_size = "t3.small"

 #Kubernetes Cluster
-
 aws_kube_master_num = 3
-aws_kube_master_size = "t2.medium"
+aws_kube_master_size = "t3.medium"
+aws_kube_master_disk_size = 50

 aws_etcd_num = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_size = "t3.medium"
+aws_etcd_disk_size = 50

 aws_kube_worker_num = 4
-aws_kube_worker_size = "t2.medium"
-
-aws_cluster_ami = "ami-903df7ff"
+aws_kube_worker_size = "t3.medium"
+aws_kube_worker_disk_size = 50

 #Settings AWS ELB
-
 aws_elb_api_port = 6443
 k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = 0.0.0.0
+
+default_tags = { }
+
+inventory_file = "../../../inventory/hosts"
--- a/contrib/terraform/aws/variables.tf
+++ b/contrib/terraform/aws/variables.tf
@ -25,7 +25,7 @@ data "aws_ami" "distro" {

  filter {
    name   = "name"
-    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+    values = ["debian-10-amd64-*"]
  }

  filter {
@ -33,7 +33,7 @@ data "aws_ami" "distro" {
    values = ["hvm"]
  }

-  owners = ["099720109477"] # Canonical
+  owners = ["136693071363"] # Debian-10
 }

 //AWS VPC Variables
@ -63,10 +63,18 @@ variable "aws_bastion_size" {
 * The number should be divisable by the number of used
 * AWS Availability Zones without an remainder.
 */
+variable "aws_bastion_num" {
+  description = "Number of Bastion Nodes"
+}
+
 variable "aws_kube_master_num" {
  description = "Number of Kubernetes Master Nodes"
 }

+variable "aws_kube_master_disk_size" {
+  description = "Disk size for Kubernetes Master Nodes (in GiB)"
+}
+
 variable "aws_kube_master_size" {
  description = "Instance size of Kube Master Nodes"
 }
@ -75,6 +83,10 @@ variable "aws_etcd_num" {
  description = "Number of etcd Nodes"
 }

+variable "aws_etcd_disk_size" {
+  description = "Disk size for etcd Nodes (in GiB)"
+}
+
 variable "aws_etcd_size" {
  description = "Instance size of etcd Nodes"
 }
@ -83,6 +95,10 @@ variable "aws_kube_worker_num" {
  description = "Number of Kubernetes Worker Nodes"
 }

+variable "aws_kube_worker_disk_size" {
+  description = "Disk size for Kubernetes Worker Nodes (in GiB)"
+}
+
 variable "aws_kube_worker_size" {
  description = "Instance size of Kubernetes Worker Nodes"
 }