diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index c0a1ca21f..bd5adb4d4 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -100,6 +100,12 @@ tf-0.13.x-validate-exoscale: variables: TF_VERSION: 0.13.5 PROVIDER: exoscale + +tf-0.13.x-validate-vsphere: + extends: .terraform_validate + variables: + TF_VERSION: 0.13.5 + PROVIDER: vsphere CLUSTER: $CI_COMMIT_REF_NAME tf-0.14.x-validate-openstack: @@ -128,6 +134,12 @@ tf-0.14.x-validate-exoscale: variables: TF_VERSION: 0.14.3 PROVIDER: exoscale + +tf-0.14.x-validate-vsphere: + extends: .terraform_validate + variables: + TF_VERSION: 0.14.3 + PROVIDER: vsphere CLUSTER: $CI_COMMIT_REF_NAME # tf-packet-ubuntu16-default: diff --git a/contrib/terraform/vsphere/README.md b/contrib/terraform/vsphere/README.md new file mode 100644 index 000000000..dfb53485c --- /dev/null +++ b/contrib/terraform/vsphere/README.md @@ -0,0 +1,117 @@ +# Kubernetes on Exoscale with Terraform + +Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/se/products/vsphere.html) using Terraform and Kubespray. + +## Overview + +The setup looks like following. + +```text + Kubernetes cluster ++-----------------------+ +| +--------------+ | +| | +--------------+ | +| | | | | +| | | Master/etcd | | +| | | node(s) | | +| +-+ | | +| +--------------+ | +| ^ | +| | | +| v | +| +--------------+ | +| | +--------------+ | +| | | | | +| | | Worker | | +| | | node(s) | | +| +-+ | | +| +--------------+ | ++-----------------------+ +``` + +## Warning + +This setup assumes that the DHCP is disabled in the vSphere cluster and IP addresses have to be provided in the configuration file. + +## Requirements + +* Terraform 0.13.0 or newer + +*0.12 also works if you modify the provider block to include version and remove all `versions.tf` files* + +## Quickstart + +NOTE: *Assumes you are at the root of the kubespray repo* + +Copy the sample inventory for your cluster and copy the default terraform variables. + +```bash +CLUSTER=my-vsphere-cluster +cp -r inventory/sample inventory/$CLUSTER +cp contrib/terraform/vsphere/default.tfvars inventory/$CLUSTER/ +cd inventory/$CLUSTER +``` + +Edit `default.tfvars` to match your setup. You MUST set values specific for you network and vSphere cluster. + +```bash +# Ensure $EDITOR points to your favorite editor, e.g., vim, emacs, VS Code, etc. +$EDITOR default.tfvars +``` + +For authentication in your vSphere cluster you can use the environment variables. + +```bash +export TF_VAR_vsphere_user=username +export TF_VAR_vsphere_password=password +``` + +Run Terraform to create the infrastructure. + +```bash +terraform init ../../contrib/terraform/vsphere +terraform apply \ + -var-file default.tfvars \ + -state=tfstate-$CLUSTER.tfstate \ + ../../contrib/terraform/vsphere +``` + +You should now have a inventory file named `inventory.ini` that you can use with kubespray. +You can now copy your inventory file and use it with kubespray to set up a cluster. +You can type `terraform output` to find out the IP addresses of the nodes. + +It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by: + +```bash +ansible -i inventory.ini -m ping all +``` + +Example to use this with the default sample inventory: + +```bash +ansible-playbook -i inventory.ini ../../cluster.yml -b -v +``` + +## Variables + +### Required + +* `machines`: Machines to provision. Key of this object will be used as the name of the machine + * `node_type`: The role of this node *(master|worker)* + * `ip`: The IP address with the netmask (CIDR notation) +* `gateway`: The IP address of the network gateway +* `ssh_public_keys`: List of public SSH keys to install on all machines +* `vsphere_datacenter`: The identifier of vSphere data center +* `vsphere_compute_cluster`: The identifier of vSphere compute cluster +* `vsphere_datastore`: The identifier of vSphere data store +* `vsphere_server`: The address of vSphere server +* `vsphere_hostname`: The IP address of vSphere hostname +* `template_name`: The name of a base image (the image has to be uploaded to vSphere beforehand) + +### Optional + +* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)* +* `dns_primary`: The IP address of primary DNS server *(Defaults to `8.8.4.4`)* +* `dns_secondary`:The IP address of secondary DNS server *(Defaults to `8.8.8.8`)* + +An example variables file can be found `default.tfvars` diff --git a/contrib/terraform/vsphere/default.tfvars b/contrib/terraform/vsphere/default.tfvars new file mode 100644 index 000000000..f0ee03ed8 --- /dev/null +++ b/contrib/terraform/vsphere/default.tfvars @@ -0,0 +1,34 @@ +prefix = "default" + +inventory_file = "inventory.ini" + +machines = { + "master-0" : { + "node_type" : "master", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + }, + "worker-0" : { + "node_type" : "worker", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + }, + "worker-1" : { + "node_type" : "worker", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + } +} + +gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2 + +ssh_public_keys = [ + # Put your public SSH key here + "ssh-rsa I-did-not-read-the-docs", + "ssh-rsa I-did-not-read-the-docs 2", +] + +vsphere_datacenter = "i-did-not-read-the-docs" +vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster +vsphere_datastore = "i-did-not-read-the-docs" # e.g. ssd-000000 +vsphere_server = "i-did-not-read-the-docs" # e.g. vsphere.server.com +vsphere_hostname = "i-did-not-read-the-docs" # e.g. 192.168.0.2 + +template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg diff --git a/contrib/terraform/vsphere/main.tf b/contrib/terraform/vsphere/main.tf new file mode 100644 index 000000000..41eff4170 --- /dev/null +++ b/contrib/terraform/vsphere/main.tf @@ -0,0 +1,116 @@ +provider "vsphere" { + # Username and password set through env vars VSPHERE_USER and VSPHERE_PASSWORD + user = var.vsphere_user + password = var.vsphere_password + + vsphere_server = var.vsphere_server + + # If you have a self-signed cert + allow_unverified_ssl = true +} + +data "vsphere_datacenter" "dc" { + name = var.vsphere_datacenter +} + +data "vsphere_datastore" "datastore" { + name = var.vsphere_datastore + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_network" "network" { + name = "VM Network" + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_host" "host" { + name = var.vsphere_hostname + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_virtual_machine" "template" { + name = var.template_name + datacenter_id = data.vsphere_datacenter.dc.id +} + +data "vsphere_compute_cluster" "compute_cluster" { + name = var.vsphere_compute_cluster + datacenter_id = data.vsphere_datacenter.dc.id +} + +resource "vsphere_resource_pool" "pool" { + name = "${var.prefix}-cluster-pool" + parent_resource_pool_id = data.vsphere_host.host.resource_pool_id +} + +module "kubernetes" { + source = "./modules/kubernetes-cluster" + + prefix = var.prefix + + machines = var.machines + + ## Master ## + master_cores = var.master_cores + master_memory = var.master_memory + master_disk_size = var.master_disk_size + + ## Worker ## + worker_cores = var.worker_cores + worker_memory = var.worker_memory + worker_disk_size = var.worker_disk_size + + ## Global ## + + gateway = var.gateway + dns_primary = var.dns_primary + dns_secondary = var.dns_secondary + + pool_id = vsphere_resource_pool.pool.id + datastore_id = data.vsphere_datastore.datastore.id + + folder = "" + guest_id = data.vsphere_virtual_machine.template.guest_id + scsi_type = data.vsphere_virtual_machine.template.scsi_type + network_id = data.vsphere_network.network.id + adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0] + firmware = var.firmware + hardware_version = var.hardware_version + disk_thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned + + template_id = data.vsphere_virtual_machine.template.id + + ssh_public_keys = var.ssh_public_keys +} + +# +# Generate ansible inventory +# + +data "template_file" "inventory" { + template = file("${path.module}/templates/inventory.tpl") + + vars = { + connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d", + keys(module.kubernetes.master_ip), + values(module.kubernetes.master_ip), + range(1, length(module.kubernetes.master_ip) + 1))) + connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s", + keys(module.kubernetes.worker_ip), + values(module.kubernetes.worker_ip))) + list_master = join("\n", formatlist("%s", + keys(module.kubernetes.master_ip))) + list_worker = join("\n", formatlist("%s", + keys(module.kubernetes.worker_ip))) + } +} + +resource "null_resource" "inventories" { + provisioner "local-exec" { + command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" + } + + triggers = { + template = data.template_file.inventory.rendered + } +} diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf new file mode 100644 index 000000000..ecb3165b8 --- /dev/null +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf @@ -0,0 +1,109 @@ +resource "vsphere_virtual_machine" "worker" { + for_each = { + for name, machine in var.machines : + name => machine + if machine.node_type == "worker" + } + + name = each.key + resource_pool_id = var.pool_id + datastore_id = var.datastore_id + + num_cpus = var.worker_cores + memory = var.worker_memory + memory_reservation = var.worker_memory + guest_id = var.guest_id + enable_disk_uuid = "true" + scsi_type = var.scsi_type + folder = var.folder + firmware = var.firmware + hardware_version = var.hardware_version + + wait_for_guest_net_routable = false + + network_interface { + network_id = var.network_id + adapter_type = var.adapter_type + } + + disk { + label = "disk0" + size = var.worker_disk_size + thin_provisioned = var.disk_thin_provisioned + } + + lifecycle { + ignore_changes = [disk] + } + + clone { + template_uuid = var.template_id + } + + cdrom { + client_device = true + } + + vapp { + properties = { + "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip, + gw = var.gateway, + dns = var.dns_primary, + ssh_public_keys = var.ssh_public_keys})) + } + } +} + +resource "vsphere_virtual_machine" "master" { + for_each = { + for name, machine in var.machines : + name => machine + if machine.node_type == "master" + } + + name = each.key + resource_pool_id = var.pool_id + datastore_id = var.datastore_id + + num_cpus = var.master_cores + memory = var.master_memory + memory_reservation = var.master_memory + guest_id = var.guest_id + enable_disk_uuid = "true" + scsi_type = var.scsi_type + folder = var.folder + firmware = var.firmware + hardware_version = var.hardware_version + + network_interface { + network_id = var.network_id + adapter_type = var.adapter_type + } + + disk { + label = "disk0" + size = var.master_disk_size + thin_provisioned = var.disk_thin_provisioned + } + + lifecycle { + ignore_changes = [disk] + } + + clone { + template_uuid = var.template_id + } + + cdrom { + client_device = true + } + + vapp { + properties = { + "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip, + gw = var.gateway, + dns = var.dns_primary, + ssh_public_keys = var.ssh_public_keys})) + } + } +} diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf new file mode 100644 index 000000000..e4f576cc0 --- /dev/null +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf @@ -0,0 +1,13 @@ +output "master_ip" { + value = { + for instance in vsphere_virtual_machine.master : + instance.name => instance.default_ip_address + } +} + +output "worker_ip" { + value = { + for instance in vsphere_virtual_machine.worker : + instance.name => instance.default_ip_address + } +} diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl new file mode 100644 index 000000000..a7bd0e785 --- /dev/null +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl @@ -0,0 +1,35 @@ +#cloud-config + +ssh_authorized_keys: +%{ for ssh_public_key in ssh_public_keys ~} + - ${ssh_public_key} +%{ endfor ~} + +write_files: + - path: /etc/netplan/20-internal-network.yaml + content: | + network: + version: 2 + ethernets: + "lo:0": + match: + name: lo + dhcp4: false + addresses: + - 172.17.0.100/32 + - path: /etc/netplan/10-user-network.yaml + content: | + network: + version: 2 + ethernets: + ens192: + dhcp4: false #true to use dhcp + addresses: + - ${ip} + gateway4: ${gw} # Set gw here + nameservers: + addresses: + - ${dns} # Set DNS ip address here + +runcmd: + - netplan apply diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf new file mode 100644 index 000000000..5630fe9e1 --- /dev/null +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf @@ -0,0 +1,38 @@ +## Global ## +variable "prefix" {} + +variable "machines" { + description = "Cluster machines" + type = map(object({ + node_type = string + ip = string + })) +} + +variable "gateway" {} +variable "dns_primary" {} +variable "dns_secondary" {} +variable "pool_id" {} +variable "datastore_id" {} +variable "guest_id" {} +variable "scsi_type" {} +variable "network_id" {} +variable "adapter_type" {} +variable "disk_thin_provisioned" {} +variable "template_id" {} +variable "firmware" {} +variable "folder" {} +variable "ssh_public_keys" { + type = list(string) +} +variable "hardware_version" {} + +## Master ## +variable "master_cores" {} +variable "master_memory" {} +variable "master_disk_size" {} + +## Worker ## +variable "worker_cores" {} +variable "worker_memory" {} +variable "worker_disk_size" {} diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/versions.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/versions.tf new file mode 100644 index 000000000..8c622fdfc --- /dev/null +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + vsphere = { + source = "hashicorp/vsphere" + version = ">= 1.24.3" + } + } + required_version = ">= 0.13" +} diff --git a/contrib/terraform/vsphere/output.tf b/contrib/terraform/vsphere/output.tf new file mode 100644 index 000000000..bdd67dc83 --- /dev/null +++ b/contrib/terraform/vsphere/output.tf @@ -0,0 +1,31 @@ +output "master_ip_addresses" { + value = module.kubernetes.master_ip +} + +output "worker_ip_addresses" { + value = module.kubernetes.worker_ip +} + +output "vsphere_datacenter" { + value = var.vsphere_datacenter +} + +output "vsphere_server" { + value = var.vsphere_server +} + +output "vsphere_datastore" { + value = var.vsphere_datastore +} + +output "vsphere_network" { + value = var.network +} + +output "vsphere_folder" { + value = terraform.workspace +} + +output "vsphere_pool" { + value = "${terraform.workspace}-cluster-pool" +} diff --git a/contrib/terraform/vsphere/sample-inventory/cluster.tfvars b/contrib/terraform/vsphere/sample-inventory/cluster.tfvars new file mode 100644 index 000000000..f0ee03ed8 --- /dev/null +++ b/contrib/terraform/vsphere/sample-inventory/cluster.tfvars @@ -0,0 +1,34 @@ +prefix = "default" + +inventory_file = "inventory.ini" + +machines = { + "master-0" : { + "node_type" : "master", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + }, + "worker-0" : { + "node_type" : "worker", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + }, + "worker-1" : { + "node_type" : "worker", + "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + } +} + +gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2 + +ssh_public_keys = [ + # Put your public SSH key here + "ssh-rsa I-did-not-read-the-docs", + "ssh-rsa I-did-not-read-the-docs 2", +] + +vsphere_datacenter = "i-did-not-read-the-docs" +vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster +vsphere_datastore = "i-did-not-read-the-docs" # e.g. ssd-000000 +vsphere_server = "i-did-not-read-the-docs" # e.g. vsphere.server.com +vsphere_hostname = "i-did-not-read-the-docs" # e.g. 192.168.0.2 + +template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg diff --git a/contrib/terraform/vsphere/sample-inventory/group_vars b/contrib/terraform/vsphere/sample-inventory/group_vars new file mode 120000 index 000000000..373595823 --- /dev/null +++ b/contrib/terraform/vsphere/sample-inventory/group_vars @@ -0,0 +1 @@ +../../../../inventory/sample/group_vars \ No newline at end of file diff --git a/contrib/terraform/vsphere/templates/inventory.tpl b/contrib/terraform/vsphere/templates/inventory.tpl new file mode 100644 index 000000000..26d65e67b --- /dev/null +++ b/contrib/terraform/vsphere/templates/inventory.tpl @@ -0,0 +1,17 @@ + +[all] +${connection_strings_master} +${connection_strings_worker} + +[kube-master] +${list_master} + +[etcd] +${list_master} + +[kube-node] +${list_worker} + +[k8s-cluster:children] +kube-master +kube-node diff --git a/contrib/terraform/vsphere/variables.tf b/contrib/terraform/vsphere/variables.tf new file mode 100644 index 000000000..346c10c13 --- /dev/null +++ b/contrib/terraform/vsphere/variables.tf @@ -0,0 +1,89 @@ +## Global ## + +variable "prefix" { + default = "" +} + +variable "machines" { + description = "Cluster machines" + type = map(object({ + node_type = string + ip = string + })) +} + +variable "inventory_file" { + default = "inventory.ini" +} + +variable "network" { + default = "VM Network" +} + +variable "gateway" {} + +variable "dns_primary" { + default = "8.8.4.4" +} + +variable "dns_secondary" { + default = "8.8.8.8" +} + +variable "vsphere_datacenter" {} + +variable "vsphere_compute_cluster" {} + +variable "vsphere_datastore" {} + +variable "vsphere_user" {} + +variable "vsphere_password" {} + +variable "vsphere_server" {} + +variable "vsphere_hostname" {} + +variable "firmware" { + default = "bios" +} + +variable "hardware_version" { + default = "15" +} + +variable "template_name" { + default = "ubuntu-focal-20.04-cloudimg" +} + +variable "ssh_public_keys" { + description = "List of public SSH keys which are injected into the VMs." + type = list(string) +} + +## Master ## + +variable "master_cores" { + default = 4 +} + +variable "master_memory" { + default = 4096 +} + +variable "master_disk_size" { + default = "20" +} + +## Worker ## + +variable "worker_cores" { + default = 16 +} + +variable "worker_memory" { + default = 8192 +} +variable "worker_disk_size" { + default = "100" +} diff --git a/contrib/terraform/vsphere/versions.tf b/contrib/terraform/vsphere/versions.tf new file mode 100644 index 000000000..2a2a7c6e0 --- /dev/null +++ b/contrib/terraform/vsphere/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + vsphere = { + source = "hashicorp/vsphere" + version = ">= 1.24.3" + } + null = { + source = "hashicorp/null" + } + template = { + source = "hashicorp/template" + } + } + required_version = ">= 0.13" +}