Defaults: replace docker with containerd as our default container_manager (#8175)

* Defaults: replace docker with containerd as our default container_manager

* CI: Use docker for download_localhost test

* Defaults: with container_manager=containerd we need etcd_deployment_type=host

* CI: Run weave jobs with docker

* CI: Vagrant don't download_force_cache

* CI: Fix upgrade tests

* should run compatible with old settings, this means docker
* we need to run with a distro that has at least modern containerd,
  this means move from debian9 to debian10 to allow `containerd_version`
  to match between 2.17 and master

.gitlab-ci/packet.yml

@@ -22,11 +22,6 @@
   allow_failure: true
   extends: .packet
 
-packet_ubuntu18-calico-aio:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
 # The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-aio:
   stage: deploy-part1
@@ -54,7 +49,12 @@ packet_ubuntu20-calico-aio-ansible-2_11:
 
 # ### PR JOBS PART2
 
-packet_centos7-flannel-containerd-addons-ha:
+packet_ubuntu18-calico-aio:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_centos7-flannel-addons-ha:
   extends: .packet_pr
   stage: deploy-part2
   when: on_success
@@ -70,7 +70,7 @@ packet_ubuntu18-crio:
   stage: deploy-part2
   when: manual
 
-packet_ubuntu16-canal-kubeadm-ha:
+packet_ubuntu16-canal-ha:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -100,7 +100,12 @@ packet_debian10-cilium-svc-proxy:
   extends: .packet_periodic
   when: on_success
 
-packet_debian10-containerd:
+packet_debian10-aio:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian10-docker:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -110,6 +115,11 @@ packet_debian11-calico:
   extends: .packet_pr
   when: on_success
 
+packet_debian11-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
 packet_centos7-calico-ha-once-localhost:
   stage: deploy-part2
   extends: .packet_pr
@@ -130,7 +140,12 @@ packet_centos8-calico:
   extends: .packet_pr
   when: on_success
 
-packet_fedora34-weave:
+packet_centos8-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_fedora34-docker-weave:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -147,7 +162,7 @@ packet_ubuntu18-ovn4nfv:
 
 # ### MANUAL JOBS
 
-packet_ubuntu16-weave-sep:
+packet_ubuntu16-docker-weave-sep:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
@@ -157,12 +172,12 @@ packet_ubuntu18-cilium-sep:
   extends: .packet_pr
   when: manual
 
-packet_ubuntu18-flannel-containerd-ha:
+packet_ubuntu18-flannel-ha:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_ubuntu18-flannel-containerd-ha-once:
+packet_ubuntu18-flannel-ha-once:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
@@ -220,7 +235,7 @@ packet_centos8-calico-nodelocaldns-secondary:
   extends: .packet_pr
   when: manual
 
-packet_fedora34-kube-ovn-containerd:
+packet_fedora34-kube-ovn:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -228,7 +243,7 @@ packet_fedora34-kube-ovn-containerd:
 # ### PR JOBS PART3
 # Long jobs (45min+)
 
-packet_centos7-weave-upgrade-ha:
+packet_centos7-docker-weave-upgrade-ha:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success
@@ -241,14 +256,14 @@ packet_ubuntu20-calico-ha-wireguard:
   extends: .packet_pr
   when: manual
 
-packet_debian9-calico-upgrade:
+packet_debian10-calico-upgrade:
   stage: deploy-part3
   extends: .packet_pr
   when: on_success
   variables:
     UPGRADE_TEST: graceful
 
-packet_debian9-calico-upgrade-once:
+packet_debian10-calico-upgrade-once:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success

Vagrantfile

@@ -55,7 +55,7 @@ $network_plugin ||= "flannel"
 # Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
 $multi_networking ||= false
 $download_run_once ||= "True"
-$download_force_cache ||= "True"
+$download_force_cache ||= "False"
 # The first three nodes are etcd servers
 $etcd_instances ||= $num_instances
 # The first two nodes are kube masters

docs/ci.md

@@ -2,21 +2,21 @@
 
 To generate this Matrix run `./tests/scripts/md-table/main.py`
 
-## docker
+## containerd
 
 | OS / CNI | calico | canal | cilium | flannel | kube-ovn | kube-router | macvlan | ovn4nfv | weave |
 |---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+centos7 |  :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: |
 centos8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
-debian10 |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 fedora33 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+fedora34 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: |
 ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 
@@ -38,20 +38,20 @@ ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
-## containerd
+## docker
 
 | OS / CNI | calico | canal | cilium | flannel | kube-ovn | kube-router | macvlan | ovn4nfv | weave |
 |---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+centos8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu18 |  :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |

inventory/sample/group_vars/etcd.yml

@@ -19,4 +19,5 @@
 # etcd_peer_client_auth: true
 
 ## Settings for etcd deployment type
-etcd_deployment_type: docker
+# Set this to docker if you are using container_manager: docker
+etcd_deployment_type: host

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -202,7 +202,8 @@ dns_domain: "{{ cluster_name }}"
 
 ## Container runtime
 ## docker for docker, crio for cri-o and containerd for containerd.
-container_manager: docker
+## Default: containerd
+container_manager: containerd
 
 # Additional container runtimes
 kata_containers_enabled: false

roles/kubespray-defaults/defaults/main.yaml

@@ -253,7 +253,7 @@ kubelet_shutdown_grace_period_critical_pods: 20s
 deploy_container_engine: inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type != 'host'
 
 # Container for runtime
-container_manager: docker
+container_manager: containerd
 
 # Enable Kata Containers as additional container runtime
 # When enabled, it requires `container_manager` different than Docker
@@ -344,7 +344,7 @@ docker_containerd_version: 1.4.9
 
 # Settings for containerized control plane (etcd/kubelet/secrets)
 # deployment type for legacy etcd mode
-etcd_deployment_type: docker
+etcd_deployment_type: host
 cert_management: script
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts

tests/files/packet_centos7-calico-ha-once-localhost.yml

@@ -15,3 +15,7 @@ typha_secure: true
 disable_ipv6_dns: true
 
 auto_renew_certificates: true
+
+# Docker settings
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_centos7-weave-upgrade-ha.yml → tests/files/packet_centos7-docker-weave-upgrade-ha.yml

@@ -9,5 +9,9 @@ deploy_netchecker: true
 kubernetes_audit: true
 dns_min_replicas: 1
 
+# Docker specific settings:
+container_manager: docker
+etcd_deployment_type: docker
+
 # Needed to upgrade from 1.16 to 1.17, otherwise upgrade is partial and bug followed
 upgrade_cluster_setup: true

tests/files/packet_centos7-flannel-containerd-addons-ha.yml → tests/files/packet_centos7-flannel-addons-ha.yml

@@ -12,10 +12,8 @@ download_run_once: true
 helm_enabled: true
 krew_enabled: true
 kubernetes_audit: true
-container_manager: containerd
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-etcd_deployment_type: host
 deploy_netchecker: true
 dns_min_replicas: 1
 kube_encrypt_secret_data: true

tests/files/packet_centos8-crio.yml

@@ -8,9 +8,6 @@ deploy_netchecker: true
 dns_min_replicas: 1
 container_manager: crio
 
-# CRI-O requirements
-etcd_deployment_type: host
-
 # required
 calico_iptables_backend: "Auto"
 

tests/files/packet_centos8-docker.yml

@@ -0,0 +1,16 @@
+---
+# Instance settings
+cloud_image: centos-8
+mode: default
+vm_memory: 3072Mi
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+# required
+calico_iptables_backend: "Auto"
+
+# Use docker
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_debian10-aio.yml

@@ -0,0 +1,13 @@
+---
+# Instance settings
+cloud_image: debian-10
+mode: default
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+helm_enabled: true
+krew_enabled: true
+
+auto_renew_certificates: true

tests/files/packet_debian9-calico-upgrade-once.yml → tests/files/packet_debian10-calico-upgrade-once.yml

@@ -1,6 +1,6 @@
 ---
 # Instance settings
-cloud_image: debian-9
+cloud_image: debian-10
 mode: default
 
 # Kubespray settings
@@ -9,5 +9,9 @@ deploy_netchecker: true
 dns_min_replicas: 1
 download_run_once: true
 
+# Docker specific settings:
+container_manager: docker
+etcd_deployment_type: docker
+
 # Make docker happy
 docker_containerd_version: latest

tests/files/packet_debian9-calico-upgrade.yml → tests/files/packet_debian10-calico-upgrade.yml

@@ -1,6 +1,6 @@
 ---
 # Instance settings
-cloud_image: debian-9
+cloud_image: debian-10
 mode: default
 
 # Kubespray settings
@@ -8,5 +8,6 @@ kube_network_plugin: calico
 deploy_netchecker: true
 dns_min_replicas: 1
 
-# Make docker happy
-docker_containerd_version: latest
+# Docker specific settings:
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_debian10-containerd.yml

@@ -1,19 +0,0 @@
----
-# Instance settings
-cloud_image: debian-10
-mode: default
-
-# Kubespray settings
-container_manager: containerd
-etcd_deployment_type: host
-deploy_netchecker: true
-dns_min_replicas: 1
-
-helm_enabled: true
-krew_enabled: true
-
-# https://gitlab.com/miouge/kubespray-ci/-/blob/a4fd5ed6857807f1c353cb60848aedebaf7d2c94/manifests/http-proxy.yml#L42
-http_proxy: http://172.30.30.30:8888
-https_proxy: http://172.30.30.30:8888
-
-auto_renew_certificates: true

tests/files/packet_debian10-docker.yml

@@ -0,0 +1,12 @@
+---
+# Instance settings
+cloud_image: debian-10
+mode: default
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+# Use docker
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_debian11-calico.yml

@@ -4,6 +4,5 @@ cloud_image: debian-11
 mode: default
 
 # Kubespray settings
-etcd_deployment_type: host
 deploy_netchecker: true
 dns_min_replicas: 1

tests/files/packet_debian11-docker.yml

@@ -0,0 +1,12 @@
+---
+# Instance settings
+cloud_image: debian-11
+mode: default
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+# Use docker
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_debian9-macvlan.yml

@@ -12,6 +12,3 @@ kube_proxy_masquerade_all: true
 macvlan_interface: "eth0"
 
 auto_renew_certificates: true
-
-# Make docker happy
-docker_containerd_version: latest

tests/files/packet_fedora34-kube-ovn-containerd.yml → tests/files/packet_fedora34-docker-weave.yml

@@ -4,8 +4,10 @@ cloud_image: fedora-34
 mode: default
 
 # Kubespray settings
-container_manager: containerd
-etcd_deployment_type: host
 deploy_netchecker: true
 dns_min_replicas: 1
-kube_network_plugin: kube-ovn
+kube_network_plugin: weave
+
+# Docker specific settings:
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_fedora34-weave.yml → tests/files/packet_fedora34-kube-ovn.yml

@@ -6,4 +6,4 @@ mode: default
 # Kubespray settings
 deploy_netchecker: true
 dns_min_replicas: 1
-kube_network_plugin: weave
+kube_network_plugin: kube-ovn

tests/files/packet_ubuntu16-canal-kubeadm-ha.yml → tests/files/packet_ubuntu16-canal-ha.yml

@@ -8,6 +8,3 @@ calico_datastore: etcd
 kube_network_plugin: canal
 deploy_netchecker: true
 dns_min_replicas: 1
-
-# Make docker jobs happy
-docker_containerd_version: latest

tests/files/packet_ubuntu16-canal-sep.yml

@@ -8,6 +8,3 @@ calico_datastore: etcd
 kube_network_plugin: canal
 deploy_netchecker: true
 dns_min_replicas: 1
-
-# Make docker jobs happy
-docker_containerd_version: latest

tests/files/packet_ubuntu16-weave-sep.yml → tests/files/packet_ubuntu16-docker-weave-sep.yml

@@ -8,7 +8,8 @@ kube_network_plugin: weave
 deploy_netchecker: true
 dns_min_replicas: 1
 
-auto_renew_certificates: true
+# Docker specific settings:
+container_manager: docker
+etcd_deployment_type: docker
 
-# Make docker jobs happy
-docker_containerd_version: latest
+auto_renew_certificates: true

tests/files/packet_ubuntu16-flannel-ha.yml

@@ -10,6 +10,3 @@ kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c
 skip_non_kubeadm_warning: true
 deploy_netchecker: true
 dns_min_replicas: 1
-
-# Make docker jobs happy
-docker_containerd_version: latest

tests/files/packet_ubuntu16-kube-router-sep.yml

@@ -8,6 +8,3 @@ bootstrap_os: ubuntu
 kube_network_plugin: kube-router
 deploy_netchecker: true
 dns_min_replicas: 1
-
-# Make docker jobs happy
-docker_containerd_version: latest

tests/files/packet_ubuntu16-kube-router-svc-proxy.yml

@@ -10,6 +10,3 @@ deploy_netchecker: true
 dns_min_replicas: 1
 
 kube_router_run_service_proxy: true
-
-# Make docker jobs happy
-docker_containerd_version: latest

tests/files/packet_ubuntu18-crio.yml

@@ -10,6 +10,3 @@ container_manager: crio
 
 download_localhost: false
 download_run_once: true
-
-# CRI-O requirements
-etcd_deployment_type: host

tests/files/packet_ubuntu18-docker.yml

@@ -0,0 +1,13 @@
+---
+# Instance settings
+cloud_image: ubuntu-1804
+mode: aio
+vm_memory: 1600Mi
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+# Use docker
+container_manager: docker
+etcd_deployment_type: docker

tests/files/packet_ubuntu18-flannel-containerd-ha-once.yml → tests/files/packet_ubuntu18-flannel-ha-once.yml

@@ -11,10 +11,8 @@ kube_network_plugin: flannel
 helm_enabled: true
 krew_enabled: true
 kubernetes_audit: true
-container_manager: containerd
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-etcd_deployment_type: host
 deploy_netchecker: true
 dns_min_replicas: 1
 kube_encrypt_secret_data: true

tests/files/packet_ubuntu18-flannel-containerd-ha.yml → tests/files/packet_ubuntu18-flannel-ha.yml

@@ -11,10 +11,8 @@ kube_network_plugin: flannel
 helm_enabled: true
 krew_enabled: true
 kubernetes_audit: true
-container_manager: containerd
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-etcd_deployment_type: host
 deploy_netchecker: true
 dns_min_replicas: 1
 kube_encrypt_secret_data: true

tests/files/packet_ubuntu20-docker.yml

@@ -0,0 +1,19 @@
+---
+# Instance settings
+cloud_image: ubuntu-2004
+mode: aio
+vm_memory: 1600Mi
+
+# Kubespray settings
+deploy_netchecker: true
+dns_min_replicas: 1
+
+# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
+kube_proxy_mode: iptables
+enable_nodelocaldns: False
+
+auto_renew_certificates: true
+
+# Use docker
+container_manager: docker
+etcd_deployment_type: docker

tests/scripts/md-table/main.py

@@ -41,7 +41,6 @@ class Data:
         operating_systems = list(self.db.get_unique_ids("operating_system"))
 
         container_engines.sort()
-        container_engines.reverse() # reverse sort container_engines to get Docker first in the list
         network_plugins.sort()
         operating_systems.sort()
 
@@ -88,7 +87,7 @@ files = p.glob('*.yml')
 for f in files:
     y = yaml.load(f.open(), Loader=yaml.FullLoader)
 
-    container_manager = y.get('container_manager', 'docker')
+    container_manager = y.get('container_manager', 'containerd')
     network_plugin = y.get('kube_network_plugin', 'calico')
     x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
     operating_system = x.group(1)