Move flannel to etcd datastore

roles/network_plugin/canal/tasks/main.yml

@@ -28,10 +28,28 @@
     - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
     - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
+- name: Slurp etcd cacert file
+  slurp:
+    src: "{{ canal_cert_dir }}/ca_cert.crt"
+  register: etcd_ca_cert_file
+  failed_when: false
+
+- name: Slurp etcd cert file
+  slurp:
+    src: "{{ canal_cert_dir }}/cert.crt"
+  register: etcd_cert_file
+  failed_when: false
+
+- name: Slurp etcd key file
+  slurp:
+    src: "{{ canal_cert_dir }}/key.pem"
+  register: etcd_key_file
+  failed_when: false
+
 # Flannel need etcd v2 API
 - name: Canal | Set Flannel etcd configuration
   command: |-
-    {{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \
+    {{ bin_dir }}/etcdctl set /coreos.com/network/config \
     '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
   register: output
   retries: 4
@@ -53,14 +71,17 @@
     dest: "{{ kube_config_dir }}/{{ item.file }}"
     mode: 0644
   with_items:
+    - {name: canal-calico-etcd-secret, file: canal-secret-calico-etcd.yml, type: secret}
     - {name: canal-config, file: canal-config.yaml, type: cm}
     - {name: canal-node, file: canal-node.yaml, type: ds}
     - {name: canal-kube-controllers, file: canal-calico-kube-controllers.yml, type: deployment}
+    - {name: canal-cr, file: canal-cr.yml, type: clusterrole}
     - {name: canal, file: canal-node-sa.yml, type: sa}
     - {name: calico-cr, file: canal-cr-calico-node.yml, type: clusterrole}
     - {name: calico-kube-cr, file: canal-cr-calico-kube-controllers.yml, type: clusterrole}
     - {name: calico-crd, file: canal-crd-calico.yml, type: crd}
     - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
+    - {name: canal, file: canal-crb-canal.yml, type: clusterrolebinding}
     - {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding}
     - {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding}
   register: canal_manifests

roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2

@@ -33,15 +33,45 @@ spec:
           effect: NoSchedule
       serviceAccountName: calico-kube-controllers
       priorityClassName: system-cluster-critical
+      # The controllers must run in the host network namespace so that
+      # it isn't governed by policy that would prevent it from working.
+      hostNetwork: true
       containers:
         - name: calico-kube-controllers
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           env:
+            # The location of the etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_cert
             # Choose which controllers to run.
             - name: ENABLED_CONTROLLERS
-              value: node
-            - name: DATASTORE_TYPE
-              value: kubernetes
+              value: policy,namespace,serviceaccount,workloadendpoint,node
+          volumeMounts:
+            # Mount in the etcd TLS secrets.
+            - mountPath: /calico-secrets
+              name: etcd-certs
           livenessProbe:
             exec:
               command:
@@ -57,3 +87,10 @@ spec:
               - /usr/bin/check-status
               - -r
             periodSeconds: 10
+      volumes:
+        # Mount in the etcd TLS secrets with mode 400.
+        # See https://kubernetes.io/docs/concepts/configuration/secret/
+        - name: etcd-certs
+          secret:
+            secretName: calico-etcd-secrets
+            defaultMode: 0440

roles/network_plugin/canal/templates/canal-config.yaml.j2

@@ -7,6 +7,14 @@ metadata:
   name: canal-config
   namespace: kube-system
 data:
+  # Configure this with the location of your etcd cluster.
+  etcd_endpoints: "{{ etcd_access_addresses }}"
+  # If you're using TLS enabled etcd uncomment the following.
+  # You must also populate the Secret below with these files.
+  etcd_ca: "/calico-secrets/etcd-ca"
+  etcd_cert: "/calico-secrets/etcd-cert"
+  etcd_key: "/calico-secrets/etcd-key"
+
   # Typha is disabled.
   typha_service_name: "none"
 
@@ -28,41 +36,39 @@ data:
   # values in this config will be automatically populated.
   cni_network_config: |-
     {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
+        "name": "canal",
+        "cniVersion": "0.3.1",
+        "plugins": [
+            {
+                "type": "flannel",
+                "delegate": {
+                    "type": "calico",
+                    "include_default_routes": true,
+                    "etcd_endpoints": "__ETCD_ENDPOINTS__",
+                    "etcd_key_file": "__ETCD_KEY_FILE__",
+                    "etcd_cert_file": "__ETCD_CERT_FILE__",
+                    "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
+                    "log_level": "info",
 {% if calico_cni_log_file_path %}
-          "log_file_path": "{{ calico_cni_log_file_path }}",
+                    "log_file_path": "{{ calico_cni_log_file_path }}",
 {% endif %}
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-              "type": "host-local",
-              "subnet": "usePodCidr"
-          },
-          "policy": {
-              "type": "k8s"
-          },
-          "kubernetes": {
-              "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        },
-        {
-          "type": "bandwidth",
-          "capabilities": {"bandwidth": true}
-        }
-      ]
+                    "policy": {
+                        "type": "k8s",
+                        "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+                        "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+                    },
+                    "kubernetes": {
+                        "kubeconfig": "__KUBECONFIG_FILEPATH__"
+                    }
+                }
+            },
+            {
+                "type": "portmap",
+                "capabilities": {"portMappings": true},
+                "snat": true
+            }
+        ]
     }
-
   # Flannel network configuration. Mounted into the flannel container.
   net-conf.json: |
     {
@@ -71,3 +77,4 @@ data:
         "Type": "vxlan"
       }
     }
+

roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2

@@ -1,4 +1,4 @@
----
+# Flannel ClusterRole
 # Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1

roles/network_plugin/canal/templates/canal-cr.yml.j2

@@ -0,0 +1,30 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: canal
+rules:
+  # Used for creating service account tokens to be used by the CNI plugin
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    verbs:
+      - create
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+      - namespaces
+    verbs:
+      - get
+  # Pod CIDR auto-detection on kubeadm needs access to config maps.
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list

roles/network_plugin/canal/templates/canal-crb-canal.yml.j2

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: canal
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: canal
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: kube-system

roles/network_plugin/canal/templates/canal-node-sa.yml.j2

@@ -4,3 +4,9 @@ kind: ServiceAccount
 metadata:
   name: canal
   namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -44,6 +44,7 @@ spec:
         # and CNI network config file on each node.
         - name: install-cni
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           command: ["/opt/cni/bin/install"]
           envFrom:
           - configMapRef:
@@ -71,6 +72,30 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # The location of the etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_cert
             # CNI MTU Config variable
             - name: CNI_MTU
               valueFrom:
@@ -85,6 +110,8 @@ spec:
               name: cni-bin-dir
             - mountPath: /host/etc/cni/net.d
               name: cni-net-dir
+            - mountPath: /calico-secrets
+              name: etcd-certs
           securityContext:
             privileged: true
         # This init container mounts the necessary filesystems needed by the BPF data plane
@@ -125,17 +152,32 @@ spec:
               name: kubernetes-services-endpoint
               optional: true
           env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Configure route aggregation based on pod CIDR.
-            - name: USE_POD_CIDR
-              value: "true"
-            # Wait for the datastore.
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # Set based on the k8s node name.
-            - name: NODENAME
+            # The location of the etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_cert
+            # Set noderef for node controller.
+            - name: CALICO_K8S_NODE_REF
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
@@ -221,6 +263,8 @@ spec:
             - mountPath: /var/lib/calico
               name: var-lib-calico
               readOnly: false
+            - mountPath: /calico-secrets
+              name: etcd-certs
             - name: policysync
               mountPath: /var/run/nodeagent
             # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
@@ -230,48 +274,94 @@ spec:
             - name: cni-log-dir
               mountPath: /var/log/calico/cni
               readOnly: true
-        # This container runs flannel using the kube-subnet-mgr backend
-        # for allocating subnets.
-        - name: kube-flannel
+        # Runs the flannel daemon to enable vxlan networking between
+        # container hosts.
+        - name: flannel
           image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
+          command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"]
           imagePullPolicy: {{ k8s_image_pull_policy }}
-          command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
-          securityContext:
-            privileged: true
-          resources:
-            limits:
-              cpu: {{ flannel_cpu_limit }}
-              memory: {{ flannel_memory_limit }}
-            requests:
-              cpu: {{ flannel_cpu_requests }}
-              memory: {{ flannel_memory_requests }}
           env:
-            - name: POD_NAME
+            # The location of the etcd cluster.
+            - name: FLANNELD_ETCD_ENDPOINTS
               valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
               valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_cert
+            # Location of the CA certificate for etcd.
+            - name: FLANNELD_ETCD_CAFILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: FLANNELD_ETCD_KEYFILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: FLANNELD_ETCD_CERTFILE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_cert
+            # The interface flannel should run on.
             - name: FLANNELD_IFACE
               valueFrom:
                 configMapKeyRef:
                   name: canal-config
                   key: canal_iface
+            # Perform masquerade on traffic leaving the pod cidr.
             - name: FLANNELD_IP_MASQ
               valueFrom:
                 configMapKeyRef:
                   name: canal-config
                   key: masquerade
+            # Write the subnet.env file to the mounted directory.
+            - name: FLANNELD_SUBNET_FILE
+              value: "/run/flannel/subnet.env"
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          securityContext:
+            privileged: true
           volumeMounts:
-          - mountPath: /run/xtables.lock
-            name: xtables-lock
-            readOnly: false
-          - name: flannel-cfg
-            mountPath: /etc/kube-flannel/
+            - mountPath: /etc/resolv.conf
+              name: resolv
+            - mountPath: /run/flannel
+              name: run-flannel
+            - mountPath: /calico-secrets
+              name: etcd-certs
+            - name: flannel-cfg
+              mountPath: /etc/kube-flannel/
       volumes:
-        # Used by canal.
+        - name: flannel-cfg
+          configMap:
+            name: canal-config
+        # Used by canal-node.
         - name: lib-modules
           hostPath:
             path: /lib/modules
@@ -298,9 +388,12 @@ spec:
           hostPath:
             path: /proc
         # Used by flannel.
-        - name: flannel-cfg
-          configMap:
-            name: canal-config
+        - name: run-flannel
+          hostPath:
+            path: /run/flannel
+        - name: resolv
+          hostPath:
+            path: /etc/resolv.conf
         # Used to install CNI.
         - name: cni-bin-dir
           hostPath:
@@ -312,6 +405,12 @@ spec:
         - name: cni-log-dir
           hostPath:
             path: /var/log/calico/cni
+        # Mount in the etcd TLS secrets with mode 400.
+        # See https://kubernetes.io/docs/concepts/configuration/secret/
+        - name: etcd-certs
+          secret:
+            secretName: calico-etcd-secrets
+            defaultMode: 0400
         # Used to create per-pod Unix Domain Sockets
         - name: policysync
           hostPath:

roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2

@@ -0,0 +1,18 @@
+# Source: calico/templates/calico-etcd-secrets.yaml
+# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
+# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: calico-etcd-secrets
+  namespace: kube-system
+data:
+  # Populate the following with etcd TLS configuration if desired, but leave blank if
+  # not using TLS for etcd.
+  # The keys below should be uncommented and the values populated with the base64
+  # encoded contents of each file that would be associated with the TLS data.
+  # Example command for encoding a file contents: cat <file> | base64 -w 0
+  etcd-key: {{ etcd_key_file.content }}
+  etcd-cert: {{ etcd_cert_file.content }}
+  etcd-ca: {{ etcd_ca_cert_file.content }}

roles/network_plugin/canal/templates/cni-canal.conflist.j2

@@ -1,33 +1,34 @@
-{
-  "name": "cni0",
-  "cniVersion":"0.3.1",
-  "plugins":[
     {
-      "type": "flannel",
-      "delegate": {
-        "type": "calico",
-	    "include_default_routes": true,
-        "etcd_endpoints": "{{ etcd_access_addresses }}",
-        "etcd_key_file": "{{ canal_cert_dir }}/key.pem",
-        "etcd_cert_file": "{{ canal_cert_dir }}/cert.crt",
-        "etcd_ca_cert_file": "{{ canal_cert_dir }}/ca_cert.crt",
-        "log_level": "info",
+        "name": "canal",
+        "cniVersion": "0.3.1",
+        "plugins": [
+            {
+                "type": "flannel",
+                "delegate": {
+                    "type": "calico",
+                    "include_default_routes": true,
+                    "etcd_endpoints": "__ETCD_ENDPOINTS__",
+                    "etcd_key_file": "__ETCD_KEY_FILE__",
+                    "etcd_cert_file": "__ETCD_CERT_FILE__",
+                    "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
+                    "log_level": "info",
 {% if calico_cni_log_file_path %}
-        "log_file_path": "{{ calico_cni_log_file_path }}",
+                    "log_file_path": "{{ calico_cni_log_file_path }}",
 {% endif %}
-        "policy": {
-          "type": "k8s"
-        },
-        "kubernetes": {
-          "kubeconfig": "__KUBECONFIG_FILEPATH__"
-        }
-      }
-    },
-    {
-      "type":"portmap",
-      "capabilities":{
-        "portMappings":true
-      }
+                    "policy": {
+                        "type": "k8s",
+                        "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+                        "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+                    },
+                    "kubernetes": {
+                        "kubeconfig": "__KUBECONFIG_FILEPATH__"
+                    }
+                }
+            },
+            {
+                "type": "portmap",
+                "capabilities": {"portMappings": true},
+                "snat": true
+            }
+        ]
     }
-  ]
-}