Enable weave seed mode for kubespray (#1414)

* Enable weave seed mode for kubespray * fix task Weave seed | Set peers if existing peers * fix mac address variabilisation * fix default values * fix include seed condition * change weave var to default values * fix Set peers if existing peers
7 years ago · 3e457e4edf
8 changed files with 226 additions and 100 deletions
--- a/inventory/group_vars/k8s-cluster.yml
+++ b/inventory/group_vars/k8s-cluster.yml
@ -74,6 +74,22 @@ kube_users:
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico

+# weave's network password for encryption
+# if null then no network encryption
+# you can use --extra-vars to pass the password in command line
+weave_password: EnterPasswordHere
+
+# Weave uses consensus mode by default
+# Enabling seed mode allow to dynamically add or remove hosts
+# https://www.weave.works/docs/net/latest/ipam/
+weave_mode_seed: false
+
+# This two variable are automatically changed by the weave's role, do not manually change these values
+# To reset values :
+# weave_seed: uninitialized
+# weave_peers: uninitialized
+weave_seed: uninitialized
+weave_peers: uninitialized

 # Enable kubernetes network policies
 enable_network_policy: false
@ -136,8 +152,3 @@ efk_enabled: false

 # Helm deployment
 helm_enabled: false
-
-# dnsmasq
-# dnsmasq_upstream_dns_servers:
-#  - /resolvethiszone.with/10.0.4.250
-#  - 8.8.8.8
--- a/inventory/inventory.example
+++ b/inventory/inventory.example
@ -28,4 +28,4 @@

 # [k8s-cluster:children]
 # kube-node
-# kube-master
+# kube-master
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -25,7 +25,7 @@ etcd_version: v3.0.17
 calico_version: "v1.1.3"
 calico_cni_version: "v1.7.0"
 calico_policy_version: "v0.5.4"
-weave_version: 1.8.2
+weave_version: 2.0.1
 flannel_version: v0.6.2
 pod_infra_version: 3.0

--- a/roles/network_plugin/weave/defaults/main.yml
+++ b/roles/network_plugin/weave/defaults/main.yml
@ -4,3 +4,13 @@ weave_memory_limit: 400M
 weave_cpu_limit: 30m
 weave_memory_requests: 64M
 weave_cpu_requests: 10m
+
+# This two variable are automatically changed by the weave's role, do not manually change these values
+# To reset values :
+# weave_seed: unset
+# weave_peers: unset
+weave_seed: uninitialized
+weave_peers: uninitialized
+
+# this variable is use in seed mode
+weave_ip_current_cluster: "{% for host in groups['k8s-cluster'] %}{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}{% if not loop.last %} {% endif %}{% endfor %}"
--- a/roles/network_plugin/weave/tasks/main.yml
+++ b/roles/network_plugin/weave/tasks/main.yml
@ -1,6 +1,9 @@
 ---
 - include: pre-upgrade.yml

+- include: seed.yml
+  when: weave_mode_seed
+
 - name: Weave | enable br_netfilter module
  modprobe:
    name: br_netfilter
--- a/roles/network_plugin/weave/tasks/seed.yml
+++ b/roles/network_plugin/weave/tasks/seed.yml
@ -0,0 +1,50 @@
+---
+- name: Weave seed | Set seed if first time
+  set_fact:
+    seed: '{% for host in groups["k8s-cluster"] %}{{ hostvars[host]["ansible_default_ipv4"]["macaddress"] }}{% if not loop.last %},{% endif %}{% endfor %}'
+  when: "weave_seed == 'uninitialized'"
+  run_once: true
+  tags: confweave
+
+- name: Weave seed | Set seed if not first time
+  set_fact:
+    seed: '{{ weave_seed }}'
+  when: "weave_seed != 'uninitialized'"
+  run_once: true
+  tags: confweave
+
+- name: Weave seed | Set peers if fist time
+  set_fact:
+    peers: '{{ weave_ip_current_cluster }}'
+  when: "weave_peers == 'uninitialized'"
+  run_once: true
+  tags: confweave
+
+- name: Weave seed | Set peers if existing peers
+  set_fact:
+    peers: '{{ weave_peers }}{% for ip in weave_ip_current_cluster.split(" ") %}{% if ip not in weave_peers.split(" ") %} {{ ip }}{% endif %}{% endfor %}'
+  when: "weave_peers != 'uninitialized'"
+  run_once: true
+  tags: confweave
+
+- name: Weave seed | Save seed
+  lineinfile:
+    dest: "./inventory/group_vars/k8s-cluster.yml"
+    state: present
+    regexp: '^weave_seed:'
+    line: 'weave_seed: {{ seed }}'
+  become: no
+  delegate_to: 127.0.0.1
+  run_once: true
+  tags: confweave
+
+- name: Weave seed | Save peers
+  lineinfile:
+    dest: "./inventory/group_vars/k8s-cluster.yml"
+    state: present
+    regexp: '^weave_peers:'
+    line: 'weave_peers: {{ peers }}'
+  become: no
+  delegate_to: 127.0.0.1
+  run_once: true
+  tags: confweave
--- a/roles/network_plugin/weave/templates/weave-net.yml.j2
+++ b/roles/network_plugin/weave/templates/weave-net.yml.j2
@ -1,104 +1,156 @@
 ---
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: weave-net
-  namespace: {{ system_namespace }}
-  labels:
-    version: {{ weave_version }}
-spec:
-  template:
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: ServiceAccount
    metadata:
+      name: weave-net
      labels:
        name: weave-net
-      annotations:
-        scheduler.alpha.kubernetes.io/tolerations: |
-          [
-            {
-              "key": "dedicated",
-              "operator": "Equal",
-              "value": "master",
-              "effect": "NoSchedule"
-            }
-          ]
+      namespace: {{ system_namespace }}
+  - apiVersion: rbac.authorization.k8s.io/v1beta1
+    kind: ClusterRole
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+          - ''
+        resources:
+          - pods
+          - namespaces
+          - nodes
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - extensions
+        resources:
+          - networkpolicies
+        verbs:
+          - get
+          - list
+          - watch
+  - apiVersion: rbac.authorization.k8s.io/v1beta1
+    kind: ClusterRoleBinding
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    roleRef:
+      kind: ClusterRole
+      name: weave-net
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
+  - apiVersion: extensions/v1beta1
+    kind: DaemonSet
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+        version: {{ weave_version }}
+      namespace: {{ system_namespace }}
    spec:
-      hostNetwork: true
-      hostPID: true
-      containers:
-        - name: weave
-          image: {{ weave_kube_image_repo }}:{{ weave_kube_image_tag }}
-          imagePullPolicy: Always
-          command:
-            - /home/weave/launch.sh
-          env:
-            - name: IPALLOC_RANGE
-              value: {{ kube_pods_subnet }}
-{% if weave_checkpoint_disable is defined %}
-            - name: CHECKPOINT_DISABLE
-              value: {{ weave_checkpoint_disable }}
-{% endif %}
-{% if weave_expect_npc is defined %}
-            - name: EXPECT_NPC
-              value: {{ weave_expect_npc }}
+      template:
+        metadata:
+          labels:
+            name: weave-net
+        spec:
+          containers:
+            - name: weave
+              command:
+{% if weave_mode_seed == true %}
+                - /bin/sh
+                - -c
+                - export EXTRA_ARGS=--name=$(cat /sys/class/net/{{ ansible_default_ipv4['interface'] }}/address) && /home/weave/launch.sh
+{% else %}
+                - /home/weave/launch.sh
 {% endif %}
-{% if weave_kube_peers is defined %}
-            - name: KUBE_PEERS
-              value: {{ weave_kube_peers }}
+              env:
+                - name: HOSTNAME
+                  valueFrom:
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: spec.nodeName
+                - name: IPALLOC_RANGE
+                  value: {{ kube_pods_subnet }}
+{% if weave_mode_seed == true %}
+                - name: KUBE_PEERS
+                  value: {{ peers }}
+                - name: IPALLOC_INIT
+                  value: seed={{ seed }}
 {% endif %}
-{% if weave_ipalloc_init is defined %}
-            - name: IPALLOC_INIT
-              value: {{ weave_ipalloc_init }}
-{% endif %}
-{% if weave_expose_ip is defined %}
-            - name: WEAVE_EXPOSE_IP
-              value: {{ weave_expose_ip }}
-{% endif %}
-          livenessProbe:
-            initialDelaySeconds: 60
-            httpGet:
-              host: 127.0.0.1
-              path: /status
-              port: 6784
+                - name: WEAVE_PASSWORD
+                  value: {{ weave_password }}
+              image: {{ weave_kube_image_repo }}:{{ weave_kube_image_tag }}
+              imagePullPolicy: Always
+              livenessProbe:
+                httpGet:
+                  host: 127.0.0.1
+                  path: /status
+                  port: 6784
+                initialDelaySeconds: 30
+              resources:
+                requests:
+                  cpu: 10m
+              securityContext:
+                privileged: true
+              volumeMounts:
+                - name: weavedb
+                  mountPath: /weavedb
+                - name: cni-bin
+                  mountPath: /host/opt
+                - name: cni-bin2
+                  mountPath: /host/home
+                - name: cni-conf
+                  mountPath: /host/etc
+                - name: dbus
+                  mountPath: /host/var/lib/dbus
+                - name: lib-modules
+                  mountPath: /lib/modules
+            - name: weave-npc
+              image: {{ weave_npc_image_repo }}:{{ weave_npc_image_tag }}
+              imagePullPolicy: Always
+              resources:
+                requests:
+                  cpu: {{ weave_cpu_requests }}
+                  memory: {{ weave_memory_requests }}
+                limits:
+                  cpu: {{ weave_cpu_limit }}
+                  memory: {{ weave_memory_limit }}
+              securityContext:
+                privileged: true
+          hostNetwork: true
+          hostPID: true
+          restartPolicy: Always
          securityContext:
-            privileged: true
-          volumeMounts:
+            seLinuxOptions: {}
+          serviceAccountName: weave-net
+          tolerations:
+            - effect: NoSchedule
+              operator: Exists
+          volumes:
            - name: weavedb
-              mountPath: /weavedb
+              hostPath:
+                path: /var/lib/weave
            - name: cni-bin
-              mountPath: /opt
+              hostPath:
+                path: /opt
            - name: cni-bin2
-              mountPath: /host_home
+              hostPath:
+                path: /home
            - name: cni-conf
-              mountPath: /etc
-          resources:
-            requests:
-              cpu: {{ weave_cpu_requests }}
-              memory: {{ weave_memory_requests }}
-            limits:
-              cpu: {{ weave_cpu_limit }}
-              memory: {{ weave_memory_limit }}
-        - name: weave-npc
-          image: {{ weave_npc_image_repo }}:{{ weave_npc_image_tag }}
-          imagePullPolicy: Always
-          resources:
-            requests:
-              cpu: {{ weave_cpu_requests }}
-              memory: {{ weave_memory_requests }}
-            limits:
-              cpu: {{ weave_cpu_limit }}
-              memory: {{ weave_memory_limit }}
-          securityContext:
-            privileged: true
-      restartPolicy: Always
-      volumes:
-        - name: weavedb
-          emptyDir: {}
-        - name: cni-bin
-          hostPath:
-            path: /opt
-        - name: cni-bin2
-          hostPath:
-            path: /home
-        - name: cni-conf
-          hostPath:
-            path: /etc
+              hostPath:
+                path: /etc
+            - name: dbus
+              hostPath:
+                path: /var/lib/dbus
+            - name: lib-modules
+              hostPath:
+                path: /lib/modules
--- a/roles/uploads/defaults/main.yml
+++ b/roles/uploads/defaults/main.yml
@ -5,7 +5,7 @@ local_release_dir: /tmp
 etcd_version: v3.0.17
 calico_version: v0.23.0
 calico_cni_version: v1.5.6
-weave_version: v1.8.2
+weave_version: v2.0.1

 # Download URL's
 etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"