diff --git a/cluster.yml b/cluster.yml index 8462ea894..d7ff55045 100644 --- a/cluster.yml +++ b/cluster.yml @@ -93,6 +93,7 @@ roles: - { role: kubespray-defaults} - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } + - { role: win_nodes/kubernetes_patch, tags: win_nodes, when: "kubeadm_enabled" } - hosts: kube-master any_errors_fatal: "{{ any_errors_fatal | default(true) }}" diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 index a6d1df934..4489e2418 100644 --- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 +++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 @@ -52,3 +52,6 @@ spec: - --default-params={"linear":{"nodesPerReplica":{{ dnsmasq_nodes_per_replica }},"preventSinglePointFailure":true}} - --logtostderr=true - --v={{ kube_log_level }} + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux diff --git a/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2 b/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2 index 0fb6045e8..c3a32f02e 100644 --- a/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2 +++ b/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2 @@ -24,6 +24,9 @@ spec: tolerations: - effect: NoSchedule operator: Exists + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: dnsmasq image: "{{ dnsmasq_image_repo }}:{{ dnsmasq_image_tag }}" diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 70e98b53f..feba0ea38 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -136,6 +136,15 @@ msg: "{{available_packages}}" when: docker_task_result|failed +# This is required to ensure any apt upgrade will not break kubernetes +- name: Set docker pin priority to apt_preferences on Debian family + template: + src: "apt_preferences.d/debian_docker.j2" + dest: "/etc/apt/preferences.d/docker" + owner: "root" + mode: 0644 + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic) + - name: ensure service is started if docker packages are already present service: name: docker diff --git a/roles/docker/templates/apt_preferences.d/debian_docker.j2 b/roles/docker/templates/apt_preferences.d/debian_docker.j2 new file mode 100644 index 000000000..f21008b6c --- /dev/null +++ b/roles/docker/templates/apt_preferences.d/debian_docker.j2 @@ -0,0 +1,3 @@ +Package: docker-ce +Pin: version {{ docker_version }}.* +Pin-Priority: 1001 \ No newline at end of file diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 38df04d73..db59a983f 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -6,7 +6,6 @@ - facts - include_tasks: "gen_certs_{{ cert_management }}.yml" - when: tags: - etcd-secrets diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 index 11c8d37f0..e726e8d2a 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 @@ -28,6 +28,9 @@ spec: labels: k8s-app: kubedns-autoscaler spec: + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux tolerations: - effect: NoSchedule operator: Equal diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 index 549d93c14..96ef72283 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 @@ -27,6 +27,9 @@ spec: annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux tolerations: - key: "CriticalAddonsOnly" operator: "Exists" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 index 431448231..a2c4850c4 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -15,6 +15,9 @@ spec: tolerations: - effect: NoSchedule operator: Exists + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: netchecker-agent image: "{{ agent_img }}" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 index ad32d509a..f046e8f4b 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -13,6 +13,9 @@ spec: app: netchecker-agent-hostnet spec: hostNetwork: True + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux {% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirstWithHostNet {% endif %} diff --git a/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 index 6e9ad30c0..03b118f8d 100644 --- a/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 +++ b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 @@ -29,6 +29,9 @@ spec: spec: priorityClassName: system-node-critical serviceAccountName: efk + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: fluentd-es image: "{{ fluentd_image_repo }}:{{ fluentd_image_tag }}" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 index 76d71dd96..470950b03 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 @@ -42,3 +42,6 @@ spec: requests: cpu: 10m memory: 20Mi + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 69ad06e4f..d2d2f89f4 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -121,6 +121,7 @@ --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades + --force register: kubeadm_upgrade # Retry is because upload config sometimes fails retries: 3 diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index d1292887a..ece9be10c 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -12,6 +12,9 @@ spec: {% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirst {% endif %} + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: kube-proxy image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} diff --git a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 index a1e9a7815..756eba7ee 100644 --- a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 @@ -7,6 +7,9 @@ metadata: k8s-app: kube-nginx spec: hostNetwork: true + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: nginx-proxy image: {{ nginx_image_repo }}:{{ nginx_image_tag }} diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index b201e8e7f..de9be8d9e 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -53,6 +53,9 @@ spec: k8s-app: flannel spec: serviceAccountName: flannel + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: kube-flannel image: {{ flannel_image_repo }}:{{ flannel_image_tag }} diff --git a/roles/win_nodes/kubernetes_patch/defaults/main.yml b/roles/win_nodes/kubernetes_patch/defaults/main.yml new file mode 100644 index 000000000..587f73ab4 --- /dev/null +++ b/roles/win_nodes/kubernetes_patch/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +kubernetes_user_manifests_path: "{{ ansible_env.HOME }}/kube-manifests" diff --git a/roles/win_nodes/kubernetes_patch/files/nodeselector-os-linux-patch.json b/roles/win_nodes/kubernetes_patch/files/nodeselector-os-linux-patch.json new file mode 100644 index 000000000..d718ff446 --- /dev/null +++ b/roles/win_nodes/kubernetes_patch/files/nodeselector-os-linux-patch.json @@ -0,0 +1 @@ +{"spec":{"template":{"spec":{"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}} \ No newline at end of file diff --git a/roles/win_nodes/kubernetes_patch/tasks/main.yml b/roles/win_nodes/kubernetes_patch/tasks/main.yml new file mode 100644 index 000000000..8d88818a5 --- /dev/null +++ b/roles/win_nodes/kubernetes_patch/tasks/main.yml @@ -0,0 +1,34 @@ +--- + +- name: Ensure that user manifests directory exists + file: + path: "{{ kubernetes_user_manifests_path }}/kubernetes" + state: directory + recurse: yes + tags: [init, cni] + +- name: Apply kube-proxy nodeselector + block: + - name: Copy kube-proxy daemonset nodeselector patch + copy: + src: nodeselector-os-linux-patch.json + dest: "{{ kubernetes_user_manifests_path }}/nodeselector-os-linux-patch.json" + + # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for "kubectl patch" + - name: Check current nodeselector for kube-proxy daemonset + shell: kubectl get ds kube-proxy --namespace=kube-system -o jsonpath='{.spec.template.spec.nodeSelector.beta\.kubernetes\.io/os}' + register: current_kube_proxy_state + + - name: Apply nodeselector patch for kube-proxy daemonset + shell: kubectl patch ds kube-proxy --namespace=kube-system --type=strategic -p "$(cat nodeselector-os-linux-patch.json)" + args: + chdir: "{{ kubernetes_user_manifests_path }}" + register: patch_kube_proxy_state + when: current_kube_proxy_state.stdout | trim | lower != "linux" + + - debug: msg={{ patch_kube_proxy_state.stdout_lines }} + when: patch_kube_proxy_state is not skipped + + - debug: msg={{ patch_kube_proxy_state.stderr_lines }} + when: patch_kube_proxy_state is not skipped + tags: init