You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

179 lines
5.6 KiB

Issue 8004: Fix typha prometheus (#8005) The typha prometheus settings were in the `volumeMounts` section of the spec and not in the `envs` section. This was cauing the deployment to fail because it was looking for a volumeMount. ``` failed: [controller-001.a2.da.dev.logdna.net] (item=calico-typha.yml) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": true, "checksum": "598ac79530749e8e2110793b53fc49ac208e7130", "dest": "/etc/kubernetes/calico-typha.yml", "diff": [], "failed": false, "gid": 0, "group": "root", "invocation": {"module_args": {"_original_basename": "calico-typha.yml.j2", "attributes": null, "backup": false, "checksum": "598ac79530749e8e2110793b53fc49ac208e7130", "content": null, "delimiter": null, "dest": "/etc/kubernetes/calico-typha.yml", "directory_mode": null, "follow": false, "force": true, "group": null, "local_follow": null, "mode": null, "owner": null, "regexp": null, "remote_src": null, "selevel": null, "serole": null, "setype": null, "seuser": null, "src": "/home/core/.ansible/tmp/ansible-tmp-1632349768.56-75434-32452975679246/source", "unsafe_writes": null, "validate": null}}, "item": {"file": "calico-typha.yml", "name": "calico", "type": "typha"}, "md5sum": "53c00ac7f562cf9ecbbfd27899ea066d", "mode": "0644", "owner": "root", "size": 5378, "src": "/home/core/.ansible/tmp/ansible-tmp-1632349768.56-75434-32452975679246/source", "state": "file", "uid": 0}, "msg": "error running kubectl (/opt/bin/kubectl --namespace=kube-system apply --force --filename=/etc/kubernetes/calico-typha.yml) command (rc=1), out='service/calico-typha unchanged\n', err='error: error validating \"/etc/kubernetes/calico-typha.yml\": error validating data: [ValidationError(Deployment.spec.template.spec.containers[0].volumeMounts[2]): unknown field \"value\" in io.k8s.api.core.v1.VolumeMount, ValidationError(Deployment.spec.template.spec.containers[0].volumeMounts[2]): missing required field \"mountPath\" in io.k8s.api.core.v1.VolumeMount, ValidationError(Deployment.spec.template.spec.containers[0].volumeMounts[3]): unknown field \"value\" in io.k8s.api.core.v1.VolumeMount, ValidationError(Deployment.spec.template.spec.containers[0].volumeMounts[3]): missing required field \"mountPath\" in io.k8s.api.core.v1.VolumeMount]; if you choose to ignore these errors, turn validation off with --validate=false\n'"} ```
3 years ago
  1. # This manifest creates a Service, which will be backed by Calico's Typha daemon.
  2. # Typha sits in between Felix and the API server, reducing Calico's load on the API server.
  3. apiVersion: v1
  4. kind: Service
  5. metadata:
  6. name: calico-typha
  7. namespace: kube-system
  8. labels:
  9. k8s-app: calico-typha
  10. spec:
  11. ports:
  12. - port: 5473
  13. protocol: TCP
  14. targetPort: calico-typha
  15. name: calico-typha
  16. selector:
  17. k8s-app: calico-typha
  18. ---
  19. # This manifest creates a Deployment of Typha to back the above service.
  20. apiVersion: apps/v1
  21. kind: Deployment
  22. metadata:
  23. name: calico-typha
  24. namespace: kube-system
  25. labels:
  26. k8s-app: calico-typha
  27. spec:
  28. # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the
  29. # typha_service_name variable in the calico-config ConfigMap above.
  30. #
  31. # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential
  32. # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In
  33. # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
  34. replicas: {{ typha_replicas }}
  35. revisionHistoryLimit: 2
  36. selector:
  37. matchLabels:
  38. k8s-app: calico-typha
  39. template:
  40. metadata:
  41. labels:
  42. k8s-app: calico-typha
  43. annotations:
  44. cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
  45. {% if typha_prometheusmetricsenabled %}
  46. prometheus.io/scrape: 'true'
  47. prometheus.io/port: "{{ typha_prometheusmetricsport }}"
  48. {% endif %}
  49. spec:
  50. nodeSelector:
  51. kubernetes.io/os: linux
  52. hostNetwork: true
  53. tolerations:
  54. - key: node-role.kubernetes.io/master
  55. operator: Exists
  56. effect: NoSchedule
  57. - key: node-role.kubernetes.io/control-plane
  58. operator: Exists
  59. effect: NoSchedule
  60. # Since Calico can't network a pod until Typha is up, we need to run Typha itself
  61. # as a host-networked pod.
  62. serviceAccountName: calico-node
  63. priorityClassName: system-cluster-critical
  64. # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573
  65. securityContext:
  66. fsGroup: 65534
  67. containers:
  68. - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
  69. imagePullPolicy: {{ k8s_image_pull_policy }}
  70. name: calico-typha
  71. ports:
  72. - containerPort: 5473
  73. name: calico-typha
  74. protocol: TCP
  75. envFrom:
  76. - configMapRef:
  77. # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
  78. name: kubernetes-services-endpoint
  79. optional: true
  80. env:
  81. # Enable "info" logging by default. Can be set to "debug" to increase verbosity.
  82. - name: TYPHA_LOGSEVERITYSCREEN
  83. value: "info"
  84. # Disable logging to file and syslog since those don't make sense in Kubernetes.
  85. - name: TYPHA_LOGFILEPATH
  86. value: "none"
  87. - name: TYPHA_LOGSEVERITYSYS
  88. value: "none"
  89. # Monitor the Kubernetes API to find the number of running instances and rebalance
  90. # connections.
  91. - name: TYPHA_CONNECTIONREBALANCINGMODE
  92. value: "kubernetes"
  93. - name: TYPHA_DATASTORETYPE
  94. value: "kubernetes"
  95. - name: TYPHA_HEALTHENABLED
  96. value: "true"
  97. - name: TYPHA_MAXCONNECTIONSLOWERLIMIT
  98. value: "{{ typha_max_connections_lower_limit }}"
  99. {% if typha_secure %}
  100. - name: TYPHA_CAFILE
  101. value: /etc/ca/ca.crt
  102. - name: TYPHA_CLIENTCN
  103. value: typha-client
  104. - name: TYPHA_SERVERCERTFILE
  105. value: /etc/typha/server_certificate.pem
  106. - name: TYPHA_SERVERKEYFILE
  107. value: /etc/typha/server_key.pem
  108. {% endif %}
  109. {% if typha_prometheusmetricsenabled %}
  110. # Since Typha is host-networked,
  111. # this opens a port on the host, which may need to be secured.
  112. - name: TYPHA_PROMETHEUSMETRICSENABLED
  113. value: "true"
  114. - name: TYPHA_PROMETHEUSMETRICSPORT
  115. value: "{{ typha_prometheusmetricsport }}"
  116. {% endif %}
  117. {% if typha_secure %}
  118. volumeMounts:
  119. - mountPath: /etc/typha
  120. name: typha-server
  121. readOnly: true
  122. - mountPath: /etc/ca/ca.crt
  123. subPath: ca.crt
  124. name: cacert
  125. readOnly: true
  126. {% endif %}
  127. # Needed for version >=3.7 when the 'host-local' ipam is used
  128. # Should never happen given templates/cni-calico.conflist.j2
  129. # Configure route aggregation based on pod CIDR.
  130. # - name: USE_POD_CIDR
  131. # value: "true"
  132. livenessProbe:
  133. httpGet:
  134. path: /liveness
  135. port: 9098
  136. host: localhost
  137. periodSeconds: 30
  138. initialDelaySeconds: 30
  139. readinessProbe:
  140. httpGet:
  141. path: /readiness
  142. port: 9098
  143. host: localhost
  144. periodSeconds: 10
  145. {% if typha_secure %}
  146. volumes:
  147. - name: typha-server
  148. secret:
  149. secretName: typha-server
  150. items:
  151. - key: tls.crt
  152. path: server_certificate.pem
  153. - key: tls.key
  154. path: server_key.pem
  155. - name: cacert
  156. hostPath:
  157. path: "{{ kube_cert_dir }}"
  158. {% endif %}
  159. ---
  160. # This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict
  161. apiVersion: policy/v1beta1
  162. kind: PodDisruptionBudget
  163. metadata:
  164. name: calico-typha
  165. namespace: kube-system
  166. labels:
  167. k8s-app: calico-typha
  168. spec:
  169. maxUnavailable: 1
  170. selector:
  171. matchLabels:
  172. k8s-app: calico-typha