Browse Source
Added cilium support (#2236)
Added cilium support (#2236)
* Added cilium support * Fix typo in debian test config * Remove empty lines * Changed cilium version from <latest> to <v1.0.0-rc3> * Add missing changes for cilium * Add cilium to CI pipeline * Fix wrong file name * Check kernel version for cilium * fixed ci error * fixed cilium-ds.j2 template * added waiting for cilium pods to run * Fixed missing EOF * Fixed trailing spaces * Fixed trailing spaces * Fixed trailing spaces * Fixed too many blank lines * Updated tolerations,annotations in cilium DS template * Set cilium_version to iptables-1.9 to see if bug is fixed in CI * Update cilium image tag to v1.0.0-rc4 * Update Cilium test case CI vars filenames * Add optional prometheus flag, adjust initial readiness delay * Update README.md with cilium infopull/2355/merge
melkosoft
7 years ago
committed by
Chad Swenson
Chad Swenson
26 changed files with 486 additions and 8 deletions
Split View
Diff Options
-
15.gitlab-ci.yml
-
5README.md
-
2inventory/sample/group_vars/k8s-cluster.yml
-
9roles/download/defaults/main.yml
-
20roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml
-
5roles/kubernetes-apps/network_plugin/meta/main.yml
-
2roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
-
2roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
-
2roles/kubernetes/node/templates/kubelet.rkt.service.j2
-
2roles/kubernetes/node/templates/kubelet.standard.env.j2
-
3roles/kubernetes/preinstall/tasks/main.yml
-
6roles/kubernetes/preinstall/tasks/verify-settings.yml
-
2roles/kubespray-defaults/defaults/main.yaml
-
25roles/network_plugin/cilium/defaults/main.yml
-
14roles/network_plugin/cilium/handlers/main.yml
-
55roles/network_plugin/cilium/tasks/main.yml
-
29roles/network_plugin/cilium/templates/cilium-config.yml.j2
-
62roles/network_plugin/cilium/templates/cilium-cr.yml.j2
-
15roles/network_plugin/cilium/templates/cilium-crb.yml.j2
-
163roles/network_plugin/cilium/templates/cilium-ds.yml.j2
-
6roles/network_plugin/cilium/templates/cilium-sa.yml.j2
-
5roles/network_plugin/meta/main.yml
-
11tests/files/gce_centos7-cilium.yml
-
13tests/files/gce_coreos-cilium.yml
-
10tests/files/gce_rhel7-cilium.yml
-
11tests/files/gce_ubuntu-cilium-sep.yml
@ -0,0 +1,20 @@ |
|||
--- |
|||
- name: Cilium | Start Resources |
|||
kube: |
|||
name: "{{item.item.name}}" |
|||
namespace: "{{ system_namespace }}" |
|||
kubectl: "{{bin_dir}}/kubectl" |
|||
resource: "{{item.item.type}}" |
|||
filename: "{{kube_config_dir}}/{{item.item.file}}" |
|||
state: "latest" |
|||
with_items: "{{ cilium_node_manifests.results }}" |
|||
when: inventory_hostname == groups['kube-master'][0] and not item|skipped |
|||
|
|||
- name: Cilium | Wait for pods to run |
|||
command: "{{bin_dir}}/kubectl -n {{system_namespace}} get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" |
|||
register: pods_not_ready |
|||
until: pods_not_ready.stdout.find("cilium")==-1 |
|||
retries: 30 |
|||
delay: 10 |
|||
ignore_errors: yes |
|||
when: inventory_hostname == groups['kube-master'][0] |
|||
@ -0,0 +1,25 @@ |
|||
--- |
|||
# Log-level |
|||
cilium_debug: false |
|||
cilium_disable_ipv4: false |
|||
|
|||
# Etcd SSL dirs |
|||
cilium_cert_dir: /etc/cilium/certs |
|||
etcd_cert_dir: /etc/ssl/etcd/ssl |
|||
|
|||
# Cilium Network Policy directory |
|||
cilium_policy_dir: /etc/kubernetes/policy |
|||
|
|||
# Limits for apps |
|||
cilium_memory_limit: 500M |
|||
cilium_cpu_limit: 200m |
|||
cilium_memory_requests: 64M |
|||
cilium_cpu_requests: 50m |
|||
|
|||
# Optional features |
|||
cilium_enable_prometheus: false |
|||
|
|||
rbac_resources: |
|||
- sa |
|||
- clusterrole |
|||
- clusterrolebinding |
|||
@ -0,0 +1,14 @@ |
|||
--- |
|||
- name: restart kubelet |
|||
command: /bin/true |
|||
notify: |
|||
- Kubelet | reload systemd |
|||
- Kubelet | reload kubelet |
|||
|
|||
- name: Kubelet | reload systemd |
|||
command: systemctl daemon-reload |
|||
|
|||
- name: Kubelet | reload kubelet |
|||
service: |
|||
name: kubelet |
|||
state: restarted |
|||
@ -0,0 +1,55 @@ |
|||
--- |
|||
- name: Cilium | Ensure BFPFS mounted |
|||
mount: |
|||
fstype: bpf |
|||
path: /sys/fs/bpf |
|||
src: bpffs |
|||
state: mounted |
|||
|
|||
- name: Cilium | Create Cilium certs directory |
|||
file: |
|||
dest: "{{ cilium_cert_dir }}" |
|||
state: directory |
|||
mode: 0750 |
|||
owner: root |
|||
group: root |
|||
|
|||
- name: Cilium | Link etcd certificates for cilium |
|||
file: |
|||
src: "{{ etcd_cert_dir }}/{{ item.s }}" |
|||
dest: "{{ cilium_cert_dir }}/{{ item.d }}" |
|||
state: hard |
|||
force: yes |
|||
with_items: |
|||
- {s: "ca.pem", d: "ca_cert.crt"} |
|||
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} |
|||
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} |
|||
|
|||
- name: Cilium | Create Cilium node manifests |
|||
template: |
|||
src: "{{item.file}}.j2" |
|||
dest: "{{kube_config_dir}}/{{item.file}}" |
|||
with_items: |
|||
- {name: cilium, file: cilium-config.yml, type: cm} |
|||
- {name: cilium, file: cilium-crb.yml, type: clusterrolebinding} |
|||
- {name: cilium, file: cilium-cr.yml, type: clusterrole} |
|||
- {name: cilium, file: cilium-ds.yml, type: ds} |
|||
- {name: cilium, file: cilium-sa.yml, type: sa} |
|||
register: cilium_node_manifests |
|||
when: |
|||
- inventory_hostname in groups['kube-master'] |
|||
- rbac_enabled or item.type not in rbac_resources |
|||
|
|||
- name: Cilium | Set CNI directory permissions |
|||
file: |
|||
path: /opt/cni/bin |
|||
state: directory |
|||
owner: kube |
|||
recurse: true |
|||
mode: 0755 |
|||
register: cni_bin_dir |
|||
|
|||
- name: Cilium | Create network policy directory |
|||
file: |
|||
path: "{{ cilium_policy_dir }}" |
|||
state: directory |
|||
@ -0,0 +1,29 @@ |
|||
kind: ConfigMap |
|||
apiVersion: v1 |
|||
metadata: |
|||
name: cilium-config |
|||
namespace: {{ system_namespace }} |
|||
data: |
|||
# This etcd-config contains the etcd endpoints of your cluster. If you use |
|||
# TLS please make sure you uncomment the ca-file line and add the respective |
|||
# certificate has a k8s secret, see explanation bellow in the comment labeled |
|||
# "ETCD-CERT" |
|||
etcd-config: |- |
|||
--- |
|||
endpoints: |
|||
{% for ip_addr in etcd_access_addresses.split(',') %} |
|||
- {{ ip_addr }} |
|||
{% endfor %} |
|||
# |
|||
# In case you want to use TLS in etcd, uncomment the following line |
|||
# and add the certificate as explained in the comment labeled "ETCD-CERT" |
|||
ca-file: "{{ cilium_cert_dir }}/ca_cert.crt" |
|||
# |
|||
# In case you want client to server authentication, uncomment the following |
|||
# lines and add the certificate and key in cilium-etcd-secrets bellow |
|||
key-file: "{{ cilium_cert_dir }}/key.pem" |
|||
cert-file: "{{ cilium_cert_dir }}/cert.crt" |
|||
|
|||
# If you want to run cilium in debug mode change this value to true |
|||
debug: "{{ cilium_debug }}" |
|||
disable-ipv4: "{{ cilium_disable_ipv4 }}" |
|||
@ -0,0 +1,62 @@ |
|||
--- |
|||
kind: ClusterRole |
|||
apiVersion: rbac.authorization.k8s.io/v1beta1 |
|||
metadata: |
|||
name: cilium |
|||
rules: |
|||
- apiGroups: |
|||
- "networking.k8s.io" |
|||
resources: |
|||
- networkpolicies |
|||
verbs: |
|||
- get |
|||
- list |
|||
- watch |
|||
- apiGroups: |
|||
- "" |
|||
resources: |
|||
- namespaces |
|||
- services |
|||
- nodes |
|||
- endpoints |
|||
- componentstatuses |
|||
verbs: |
|||
- get |
|||
- list |
|||
- watch |
|||
- apiGroups: |
|||
- "" |
|||
resources: |
|||
- pods |
|||
- nodes |
|||
verbs: |
|||
- get |
|||
- list |
|||
- watch |
|||
- update |
|||
- apiGroups: |
|||
- extensions |
|||
resources: |
|||
- networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202 |
|||
- thirdpartyresources |
|||
- ingresses |
|||
verbs: |
|||
- create |
|||
- get |
|||
- list |
|||
- watch |
|||
- apiGroups: |
|||
- "apiextensions.k8s.io" |
|||
resources: |
|||
- customresourcedefinitions |
|||
verbs: |
|||
- create |
|||
- get |
|||
- list |
|||
- watch |
|||
- apiGroups: |
|||
- cilium.io |
|||
resources: |
|||
- ciliumnetworkpolicies |
|||
verbs: |
|||
- "*" |
|||
@ -0,0 +1,15 @@ |
|||
--- |
|||
kind: ClusterRoleBinding |
|||
apiVersion: rbac.authorization.k8s.io/v1beta1 |
|||
metadata: |
|||
name: cilium |
|||
roleRef: |
|||
apiGroup: rbac.authorization.k8s.io |
|||
kind: ClusterRole |
|||
name: cilium |
|||
subjects: |
|||
- kind: ServiceAccount |
|||
name: cilium |
|||
namespace: {{ system_namespace }} |
|||
- kind: Group |
|||
name: system:nodes |
|||
@ -0,0 +1,163 @@ |
|||
--- |
|||
apiVersion: extensions/v1beta1 |
|||
kind: DaemonSet |
|||
metadata: |
|||
name: cilium |
|||
namespace: {{ system_namespace }} |
|||
spec: |
|||
template: |
|||
metadata: |
|||
labels: |
|||
k8s-app: cilium |
|||
kubernetes.io/cluster-service: "true" |
|||
annotations: |
|||
# This annotation plus the CriticalAddonsOnly toleration makes |
|||
# cilium to be a critical pod in the cluster, which ensures cilium |
|||
# gets priority scheduling. |
|||
# https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ |
|||
scheduler.alpha.kubernetes.io/critical-pod: '' |
|||
scheduler.alpha.kubernetes.io/tolerations: >- |
|||
[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}] |
|||
{% if cilium_enable_prometheus %} |
|||
prometheus.io/scrape: "true" |
|||
prometheus.io/port: "9090" |
|||
{% endif %} |
|||
spec: |
|||
{% if rbac_enabled %} |
|||
serviceAccountName: cilium |
|||
{% endif %} |
|||
containers: |
|||
- image: {{ cilium_image_repo }}:{{ cilium_image_tag }} |
|||
imagePullPolicy: Always |
|||
name: cilium-agent |
|||
command: [ "cilium-agent" ] |
|||
args: |
|||
- "--debug=$(CILIUM_DEBUG)" |
|||
- "-t" |
|||
- "vxlan" |
|||
- "--kvstore" |
|||
- "etcd" |
|||
- "--kvstore-opt" |
|||
- "etcd.config=/var/lib/etcd-config/etcd.config" |
|||
- "--disable-ipv4=$(DISABLE_IPV4)" |
|||
{% if cilium_enable_prometheus %} |
|||
ports: |
|||
- name: prometheus |
|||
containerPort: 9090 |
|||
{% endif %} |
|||
lifecycle: |
|||
postStart: |
|||
exec: |
|||
command: |
|||
- "/cni-install.sh" |
|||
preStop: |
|||
exec: |
|||
command: |
|||
- "/cni-uninstall.sh" |
|||
env: |
|||
- name: "K8S_NODE_NAME" |
|||
valueFrom: |
|||
fieldRef: |
|||
fieldPath: spec.nodeName |
|||
- name: "CILIUM_DEBUG" |
|||
valueFrom: |
|||
configMapKeyRef: |
|||
name: cilium-config |
|||
key: debug |
|||
- name: "DISABLE_IPV4" |
|||
valueFrom: |
|||
configMapKeyRef: |
|||
name: cilium-config |
|||
key: disable-ipv4 |
|||
{% if cilium_enable_prometheus %} |
|||
# Note: this variable is a no-op if not defined, and is used in the |
|||
# prometheus examples. |
|||
- name: "CILIUM_PROMETHEUS_SERVE_ADDR" |
|||
valueFrom: |
|||
configMapKeyRef: |
|||
name: cilium-metrics-config |
|||
optional: true |
|||
key: prometheus-serve-addr |
|||
{% endif %} |
|||
livenessProbe: |
|||
exec: |
|||
command: |
|||
- cilium |
|||
- status |
|||
# The initial delay for the liveness probe is intentionally large to |
|||
# avoid an endless kill & restart cycle if in the event that the initial |
|||
# bootstrapping takes longer than expected. |
|||
initialDelaySeconds: 120 |
|||
failureThreshold: 10 |
|||
periodSeconds: 10 |
|||
readinessProbe: |
|||
exec: |
|||
command: |
|||
- cilium |
|||
- status |
|||
initialDelaySeconds: 5 |
|||
periodSeconds: 5 |
|||
volumeMounts: |
|||
- name: bpf-maps |
|||
mountPath: /sys/fs/bpf |
|||
- name: cilium-run |
|||
mountPath: /var/run/cilium |
|||
- name: cni-path |
|||
mountPath: /host/opt/cni/bin |
|||
- name: etc-cni-netd |
|||
mountPath: /host/etc/cni/net.d |
|||
- name: docker-socket |
|||
mountPath: /var/run/docker.sock |
|||
readOnly: true |
|||
- name: etcd-config-path |
|||
mountPath: /var/lib/etcd-config |
|||
readOnly: true |
|||
- name: cilium-certs |
|||
mountPath: {{ cilium_cert_dir }} |
|||
readOnly: true |
|||
securityContext: |
|||
capabilities: |
|||
add: |
|||
- "NET_ADMIN" |
|||
privileged: true |
|||
hostNetwork: true |
|||
volumes: |
|||
# To keep state between restarts / upgrades |
|||
- name: cilium-run |
|||
hostPath: |
|||
path: /var/run/cilium |
|||
# To keep state between restarts / upgrades |
|||
- name: bpf-maps |
|||
hostPath: |
|||
path: /sys/fs/bpf |
|||
# To read docker events from the node |
|||
- name: docker-socket |
|||
hostPath: |
|||
path: /var/run/docker.sock |
|||
# To install cilium cni plugin in the host |
|||
- name: cni-path |
|||
hostPath: |
|||
path: /opt/cni/bin |
|||
# To install cilium cni configuration in the host |
|||
- name: etc-cni-netd |
|||
hostPath: |
|||
path: /etc/cni/net.d |
|||
- name: cilium-certs |
|||
hostPath: |
|||
path: {{ cilium_cert_dir }} |
|||
# To read the etcd config stored in config maps |
|||
- name: etcd-config-path |
|||
configMap: |
|||
name: cilium-config |
|||
items: |
|||
- key: etcd-config |
|||
path: etcd.config |
|||
tolerations: |
|||
- effect: NoSchedule |
|||
key: node-role.kubernetes.io/master |
|||
- effect: NoSchedule |
|||
key: node.cloudprovider.kubernetes.io/uninitialized |
|||
value: "true" |
|||
# Mark cilium's pod as critical for rescheduling |
|||
- key: CriticalAddonsOnly |
|||
operator: "Exists" |
|||
@ -0,0 +1,6 @@ |
|||
--- |
|||
apiVersion: v1 |
|||
kind: ServiceAccount |
|||
metadata: |
|||
name: cilium |
|||
namespace: {{ system_namespace }} |
|||
@ -0,0 +1,11 @@ |
|||
# Instance settings |
|||
cloud_image_family: centos-7 |
|||
cloud_region: us-central1-c |
|||
cloud_machine_type: "n1-standard-1" |
|||
mode: default |
|||
|
|||
# Deployment settings |
|||
kube_network_plugin: cilium |
|||
deploy_netchecker: true |
|||
kubedns_min_replicas: 1 |
|||
cloud_provider: gce |
|||
@ -0,0 +1,13 @@ |
|||
# Instance settings |
|||
cloud_image_family: coreos-stable |
|||
cloud_region: us-central1-c |
|||
mode: default |
|||
startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' |
|||
|
|||
# Deployment settings |
|||
kube_network_plugin: cilium |
|||
bootstrap_os: coreos |
|||
resolvconf_mode: host_resolvconf # this is required as long as the coreos stable channel uses docker < 1.12 |
|||
deploy_netchecker: true |
|||
kubedns_min_replicas: 1 |
|||
cloud_provider: gce |
|||
@ -0,0 +1,10 @@ |
|||
# Instance settings |
|||
cloud_image_family: rhel-7 |
|||
cloud_region: us-central1-b |
|||
mode: default |
|||
|
|||
# Deployment settings |
|||
kube_network_plugin: cilium |
|||
deploy_netchecker: true |
|||
kubedns_min_replicas: 1 |
|||
cloud_provider: gce |
|||
@ -0,0 +1,11 @@ |
|||
# Instance settings |
|||
cloud_image_family: ubuntu-1604-lts |
|||
cloud_region: us-central1-b |
|||
mode: separate |
|||
|
|||
# Deployment settings |
|||
kube_network_plugin: cilium |
|||
deploy_netchecker: true |
|||
kubedns_min_replicas: 1 |
|||
cloud_provider: gce |
|||
|
|||
Write
Preview
Loading…
Cancel
Save