From 19e1b11d98c471e0c1841a327fc4cfce10182b29 Mon Sep 17 00:00:00 2001 From: Vladimir Vasilkin Date: Wed, 28 Mar 2018 21:23:30 +0300 Subject: [PATCH 1/5] prometheus operator, metrics for k8s cluster install using Helm: - Prometheus Operator - metrics for k8s cluster including: grafana dashboard, alertmanager, node exporters base project: https://github.com/coreos/prometheus-operator the issue: https://github.com/kubernetes-incubator/kubespray/issues/2042 Previous PR, raw ansible without Helm: https://github.com/kubernetes-incubator/kubespray/pull/2499 --- inventory/sample/group_vars/k8s-cluster.yml | 8 +++++++- roles/kubernetes-apps/meta/main.yml | 6 ++++++ roles/kubernetes-apps/metrics/defaults/main.yml | 9 +++++++++ roles/kubernetes-apps/metrics/tasks/main.yml | 17 +++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 roles/kubernetes-apps/metrics/defaults/main.yml create mode 100644 roles/kubernetes-apps/metrics/tasks/main.yml diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 5f4889e8b..3e208bdaf 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -163,9 +163,15 @@ dashboard_enabled: true # Monitoring apps for k8s efk_enabled: false -# Helm deployment +# Helm deployment. Needs for Prometheus Operator, k8s metrics. helm_enabled: false +# Prometheus Operator. Needs for k8s metrics. Installed Helm is required. +prometheus_operator_enabled: false + +# K8s cluster metrics. Installed Helm and Prometheus Operator are required. +k8s_metrics_enabled: false + # Istio deployment istio_enabled: false diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index fca51a3b6..bc05e6f8c 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -27,6 +27,12 @@ dependencies: - apps - registry + - role: kubernetes-apps/metrics + when: prometheus_operator_enabled + tags: + - apps + - metrics + # istio role should be last because it takes a long time to initialize and # will cause timeouts trying to start other addons. - role: kubernetes-apps/istio diff --git a/roles/kubernetes-apps/metrics/defaults/main.yml b/roles/kubernetes-apps/metrics/defaults/main.yml new file mode 100644 index 000000000..72018e6f5 --- /dev/null +++ b/roles/kubernetes-apps/metrics/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# Prometheus Operator. Needs for k8s metrics. Installed Helm is required. +prometheus_operator_enabled: false + +# K8s cluster metrics. Installed Helm and Prometheus Operators are required. +k8s_metrics_enabled: false + +# Separate namespace for monitoring/metrics +monitoring_namespace: "monitoring" diff --git a/roles/kubernetes-apps/metrics/tasks/main.yml b/roles/kubernetes-apps/metrics/tasks/main.yml new file mode 100644 index 000000000..f64433c69 --- /dev/null +++ b/roles/kubernetes-apps/metrics/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Metrics | Make sure Helm is installed + command: "{{ bin_dir }}/helm version" + +- name: Metrics | Add coreos repo + command: "{{ bin_dir }}/helm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/" + run_once: true + +- name: Metrics | Install Prometheus Operator + command: "{{ bin_dir }}/helm upgrade --install prometheus-operator coreos/prometheus-operator --namespace {{ monitoring_namespace }}" + when: prometheus_operator_enabled + run_once: true + +- name: Metrics | Install K8s cluster metrics + command: "{{ bin_dir }}/helm upgrade --install kube-prometheus coreos/kube-prometheus --namespace {{ monitoring_namespace }}" + when: k8s_metrics_enabled + run_once: true From 23b3833806697c7488c1c6865e7ed4eb2e99db95 Mon Sep 17 00:00:00 2001 From: Vladimir Vasilkin Date: Thu, 29 Mar 2018 22:51:46 +0300 Subject: [PATCH 2/5] running on the first master only. --- roles/kubernetes-apps/metrics/tasks/main.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes-apps/metrics/tasks/main.yml b/roles/kubernetes-apps/metrics/tasks/main.yml index f64433c69..bda7d36e6 100644 --- a/roles/kubernetes-apps/metrics/tasks/main.yml +++ b/roles/kubernetes-apps/metrics/tasks/main.yml @@ -1,17 +1,25 @@ --- - name: Metrics | Make sure Helm is installed command: "{{ bin_dir }}/helm version" + when: + - inventory_hostname == groups['kube-master'][0] - name: Metrics | Add coreos repo command: "{{ bin_dir }}/helm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/" + when: + - inventory_hostname == groups['kube-master'][0] run_once: true - name: Metrics | Install Prometheus Operator command: "{{ bin_dir }}/helm upgrade --install prometheus-operator coreos/prometheus-operator --namespace {{ monitoring_namespace }}" - when: prometheus_operator_enabled + when: + - prometheus_operator_enabled + - inventory_hostname == groups['kube-master'][0] run_once: true - name: Metrics | Install K8s cluster metrics command: "{{ bin_dir }}/helm upgrade --install kube-prometheus coreos/kube-prometheus --namespace {{ monitoring_namespace }}" - when: k8s_metrics_enabled + when: + - k8s_metrics_enabled + - inventory_hostname == groups['kube-master'][0] run_once: true From 760ca1c3a956fb0033510633c0ea67b8786fcb41 Mon Sep 17 00:00:00 2001 From: Vladimir Vasilkin Date: Thu, 29 Mar 2018 23:03:43 +0300 Subject: [PATCH 3/5] adding checking for prometheus_operator_enabled --- roles/kubernetes-apps/metrics/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/kubernetes-apps/metrics/tasks/main.yml b/roles/kubernetes-apps/metrics/tasks/main.yml index bda7d36e6..ee858602b 100644 --- a/roles/kubernetes-apps/metrics/tasks/main.yml +++ b/roles/kubernetes-apps/metrics/tasks/main.yml @@ -2,11 +2,13 @@ - name: Metrics | Make sure Helm is installed command: "{{ bin_dir }}/helm version" when: + - prometheus_operator_enabled - inventory_hostname == groups['kube-master'][0] - name: Metrics | Add coreos repo command: "{{ bin_dir }}/helm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/" when: + - prometheus_operator_enabled - inventory_hostname == groups['kube-master'][0] run_once: true @@ -20,6 +22,7 @@ - name: Metrics | Install K8s cluster metrics command: "{{ bin_dir }}/helm upgrade --install kube-prometheus coreos/kube-prometheus --namespace {{ monitoring_namespace }}" when: + - prometheus_operator_enabled - k8s_metrics_enabled - inventory_hostname == groups['kube-master'][0] run_once: true From f0a04b4d65bdfe7c3c4a403eaa629c2446864c30 Mon Sep 17 00:00:00 2001 From: Vladimir Vasilkin Date: Fri, 30 Mar 2018 00:09:36 +0300 Subject: [PATCH 4/5] wait 5 * 4 secs until Tiller starts --- roles/kubernetes-apps/metrics/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/metrics/tasks/main.yml b/roles/kubernetes-apps/metrics/tasks/main.yml index ee858602b..e2280e98b 100644 --- a/roles/kubernetes-apps/metrics/tasks/main.yml +++ b/roles/kubernetes-apps/metrics/tasks/main.yml @@ -1,6 +1,10 @@ --- - name: Metrics | Make sure Helm is installed command: "{{ bin_dir }}/helm version" + register: helm_ready_result + until: helm_ready_result|succeeded + retries: 4 + delay: 5 when: - prometheus_operator_enabled - inventory_hostname == groups['kube-master'][0] @@ -21,7 +25,7 @@ - name: Metrics | Install K8s cluster metrics command: "{{ bin_dir }}/helm upgrade --install kube-prometheus coreos/kube-prometheus --namespace {{ monitoring_namespace }}" - when: + when: - prometheus_operator_enabled - k8s_metrics_enabled - inventory_hostname == groups['kube-master'][0] From 94a0562c93ac6e5a17fa1b6bf8caa44316d5500d Mon Sep 17 00:00:00 2001 From: Vladimir Vasilkin Date: Fri, 30 Mar 2018 12:29:04 +0300 Subject: [PATCH 5/5] adding prometheus_operator_enabled, k8s_metrics_enabled parameters to tests --- tests/files/gce_centos7-flannel-addons.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 8ac8a901b..467bee2d0 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -16,3 +16,5 @@ deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce kube_encrypt_secret_data: true +prometheus_operator_enabled: true +k8s_metrics_enabled: true