From c39835628d1df9c5fd5affb0a71e8d8cf69ad415 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Thu, 5 Jul 2018 02:14:36 +0200 Subject: [PATCH] prevent some race conditions, increase over all time limits --- .../heketi/roles/provision/tasks/kubernetes.yml | 14 +++++++++++--- .../heketi/roles/provision/tasks/setup/boot.yml | 4 ++-- .../heketi/roles/provision/tasks/setup/rest.yml | 4 ++-- .../heketi/roles/provision/tasks/setup/storage.yml | 8 +++++--- .../provision/tasks/setup/tear-down-bootstrap.yml | 2 +- .../roles/provision/tasks/setup/topology.yml | 8 +++----- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml index d78ce8f2e..813a434cf 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -6,9 +6,17 @@ when: "daemonset_state.stdout == \"\"" command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" - register: "daemonset_state" - command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" - changed_when: false -- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." } + command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" +- name: "Wait for daemonset to become available." + register: "daemonset_state" + command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + changed_when: false + vars: + ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" + desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" + until: "ready == desired" + retries: 60 + delay: 5 - name: "Label Gluster nodes" with_items: "{{ groups['heketi-node'] }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml index 3de6922f1..358881185 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -30,5 +30,5 @@ until: - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml index 120768fd2..1b0d475e2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml @@ -29,5 +29,5 @@ uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } register: "rest_hello_check" until: "rest_hello_check.content == \"Hello from Heketi\"" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index e0ae12246..e84303d3e 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -15,6 +15,8 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" +- command: "kubectl get secrets,endpoints,services,jobs --output=json" + register: "heketi_storage_state" - name: "Get state of heketi storage service, endpoint, secret and job." command: "kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false @@ -23,11 +25,11 @@ secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" - job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.active==0]" until: - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml index 019f92dfa..0fa3a1947 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml @@ -10,5 +10,5 @@ command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" register: "heketi_result" until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" - retries: 10 + retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml index cc73fd62a..378aa20de 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -13,8 +13,6 @@ - name: "Get heketi topology." register: "heketi_topology" command: "heketi-cli -s http://localhost:48080 topology info --json" -- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" } -- name: "Ensure heketi nodes are configured." - assert: - that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0" - msg: "Heketi topology missing." + until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" + retries: 60 + delay: 5