Browse Source
recator to use kube module, finally fix race condition in storage tasks
pull/2931/head
recator to use kube module, finally fix race condition in storage tasks
pull/2931/head
Sascha Marcel Schmidt
6 years ago
No known key found for this signature in database
GPG Key ID: D1D6CE184437796D
18 changed files with 242 additions and 151 deletions
Split View
Diff Options
-
16contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml
-
23contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
-
13contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml
-
0contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml
-
1contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
-
6contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml
-
37contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
-
0contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml
-
25contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
-
64contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml
-
45contrib/network-storage/heketi/roles/provision/tasks/main.yml
-
27contrib/network-storage/heketi/roles/provision/tasks/secret.yml
-
27contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml
-
10contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml
-
11contrib/network-storage/heketi/roles/provision/tasks/storage.yml
-
9contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
-
25contrib/network-storage/heketi/roles/provision/tasks/topology.yml
-
54contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2
@ -0,0 +1,23 @@ |
|||
--- |
|||
- name: "Kubernetes Apps | Lay Down Heketi Bootstrap" |
|||
become: true |
|||
template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" } |
|||
register: "rendering" |
|||
- name: "Kubernetes Apps | Install and configure Heketi Bootstrap" |
|||
kube: |
|||
name: "GlusterFS" |
|||
filename: "{{ kube_config_dir }}/heketi-bootstrap.json" |
|||
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
|||
- name: "Wait for heketi bootstrap to complete." |
|||
changed_when: false |
|||
register: "initial_heketi_state" |
|||
vars: |
|||
initial_heketi_state: { stdout: "{}" } |
|||
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
|||
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
|||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|||
until: |
|||
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
|||
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
|||
retries: 60 |
|||
delay: 5 |
@ -0,0 +1,37 @@ |
|||
--- |
|||
- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset" |
|||
template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" } |
|||
become: true |
|||
register: "rendering" |
|||
- name: "Kubernetes Apps | Install and configure GlusterFS daemonset" |
|||
kube: |
|||
name: "GlusterFS" |
|||
filename: "{{ kube_config_dir }}/glusterfs-daemonset.json" |
|||
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
|||
- name: "Kubernetes Apps | Wait for daemonset to become available." |
|||
register: "daemonset_state" |
|||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
|||
changed_when: false |
|||
vars: |
|||
daemonset_state: { stdout: "{}" } |
|||
ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" |
|||
desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" |
|||
until: "ready == desired" |
|||
retries: 60 |
|||
delay: 5 |
|||
|
|||
- name: "Kubernetes Apps | Label GlusterFS nodes" |
|||
include_tasks: "glusterfs/label.yml" |
|||
with_items: "{{ groups['heketi-node'] }}" |
|||
loop_control: |
|||
loop_var: "node" |
|||
|
|||
- name: "Kubernetes Apps | Lay Down Heketi Service Account" |
|||
template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" } |
|||
become: true |
|||
register: "rendering" |
|||
- name: "Kubernetes Apps | Install and configure Heketi Service Account" |
|||
kube: |
|||
name: "GlusterFS" |
|||
filename: "{{ kube_config_dir }}/heketi-service-account.json" |
|||
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
@ -0,0 +1,25 @@ |
|||
--- |
|||
- name: "Kubernetes Apps | Lay Down Heketi" |
|||
become: true |
|||
template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" } |
|||
register: "rendering" |
|||
- name: "Kubernetes Apps | Install and configure Heketi" |
|||
kube: |
|||
name: "GlusterFS" |
|||
filename: "{{ kube_config_dir }}/heketi-deployment.json" |
|||
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
|||
- name: "Ensure heketi is up and running." |
|||
changed_when: false |
|||
register: "heketi_state" |
|||
vars: |
|||
heketi_state: { stdout: "{}" } |
|||
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
|||
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
|||
command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json" |
|||
until: |
|||
- "heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
|||
- "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
|||
retries: 60 |
|||
delay: 5 |
|||
- set_fact: |
|||
heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}" |
@ -1,64 +0,0 @@ |
|||
--- |
|||
- register: "daemonset_state" |
|||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Deploy the GlusterFS DaemonSet" |
|||
when: "daemonset_state.stdout == \"\"" |
|||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" |
|||
- register: "daemonset_state" |
|||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Wait for daemonset to become available." |
|||
register: "daemonset_state" |
|||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
|||
changed_when: false |
|||
vars: |
|||
ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" |
|||
desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" |
|||
until: "ready == desired" |
|||
retries: 60 |
|||
delay: 5 |
|||
|
|||
- name: "Label Gluster nodes" |
|||
with_items: "{{ groups['heketi-node'] }}" |
|||
loop_control: |
|||
loop_var: "node" |
|||
include_tasks: "kubernetes/label.yml" |
|||
|
|||
- register: "service_account_state" |
|||
command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Deploy the Heketi service account" |
|||
when: "service_account_state.stdout == \"\"" |
|||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" |
|||
- register: "service_account_state" |
|||
command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } |
|||
|
|||
- register: "clusterrolebinding_state" |
|||
command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Deploy cluster role binding." |
|||
when: "clusterrolebinding_state.stdout == \"\"" |
|||
command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" |
|||
- register: "clusterrolebinding_state" |
|||
command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } |
|||
|
|||
- register: "secret_state" |
|||
command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Render Heketi secret configuration." |
|||
become: true |
|||
template: |
|||
src: "heketi.json.j2" |
|||
dest: "{{ kube_config_dir }}/heketi.json" |
|||
- name: "Deploy Heketi config secret" |
|||
when: "secret_state.stdout == \"\"" |
|||
command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" |
|||
- register: "secret_state" |
|||
command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } |
@ -1,34 +1,27 @@ |
|||
--- |
|||
- name: "Render configuration." |
|||
become: true |
|||
template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" } |
|||
with_items: |
|||
- { file: "glusterfs-daemonset.json" } |
|||
- { file: "heketi-bootstrap.json" } |
|||
- { file: "heketi-deployment.json" } |
|||
- { file: "heketi-service-account.json" } |
|||
- name: "Prepare kubernetes." |
|||
include_tasks: "kubernetes.yml" |
|||
- name: "Kubernetes Apps | GlusterFS" |
|||
include_tasks: "glusterfs.yml" |
|||
|
|||
- name: "Test heketi setup." |
|||
- name: "Kubernetes Apps | Heketi Secrets" |
|||
include_tasks: "secret.yml" |
|||
|
|||
- name: "Kubernetes Apps | Test Heketi" |
|||
register: "heketi_service_state" |
|||
command: "{{ bin_dir }}/kubectl get service heketi -o=name --ignore-not-found=true" |
|||
command: "kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
|
|||
- name: "Setup heketi." |
|||
- name: "Kubernetes Apps | Bootstrap Heketi" |
|||
when: "heketi_service_state.stdout == \"\"" |
|||
include_tasks: "setup.yml" |
|||
include_tasks: "bootstrap.yml" |
|||
|
|||
- name: "Test storage class." |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" |
|||
register: "storageclass" |
|||
- name: "Setup storage class." |
|||
when: "storageclass.stdout == \"\"" |
|||
- name: "Kubernetes Apps | Heketi" |
|||
include_tasks: "heketi.yml" |
|||
|
|||
- name: "Kubernetes Apps | Heketi Topology" |
|||
include_tasks: "topology.yml" |
|||
|
|||
- name: "Kubernetes Apps | Heketi Storage" |
|||
include_tasks: "storage.yml" |
|||
|
|||
- name: "Kubernetes Apps | Storage Class" |
|||
include_tasks: "storageclass.yml" |
|||
- name: "Test storage class." |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" |
|||
register: "storageclass" |
|||
- name: "Ensure storage class is up." |
|||
assert: { that: "storageclass.stdout != \"\"" } |
@ -0,0 +1,27 @@ |
|||
--- |
|||
- register: "clusterrolebinding_state" |
|||
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Kubernetes Apps | Deploy cluster role binding." |
|||
when: "clusterrolebinding_state.stdout == \"\"" |
|||
command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" |
|||
- register: "clusterrolebinding_state" |
|||
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } |
|||
|
|||
- register: "secret_state" |
|||
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Render Heketi secret configuration." |
|||
become: true |
|||
template: |
|||
src: "heketi.json.j2" |
|||
dest: "{{ kube_config_dir }}/heketi.json" |
|||
- name: "Deploy Heketi config secret" |
|||
when: "secret_state.stdout == \"\"" |
|||
command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" |
|||
- register: "secret_state" |
|||
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } |
@ -1,27 +0,0 @@ |
|||
--- |
|||
- name: "Get state of heketi service, deployment and pods." |
|||
register: "initial_heketi_state" |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|||
- name: "Create Heketi initial service and deployment" |
|||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" |
|||
when: |
|||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" |
|||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" |
|||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" |
|||
- name: "Get state of heketi service, deployment and pods." |
|||
register: "initial_heketi_state" |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|||
- name: "Wait for heketi bootstrap to complete." |
|||
changed_when: false |
|||
register: "initial_heketi_state" |
|||
vars: |
|||
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
|||
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
|||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|||
until: |
|||
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
|||
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
|||
retries: 60 |
|||
delay: 5 |
@ -1,10 +0,0 @@ |
|||
--- |
|||
- name: "Create long term Heketi instance." |
|||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" |
|||
- name: "Get heketi deployment state." |
|||
register: "heketi_deployment_state" |
|||
command: "{{ bin_dir }}/kubectl get deployment heketi -o=name --ignore-not-found=true" |
|||
changed_when: false |
|||
- name: "Ensure heketi is up and running." |
|||
assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } |
|||
|
@ -0,0 +1,11 @@ |
|||
--- |
|||
- name: "Kubernetes Apps | Lay Down Heketi Storage" |
|||
become: true |
|||
vars: { nodes: "{{ groups['heketi-node'] }}" } |
|||
template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" } |
|||
register: "rendering" |
|||
- name: "Kubernetes Apps | Install and configure Heketi Storage" |
|||
kube: |
|||
name: "GlusterFS" |
|||
filename: "{{ kube_config_dir }}/heketi-storage.json" |
|||
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
@ -0,0 +1,25 @@ |
|||
--- |
|||
- name: "Get heketi topology." |
|||
register: "heketi_topology" |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" |
|||
- name: "Render heketi topology template." |
|||
become: true |
|||
vars: { nodes: "{{ groups['heketi-node'] }}" } |
|||
register: "rendering" |
|||
template: |
|||
src: "topology.json.j2" |
|||
dest: "{{ kube_config_dir }}/topology.json" |
|||
- name: "Copy topology configuration into container." |
|||
when: "rendering.changed" |
|||
command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" |
|||
- name: "Load heketi topology." |
|||
when: "rendering.changed" |
|||
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" |
|||
- name: "Get heketi topology." |
|||
register: "heketi_topology" |
|||
changed_when: false |
|||
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" |
|||
until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" |
|||
retries: 60 |
|||
delay: 5 |
@ -0,0 +1,54 @@ |
|||
{ |
|||
"apiVersion": "v1", |
|||
"kind": "List", |
|||
"items": [ |
|||
{ |
|||
"kind": "Endpoints", |
|||
"apiVersion": "v1", |
|||
"metadata": { |
|||
"name": "heketi-storage-endpoints", |
|||
"creationTimestamp": null |
|||
}, |
|||
"subsets": [ |
|||
{% set nodeblocks = [] %} |
|||
{% for node in nodes %} |
|||
{% set nodeblock %} |
|||
{ |
|||
"addresses": [ |
|||
{ |
|||
"ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" |
|||
} |
|||
], |
|||
"ports": [ |
|||
{ |
|||
"port": 1 |
|||
} |
|||
] |
|||
} |
|||
{% endset %} |
|||
{% if nodeblocks.append(nodeblock) %}{% endif %} |
|||
{% endfor %} |
|||
{{ nodeblocks|join(',') }} |
|||
] |
|||
}, |
|||
{ |
|||
"kind": "Service", |
|||
"apiVersion": "v1", |
|||
"metadata": { |
|||
"name": "heketi-storage-endpoints", |
|||
"creationTimestamp": null |
|||
}, |
|||
"spec": { |
|||
"ports": [ |
|||
{ |
|||
"port": 1, |
|||
"targetPort": 0 |
|||
} |
|||
] |
|||
}, |
|||
"status": { |
|||
"loadBalancer": {} |
|||
} |
|||
} |
|||
] |
|||
} |
Write
Preview
Loading…
Cancel
Save