Browse Source
recator to use kube module, finally fix race condition in storage tasks
pull/2931/head
recator to use kube module, finally fix race condition in storage tasks
pull/2931/head
No known key found for this signature in database
GPG Key ID: D1D6CE184437796D
18 changed files with 242 additions and 151 deletions
Unified View
Diff Options
-
16contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml
-
23contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
-
13contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml
-
0contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml
-
1contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
-
6contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml
-
37contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
-
0contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml
-
25contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
-
64contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml
-
45contrib/network-storage/heketi/roles/provision/tasks/main.yml
-
27contrib/network-storage/heketi/roles/provision/tasks/secret.yml
-
27contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml
-
10contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml
-
11contrib/network-storage/heketi/roles/provision/tasks/storage.yml
-
9contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
-
25contrib/network-storage/heketi/roles/provision/tasks/topology.yml
-
54contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2
@ -0,0 +1,23 @@ |
|||||
|
--- |
||||
|
- name: "Kubernetes Apps | Lay Down Heketi Bootstrap" |
||||
|
become: true |
||||
|
template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" } |
||||
|
register: "rendering" |
||||
|
- name: "Kubernetes Apps | Install and configure Heketi Bootstrap" |
||||
|
kube: |
||||
|
name: "GlusterFS" |
||||
|
filename: "{{ kube_config_dir }}/heketi-bootstrap.json" |
||||
|
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
||||
|
- name: "Wait for heketi bootstrap to complete." |
||||
|
changed_when: false |
||||
|
register: "initial_heketi_state" |
||||
|
vars: |
||||
|
initial_heketi_state: { stdout: "{}" } |
||||
|
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
||||
|
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
||||
|
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
||||
|
until: |
||||
|
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
||||
|
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
||||
|
retries: 60 |
||||
|
delay: 5 |
@ -0,0 +1,37 @@ |
|||||
|
--- |
||||
|
- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset" |
||||
|
template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" } |
||||
|
become: true |
||||
|
register: "rendering" |
||||
|
- name: "Kubernetes Apps | Install and configure GlusterFS daemonset" |
||||
|
kube: |
||||
|
name: "GlusterFS" |
||||
|
filename: "{{ kube_config_dir }}/glusterfs-daemonset.json" |
||||
|
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
||||
|
- name: "Kubernetes Apps | Wait for daemonset to become available." |
||||
|
register: "daemonset_state" |
||||
|
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
vars: |
||||
|
daemonset_state: { stdout: "{}" } |
||||
|
ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" |
||||
|
desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" |
||||
|
until: "ready == desired" |
||||
|
retries: 60 |
||||
|
delay: 5 |
||||
|
|
||||
|
- name: "Kubernetes Apps | Label GlusterFS nodes" |
||||
|
include_tasks: "glusterfs/label.yml" |
||||
|
with_items: "{{ groups['heketi-node'] }}" |
||||
|
loop_control: |
||||
|
loop_var: "node" |
||||
|
|
||||
|
- name: "Kubernetes Apps | Lay Down Heketi Service Account" |
||||
|
template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" } |
||||
|
become: true |
||||
|
register: "rendering" |
||||
|
- name: "Kubernetes Apps | Install and configure Heketi Service Account" |
||||
|
kube: |
||||
|
name: "GlusterFS" |
||||
|
filename: "{{ kube_config_dir }}/heketi-service-account.json" |
||||
|
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
@ -0,0 +1,25 @@ |
|||||
|
--- |
||||
|
- name: "Kubernetes Apps | Lay Down Heketi" |
||||
|
become: true |
||||
|
template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" } |
||||
|
register: "rendering" |
||||
|
- name: "Kubernetes Apps | Install and configure Heketi" |
||||
|
kube: |
||||
|
name: "GlusterFS" |
||||
|
filename: "{{ kube_config_dir }}/heketi-deployment.json" |
||||
|
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
||||
|
- name: "Ensure heketi is up and running." |
||||
|
changed_when: false |
||||
|
register: "heketi_state" |
||||
|
vars: |
||||
|
heketi_state: { stdout: "{}" } |
||||
|
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
||||
|
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
||||
|
command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json" |
||||
|
until: |
||||
|
- "heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
||||
|
- "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
||||
|
retries: 60 |
||||
|
delay: 5 |
||||
|
- set_fact: |
||||
|
heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}" |
@ -1,64 +0,0 @@ |
|||||
--- |
|
||||
- register: "daemonset_state" |
|
||||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Deploy the GlusterFS DaemonSet" |
|
||||
when: "daemonset_state.stdout == \"\"" |
|
||||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" |
|
||||
- register: "daemonset_state" |
|
||||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Wait for daemonset to become available." |
|
||||
register: "daemonset_state" |
|
||||
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
vars: |
|
||||
ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" |
|
||||
desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" |
|
||||
until: "ready == desired" |
|
||||
retries: 60 |
|
||||
delay: 5 |
|
||||
|
|
||||
- name: "Label Gluster nodes" |
|
||||
with_items: "{{ groups['heketi-node'] }}" |
|
||||
loop_control: |
|
||||
loop_var: "node" |
|
||||
include_tasks: "kubernetes/label.yml" |
|
||||
|
|
||||
- register: "service_account_state" |
|
||||
command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Deploy the Heketi service account" |
|
||||
when: "service_account_state.stdout == \"\"" |
|
||||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" |
|
||||
- register: "service_account_state" |
|
||||
command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } |
|
||||
|
|
||||
- register: "clusterrolebinding_state" |
|
||||
command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Deploy cluster role binding." |
|
||||
when: "clusterrolebinding_state.stdout == \"\"" |
|
||||
command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" |
|
||||
- register: "clusterrolebinding_state" |
|
||||
command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } |
|
||||
|
|
||||
- register: "secret_state" |
|
||||
command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Render Heketi secret configuration." |
|
||||
become: true |
|
||||
template: |
|
||||
src: "heketi.json.j2" |
|
||||
dest: "{{ kube_config_dir }}/heketi.json" |
|
||||
- name: "Deploy Heketi config secret" |
|
||||
when: "secret_state.stdout == \"\"" |
|
||||
command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" |
|
||||
- register: "secret_state" |
|
||||
command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } |
|
@ -1,34 +1,27 @@ |
|||||
--- |
--- |
||||
- name: "Render configuration." |
|
||||
become: true |
|
||||
template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" } |
|
||||
with_items: |
|
||||
- { file: "glusterfs-daemonset.json" } |
|
||||
- { file: "heketi-bootstrap.json" } |
|
||||
- { file: "heketi-deployment.json" } |
|
||||
- { file: "heketi-service-account.json" } |
|
||||
- name: "Prepare kubernetes." |
|
||||
include_tasks: "kubernetes.yml" |
|
||||
|
- name: "Kubernetes Apps | GlusterFS" |
||||
|
include_tasks: "glusterfs.yml" |
||||
|
|
||||
- name: "Test heketi setup." |
|
||||
|
- name: "Kubernetes Apps | Heketi Secrets" |
||||
|
include_tasks: "secret.yml" |
||||
|
|
||||
|
- name: "Kubernetes Apps | Test Heketi" |
||||
register: "heketi_service_state" |
register: "heketi_service_state" |
||||
command: "{{ bin_dir }}/kubectl get service heketi -o=name --ignore-not-found=true" |
|
||||
|
command: "kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" |
||||
changed_when: false |
changed_when: false |
||||
|
|
||||
- name: "Setup heketi." |
|
||||
|
- name: "Kubernetes Apps | Bootstrap Heketi" |
||||
when: "heketi_service_state.stdout == \"\"" |
when: "heketi_service_state.stdout == \"\"" |
||||
include_tasks: "setup.yml" |
|
||||
|
include_tasks: "bootstrap.yml" |
||||
|
|
||||
- name: "Test storage class." |
|
||||
changed_when: false |
|
||||
command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" |
|
||||
register: "storageclass" |
|
||||
- name: "Setup storage class." |
|
||||
when: "storageclass.stdout == \"\"" |
|
||||
|
- name: "Kubernetes Apps | Heketi" |
||||
|
include_tasks: "heketi.yml" |
||||
|
|
||||
|
- name: "Kubernetes Apps | Heketi Topology" |
||||
|
include_tasks: "topology.yml" |
||||
|
|
||||
|
- name: "Kubernetes Apps | Heketi Storage" |
||||
|
include_tasks: "storage.yml" |
||||
|
|
||||
|
- name: "Kubernetes Apps | Storage Class" |
||||
include_tasks: "storageclass.yml" |
include_tasks: "storageclass.yml" |
||||
- name: "Test storage class." |
|
||||
changed_when: false |
|
||||
command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" |
|
||||
register: "storageclass" |
|
||||
- name: "Ensure storage class is up." |
|
||||
assert: { that: "storageclass.stdout != \"\"" } |
|
@ -0,0 +1,27 @@ |
|||||
|
--- |
||||
|
- register: "clusterrolebinding_state" |
||||
|
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Kubernetes Apps | Deploy cluster role binding." |
||||
|
when: "clusterrolebinding_state.stdout == \"\"" |
||||
|
command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" |
||||
|
- register: "clusterrolebinding_state" |
||||
|
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } |
||||
|
|
||||
|
- register: "secret_state" |
||||
|
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Render Heketi secret configuration." |
||||
|
become: true |
||||
|
template: |
||||
|
src: "heketi.json.j2" |
||||
|
dest: "{{ kube_config_dir }}/heketi.json" |
||||
|
- name: "Deploy Heketi config secret" |
||||
|
when: "secret_state.stdout == \"\"" |
||||
|
command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" |
||||
|
- register: "secret_state" |
||||
|
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } |
@ -1,27 +0,0 @@ |
|||||
--- |
|
||||
- name: "Get state of heketi service, deployment and pods." |
|
||||
register: "initial_heketi_state" |
|
||||
changed_when: false |
|
||||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|
||||
- name: "Create Heketi initial service and deployment" |
|
||||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" |
|
||||
when: |
|
||||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" |
|
||||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" |
|
||||
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" |
|
||||
- name: "Get state of heketi service, deployment and pods." |
|
||||
register: "initial_heketi_state" |
|
||||
changed_when: false |
|
||||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|
||||
- name: "Wait for heketi bootstrap to complete." |
|
||||
changed_when: false |
|
||||
register: "initial_heketi_state" |
|
||||
vars: |
|
||||
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
|
||||
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
|
||||
command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
|
||||
until: |
|
||||
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
|
||||
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
|
||||
retries: 60 |
|
||||
delay: 5 |
|
@ -1,10 +0,0 @@ |
|||||
--- |
|
||||
- name: "Create long term Heketi instance." |
|
||||
command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" |
|
||||
- name: "Get heketi deployment state." |
|
||||
register: "heketi_deployment_state" |
|
||||
command: "{{ bin_dir }}/kubectl get deployment heketi -o=name --ignore-not-found=true" |
|
||||
changed_when: false |
|
||||
- name: "Ensure heketi is up and running." |
|
||||
assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } |
|
||||
|
|
@ -0,0 +1,11 @@ |
|||||
|
--- |
||||
|
- name: "Kubernetes Apps | Lay Down Heketi Storage" |
||||
|
become: true |
||||
|
vars: { nodes: "{{ groups['heketi-node'] }}" } |
||||
|
template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" } |
||||
|
register: "rendering" |
||||
|
- name: "Kubernetes Apps | Install and configure Heketi Storage" |
||||
|
kube: |
||||
|
name: "GlusterFS" |
||||
|
filename: "{{ kube_config_dir }}/heketi-storage.json" |
||||
|
state: "{{ rendering.changed | ternary('latest', 'present') }}" |
@ -0,0 +1,25 @@ |
|||||
|
--- |
||||
|
- name: "Get heketi topology." |
||||
|
register: "heketi_topology" |
||||
|
changed_when: false |
||||
|
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" |
||||
|
- name: "Render heketi topology template." |
||||
|
become: true |
||||
|
vars: { nodes: "{{ groups['heketi-node'] }}" } |
||||
|
register: "rendering" |
||||
|
template: |
||||
|
src: "topology.json.j2" |
||||
|
dest: "{{ kube_config_dir }}/topology.json" |
||||
|
- name: "Copy topology configuration into container." |
||||
|
when: "rendering.changed" |
||||
|
command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" |
||||
|
- name: "Load heketi topology." |
||||
|
when: "rendering.changed" |
||||
|
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" |
||||
|
- name: "Get heketi topology." |
||||
|
register: "heketi_topology" |
||||
|
changed_when: false |
||||
|
command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" |
||||
|
until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" |
||||
|
retries: 60 |
||||
|
delay: 5 |
@ -0,0 +1,54 @@ |
|||||
|
{ |
||||
|
"apiVersion": "v1", |
||||
|
"kind": "List", |
||||
|
"items": [ |
||||
|
{ |
||||
|
"kind": "Endpoints", |
||||
|
"apiVersion": "v1", |
||||
|
"metadata": { |
||||
|
"name": "heketi-storage-endpoints", |
||||
|
"creationTimestamp": null |
||||
|
}, |
||||
|
"subsets": [ |
||||
|
{% set nodeblocks = [] %} |
||||
|
{% for node in nodes %} |
||||
|
{% set nodeblock %} |
||||
|
{ |
||||
|
"addresses": [ |
||||
|
{ |
||||
|
"ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" |
||||
|
} |
||||
|
], |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"port": 1 |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
{% endset %} |
||||
|
{% if nodeblocks.append(nodeblock) %}{% endif %} |
||||
|
{% endfor %} |
||||
|
{{ nodeblocks|join(',') }} |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"kind": "Service", |
||||
|
"apiVersion": "v1", |
||||
|
"metadata": { |
||||
|
"name": "heketi-storage-endpoints", |
||||
|
"creationTimestamp": null |
||||
|
}, |
||||
|
"spec": { |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"port": 1, |
||||
|
"targetPort": 0 |
||||
|
} |
||||
|
] |
||||
|
}, |
||||
|
"status": { |
||||
|
"loadBalancer": {} |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
Write
Preview
Loading…
Cancel
Save