Sascha Marcel Schmidt
7 years ago
No known key found for this signature in database
GPG Key ID: D1D6CE184437796D
23 changed files with 896 additions and 0 deletions
Unified View
Diff Options
-
10contrib/network-storage/heketi/roles/heketi.yml
-
16contrib/network-storage/heketi/roles/prepare/tasks/main.yml
-
2contrib/network-storage/heketi/roles/provision/defaults/main.yml
-
144contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json
-
3contrib/network-storage/heketi/roles/provision/handlers/main.yml
-
133contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json
-
159contrib/network-storage/heketi/roles/provision/heketi-deployment.json
-
7contrib/network-storage/heketi/roles/provision/heketi-service-account.json
-
54contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml
-
11contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml
-
26contrib/network-storage/heketi/roles/provision/tasks/main.yml
-
51contrib/network-storage/heketi/roles/provision/tasks/setup.yml
-
34contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml
-
10contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml
-
38contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml
-
36contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml
-
14contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml
-
20contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml
-
20contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml
-
20contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
-
44contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2
-
10contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2
-
34contrib/network-storage/heketi/roles/provision/templates/topology.json.j2
@ -0,0 +1,10 @@ |
|||||
|
--- |
||||
|
- hosts: heketi-node |
||||
|
roles: |
||||
|
- { role: prepare } |
||||
|
|
||||
|
- hosts: localhost |
||||
|
tags: |
||||
|
- "provision" |
||||
|
roles: |
||||
|
- { role: provision } |
||||
@ -0,0 +1,16 @@ |
|||||
|
--- |
||||
|
- name: "Load lvm kernel modules" |
||||
|
become: true |
||||
|
with_items: |
||||
|
- "dm_snapshot" |
||||
|
- "dm_mirror" |
||||
|
- "dm_thin_pool" |
||||
|
modprobe: |
||||
|
name: "{{ item }}" |
||||
|
state: "present" |
||||
|
|
||||
|
- name: "Install glusterfs mount utils" |
||||
|
become: true |
||||
|
yum: |
||||
|
name: "glusterfs-fuse" |
||||
|
state: "present" |
||||
@ -0,0 +1,2 @@ |
|||||
|
--- |
||||
|
artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts" |
||||
@ -0,0 +1,144 @@ |
|||||
|
{ |
||||
|
"kind": "DaemonSet", |
||||
|
"apiVersion": "extensions/v1beta1", |
||||
|
"metadata": { |
||||
|
"name": "glusterfs", |
||||
|
"labels": { |
||||
|
"glusterfs": "deployment" |
||||
|
}, |
||||
|
"annotations": { |
||||
|
"description": "GlusterFS Daemon Set", |
||||
|
"tags": "glusterfs" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"template": { |
||||
|
"metadata": { |
||||
|
"name": "glusterfs", |
||||
|
"labels": { |
||||
|
"glusterfs-node": "daemonset" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"nodeSelector": { |
||||
|
"storagenode" : "glusterfs" |
||||
|
}, |
||||
|
"hostNetwork": true, |
||||
|
"containers": [ |
||||
|
{ |
||||
|
"image": "gluster/gluster-centos:latest", |
||||
|
"imagePullPolicy": "Always", |
||||
|
"name": "glusterfs", |
||||
|
"volumeMounts": [ |
||||
|
{ |
||||
|
"name": "glusterfs-heketi", |
||||
|
"mountPath": "/var/lib/heketi" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-run", |
||||
|
"mountPath": "/run" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-lvm", |
||||
|
"mountPath": "/run/lvm" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-etc", |
||||
|
"mountPath": "/etc/glusterfs" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-logs", |
||||
|
"mountPath": "/var/log/glusterfs" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-config", |
||||
|
"mountPath": "/var/lib/glusterd" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-dev", |
||||
|
"mountPath": "/dev" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-cgroup", |
||||
|
"mountPath": "/sys/fs/cgroup" |
||||
|
} |
||||
|
], |
||||
|
"securityContext": { |
||||
|
"capabilities": {}, |
||||
|
"privileged": true |
||||
|
}, |
||||
|
"readinessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 60, |
||||
|
"exec": { |
||||
|
"command": [ |
||||
|
"/bin/bash", |
||||
|
"-c", |
||||
|
"systemctl status glusterd.service" |
||||
|
] |
||||
|
} |
||||
|
}, |
||||
|
"livenessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 60, |
||||
|
"exec": { |
||||
|
"command": [ |
||||
|
"/bin/bash", |
||||
|
"-c", |
||||
|
"systemctl status glusterd.service" |
||||
|
] |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
], |
||||
|
"volumes": [ |
||||
|
{ |
||||
|
"name": "glusterfs-heketi", |
||||
|
"hostPath": { |
||||
|
"path": "/var/lib/heketi" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-run" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-lvm", |
||||
|
"hostPath": { |
||||
|
"path": "/run/lvm" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-etc", |
||||
|
"hostPath": { |
||||
|
"path": "/etc/glusterfs" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-logs", |
||||
|
"hostPath": { |
||||
|
"path": "/var/log/glusterfs" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-config", |
||||
|
"hostPath": { |
||||
|
"path": "/var/lib/glusterd" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-dev", |
||||
|
"hostPath": { |
||||
|
"path": "/dev" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "glusterfs-cgroup", |
||||
|
"hostPath": { |
||||
|
"path": "/sys/fs/cgroup" |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,3 @@ |
|||||
|
--- |
||||
|
- name: "stop port forwarding" |
||||
|
command: "killall " |
||||
@ -0,0 +1,133 @@ |
|||||
|
{ |
||||
|
"kind": "List", |
||||
|
"apiVersion": "v1", |
||||
|
"items": [ |
||||
|
{ |
||||
|
"kind": "Service", |
||||
|
"apiVersion": "v1", |
||||
|
"metadata": { |
||||
|
"name": "deploy-heketi", |
||||
|
"labels": { |
||||
|
"glusterfs": "heketi-service", |
||||
|
"deploy-heketi": "support" |
||||
|
}, |
||||
|
"annotations": { |
||||
|
"description": "Exposes Heketi Service" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"selector": { |
||||
|
"name": "deploy-heketi" |
||||
|
}, |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"name": "deploy-heketi", |
||||
|
"port": 8080, |
||||
|
"targetPort": 8080 |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"kind": "Deployment", |
||||
|
"apiVersion": "extensions/v1beta1", |
||||
|
"metadata": { |
||||
|
"name": "deploy-heketi", |
||||
|
"labels": { |
||||
|
"glusterfs": "heketi-deployment", |
||||
|
"deploy-heketi": "deployment" |
||||
|
}, |
||||
|
"annotations": { |
||||
|
"description": "Defines how to deploy Heketi" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"replicas": 1, |
||||
|
"template": { |
||||
|
"metadata": { |
||||
|
"name": "deploy-heketi", |
||||
|
"labels": { |
||||
|
"name": "deploy-heketi", |
||||
|
"glusterfs": "heketi-pod", |
||||
|
"deploy-heketi": "pod" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"serviceAccountName": "heketi-service-account", |
||||
|
"containers": [ |
||||
|
{ |
||||
|
"image": "heketi/heketi:dev", |
||||
|
"imagePullPolicy": "Always", |
||||
|
"name": "deploy-heketi", |
||||
|
"env": [ |
||||
|
{ |
||||
|
"name": "HEKETI_EXECUTOR", |
||||
|
"value": "kubernetes" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_DB_PATH", |
||||
|
"value": "/var/lib/heketi/heketi.db" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_FSTAB", |
||||
|
"value": "/var/lib/heketi/fstab" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_SNAPSHOT_LIMIT", |
||||
|
"value": "14" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET", |
||||
|
"value": "y" |
||||
|
} |
||||
|
], |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"containerPort": 8080 |
||||
|
} |
||||
|
], |
||||
|
"volumeMounts": [ |
||||
|
{ |
||||
|
"name": "db", |
||||
|
"mountPath": "/var/lib/heketi" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "config", |
||||
|
"mountPath": "/etc/heketi" |
||||
|
} |
||||
|
], |
||||
|
"readinessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 3, |
||||
|
"httpGet": { |
||||
|
"path": "/hello", |
||||
|
"port": 8080 |
||||
|
} |
||||
|
}, |
||||
|
"livenessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 30, |
||||
|
"httpGet": { |
||||
|
"path": "/hello", |
||||
|
"port": 8080 |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
], |
||||
|
"volumes": [ |
||||
|
{ |
||||
|
"name": "db" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "config", |
||||
|
"secret": { |
||||
|
"secretName": "heketi-config-secret" |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
||||
@ -0,0 +1,159 @@ |
|||||
|
{ |
||||
|
"kind": "List", |
||||
|
"apiVersion": "v1", |
||||
|
"items": [ |
||||
|
{ |
||||
|
"kind": "Secret", |
||||
|
"apiVersion": "v1", |
||||
|
"metadata": { |
||||
|
"name": "heketi-db-backup", |
||||
|
"labels": { |
||||
|
"glusterfs": "heketi-db", |
||||
|
"heketi": "db" |
||||
|
} |
||||
|
}, |
||||
|
"data": { |
||||
|
}, |
||||
|
"type": "Opaque" |
||||
|
}, |
||||
|
{ |
||||
|
"kind": "Service", |
||||
|
"apiVersion": "v1", |
||||
|
"metadata": { |
||||
|
"name": "heketi", |
||||
|
"labels": { |
||||
|
"glusterfs": "heketi-service", |
||||
|
"deploy-heketi": "support" |
||||
|
}, |
||||
|
"annotations": { |
||||
|
"description": "Exposes Heketi Service" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"selector": { |
||||
|
"name": "heketi" |
||||
|
}, |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"name": "heketi", |
||||
|
"port": 8080, |
||||
|
"targetPort": 8080 |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"kind": "Deployment", |
||||
|
"apiVersion": "extensions/v1beta1", |
||||
|
"metadata": { |
||||
|
"name": "heketi", |
||||
|
"labels": { |
||||
|
"glusterfs": "heketi-deployment" |
||||
|
}, |
||||
|
"annotations": { |
||||
|
"description": "Defines how to deploy Heketi" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"replicas": 1, |
||||
|
"template": { |
||||
|
"metadata": { |
||||
|
"name": "heketi", |
||||
|
"labels": { |
||||
|
"name": "heketi", |
||||
|
"glusterfs": "heketi-pod" |
||||
|
} |
||||
|
}, |
||||
|
"spec": { |
||||
|
"serviceAccountName": "heketi-service-account", |
||||
|
"containers": [ |
||||
|
{ |
||||
|
"image": "heketi/heketi:dev", |
||||
|
"imagePullPolicy": "Always", |
||||
|
"name": "heketi", |
||||
|
"env": [ |
||||
|
{ |
||||
|
"name": "HEKETI_EXECUTOR", |
||||
|
"value": "kubernetes" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_DB_PATH", |
||||
|
"value": "/var/lib/heketi/heketi.db" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_FSTAB", |
||||
|
"value": "/var/lib/heketi/fstab" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_SNAPSHOT_LIMIT", |
||||
|
"value": "14" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET", |
||||
|
"value": "y" |
||||
|
} |
||||
|
], |
||||
|
"ports": [ |
||||
|
{ |
||||
|
"containerPort": 8080 |
||||
|
} |
||||
|
], |
||||
|
"volumeMounts": [ |
||||
|
{ |
||||
|
"mountPath": "/backupdb", |
||||
|
"name": "heketi-db-secret" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "db", |
||||
|
"mountPath": "/var/lib/heketi" |
||||
|
}, |
||||
|
{ |
||||
|
"name": "config", |
||||
|
"mountPath": "/etc/heketi" |
||||
|
} |
||||
|
], |
||||
|
"readinessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 3, |
||||
|
"httpGet": { |
||||
|
"path": "/hello", |
||||
|
"port": 8080 |
||||
|
} |
||||
|
}, |
||||
|
"livenessProbe": { |
||||
|
"timeoutSeconds": 3, |
||||
|
"initialDelaySeconds": 30, |
||||
|
"httpGet": { |
||||
|
"path": "/hello", |
||||
|
"port": 8080 |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
], |
||||
|
"volumes": [ |
||||
|
{ |
||||
|
"name": "db", |
||||
|
"glusterfs": { |
||||
|
"endpoints": "heketi-storage-endpoints", |
||||
|
"path": "heketidbstorage" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "heketi-db-secret", |
||||
|
"secret": { |
||||
|
"secretName": "heketi-db-backup" |
||||
|
} |
||||
|
}, |
||||
|
{ |
||||
|
"name": "config", |
||||
|
"secret": { |
||||
|
"secretName": "heketi-config-secret" |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
] |
||||
|
} |
||||
@ -0,0 +1,7 @@ |
|||||
|
{ |
||||
|
"apiVersion": "v1", |
||||
|
"kind": "ServiceAccount", |
||||
|
"metadata": { |
||||
|
"name": "heketi-service-account" |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,54 @@ |
|||||
|
--- |
||||
|
- register: "daemonset_state" |
||||
|
command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Deploy the GlusterFS DaemonSet" |
||||
|
when: "daemonset_state.stdout == \"\"" |
||||
|
command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" |
||||
|
- register: "daemonset_state" |
||||
|
command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." } |
||||
|
|
||||
|
- name: "Label Gluster nodes" |
||||
|
with_items: "{{ groups['heketi-node'] }}" |
||||
|
loop_control: |
||||
|
loop_var: "node" |
||||
|
include_tasks: "kubernetes/label.yml" |
||||
|
|
||||
|
- register: "service_account_state" |
||||
|
command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Deploy the Heketi service account" |
||||
|
when: "service_account_state.stdout == \"\"" |
||||
|
command: "kubectl create -f {{ role_path }}/heketi-service-account.json" |
||||
|
- register: "service_account_state" |
||||
|
command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } |
||||
|
|
||||
|
- register: "clusterrolebinding_state" |
||||
|
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Deploy cluster role binding." |
||||
|
when: "clusterrolebinding_state.stdout == \"\"" |
||||
|
command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" |
||||
|
- register: "clusterrolebinding_state" |
||||
|
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } |
||||
|
|
||||
|
- register: "secret_state" |
||||
|
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Render Heketi secret configuration." |
||||
|
template: |
||||
|
src: "heketi.json.j2" |
||||
|
dest: "{{ artifacts_dir }}/heketi.json" |
||||
|
- name: "Deploy Heketi config secret" |
||||
|
when: "secret_state.stdout == \"\"" |
||||
|
command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json" |
||||
|
- register: "secret_state" |
||||
|
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } |
||||
@ -0,0 +1,11 @@ |
|||||
|
--- |
||||
|
- register: "label_present" |
||||
|
command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Assign storage label" |
||||
|
when: "label_present.stdout_lines|length == 0" |
||||
|
command: "kubectl label node {{ node }} storagenode=glusterfs" |
||||
|
- register: "label_present" |
||||
|
command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." } |
||||
@ -0,0 +1,26 @@ |
|||||
|
--- |
||||
|
- name: "Prepare kubernetes." |
||||
|
include_tasks: "kubernetes.yml" |
||||
|
|
||||
|
- name: "Test heketi setup." |
||||
|
register: "heketi_service_state" |
||||
|
command: "kubectl get service heketi -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
|
||||
|
- name: "Setup heketi." |
||||
|
when: "heketi_service_state.stdout == \"\"" |
||||
|
include_tasks: "setup.yml" |
||||
|
|
||||
|
- name: "Test storage class." |
||||
|
changed_when: false |
||||
|
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" |
||||
|
register: "storageclass" |
||||
|
- name: "Setup storage class." |
||||
|
when: "storageclass.stdout == \"\"" |
||||
|
include_tasks: "storageclass.yml" |
||||
|
- name: "Test storage class." |
||||
|
changed_when: false |
||||
|
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" |
||||
|
register: "storageclass" |
||||
|
- name: "Ensure storage class is up." |
||||
|
assert: { that: "storageclass.stdout != \"\"" } |
||||
@ -0,0 +1,51 @@ |
|||||
|
--- |
||||
|
- name: "Test REST endpoint." |
||||
|
uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } |
||||
|
register: "rest_hello_check" |
||||
|
ignore_errors: true |
||||
|
|
||||
|
# Bootstrap heketi |
||||
|
- name: "Bootstrap heketi and start REST endpoint." |
||||
|
when: "rest_hello_check.content != \"Hello from Heketi\"" |
||||
|
include_tasks: "{{ item }}" |
||||
|
with_items: [ "setup/boot.yml", "setup/rest.yml" ] |
||||
|
- name: "Bootstrap heketi." |
||||
|
when: "rest_hello_check.content == \"Hello from Heketi\"" |
||||
|
include_tasks: "setup/boot.yml" |
||||
|
|
||||
|
# Prepare heketi topology |
||||
|
- name: "Test heketi topology." |
||||
|
changed_when: false |
||||
|
register: "heketi_topology" |
||||
|
command: "heketi-cli -s http://localhost:48080 topology info --json" |
||||
|
- name: "Load heketi topology." |
||||
|
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" |
||||
|
include_tasks: "setup/topology.yml" |
||||
|
|
||||
|
# Provision heketi database volume |
||||
|
- name: "Prepare heketi volumes." |
||||
|
include_tasks: "setup/volumes.yml" |
||||
|
|
||||
|
# Prepare heketi storage |
||||
|
- name: "Test heketi storage." |
||||
|
command: "kubectl get secrets,endpoints,services,jobs --output=json" |
||||
|
changed_when: false |
||||
|
register: "heketi_storage_state" |
||||
|
- name: "Create heketi storage." |
||||
|
include_tasks: "setup/storage.yml" |
||||
|
vars: |
||||
|
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" |
||||
|
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" |
||||
|
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" |
||||
|
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" |
||||
|
when: |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" |
||||
|
|
||||
|
# Finalize setup |
||||
|
- name: "Tear down bootstrap." |
||||
|
include_tasks: "setup/tear-down-bootstrap.yml" |
||||
|
- name: "Setup final heketi instance." |
||||
|
include_tasks: "setup/heketi.yml" |
||||
@ -0,0 +1,34 @@ |
|||||
|
--- |
||||
|
- name: "Get state of heketi service, deployment and pods." |
||||
|
register: "initial_heketi_state" |
||||
|
changed_when: false |
||||
|
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
||||
|
- name: "Create Heketi initial service and deployment" |
||||
|
command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json" |
||||
|
when: |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" |
||||
|
- name: "Get state of heketi service, deployment and pods." |
||||
|
register: "initial_heketi_state" |
||||
|
changed_when: false |
||||
|
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
||||
|
- name: "Ensure heketi bootstrap environment exists." |
||||
|
assert: |
||||
|
that: |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'" |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'" |
||||
|
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'" |
||||
|
msg: "Heketi deployment did not succeed." |
||||
|
- name: "Wait for heketi bootstrap to complete." |
||||
|
changed_when: false |
||||
|
register: "initial_heketi_state" |
||||
|
vars: |
||||
|
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" |
||||
|
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" |
||||
|
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" |
||||
|
until: |
||||
|
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" |
||||
|
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" |
||||
|
retries: 10 |
||||
|
delay: 10 |
||||
@ -0,0 +1,10 @@ |
|||||
|
--- |
||||
|
- name: "Create long term Heketi instance." |
||||
|
command: "kubectl create -f {{ role_path }}/heketi-deployment.json" |
||||
|
- name: "Get heketi deployment state." |
||||
|
register: "heketi_deployment_state" |
||||
|
command: "kubectl get deployment heketi -o=name --ignore-not-found=true" |
||||
|
changed_when: false |
||||
|
- name: "Ensure heketi is up and running." |
||||
|
assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } |
||||
|
|
||||
@ -0,0 +1,38 @@ |
|||||
|
--- |
||||
|
# Enable local REST-Interface |
||||
|
- name: "Get heketi initial pod state." |
||||
|
tags: ["test"] |
||||
|
register: "initial_heketi_pod" |
||||
|
command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" |
||||
|
changed_when: false |
||||
|
- name: "Ensure heketi bootstrap pod is up." |
||||
|
tags: ["test"] |
||||
|
assert: |
||||
|
that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" |
||||
|
- name: "Temporarily enable local port forwarding to heketi REST interface" |
||||
|
tags: ["test"] |
||||
|
vars: |
||||
|
initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" |
||||
|
command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080" |
||||
|
async: 180 |
||||
|
poll: 0 |
||||
|
ignore_errors: "yes" |
||||
|
register: "heketi_port_forwarding" |
||||
|
changed_when: false |
||||
|
- name: "Ensure port forwarding is enabled." |
||||
|
tags: ["test"] |
||||
|
retries: 10 |
||||
|
delay: 5 |
||||
|
assert: |
||||
|
that: |
||||
|
- "heketi_port_forwarding.finished == 0" |
||||
|
- "heketi_port_forwarding.started == 1" |
||||
|
- "heketi_port_forwarding.failed == false" |
||||
|
msg: "Port forwarding does not work." |
||||
|
- name: "Test REST endpoint." |
||||
|
tags: ["test"] |
||||
|
uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } |
||||
|
register: "rest_hello_check" |
||||
|
until: "rest_hello_check.content == \"Hello from Heketi\"" |
||||
|
retries: 10 |
||||
|
delay: 10 |
||||
@ -0,0 +1,36 @@ |
|||||
|
--- |
||||
|
- name: "Test heketi storage." |
||||
|
tags: ["test"] |
||||
|
command: "kubectl get secrets,endpoints,services,jobs --output=json" |
||||
|
changed_when: false |
||||
|
register: "heketi_storage_state" |
||||
|
- name: "Create heketi storage." |
||||
|
tags: ["test"] |
||||
|
command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" |
||||
|
vars: |
||||
|
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" |
||||
|
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" |
||||
|
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" |
||||
|
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" |
||||
|
when: |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" |
||||
|
- name: "Get state of heketi storage service, endpoint, secret and job." |
||||
|
tags: ["test"] |
||||
|
command: "kubectl get secrets,endpoints,services,jobs --output=json" |
||||
|
changed_when: false |
||||
|
register: "heketi_storage_state" |
||||
|
vars: |
||||
|
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" |
||||
|
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" |
||||
|
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" |
||||
|
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status" |
||||
|
until: |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" |
||||
|
- "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'" |
||||
|
retries: 10 |
||||
|
delay: 10 |
||||
@ -0,0 +1,14 @@ |
|||||
|
--- |
||||
|
- name: "Get existing Heketi deploy resources." |
||||
|
command: "kubectl get all --selector=\"deploy-heketi\" -o=json" |
||||
|
register: "heketi_resources" |
||||
|
changed_when: false |
||||
|
- name: "Delete bootstrap Heketi." |
||||
|
command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" |
||||
|
when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0" |
||||
|
- name: "Ensure there is nothing left over." |
||||
|
command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" |
||||
|
register: "heketi_result" |
||||
|
until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" |
||||
|
retries: 10 |
||||
|
delay: 5 |
||||
@ -0,0 +1,20 @@ |
|||||
|
--- |
||||
|
- name: "Get heketi topology." |
||||
|
register: "heketi_topology" |
||||
|
command: "heketi-cli -s http://localhost:48080 topology info --json" |
||||
|
- name: "Render heketi topology template." |
||||
|
vars: { nodes: "{{ groups['heketi-node'] }}" } |
||||
|
template: |
||||
|
src: "topology.json.j2" |
||||
|
dest: "{{ artifacts_dir }}/topology.json" |
||||
|
- name: "Load heketi topology." |
||||
|
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" |
||||
|
command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json" |
||||
|
- name: "Get heketi topology." |
||||
|
register: "heketi_topology" |
||||
|
command: "heketi-cli -s http://localhost:48080 topology info --json" |
||||
|
- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" } |
||||
|
- name: "Ensure heketi nodes are configured." |
||||
|
assert: |
||||
|
that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0" |
||||
|
msg: "Heketi topology missing." |
||||
@ -0,0 +1,20 @@ |
|||||
|
--- |
||||
|
- name: "Get heketi volume ids." |
||||
|
command: "heketi-cli -s http://localhost:48080 volume list --json" |
||||
|
changed_when: false |
||||
|
register: "heketi_volumes" |
||||
|
- name: "Get heketi volumes." |
||||
|
changed_when: false |
||||
|
command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" |
||||
|
with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" |
||||
|
loop_control: { loop_var: "volume_id" } |
||||
|
register: "volumes_information" |
||||
|
- name: "Test heketi database volume." |
||||
|
set_fact: { heketi_database_volume_exists: true } |
||||
|
with_items: "{{ volumes_information.results }}" |
||||
|
loop_control: { loop_var: "volume_information" } |
||||
|
vars: { volume: "{{ volume_information.stdout|from_json }}" } |
||||
|
when: "volume.name == 'heketidbstorage'" |
||||
|
- name: "Provision database volume." |
||||
|
command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" |
||||
|
when: "heketi_database_volume_exists != true" |
||||
@ -0,0 +1,20 @@ |
|||||
|
--- |
||||
|
- name: "Test storage class." |
||||
|
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" |
||||
|
register: "storageclass" |
||||
|
changed_when: false |
||||
|
- name: "Test heketi service." |
||||
|
command: "kubectl get service heketi --ignore-not-found=true --output=json" |
||||
|
register: "heketi_service" |
||||
|
changed_when: false |
||||
|
- name: "Ensure heketi service is available." |
||||
|
assert: { that: "heketi_service.stdout != \"\"" } |
||||
|
- name: "Render storage class configuration." |
||||
|
vars: |
||||
|
endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}" |
||||
|
template: |
||||
|
src: "storageclass.yml.j2" |
||||
|
dest: "{{ artifacts_dir }}/storageclass.yml" |
||||
|
- name: "Setup storage class." |
||||
|
when: "storageclass.stdout == \"\"" |
||||
|
command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml" |
||||
@ -0,0 +1,44 @@ |
|||||
|
{ |
||||
|
"_port_comment": "Heketi Server Port Number", |
||||
|
"port": "8080", |
||||
|
|
||||
|
"_use_auth": "Enable JWT authorization. Please enable for deployment", |
||||
|
"use_auth": false, |
||||
|
|
||||
|
"_jwt": "Private keys for access", |
||||
|
"jwt": { |
||||
|
"_admin": "Admin has access to all APIs", |
||||
|
"admin": { |
||||
|
"key": "{{ heketi_admin_key }}" |
||||
|
}, |
||||
|
"_user": "User only has access to /volumes endpoint", |
||||
|
"user": { |
||||
|
"key": "{{ heketi_user_key }}" |
||||
|
} |
||||
|
}, |
||||
|
|
||||
|
"_glusterfs_comment": "GlusterFS Configuration", |
||||
|
"glusterfs": { |
||||
|
"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh", |
||||
|
"executor": "kubernetes", |
||||
|
|
||||
|
"_db_comment": "Database file name", |
||||
|
"db": "/var/lib/heketi/heketi.db", |
||||
|
|
||||
|
"kubeexec": { |
||||
|
"rebalance_on_expansion": true |
||||
|
}, |
||||
|
|
||||
|
"sshexec": { |
||||
|
"rebalance_on_expansion": true, |
||||
|
"keyfile": "/etc/heketi/private_key", |
||||
|
"fstab": "/etc/fstab", |
||||
|
"port": "22", |
||||
|
"user": "root", |
||||
|
"sudo": false |
||||
|
} |
||||
|
}, |
||||
|
|
||||
|
"_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.", |
||||
|
"backup_db_to_kube_secret": false |
||||
|
} |
||||
@ -0,0 +1,10 @@ |
|||||
|
--- |
||||
|
apiVersion: storage.k8s.io/v1 |
||||
|
kind: StorageClass |
||||
|
metadata: |
||||
|
name: gluster |
||||
|
provisioner: kubernetes.io/glusterfs |
||||
|
parameters: |
||||
|
resturl: "http://{{ endpoint_address }}:8080" |
||||
|
restuser: "user" |
||||
|
restuserkey: "{{ heketi_user_key }}" |
||||
@ -0,0 +1,34 @@ |
|||||
|
{ |
||||
|
"clusters": [ |
||||
|
{ |
||||
|
"nodes": [ |
||||
|
{% set nodeblocks = [] %} |
||||
|
{% for node in nodes %} |
||||
|
{% set nodeblock %} |
||||
|
{ |
||||
|
"node": { |
||||
|
"hostnames": { |
||||
|
"manage": [ |
||||
|
"{{ node }}" |
||||
|
], |
||||
|
"storage": [ |
||||
|
"{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" |
||||
|
] |
||||
|
}, |
||||
|
"zone": 1 |
||||
|
}, |
||||
|
"devices": [ |
||||
|
{ |
||||
|
"name": "{{ hostvars[node]['disk_volume_device_1'] }}", |
||||
|
"destroydata": false |
||||
|
} |
||||
|
] |
||||
|
} |
||||
|
{% endset %} |
||||
|
{% if nodeblocks.append(nodeblock) %}{% endif %} |
||||
|
{% endfor %} |
||||
|
{{ nodeblocks|join(',') }} |
||||
|
] |
||||
|
} |
||||
|
] |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save