From 9d2fabc9b9567daee8c2c04912f0fba974289ab8 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 13:23:20 +0200 Subject: [PATCH 01/35] add heketi/glusterfs as additional contributional network storage --- .../network-storage/heketi/roles/heketi.yml | 10 ++ .../heketi/roles/prepare/tasks/main.yml | 16 ++ .../heketi/roles/provision/defaults/main.yml | 2 + .../roles/provision/glusterfs-daemonset.json | 144 ++++++++++++++++ .../heketi/roles/provision/handlers/main.yml | 3 + .../roles/provision/heketi-bootstrap.json | 133 +++++++++++++++ .../roles/provision/heketi-deployment.json | 159 ++++++++++++++++++ .../provision/heketi-service-account.json | 7 + .../roles/provision/tasks/kubernetes.yml | 54 ++++++ .../provision/tasks/kubernetes/label.yml | 11 ++ .../heketi/roles/provision/tasks/main.yml | 26 +++ .../heketi/roles/provision/tasks/setup.yml | 51 ++++++ .../roles/provision/tasks/setup/boot.yml | 34 ++++ .../roles/provision/tasks/setup/heketi.yml | 10 ++ .../roles/provision/tasks/setup/rest.yml | 38 +++++ .../roles/provision/tasks/setup/storage.yml | 36 ++++ .../tasks/setup/tear-down-bootstrap.yml | 14 ++ .../roles/provision/tasks/setup/topology.yml | 20 +++ .../roles/provision/tasks/setup/volumes.yml | 20 +++ .../roles/provision/tasks/storageclass.yml | 20 +++ .../roles/provision/templates/heketi.json.j2 | 44 +++++ .../provision/templates/storageclass.yml.j2 | 10 ++ .../provision/templates/topology.json.j2 | 34 ++++ 23 files changed, 896 insertions(+) create mode 100644 contrib/network-storage/heketi/roles/heketi.yml create mode 100644 contrib/network-storage/heketi/roles/prepare/tasks/main.yml create mode 100644 contrib/network-storage/heketi/roles/provision/defaults/main.yml create mode 100644 contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json create mode 100644 contrib/network-storage/heketi/roles/provision/handlers/main.yml create mode 100644 contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json create mode 100644 contrib/network-storage/heketi/roles/provision/heketi-deployment.json create mode 100644 contrib/network-storage/heketi/roles/provision/heketi-service-account.json create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/main.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml create mode 100644 contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 create mode 100644 contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 create mode 100644 contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 diff --git a/contrib/network-storage/heketi/roles/heketi.yml b/contrib/network-storage/heketi/roles/heketi.yml new file mode 100644 index 000000000..8ac74b980 --- /dev/null +++ b/contrib/network-storage/heketi/roles/heketi.yml @@ -0,0 +1,10 @@ +--- +- hosts: heketi-node + roles: + - { role: prepare } + +- hosts: localhost + tags: + - "provision" + roles: + - { role: provision } diff --git a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml new file mode 100644 index 000000000..cb644364c --- /dev/null +++ b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: "Load lvm kernel modules" + become: true + with_items: + - "dm_snapshot" + - "dm_mirror" + - "dm_thin_pool" + modprobe: + name: "{{ item }}" + state: "present" + +- name: "Install glusterfs mount utils" + become: true + yum: + name: "glusterfs-fuse" + state: "present" diff --git a/contrib/network-storage/heketi/roles/provision/defaults/main.yml b/contrib/network-storage/heketi/roles/provision/defaults/main.yml new file mode 100644 index 000000000..df38376d1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/defaults/main.yml @@ -0,0 +1,2 @@ +--- +artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts" diff --git a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json new file mode 100644 index 000000000..3c057fddd --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json @@ -0,0 +1,144 @@ +{ + "kind": "DaemonSet", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs": "deployment" + }, + "annotations": { + "description": "GlusterFS Daemon Set", + "tags": "glusterfs" + } + }, + "spec": { + "template": { + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs-node": "daemonset" + } + }, + "spec": { + "nodeSelector": { + "storagenode" : "glusterfs" + }, + "hostNetwork": true, + "containers": [ + { + "image": "gluster/gluster-centos:latest", + "imagePullPolicy": "Always", + "name": "glusterfs", + "volumeMounts": [ + { + "name": "glusterfs-heketi", + "mountPath": "/var/lib/heketi" + }, + { + "name": "glusterfs-run", + "mountPath": "/run" + }, + { + "name": "glusterfs-lvm", + "mountPath": "/run/lvm" + }, + { + "name": "glusterfs-etc", + "mountPath": "/etc/glusterfs" + }, + { + "name": "glusterfs-logs", + "mountPath": "/var/log/glusterfs" + }, + { + "name": "glusterfs-config", + "mountPath": "/var/lib/glusterd" + }, + { + "name": "glusterfs-dev", + "mountPath": "/dev" + }, + { + "name": "glusterfs-cgroup", + "mountPath": "/sys/fs/cgroup" + } + ], + "securityContext": { + "capabilities": {}, + "privileged": true + }, + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + } + } + ], + "volumes": [ + { + "name": "glusterfs-heketi", + "hostPath": { + "path": "/var/lib/heketi" + } + }, + { + "name": "glusterfs-run" + }, + { + "name": "glusterfs-lvm", + "hostPath": { + "path": "/run/lvm" + } + }, + { + "name": "glusterfs-etc", + "hostPath": { + "path": "/etc/glusterfs" + } + }, + { + "name": "glusterfs-logs", + "hostPath": { + "path": "/var/log/glusterfs" + } + }, + { + "name": "glusterfs-config", + "hostPath": { + "path": "/var/lib/glusterd" + } + }, + { + "name": "glusterfs-dev", + "hostPath": { + "path": "/dev" + } + }, + { + "name": "glusterfs-cgroup", + "hostPath": { + "path": "/sys/fs/cgroup" + } + } + ] + } + } + } +} diff --git a/contrib/network-storage/heketi/roles/provision/handlers/main.yml b/contrib/network-storage/heketi/roles/provision/handlers/main.yml new file mode 100644 index 000000000..9e876de17 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: "stop port forwarding" + command: "killall " diff --git a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json new file mode 100644 index 000000000..cbd2bfa8d --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json @@ -0,0 +1,133 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "deploy-heketi" + }, + "ports": [ + { + "name": "deploy-heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-deployment", + "deploy-heketi": "deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "deploy-heketi", + "labels": { + "name": "deploy-heketi", + "glusterfs": "heketi-pod", + "deploy-heketi": "pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:dev", + "imagePullPolicy": "Always", + "name": "deploy-heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db" + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json new file mode 100644 index 000000000..6670bc874 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json @@ -0,0 +1,159 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "heketi-db-backup", + "labels": { + "glusterfs": "heketi-db", + "heketi": "db" + } + }, + "data": { + }, + "type": "Opaque" + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "heketi" + }, + "ports": [ + { + "name": "heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "heketi", + "labels": { + "name": "heketi", + "glusterfs": "heketi-pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:dev", + "imagePullPolicy": "Always", + "name": "heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "mountPath": "/backupdb", + "name": "heketi-db-secret" + }, + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db", + "glusterfs": { + "endpoints": "heketi-storage-endpoints", + "path": "heketidbstorage" + } + }, + { + "name": "heketi-db-secret", + "secret": { + "secretName": "heketi-db-backup" + } + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-service-account.json b/contrib/network-storage/heketi/roles/provision/heketi-service-account.json new file mode 100644 index 000000000..1dbcb9e96 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-service-account.json @@ -0,0 +1,7 @@ +{ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "heketi-service-account" + } +} diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml new file mode 100644 index 000000000..d78ce8f2e --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -0,0 +1,54 @@ +--- +- register: "daemonset_state" + command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy the GlusterFS DaemonSet" + when: "daemonset_state.stdout == \"\"" + command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" +- register: "daemonset_state" + command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." } + +- name: "Label Gluster nodes" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" + include_tasks: "kubernetes/label.yml" + +- register: "service_account_state" + command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy the Heketi service account" + when: "service_account_state.stdout == \"\"" + command: "kubectl create -f {{ role_path }}/heketi-service-account.json" +- register: "service_account_state" + command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } + +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy cluster role binding." + when: "clusterrolebinding_state.stdout == \"\"" + command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } + +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- name: "Render Heketi secret configuration." + template: + src: "heketi.json.j2" + dest: "{{ artifacts_dir }}/heketi.json" +- name: "Deploy Heketi config secret" + when: "secret_state.stdout == \"\"" + command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json" +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml new file mode 100644 index 000000000..7ff9ee4ae --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml @@ -0,0 +1,11 @@ +--- +- register: "label_present" + command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- name: "Assign storage label" + when: "label_present.stdout_lines|length == 0" + command: "kubectl label node {{ node }} storagenode=glusterfs" +- register: "label_present" + command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml new file mode 100644 index 000000000..983f5e672 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: "Prepare kubernetes." + include_tasks: "kubernetes.yml" + +- name: "Test heketi setup." + register: "heketi_service_state" + command: "kubectl get service heketi -o=name --ignore-not-found=true" + changed_when: false + +- name: "Setup heketi." + when: "heketi_service_state.stdout == \"\"" + include_tasks: "setup.yml" + +- name: "Test storage class." + changed_when: false + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" +- name: "Setup storage class." + when: "storageclass.stdout == \"\"" + include_tasks: "storageclass.yml" +- name: "Test storage class." + changed_when: false + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" +- name: "Ensure storage class is up." + assert: { that: "storageclass.stdout != \"\"" } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml new file mode 100644 index 000000000..1de5073a7 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml @@ -0,0 +1,51 @@ +--- +- name: "Test REST endpoint." + uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } + register: "rest_hello_check" + ignore_errors: true + +# Bootstrap heketi +- name: "Bootstrap heketi and start REST endpoint." + when: "rest_hello_check.content != \"Hello from Heketi\"" + include_tasks: "{{ item }}" + with_items: [ "setup/boot.yml", "setup/rest.yml" ] +- name: "Bootstrap heketi." + when: "rest_hello_check.content == \"Hello from Heketi\"" + include_tasks: "setup/boot.yml" + +# Prepare heketi topology +- name: "Test heketi topology." + changed_when: false + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- name: "Load heketi topology." + when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + include_tasks: "setup/topology.yml" + +# Provision heketi database volume +- name: "Prepare heketi volumes." + include_tasks: "setup/volumes.yml" + +# Prepare heketi storage +- name: "Test heketi storage." + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +- name: "Create heketi storage." + include_tasks: "setup/storage.yml" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" + +# Finalize setup +- name: "Tear down bootstrap." + include_tasks: "setup/tear-down-bootstrap.yml" +- name: "Setup final heketi instance." + include_tasks: "setup/heketi.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml new file mode 100644 index 000000000..3de6922f1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -0,0 +1,34 @@ +--- +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" +- name: "Create Heketi initial service and deployment" + command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json" + when: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" +- name: "Ensure heketi bootstrap environment exists." + assert: + that: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'" + msg: "Heketi deployment did not succeed." +- name: "Wait for heketi bootstrap to complete." + changed_when: false + register: "initial_heketi_state" + vars: + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + until: + - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml new file mode 100644 index 000000000..ff43568ca --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml @@ -0,0 +1,10 @@ +--- +- name: "Create long term Heketi instance." + command: "kubectl create -f {{ role_path }}/heketi-deployment.json" +- name: "Get heketi deployment state." + register: "heketi_deployment_state" + command: "kubectl get deployment heketi -o=name --ignore-not-found=true" + changed_when: false +- name: "Ensure heketi is up and running." + assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } + diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml new file mode 100644 index 000000000..b78395eeb --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml @@ -0,0 +1,38 @@ +--- +# Enable local REST-Interface +- name: "Get heketi initial pod state." + tags: ["test"] + register: "initial_heketi_pod" + command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" + changed_when: false +- name: "Ensure heketi bootstrap pod is up." + tags: ["test"] + assert: + that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" +- name: "Temporarily enable local port forwarding to heketi REST interface" + tags: ["test"] + vars: + initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" + command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080" + async: 180 + poll: 0 + ignore_errors: "yes" + register: "heketi_port_forwarding" + changed_when: false +- name: "Ensure port forwarding is enabled." + tags: ["test"] + retries: 10 + delay: 5 + assert: + that: + - "heketi_port_forwarding.finished == 0" + - "heketi_port_forwarding.started == 1" + - "heketi_port_forwarding.failed == false" + msg: "Port forwarding does not work." +- name: "Test REST endpoint." + tags: ["test"] + uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } + register: "rest_hello_check" + until: "rest_hello_check.content == \"Hello from Heketi\"" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml new file mode 100644 index 000000000..3a2d0df8a --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -0,0 +1,36 @@ +--- +- name: "Test heketi storage." + tags: ["test"] + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +- name: "Create heketi storage." + tags: ["test"] + command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" +- name: "Get state of heketi storage service, endpoint, secret and job." + tags: ["test"] + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status" + until: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml new file mode 100644 index 000000000..019f92dfa --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml @@ -0,0 +1,14 @@ +--- +- name: "Get existing Heketi deploy resources." + command: "kubectl get all --selector=\"deploy-heketi\" -o=json" + register: "heketi_resources" + changed_when: false +- name: "Delete bootstrap Heketi." + command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" + when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0" +- name: "Ensure there is nothing left over." + command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 10 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml new file mode 100644 index 000000000..cc73fd62a --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -0,0 +1,20 @@ +--- +- name: "Get heketi topology." + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- name: "Render heketi topology template." + vars: { nodes: "{{ groups['heketi-node'] }}" } + template: + src: "topology.json.j2" + dest: "{{ artifacts_dir }}/topology.json" +- name: "Load heketi topology." + when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json" +- name: "Get heketi topology." + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" } +- name: "Ensure heketi nodes are configured." + assert: + that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0" + msg: "Heketi topology missing." diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml new file mode 100644 index 000000000..d7b710c10 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -0,0 +1,20 @@ +--- +- name: "Get heketi volume ids." + command: "heketi-cli -s http://localhost:48080 volume list --json" + changed_when: false + register: "heketi_volumes" +- name: "Get heketi volumes." + changed_when: false + command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" + with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" + loop_control: { loop_var: "volume_id" } + register: "volumes_information" +- name: "Test heketi database volume." + set_fact: { heketi_database_volume_exists: true } + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" +- name: "Provision database volume." + command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" + when: "heketi_database_volume_exists != true" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml new file mode 100644 index 000000000..d2d8e7725 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -0,0 +1,20 @@ +--- +- name: "Test storage class." + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" + changed_when: false +- name: "Test heketi service." + command: "kubectl get service heketi --ignore-not-found=true --output=json" + register: "heketi_service" + changed_when: false +- name: "Ensure heketi service is available." + assert: { that: "heketi_service.stdout != \"\"" } +- name: "Render storage class configuration." + vars: + endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}" + template: + src: "storageclass.yml.j2" + dest: "{{ artifacts_dir }}/storageclass.yml" +- name: "Setup storage class." + when: "storageclass.stdout == \"\"" + command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml" diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 new file mode 100644 index 000000000..7ba980dca --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 @@ -0,0 +1,44 @@ +{ + "_port_comment": "Heketi Server Port Number", + "port": "8080", + + "_use_auth": "Enable JWT authorization. Please enable for deployment", + "use_auth": false, + + "_jwt": "Private keys for access", + "jwt": { + "_admin": "Admin has access to all APIs", + "admin": { + "key": "{{ heketi_admin_key }}" + }, + "_user": "User only has access to /volumes endpoint", + "user": { + "key": "{{ heketi_user_key }}" + } + }, + + "_glusterfs_comment": "GlusterFS Configuration", + "glusterfs": { + "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh", + "executor": "kubernetes", + + "_db_comment": "Database file name", + "db": "/var/lib/heketi/heketi.db", + + "kubeexec": { + "rebalance_on_expansion": true + }, + + "sshexec": { + "rebalance_on_expansion": true, + "keyfile": "/etc/heketi/private_key", + "fstab": "/etc/fstab", + "port": "22", + "user": "root", + "sudo": false + } + }, + + "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.", + "backup_db_to_kube_secret": false +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 new file mode 100644 index 000000000..5eec1f5d1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gluster +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://{{ endpoint_address }}:8080" + restuser: "user" + restuserkey: "{{ heketi_user_key }}" diff --git a/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 new file mode 100644 index 000000000..b0ac29d7b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 @@ -0,0 +1,34 @@ +{ + "clusters": [ + { + "nodes": [ +{% set nodeblocks = [] %} +{% for node in nodes %} +{% set nodeblock %} + { + "node": { + "hostnames": { + "manage": [ + "{{ node }}" + ], + "storage": [ + "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" + ] + }, + "zone": 1 + }, + "devices": [ + { + "name": "{{ hostvars[node]['disk_volume_device_1'] }}", + "destroydata": false + } + ] + } +{% endset %} +{% if nodeblocks.append(nodeblock) %}{% endif %} +{% endfor %} +{{ nodeblocks|join(',') }} + ] + } + ] +} From 61046a692304398ebb61aa7cdac7a339c61bb8dc Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 13:26:21 +0200 Subject: [PATCH 02/35] move heketi playbook --- contrib/network-storage/heketi/{roles => }/heketi.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename contrib/network-storage/heketi/{roles => }/heketi.yml (100%) diff --git a/contrib/network-storage/heketi/roles/heketi.yml b/contrib/network-storage/heketi/heketi.yml similarity index 100% rename from contrib/network-storage/heketi/roles/heketi.yml rename to contrib/network-storage/heketi/heketi.yml From 8c5bfc7718cb2bdcde018786ede8ab7e58534c61 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 13:37:58 +0200 Subject: [PATCH 03/35] add debian compatibility --- .../heketi/roles/prepare/tasks/main.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml index cb644364c..e4db23365 100644 --- a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml @@ -9,8 +9,16 @@ name: "{{ item }}" state: "present" -- name: "Install glusterfs mount utils" +- name: "Install glusterfs mount utils (RedHat)" become: true yum: name: "glusterfs-fuse" state: "present" + when: "ansible_os_family == 'RedHat'" + +- name: "Install glusterfs mount utils (Debian)" + become: true + apt: + name: "glusterfs-client" + state: "present" + when: "ansible_os_family == 'Debian'" From 9c7e30e4b4f7edb4fef4f207f225a051da284ac9 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 13:56:27 +0200 Subject: [PATCH 04/35] add sample inventory --- .../heketi/inventory.yml.sample | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 contrib/network-storage/heketi/inventory.yml.sample diff --git a/contrib/network-storage/heketi/inventory.yml.sample b/contrib/network-storage/heketi/inventory.yml.sample new file mode 100644 index 000000000..7d488d1ba --- /dev/null +++ b/contrib/network-storage/heketi/inventory.yml.sample @@ -0,0 +1,26 @@ +all: + vars: + heketi_admin_key: "11elfeinhundertundelf" + heketi_user_key: "!!einseinseins" + children: + k8s-cluster: + vars: + kubelet_fail_swap_on: false + children: + kube-master: + hosts: + node1: + etcd: + hosts: + node2: + kube-node: + hosts: &kube_nodes + node1: + node2: + node3: + node4: + heketi-node: + vars: + disk_volume_device_1: "/dev/vdb" + hosts: + <<: *kube_nodes From 831ef7ea2c69c19f0300f5c389fa948a2905fedf Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 14:14:58 +0200 Subject: [PATCH 05/35] add readme --- contrib/network-storage/heketi/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 contrib/network-storage/heketi/README.md diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md new file mode 100644 index 000000000..d06eb8af9 --- /dev/null +++ b/contrib/network-storage/heketi/README.md @@ -0,0 +1,6 @@ +# Deploy Heketi/Glusterfs into Kubespray/Kubernetes +This playbook aims to automate (this)[https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md] tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass. + +## Install +Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. +ansible-playbook -b --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml From 5aefa847dfbed079bcb8076dc4feebf1a881cad6 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 14:16:12 +0200 Subject: [PATCH 06/35] add fences --- contrib/network-storage/heketi/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md index d06eb8af9..4f8915157 100644 --- a/contrib/network-storage/heketi/README.md +++ b/contrib/network-storage/heketi/README.md @@ -3,4 +3,6 @@ This playbook aims to automate (this)[https://github.com/heketi/heketi/blob/mast ## Install Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. +``` ansible-playbook -b --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml +``` From 9516170ce5e39d64c5faf974cc5cbe28e8fb2b06 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 14:17:13 +0200 Subject: [PATCH 07/35] remove unnecessary become flag --- contrib/network-storage/heketi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md index 4f8915157..bc8488a3e 100644 --- a/contrib/network-storage/heketi/README.md +++ b/contrib/network-storage/heketi/README.md @@ -4,5 +4,5 @@ This playbook aims to automate (this)[https://github.com/heketi/heketi/blob/mast ## Install Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. ``` -ansible-playbook -b --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml +ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml ``` From 8ef0cf771f305f4bf6851e2016f39cb0318d3697 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Mon, 25 Jun 2018 14:18:17 +0200 Subject: [PATCH 08/35] update link --- contrib/network-storage/heketi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md index bc8488a3e..e4492817f 100644 --- a/contrib/network-storage/heketi/README.md +++ b/contrib/network-storage/heketi/README.md @@ -1,5 +1,5 @@ # Deploy Heketi/Glusterfs into Kubespray/Kubernetes -This playbook aims to automate (this)[https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md] tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass. +This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass. ## Install Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. From 74cad6b811ad082156cb35ad6357f53d128cdc90 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 27 Jun 2018 10:11:14 +0200 Subject: [PATCH 09/35] pin versions of container images --- .../heketi/roles/provision/glusterfs-daemonset.json | 2 +- .../heketi/roles/provision/heketi-bootstrap.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json index 3c057fddd..8f46bea24 100644 --- a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json +++ b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json @@ -26,7 +26,7 @@ "hostNetwork": true, "containers": [ { - "image": "gluster/gluster-centos:latest", + "image": "gluster/gluster-centos:gluster4u0_centos7", "imagePullPolicy": "Always", "name": "glusterfs", "volumeMounts": [ diff --git a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json index cbd2bfa8d..bdcf3e958 100644 --- a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json +++ b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json @@ -56,7 +56,7 @@ "serviceAccountName": "heketi-service-account", "containers": [ { - "image": "heketi/heketi:dev", + "image": "heketi/heketi:7", "imagePullPolicy": "Always", "name": "deploy-heketi", "env": [ From b56f4651450829451cf320788248435390eb433c Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 27 Jun 2018 10:12:23 +0200 Subject: [PATCH 10/35] fix creation of heketi volumes and storage provisioning validation --- .../roles/provision/tasks/setup/storage.yml | 7 ++---- .../roles/provision/tasks/setup/volumes.yml | 22 +++++++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index 3a2d0df8a..e0ae12246 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -1,11 +1,9 @@ --- - name: "Test heketi storage." - tags: ["test"] command: "kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" - name: "Create heketi storage." - tags: ["test"] command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" @@ -18,7 +16,6 @@ - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" - name: "Get state of heketi storage service, endpoint, secret and job." - tags: ["test"] command: "kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" @@ -26,11 +23,11 @@ secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" - job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" until: - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" - - "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" retries: 10 delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index d7b710c10..268ca94a2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -16,5 +16,23 @@ vars: { volume: "{{ volume_information.stdout|from_json }}" } when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." - command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" - when: "heketi_database_volume_exists != true" + command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" + when: "heketi_database_volume_exists is undefined" +- name: "Get heketi volume ids." + command: "heketi-cli -s http://localhost:48080 volume list --json" + changed_when: false + register: "heketi_volumes" +- name: "Get heketi volumes." + changed_when: false + command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" + with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" + loop_control: { loop_var: "volume_id" } + register: "volumes_information" +- name: "Test heketi database volume." + set_fact: { heketi_database_volume_created: true } + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" +- name: "Ensure heketi database volume exists." + assert: { that: "heketi_database_volume_created is defined" , msg: "Heketi database volume does not exist." } From 8e275ab2bd081e459ea1644f9155d64ea53289a7 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 27 Jun 2018 12:30:14 +0200 Subject: [PATCH 11/35] change order and validation of bootstrap and rest tasks as well as volumes --- .../heketi/roles/provision/tasks/setup.yml | 22 ++++++++++--------- .../roles/provision/tasks/setup/rest.yml | 7 +----- .../roles/provision/tasks/setup/volumes.yml | 1 + 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml index 1de5073a7..c2eb0502c 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml @@ -1,17 +1,19 @@ --- -- name: "Test REST endpoint." - uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } - register: "rest_hello_check" - ignore_errors: true - # Bootstrap heketi -- name: "Bootstrap heketi and start REST endpoint." - when: "rest_hello_check.content != \"Hello from Heketi\"" - include_tasks: "{{ item }}" - with_items: [ "setup/boot.yml", "setup/rest.yml" ] +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - name: "Bootstrap heketi." - when: "rest_hello_check.content == \"Hello from Heketi\"" + when: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" include_tasks: "setup/boot.yml" +- name: "Test rest endpoint port." + wait_for: { host: "localhost", port: "48080", state: "absent" } +- name: "Start REST endpoint." + include_tasks: "setup/rest.yml" # Prepare heketi topology - name: "Test heketi topology." diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml index b78395eeb..120768fd2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml @@ -1,26 +1,22 @@ --- # Enable local REST-Interface - name: "Get heketi initial pod state." - tags: ["test"] register: "initial_heketi_pod" command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" changed_when: false - name: "Ensure heketi bootstrap pod is up." - tags: ["test"] assert: that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" - name: "Temporarily enable local port forwarding to heketi REST interface" - tags: ["test"] vars: initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080" - async: 180 + async: 600 poll: 0 ignore_errors: "yes" register: "heketi_port_forwarding" changed_when: false - name: "Ensure port forwarding is enabled." - tags: ["test"] retries: 10 delay: 5 assert: @@ -30,7 +26,6 @@ - "heketi_port_forwarding.failed == false" msg: "Port forwarding does not work." - name: "Test REST endpoint." - tags: ["test"] uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } register: "rest_hello_check" until: "rest_hello_check.content == \"Hello from Heketi\"" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index 268ca94a2..e2103973d 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -17,6 +17,7 @@ when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" + args: { chdir: "{{ artifacts_dir }}", creates: "{{ artifacts_dir }}/{{ heketi-storage.json }}" } when: "heketi_database_volume_exists is undefined" - name: "Get heketi volume ids." command: "heketi-cli -s http://localhost:48080 volume list --json" From 0b7aa33bc233c55bcebee7ed2dcb9e865e27b1ff Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 4 Jul 2018 18:25:35 +0200 Subject: [PATCH 12/35] add jmespath as requirement --- contrib/network-storage/heketi/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 contrib/network-storage/heketi/requirements.txt diff --git a/contrib/network-storage/heketi/requirements.txt b/contrib/network-storage/heketi/requirements.txt new file mode 100644 index 000000000..45c1e038e --- /dev/null +++ b/contrib/network-storage/heketi/requirements.txt @@ -0,0 +1 @@ +jmespath From 78aeef074e93780110daba01ddc9b3ce9ddbbedd Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 4 Jul 2018 18:40:48 +0200 Subject: [PATCH 13/35] add hint on how to install heketi-cli --- contrib/network-storage/heketi/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md index e4492817f..4ea825b7b 100644 --- a/contrib/network-storage/heketi/README.md +++ b/contrib/network-storage/heketi/README.md @@ -1,6 +1,9 @@ # Deploy Heketi/Glusterfs into Kubespray/Kubernetes This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass. +## Client Setup +Heketi provides a CLI that provides users with a means to administer the deployment and configuration of GlusterFS in Kubernetes. [Download and install the heketi-cli](https://github.com/heketi/heketi/releases) on your client machine. + ## Install Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. ``` From d7abdced05df18d02660f437641ccc5fc800de05 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 4 Jul 2018 18:58:45 +0200 Subject: [PATCH 14/35] fix typo --- .../heketi/roles/provision/tasks/setup/volumes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index e2103973d..146c2b69d 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -17,7 +17,7 @@ when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" - args: { chdir: "{{ artifacts_dir }}", creates: "{{ artifacts_dir }}/{{ heketi-storage.json }}" } + args: { chdir: "{{ artifacts_dir }}", creates: "{{ artifacts_dir }}/heketi-storage.json" } when: "heketi_database_volume_exists is undefined" - name: "Get heketi volume ids." command: "heketi-cli -s http://localhost:48080 volume list --json" From f4c1d6a5d7ee6e290ab0f9eae866d1dec9393e29 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 4 Jul 2018 19:08:02 +0200 Subject: [PATCH 15/35] remove unnecessary check for existing artifact --- .../heketi/roles/provision/tasks/setup/volumes.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index 146c2b69d..268ca94a2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -17,7 +17,6 @@ when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" - args: { chdir: "{{ artifacts_dir }}", creates: "{{ artifacts_dir }}/heketi-storage.json" } when: "heketi_database_volume_exists is undefined" - name: "Get heketi volume ids." command: "heketi-cli -s http://localhost:48080 volume list --json" From 125372597561433ff8e292c980a4e31312ef00c4 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 4 Jul 2018 19:31:25 +0200 Subject: [PATCH 16/35] add necessary chdir --- .../heketi/roles/provision/tasks/setup/volumes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index 268ca94a2..265cdff57 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -17,6 +17,7 @@ when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" + args: { chdir: "{{ artifacts_dir }}" } when: "heketi_database_volume_exists is undefined" - name: "Get heketi volume ids." command: "heketi-cli -s http://localhost:48080 volume list --json" From c39835628d1df9c5fd5affb0a71e8d8cf69ad415 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Thu, 5 Jul 2018 02:14:36 +0200 Subject: [PATCH 17/35] prevent some race conditions, increase over all time limits --- .../heketi/roles/provision/tasks/kubernetes.yml | 14 +++++++++++--- .../heketi/roles/provision/tasks/setup/boot.yml | 4 ++-- .../heketi/roles/provision/tasks/setup/rest.yml | 4 ++-- .../heketi/roles/provision/tasks/setup/storage.yml | 8 +++++--- .../provision/tasks/setup/tear-down-bootstrap.yml | 2 +- .../roles/provision/tasks/setup/topology.yml | 8 +++----- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml index d78ce8f2e..813a434cf 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -6,9 +6,17 @@ when: "daemonset_state.stdout == \"\"" command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" - register: "daemonset_state" - command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" - changed_when: false -- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." } + command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" +- name: "Wait for daemonset to become available." + register: "daemonset_state" + command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + changed_when: false + vars: + ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" + desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" + until: "ready == desired" + retries: 60 + delay: 5 - name: "Label Gluster nodes" with_items: "{{ groups['heketi-node'] }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml index 3de6922f1..358881185 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -30,5 +30,5 @@ until: - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml index 120768fd2..1b0d475e2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml @@ -29,5 +29,5 @@ uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } register: "rest_hello_check" until: "rest_hello_check.content == \"Hello from Heketi\"" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index e0ae12246..e84303d3e 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -15,6 +15,8 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" +- command: "kubectl get secrets,endpoints,services,jobs --output=json" + register: "heketi_storage_state" - name: "Get state of heketi storage service, endpoint, secret and job." command: "kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false @@ -23,11 +25,11 @@ secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" - job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.active==0]" until: - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" - retries: 10 - delay: 10 + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml index 019f92dfa..0fa3a1947 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml @@ -10,5 +10,5 @@ command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" register: "heketi_result" until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" - retries: 10 + retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml index cc73fd62a..378aa20de 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -13,8 +13,6 @@ - name: "Get heketi topology." register: "heketi_topology" command: "heketi-cli -s http://localhost:48080 topology info --json" -- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" } -- name: "Ensure heketi nodes are configured." - assert: - that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0" - msg: "Heketi topology missing." + until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" + retries: 60 + delay: 5 From f70381456141921e244f79c52c6fc508e8da7d6d Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Thu, 5 Jul 2018 02:15:05 +0200 Subject: [PATCH 18/35] add tear down playbook --- contrib/network-storage/heketi/README.md | 5 ++ .../heketi/heketi-tear-down.yml | 4 ++ .../heketi/roles/tear-down/tasks/disks.yml | 51 ++++++++++++++++++ .../heketi/roles/tear-down/tasks/main.yml | 53 +++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 contrib/network-storage/heketi/heketi-tear-down.yml create mode 100644 contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml create mode 100644 contrib/network-storage/heketi/roles/tear-down/tasks/main.yml diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md index 4ea825b7b..aa1b656e5 100644 --- a/contrib/network-storage/heketi/README.md +++ b/contrib/network-storage/heketi/README.md @@ -9,3 +9,8 @@ Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml ``` ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml ``` + +## Tear down +``` +ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi-tear-down.yml +``` diff --git a/contrib/network-storage/heketi/heketi-tear-down.yml b/contrib/network-storage/heketi/heketi-tear-down.yml new file mode 100644 index 000000000..da2c05eb7 --- /dev/null +++ b/contrib/network-storage/heketi/heketi-tear-down.yml @@ -0,0 +1,4 @@ +--- +- hosts: localhost + roles: + - { role: tear-down } diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml new file mode 100644 index 000000000..dace142ed --- /dev/null +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml @@ -0,0 +1,51 @@ +--- +- name: "Install lvm utils (RedHat)" + delegate_to: "{{ node }}" + become: true + yum: + name: "lvm2" + state: "present" + when: "ansible_os_family == 'RedHat'" + +- name: "Install lvm utils (Debian)" + delegate_to: "{{ node }}" + become: true + apt: + name: "lvm2" + state: "present" + when: "ansible_os_family == 'Debian'" +- name: "Get volume group information." + delegate_to: "{{ node }}" + become: true + shell: "pvs {{ disk }} --option vg_name | tail -n+2" + vars: { disk: "{{ hostvars[node]['disk_volume_device_1'] }}" } + register: "volume_groups" + ignore_errors: true + changed_when: false +- name: "Remove volume groups." + delegate_to: "{{ node }}" + become: true + command: "vgremove {{ volume_group }} --yes" + with_items: "{{ volume_groups.stdout_lines }}" + loop_control: { loop_var: "volume_group" } +- name: "Remove physical volume from cluster disks." + delegate_to: "{{ node }}" + become: true + command: "pvremove {{ disk }} --yes" + vars: { disk: "{{ hostvars[node]['disk_volume_device_1'] }}" } + ignore_errors: true +- name: "Remove lvm utils (RedHat)" + delegate_to: "{{ node }}" + become: true + yum: + name: "lvm2" + state: "absent" + when: "ansible_os_family == 'RedHat'" + +- name: "Remove lvm utils (Debian)" + delegate_to: "{{ node }}" + become: true + apt: + name: "lvm2" + state: "absent" + when: "ansible_os_family == 'Debian'" diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml new file mode 100644 index 000000000..a62d59d97 --- /dev/null +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: "Tear down heketi." + command: "kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\"" + ignore_errors: true +- name: "Tear down heketi." + command: "kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\"" + ignore_errors: true +- name: "Tear down bootstrap." + include_tasks: "../provision/tasks/setup/tear-down-bootstrap.yml" +- name: "Ensure there is nothing left over." + command: "kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 60 + delay: 5 +- name: "Ensure there is nothing left over." + command: "kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 60 + delay: 5 +- name: "Tear down glusterfs." + command: "kubectl delete daemonset.extensions/glusterfs" + ignore_errors: true +- name: "Remove heketi storage service." + command: "kubectl delete service heketi-storage-endpoints" + ignore_errors: true +- name: "Remove heketi gluster role binding" + command: "kubectl delete clusterrolebinding heketi-gluster-admin" + ignore_errors: true +- name: "Remove heketi config secret" + command: "kubectl delete secret heketi-config-secret" + ignore_errors: true +- name: "Remove heketi db backup" + command: "kubectl delete secret heketi-db-backup" + ignore_errors: true +- name: "Remove heketi service account" + command: "kubectl delete serviceaccount heketi-service-account" + ignore_errors: true +- name: "Get secrets" + command: "kubectl get secrets --output=\"json\"" + register: "secrets" + changed_when: false +- name: "Remove heketi storage secret" + vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" } + command: "kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}" + when: "storage_query is defined" + ignore_errors: true +- name: "Prepare cluster disks." + include_tasks: "disks.yml" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" From ee67ece641f16a462f45196f162026e1fa85c477 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Thu, 5 Jul 2018 14:18:27 +0200 Subject: [PATCH 19/35] suppress unnecessary change --- .../network-storage/heketi/roles/provision/tasks/kubernetes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml index 813a434cf..257f85c04 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -7,6 +7,7 @@ command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" - register: "daemonset_state" command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + changed_when: false - name: "Wait for daemonset to become available." register: "daemonset_state" command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" From 6d1804d8a4a13f5371be4817481367739c725e55 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Thu, 5 Jul 2018 14:19:18 +0200 Subject: [PATCH 20/35] also remove storage class --- contrib/network-storage/heketi/roles/tear-down/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml index a62d59d97..3943047ec 100644 --- a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -1,4 +1,7 @@ --- +- name: "Remove storage class." + command: "kubectl delete storageclass gluster" + ignore_errors: true - name: "Tear down heketi." command: "kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\"" ignore_errors: true From 318c69350e2788851b2b574986b095104055885d Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Sun, 8 Jul 2018 13:15:54 +0200 Subject: [PATCH 21/35] pin heketi image version --- .../heketi/roles/provision/heketi-deployment.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json index 6670bc874..5eb71718c 100644 --- a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json +++ b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json @@ -68,7 +68,7 @@ "serviceAccountName": "heketi-service-account", "containers": [ { - "image": "heketi/heketi:dev", + "image": "heketi/heketi:7", "imagePullPolicy": "Always", "name": "heketi", "env": [ From 306a6a751f60da84c0079755570413ceee685194 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Sun, 8 Jul 2018 13:16:25 +0200 Subject: [PATCH 22/35] wait for job to complete --- .../heketi/roles/provision/tasks/setup/storage.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index e84303d3e..16b300600 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -15,6 +15,8 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" + register: "heketi_storage_result" +- debug: { var: "heketi_storage_result" } - command: "kubectl get secrets,endpoints,services,jobs --output=json" register: "heketi_storage_state" - name: "Get state of heketi storage service, endpoint, secret and job." @@ -25,7 +27,7 @@ secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" - job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.active==0]" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete'].status=='True']" until: - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" From c42397d7dbc146d0694dfd61a2b2b9b055ba0d67 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 25 Jul 2018 16:42:30 +0200 Subject: [PATCH 23/35] run kubectl on one of the masters --- contrib/network-storage/heketi/heketi.yml | 2 +- .../heketi/roles/provision/defaults/main.yml | 1 - .../roles/provision/glusterfs-daemonset.json | 144 ---------------- .../roles/provision/heketi-bootstrap.json | 133 --------------- .../roles/provision/heketi-deployment.json | 159 ------------------ .../provision/heketi-service-account.json | 7 - .../roles/provision/tasks/kubernetes.yml | 9 +- .../heketi/roles/provision/tasks/main.yml | 8 + .../heketi/roles/provision/tasks/setup.yml | 18 +- .../roles/provision/tasks/setup/boot.yml | 9 +- .../roles/provision/tasks/setup/heketi.yml | 2 +- .../roles/provision/tasks/setup/rest.yml | 33 ---- .../roles/provision/tasks/setup/storage.yml | 2 +- .../roles/provision/tasks/setup/topology.yml | 11 +- .../roles/provision/tasks/setup/volumes.yml | 20 ++- .../roles/provision/tasks/storageclass.yml | 5 +- .../provision/templates/storageclass.yml.j2 | 2 + 17 files changed, 55 insertions(+), 510 deletions(-) delete mode 100644 contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json delete mode 100644 contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json delete mode 100644 contrib/network-storage/heketi/roles/provision/heketi-deployment.json delete mode 100644 contrib/network-storage/heketi/roles/provision/heketi-service-account.json delete mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml diff --git a/contrib/network-storage/heketi/heketi.yml b/contrib/network-storage/heketi/heketi.yml index 8ac74b980..3ec719e95 100644 --- a/contrib/network-storage/heketi/heketi.yml +++ b/contrib/network-storage/heketi/heketi.yml @@ -3,7 +3,7 @@ roles: - { role: prepare } -- hosts: localhost +- hosts: kube-master[0] tags: - "provision" roles: diff --git a/contrib/network-storage/heketi/roles/provision/defaults/main.yml b/contrib/network-storage/heketi/roles/provision/defaults/main.yml index df38376d1..ed97d539c 100644 --- a/contrib/network-storage/heketi/roles/provision/defaults/main.yml +++ b/contrib/network-storage/heketi/roles/provision/defaults/main.yml @@ -1,2 +1 @@ --- -artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts" diff --git a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json deleted file mode 100644 index 8f46bea24..000000000 --- a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "kind": "DaemonSet", - "apiVersion": "extensions/v1beta1", - "metadata": { - "name": "glusterfs", - "labels": { - "glusterfs": "deployment" - }, - "annotations": { - "description": "GlusterFS Daemon Set", - "tags": "glusterfs" - } - }, - "spec": { - "template": { - "metadata": { - "name": "glusterfs", - "labels": { - "glusterfs-node": "daemonset" - } - }, - "spec": { - "nodeSelector": { - "storagenode" : "glusterfs" - }, - "hostNetwork": true, - "containers": [ - { - "image": "gluster/gluster-centos:gluster4u0_centos7", - "imagePullPolicy": "Always", - "name": "glusterfs", - "volumeMounts": [ - { - "name": "glusterfs-heketi", - "mountPath": "/var/lib/heketi" - }, - { - "name": "glusterfs-run", - "mountPath": "/run" - }, - { - "name": "glusterfs-lvm", - "mountPath": "/run/lvm" - }, - { - "name": "glusterfs-etc", - "mountPath": "/etc/glusterfs" - }, - { - "name": "glusterfs-logs", - "mountPath": "/var/log/glusterfs" - }, - { - "name": "glusterfs-config", - "mountPath": "/var/lib/glusterd" - }, - { - "name": "glusterfs-dev", - "mountPath": "/dev" - }, - { - "name": "glusterfs-cgroup", - "mountPath": "/sys/fs/cgroup" - } - ], - "securityContext": { - "capabilities": {}, - "privileged": true - }, - "readinessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 60, - "exec": { - "command": [ - "/bin/bash", - "-c", - "systemctl status glusterd.service" - ] - } - }, - "livenessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 60, - "exec": { - "command": [ - "/bin/bash", - "-c", - "systemctl status glusterd.service" - ] - } - } - } - ], - "volumes": [ - { - "name": "glusterfs-heketi", - "hostPath": { - "path": "/var/lib/heketi" - } - }, - { - "name": "glusterfs-run" - }, - { - "name": "glusterfs-lvm", - "hostPath": { - "path": "/run/lvm" - } - }, - { - "name": "glusterfs-etc", - "hostPath": { - "path": "/etc/glusterfs" - } - }, - { - "name": "glusterfs-logs", - "hostPath": { - "path": "/var/log/glusterfs" - } - }, - { - "name": "glusterfs-config", - "hostPath": { - "path": "/var/lib/glusterd" - } - }, - { - "name": "glusterfs-dev", - "hostPath": { - "path": "/dev" - } - }, - { - "name": "glusterfs-cgroup", - "hostPath": { - "path": "/sys/fs/cgroup" - } - } - ] - } - } - } -} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json deleted file mode 100644 index bdcf3e958..000000000 --- a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "kind": "List", - "apiVersion": "v1", - "items": [ - { - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "deploy-heketi", - "labels": { - "glusterfs": "heketi-service", - "deploy-heketi": "support" - }, - "annotations": { - "description": "Exposes Heketi Service" - } - }, - "spec": { - "selector": { - "name": "deploy-heketi" - }, - "ports": [ - { - "name": "deploy-heketi", - "port": 8080, - "targetPort": 8080 - } - ] - } - }, - { - "kind": "Deployment", - "apiVersion": "extensions/v1beta1", - "metadata": { - "name": "deploy-heketi", - "labels": { - "glusterfs": "heketi-deployment", - "deploy-heketi": "deployment" - }, - "annotations": { - "description": "Defines how to deploy Heketi" - } - }, - "spec": { - "replicas": 1, - "template": { - "metadata": { - "name": "deploy-heketi", - "labels": { - "name": "deploy-heketi", - "glusterfs": "heketi-pod", - "deploy-heketi": "pod" - } - }, - "spec": { - "serviceAccountName": "heketi-service-account", - "containers": [ - { - "image": "heketi/heketi:7", - "imagePullPolicy": "Always", - "name": "deploy-heketi", - "env": [ - { - "name": "HEKETI_EXECUTOR", - "value": "kubernetes" - }, - { - "name": "HEKETI_DB_PATH", - "value": "/var/lib/heketi/heketi.db" - }, - { - "name": "HEKETI_FSTAB", - "value": "/var/lib/heketi/fstab" - }, - { - "name": "HEKETI_SNAPSHOT_LIMIT", - "value": "14" - }, - { - "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", - "value": "y" - } - ], - "ports": [ - { - "containerPort": 8080 - } - ], - "volumeMounts": [ - { - "name": "db", - "mountPath": "/var/lib/heketi" - }, - { - "name": "config", - "mountPath": "/etc/heketi" - } - ], - "readinessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 3, - "httpGet": { - "path": "/hello", - "port": 8080 - } - }, - "livenessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 30, - "httpGet": { - "path": "/hello", - "port": 8080 - } - } - } - ], - "volumes": [ - { - "name": "db" - }, - { - "name": "config", - "secret": { - "secretName": "heketi-config-secret" - } - } - ] - } - } - } - } - ] -} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json deleted file mode 100644 index 5eb71718c..000000000 --- a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "kind": "List", - "apiVersion": "v1", - "items": [ - { - "kind": "Secret", - "apiVersion": "v1", - "metadata": { - "name": "heketi-db-backup", - "labels": { - "glusterfs": "heketi-db", - "heketi": "db" - } - }, - "data": { - }, - "type": "Opaque" - }, - { - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "heketi", - "labels": { - "glusterfs": "heketi-service", - "deploy-heketi": "support" - }, - "annotations": { - "description": "Exposes Heketi Service" - } - }, - "spec": { - "selector": { - "name": "heketi" - }, - "ports": [ - { - "name": "heketi", - "port": 8080, - "targetPort": 8080 - } - ] - } - }, - { - "kind": "Deployment", - "apiVersion": "extensions/v1beta1", - "metadata": { - "name": "heketi", - "labels": { - "glusterfs": "heketi-deployment" - }, - "annotations": { - "description": "Defines how to deploy Heketi" - } - }, - "spec": { - "replicas": 1, - "template": { - "metadata": { - "name": "heketi", - "labels": { - "name": "heketi", - "glusterfs": "heketi-pod" - } - }, - "spec": { - "serviceAccountName": "heketi-service-account", - "containers": [ - { - "image": "heketi/heketi:7", - "imagePullPolicy": "Always", - "name": "heketi", - "env": [ - { - "name": "HEKETI_EXECUTOR", - "value": "kubernetes" - }, - { - "name": "HEKETI_DB_PATH", - "value": "/var/lib/heketi/heketi.db" - }, - { - "name": "HEKETI_FSTAB", - "value": "/var/lib/heketi/fstab" - }, - { - "name": "HEKETI_SNAPSHOT_LIMIT", - "value": "14" - }, - { - "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", - "value": "y" - } - ], - "ports": [ - { - "containerPort": 8080 - } - ], - "volumeMounts": [ - { - "mountPath": "/backupdb", - "name": "heketi-db-secret" - }, - { - "name": "db", - "mountPath": "/var/lib/heketi" - }, - { - "name": "config", - "mountPath": "/etc/heketi" - } - ], - "readinessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 3, - "httpGet": { - "path": "/hello", - "port": 8080 - } - }, - "livenessProbe": { - "timeoutSeconds": 3, - "initialDelaySeconds": 30, - "httpGet": { - "path": "/hello", - "port": 8080 - } - } - } - ], - "volumes": [ - { - "name": "db", - "glusterfs": { - "endpoints": "heketi-storage-endpoints", - "path": "heketidbstorage" - } - }, - { - "name": "heketi-db-secret", - "secret": { - "secretName": "heketi-db-backup" - } - }, - { - "name": "config", - "secret": { - "secretName": "heketi-config-secret" - } - } - ] - } - } - } - } - ] -} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-service-account.json b/contrib/network-storage/heketi/roles/provision/heketi-service-account.json deleted file mode 100644 index 1dbcb9e96..000000000 --- a/contrib/network-storage/heketi/roles/provision/heketi-service-account.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "apiVersion": "v1", - "kind": "ServiceAccount", - "metadata": { - "name": "heketi-service-account" - } -} diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml index 257f85c04..e249fc1d1 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -4,7 +4,7 @@ changed_when: false - name: "Deploy the GlusterFS DaemonSet" when: "daemonset_state.stdout == \"\"" - command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" + command: "kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" - register: "daemonset_state" command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" changed_when: false @@ -30,7 +30,7 @@ changed_when: false - name: "Deploy the Heketi service account" when: "service_account_state.stdout == \"\"" - command: "kubectl create -f {{ role_path }}/heketi-service-account.json" + command: "kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" - register: "service_account_state" command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" changed_when: false @@ -51,12 +51,13 @@ command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false - name: "Render Heketi secret configuration." + become: true template: src: "heketi.json.j2" - dest: "{{ artifacts_dir }}/heketi.json" + dest: "{{ kube_config_dir }}/heketi.json" - name: "Deploy Heketi config secret" when: "secret_state.stdout == \"\"" - command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json" + command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" - register: "secret_state" command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml index 983f5e672..f85306b47 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -1,4 +1,12 @@ --- +- name: "Render configuration." + become: true + template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" } + with_items: + - { file: "glusterfs-daemonset.json" } + - { file: "heketi-bootstrap.json" } + - { file: "heketi-deployment.json" } + - { file: "heketi-service-account.json" } - name: "Prepare kubernetes." include_tasks: "kubernetes.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml index c2eb0502c..a2c532fa0 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml @@ -10,16 +10,21 @@ - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" include_tasks: "setup/boot.yml" -- name: "Test rest endpoint port." - wait_for: { host: "localhost", port: "48080", state: "absent" } -- name: "Start REST endpoint." - include_tasks: "setup/rest.yml" # Prepare heketi topology +- name: "Get heketi initial pod state." + register: "initial_heketi_pod" + command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" + changed_when: false +- name: "Ensure heketi bootstrap pod is up." + assert: + that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" +- set_fact: + initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" - name: "Test heketi topology." changed_when: false register: "heketi_topology" - command: "heketi-cli -s http://localhost:48080 topology info --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" include_tasks: "setup/topology.yml" @@ -33,6 +38,8 @@ command: "kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" +- command: "kubectl get secrets,endpoints,services,jobs --output=json" + register: "job" - name: "Create heketi storage." include_tasks: "setup/storage.yml" vars: @@ -45,7 +52,6 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" - # Finalize setup - name: "Tear down bootstrap." include_tasks: "setup/tear-down-bootstrap.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml index 358881185..6aab82cca 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -4,7 +4,7 @@ changed_when: false command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - name: "Create Heketi initial service and deployment" - command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json" + command: "kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" when: - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" @@ -13,13 +13,6 @@ register: "initial_heketi_state" changed_when: false command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" -- name: "Ensure heketi bootstrap environment exists." - assert: - that: - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'" - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'" - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'" - msg: "Heketi deployment did not succeed." - name: "Wait for heketi bootstrap to complete." changed_when: false register: "initial_heketi_state" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml index ff43568ca..cba5e57ed 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml @@ -1,6 +1,6 @@ --- - name: "Create long term Heketi instance." - command: "kubectl create -f {{ role_path }}/heketi-deployment.json" + command: "kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" - name: "Get heketi deployment state." register: "heketi_deployment_state" command: "kubectl get deployment heketi -o=name --ignore-not-found=true" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml deleted file mode 100644 index 1b0d475e2..000000000 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Enable local REST-Interface -- name: "Get heketi initial pod state." - register: "initial_heketi_pod" - command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" - changed_when: false -- name: "Ensure heketi bootstrap pod is up." - assert: - that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" -- name: "Temporarily enable local port forwarding to heketi REST interface" - vars: - initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" - command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080" - async: 600 - poll: 0 - ignore_errors: "yes" - register: "heketi_port_forwarding" - changed_when: false -- name: "Ensure port forwarding is enabled." - retries: 10 - delay: 5 - assert: - that: - - "heketi_port_forwarding.finished == 0" - - "heketi_port_forwarding.started == 1" - - "heketi_port_forwarding.failed == false" - msg: "Port forwarding does not work." -- name: "Test REST endpoint." - uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } - register: "rest_hello_check" - until: "rest_hello_check.content == \"Hello from Heketi\"" - retries: 60 - delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index 16b300600..64a78714c 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -4,7 +4,7 @@ changed_when: false register: "heketi_storage_state" - name: "Create heketi storage." - command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" + command: "kubectl create -f {{ kube_config_dir }}/heketi-storage.json" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml index 378aa20de..190a131c3 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -1,18 +1,21 @@ --- - name: "Get heketi topology." register: "heketi_topology" - command: "heketi-cli -s http://localhost:48080 topology info --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" - name: "Render heketi topology template." + become: true vars: { nodes: "{{ groups['heketi-node'] }}" } template: src: "topology.json.j2" - dest: "{{ artifacts_dir }}/topology.json" + dest: "{{ kube_config_dir }}/topology.json" +- name: "Copy topology configuration into container." + command: "kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" - command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" - name: "Get heketi topology." register: "heketi_topology" - command: "heketi-cli -s http://localhost:48080 topology info --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index 265cdff57..d8e7bf4b8 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -1,11 +1,11 @@ --- - name: "Get heketi volume ids." - command: "heketi-cli -s http://localhost:48080 volume list --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" @@ -16,19 +16,27 @@ vars: { volume: "{{ volume_information.stdout|from_json }}" } when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." - command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage" - args: { chdir: "{{ artifacts_dir }}" } + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage" when: "heketi_database_volume_exists is undefined" +- name: "Copy configuration from pod." + become: true + command: "/usr/local/bin/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json" - name: "Get heketi volume ids." - command: "heketi-cli -s http://localhost:48080 volume list --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" + command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" +- name: "debug heketi db vol." + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" + debug: { var: "volume" } - name: "Test heketi database volume." set_fact: { heketi_database_volume_created: true } with_items: "{{ volumes_information.results }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml index d2d8e7725..761ec6533 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -10,11 +10,12 @@ - name: "Ensure heketi service is available." assert: { that: "heketi_service.stdout != \"\"" } - name: "Render storage class configuration." + become: true vars: endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}" template: src: "storageclass.yml.j2" - dest: "{{ artifacts_dir }}/storageclass.yml" + dest: "{{ kube_config_dir }}/storageclass.yml" - name: "Setup storage class." when: "storageclass.stdout == \"\"" - command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml" + command: "kubectl create -f {{ kube_config_dir }}/storageclass.yml" diff --git a/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 index 5eec1f5d1..65104e143 100644 --- a/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 +++ b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 @@ -3,6 +3,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: gluster + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" provisioner: kubernetes.io/glusterfs parameters: resturl: "http://{{ endpoint_address }}:8080" From 205ea33b10340ab6431b67c8adf2c5f020622ae5 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 25 Jul 2018 16:42:57 +0200 Subject: [PATCH 24/35] "fix" race condition --- .../heketi/roles/provision/tasks/setup/storage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index 64a78714c..1c471db3b 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -16,7 +16,6 @@ - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" register: "heketi_storage_result" -- debug: { var: "heketi_storage_result" } - command: "kubectl get secrets,endpoints,services,jobs --output=json" register: "heketi_storage_state" - name: "Get state of heketi storage service, endpoint, secret and job." @@ -35,3 +34,5 @@ - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" retries: 60 delay: 5 +# looks like there is some race condition that leads to "Database file did not appear, exiting.", can't figure out where +- command: "sleep 10" From 2bd8fbb2dd15e0348bb7c5dc59bee7a2bff4646c Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 25 Jul 2018 16:46:12 +0200 Subject: [PATCH 25/35] add missing templates --- .../templates/glusterfs-daemonset.json.j2 | 144 ++++++++++++++++ .../templates/heketi-bootstrap.json.j2 | 133 +++++++++++++++ .../templates/heketi-deployment.json.j2 | 159 ++++++++++++++++++ .../templates/heketi-service-account.json.j2 | 7 + 4 files changed, 443 insertions(+) create mode 100644 contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 create mode 100644 contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 create mode 100644 contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 create mode 100644 contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 diff --git a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 new file mode 100644 index 000000000..8f46bea24 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 @@ -0,0 +1,144 @@ +{ + "kind": "DaemonSet", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs": "deployment" + }, + "annotations": { + "description": "GlusterFS Daemon Set", + "tags": "glusterfs" + } + }, + "spec": { + "template": { + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs-node": "daemonset" + } + }, + "spec": { + "nodeSelector": { + "storagenode" : "glusterfs" + }, + "hostNetwork": true, + "containers": [ + { + "image": "gluster/gluster-centos:gluster4u0_centos7", + "imagePullPolicy": "Always", + "name": "glusterfs", + "volumeMounts": [ + { + "name": "glusterfs-heketi", + "mountPath": "/var/lib/heketi" + }, + { + "name": "glusterfs-run", + "mountPath": "/run" + }, + { + "name": "glusterfs-lvm", + "mountPath": "/run/lvm" + }, + { + "name": "glusterfs-etc", + "mountPath": "/etc/glusterfs" + }, + { + "name": "glusterfs-logs", + "mountPath": "/var/log/glusterfs" + }, + { + "name": "glusterfs-config", + "mountPath": "/var/lib/glusterd" + }, + { + "name": "glusterfs-dev", + "mountPath": "/dev" + }, + { + "name": "glusterfs-cgroup", + "mountPath": "/sys/fs/cgroup" + } + ], + "securityContext": { + "capabilities": {}, + "privileged": true + }, + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + } + } + ], + "volumes": [ + { + "name": "glusterfs-heketi", + "hostPath": { + "path": "/var/lib/heketi" + } + }, + { + "name": "glusterfs-run" + }, + { + "name": "glusterfs-lvm", + "hostPath": { + "path": "/run/lvm" + } + }, + { + "name": "glusterfs-etc", + "hostPath": { + "path": "/etc/glusterfs" + } + }, + { + "name": "glusterfs-logs", + "hostPath": { + "path": "/var/log/glusterfs" + } + }, + { + "name": "glusterfs-config", + "hostPath": { + "path": "/var/lib/glusterd" + } + }, + { + "name": "glusterfs-dev", + "hostPath": { + "path": "/dev" + } + }, + { + "name": "glusterfs-cgroup", + "hostPath": { + "path": "/sys/fs/cgroup" + } + } + ] + } + } + } +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 new file mode 100644 index 000000000..bdcf3e958 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 @@ -0,0 +1,133 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "deploy-heketi" + }, + "ports": [ + { + "name": "deploy-heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-deployment", + "deploy-heketi": "deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "deploy-heketi", + "labels": { + "name": "deploy-heketi", + "glusterfs": "heketi-pod", + "deploy-heketi": "pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:7", + "imagePullPolicy": "Always", + "name": "deploy-heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db" + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 new file mode 100644 index 000000000..5eb71718c --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 @@ -0,0 +1,159 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "heketi-db-backup", + "labels": { + "glusterfs": "heketi-db", + "heketi": "db" + } + }, + "data": { + }, + "type": "Opaque" + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "heketi" + }, + "ports": [ + { + "name": "heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "heketi", + "labels": { + "name": "heketi", + "glusterfs": "heketi-pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:7", + "imagePullPolicy": "Always", + "name": "heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "mountPath": "/backupdb", + "name": "heketi-db-secret" + }, + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db", + "glusterfs": { + "endpoints": "heketi-storage-endpoints", + "path": "heketidbstorage" + } + }, + { + "name": "heketi-db-secret", + "secret": { + "secretName": "heketi-db-backup" + } + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 new file mode 100644 index 000000000..1dbcb9e96 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 @@ -0,0 +1,7 @@ +{ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "heketi-service-account" + } +} From 8ac57201a70484eefc9715f15bcf6b4afef8fd31 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 30 Jul 2018 16:44:02 +0100 Subject: [PATCH 26/35] Prefix heketi kubectl calls with {{ bin_dir }} --- .../roles/provision/tasks/kubernetes.yml | 26 +++++++++---------- .../provision/tasks/kubernetes/label.yml | 6 ++--- .../heketi/roles/provision/tasks/main.yml | 6 ++--- .../heketi/roles/provision/tasks/setup.yml | 10 +++---- .../roles/provision/tasks/setup/boot.yml | 8 +++--- .../roles/provision/tasks/setup/heketi.yml | 4 +-- .../roles/provision/tasks/setup/storage.yml | 8 +++--- .../tasks/setup/tear-down-bootstrap.yml | 6 ++--- .../roles/provision/tasks/setup/topology.yml | 8 +++--- .../roles/provision/tasks/setup/volumes.yml | 12 ++++----- .../roles/provision/tasks/storageclass.yml | 6 ++--- .../heketi/roles/tear-down/tasks/main.yml | 26 +++++++++---------- 12 files changed, 63 insertions(+), 63 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml index e249fc1d1..6ab92713b 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -1,16 +1,16 @@ --- - register: "daemonset_state" - command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get daemonset glusterfs -o=name --ignore-not-found=true" changed_when: false - name: "Deploy the GlusterFS DaemonSet" when: "daemonset_state.stdout == \"\"" - command: "kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" - register: "daemonset_state" - command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" changed_when: false - name: "Wait for daemonset to become available." register: "daemonset_state" - command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" changed_when: false vars: ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" @@ -26,29 +26,29 @@ include_tasks: "kubernetes/label.yml" - register: "service_account_state" - command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" changed_when: false - name: "Deploy the Heketi service account" when: "service_account_state.stdout == \"\"" - command: "kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" - register: "service_account_state" - command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" changed_when: false - assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } - register: "clusterrolebinding_state" - command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" changed_when: false - name: "Deploy cluster role binding." when: "clusterrolebinding_state.stdout == \"\"" - command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" + command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" - register: "clusterrolebinding_state" - command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" changed_when: false - assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } - register: "secret_state" - command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false - name: "Render Heketi secret configuration." become: true @@ -57,8 +57,8 @@ dest: "{{ kube_config_dir }}/heketi.json" - name: "Deploy Heketi config secret" when: "secret_state.stdout == \"\"" - command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" + command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" - register: "secret_state" - command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false - assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml index 7ff9ee4ae..61729a5e2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml @@ -1,11 +1,11 @@ --- - register: "label_present" - command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" changed_when: false - name: "Assign storage label" when: "label_present.stdout_lines|length == 0" - command: "kubectl label node {{ node }} storagenode=glusterfs" + command: "{{ bin_dir }}/kubectl label node {{ node }} storagenode=glusterfs" - register: "label_present" - command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" changed_when: false - assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml index f85306b47..c247929a7 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -12,7 +12,7 @@ - name: "Test heketi setup." register: "heketi_service_state" - command: "kubectl get service heketi -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get service heketi -o=name --ignore-not-found=true" changed_when: false - name: "Setup heketi." @@ -21,14 +21,14 @@ - name: "Test storage class." changed_when: false - command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" register: "storageclass" - name: "Setup storage class." when: "storageclass.stdout == \"\"" include_tasks: "storageclass.yml" - name: "Test storage class." changed_when: false - command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" register: "storageclass" - name: "Ensure storage class is up." assert: { that: "storageclass.stdout != \"\"" } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml index a2c532fa0..96462dd11 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml @@ -3,7 +3,7 @@ - name: "Get state of heketi service, deployment and pods." register: "initial_heketi_state" changed_when: false - command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - name: "Bootstrap heketi." when: - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" @@ -14,7 +14,7 @@ # Prepare heketi topology - name: "Get heketi initial pod state." register: "initial_heketi_pod" - command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" + command: "{{ bin_dir }}/kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" changed_when: false - name: "Ensure heketi bootstrap pod is up." assert: @@ -24,7 +24,7 @@ - name: "Test heketi topology." changed_when: false register: "heketi_topology" - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" include_tasks: "setup/topology.yml" @@ -35,10 +35,10 @@ # Prepare heketi storage - name: "Test heketi storage." - command: "kubectl get secrets,endpoints,services,jobs --output=json" + command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" -- command: "kubectl get secrets,endpoints,services,jobs --output=json" +- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" register: "job" - name: "Create heketi storage." include_tasks: "setup/storage.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml index 6aab82cca..806e91570 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -2,9 +2,9 @@ - name: "Get state of heketi service, deployment and pods." register: "initial_heketi_state" changed_when: false - command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - name: "Create Heketi initial service and deployment" - command: "kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" when: - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" @@ -12,14 +12,14 @@ - name: "Get state of heketi service, deployment and pods." register: "initial_heketi_state" changed_when: false - command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - name: "Wait for heketi bootstrap to complete." changed_when: false register: "initial_heketi_state" vars: pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" - command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" until: - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml index cba5e57ed..c38b151ac 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml @@ -1,9 +1,9 @@ --- - name: "Create long term Heketi instance." - command: "kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" - name: "Get heketi deployment state." register: "heketi_deployment_state" - command: "kubectl get deployment heketi -o=name --ignore-not-found=true" + command: "{{ bin_dir }}/kubectl get deployment heketi -o=name --ignore-not-found=true" changed_when: false - name: "Ensure heketi is up and running." assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml index 1c471db3b..7c49a2258 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -1,10 +1,10 @@ --- - name: "Test heketi storage." - command: "kubectl get secrets,endpoints,services,jobs --output=json" + command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" - name: "Create heketi storage." - command: "kubectl create -f {{ kube_config_dir }}/heketi-storage.json" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-storage.json" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" @@ -16,10 +16,10 @@ - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" register: "heketi_storage_result" -- command: "kubectl get secrets,endpoints,services,jobs --output=json" +- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" register: "heketi_storage_state" - name: "Get state of heketi storage service, endpoint, secret and job." - command: "kubectl get secrets,endpoints,services,jobs --output=json" + command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" vars: diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml index 0fa3a1947..0ffd6f469 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml @@ -1,13 +1,13 @@ --- - name: "Get existing Heketi deploy resources." - command: "kubectl get all --selector=\"deploy-heketi\" -o=json" + command: "{{ bin_dir }}/kubectl get all --selector=\"deploy-heketi\" -o=json" register: "heketi_resources" changed_when: false - name: "Delete bootstrap Heketi." - command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0" - name: "Ensure there is nothing left over." - command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" register: "heketi_result" until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" retries: 60 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml index 190a131c3..fd148d8d9 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -1,7 +1,7 @@ --- - name: "Get heketi topology." register: "heketi_topology" - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" - name: "Render heketi topology template." become: true vars: { nodes: "{{ groups['heketi-node'] }}" } @@ -9,13 +9,13 @@ src: "topology.json.j2" dest: "{{ kube_config_dir }}/topology.json" - name: "Copy topology configuration into container." - command: "kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" + command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" - name: "Get heketi topology." register: "heketi_topology" - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml index d8e7bf4b8..0d24a3a0b 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -1,11 +1,11 @@ --- - name: "Get heketi volume ids." - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" @@ -16,18 +16,18 @@ vars: { volume: "{{ volume_information.stdout|from_json }}" } when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage" when: "heketi_database_volume_exists is undefined" - name: "Copy configuration from pod." become: true - command: "/usr/local/bin/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json" + command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json" - name: "Get heketi volume ids." - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml index 761ec6533..393d76e64 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -1,10 +1,10 @@ --- - name: "Test storage class." - command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" register: "storageclass" changed_when: false - name: "Test heketi service." - command: "kubectl get service heketi --ignore-not-found=true --output=json" + command: "{{ bin_dir }}/kubectl get service heketi --ignore-not-found=true --output=json" register: "heketi_service" changed_when: false - name: "Ensure heketi service is available." @@ -18,4 +18,4 @@ dest: "{{ kube_config_dir }}/storageclass.yml" - name: "Setup storage class." when: "storageclass.stdout == \"\"" - command: "kubectl create -f {{ kube_config_dir }}/storageclass.yml" + command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/storageclass.yml" diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml index 3943047ec..3c9659bae 100644 --- a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -1,52 +1,52 @@ --- - name: "Remove storage class." - command: "kubectl delete storageclass gluster" + command: "{{ bin_dir }}/kubectl delete storageclass gluster" ignore_errors: true - name: "Tear down heketi." - command: "kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\"" + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\"" ignore_errors: true - name: "Tear down heketi." - command: "kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\"" + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\"" ignore_errors: true - name: "Tear down bootstrap." include_tasks: "../provision/tasks/setup/tear-down-bootstrap.yml" - name: "Ensure there is nothing left over." - command: "kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json" + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json" register: "heketi_result" until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" retries: 60 delay: 5 - name: "Ensure there is nothing left over." - command: "kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json" + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json" register: "heketi_result" until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" retries: 60 delay: 5 - name: "Tear down glusterfs." - command: "kubectl delete daemonset.extensions/glusterfs" + command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs" ignore_errors: true - name: "Remove heketi storage service." - command: "kubectl delete service heketi-storage-endpoints" + command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints" ignore_errors: true - name: "Remove heketi gluster role binding" - command: "kubectl delete clusterrolebinding heketi-gluster-admin" + command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin" ignore_errors: true - name: "Remove heketi config secret" - command: "kubectl delete secret heketi-config-secret" + command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret" ignore_errors: true - name: "Remove heketi db backup" - command: "kubectl delete secret heketi-db-backup" + command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup" ignore_errors: true - name: "Remove heketi service account" - command: "kubectl delete serviceaccount heketi-service-account" + command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account" ignore_errors: true - name: "Get secrets" - command: "kubectl get secrets --output=\"json\"" + command: "{{ bin_dir }}/kubectl get secrets --output=\"json\"" register: "secrets" changed_when: false - name: "Remove heketi storage secret" vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" } - command: "kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}" + command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}" when: "storage_query is defined" ignore_errors: true - name: "Prepare cluster disks." From d284961d4717b141b4bdd9df1d02e474d07ff06a Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 27 Jul 2018 16:33:43 +0100 Subject: [PATCH 27/35] Change heketi-tear-down to run on nodes instead of localhost delegate_to --- .../network-storage/heketi/heketi-tear-down.yml | 7 ++++++- .../tasks/main.yml} | 17 ++++++----------- .../heketi/roles/tear-down/tasks/main.yml | 5 ----- 3 files changed, 12 insertions(+), 17 deletions(-) rename contrib/network-storage/heketi/roles/{tear-down/tasks/disks.yml => tear-down-disks/tasks/main.yml} (69%) diff --git a/contrib/network-storage/heketi/heketi-tear-down.yml b/contrib/network-storage/heketi/heketi-tear-down.yml index da2c05eb7..92b9f92d6 100644 --- a/contrib/network-storage/heketi/heketi-tear-down.yml +++ b/contrib/network-storage/heketi/heketi-tear-down.yml @@ -1,4 +1,9 @@ --- -- hosts: localhost +- hosts: kube-master[0] roles: - { role: tear-down } + +- hosts: heketi-node + become: yes + roles: + - { role: tear-down-disks } diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml b/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml similarity index 69% rename from contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml rename to contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml index dace142ed..01e03660c 100644 --- a/contrib/network-storage/heketi/roles/tear-down/tasks/disks.yml +++ b/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml @@ -1,6 +1,5 @@ --- - name: "Install lvm utils (RedHat)" - delegate_to: "{{ node }}" become: true yum: name: "lvm2" @@ -8,34 +7,31 @@ when: "ansible_os_family == 'RedHat'" - name: "Install lvm utils (Debian)" - delegate_to: "{{ node }}" become: true apt: name: "lvm2" state: "present" when: "ansible_os_family == 'Debian'" + - name: "Get volume group information." - delegate_to: "{{ node }}" become: true - shell: "pvs {{ disk }} --option vg_name | tail -n+2" - vars: { disk: "{{ hostvars[node]['disk_volume_device_1'] }}" } + shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2" register: "volume_groups" ignore_errors: true changed_when: false + - name: "Remove volume groups." - delegate_to: "{{ node }}" become: true command: "vgremove {{ volume_group }} --yes" with_items: "{{ volume_groups.stdout_lines }}" loop_control: { loop_var: "volume_group" } + - name: "Remove physical volume from cluster disks." - delegate_to: "{{ node }}" become: true - command: "pvremove {{ disk }} --yes" - vars: { disk: "{{ hostvars[node]['disk_volume_device_1'] }}" } + command: "pvremove {{ disk_volume_device_1 }} --yes" ignore_errors: true + - name: "Remove lvm utils (RedHat)" - delegate_to: "{{ node }}" become: true yum: name: "lvm2" @@ -43,7 +39,6 @@ when: "ansible_os_family == 'RedHat'" - name: "Remove lvm utils (Debian)" - delegate_to: "{{ node }}" become: true apt: name: "lvm2" diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml index 3c9659bae..5c5e69ab8 100644 --- a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -49,8 +49,3 @@ command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}" when: "storage_query is defined" ignore_errors: true -- name: "Prepare cluster disks." - include_tasks: "disks.yml" - with_items: "{{ groups['heketi-node'] }}" - loop_control: - loop_var: "node" From 9fba448053ae7070273ab980138ce6e1300bc344 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 8 Aug 2018 14:22:50 +0200 Subject: [PATCH 28/35] recator to use kube module, finally fix race condition in storage tasks --- .../tasks/{setup.yml => bootstrap.yml} | 16 ++--- .../provision/tasks/bootstrap/deploy.yml | 23 +++++++ .../tasks/{setup => bootstrap}/storage.yml | 13 ++-- .../tear-down.yml} | 0 .../tasks/{setup => bootstrap}/topology.yml | 1 + .../tasks/{setup => bootstrap}/volumes.yml | 6 -- .../roles/provision/tasks/glusterfs.yml | 37 +++++++++++ .../tasks/{kubernetes => glusterfs}/label.yml | 0 .../heketi/roles/provision/tasks/heketi.yml | 25 ++++++++ .../roles/provision/tasks/kubernetes.yml | 64 ------------------- .../heketi/roles/provision/tasks/main.yml | 45 ++++++------- .../heketi/roles/provision/tasks/secret.yml | 27 ++++++++ .../roles/provision/tasks/setup/boot.yml | 27 -------- .../roles/provision/tasks/setup/heketi.yml | 10 --- .../heketi/roles/provision/tasks/storage.yml | 11 ++++ .../roles/provision/tasks/storageclass.yml | 9 ++- .../heketi/roles/provision/tasks/topology.yml | 25 ++++++++ .../templates/heketi-storage.json.j2 | 54 ++++++++++++++++ 18 files changed, 242 insertions(+), 151 deletions(-) rename contrib/network-storage/heketi/roles/provision/tasks/{setup.yml => bootstrap.yml} (89%) create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml rename contrib/network-storage/heketi/roles/provision/tasks/{setup => bootstrap}/storage.yml (81%) rename contrib/network-storage/heketi/roles/provision/tasks/{setup/tear-down-bootstrap.yml => bootstrap/tear-down.yml} (100%) rename contrib/network-storage/heketi/roles/provision/tasks/{setup => bootstrap}/topology.yml (97%) rename contrib/network-storage/heketi/roles/provision/tasks/{setup => bootstrap}/volumes.yml (89%) create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml rename contrib/network-storage/heketi/roles/provision/tasks/{kubernetes => glusterfs}/label.yml (100%) create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/heketi.yml delete mode 100644 contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/secret.yml delete mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml delete mode 100644 contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/storage.yml create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/topology.yml create mode 100644 contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml similarity index 89% rename from contrib/network-storage/heketi/roles/provision/tasks/setup.yml rename to contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml index 96462dd11..a21cfadd6 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml @@ -1,4 +1,3 @@ ---- # Bootstrap heketi - name: "Get state of heketi service, deployment and pods." register: "initial_heketi_state" @@ -9,7 +8,7 @@ - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" - include_tasks: "setup/boot.yml" + include_tasks: "bootstrap/deploy.yml" # Prepare heketi topology - name: "Get heketi initial pod state." @@ -27,11 +26,11 @@ command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" - include_tasks: "setup/topology.yml" + include_tasks: "bootstrap/topology.yml" # Provision heketi database volume - name: "Prepare heketi volumes." - include_tasks: "setup/volumes.yml" + include_tasks: "bootstrap/volumes.yml" # Prepare heketi storage - name: "Test heketi storage." @@ -41,7 +40,7 @@ - command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" register: "job" - name: "Create heketi storage." - include_tasks: "setup/storage.yml" + include_tasks: "bootstrap/storage.yml" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" @@ -52,8 +51,7 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" -# Finalize setup + +# Remove bootstrap heketi - name: "Tear down bootstrap." - include_tasks: "setup/tear-down-bootstrap.yml" -- name: "Setup final heketi instance." - include_tasks: "setup/heketi.yml" + include_tasks: "bootstrap/tear-down.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml new file mode 100644 index 000000000..3580707d5 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml @@ -0,0 +1,23 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi Bootstrap" + become: true + template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Bootstrap" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-bootstrap.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Wait for heketi bootstrap to complete." + changed_when: false + register: "initial_heketi_state" + vars: + initial_heketi_state: { stdout: "{}" } + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + until: + - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml similarity index 81% rename from contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml rename to contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml index 7c49a2258..1daa72ac1 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml @@ -4,7 +4,10 @@ changed_when: false register: "heketi_storage_state" - name: "Create heketi storage." - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-storage.json" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-storage.json" + state: "present" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" @@ -16,17 +19,17 @@ - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" register: "heketi_storage_result" -- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" - register: "heketi_storage_state" + - name: "Get state of heketi storage service, endpoint, secret and job." command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" vars: + heketi_storage_state: { stdout: "{}" } secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" - job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete'].status=='True']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]" until: - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" @@ -34,5 +37,3 @@ - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" retries: 60 delay: 5 -# looks like there is some race condition that leads to "Database file did not appear, exiting.", can't figure out where -- command: "sleep 10" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml similarity index 100% rename from contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml rename to contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml similarity index 97% rename from contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml rename to contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml index fd148d8d9..8c29aa1a6 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml @@ -13,6 +13,7 @@ - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" + register: "load_heketi" - name: "Get heketi topology." register: "heketi_topology" command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml similarity index 89% rename from contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml rename to contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml index 0d24a3a0b..e6226a7c3 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml @@ -31,12 +31,6 @@ with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" -- name: "debug heketi db vol." - with_items: "{{ volumes_information.results }}" - loop_control: { loop_var: "volume_information" } - vars: { volume: "{{ volume_information.stdout|from_json }}" } - when: "volume.name == 'heketidbstorage'" - debug: { var: "volume" } - name: "Test heketi database volume." set_fact: { heketi_database_volume_created: true } with_items: "{{ volumes_information.results }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml new file mode 100644 index 000000000..e9650276a --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml @@ -0,0 +1,37 @@ +--- +- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset" + template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" } + become: true + register: "rendering" +- name: "Kubernetes Apps | Install and configure GlusterFS daemonset" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/glusterfs-daemonset.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Kubernetes Apps | Wait for daemonset to become available." + register: "daemonset_state" + command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + changed_when: false + vars: + daemonset_state: { stdout: "{}" } + ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" + desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" + until: "ready == desired" + retries: 60 + delay: 5 + +- name: "Kubernetes Apps | Label GlusterFS nodes" + include_tasks: "glusterfs/label.yml" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" + +- name: "Kubernetes Apps | Lay Down Heketi Service Account" + template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" } + become: true + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Service Account" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-service-account.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml similarity index 100% rename from contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml rename to contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml diff --git a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml new file mode 100644 index 000000000..44016df5e --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml @@ -0,0 +1,25 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi" + become: true + template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-deployment.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Ensure heketi is up and running." + changed_when: false + register: "heketi_state" + vars: + heketi_state: { stdout: "{}" } + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json" + until: + - "heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 60 + delay: 5 +- set_fact: + heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml deleted file mode 100644 index 6ab92713b..000000000 --- a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- register: "daemonset_state" - command: "{{ bin_dir }}/kubectl get daemonset glusterfs -o=name --ignore-not-found=true" - changed_when: false -- name: "Deploy the GlusterFS DaemonSet" - when: "daemonset_state.stdout == \"\"" - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json" -- register: "daemonset_state" - command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" - changed_when: false -- name: "Wait for daemonset to become available." - register: "daemonset_state" - command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" - changed_when: false - vars: - ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" - desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" - until: "ready == desired" - retries: 60 - delay: 5 - -- name: "Label Gluster nodes" - with_items: "{{ groups['heketi-node'] }}" - loop_control: - loop_var: "node" - include_tasks: "kubernetes/label.yml" - -- register: "service_account_state" - command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" - changed_when: false -- name: "Deploy the Heketi service account" - when: "service_account_state.stdout == \"\"" - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-service-account.json" -- register: "service_account_state" - command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" - changed_when: false -- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } - -- register: "clusterrolebinding_state" - command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" - changed_when: false -- name: "Deploy cluster role binding." - when: "clusterrolebinding_state.stdout == \"\"" - command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" -- register: "clusterrolebinding_state" - command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" - changed_when: false -- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } - -- register: "secret_state" - command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" - changed_when: false -- name: "Render Heketi secret configuration." - become: true - template: - src: "heketi.json.j2" - dest: "{{ kube_config_dir }}/heketi.json" -- name: "Deploy Heketi config secret" - when: "secret_state.stdout == \"\"" - command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" -- register: "secret_state" - command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" - changed_when: false -- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml index c247929a7..cae035893 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -1,34 +1,27 @@ --- -- name: "Render configuration." - become: true - template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" } - with_items: - - { file: "glusterfs-daemonset.json" } - - { file: "heketi-bootstrap.json" } - - { file: "heketi-deployment.json" } - - { file: "heketi-service-account.json" } -- name: "Prepare kubernetes." - include_tasks: "kubernetes.yml" +- name: "Kubernetes Apps | GlusterFS" + include_tasks: "glusterfs.yml" -- name: "Test heketi setup." +- name: "Kubernetes Apps | Heketi Secrets" + include_tasks: "secret.yml" + +- name: "Kubernetes Apps | Test Heketi" register: "heketi_service_state" - command: "{{ bin_dir }}/kubectl get service heketi -o=name --ignore-not-found=true" + command: "kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" changed_when: false -- name: "Setup heketi." +- name: "Kubernetes Apps | Bootstrap Heketi" when: "heketi_service_state.stdout == \"\"" - include_tasks: "setup.yml" + include_tasks: "bootstrap.yml" -- name: "Test storage class." - changed_when: false - command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" - register: "storageclass" -- name: "Setup storage class." - when: "storageclass.stdout == \"\"" +- name: "Kubernetes Apps | Heketi" + include_tasks: "heketi.yml" + +- name: "Kubernetes Apps | Heketi Topology" + include_tasks: "topology.yml" + +- name: "Kubernetes Apps | Heketi Storage" + include_tasks: "storage.yml" + +- name: "Kubernetes Apps | Storage Class" include_tasks: "storageclass.yml" -- name: "Test storage class." - changed_when: false - command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" - register: "storageclass" -- name: "Ensure storage class is up." - assert: { that: "storageclass.stdout != \"\"" } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml new file mode 100644 index 000000000..699eaf34b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml @@ -0,0 +1,27 @@ +--- +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- name: "Kubernetes Apps | Deploy cluster role binding." + when: "clusterrolebinding_state.stdout == \"\"" + command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } + +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- name: "Render Heketi secret configuration." + become: true + template: + src: "heketi.json.j2" + dest: "{{ kube_config_dir }}/heketi.json" +- name: "Deploy Heketi config secret" + when: "secret_state.stdout == \"\"" + command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml deleted file mode 100644 index 806e91570..000000000 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: "Get state of heketi service, deployment and pods." - register: "initial_heketi_state" - changed_when: false - command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" -- name: "Create Heketi initial service and deployment" - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json" - when: - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" - - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" -- name: "Get state of heketi service, deployment and pods." - register: "initial_heketi_state" - changed_when: false - command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" -- name: "Wait for heketi bootstrap to complete." - changed_when: false - register: "initial_heketi_state" - vars: - pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" - deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" - command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" - until: - - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" - - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" - retries: 60 - delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml deleted file mode 100644 index c38b151ac..000000000 --- a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: "Create long term Heketi instance." - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-deployment.json" -- name: "Get heketi deployment state." - register: "heketi_deployment_state" - command: "{{ bin_dir }}/kubectl get deployment heketi -o=name --ignore-not-found=true" - changed_when: false -- name: "Ensure heketi is up and running." - assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } - diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml new file mode 100644 index 000000000..881084bbe --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml @@ -0,0 +1,11 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi Storage" + become: true + vars: { nodes: "{{ groups['heketi-node'] }}" } + template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Storage" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-storage.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml index 393d76e64..835dc78b2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -16,6 +16,9 @@ template: src: "storageclass.yml.j2" dest: "{{ kube_config_dir }}/storageclass.yml" -- name: "Setup storage class." - when: "storageclass.stdout == \"\"" - command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/storageclass.yml" + register: "rendering" +- name: "Kubernetes Apps | Install and configure Storace Class" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/storageclass.yml" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml new file mode 100644 index 000000000..52c709f37 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml @@ -0,0 +1,25 @@ +--- +- name: "Get heketi topology." + register: "heketi_topology" + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" +- name: "Render heketi topology template." + become: true + vars: { nodes: "{{ groups['heketi-node'] }}" } + register: "rendering" + template: + src: "topology.json.j2" + dest: "{{ kube_config_dir }}/topology.json" +- name: "Copy topology configuration into container." + when: "rendering.changed" + command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" +- name: "Load heketi topology." + when: "rendering.changed" + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" +- name: "Get heketi topology." + register: "heketi_topology" + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" + until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 new file mode 100644 index 000000000..3089256c9 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 @@ -0,0 +1,54 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "kind": "Endpoints", + "apiVersion": "v1", + "metadata": { + "name": "heketi-storage-endpoints", + "creationTimestamp": null + }, + "subsets": [ +{% set nodeblocks = [] %} +{% for node in nodes %} +{% set nodeblock %} + { + "addresses": [ + { + "ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" + } + ], + "ports": [ + { + "port": 1 + } + ] + } +{% endset %} +{% if nodeblocks.append(nodeblock) %}{% endif %} +{% endfor %} +{{ nodeblocks|join(',') }} + ] + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi-storage-endpoints", + "creationTimestamp": null + }, + "spec": { + "ports": [ + { + "port": 1, + "targetPort": 0 + } + ] + }, + "status": { + "loadBalancer": {} + } + } + ] +} From 19e28684844f957d8a2e11c5c5cf6d594ba99c21 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Fri, 10 Aug 2018 13:42:28 +0200 Subject: [PATCH 29/35] fix path to bootstrap tear down --- contrib/network-storage/heketi/roles/tear-down/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml index 5c5e69ab8..ddc56b256 100644 --- a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -9,7 +9,7 @@ command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\"" ignore_errors: true - name: "Tear down bootstrap." - include_tasks: "../provision/tasks/setup/tear-down-bootstrap.yml" + include_tasks: "../provision/tasks/bootstrap/tear-down.yml" - name: "Ensure there is nothing left over." command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json" register: "heketi_result" From 949984601ffd147ef306ed96715308921c7344f6 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 15:58:44 +0200 Subject: [PATCH 30/35] actually use heketi auth --- .../roles/provision/tasks/bootstrap/topology.yml | 6 +++--- .../heketi/roles/provision/tasks/bootstrap/volumes.yml | 10 +++++----- .../heketi/roles/provision/tasks/topology.yml | 4 ++-- .../heketi/roles/provision/templates/heketi.json.j2 | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml index 8c29aa1a6..31d1d48cb 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml @@ -1,7 +1,7 @@ --- - name: "Get heketi topology." register: "heketi_topology" - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" - name: "Render heketi topology template." become: true vars: { nodes: "{{ groups['heketi-node'] }}" } @@ -12,11 +12,11 @@ command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json" register: "load_heketi" - name: "Get heketi topology." register: "heketi_topology" - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml index e6226a7c3..bb088173e 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml @@ -1,11 +1,11 @@ --- - name: "Get heketi volume ids." - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" @@ -16,18 +16,18 @@ vars: { volume: "{{ volume_information.stdout|from_json }}" } when: "volume.name == 'heketidbstorage'" - name: "Provision database volume." - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} setup-openshift-heketi-storage" when: "heketi_database_volume_exists is undefined" - name: "Copy configuration from pod." become: true command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json" - name: "Get heketi volume ids." - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json" changed_when: false register: "heketi_volumes" - name: "Get heketi volumes." changed_when: false - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json" with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" loop_control: { loop_var: "volume_id" } register: "volumes_information" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml index 52c709f37..b9cb3e0d2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml @@ -15,11 +15,11 @@ command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" - name: "Load heketi topology." when: "rendering.changed" - command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json" + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json" - name: "Get heketi topology." register: "heketi_topology" changed_when: false - command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" retries: 60 delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 index 7ba980dca..5861b684b 100644 --- a/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 @@ -3,7 +3,7 @@ "port": "8080", "_use_auth": "Enable JWT authorization. Please enable for deployment", - "use_auth": false, + "use_auth": true, "_jwt": "Private keys for access", "jwt": { From ce776f0f6aae6ddec6d7f47864970198e74cab87 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 15:59:56 +0200 Subject: [PATCH 31/35] actually use heketi auth --- .../network-storage/heketi/roles/provision/tasks/bootstrap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml index a21cfadd6..788f01f4b 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml @@ -23,7 +23,7 @@ - name: "Test heketi topology." changed_when: false register: "heketi_topology" - command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" - name: "Load heketi topology." when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" include_tasks: "bootstrap/topology.yml" From 64b0ce974dea21451901728ab2aa604251f3e736 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 16:02:55 +0200 Subject: [PATCH 32/35] use bin_dir variable --- .../heketi/roles/provision/tasks/main.yml | 2 +- .../heketi/roles/provision/tasks/secret.yml | 12 ++++++------ .../heketi/roles/provision/tasks/storageclass.yml | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml index cae035893..4adc2dbb2 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -7,7 +7,7 @@ - name: "Kubernetes Apps | Test Heketi" register: "heketi_service_state" - command: "kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" + command: "{{bin_dir}}/kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" changed_when: false - name: "Kubernetes Apps | Bootstrap Heketi" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml index 699eaf34b..8ca21bcb6 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml @@ -1,17 +1,17 @@ --- - register: "clusterrolebinding_state" - command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" changed_when: false - name: "Kubernetes Apps | Deploy cluster role binding." when: "clusterrolebinding_state.stdout == \"\"" - command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" + command: "{{bin_dir}}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" - register: "clusterrolebinding_state" - command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" changed_when: false - assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } - register: "secret_state" - command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false - name: "Render Heketi secret configuration." become: true @@ -20,8 +20,8 @@ dest: "{{ kube_config_dir }}/heketi.json" - name: "Deploy Heketi config secret" when: "secret_state.stdout == \"\"" - command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" + command: "{{bin_dir}}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" - register: "secret_state" - command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" changed_when: false - assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml index 835dc78b2..afd818eb3 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -20,5 +20,6 @@ - name: "Kubernetes Apps | Install and configure Storace Class" kube: name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" filename: "{{ kube_config_dir }}/storageclass.yml" state: "{{ rendering.changed | ternary('latest', 'present') }}" From 6ffddbff24cec4cfc42def0b466f504aac8f5610 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 16:03:32 +0200 Subject: [PATCH 33/35] fix database not available heketi error --- .../roles/provision/tasks/bootstrap.yml | 11 ++++----- .../provision/tasks/bootstrap/deploy.yml | 1 + .../provision/tasks/bootstrap/storage.yml | 23 +++++++++++-------- .../provision/tasks/bootstrap/topology.yml | 6 ++++- .../provision/tasks/bootstrap/volumes.yml | 2 +- .../roles/provision/tasks/glusterfs.yml | 15 ++++++------ .../heketi/roles/provision/tasks/heketi.yml | 1 + .../heketi/roles/provision/tasks/main.yml | 3 +++ .../heketi/roles/provision/tasks/topology.yml | 2 +- .../templates/glusterfs-daemonset.json.j2 | 2 +- 10 files changed, 39 insertions(+), 27 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml index 788f01f4b..572913a63 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml @@ -32,13 +32,16 @@ - name: "Prepare heketi volumes." include_tasks: "bootstrap/volumes.yml" +# Remove bootstrap heketi +- name: "Tear down bootstrap." + include_tasks: "bootstrap/tear-down.yml" + # Prepare heketi storage - name: "Test heketi storage." command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" changed_when: false register: "heketi_storage_state" -- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" - register: "job" +# ensure endpoints actually exist before trying to move database data to it - name: "Create heketi storage." include_tasks: "bootstrap/storage.yml" vars: @@ -51,7 +54,3 @@ - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" - -# Remove bootstrap heketi -- name: "Tear down bootstrap." - include_tasks: "bootstrap/tear-down.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml index 3580707d5..3037d8b77 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml @@ -6,6 +6,7 @@ - name: "Kubernetes Apps | Install and configure Heketi Bootstrap" kube: name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" filename: "{{ kube_config_dir }}/heketi-bootstrap.json" state: "{{ rendering.changed | ternary('latest', 'present') }}" - name: "Wait for heketi bootstrap to complete." diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml index 1daa72ac1..ff32ef201 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml @@ -6,7 +6,8 @@ - name: "Create heketi storage." kube: name: "GlusterFS" - filename: "{{ kube_config_dir }}/heketi-storage.json" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json" state: "present" vars: secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" @@ -19,21 +20,23 @@ - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" register: "heketi_storage_result" - -- name: "Get state of heketi storage service, endpoint, secret and job." - command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" +- name: "Get state of heketi database copy job." + command: "{{ bin_dir }}/kubectl get jobs --output=json" changed_when: false register: "heketi_storage_state" vars: heketi_storage_state: { stdout: "{}" } - secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" - endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" - service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]" until: - - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" - - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" - - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" retries: 60 delay: 5 +- become: true + template: { src: "heketi-storage-test.json.j2", dest: "{{ kube_config_dir }}/heketi-storage-test.json" } + register: "rendering" +- kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-storage-test.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" + register: "state" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml index 31d1d48cb..7d2c5981e 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml @@ -1,20 +1,24 @@ --- - name: "Get heketi topology." + changed_when: false register: "heketi_topology" command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" - name: "Render heketi topology template." become: true vars: { nodes: "{{ groups['heketi-node'] }}" } + register: "render" template: src: "topology.json.j2" dest: "{{ kube_config_dir }}/topology.json" - name: "Copy topology configuration into container." + changed_when: false command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" - name: "Load heketi topology." - when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + when: "render.changed" command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json" register: "load_heketi" - name: "Get heketi topology." + changed_when: false register: "heketi_topology" command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml index bb088173e..d5da1a125 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml @@ -20,7 +20,7 @@ when: "heketi_database_volume_exists is undefined" - name: "Copy configuration from pod." become: true - command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json" + command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json" - name: "Get heketi volume ids." command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json" changed_when: false diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml index e9650276a..e46159969 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml @@ -6,8 +6,14 @@ - name: "Kubernetes Apps | Install and configure GlusterFS daemonset" kube: name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" filename: "{{ kube_config_dir }}/glusterfs-daemonset.json" state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Kubernetes Apps | Label GlusterFS nodes" + include_tasks: "glusterfs/label.yml" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" - name: "Kubernetes Apps | Wait for daemonset to become available." register: "daemonset_state" command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" @@ -16,16 +22,10 @@ daemonset_state: { stdout: "{}" } ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" - until: "ready == desired" + until: "ready >= 3" retries: 60 delay: 5 -- name: "Kubernetes Apps | Label GlusterFS nodes" - include_tasks: "glusterfs/label.yml" - with_items: "{{ groups['heketi-node'] }}" - loop_control: - loop_var: "node" - - name: "Kubernetes Apps | Lay Down Heketi Service Account" template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" } become: true @@ -33,5 +33,6 @@ - name: "Kubernetes Apps | Install and configure Heketi Service Account" kube: name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" filename: "{{ kube_config_dir }}/heketi-service-account.json" state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml index 44016df5e..029baef94 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml @@ -6,6 +6,7 @@ - name: "Kubernetes Apps | Install and configure Heketi" kube: name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" filename: "{{ kube_config_dir }}/heketi-deployment.json" state: "{{ rendering.changed | ternary('latest', 'present') }}" - name: "Ensure heketi is up and running." diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml index 4adc2dbb2..23a2b4f9c 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -25,3 +25,6 @@ - name: "Kubernetes Apps | Storage Class" include_tasks: "storageclass.yml" + +- name: "Clean up" + include_tasks: "cleanup.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml index b9cb3e0d2..dd1e272be 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml @@ -2,7 +2,7 @@ - name: "Get heketi topology." register: "heketi_topology" changed_when: false - command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json" + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" - name: "Render heketi topology template." become: true vars: { nodes: "{{ groups['heketi-node'] }}" } diff --git a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 index 8f46bea24..eddd57eb8 100644 --- a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 +++ b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 @@ -27,7 +27,7 @@ "containers": [ { "image": "gluster/gluster-centos:gluster4u0_centos7", - "imagePullPolicy": "Always", + "imagePullPolicy": "IfNotPresent", "name": "glusterfs", "volumeMounts": [ { From 5cf1396cb7ab37546f54ea77cab7cf19ec0d924f Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 17:17:49 +0200 Subject: [PATCH 34/35] removes unnecessary check --- .../heketi/roles/provision/tasks/bootstrap/storage.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml index ff32ef201..be3c42caf 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml @@ -31,12 +31,3 @@ - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" retries: 60 delay: 5 -- become: true - template: { src: "heketi-storage-test.json.j2", dest: "{{ kube_config_dir }}/heketi-storage-test.json" } - register: "rendering" -- kube: - name: "GlusterFS" - kubectl: "{{bin_dir}}/kubectl" - filename: "{{ kube_config_dir }}/heketi-storage-test.json" - state: "{{ rendering.changed | ternary('latest', 'present') }}" - register: "state" From df6cf9aa5156f5155e0e4c369e678b18fc5c0bcb Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Wed, 5 Sep 2018 17:18:53 +0200 Subject: [PATCH 35/35] add cleanup --- .../network-storage/heketi/roles/provision/tasks/cleanup.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml diff --git a/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml b/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml new file mode 100644 index 000000000..238f29bc2 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml @@ -0,0 +1,4 @@ +--- +- name: "Clean up left over jobs." + command: "{{ bin_dir }}/kubectl delete jobs,pods --selector=\"deploy-heketi\"" + changed_when: false