add heketi/glusterfs as additional contributional network storage

contrib/network-storage/heketi/roles/heketi.yml

@@ -0,0 +1,10 @@
+---
+- hosts: heketi-node
+  roles:
+    - { role: prepare }
+
+- hosts: localhost
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: "Load lvm kernel modules"
+  become: true
+  with_items:
+      - "dm_snapshot"
+      - "dm_mirror"
+      - "dm_thin_pool"
+  modprobe:
+      name: "{{ item }}"
+      state: "present"
+
+- name: "Install glusterfs mount utils"
+  become: true
+  yum:
+      name: "glusterfs-fuse"
+      state: "present"

contrib/network-storage/heketi/roles/provision/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts"

contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json

@@ -0,0 +1,144 @@
+{
+    "kind": "DaemonSet",
+    "apiVersion": "extensions/v1beta1",
+    "metadata": {
+        "name": "glusterfs",
+        "labels": {
+            "glusterfs": "deployment"
+        },
+        "annotations": {
+            "description": "GlusterFS Daemon Set",
+            "tags": "glusterfs"
+        }
+    },
+    "spec": {
+        "template": {
+            "metadata": {
+                "name": "glusterfs",
+                "labels": {
+                    "glusterfs-node": "daemonset"
+                }
+            },
+            "spec": {
+                "nodeSelector": {
+                    "storagenode" : "glusterfs"
+                },
+                "hostNetwork": true,
+                "containers": [
+                    {
+                        "image": "gluster/gluster-centos:latest",
+                        "imagePullPolicy": "Always",
+                        "name": "glusterfs",
+                        "volumeMounts": [
+                            {
+                                "name": "glusterfs-heketi",
+                                "mountPath": "/var/lib/heketi"
+                            },
+                            {
+                                "name": "glusterfs-run",
+                                "mountPath": "/run"
+                            },
+                            {
+                                "name": "glusterfs-lvm",
+                                "mountPath": "/run/lvm"
+                            },
+                            {
+                                "name": "glusterfs-etc",
+                                "mountPath": "/etc/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-logs",
+                                "mountPath": "/var/log/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-config",
+                                "mountPath": "/var/lib/glusterd"
+                            },
+                            {
+                                "name": "glusterfs-dev",
+                                "mountPath": "/dev"
+                            },
+                            {
+                                "name": "glusterfs-cgroup",
+                                "mountPath": "/sys/fs/cgroup"
+                            }
+                        ],
+                        "securityContext": {
+                            "capabilities": {},
+                            "privileged": true
+                        },
+                        "readinessProbe": {
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 60,
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        },
+                        "livenessProbe": {
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 60,
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        }
+                    }
+                ],
+                "volumes": [
+                    {
+                        "name": "glusterfs-heketi",
+                        "hostPath": {
+                            "path": "/var/lib/heketi"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-run"
+                    },
+                    {
+                        "name": "glusterfs-lvm",
+                        "hostPath": {
+                            "path": "/run/lvm"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-etc",
+                        "hostPath": {
+                            "path": "/etc/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-logs",
+                        "hostPath": {
+                            "path": "/var/log/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-config",
+                        "hostPath": {
+                            "path": "/var/lib/glusterd"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-dev",
+                        "hostPath": {
+                            "path": "/dev"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-cgroup",
+                        "hostPath": {
+                            "path": "/sys/fs/cgroup"
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

contrib/network-storage/heketi/roles/provision/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: "stop port forwarding"
+  command: "killall "

contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json

@@ -0,0 +1,133 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "deploy-heketi"
+        },
+        "ports": [
+          {
+            "name": "deploy-heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "extensions/v1beta1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment",
+          "deploy-heketi": "deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "deploy-heketi",
+            "labels": {
+              "name": "deploy-heketi",
+              "glusterfs": "heketi-pod",
+              "deploy-heketi": "pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:dev",
+                "imagePullPolicy": "Always",
+                "name": "deploy-heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 30,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db"
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/heketi-deployment.json

@@ -0,0 +1,159 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Secret",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-db-backup",
+        "labels": {
+          "glusterfs": "heketi-db",
+          "heketi": "db"
+        }
+      },
+      "data": {
+      },
+      "type": "Opaque"
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "heketi"
+        },
+        "ports": [
+          {
+            "name": "heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "extensions/v1beta1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "heketi",
+            "labels": {
+              "name": "heketi",
+              "glusterfs": "heketi-pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:dev",
+                "imagePullPolicy": "Always",
+                "name": "heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "mountPath": "/backupdb",
+                    "name": "heketi-db-secret"
+                  },
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 30,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db",
+                "glusterfs": {
+                  "endpoints": "heketi-storage-endpoints",
+                  "path": "heketidbstorage"
+                }
+              },
+              {
+                "name": "heketi-db-secret",
+                "secret": {
+                  "secretName": "heketi-db-backup"
+                }
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/heketi-service-account.json

@@ -0,0 +1,7 @@
+{
+  "apiVersion": "v1",
+  "kind": "ServiceAccount",
+  "metadata": {
+    "name": "heketi-service-account"
+  }
+}

contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml

@@ -0,0 +1,54 @@
+---
+- register: "daemonset_state"
+  command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy the GlusterFS DaemonSet"
+  when: "daemonset_state.stdout == \"\""
+  command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
+- register: "daemonset_state"
+  command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." }
+
+- name: "Label Gluster nodes"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+  include_tasks: "kubernetes/label.yml"
+
+- register: "service_account_state"
+  command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy the Heketi service account"
+  when: "service_account_state.stdout == \"\""
+  command: "kubectl create -f {{ role_path }}/heketi-service-account.json"
+- register: "service_account_state"
+  command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." }
+
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout == \"\""
+  command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
+
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Render Heketi secret configuration."
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ artifacts_dir }}/heketi.json"
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout == \"\""
+  command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json"
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml

@@ -0,0 +1,11 @@
+---
+- register: "label_present"
+  command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- name: "Assign storage label"
+  when: "label_present.stdout_lines|length == 0"
+  command: "kubectl label node {{ node }} storagenode=glusterfs"
+- register: "label_present"
+  command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." }

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: "Prepare kubernetes."
+  include_tasks: "kubernetes.yml"
+
+- name: "Test heketi setup."
+  register: "heketi_service_state"
+  command: "kubectl get service heketi -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Setup heketi."
+  when: "heketi_service_state.stdout == \"\""
+  include_tasks: "setup.yml"
+
+- name: "Test storage class."
+  changed_when: false
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+- name: "Setup storage class."
+  when: "storageclass.stdout == \"\""
+  include_tasks: "storageclass.yml"
+- name: "Test storage class."
+  changed_when: false
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+- name: "Ensure storage class is up."
+  assert: { that: "storageclass.stdout != \"\"" }

contrib/network-storage/heketi/roles/provision/tasks/setup.yml

@@ -0,0 +1,51 @@
+---
+- name: "Test REST endpoint."
+  uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
+  register: "rest_hello_check"
+  ignore_errors: true
+
+# Bootstrap heketi
+- name: "Bootstrap heketi and start REST endpoint."
+  when: "rest_hello_check.content != \"Hello from Heketi\""
+  include_tasks: "{{ item }}"
+  with_items: [ "setup/boot.yml", "setup/rest.yml" ]
+- name: "Bootstrap heketi."
+  when: "rest_hello_check.content == \"Hello from Heketi\""
+  include_tasks: "setup/boot.yml"
+
+# Prepare heketi topology
+- name: "Test heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  include_tasks: "setup/topology.yml"
+
+# Provision heketi database volume
+- name: "Prepare heketi volumes."
+  include_tasks: "setup/volumes.yml"
+
+# Prepare heketi storage
+- name: "Test heketi storage."
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  include_tasks: "setup/storage.yml"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+
+# Finalize setup
+- name: "Tear down bootstrap."
+  include_tasks: "setup/tear-down-bootstrap.yml"
+- name: "Setup final heketi instance."
+  include_tasks: "setup/heketi.yml"

contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml

@@ -0,0 +1,34 @@
+---
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+- name: "Create Heketi initial service and deployment"
+  command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json"
+  when:
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+- name: "Ensure heketi bootstrap environment exists."
+  assert:
+    that:
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'"
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'"
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'"
+    msg: "Heketi deployment did not succeed."
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+      - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml

@@ -0,0 +1,10 @@
+---
+- name: "Create long term Heketi instance."
+  command: "kubectl create -f {{ role_path }}/heketi-deployment.json"
+- name: "Get heketi deployment state."
+  register: "heketi_deployment_state"
+  command: "kubectl get deployment heketi -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Ensure heketi is up and running."
+  assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." }
+

contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml

@@ -0,0 +1,38 @@
+---
+# Enable local REST-Interface
+- name: "Get heketi initial pod state."
+  tags: ["test"]
+  register: "initial_heketi_pod"
+  command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+- name: "Ensure heketi bootstrap pod is up."
+  tags: ["test"]
+  assert:
+    that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
+- name: "Temporarily enable local port forwarding to heketi REST interface"
+  tags: ["test"]
+  vars:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
+  command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080"
+  async: 180
+  poll: 0
+  ignore_errors: "yes"
+  register: "heketi_port_forwarding"
+  changed_when: false
+- name: "Ensure port forwarding is enabled."
+  tags: ["test"]
+  retries: 10
+  delay: 5
+  assert:
+    that:
+      - "heketi_port_forwarding.finished == 0"
+      - "heketi_port_forwarding.started == 1"
+      - "heketi_port_forwarding.failed == false"
+    msg: "Port forwarding does not work."
+- name: "Test REST endpoint."
+  tags: ["test"]
+  uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
+  register: "rest_hello_check"
+  until: "rest_hello_check.content == \"Hello from Heketi\""
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml

@@ -0,0 +1,36 @@
+---
+- name: "Test heketi storage."
+  tags: ["test"]
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  tags: ["test"]
+  command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+- name: "Get state of heketi storage service, endpoint, secret and job."
+  tags: ["test"]
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status"
+  until:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'"
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml

@@ -0,0 +1,14 @@
+---
+- name: "Get existing Heketi deploy resources."
+  command: "kubectl get all --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_resources"
+  changed_when: false
+- name: "Delete bootstrap Heketi."
+  command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
+  when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0"
+- name: "Ensure there is nothing left over."
+  command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
+  retries: 10
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml

@@ -0,0 +1,20 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- name: "Render heketi topology template."
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template:
+    src: "topology.json.j2"
+    dest: "{{ artifacts_dir }}/topology.json"
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" }
+- name: "Ensure heketi nodes are configured."
+  assert:
+    that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0"
+    msg: "Heketi topology missing."

contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml

@@ -0,0 +1,20 @@
+---
+- name: "Get heketi volume ids."
+  command: "heketi-cli -s http://localhost:48080 volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_exists: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout|from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Provision database volume."
+  command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
+  when: "heketi_database_volume_exists != true"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -0,0 +1,20 @@
+---
+- name: "Test storage class."
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+  changed_when: false
+- name: "Test heketi service."
+  command: "kubectl get service heketi --ignore-not-found=true --output=json"
+  register: "heketi_service"
+  changed_when: false
+- name: "Ensure heketi service is available."
+  assert: { that: "heketi_service.stdout != \"\""  }
+- name: "Render storage class configuration."
+  vars:
+    endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
+  template:
+    src: "storageclass.yml.j2"
+    dest: "{{ artifacts_dir }}/storageclass.yml"
+- name: "Setup storage class."
+  when: "storageclass.stdout == \"\""
+  command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml"

contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2

@@ -0,0 +1,44 @@
+{
+  "_port_comment": "Heketi Server Port Number",
+  "port": "8080",
+
+  "_use_auth": "Enable JWT authorization. Please enable for deployment",
+  "use_auth": false,
+
+  "_jwt": "Private keys for access",
+  "jwt": {
+    "_admin": "Admin has access to all APIs",
+    "admin": {
+      "key": "{{ heketi_admin_key }}"
+    },
+    "_user": "User only has access to /volumes endpoint",
+    "user": {
+      "key": "{{ heketi_user_key }}"
+    }
+  },
+
+  "_glusterfs_comment": "GlusterFS Configuration",
+  "glusterfs": {
+    "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+    "executor": "kubernetes",
+
+    "_db_comment": "Database file name",
+    "db": "/var/lib/heketi/heketi.db",
+
+    "kubeexec": {
+      "rebalance_on_expansion": true
+    },
+
+    "sshexec": {
+      "rebalance_on_expansion": true,
+      "keyfile": "/etc/heketi/private_key",
+      "fstab": "/etc/fstab",
+      "port": "22",
+      "user": "root",
+      "sudo": false
+    }
+  },
+
+  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
+  "backup_db_to_kube_secret": false
+}

contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: gluster
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://{{ endpoint_address }}:8080"
+  restuser: "user"
+  restuserkey: "{{ heketi_user_key }}"

contrib/network-storage/heketi/roles/provision/templates/topology.json.j2

@@ -0,0 +1,34 @@
+{
+  "clusters": [
+    {
+      "nodes": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "node": {
+            "hostnames": {
+              "manage": [
+                "{{ node }}"
+              ],
+              "storage": [
+                "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}"
+              ]
+            },
+            "zone": 1
+          },
+          "devices": [
+            {
+              "name": "{{ hostvars[node]['disk_volume_device_1'] }}",
+              "destroydata": false
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    }
+  ]
+}