diff --git a/README.md b/README.md index 1bd978e0e..32943c7b2 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,6 @@ Note: - [metallb](https://metallb.universe.tf/) 0.13.9 - [registry](https://github.com/distribution/distribution) 2.8.1 - Storage Plugin - - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11 - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11 - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0 - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0 diff --git a/docs/_sidebar.md b/docs/_sidebar.md index b8acf2322..8400edcb3 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -52,7 +52,6 @@ * [Test Cases](/docs/developers/test_cases.md) * [Vagrant](/docs/developers/vagrant.md) * External Storage Provisioners - * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md) * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md) * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md) * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md) diff --git a/docs/ansible/ansible.md b/docs/ansible/ansible.md index 7ba31cf7b..eb960821a 100644 --- a/docs/ansible/ansible.md +++ b/docs/ansible/ansible.md @@ -65,7 +65,6 @@ The following tags are defined in playbooks: | bootstrap-os | Anything related to host OS configuration | | calico | Network plugin Calico | | calico_rr | Configuring Calico route reflector | -| cephfs-provisioner | Configuring CephFS | | cert-manager | Configuring certificate manager for K8s | | cilium | Network plugin Cilium | | cinder-csi-driver | Configuring csi driver: cinder | diff --git a/docs/external_storage_provisioners/cephfs_provisioner.md b/docs/external_storage_provisioners/cephfs_provisioner.md deleted file mode 100644 index c5c18db1f..000000000 --- a/docs/external_storage_provisioners/cephfs_provisioner.md +++ /dev/null @@ -1,73 +0,0 @@ -# CephFS Volume Provisioner for Kubernetes 1.5+ - -[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner) - -Using Ceph volume client - -## Development - -Compile the provisioner - -``` console -make -``` - -Make the container image and push to the registry - -``` console -make push -``` - -## Test instruction - -- Start Kubernetes local cluster - -See [Kubernetes](https://kubernetes.io/) - -- Create a Ceph admin secret - -``` bash -ceph auth get client.admin 2>&1 |grep "key = " |awk '{print $3'} |xargs echo -n > /tmp/secret -kubectl create ns cephfs -kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs -``` - -- Start CephFS provisioner - -The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity. - -``` bash -docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1 -``` - -Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md). - -- Create a CephFS Storage Class - -Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class: - -``` bash -kubectl create -f example/class.yaml -``` - -- Create a claim - -``` bash -kubectl create -f example/claim.yaml -``` - -- Create a Pod using the claim - -``` bash -kubectl create -f example/test-pod.yaml -``` - -## Known limitations - -- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work. -- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated. -- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount. - -## Acknowledgement - -Inspired by CephFS Manila provisioner and conversation with John Spray diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index 64b6bd702..c964bf1d0 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -65,18 +65,6 @@ local_volume_provisioner_enabled: false # csi snapshot namespace # snapshot_controller_namespace: kube-system -# CephFS provisioner deployment -cephfs_provisioner_enabled: false -# cephfs_provisioner_namespace: "cephfs-provisioner" -# cephfs_provisioner_cluster: ceph -# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" -# cephfs_provisioner_admin_id: admin -# cephfs_provisioner_secret: secret -# cephfs_provisioner_storage_class: cephfs -# cephfs_provisioner_reclaim_policy: Delete -# cephfs_provisioner_claim_root: /volumes -# cephfs_provisioner_deterministic_names: true - # RBD provisioner deployment rbd_provisioner_enabled: false # rbd_provisioner_namespace: rbd-provisioner diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml deleted file mode 100644 index 577fbff1e..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -cephfs_provisioner_namespace: "cephfs-provisioner" -cephfs_provisioner_cluster: ceph -cephfs_provisioner_monitors: ~ -cephfs_provisioner_admin_id: admin -cephfs_provisioner_secret: secret -cephfs_provisioner_storage_class: cephfs -cephfs_provisioner_reclaim_policy: Delete -cephfs_provisioner_claim_root: /volumes -cephfs_provisioner_deterministic_names: true diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml deleted file mode 100644 index 4993eebab..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- - -- name: CephFS Provisioner | Remove legacy addon dir and manifests - file: - path: "{{ kube_config_dir }}/addons/cephfs_provisioner" - state: absent - when: - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - upgrade - -- name: CephFS Provisioner | Remove legacy namespace - command: > - {{ kubectl }} delete namespace {{ cephfs_provisioner_namespace }} - ignore_errors: true # noqa ignore-errors - when: - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - upgrade - -- name: CephFS Provisioner | Remove legacy storageclass - command: > - {{ kubectl }} delete storageclass {{ cephfs_provisioner_storage_class }} - ignore_errors: true # noqa ignore-errors - when: - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - upgrade - -- name: CephFS Provisioner | Create addon dir - file: - path: "{{ kube_config_dir }}/addons/cephfs_provisioner" - state: directory - owner: root - group: root - mode: "0755" - when: - - inventory_hostname == groups['kube_control_plane'][0] - -- name: CephFS Provisioner | Templates list - set_fact: - cephfs_provisioner_templates: - - { name: 00-namespace, file: 00-namespace.yml, type: ns } - - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret } - - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa } - - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole } - - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding } - - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role } - - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding } - - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy } - - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc } - -- name: CephFS Provisioner | Create manifests - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}" - mode: "0644" - with_items: "{{ cephfs_provisioner_templates }}" - register: cephfs_provisioner_manifests - when: inventory_hostname == groups['kube_control_plane'][0] - -- name: CephFS Provisioner | Apply manifests - kube: - name: "{{ item.item.name }}" - namespace: "{{ cephfs_provisioner_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item.item.type }}" - filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}" - state: "latest" - with_items: "{{ cephfs_provisioner_manifests.results }}" - when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 deleted file mode 100644 index 2a2a67cf6..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ cephfs_provisioner_namespace }} - labels: - name: {{ cephfs_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 deleted file mode 100644 index c6a149086..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,22 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "delete"] diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 deleted file mode 100644 index cc5d5ff5b..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cephfs-provisioner -subjects: - - kind: ServiceAccount - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -roleRef: - kind: ClusterRole - name: cephfs-provisioner - apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 deleted file mode 100644 index 8d9eb08bb..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} - labels: - app: cephfs-provisioner - version: {{ cephfs_provisioner_image_tag }} -spec: - replicas: 1 - selector: - matchLabels: - app: cephfs-provisioner - version: {{ cephfs_provisioner_image_tag }} - template: - metadata: - labels: - app: cephfs-provisioner - version: {{ cephfs_provisioner_image_tag }} - spec: - priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} - serviceAccount: cephfs-provisioner - containers: - - name: cephfs-provisioner - image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }} - imagePullPolicy: {{ k8s_image_pull_policy }} - env: - - name: PROVISIONER_NAME - value: ceph.com/cephfs - command: - - "/usr/local/bin/cephfs-provisioner" - args: - - "-id=cephfs-provisioner-1" diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 deleted file mode 100644 index 1fb80a13a..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "delete"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 deleted file mode 100644 index 01ab87b7d..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -subjects: - - kind: ServiceAccount - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cephfs-provisioner diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 deleted file mode 100644 index 31f87bdc4..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 deleted file mode 100644 index dd0e37eb5..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: {{ cephfs_provisioner_storage_class }} -provisioner: ceph.com/cephfs -reclaimPolicy: {{ cephfs_provisioner_reclaim_policy }} -parameters: - cluster: {{ cephfs_provisioner_cluster }} - monitors: {{ cephfs_provisioner_monitors }} - adminId: {{ cephfs_provisioner_admin_id }} - adminSecretName: cephfs-provisioner - adminSecretNamespace: {{ cephfs_provisioner_namespace }} - claimRoot: {{ cephfs_provisioner_claim_root }} - deterministicNames: "{{ cephfs_provisioner_deterministic_names | bool | lower }}" diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 deleted file mode 100644 index 6d73c0c15..000000000 --- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: Secret -apiVersion: v1 -metadata: - name: cephfs-provisioner - namespace: {{ cephfs_provisioner_namespace }} -type: Opaque -data: - secret: {{ cephfs_provisioner_secret | b64encode }} diff --git a/roles/kubernetes-apps/external_provisioner/meta/main.yml b/roles/kubernetes-apps/external_provisioner/meta/main.yml index 13bc8b6e8..19123d76e 100644 --- a/roles/kubernetes-apps/external_provisioner/meta/main.yml +++ b/roles/kubernetes-apps/external_provisioner/meta/main.yml @@ -9,13 +9,6 @@ dependencies: - local-volume-provisioner - external-provisioner - - role: kubernetes-apps/external_provisioner/cephfs_provisioner - when: cephfs_provisioner_enabled - tags: - - apps - - cephfs-provisioner - - external-provisioner - - role: kubernetes-apps/external_provisioner/rbd_provisioner when: rbd_provisioner_enabled tags: diff --git a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml index 4aa69382e..4a9792ba6 100644 --- a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml @@ -15,7 +15,6 @@ - not calico_apiserver_version.startswith('v') - not calico_ctl_version.startswith('v') - not calico_typha_version.startswith('v') - - not cephfs_provisioner_version.startswith('v') - not cert_manager_version.startswith('v') - not cilium_cli_version.startswith('v') - not cilium_version.startswith('v') diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 96ed04aca..1c20a0130 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -314,9 +314,6 @@ metrics_server_image_tag: "v{{ metrics_server_version }}" local_volume_provisioner_version: "2.5.0" local_volume_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/local-volume-provisioner" local_volume_provisioner_image_tag: "v{{ local_volume_provisioner_version }}" -cephfs_provisioner_version: "2.1.0-k8s1.11" -cephfs_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/cephfs-provisioner" -cephfs_provisioner_image_tag: "v{{ cephfs_provisioner_version }}" rbd_provisioner_version: "2.1.1-k8s1.11" rbd_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/rbd-provisioner" rbd_provisioner_image_tag: "v{{ rbd_provisioner_version }}" @@ -947,15 +944,6 @@ downloads: groups: - kube_node - cephfs_provisioner: - enabled: "{{ cephfs_provisioner_enabled }}" - container: true - repo: "{{ cephfs_provisioner_image_repo }}" - tag: "{{ cephfs_provisioner_image_tag }}" - checksum: "{{ cephfs_provisioner_digest_checksum | default(None) }}" - groups: - - kube_node - rbd_provisioner: enabled: "{{ rbd_provisioner_enabled }}" container: true diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index ba37674b2..5c40d3ac8 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -435,7 +435,6 @@ vsphere_csi_enabled: false upcloud_csi_enabled: false csi_snapshot_controller_enabled: false persistent_volumes_enabled: false -cephfs_provisioner_enabled: false rbd_provisioner_enabled: false ingress_nginx_enabled: false ingress_alb_enabled: false diff --git a/scripts/readme_versions.md.j2 b/scripts/readme_versions.md.j2 index d8742ca0b..2d58abe3f 100644 --- a/scripts/readme_versions.md.j2 +++ b/scripts/readme_versions.md.j2 @@ -23,7 +23,6 @@ - [metallb](https://metallb.universe.tf/) {{ metallb_version }} - [registry](https://github.com/distribution/distribution) {{ registry_version }} - Storage Plugin - - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ cephfs_provisioner_version }} - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ rbd_provisioner_version }} - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) {{ aws_ebs_csi_plugin_version }} - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) {{ azure_csi_plugin_version }}