From 90b0151cafa718b33c1e5e3bafaf76aa01a09325 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Wed, 6 Mar 2024 00:36:08 +0800 Subject: [PATCH] support node feature discovery (#10861) Signed-off-by: Kay Yan --- README.md | 1 + .../sample/group_vars/k8s_cluster/addons.yml | 9 + roles/kubernetes-apps/meta/main.yml | 7 + .../node_feature_discovery/defaults/main.yml | 16 + .../node_feature_discovery/tasks/main.yml | 49 +++ .../templates/nfd-api-crds.yaml.j2 | 361 ++++++++++++++++++ .../templates/nfd-clusterrole.yaml.j2 | 72 ++++ .../templates/nfd-clusterrolebinding.yaml.j2 | 25 ++ .../templates/nfd-gc.yaml.j2 | 42 ++ .../templates/nfd-master-conf.yaml.j2 | 12 + .../templates/nfd-master.yaml.j2 | 115 ++++++ .../templates/nfd-ns.yaml.j2 | 7 + .../templates/nfd-role.yaml.j2 | 14 + .../templates/nfd-rolebinding.yaml.j2 | 14 + .../templates/nfd-service.yaml.j2 | 18 + .../templates/nfd-serviceaccount.yaml.j2 | 22 ++ .../nfd-topologyupdater-conf.yaml.j2 | 7 + .../templates/nfd-worker-conf.yaml.j2 | 12 + .../templates/nfd-worker.yaml.j2 | 105 +++++ .../defaults/main/download.yml | 4 + tests/files/packet_rockylinux9-cilium.yml | 3 + tests/scripts/check_readme_versions.sh | 2 +- 22 files changed, 916 insertions(+), 1 deletion(-) create mode 100644 roles/kubernetes-apps/node_feature_discovery/defaults/main.yml create mode 100644 roles/kubernetes-apps/node_feature_discovery/tasks/main.yml create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2 create mode 100644 roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2 diff --git a/README.md b/README.md index 57ae5abd2..fd01875fd 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ Note: Upstart/SysV init based OS types are not supported. - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2 - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24 - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0 + - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.14.2 ## Container Runtime Notes diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index 5fc115fc6..bf0588dda 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -259,3 +259,12 @@ kube_vip_enabled: false # port: 6443 # kube_vip_interface: eth0 # kube_vip_services_enabled: false + +# Node Feature Discovery +node_feature_discovery_enabled: false +# node_feature_discovery_gc_sa_name: node-feature-discovery +# node_feature_discovery_gc_sa_create: false +# node_feature_discovery_worker_sa_name: node-feature-discovery +# node_feature_discovery_worker_sa_create: false +# node_feature_discovery_master_config: +# extraLabelNs: ["nvidia.com"] diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 1b9cd6be8..b97dc5f77 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -132,3 +132,10 @@ dependencies: - inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins + + - role: kubernetes-apps/node_feature_discovery + when: + - node_feature_discovery_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - node_feature_discovery diff --git a/roles/kubernetes-apps/node_feature_discovery/defaults/main.yml b/roles/kubernetes-apps/node_feature_discovery/defaults/main.yml new file mode 100644 index 000000000..19f09ca1c --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/defaults/main.yml @@ -0,0 +1,16 @@ +--- +node_feature_discovery_enabled: false +node_feature_discovery_namespace: node-feature-discovery +node_feature_discovery_enable_nodefeature_api: true +node_feature_discovery_gc_replicas: 1 +node_feature_discovery_gc_interval: 1h +node_feature_discovery_gc_sa_name: node-feature-discovery-gc +node_feature_discovery_gc_sa_create: true +node_feature_discovery_master_replicas: 1 +node_feature_discovery_master_crd_controller: null +node_feature_discovery_master_instance: null +node_feature_discovery_master_config: null +node_feature_discovery_worker_sa_name: node-feature-discovery-worker +node_feature_discovery_worker_sa_create: true +node_feature_discovery_worker_config: null +node_feature_discovery_worker_tolerations: null diff --git a/roles/kubernetes-apps/node_feature_discovery/tasks/main.yml b/roles/kubernetes-apps/node_feature_discovery/tasks/main.yml new file mode 100644 index 000000000..b7e930afe --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: Node Feature Discovery | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/node_feature_discovery" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Node Feature Discovery | Templates list + set_fact: + node_feature_discovery_templates: + - { name: nfd-ns, file: nfd-ns.yaml, type: ns } + - { name: nfd-api-crd, file: nfd-api-crds.yaml, type: crd } + - { name: nfd-serviceaccount, file: nfd-serviceaccount.yaml, type: sa } + - { name: nfd-role, file: nfd-role.yaml, type: role } + - { name: nfd-clusterrole, file: nfd-clusterrole.yaml, type: clusterrole } + - { name: nfd-rolebinding, file: nfd-rolebinding.yaml, type: rolebinding } + - { name: nfd-clusterrolebinding, file: nfd-clusterrolebinding.yaml, type: clusterrolebinding } + - { name: nfd-master-conf, file: nfd-master-conf.yaml, type: cm } + - { name: nfd-worker-conf, file: nfd-worker-conf.yaml, type: cm } + - { name: nfd-topologyupdater-conf, file: nfd-topologyupdater-conf.yaml, type: cm } + - { name: nfd-gc, file: nfd-gc.yaml, type: deploy } + - { name: nfd-master, file: nfd-master.yaml, type: deploy } + - { name: nfd-worker, file: nfd-worker.yaml, type: ds } + - { name: nfd-service, file: nfd-service.yaml, type: srv } + +- name: Node Feature Discovery | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.file }}" + mode: 0644 + with_items: "{{ node_feature_discovery_templates }}" + register: node_feature_discovery_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Node Feature Discovery | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.item.file }}" + state: "latest" + with_items: "{{ node_feature_discovery_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2 new file mode 100644 index 000000000..6866c7ffe --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2 @@ -0,0 +1,361 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: nodefeatures.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeature + listKind: NodeFeatureList + plural: nodefeatures + singular: nodefeature + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeature resource holds the features discovered for one node + in the cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeFeatureSpec describes a NodeFeature object. + properties: + features: + description: Features is the full "raw" features data that has been + discovered. + properties: + attributes: + additionalProperties: + description: AttributeFeatureSet is a set of features having + string value. + properties: + elements: + additionalProperties: + type: string + type: object + required: + - elements + type: object + description: Attributes contains all the attribute-type features + of the node. + type: object + flags: + additionalProperties: + description: FlagFeatureSet is a set of simple features only + containing names without values. + properties: + elements: + additionalProperties: + description: Nil is a dummy empty struct for protobuf + compatibility + type: object + type: object + required: + - elements + type: object + description: Flags contains all the flag-type features of the + node. + type: object + instances: + additionalProperties: + description: InstanceFeatureSet is a set of features each of + which is an instance having multiple attributes. + properties: + elements: + items: + description: InstanceFeature represents one instance of + a complex features, e.g. a device. + properties: + attributes: + additionalProperties: + type: string + type: object + required: + - attributes + type: object + type: array + required: + - elements + type: object + description: Instances contains all the instance-type features + of the node. + type: object + type: object + labels: + additionalProperties: + type: string + description: Labels is the set of node labels that are requested to + be created. + type: object + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: nodefeaturerules.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeatureRule + listKind: NodeFeatureRuleList + plural: nodefeaturerules + shortNames: + - nfr + singular: nodefeaturerule + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeatureRule resource specifies a configuration for feature-based + customization of node objects, such as node labeling. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeFeatureRuleSpec describes a NodeFeatureRule. + properties: + rules: + description: Rules is a list of node customization rules. + items: + description: Rule defines a rule for node customization such as + labeling. + properties: + extendedResources: + additionalProperties: + type: string + description: ExtendedResources to create if the rule matches. + type: object + labels: + additionalProperties: + type: string + description: Labels to create if the rule matches. + type: object + labelsTemplate: + description: LabelsTemplate specifies a template to expand for + dynamically generating multiple labels. Data (after template + expansion) must be keys with an optional value ([=]) + separated by newlines. + type: string + matchAny: + description: MatchAny specifies a list of matchers one of which + must match. + items: + description: MatchAnyElem specifies one sub-matcher of MatchAny. + properties: + matchFeatures: + description: MatchFeatures specifies a set of matcher + terms all of which must match. + items: + description: FeatureMatcherTerm defines requirements + against one feature set. All requirements (specified + as MatchExpressions) are evaluated against each element + in the feature set. + properties: + feature: + type: string + matchExpressions: + additionalProperties: + description: "MatchExpression specifies an expression + to evaluate against a set of input values. It + contains an operator that is applied when matching + the input and an array of values that the operator + evaluates the input against. \n NB: CreateMatchExpression + or MustCreateMatchExpression() should be used + for creating new instances. \n NB: Validate() + must be called if Op or Value fields are modified + or if a new instance is created from scratch + without using the helper functions." + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: Value is the list of values that + the operand evaluates the input against. + Value should be empty if the operator is + Exists, DoesNotExist, IsTrue or IsFalse. + Value should contain exactly one element + if the operator is Gt or Lt and exactly + two elements if the operator is GtLt. In + other cases Value should contain at least + one element. + items: + type: string + type: array + required: + - op + type: object + description: MatchExpressionSet contains a set of + MatchExpressions, each of which is evaluated against + a set of input values. + type: object + required: + - feature + - matchExpressions + type: object + type: array + required: + - matchFeatures + type: object + type: array + matchFeatures: + description: MatchFeatures specifies a set of matcher terms + all of which must match. + items: + description: FeatureMatcherTerm defines requirements against + one feature set. All requirements (specified as MatchExpressions) + are evaluated against each element in the feature set. + properties: + feature: + type: string + matchExpressions: + additionalProperties: + description: "MatchExpression specifies an expression + to evaluate against a set of input values. It contains + an operator that is applied when matching the input + and an array of values that the operator evaluates + the input against. \n NB: CreateMatchExpression or + MustCreateMatchExpression() should be used for creating + new instances. \n NB: Validate() must be called if + Op or Value fields are modified or if a new instance + is created from scratch without using the helper functions." + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: Value is the list of values that the + operand evaluates the input against. Value should + be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly + one element if the operator is Gt or Lt and exactly + two elements if the operator is GtLt. In other + cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: MatchExpressionSet contains a set of MatchExpressions, + each of which is evaluated against a set of input values. + type: object + required: + - feature + - matchExpressions + type: object + type: array + name: + description: Name of the rule. + type: string + taints: + description: Taints to create if the rule matches. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the + taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array + vars: + additionalProperties: + type: string + description: Vars is the variables to store if the rule matches. + Variables do not directly inflict any changes in the node + object. However, they can be referenced from other rules enabling + more complex rule hierarchies, without exposing intermediary + output values as labels. + type: object + varsTemplate: + description: VarsTemplate specifies a template to expand for + dynamically generating multiple variables. Data (after template + expansion) must be keys with an optional value ([=]) + separated by newlines. + type: string + required: + - name + type: object + type: array + required: + - rules + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2 new file mode 100644 index 000000000..ce880d86e --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2 @@ -0,0 +1,72 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: node-feature-discovery +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + - nodefeaturerules + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - "nfd-master.nfd.kubernetes.io" + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: node-feature-discovery-gc +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - delete + - list diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2 new file mode 100644 index 000000000..c3768c37c --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2 @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: node-feature-discovery +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-feature-discovery +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: {{ node_feature_discovery_namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: node-feature-discovery-gc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-feature-discovery-gc +subjects: +- kind: ServiceAccount + name: {{ node_feature_discovery_gc_sa_name }} + namespace: {{ node_feature_discovery_namespace }} diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2 new file mode 100644 index 000000000..78edbf08c --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2 @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node-feature-discovery-gc + namespace: {{ node_feature_discovery_namespace }} + labels: + app.kubernetes.io/name: node-feature-discovery + role: gc +spec: + replicas: {{ node_feature_discovery_gc_replicas }} + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + role: gc + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + role: gc + spec: + serviceAccountName: {{ node_feature_discovery_gc_sa_name }} + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: gc + image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} + imagePullPolicy: IfNotPresent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-gc" + args: + - "-gc-interval={{ node_feature_discovery_gc_interval }}" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2 new file mode 100644 index 000000000..92b58e479 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2 @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-feature-discovery-master-conf + namespace: {{ node_feature_discovery_namespace }} +data: +{% if node_feature_discovery_master_config %} + nfd-master.conf: |- + {{ node_feature_discovery_master_config | to_yaml(indent=2, width=1337) | indent(width=4) }} +{% else %} + nfd-master.conf: "null" +{% endif %} diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2 new file mode 100644 index 000000000..348673014 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2 @@ -0,0 +1,115 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node-feature-discovery-master + namespace: {{ node_feature_discovery_namespace }} + labels: + app.kubernetes.io/name: node-feature-discovery + role: master +spec: + replicas: {{ node_feature_discovery_master_replicas }} + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + role: master + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + role: master + spec: + serviceAccountName: node-feature-discovery + enableServiceLinks: false + containers: + - name: master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "/usr/bin/grpc_health_probe" + - "-addr=:8080" + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + exec: + command: + - "/usr/bin/grpc_health_probe" + - "-addr=:8080" + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 10 + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8081 + name: metrics + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-master" + args: + - "-port=8080" +{% if not node_feature_discovery_enable_nodefeature_api %} + - "-enable-nodefeature-api=false" +{% elif node_feature_discovery_master_replicas > 1 %} + - "-enable-leader-election" +{% endif %} +{% if node_feature_discovery_master_crd_controller != none %} + - "-crd-controller={{ node_feature_discovery_master_crd_controller }}" +{% else %} +{% if node_feature_discovery_master_instance %} + ## By default, disable crd controller for other than the default instances + - "-crd-controller=false" +{% else %} + ## By default, disable crd controller for other than the default instances + - "-crd-controller=true" +{% endif %} +{% endif %} + - "-metrics=8081" + volumeMounts: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: nfd-master-conf + configMap: + name: node-feature-discovery-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2 new file mode 100644 index 000000000..953ae590d --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ node_feature_discovery_namespace }} + labels: + name: {{ node_feature_discovery_namespace }} diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2 new file mode 100644 index 000000000..628134598 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: node-feature-discovery-worker + namespace: {{ node_feature_discovery_namespace }} +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 new file mode 100644 index 000000000..549308761 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: node-feature-discovery-worker + namespace: {{ node_feature_discovery_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: node-feature-discovery-worker +subjects: +- kind: ServiceAccount + name: {{ node_feature_discovery_worker_sa_name }} + namespace: {{ node_feature_discovery_namespace }} + diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2 new file mode 100644 index 000000000..82703b934 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2 @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: node-feature-discovery-master + namespace: {{ node_feature_discovery_namespace }} + labels: + app.kubernetes.io/name: node-feature-discovery + role: master +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: grpc + protocol: TCP + name: grpc + selector: + app.kubernetes.io/name: node-feature-discovery + role: master diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2 new file mode 100644 index 000000000..f79e1b1ae --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-feature-discovery + namespace: {{ node_feature_discovery_namespace }} +{% if node_feature_discovery_gc_sa_create %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ node_feature_discovery_gc_sa_name }} + namespace: {{ node_feature_discovery_namespace }} +{% endif %} +{% if node_feature_discovery_worker_sa_create %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ node_feature_discovery_worker_sa_name }} + namespace: {{ node_feature_discovery_namespace }} +{% endif %} diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2 new file mode 100644 index 000000000..0f8feca28 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-feature-discovery-topology-updater-conf + namespace: {{ node_feature_discovery_namespace }} +data: + nfd-topology-updater.conf: "null" diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2 new file mode 100644 index 000000000..280ae566a --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2 @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-feature-discovery-worker-conf + namespace: {{ node_feature_discovery_namespace }} +data: +{% if node_feature_discovery_worker_config %} + nfd-worker.conf: |- + {{ node_feature_discovery_worker_config | to_yaml(indent=2, width=1337) | indent(width=4) }} +{% else %} + nfd-worker.conf: "null" +{% endif %} diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2 new file mode 100644 index 000000000..fc6f94ce3 --- /dev/null +++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2 @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-feature-discovery-worker + namespace: {{ node_feature_discovery_namespace }} + labels: + app.kubernetes.io/name: node-feature-discovery + role: worker +spec: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + role: worker + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + role: worker + spec: + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ node_feature_discovery_worker_sa_name }} + containers: + - name: worker + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} + imagePullPolicy: IfNotPresent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-worker" + args: + - "-server=node-feature-discovery-master:8080" +{% if not node_feature_discovery_enable_nodefeature_api %} + - "-enable-nodefeature-api=false" +{% endif %} + - "-metrics=8081" + ports: + - name: metrics + containerPort: 8081 + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true + - name: host-lib + mountPath: "/host-lib" + readOnly: true + - name: source-d + mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" + readOnly: true + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" + - name: host-lib + hostPath: + path: "/lib" + - name: source-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/source.d/" + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: nfd-worker-conf + configMap: + name: node-feature-discovery-worker-conf + items: + - key: nfd-worker.conf + path: nfd-worker.conf +{% if node_feature_discovery_worker_tolerations %} + tolerations: + {{ node_feature_discovery_worker_tolerations | to_yaml(indent=2, width=1337) | indent(width=8) }} +{% endif %} diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 300d297fc..09b655655 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -398,6 +398,10 @@ metallb_speaker_image_repo: "{{ quay_image_repo }}/metallb/speaker" metallb_controller_image_repo: "{{ quay_image_repo }}/metallb/controller" metallb_version: v0.13.9 +node_feature_discovery_version: v0.14.2 +node_feature_discovery_image_repo: "{{ kube_image_repo }}/nfd/node-feature-discovery" +node_feature_discovery_image_tag: "{{ node_feature_discovery_version }}" + downloads: netcheck_server: enabled: "{{ deploy_netchecker }}" diff --git a/tests/files/packet_rockylinux9-cilium.yml b/tests/files/packet_rockylinux9-cilium.yml index 038e600c1..033fde1b8 100644 --- a/tests/files/packet_rockylinux9-cilium.yml +++ b/tests/files/packet_rockylinux9-cilium.yml @@ -8,3 +8,6 @@ vm_memory: 3072Mi kube_network_plugin: cilium cilium_kube_proxy_replacement: strict + +# Node Feature Discovery +node_feature_discovery_enabled: true diff --git a/tests/scripts/check_readme_versions.sh b/tests/scripts/check_readme_versions.sh index 0f1836f9d..594391e89 100755 --- a/tests/scripts/check_readme_versions.sh +++ b/tests/scripts/check_readme_versions.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -TARGET_COMPONENTS="containerd calico cilium flannel kube-ovn kube-router weave cert-manager krew helm metallb registry cephfs-provisioner rbd-provisioner aws-ebs-csi-plugin azure-csi-plugin cinder-csi-plugin gcp-pd-csi-plugin local-path-provisioner local-volume-provisioner kube-vip ingress-nginx" +TARGET_COMPONENTS="containerd calico cilium flannel kube-ovn kube-router weave cert-manager krew helm metallb registry cephfs-provisioner rbd-provisioner aws-ebs-csi-plugin azure-csi-plugin cinder-csi-plugin gcp-pd-csi-plugin local-path-provisioner local-volume-provisioner kube-vip ingress-nginx node-feature-discovery" # cd to the root directory of kubespray cd $(dirname $0)/../../