From f26f544ff6e0203cf1abbcd242ea2ffa9530f64b Mon Sep 17 00:00:00 2001 From: Samuel Liu Date: Thu, 12 May 2022 12:35:15 +0800 Subject: [PATCH] [kube-ovn]: update kube-ovn version and sync some feature (#8790) * [kube-ovn]: some feature kube-ovn vlan mode ipv6/ipv4 dual stack ... * remove unused env * fix readinessprobe --- .../group_vars/k8s_cluster/k8s-cluster.yml | 2 +- .../k8s_cluster/k8s-net-kube-ovn.yml | 57 ++++++++ roles/download/defaults/main.yml | 9 +- roles/kubespray-defaults/defaults/main.yaml | 2 +- .../network_plugin/kube-ovn/defaults/main.yml | 65 ++++++++- .../templates/cni-kube-ovn-crd.yml.j2 | 117 +++++++++++++++ .../kube-ovn/templates/cni-kube-ovn.yml.j2 | 109 ++++++++++---- .../kube-ovn/templates/cni-ovn.yml.j2 | 134 +++++++++++------- 8 files changed, 407 insertions(+), 88 deletions(-) create mode 100644 inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index ba324967e..819843336 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -62,7 +62,7 @@ credentials_dir: "{{ inventory_dir }}/credentials" # kube_webhook_authorization_url: https://... # kube_webhook_authorization_url_skip_tls_verify: false -# Choose network plugin (cilium, calico, weave or flannel. Use cni for generic cni plugin) +# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml new file mode 100644 index 000000000..d580e15fc --- /dev/null +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml @@ -0,0 +1,57 @@ +--- + +# geneve or vlan +kube_ovn_network_type: geneve + +# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module +kube_ovn_tunnel_type: geneve + +## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. +# kube_ovn_iface: eth1 +## The MTU used by pod iface in overlay networks (default iface MTU - 100) +# kube_ovn_mtu: 1333 + +## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. +kube_ovn_hw_offload: false +# traffic mirror +kube_ovn_traffic_mirror: false + +# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 +# kube_ovn_default_interface_name: eth0 + +kube_ovn_external_address: 8.8.8.8 +kube_ovn_external_address_ipv6: 2400:3200::1 +kube_ovn_external_dns: alauda.cn + +# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 +kube_ovn_default_gateway_check: true +kube_ovn_default_logical_gateway: false +# kube_ovn_default_exclude_ips: 10.16.0.1 +kube_ovn_node_switch_cidr: 100.64.0.0/16 +kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 + +## vlan config, set default interface name and vlan id +# kube_ovn_default_interface_name: eth0 +kube_ovn_default_vlan_id: 100 +kube_ovn_vlan_name: product + +## pod nic type, support: veth-pair or internal-port +kube_ovn_pod_nic_type: veth_pair + +## Enable load balancer +kube_ovn_enable_lb: true + +## Enable network policy support +kube_ovn_enable_np: true + +## Enable external vpc support +kube_ovn_enable_external_vpc: true + +## Enable checksum +kube_ovn_encap_checksum: true + +## enable ssl +kube_ovn_enable_ssl: false + +## dpdk +kube_ovn_dpdk_enabled: false diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 6ba48bb3c..8ce00081d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -111,7 +111,8 @@ cni_version: "v1.0.1" weave_version: 2.8.1 pod_infra_version: "3.3" cilium_version: "v1.11.3" -kube_ovn_version: "v1.8.1" +kube_ovn_version: "v1.9.2" +kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v1.4.0" multus_version: "v3.8" helm_version: "v3.8.2" @@ -918,8 +919,10 @@ cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-bac cilium_hubble_ui_backend_image_tag: "v0.7.3" cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy" cilium_hubble_envoy_image_tag: "v1.14.5" -kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" -kube_ovn_container_image_tag: "{{ kube_ovn_version }}" +kube_ovn_dpdk_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" +kube_ovn_dpdk_container_image_tag: "{{ kube_ovn_version }}" +kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-dpdk" +kube_ovn_container_image_tag: "{{ kube_ovn_dpdk_version }}" kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" kube_router_image_tag: "{{ kube_router_version }}" multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni" diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index f0ce2598d..a3d51bed1 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -166,7 +166,7 @@ kube_external_ca_mode: false # Cluster Loglevel configuration kube_log_level: 2 -# Choose network plugin (cilium, calico, weave or flannel) +# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico kube_network_plugin_multus: false diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml index 831c26bd4..78862b71d 100644 --- a/roles/network_plugin/kube-ovn/defaults/main.yml +++ b/roles/network_plugin/kube-ovn/defaults/main.yml @@ -23,7 +23,66 @@ kube_ovn_monitor_memory_request: 200Mi kube_ovn_monitor_cpu_request: 200m kube_ovn_monitor_memory_limit: 200Mi kube_ovn_monitor_cpu_limit: 200m +kube_ovn_dpdk_node_cpu_request: 1000m +kube_ovn_dpdk_node_memory_request: 2Gi +kube_ovn_dpdk_node_cpu_limit: 1000m +kube_ovn_dpdk_node_memory_limit: 2Gi -traffic_mirror: true -encap_checksum: false -enable_ssl: false +kube_ovn_central_replics: 1 +kube_ovn_controller_replics: 1 + +# geneve or vlan +kube_ovn_network_type: geneve + +# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module +kube_ovn_tunnel_type: geneve + +## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. +# kube_ovn_iface: eth1 +## The MTU used by pod iface in overlay networks (default iface MTU - 100) +# kube_ovn_mtu: 1333 + +## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. +kube_ovn_hw_offload: false +# traffic mirror +kube_ovn_traffic_mirror: false + +# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 +# kube_ovn_default_interface_name: eth0 + +kube_ovn_external_address: 8.8.8.8 +kube_ovn_external_address_ipv6: 2400:3200::1 +kube_ovn_external_dns: alauda.cn + +# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 +kube_ovn_default_gateway_check: true +kube_ovn_default_logical_gateway: false +# kube_ovn_default_exclude_ips: 10.16.0.1 +kube_ovn_node_switch_cidr: 100.64.0.0/16 +kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 + +## vlan config, set default interface name and vlan id +# kube_ovn_default_interface_name: eth0 +kube_ovn_default_vlan_id: 100 +kube_ovn_vlan_name: product + +## pod nic type, support: veth-pair or internal-port +kube_ovn_pod_nic_type: veth_pair + +## Enable load balancer +kube_ovn_enable_lb: true + +## Enable network policy support +kube_ovn_enable_np: true + +## Enable external vpc support +kube_ovn_enable_external_vpc: true + +## Enable checksum +kube_ovn_encap_checksum: true + +## enable ssl +kube_ovn_enable_ssl: false + +## dpdk +kube_ovn_dpdk_enabled: false diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 9234006e2..5aa61f4a5 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -61,6 +61,8 @@ spec: type: string containerID: type: string + podType: + type: string scope: Cluster names: plural: ips @@ -138,6 +140,10 @@ spec: type: number activateGateway: type: string + dhcpV4OptionsUUID: + type: string + dhcpV6OptionsUUID: + type: string conditions: type: array items: @@ -164,6 +170,10 @@ spec: type: boolean protocol: type: string + enum: + - IPv4 + - IPv6 + - Dual cidrBlock: type: string namespaces: @@ -178,6 +188,10 @@ spec: type: array items: type: string + vips: + type: array + items: + type: string gatewayType: type: string allowSubnets: @@ -208,10 +222,48 @@ spec: type: boolean vlan: type: string + logicalGateway: + type: boolean disableGatewayCheck: type: boolean disableInterConnection: type: boolean + enableDHCP: + type: boolean + dhcpV4Options: + type: string + dhcpV6Options: + type: string + enableIPv6RA: + type: boolean + ipv6RAConfigs: + type: string + htbqos: + type: string + acls: + type: array + items: + type: object + properties: + direction: + type: string + enum: + - from-lport + - to-lport + priority: + type: integer + minimum: 0 + maximum: 32767 + match: + type: string + action: + type: string + enum: + - allow-related + - allow-stateless + - allow + - drop + - reject scope: Cluster names: plural: subnets @@ -230,6 +282,8 @@ spec: - name: v1 served: true storage: true + subresources: + status: {} schema: openAPIV3Schema: type: object @@ -283,6 +337,8 @@ spec: - name: v1 served: true storage: true + subresources: + status: {} schema: openAPIV3Schema: type: object @@ -407,6 +463,28 @@ spec: type: string type: object type: array + policyRoutes: + items: + properties: + priority: + type: integer + action: + type: string + match: + type: string + nextHopIP: + type: string + type: object + type: array + vpcPeerings: + items: + properties: + remoteVpc: + type: string + localConnectIP: + type: string + type: object + type: array type: object status: properties: @@ -439,6 +517,10 @@ spec: items: type: string type: array + vpcPeerings: + items: + type: string + type: array tcpLoadBalancer: type: string tcpSessionLoadBalancer: @@ -545,6 +627,10 @@ spec: type: string vpc: type: string + selector: + type: array + items: + type: string subresources: status: {} conversion: @@ -642,3 +728,34 @@ spec: status: {} conversion: strategy: None +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: htbqoses.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: PRIORITY + type: string + jsonPath: .spec.priority + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + priority: + type: string # Value in range 0 to 4,294,967,295. + scope: Cluster + names: + plural: htbqoses + singular: htbqos + kind: HtbQos + shortNames: + - htbqos diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index e9cadc2b9..36e322623 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -8,7 +8,7 @@ metadata: kubernetes.io/description: | kube-ovn controller spec: - replicas: 1 + replicas: {{ kube_ovn_controller_replics }} selector: matchLabels: app: kube-ovn-controller @@ -43,14 +43,26 @@ spec: command: - /kube-ovn/start-controller.sh args: - - --default-cidr={{ kube_pods_subnet }} - - --pod-nic-type=veth-pair - - --enable-lb=true - - --enable-np=true - - --enable-external-vpc=true + - --default-cidr={{ kube_pods_subnet }}{% if enable_dual_stack_networks %},{{ kube_ovn_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}{% endif %}{{''}} + - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{''}} + - --default-gateway-check={{ kube_ovn_default_gateway_check|string }} + - --default-logical-gateway={{ kube_ovn_default_logical_gateway|string }} + - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{''}} + - --node-switch-cidr={{ kube_ovn_node_switch_cidr }}{% if enable_dual_stack_networks %},{{ kube_ovn_node_switch_cidr_ipv6 }}{% endif %}{{''}} + - --service-cluster-ip-range={{ kube_service_addresses }}{% if enable_dual_stack_networks %},{{ kube_service_addresses_ipv6 }}{% endif %}{{''}} + - --network-type={{ kube_ovn_network_type }} + - --default-interface-name={{ kube_ovn_default_interface_name|default('') }} + - --default-vlan-id={{ kube_ovn_default_vlan_id }} + - --pod-nic-type={{ kube_ovn_pod_nic_type }} + - --enable-lb={{ kube_ovn_enable_lb|string }} + - --enable-np={{ kube_ovn_enable_np|string }} + - --enable-external-vpc={{ kube_ovn_enable_external_vpc|string }} + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-controller.log env: - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_NAME valueFrom: fieldRef: @@ -66,20 +78,20 @@ spec: volumeMounts: - mountPath: /etc/localtime name: localtime + - mountPath: /var/log/kube-ovn + name: kube-ovn-log - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - - bash - - /kube-ovn/kube-ovn-controller-healthcheck.sh + - /kube-ovn/kube-ovn-controller-healthcheck periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - - bash - - /kube-ovn/kube-ovn-controller-healthcheck.sh + - /kube-ovn/kube-ovn-controller-healthcheck initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 @@ -97,10 +109,14 @@ spec: - name: localtime hostPath: path: /etc/localtime + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls + --- kind: DaemonSet apiVersion: apps/v1 @@ -146,15 +162,24 @@ spec: - bash - /kube-ovn/start-cniserver.sh args: - - --enable-mirror={{ traffic_mirror | lower }} - - --encap-checksum={{ encap_checksum | lower }} - - --service-cluster-ip-range={{ kube_service_addresses }} + - --enable-mirror={{ kube_ovn_traffic_mirror | lower }} + - --encap-checksum={{ kube_ovn_encap_checksum | lower }} + - --service-cluster-ip-range={{ kube_service_addresses }}{% if enable_dual_stack_networks %},{{ kube_service_addresses_ipv6 }}{% endif %}{{''}} + - --iface={{ kube_ovn_iface|default('') }} + - --network-type={{ kube_ovn_network_type }} + - --default-interface-name={{ kube_ovn_default_interface_name|default('') }} +{% if kube_ovn_mtu is defined %} + - --mtu={{ kube_ovn_mtu }} +{% endif %} + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-cni.log securityContext: runAsUser: 0 privileged: true env: - - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + - name: kube_ovn_enable_ssl + value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: @@ -175,6 +200,8 @@ spec: - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer + - mountPath: /var/log/kube-ovn + name: kube-ovn-log - mountPath: /etc/localtime name: localtime readinessProbe: @@ -186,6 +213,7 @@ spec: - 127.0.0.1 - "10665" periodSeconds: 3 + timeoutSeconds: 5 livenessProbe: exec: command: @@ -197,6 +225,7 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 failureThreshold: 5 + timeoutSeconds: 5 resources: requests: cpu: {{ kube_ovn_cni_server_cpu_request }} @@ -225,6 +254,9 @@ spec: - name: host-ns hostPath: path: /var/run/netns + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn - name: localtime hostPath: path: /etc/localtime @@ -251,21 +283,26 @@ spec: component: network type: infra spec: - tolerations: - - operator: Exists serviceAccountName: ovn hostPID: true containers: - name: pinger image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114"] + command: + - /kube-ovn/kube-ovn-pinger + args: + - --external-address={{ kube_ovn_external_address }}{% if enable_dual_stack_networks %},{{ kube_ovn_external_address_ipv6 }}{% endif %}{{''}} + - --external-dns={{ kube_ovn_external_dns }} + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log securityContext: runAsUser: 0 privileged: false env: - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: @@ -301,6 +338,8 @@ spec: name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn + - mountPath: /var/log/kube-ovn + name: kube-ovn-log - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls @@ -333,6 +372,9 @@ spec: - name: host-log-ovs hostPath: path: /var/log/openvswitch + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn - name: host-log-ovn hostPath: path: /var/log/ovn @@ -356,7 +398,7 @@ spec: replicas: 1 strategy: rollingUpdate: - maxSurge: 0 + maxSurge: 1 maxUnavailable: 1 type: RollingUpdate selector: @@ -380,6 +422,7 @@ spec: topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn + hostNetwork: true containers: - name: kube-ovn-monitor image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} @@ -390,7 +433,7 @@ spec: privileged: false env: - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + value: "{{ kube_ovn_enable_ssl | lower }}" - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -407,9 +450,6 @@ spec: name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn @@ -427,13 +467,13 @@ spec: command: - cat - /var/run/ovn/ovnnb_db.pid - periodSeconds: 3 + periodSeconds: 10 timeoutSeconds: 45 livenessProbe: exec: command: - cat - - /var/run/ovn/ovn-nbctl.pid + - /var/run/ovn/ovnnb_db.pid initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 5 @@ -448,9 +488,6 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-sys - hostPath: - path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch @@ -483,6 +520,9 @@ spec: - name: metrics port: 10661 type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: kube-ovn-monitor sessionAffinity: None @@ -495,6 +535,9 @@ metadata: labels: app: kube-ovn-pinger spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: kube-ovn-pinger ports: @@ -509,6 +552,9 @@ metadata: labels: app: kube-ovn-controller spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: kube-ovn-controller ports: @@ -523,6 +569,9 @@ metadata: labels: app: kube-ovn-cni spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: kube-ovn-cni ports: diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index a8505df26..d2dc9f4e5 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -1,40 +1,10 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: kube-ovn - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - ---- - apiVersion: v1 kind: ConfigMap metadata: name: ovn-config namespace: kube-system data: - defaultNetworkType: geneve + defaultNetworkType: '{{ kube_ovn_network_type }}' --- apiVersion: v1 kind: ServiceAccount @@ -49,29 +19,27 @@ metadata: rbac.authorization.k8s.io/system-only: "true" name: system:ovn rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] resourceNames: - kube-ovn - apiGroups: - "kubeovn.io" resources: - - subnets - - subnets/status - vpcs - vpcs/status - vpc-nat-gateways + - subnets + - subnets/status - ips - vlans + - vlans/status - provider-networks - provider-networks/status - - networks - security-groups - security-groups/status + - htbqoses verbs: - "*" - apiGroups: @@ -111,6 +79,7 @@ rules: - statefulsets - daemonsets - deployments + - deployments/scale verbs: - create - delete @@ -127,6 +96,24 @@ rules: - create - patch - update + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - update + - apiGroups: + - "kubevirt.io" + resources: + - virtualmachines + - virtualmachineinstances + verbs: + - get + - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -153,6 +140,9 @@ spec: port: 6641 targetPort: 6641 type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: ovn-central ovn-nb-leader: "true" @@ -170,6 +160,9 @@ spec: port: 6642 targetPort: 6642 type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: ovn-central ovn-sb-leader: "true" @@ -187,6 +180,9 @@ spec: port: 6643 targetPort: 6643 type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} selector: app: ovn-central ovn-northd-leader: "true" @@ -201,7 +197,7 @@ metadata: kubernetes.io/description: | OVN components: northd, nb and sb. spec: - replicas: 1 + replicas: {{ kube_ovn_central_replics }} strategy: rollingUpdate: maxSurge: 0 @@ -218,7 +214,7 @@ spec: type: infra spec: tolerations: - - operator: Exists + - operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -239,7 +235,7 @@ spec: add: ["SYS_NICE"] env: - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: @@ -284,7 +280,7 @@ spec: command: - bash - /kube-ovn/ovn-is-leader.sh - periodSeconds: 3 + periodSeconds: 15 timeoutSeconds: 45 livenessProbe: exec: @@ -292,7 +288,7 @@ spec: - bash - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 - periodSeconds: 7 + periodSeconds: 15 failureThreshold: 5 timeoutSeconds: 45 nodeSelector: @@ -350,28 +346,33 @@ spec: type: infra spec: tolerations: - - operator: Exists + - operator: Exists priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true containers: - name: openvswitch - image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + image: {% if kube_ovn_dpdk_enabled %}{{ kube_ovn_dpdk_container_image_repo }}:{{ kube_ovn_dpdk_container_image_tag }}{% else %}{{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}{% endif %} + imagePullPolicy: {{ k8s_image_pull_policy }} - command: ["/kube-ovn/start-ovs.sh"] + command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}] securityContext: runAsUser: 0 privileged: true env: - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" + value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP +{% if not kube_ovn_dpdk_enabled %} - name: HW_OFFLOAD - value: "false" + value: "{{ kube_ovn_hw_offload }}" + - name: TUNNEL_TYPE + value: "{{ kube_ovn_tunnel_type }}" +{% endif %} - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -397,6 +398,12 @@ spec: name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn +{% if kube_ovn_dpdk_enabled %} + - mountPath: /opt/ovs-config + name: host-config-ovs + - mountPath: /dev/hugepages + name: hugepage +{% endif %} - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls @@ -405,25 +412,43 @@ spec: exec: command: - bash +{% if kube_ovn_dpdk_enabled %} + - /kube-ovn/ovs-dpdk-healthcheck.sh +{% else %} - /kube-ovn/ovs-healthcheck.sh +{% endif %} periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - bash +{% if kube_ovn_dpdk_enabled %} + - /kube-ovn/ovs-dpdk-healthcheck.sh +{% else %} - /kube-ovn/ovs-healthcheck.sh +{% endif %} initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 5 timeoutSeconds: 45 resources: +{% if kube_ovn_dpdk_enabled %} + requests: + cpu: {{ kube_ovn_dpdk_node_cpu_request }} + memory: {{ kube_ovn_dpdk_node_memory_request }} + limits: + cpu: {{ kube_ovn_dpdk_node_cpu_limit }} + memory: {{ kube_ovn_dpdk_node_memory_limit }} + hugepages-1Gi: 1Gi +{% else %} requests: cpu: {{ kube_ovn_node_cpu_request }} memory: {{ kube_ovn_node_memory_request }} limits: cpu: {{ kube_ovn_node_cpu_limit }} memory: {{ kube_ovn_node_memory_limit }} +{% endif %} nodeSelector: kubernetes.io/os: "linux" volumes: @@ -454,6 +479,15 @@ spec: - name: host-log-ovn hostPath: path: /var/log/ovn +{% if kube_ovn_dpdk_enabled %} + - name: host-config-ovs + hostPath: + path: /opt/ovs-config + type: DirectoryOrCreate + - name: hugepage + emptyDir: + medium: HugePages +{% endif %} - name: localtime hostPath: path: /etc/localtime