Upgrade kube-ovn to v1.1.0 and move test from centos7 to centos8 (#5852)

.gitlab-ci/packet.yml

@@ -108,7 +108,7 @@ packet_centos7-calico-ha-once-localhost:
   services:
     - docker:18.09.9-dind
 
-packet_centos7-kube-ovn:
+packet_centos8-kube-ovn:
   stage: deploy-part2
   extends: .packet
   when: on_success

docs/kube-ovn.md

@@ -4,6 +4,12 @@ Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It off
 
 For more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn)
 
+**Warning:** Kernel version (`cat /proc/version`) needs to be different than `3.10.0-862` or kube-ovn won't start and will print this message:
+
+```bash
+kernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898
+```
+
 ## How to use it
 
 Enable kube-ovn in `group_vars/k8s-cluster/k8s-cluster.yml`

roles/download/defaults/main.yml

@@ -63,6 +63,9 @@ docker_image_repo: "docker.io"
 # quay image repo define
 quay_image_repo: "quay.io"
 
+# alauda.cn image repo (for kube-ovn...)
+alauda_image_repo: "index.alauda.cn"
+
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
 calico_version: "v3.13.2"
@@ -81,7 +84,7 @@ weave_version: 2.6.2
 pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.7.2"
-kube_ovn_version: "v0.6.0"
+kube_ovn_version: "v1.1.0"
 kube_router_version: "v0.4.0"
 multus_version: "v3.4.1"
 
@@ -472,14 +475,8 @@ cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
 cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
-kube_ovn_db_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-db"
-kube_ovn_node_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-node"
-kube_ovn_cni_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-cni"
-kube_ovn_controller_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-controller"
-kube_ovn_db_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_node_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_controller_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_cni_image_tag: "{{ kube_ovn_version }}"
+kube_ovn_container_image_repo: "{{ alauda_image_repo }}/alaudak8s/kube-ovn"
+kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
 kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router"
 kube_router_image_tag: "{{ kube_router_version }}"
 multus_image_repo: "{{ docker_image_repo }}/nfvpe/multus"
@@ -836,38 +833,11 @@ downloads:
     groups:
       - k8s-cluster
 
-  kube_ovn_db:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_db_image_repo }}"
-    tag: "{{ kube_ovn_db_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_node:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_node_image_repo }}"
-    tag: "{{ kube_ovn_node_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_controller:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_controller_image_repo }}"
-    tag: "{{ kube_ovn_controller_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_cni:
+  kube_ovn:
     enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
     container: true
-    repo: "{{ kube_ovn_cni_image_repo }}"
-    tag: "{{ kube_ovn_cni_image_tag }}"
+    repo: "{{ kube_ovn_container_image_repo }}"
+    tag: "{{ kube_ovn_container_image_tag }}"
     sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster

roles/network_plugin/kube-ovn/defaults/main.yml

@@ -7,5 +7,10 @@ kube_ovn_node_cpu_request: 100m
 kube_ovn_node_memory_request: 300Mi
 kube_ovn_node_cpu_limit: 200m
 kube_ovn_node_memory_limit: 500Mi
+kube_ovn_pinger_cpu_request: 100m
+kube_ovn_pinger_memory_request: 300Mi
+kube_ovn_pinger_cpu_limit: 200m
+kube_ovn_pinger_memory_limit: 400Mi
 
-traffic_mirror: true
+traffic_mirror: true
+encap_checksum: true

roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2

@@ -40,7 +40,12 @@ spec:
     kind: Subnet
     shortNames:
       - subnet
+  subresources:
+    status: {}
   additionalPrinterColumns:
+    - name: Provider
+      type: string
+      JSONPath: .spec.provider
     - name: Protocol
       type: string
       JSONPath: .spec.protocol
@@ -53,11 +58,23 @@ spec:
     - name: NAT
       type: boolean
       JSONPath: .spec.natOutgoing
+    - name: Default
+      type: boolean
+      JSONPath: .spec.default
+    - name: GatewayType
+      type: string
+      JSONPath: .spec.gatewayType
+    - name: Used
+      type: integer
+      JSONPath: .status.usingIPs
+    - name: Available
+      type: integer
+      JSONPath: .status.availableIPs
   validation:
     openAPIV3Schema:
       properties:
         spec:
-          required: ["cidrBlock","gateway"]
+          required: ["cidrBlock"]
           properties:
             cidrBlock:
               type: "string"

roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2

@@ -34,11 +34,12 @@ spec:
                 matchLabels:
                   app: kube-ovn-controller
               topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
       containers:
         - name: kube-ovn-controller
-          image: {{ kube_ovn_controller_image_repo }}:{{ kube_ovn_controller_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
           command:
           - /kube-ovn/start-controller.sh
@@ -60,25 +61,19 @@ spec:
           readinessProbe:
             exec:
               command:
-                - nc
-                - -z
-                - -w3
-                - 127.0.0.1
-                - "10660"
+                - sh
+                - /kube-ovn/kube-ovn-controller-healthcheck.sh
             periodSeconds: 3
           livenessProbe:
             exec:
               command:
-                - nc
-                - -z
-                - -w3
-                - 127.0.0.1
-                - "10660"
+                - sh
+                - /kube-ovn/kube-ovn-controller-healthcheck.sh
             initialDelaySeconds: 30
             periodSeconds: 7
             failureThreshold: 5
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
 
 ---
 kind: DaemonSet
@@ -94,7 +89,7 @@ spec:
     matchLabels:
       app: kube-ovn-cni
   updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
   template:
     metadata:
       labels:
@@ -105,14 +100,18 @@ spec:
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
       hostPID: true
       initContainers:
       - name: install-cni
-        image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
         imagePullPolicy: {{ k8s_image_pull_policy }}
         command: ["/kube-ovn/install-cni.sh"]
+        securityContext:
+          runAsUser: 0
+          privileged: true
         volumeMounts:
           - mountPath: /etc/cni/net.d
             name: cni-conf
@@ -120,16 +119,18 @@ spec:
             name: cni-bin
       containers:
       - name: cni-server
-        image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
         imagePullPolicy: {{ k8s_image_pull_policy }}
         command:
           - sh
           - /kube-ovn/start-cniserver.sh
         args:
           - --enable-mirror={{ traffic_mirror }}
+          - --encap-checksum={{ encap_checksum }}
+          - --service-cluster-ip-range={{ kube_service_addresses }}
         securityContext:
-          runAsUser: 0
-          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"]
         env:
           - name: POD_IP
             valueFrom:
@@ -142,6 +143,11 @@ spec:
         volumeMounts:
           - mountPath: /run/openvswitch
             name: host-run-ovs
+          - mountPath: /run/ovn
+            name: host-run-ovn
+          - mountPath: /var/run/netns
+            name: host-ns
+            mountPropagation: HostToContainer
         readinessProbe:
           exec:
             command:
@@ -163,14 +169,165 @@ spec:
           periodSeconds: 7
           failureThreshold: 5
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
       volumes:
         - name: host-run-ovs
           hostPath:
             path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
         - name: cni-conf
           hostPath:
             path: /etc/cni/net.d
         - name: cni-bin
           hostPath:
-            path: /opt/cni/bin
+            path: /opt/cni/bin
+        - name: host-ns
+          hostPath:
+            path: /var/run/netns
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the openvswitch daemon.
+spec:
+  selector:
+    matchLabels:
+      app: kube-ovn-pinger
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-pinger
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: ovn
+      hostPID: true
+      containers:
+        - name: pinger
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114"]
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            runAsUser: 0
+            privileged: false
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: host-modules
+              readOnly: true
+            - mountPath: /run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /sys
+              name: host-sys
+              readOnly: true
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /var/log/openvswitch
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+          resources:
+            requests:
+              cpu: {{ kube_ovn_pinger_cpu_request }}
+              memory: {{ kube_ovn_pinger_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_pinger_cpu_limit }}
+              memory: {{ kube_ovn_pinger_memory_limit }}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: host-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-pinger
+spec:
+  selector:
+    app: kube-ovn-pinger
+  ports:
+    - port: 8080
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-controller
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-controller
+spec:
+  selector:
+    app: kube-ovn-controller
+  ports:
+    - port: 10660
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-cni
+spec:
+  selector:
+    app: kube-ovn-cni
+  ports:
+    - port: 10665
+      name: metrics

roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2

@@ -29,6 +29,7 @@ rules:
       - "kubeovn.io"
     resources:
       - subnets
+      - subnets/status
       - ips
     verbs:
       - "*"
@@ -55,6 +56,7 @@ rules:
       - services
       - endpoints
       - statefulsets
+      - daemonsets
     verbs:
       - get
       - list
@@ -97,6 +99,7 @@ spec:
   type: ClusterIP
   selector:
     app: ovn-central
+    ovn-nb-leader: "true"
   sessionAffinity: None
 
 ---
@@ -114,6 +117,7 @@ spec:
   type: ClusterIP
   selector:
     app: ovn-central
+    ovn-sb-leader: "true"
   sessionAffinity: None
 
 ---
@@ -152,17 +156,30 @@ spec:
                 matchLabels:
                   app: ovn-central
               topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
       containers:
         - name: ovn-central
-          image: {{ kube_ovn_db_image_repo }}:{{ kube_ovn_db_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-db.sh"]
+          securityContext:
+            capabilities:
+              add: ["SYS_NICE"]
           env:
             - name: POD_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           resources:
             requests:
               cpu: {{ kube_ovn_db_cpu_request }}
@@ -171,47 +188,55 @@ spec:
               cpu: {{ kube_ovn_db_cpu_limit }}
               memory: {{ kube_ovn_db_memory_limit }}
           volumeMounts:
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
             - mountPath: /var/run/openvswitch
               name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
             - mountPath: /sys
               name: host-sys
               readOnly: true
             - mountPath: /etc/openvswitch
               name: host-config-openvswitch
             - mountPath: /var/log/openvswitch
-              name: host-log
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
           readinessProbe:
             exec:
               command:
                 - sh
-                - /root/ovn-is-leader.sh
+                - /kube-ovn/ovn-is-leader.sh
             periodSeconds: 3
           livenessProbe:
             exec:
               command:
               - sh
-              - /root/ovn-healthcheck.sh
+              - /kube-ovn/ovn-healthcheck.sh
             initialDelaySeconds: 30
             periodSeconds: 7
             failureThreshold: 5
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
         kube-ovn/role: "master"
       volumes:
         - name: host-run-ovs
           hostPath:
             path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
         - name: host-sys
           hostPath:
             path: /sys
         - name: host-config-openvswitch
           hostPath:
             path: /etc/origin/openvswitch
-        - name: host-log
+        - name: host-log-ovs
           hostPath:
             path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
 
 ---
 kind: DaemonSet
@@ -227,7 +252,7 @@ spec:
     matchLabels:
       app: ovs
   updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
   template:
     metadata:
       labels:
@@ -238,13 +263,15 @@ spec:
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
       hostPID: true
       containers:
         - name: openvswitch
-          image: {{ kube_ovn_node_image_repo }}:{{ kube_ovn_node_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-ovs.sh"]
           securityContext:
             runAsUser: 0
             privileged: true
@@ -257,28 +284,30 @@ spec:
             - mountPath: /lib/modules
               name: host-modules
               readOnly: true
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
             - mountPath: /var/run/openvswitch
               name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
             - mountPath: /sys
               name: host-sys
               readOnly: true
             - mountPath: /etc/openvswitch
               name: host-config-openvswitch
             - mountPath: /var/log/openvswitch
-              name: host-log
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
           readinessProbe:
             exec:
               command:
               - sh
-              - /root/ovs-healthcheck.sh
+              - /kube-ovn/ovs-healthcheck.sh
             periodSeconds: 5
           livenessProbe:
             exec:
               command:
               - sh
-              - /root/ovs-healthcheck.sh
+              - /kube-ovn/ovs-healthcheck.sh
             initialDelaySeconds: 10
             periodSeconds: 5
             failureThreshold: 5
@@ -290,7 +319,7 @@ spec:
               cpu: {{ kube_ovn_node_cpu_limit }}
               memory: {{ kube_ovn_node_memory_limit }}
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
       volumes:
         - name: host-modules
           hostPath:
@@ -298,12 +327,18 @@ spec:
         - name: host-run-ovs
           hostPath:
             path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
         - name: host-sys
           hostPath:
             path: /sys
         - name: host-config-openvswitch
           hostPath:
             path: /etc/origin/openvswitch
-        - name: host-log
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
           hostPath:
-            path: /var/log/openvswitch
+            path: /var/log/ovn

tests/files/packet_centos7-kube-ovn.yml → tests/files/packet_centos8-kube-ovn.yml

@@ -1,12 +1,9 @@
 ---
 # Instance settings
-cloud_image: centos-7
+cloud_image: centos-8
 mode: default
 
 # Kubespray settings
 kube_network_plugin: kube-ovn
 deploy_netchecker: true
-dns_min_replicas: 1
-
-# Temp set k8s ver to 1.16.8
-kube_version: v1.16.7
+dns_min_replicas: 1