From b36bb9115aeb5a39219f5c4a26db4723de697339 Mon Sep 17 00:00:00 2001 From: Samuel Liu Date: Thu, 18 Aug 2022 15:52:37 +0800 Subject: [PATCH] [calico] calico rr supports multiple groups (#9134) * update calico rr * fix bgppeer conf * fix yamllint * fix ansible lint * fix calico deploy * fix yamllint * fix some typo --- docs/calico.md | 2 + .../calico/rr/tasks/update-node.yml | 9 ++ roles/network_plugin/calico/tasks/install.yml | 140 +----------------- .../calico/tasks/peer_with_calico_rr.yml | 86 +++++++++++ .../calico/tasks/peer_with_router.yml | 77 ++++++++++ 5 files changed, 182 insertions(+), 132 deletions(-) create mode 100644 roles/network_plugin/calico/tasks/peer_with_calico_rr.yml create mode 100644 roles/network_plugin/calico/tasks/peer_with_router.yml diff --git a/docs/calico.md b/docs/calico.md index c733c3c0c..9c371fd1a 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -172,6 +172,8 @@ node5 [rack0:vars] cluster_id="1.0.0.1" +calcio_rr_id=rr1 +calico_group_id=rr1 ``` The inventory above will deploy the following topology assuming that calico's diff --git a/roles/network_plugin/calico/rr/tasks/update-node.yml b/roles/network_plugin/calico/rr/tasks/update-node.yml index 21c96a596..970cad83a 100644 --- a/roles/network_plugin/calico/rr/tasks/update-node.yml +++ b/roles/network_plugin/calico/rr/tasks/update-node.yml @@ -6,6 +6,15 @@ set_fact: retry_count: "{{ 0 if retry_count is undefined else retry_count|int + 1 }}" + - name: Calico | Set label for route reflector # noqa 301 305 + shell: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-rr-id={{ calcio_rr_id }} --overwrite" + changed_when: false + register: calico_rr_id_label + until: calico_rr_id_label is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 + when: calcio_rr_id is defined + - name: Calico-rr | Fetch current node object command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson" changed_when: false diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index f543036e5..54c3deba4 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -346,83 +346,6 @@ when: - inventory_hostname == groups['kube_control_plane'][0] -- name: Calico | Configure peering with router(s) at global scope - command: - cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" - stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" - vars: - stdin: > - {"apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "global-{{ item.name | default(item.router_id|replace(':','-')) }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "peerIP": "{{ item.router_id }}" - }} - register: output - retries: 4 - until: output.rc == 0 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - - peer_with_router|default(false) - -- name: Calico | Configure peering with route reflectors at global scope - command: - cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" - # revert when it's already a string - stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" - vars: - stdin: > - {"apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "peer-to-rrs" - }, - "spec": { - "nodeSelector": "!has(i-am-a-route-reflector)", - "peerSelector": "has(i-am-a-route-reflector)" - }} - register: output - retries: 4 - until: output.rc == 0 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ groups['calico_rr'] | default([]) }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - - peer_with_calico_rr|default(false) - -- name: Calico | Configure route reflectors to peer with each other - command: - cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" - # revert when it's already a string - stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" - vars: - stdin: > - {"apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "rr-mesh" - }, - "spec": { - "nodeSelector": "has(i-am-a-route-reflector)", - "peerSelector": "has(i-am-a-route-reflector)" - }} - register: output - retries: 4 - until: output.rc == 0 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ groups['calico_rr'] | default([]) }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - - peer_with_calico_rr|default(false) - - name: Calico | Create calico manifests template: src: "{{ item.file }}.j2" @@ -519,61 +442,6 @@ - inventory_hostname not in groups['kube_control_plane'] - calico_datastore == "kdd" -- name: Calico | Configure node asNumber for per node peering - command: - cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" - stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" - vars: - stdin: > - {"apiVersion": "projectcalico.org/v3", - "kind": "Node", - "metadata": { - "name": "{{ inventory_hostname }}" - }, - "spec": { - "bgp": { - "asNumber": "{{ local_as }}" - }, - "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }} - register: output - retries: 4 - until: output.rc == 0 - delay: "{{ retry_stagger | random + 3 }}" - when: - - peer_with_router|default(false) - - inventory_hostname in groups['k8s_cluster'] - - local_as is defined - - groups['calico_rr'] | default([]) | length == 0 - -- name: Calico | Configure peering with router(s) at node scope - command: - cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" - stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" - vars: - stdin: > - {"apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id|replace(':','-')) }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "node": "{{ inventory_hostname }}", - "peerIP": "{{ item.router_id }}", - "sourceAddress": "{{ item.sourceaddress|default('UseNodeIP') }}" - }} - register: output - retries: 4 - until: output.rc == 0 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}" - delegate_to: "{{ groups['kube_control_plane'][0] }}" - when: - - peer_with_router|default(false) - - inventory_hostname in groups['k8s_cluster'] - - name: Calico | Create Calico ipam manifests template: src: "{{ item.file }}.j2" @@ -591,3 +459,11 @@ state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] + +- include_tasks: peer_with_calico_rr.yml + when: + - peer_with_calico_rr|default(false) + +- include_tasks: peer_with_router.yml + when: + - peer_with_router|default(false) diff --git a/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml b/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml new file mode 100644 index 000000000..c95924ac6 --- /dev/null +++ b/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml @@ -0,0 +1,86 @@ +--- +- name: Calico | Set lable for groups nodes # noqa 301 305 + shell: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite" + changed_when: false + register: calico_group_id_label + until: calico_group_id_label is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 + when: + - calico_group_id is defined + +- name: Calico | Configure peering with route reflectors at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ calcio_rr_id }}-to-node" + }, + "spec": { + "peerSelector": "calico-rr-id == '{{ calcio_rr_id }}'", + "nodeSelector": "calico-group-id == '{{ calico_group_id }}'" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + when: + - calcio_rr_id is defined + - calico_group_id is defined + - inventory_hostname in groups['calico_rr'] + +- name: Calico | Configure peering with route reflectors at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "peer-to-rrs" + }, + "spec": { + "nodeSelector": "!has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico_rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calcio_rr_id is not defined or calico_group_id is not defined + +- name: Calico | Configure route reflectors to peer with each other + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "rr-mesh" + }, + "spec": { + "nodeSelector": "has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico_rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/network_plugin/calico/tasks/peer_with_router.yml b/roles/network_plugin/calico/tasks/peer_with_router.yml new file mode 100644 index 000000000..a698ed1da --- /dev/null +++ b/roles/network_plugin/calico/tasks/peer_with_router.yml @@ -0,0 +1,77 @@ +--- +- name: Calico | Configure peering with router(s) at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "global-{{ item.name | default(item.router_id|replace(':','-')) }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "peerIP": "{{ item.router_id }}" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Configure node asNumber for per node peering + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "Node", + "metadata": { + "name": "{{ inventory_hostname }}" + }, + "spec": { + "bgp": { + "asNumber": "{{ local_as }}" + }, + "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + when: + - inventory_hostname in groups['k8s_cluster'] + - local_as is defined + - groups['calico_rr'] | default([]) | length == 0 + +- name: Calico | Configure peering with router(s) at node scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id|replace(':','-')) }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "node": "{{ inventory_hostname }}", + "peerIP": "{{ item.router_id }}", + "sourceAddress": "{{ item.sourceaddress|default('UseNodeIP') }}" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - inventory_hostname in groups['k8s_cluster']