Support for Ambassador OSS as an Ingress (#6135)

Support for Ambassador OSS as an Ingress Controller when
settings `ingress_ambassador_enabled: true`.

Signed-off-by: Alvaro Saurin <alvaro.saurin@gmail.com>

docs/ansible.md

@@ -138,6 +138,7 @@ The following tags are defined in playbooks:
 |                   upload | Distributing images/binaries across hosts
 |                    weave | Network plugin Weave
 |              ingress_alb | AWS ALB Ingress Controller
+|              ambassador  | Ambassador Ingress Controller
 
 Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
 tags found in the codebase. New tags will be listed with the empty "Used for"

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -103,6 +103,11 @@ ingress_publish_status_address: ""
 # ingress_nginx_extra_args:
 #   - --default-ssl-certificate=default/foo-tls
 
+# ambassador ingress controller deployment
+ingress_ambassador_enabled: false
+# ingress_ambassador_namespace: "ambassador"
+# ingress_ambassador_version: "*"
+
 # ALB ingress controller deployment
 ingress_alb_enabled: false
 # alb_ingress_aws_region: "us-east-1"

roles/download/defaults/main.yml

@@ -482,6 +482,8 @@ local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-p
 local_path_provisioner_image_tag: "v0.0.14"
 ingress_nginx_controller_image_repo: "{{ quay_image_repo }}/kubernetes-ingress-controller/nginx-ingress-controller"
 ingress_nginx_controller_image_tag: "0.32.0"
+ingress_ambassador_image_repo: "{{ quay_image_repo }}/datawire/ambassador-operator"
+ingress_ambassador_image_tag: "v1.2.8"
 alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller"
 alb_ingress_image_tag: "v1.1.8"
 cert_manager_version: "v0.11.1"
@@ -980,6 +982,15 @@ downloads:
     groups:
     - kube-node
 
+  ingress_ambassador_controller:
+    enabled: "{{ ingress_ambassador_enabled }}"
+    container: true
+    repo: "{{ ingress_ambassador_image_repo }}"
+    tag: "{{ ingress_ambassador_image_tag }}"
+    sha256: "{{ ingress_ambassador_digest_checksum|default(None) }}"
+    groups:
+    - kube-node
+
   ingress_alb_controller:
     enabled: "{{ ingress_alb_enabled }}"
     container: true

roles/kubernetes-apps/ingress_controller/ambassador/README.md

@@ -0,0 +1,37 @@
+# Installation Guide
+
+- [Installation Guide](#installation-guide)
+  - [Ambassador](#ambassador)
+  - [Ambassador Operator](#ambassador-operator)
+  - [Configuration](#configuration)
+  - [Ingress annotations](#ingress-annotations)
+
+## Ambassador
+
+The Ambassador API Gateway provides all the functionality of a traditional ingress controller
+(e.g., path-based routing) while exposing many additional capabilities such as authentication,
+URL rewriting, CORS, rate limiting, and automatic metrics collection.
+
+## Ambassador Operator
+
+This addon deploys the Ambassador Operator, which in turn will install Ambassador in
+a kubespray cluster.
+
+The Ambassador Operator is a Kubernetes Operator that controls Ambassador's complete lifecycle
+in your cluster, automating many of the repeatable tasks you would otherwise have to perform
+yourself.  Once installed, the Operator will complete installations and seamlessly upgrade to new
+versions of Ambassador as they become available.
+
+## Configuration
+
+* `ingress_ambassador_namespace` (default `ambassador`): namespace for installing Ambassador.
+* `ingress_ambassador_update_window` (default `0 0 * * SUN`): _crontab_-like expression
+  for specifying when the Operator should try to update the Ambassador API Gateway.
+* `ingress_ambassador_version` (defaulkt: `*`): SemVer rule for versions allowed for
+  installation/updates.
+
+## Ingress annotations
+
+The Ambassador API Gateway will automatically load balance `Ingress` resources
+that include the annotation `kubernetes.io/ingress.class=ambassador`. All the other
+resources will be just ignored.

roles/kubernetes-apps/ingress_controller/ambassador/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+ingress_ambassador_namespace: "ambassador"
+ingress_ambassador_version: "*"
+ingress_ambassador_update_window: "0 0 * * SUN"
+ingress_ambassador_replicas: 1
+ingress_ambassador_insecure_port: 80
+ingress_ambassador_secure_port: 443
+ingress_ambassador_extra_args: []
+ingress_ambassador_host_network: false

roles/kubernetes-apps/ingress_controller/ambassador/tasks/main.yml

@@ -0,0 +1,72 @@
+---
+
+- name: Ambassador | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/ambassador"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Ambassador | Templates list
+  set_fact:
+    ingress_ambassador_templates:
+      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+      - { name: crd-ambassador-installation, file: crd-ambassador-installation.yml, type: customresourcedefinition }
+      - { name: sa-ambassador, file: sa-ambassador.yml, type: sa }
+      - { name: clusterrole-ambassador, file: clusterrole-ambassador.yml, type: clusterrole }
+      - { name: clusterrolebinding-ambassador, file: clusterrolebinding-ambassador.yml, type: clusterrolebinding }
+      - { name: role-ambassador, file: role-ambassador.yml, type: role }
+      - { name: rolebinding-ambassador, file: rolebinding-ambassador.yml, type: rolebinding }
+      - { name: deploy-ambassador, file: deploy-ambassador.yml, type: deploy }
+
+- name: Ambassador | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/ambassador/{{ item.file }}"
+  loop: "{{ ingress_ambassador_templates }}"
+  register: ingress_ambassador_manifests
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Ambassador | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ ingress_ambassador_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/ambassador/{{ item.item.file }}"
+    state: "latest"
+  loop: "{{ ingress_ambassador_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+# load the AmbassadorInstallation _after_ the CustomResourceDefinition has been loaded
+
+- name: Ambassador | AmbassadorInstallation template
+  set_fact:
+    ingress_ambassador_cr_templates:
+      - { name: cr-ambassador-installation, file: cr-ambassador-installation.yml, type: cr }
+
+- name: Ambassador | Create installation manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/ambassador/{{ item.file }}"
+  loop: "{{ ingress_ambassador_cr_templates }}"
+  register: ingress_ambassador_cr_manifests
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Ambassador | Apply AmbassadorInstallation
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ ingress_ambassador_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/ambassador/{{ item.item.file }}"
+    state: "latest"
+  loop: "{{ ingress_ambassador_cr_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ingress_controller/ambassador/templates/00-namespace.yml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ ingress_ambassador_namespace }}
+  labels:
+    name: {{ ingress_ambassador_namespace }}

roles/kubernetes-apps/ingress_controller/ambassador/templates/clusterrole-ambassador.yml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ambassador-operator-cluster
+  labels:
+    app.kubernetes.io/name: ambassador-operator
+    app.kubernetes.io/part-of: ambassador-operator
+rules:
+  - apiGroups: ['*']
+    resources: ['*']
+    verbs: ['*']
+  - nonResourceURLs: ['*']
+    verbs: ['*']

roles/kubernetes-apps/ingress_controller/ambassador/templates/clusterrolebinding-ambassador.yml.j2

@@ -0,0 +1,16 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ambassador-operator-cluster
+  labels:
+    app.kubernetes.io/name: ambassador-operator
+    app.kubernetes.io/part-of: ambassador-operator
+subjects:
+  - kind: ServiceAccount
+    name: ambassador-operator
+    namespace: {{ ingress_ambassador_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ambassador-operator-cluster
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/ingress_controller/ambassador/templates/cr-ambassador-installation.yml.j2

@@ -0,0 +1,37 @@
+apiVersion: getambassador.io/v2
+kind: AmbassadorInstallation
+metadata:
+  name: ambassador
+  labels:
+    app.kubernetes.io/name: ambassador-operator
+    app.kubernetes.io/part-of: ambassador-operator
+spec:
+  installOSS: true
+{% if ingress_ambassador_update_window %}
+  updateWindow: "{{ ingress_ambassador_update_window }}"
+{% endif %}
+{% if ingress_ambassador_version %}
+  version: "{{ ingress_ambassador_version }}"
+{% endif %}
+  helmValues:
+    tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: Equal
+        effect: NoSchedule
+    deploymentTool: amb-oper-kubespray
+{% if ingress_ambassador_host_network %}
+    hostNetwork: true
+{% endif %}
+    replicaCount: {{ ingress_ambassador_replicas }}
+    service:
+      ports:
+        - name: http
+          port: 80
+          hostPort: {{ ingress_ambassador_insecure_port }}
+          targetPort: 8080
+          protocol: TCP
+        - name: https
+          port: 443
+          hostPort: {{ ingress_ambassador_secure_port }}
+          targetPort: 8443
+          protocol: TCP

roles/kubernetes-apps/ingress_controller/ambassador/templates/crd-ambassador-installation.yml.j2

@@ -0,0 +1,186 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ambassadorinstallations.getambassador.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .spec.version
+    name: VERSION
+    type: string
+  - JSONPath: .spec.updateWindow
+    name: UPDATE-WINDOW
+    type: integer
+  - JSONPath: .status.lastCheckTime
+    description: Last time checked
+    name: LAST-CHECK
+    type: string
+  - JSONPath: .status.conditions[?(@.type=='Deployed')].status
+    description: Indicates if deployment has completed
+    name: DEPLOYED
+    type: string
+  - JSONPath: .status.conditions[?(@.type=='Deployed')].reason
+    description: Reason for deployment completed
+    name: REASON
+    priority: 1
+    type: string
+  - JSONPath: .status.conditions[?(@.type=='Deployed')].message
+    description: Message for deployment completed
+    name: MESSAGE
+    priority: 1
+    type: string
+  - JSONPath: .status.deployedRelease.appVersion
+    description: Deployed version of Ambassador
+    name: DEPLOYED-VERSION
+    type: string
+  - JSONPath: .status.deployedRelease.flavor
+    description: Deployed flavor of Ambassador (OSS or AES)
+    name: DEPLOYED-FLAVOR
+    type: string
+  group: getambassador.io
+  names:
+    kind: AmbassadorInstallation
+    listKind: AmbassadorInstallationList
+    plural: ambassadorinstallations
+    singular: ambassadorinstallation
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: AmbassadorInstallation is the Schema for the ambassadorinstallations
+        API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: AmbassadorInstallationSpec defines the desired state of AmbassadorInstallation
+          properties:
+            baseImage:
+              description: An (optional) image to use instead of the image specified
+                in the Helm chart.
+              type: string
+            helmRepo:
+              description: An (optional) Helm repository.
+              type: string
+            installOSS:
+              description: 'Installs [Ambassador OSS](https://www.getambassador.io/docs/latest/topics/install/install-ambassador-oss/)
+                instead of [AES](https://www.getambassador.io/docs/latest/topics/install/).
+                Default is false which means it installs AES by default. TODO: 1.
+                AES/AOSS is not installed and the user installs using `installOSS:
+                true`, then we straightaway install AOSS. 2. AOSS is installed via
+                operator and the user sets `installOSS: false`, then we perform the
+                migration as    detailed here - https://www.getambassador.io/docs/latest/topics/install/upgrade-to-edge-stack/
+                3. AES is installed and the user sets `installOSS: true`, then we
+                point users to the docs which gives them    pointers on how to do
+                that themselves.'
+              type: boolean
+            logLevel:
+              description: 'An (optional) log level: debug, info...'
+              enum:
+              - info
+              - debug
+              - warn
+              - warning
+              - error
+              - critical
+              - fatal
+              type: string
+            updateWindow:
+              description: "`updateWindow` is an optional item that will control when
+                the updates can take place. This is used to force system updates to
+                happen late at night if that’s what the sysadmins want. \n  * There
+                can be any number of `updateWindow` entries (separated by commas).
+                \ * `Never` turns off automatic updates even if there are other entries
+                in the    comma-separated list. `Never` is used by sysadmins to disable
+                all updates    during blackout periods by doing a `kubectl apply`
+                or using our Edge Policy    Console to set this. * Each `updateWindow`
+                is in crontab format (see https://crontab.guru/)   Some examples of
+                `updateWindows` are:    - `* 0-6 * * * SUN`: every Sunday, from _0am_
+                to _6am_    - `* 5 1 * * *`: every first day of the month, at _5am_
+                * The Operator cannot guarantee minute time granularity, so specifying
+                \  a minute in the crontab expression can lead to some updates happening
+                \  sooner/later than expected."
+              type: string
+            version:
+              description: "We are using SemVer for the version number and it can
+                be specified with any level of precision and can optionally end in
+                `*`. These are interpreted as: \n * `1.0` = exactly version 1.0 *
+                `1.1` = exactly version 1.1 * `1.1.*` = version 1.1 and any bug fix
+                versions `1.1.1`, `1.1.2`, `1.1.3`, etc. * `2.*` = version 2.0 and
+                any incremental and bug fix versions `2.0`, `2.0.1`,   `2.0.2`, `2.1`,
+                `2.2`, `2.2.1`, etc. * `*` = all versions. * `3.0-ea` = version `3.0-ea1`
+                and any subsequent EA releases on `3.0`.   Also selects the final
+                3.0 once the final GA version is released. * `4.*-ea` = version `4.0-ea1`
+                and any subsequent EA release on `4.0`.   Also selects the final GA
+                `4.0`. Also selects any incremental and bug   fix versions `4.*` and
+                `4.*.*`. Also selects the most recent `4.*` EA release   i.e., if
+                `4.0.5` is the last GA version and there is a `4.1-EA3`, then this
+                \  selects `4.1-EA3` over the `4.0.5` GA. \n   You can find the reference
+                docs about the SemVer syntax accepted   [here](https://github.com/Masterminds/semver#basic-comparisons)."
+              type: string
+          type: object
+        status:
+          description: AmbassadorInstallationStatus defines the observed state of
+            AmbassadorInstallation
+          properties:
+            conditions:
+              description: List of conditions the installation has experienced.
+              items:
+                description: AmbInsCondition defines an Ambassador installation condition,
+                  as well as the last time there was a transition to this condition..
+                properties:
+                  lastTransitionTime:
+                    format: date-time
+                    type: string
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                  status:
+                    type: string
+                  type:
+                    type: string
+                required:
+                - status
+                - type
+                type: object
+              type: array
+            deployedRelease:
+              description: the currently deployed Helm chart
+              nullable: true
+              properties:
+                appVersion:
+                  type: string
+                flavor:
+                  type: string
+                manifest:
+                  type: string
+                name:
+                  type: string
+                version:
+                  type: string
+              type: object
+            lastCheckTime:
+              description: Last time a successful update check was performed.
+              format: date-time
+              nullable: true
+              type: string
+          required:
+          - conditions
+          type: object
+      type: object
+  version: v2
+  versions:
+  - name: v2
+    served: true
+    storage: true

roles/kubernetes-apps/ingress_controller/ambassador/templates/deploy-ambassador.yml.j2

@@ -0,0 +1,43 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ambassador-operator
+  namespace: {{ ingress_ambassador_namespace }}
+  labels:
+    app.kubernetes.io/name: ambassador-operator
+    app.kubernetes.io/part-of: ambassador-operator
+    getambassador.io/installer: operator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: ambassador-operator
+      app.kubernetes.io/name: ambassador-operator
+      app.kubernetes.io/part-of: ambassador-operator
+  template:
+    metadata:
+      labels:
+        name: ambassador-operator
+        getambassador.io/installer: operator
+        app.kubernetes.io/name: ambassador-operator
+        app.kubernetes.io/part-of: ambassador-operator
+    spec:
+      serviceAccountName: ambassador-operator
+      containers:
+        - name: ambassador-operator
+          image: {{ ingress_ambassador_image_repo }}:{{ ingress_ambassador_image_tag }}
+          command:
+          - ambassador-operator
+          imagePullPolicy: Always
+          env:
+            - name: WATCH_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OPERATOR_NAME
+              value: "ambassador-operator"

roles/kubernetes-apps/ingress_controller/ambassador/templates/role-ambassador.yml.j2

@@ -0,0 +1,82 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: ambassador-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - services/finalizers
+      - endpoints
+      - persistentvolumeclaims
+      - events
+      - configmaps
+      - secrets
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+      - daemonsets
+      - replicasets
+      - statefulsets
+      - customresourcedefinitions
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+    verbs:
+      - get
+      - create
+  - apiGroups:
+      - apps
+    resourceNames:
+      - ambassador-operator
+    resources:
+      - deployments/finalizers
+    verbs:
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - apps
+    resources:
+      - replicasets
+      - deployments
+    verbs:
+      - get
+  - apiGroups:
+      - getambassador.io
+    resources:
+      - '*'
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch

roles/kubernetes-apps/ingress_controller/ambassador/templates/rolebinding-ambassador.yml.j2

@@ -0,0 +1,12 @@
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ambassador-operator
+subjects:
+  - kind: ServiceAccount
+    name: ambassador-operator
+roleRef:
+  kind: Role
+  name: ambassador-operator
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/ingress_controller/ambassador/templates/sa-ambassador.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ambassador-operator
+  namespace: {{ ingress_ambassador_namespace }}
+  labels:
+    app.kubernetes.io/name: ambassador-operator
+    app.kubernetes.io/part-of: ambassador-operator

roles/kubernetes-apps/ingress_controller/meta/main.yml

@@ -7,6 +7,13 @@ dependencies:
       - ingress-nginx
       - ingress-controller
 
+  - role: kubernetes-apps/ingress_controller/ambassador
+    when: ingress_ambassador_enabled
+    tags:
+      - apps
+      - ambassador
+      - ingress-controller
+
   - role: kubernetes-apps/ingress_controller/cert_manager
     when: cert_manager_enabled
     tags:

roles/kubespray-defaults/defaults/main.yaml

@@ -320,6 +320,7 @@ persistent_volumes_enabled: false
 cephfs_provisioner_enabled: false
 rbd_provisioner_enabled: false
 ingress_nginx_enabled: false
+ingress_ambassador_enabled: false
 ingress_alb_enabled: false
 cert_manager_enabled: false
 expand_persistent_volumes: false

tests/files/packet_opensuse-canal.yml

@@ -7,3 +7,6 @@ mode: default
 kube_network_plugin: canal
 deploy_netchecker: true
 dns_min_replicas: 1
+
+# test Ambassador
+ingress_ambassador_enabled: true