diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30ca25bbc..bbf22d0ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,6 +61,7 @@ before_script: KUBELET_DEPLOYMENT: "host" VAULT_DEPLOYMENT: "docker" WEAVE_CPU_LIMIT: "100m" + EXTRA_SETTINGS: "{}" MAGIC: "ci check this" .gce: &gce @@ -132,6 +133,7 @@ before_script: -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" + -e "${EXTRA_SETTINGS}" --limit "all:!fake_hosts" cluster.yml @@ -161,6 +163,7 @@ before_script: -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" + -e "${EXTRA_SETTINGS}" --limit "all:!fake_hosts" $PLAYBOOK; fi @@ -198,6 +201,7 @@ before_script: -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} + -e "${EXTRA_SETTINGS}" --limit "all:!fake_hosts" cluster.yml; fi @@ -246,6 +250,7 @@ before_script: -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} + -e "${EXTRA_SETTINGS}" --limit "all:!fake_hosts" cluster.yml; fi @@ -323,13 +328,17 @@ before_script: CLUSTER_MODE: default STARTUP_SCRIPT: "" -.centos7_flannel_variables: ¢os7_flannel_variables +.centos7_flannel_addons_variables: ¢os7_flannel_addons_variables # stage: deploy-gce-part2 KUBE_NETWORK_PLUGIN: flannel CLOUD_IMAGE: centos-7 CLOUD_REGION: us-west1-a CLOUD_MACHINE_TYPE: "n1-standard-1" CLUSTER_MODE: default + EXTRA_SETTINGS: >- + { helm_enabled: true, + istio_enabled: true, + efk_enabled: true } STARTUP_SCRIPT: "" .debian8_calico_variables: &debian8_calico_variables @@ -440,24 +449,24 @@ coreos-calico-sep-triggers: when: on_success only: ['triggers'] -centos7-flannel: +centos7-flannel-addons: stage: deploy-gce-part2 <<: *job <<: *gce variables: <<: *gce_variables - <<: *centos7_flannel_variables + <<: *centos7_flannel_addons_variables when: on_success except: ['triggers'] only: [/^pr-.*$/] -centos7-flannel-triggers: +centos7-flannel-addons-triggers: stage: deploy-gce-part1 <<: *job <<: *gce variables: <<: *gce_variables - <<: *centos7_flannel_variables + <<: *centos7_flannel_addons_variables when: on_success only: ['triggers'] diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index f5250cb7c..a1d98ec14 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -151,6 +151,9 @@ efk_enabled: false # Helm deployment helm_enabled: false +# Istio depoyment +istio_enabled: false + # Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts # kubeconfig_localhost: false # Download kubectl onto the host that runs Ansible in GITDIR/artifacts diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b9d12d20b..9c375fa61 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -38,6 +38,11 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release # Checksums kubeadm_checksum: "9f4b9cf255d5ef45481d5a1b20bfe84c1d633d67cd50eeaa5c8712fb8fc1bd5b" +istio_version: "0.2.6" + +istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" +istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370 + # Containers etcd_image_repo: "quay.io/coreos/etcd" etcd_image_tag: "{{ etcd_version }}" @@ -132,6 +137,15 @@ downloads: unarchive: false owner: "root" mode: "0755" + istioctl: + version: "{{ istio_version }}" + dest: "istio/istioctl" + sha256: "{{ istioctl_checksum }}" + source_url: "{{ istioctl_download_url }}" + url: "{{ istioctl_download_url }}" + unarchive: false + owner: "root" + mode: "0755" hyperkube: container: true repo: "{{ hyperkube_image_repo }}" diff --git a/roles/kubernetes-apps/istio/defaults/main.yml b/roles/kubernetes-apps/istio/defaults/main.yml new file mode 100644 index 000000000..dc51ea7d6 --- /dev/null +++ b/roles/kubernetes-apps/istio/defaults/main.yml @@ -0,0 +1,32 @@ +--- +istio_enabled: false + +istio_namespace: istio-system +istio_version: "0.2.6" + +istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" +istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370 + +istio_proxy_image_repo: docker.io/istio/proxy +istio_proxy_image_tag: "{{ istio_version }}" + +istio_proxy_init_image_repo: docker.io/istio/proxy_init +istio_proxy_init_image_tag: "{{ istio_version }}" + +istio_ca_image_repo: docker.io/istio/istio-ca +istio_ca_image_tag: "{{ istio_version }}" + +istio_mixer_image_repo: docker.io/istio/mixer +istio_mixer_image_tag: "{{ istio_version }}" + +istio_pilot_image_repo: docker.io/istio/pilot +istio_pilot_image_tag: "{{ istio_version }}" + +istio_proxy_debug_image_repo: docker.io/istio/proxy_debug +istio_proxy_debug_image_tag: "{{ istio_version }}" + +istio_sidecar_initializer_image_repo: docker.io/istio/sidecar_initializer +istio_sidecar_initializer_image_tag: "{{ istio_version }}" + +istio_statsd_image_repo: prom/statsd-exporter +istio_statsd_image_tag: latest diff --git a/roles/kubernetes-apps/istio/meta/main.yml b/roles/kubernetes-apps/istio/meta/main.yml new file mode 100644 index 000000000..97c4df2a0 --- /dev/null +++ b/roles/kubernetes-apps/istio/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: download + file: "{{ downloads.istioctl }}" diff --git a/roles/kubernetes-apps/istio/tasks/main.yml b/roles/kubernetes-apps/istio/tasks/main.yml new file mode 100644 index 000000000..5e36a56cc --- /dev/null +++ b/roles/kubernetes-apps/istio/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: istio | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/istio" + owner: root + group: root + mode: 0755 + recurse: yes + +- name: istio | Lay out manifests + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/addons/istio/{{item.file}}" + with_items: + - {name: istio-mixer, file: istio.yml, type: deployment } + - {name: istio-initializer, file: istio-initializer.yml, type: deployment } + register: manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: istio | Copy istioctl binary from download dir + command: rsync -piu "{{ local_release_dir }}/istio/istioctl" "{{ bin_dir }}/istioctl" + changed_when: false + +- name: istio | Set up bash completion + shell: "{{ bin_dir }}/istioctl completion >/etc/bash_completion.d/istioctl.sh" + when: ansible_os_family in ["Debian","RedHat"] + +- name: istio | Set bash completion file + file: + path: /etc/bash_completion.d/istioctl.sh + owner: root + group: root + mode: 0755 + when: ansible_os_family in ["Debian","RedHat"] + +- name: istio | apply manifests + kube: + name: "{{item.item.name}}" + namespace: "{{ istio_namespace }}" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/addons/istio/{{item.item.file}}" + state: "latest" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/istio/templates/istio-initializer.yml.j2 b/roles/kubernetes-apps/istio/templates/istio-initializer.yml.j2 new file mode 100644 index 000000000..84f957ed1 --- /dev/null +++ b/roles/kubernetes-apps/istio/templates/istio-initializer.yml.j2 @@ -0,0 +1,84 @@ +# GENERATED FILE. Use with Kubernetes 1.7+ +# TO UPDATE, modify files in install/kubernetes/templates and run install/updateVersion.sh +################################ +# Istio initializer +################################ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-inject + namespace: {{ istio_namespace }} +data: + config: |- + policy: "enabled" + namespaces: [""] # everything, aka v1.NamepsaceAll, aka cluster-wide + initializerName: "sidecar.initializer.istio.io" + params: + initImage: {{ istio_proxy_init_image_repo }}:{{ istio_proxy_init_image_tag }} + proxyImage: {{ istio_proxy_image_repo }}:{{ istio_proxy_image_tag }} + verbosity: 2 + version: 0.2.6 + meshConfigMapName: istio + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-initializer-service-account + namespace: {{ istio_namespace }} +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: istio-initializer + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" + initializers: + pending: [] + labels: + istio: istio-initializer +spec: + replicas: 1 + template: + metadata: + name: istio-initializer + labels: + istio: initializer + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-initializer-service-account + containers: + - name: initializer + image: {{ istio_sidecar_initializer_image_repo }}:{{ istio_sidecar_initializer_image_tag }} + imagePullPolicy: IfNotPresent + args: + - --port=8083 + - --namespace={{ istio_namespace }} + - -v=2 + volumeMounts: + - name: config-volume + mountPath: /etc/istio/config + volumes: + - name: config-volume + configMap: + name: istio +--- +apiVersion: admissionregistration.k8s.io/v1alpha1 +kind: InitializerConfiguration +metadata: + name: istio-sidecar +initializers: + - name: sidecar.initializer.istio.io + rules: + - apiGroups: + - "*" + apiVersions: + - "*" + resources: + - deployments + - statefulsets + - jobs + - daemonsets +--- diff --git a/roles/kubernetes-apps/istio/templates/istio.yml.j2 b/roles/kubernetes-apps/istio/templates/istio.yml.j2 new file mode 100644 index 000000000..bd0b93a7f --- /dev/null +++ b/roles/kubernetes-apps/istio/templates/istio.yml.j2 @@ -0,0 +1,1285 @@ +# GENERATED FILE. Use with Kubernetes 1.7+ +# TO UPDATE, modify files in install/kubernetes/templates and run install/updateVersion.sh +################################ +# Istio system namespace +################################ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ istio_namespace }} +--- +################################ +# Istio RBAC +################################ +# Permissions and roles for istio +# To debug: start the cluster with -vmodule=rbac,3 to enable verbose logging on RBAC DENY +# Also helps to enable logging on apiserver 'wrap' to see the URLs. +# Each RBAC deny needs to be mapped into a rule for the role. +# If using minikube, start with '--extra-config=apiserver.Authorization.Mode=RBAC' +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-pilot-istio-system +rules: +- apiGroups: ["config.istio.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["*"] +- apiGroups: ["istio.io"] + resources: ["istioconfigs", "istioconfigs.istio.io"] + verbs: ["*"] +- apiGroups: ["extensions"] + resources: ["thirdpartyresources", "thirdpartyresources.extensions", "ingresses", "ingresses/status"] + verbs: ["*"] +- apiGroups: [""] + resources: ["configmaps", "endpoints", "pods", "services"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces", "nodes", "secrets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["externaladmissionhookconfigurations"] + verbs: ["create", "update", "delete"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-initializer-istio-system +rules: +- apiGroups: ["*"] + resources: ["deployments", "statefulsets", "jobs", "cronjobs", "daemonsets", "replicasets", "replicationcontrollers"] + verbs: ["initialize", "patch", "watch", "list"] +- apiGroups: ["*"] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +# Mixer CRD needs to watch and list CRDs +# It also uses discovery API to discover Kinds of config.istio.io +# K8s adapter needs to list pods, services etc. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-mixer-istio-system +rules: +- apiGroups: ["config.istio.io"] # Istio CRD watcher + resources: ["*"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps", "endpoints", "pods", "services", "namespaces", "secrets"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-ca-istio-system +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "watch", "list", "update"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "watch", "list"] +--- +# Permissions for the sidecar proxy. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-sidecar-istio-system +rules: +- apiGroups: ["istio.io"] + resources: ["istioconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["extensions"] + resources: ["thirdpartyresources", "ingresses"] + verbs: ["get", "watch", "list", "update"] +- apiGroups: [""] + resources: ["configmaps", "pods", "endpoints", "services"] + verbs: ["get", "watch", "list"] +--- +# Grant permissions to the Pilot/discovery. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-pilot-admin-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-pilot-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-pilot-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to the Sidecar initializer +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-initializer-admin-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-initializer-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-initializer-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to the CA. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-ca-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-ca-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-ca-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to the Ingress controller. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-ingress-admin-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-ingress-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-pilot-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to the Egress controller. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-egress-admin-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-egress-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-pilot-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to the sidecar. +# TEMPORARY: the istioctl should generate a separate service account for the proxy, and permission +# granted only to that account ! +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-sidecar-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: default + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-sidecar-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Grant permissions to Mixer. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: istio-mixer-admin-role-binding-istio-system +subjects: +- kind: ServiceAccount + name: istio-mixer-service-account + namespace: {{ istio_namespace }} +roleRef: + kind: ClusterRole + name: istio-mixer-istio-system + apiGroup: rbac.authorization.k8s.io +--- +# Mixer +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-mixer + namespace: {{ istio_namespace }} +data: + mapping.conf: |- +--- +apiVersion: v1 +kind: Service +metadata: + name: istio-mixer + namespace: {{ istio_namespace }} + labels: + istio: mixer +spec: + ports: + - name: tcp + port: 9091 + - name: http-health + port: 9093 + - name: configapi + port: 9094 + - name: statsd-prom + port: 9102 + - name: statsd-udp + port: 9125 + protocol: UDP + - name: prometheus + port: 42422 + selector: + istio: mixer +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-mixer-service-account + namespace: {{ istio_namespace }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: istio-mixer + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" +spec: + replicas: 1 + template: + metadata: + labels: + istio: mixer + spec: + serviceAccountName: istio-mixer-service-account + containers: + - name: statsd-to-prometheus + image: {{ istio_statsd_image_repo }}:{{ istio_statsd_image_tag }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9102 + - containerPort: 9125 + protocol: UDP + args: + - '-statsd.mapping-config=/etc/statsd/mapping.conf' + volumeMounts: + - name: config-volume + mountPath: /etc/statsd + - name: mixer + image: {{ istio_mixer_image_repo }}:{{ istio_mixer_image_tag }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9091 + - containerPort: 9094 + - containerPort: 42422 + args: + - --configStoreURL=fs:///etc/opt/mixer/configroot + - --configStore2URL=k8s:// + - --configDefaultNamespace=istio-system + - --traceOutput=http://zipkin:9411/api/v1/spans + - --logtostderr + - -v + - "2" + volumes: + - name: config-volume + configMap: + name: istio-mixer +--- +# Mixer CRD definitions are generated using +# mixs crd all + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: rules.config.istio.io + labels: + package: istio.io.mixer + istio: core +spec: + group: config.istio.io + names: + kind: rule + plural: rules + singular: rule + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: attributemanifests.config.istio.io + labels: + package: istio.io.mixer + istio: core +spec: + group: config.istio.io + names: + kind: attributemanifest + plural: attributemanifests + singular: attributemanifest + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: deniers.config.istio.io + labels: + package: denier + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: denier + plural: deniers + singular: denier + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: listcheckers.config.istio.io + labels: + package: listchecker + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: listchecker + plural: listcheckers + singular: listchecker + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: memquotas.config.istio.io + labels: + package: memquota + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: memquota + plural: memquotas + singular: memquota + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: noops.config.istio.io + labels: + package: noop + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: noop + plural: noops + singular: noop + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: prometheuses.config.istio.io + labels: + package: prometheus + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: prometheus + plural: prometheuses + singular: prometheus + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: stackdrivers.config.istio.io + labels: + package: stackdriver + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: stackdriver + plural: stackdrivers + singular: stackdriver + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: statsds.config.istio.io + labels: + package: statsd + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: statsd + plural: statsds + singular: statsd + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: stdios.config.istio.io + labels: + package: stdio + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: stdio + plural: stdios + singular: stdio + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: svcctrls.config.istio.io + labels: + package: svcctrl + istio: mixer-adapter +spec: + group: config.istio.io + names: + kind: svcctrl + plural: svcctrls + singular: svcctrl + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: checknothings.config.istio.io + labels: + package: checknothing + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: checknothing + plural: checknothings + singular: checknothing + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: listentries.config.istio.io + labels: + package: listentry + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: listentry + plural: listentries + singular: listentry + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: logentries.config.istio.io + labels: + package: logentry + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: logentry + plural: logentries + singular: logentry + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: metrics.config.istio.io + labels: + package: metric + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: metric + plural: metrics + singular: metric + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: quotas.config.istio.io + labels: + package: quota + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: quota + plural: quotas + singular: quota + scope: Namespaced + version: v1alpha2 +--- + +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: reportnothings.config.istio.io + labels: + package: reportnothing + istio: mixer-instance +spec: + group: config.istio.io + names: + kind: reportnothing + plural: reportnothings + singular: reportnothing + scope: Namespaced + version: v1alpha2 +--- +apiVersion: "config.istio.io/v1alpha2" +kind: attributemanifest +metadata: + name: istioproxy + namespace: {{ istio_namespace }} +spec: + attributes: + origin.ip: + valueType: IP_ADDRESS + origin.uid: + valueType: STRING + origin.user: + valueType: STRING + request.headers: + valueType: STRING_MAP + request.id: + valueType: STRING + request.host: + valueType: STRING + request.method: + valueType: STRING + request.path: + valueType: STRING + request.reason: + valueType: STRING + request.referer: + valueType: STRING + request.scheme: + valueType: STRING + request.size: + valueType: INT64 + request.time: + valueType: TIMESTAMP + request.useragent: + valueType: STRING + response.code: + valueType: INT64 + response.duration: + valueType: DURATION + response.headers: + valueType: STRING_MAP + response.size: + valueType: INT64 + response.time: + valueType: TIMESTAMP + source.uid: + valueType: STRING + source.user: + valueType: STRING + destination.uid: + valueType: STRING + connection.id: + valueType: STRING + connection.received.bytes: + valueType: INT64 + connection.received.bytes_total: + valueType: INT64 + connection.sent.bytes: + valueType: INT64 + connection.sent.bytes_total: + valueType: INT64 + connection.duration: + valueType: DURATION + context.protocol: + valueType: STRING + context.timestamp: + valueType: TIMESTAMP + context.time: + valueType: TIMESTAMP + +--- +apiVersion: "config.istio.io/v1alpha2" +kind: attributemanifest +metadata: + name: kubernetes + namespace: {{ istio_namespace }} +spec: + attributes: + source.ip: + valueType: IP_ADDRESS + source.labels: + valueType: STRING_MAP + source.name: + valueType: STRING + source.namespace: + valueType: STRING + source.service: + valueType: STRING + source.serviceAccount: + valueType: STRING + destination.ip: + valueType: IP_ADDRESS + destination.labels: + valueType: STRING_MAP + destination.name: + valueType: STRING + destination.namespace: + valueType: STRING + destination.service: + valueType: STRING + destination.serviceAccount: + valueType: STRING +--- +apiVersion: "config.istio.io/v1alpha2" +kind: stdio +metadata: + name: handler + namespace: {{ istio_namespace }} +spec: + outputAsJson: true +--- +apiVersion: "config.istio.io/v1alpha2" +kind: logentry +metadata: + name: accesslog + namespace: {{ istio_namespace }} +spec: + severity: '"Default"' + timestamp: request.time + variables: + sourceIp: source.ip | ip("0.0.0.0") + destinationIp: destination.ip | ip("0.0.0.0") + sourceUser: source.user | "" + method: request.method | "" + url: request.path | "" + protocol: request.scheme | "http" + responseCode: response.code | 0 + responseSize: response.size | 0 + requestSize: request.size | 0 + latency: response.duration | "0ms" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: stdio + namespace: {{ istio_namespace }} +spec: + match: "true" # If omitted match is true. + actions: + - handler: handler.stdio + instances: + - accesslog.logentry +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestcount + namespace: {{ istio_namespace }} +spec: + value: "1" + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + response_code: response.code | 200 + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestduration + namespace: {{ istio_namespace }} +spec: + value: response.duration | "0ms" + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + response_code: response.code | 200 + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestsize + namespace: {{ istio_namespace }} +spec: + value: request.size | 0 + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + response_code: response.code | 200 + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: responsesize + namespace: {{ istio_namespace }} +spec: + value: response.size | 0 + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + response_code: response.code | 200 + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: tcpbytesent + namespace: {{ istio_namespace }} + labels: + istio-protocol: tcp # needed so that mixer will only generate when context.protocol == tcp +spec: + value: connection.sent.bytes | 0 + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: tcpbytereceived + namespace: {{ istio_namespace }} + labels: + istio-protocol: tcp # needed so that mixer will only generate when context.protocol == tcp +spec: + value: connection.received.bytes | 0 + dimensions: + source_service: source.service | "unknown" + source_version: source.labels["version"] | "unknown" + destination_service: destination.service | "unknown" + destination_version: destination.labels["version"] | "unknown" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: prometheus +metadata: + name: handler + namespace: {{ istio_namespace }} +spec: + metrics: + - name: request_count + instance_name: requestcount.metric.istio-system + kind: COUNTER + label_names: + - source_service + - source_version + - destination_service + - destination_version + - response_code + - name: request_duration + instance_name: requestduration.metric.istio-system + kind: DISTRIBUTION + label_names: + - source_service + - source_version + - destination_service + - destination_version + - response_code + buckets: + explicit_buckets: + bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + - name: request_size + instance_name: requestsize.metric.istio-system + kind: DISTRIBUTION + label_names: + - source_service + - source_version + - destination_service + - destination_version + - response_code + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: response_size + instance_name: responsesize.metric.istio-system + kind: DISTRIBUTION + label_names: + - source_service + - source_version + - destination_service + - destination_version + - response_code + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: tcp_bytes_sent + instance_name: tcpbytesent.metric.istio-system + kind: COUNTER + label_names: + - source_service + - source_version + - destination_service + - destination_version + - name: tcp_bytes_received + instance_name: tcpbytereceived.metric.istio-system + kind: COUNTER + label_names: + - source_service + - source_version + - destination_service + - destination_version +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promhttp + namespace: {{ istio_namespace }} + labels: + istio-protocol: http +spec: + actions: + - handler: handler.prometheus + instances: + - requestcount.metric + - requestduration.metric + - requestsize.metric + - responsesize.metric +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promtcp + namespace: {{ istio_namespace }} + labels: + istio-protocol: tcp # needed so that mixer will only execute when context.protocol == TCP +spec: + actions: + - handler: handler.prometheus + instances: + - tcpbytesent.metric + - tcpbytereceived.metric +--- +################################ +# Istio configMap cluster-wide +################################ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio + namespace: {{ istio_namespace }} +data: + mesh: |- + # Uncomment the following line to enable mutual TLS between proxies + # authPolicy: MUTUAL_TLS + # + # Set the following variable to true to disable policy checks by the Mixer. + # Note that metrics will still be reported to the Mixer. + disablePolicyChecks: false + # Set enableTracing to false to disable request tracing. + enableTracing: true + # + # To disable the mixer completely (including metrics), comment out + # the following line + mixerAddress: istio-mixer.istio-system:9091 + # This is the ingress service name, update if you used a different name + ingressService: istio-ingress + egressProxyAddress: istio-egress.istio-system:80 + # + # Along with discoveryRefreshDelay, this setting determines how + # frequently should Envoy fetch and update its internal configuration + # from Istio Pilot. Lower refresh delay results in higher CPU + # utilization and potential performance loss in exchange for faster + # convergence. Tweak this value according to your setup. + rdsRefreshDelay: 1s + # + defaultConfig: + # See rdsRefreshDelay for explanation about this setting. + discoveryRefreshDelay: 1s + # + # TCP connection timeout between Envoy & the application, and between Envoys. + connectTimeout: 10s + # + ### ADVANCED SETTINGS ############# + # Where should envoy's configuration be stored in the istio-proxy container + configPath: "/etc/istio/proxy" + binaryPath: "/usr/local/bin/envoy" + # The pseudo service name used for Envoy. + serviceCluster: istio-proxy + # These settings that determine how long an old Envoy + # process should be kept alive after an occasional reload. + drainDuration: 45s + parentShutdownDuration: 1m0s + # + # Port where Envoy listens (on local host) for admin commands + # You can exec into the istio-proxy container in a pod and + # curl the admin port (curl http://localhost:15000/) to obtain + # diagnostic information from Envoy. See + # https://lyft.github.io/envoy/docs/operations/admin.html + # for more details + proxyAdminPort: 15000 + # + # Address where Istio Pilot service is running + discoveryAddress: istio-pilot.istio-system:8080 + # + # Zipkin trace collector + zipkinAddress: zipkin.istio-system:9411 + # + # Statsd metrics collector. Istio mixer exposes a UDP endpoint + # to collect and convert statsd metrics into Prometheus metrics. + statsdUdpAddress: istio-mixer.istio-system:9125 +--- +################################ +# Pilot +################################ +# Pilot CRDs +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: destinationpolicies.config.istio.io +spec: + group: config.istio.io + names: + kind: DestinationPolicy + listKind: DestinationPolicyList + plural: destinationpolicies + singular: destinationpolicy + scope: Namespaced + version: v1alpha2 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: egressrules.config.istio.io +spec: + group: config.istio.io + names: + kind: EgressRule + listKind: EgressRuleList + plural: egressrules + singular: egressrule + scope: Namespaced + version: v1alpha2 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: routerules.config.istio.io +spec: + group: config.istio.io + names: + kind: RouteRule + listKind: RouteRuleList + plural: routerules + singular: routerule + scope: Namespaced + version: v1alpha2 +--- +# Pilot service for discovery +apiVersion: v1 +kind: Service +metadata: + name: istio-pilot + namespace: {{ istio_namespace }} + labels: + istio: pilot +spec: + ports: + - port: 8080 + name: http-discovery + - port: 443 + name: http-admission-webhook + selector: + istio: pilot +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-pilot-service-account + namespace: {{ istio_namespace }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: istio-pilot + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" +spec: + replicas: 1 + template: + metadata: + labels: + istio: pilot + spec: + serviceAccountName: istio-pilot-service-account + containers: + - name: discovery + image: {{ istio_pilot_image_repo }}:{{ istio_pilot_image_tag }} + imagePullPolicy: IfNotPresent + args: ["discovery", "-v", "2", "--admission-service", "istio-pilot-external"] + ports: + - containerPort: 8080 + - containerPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/istio/config + volumes: + - name: config-volume + configMap: + name: istio +--- +################################ +# Istio ingress +################################ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingress + namespace: {{ istio_namespace }} + labels: + istio: ingress +spec: + type: LoadBalancer + ports: + - port: 80 +# nodePort: 32000 + name: http + - port: 443 + name: https + selector: + istio: ingress +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-ingress-service-account + namespace: {{ istio_namespace }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: istio-ingress + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" +spec: + replicas: 1 + template: + metadata: + labels: + istio: ingress + spec: + serviceAccountName: istio-ingress-service-account + containers: + - name: istio-ingress + image: {{ istio_proxy_debug_image_repo }}:{{ istio_proxy_debug_image_tag }} + args: + - proxy + - ingress + - -v + - "2" + - --discoveryAddress + - istio-pilot:8080 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 + - containerPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + - name: ingress-certs + mountPath: /etc/istio/ingress-certs + readOnly: true + volumes: + - name: istio-certs + secret: + secretName: istio.default + optional: true + - name: ingress-certs + secret: + secretName: istio-ingress-certs + optional: true +--- +################################ +# Istio egress +################################ +apiVersion: v1 +kind: Service +metadata: + name: istio-egress + namespace: {{ istio_namespace }} +spec: + ports: + - port: 80 + selector: + istio: egress +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-egress-service-account + namespace: {{ istio_namespace }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: istio-egress + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" +spec: + replicas: 1 + template: + metadata: + labels: + istio: egress + spec: + serviceAccountName: istio-egress-service-account + containers: + - name: proxy + image: {{ istio_proxy_debug_image_repo }}:{{ istio_proxy_debug_image_tag }} + imagePullPolicy: IfNotPresent + args: + - proxy + - egress + - -v + - "2" + - --discoveryAddress + - istio-pilot:8080 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + volumes: + - name: istio-certs + secret: + secretName: istio.default + optional: true +--- +################################ +# Istio-CA cluster-wide +################################ +# Service account CA +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-ca-service-account + namespace: {{ istio_namespace }} +--- +# Istio CA watching all namespaces +apiVersion: v1 +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: istio-ca + namespace: {{ istio_namespace }} + annotations: + sidecar.istio.io/inject: "false" +spec: + replicas: 1 + template: + metadata: + labels: + istio: istio-ca + spec: + serviceAccountName: istio-ca-service-account + containers: + - name: istio-ca + image: {{ istio_ca_image_repo }}:{{ istio_ca_image_tag }} + imagePullPolicy: IfNotPresent +--- + diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 893f79424..f8866c42c 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -34,3 +34,8 @@ dependencies: tags: - apps - helm + - role: kubernetes-apps/istio + when: istio_enabled + tags: + - apps + - istio diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 076b8477c..d86088623 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -44,6 +44,12 @@ kube_apiserver_admission_control: - ServiceAccount - DefaultStorageClass - ResourceQuota + - Initializers + - GenericAdmissionWebhook + +# extra runtime config +kube_api_runtime_config: + - admissionregistration.k8s.io/v1alpha1 ## Enable/Disable Kube API Server Authentication Methods kube_basic_auth: true diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index cad57b5f2..e3fa8fc6f 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -45,6 +45,8 @@ spec: {% endif %} - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --proxy-client-cert-file={{ kube_cert_dir }}/apiserver.pem + - --proxy-client-key-file={{ kube_cert_dir }}/apiserver-key.pem {% if kube_token_auth|default(true) %} - --token-auth-file={{ kube_token_dir }}/known_tokens.csv {% endif %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index f20d6585d..e02e20ad4 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -153,7 +153,7 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes or kubeadm_enabled }}" ## List of key=value pairs that describe feature gates for ## the k8s cluster. -kube_feature_gates: [] +kube_feature_gates: ['Initializers=true'] # Vault data dirs. vault_base_dir: /etc/vault