diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f8cbaad5..cc86762ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,8 +49,10 @@ before_script: ANSIBLE_KEEP_REMOTE_FILES: "1" BOOTSTRAP_OS: none LOG_LEVEL: "-vv" + ETCD_DEPLOYMENT: "docker" + KUBELET_DEPLOYMENT: "docker" MAGIC: "ci check this" - + .gce: &gce <<: *job <<: *docker_service @@ -103,6 +105,8 @@ before_script: -e download_localhost=true -e deploy_netchecker=true -e local_release_dir=${PWD}/downloads + -e etcd_deployment_type=${ETCD_DEPLOYMENT} + -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} cluster.yml @@ -203,6 +207,15 @@ before_script: CLUSTER_MODE: ha BOOTSTRAP_OS: coreos +.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables +# stage: deploy-gce-part1 + KUBE_NETWORK_PLUGIN: flannel + CLOUD_IMAGE: ubuntu-1604-xenial + CLOUD_REGION: us-central1-b + CLUSTER_MODE: separated + ETCD_DEPLOYMENT: rkt + KUBELET_DEPLOYMENT: rkt + # Builds for PRs only (premoderated by unit-tests step) and triggers (auto) coreos-calico-sep: stage: deploy-gce-part1 @@ -406,6 +419,17 @@ coreos-alpha-weave-ha: except: ['triggers'] only: ['master', /^pr-.*$/] +ubuntu-rkt-sep: + stage: deploy-gce-part1 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_rkt_sep_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + # Premoderated with manual actions syntax-check: <<: *job diff --git a/cluster.yml b/cluster.yml index e10c0d2c9..553261063 100644 --- a/cluster.yml +++ b/cluster.yml @@ -28,6 +28,7 @@ roles: - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker } + - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" } - hosts: etcd:!k8s-cluster any_errors_fatal: true diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 839da100a..04e20922e 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -197,3 +197,7 @@ k8s_image_pull_policy: IfNotPresent # default packages to install within the cluster kpm_packages: [] # - name: kube-system/grafana + +rkt_version: 1.21.0 +etcd_deployment_type: docker +kubelet_deployment_type: docker diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index df9bd75c7..77933bfb6 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -115,13 +115,13 @@ downloads: version: "{{etcd_version}}" dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz" sha256: >- - {%- if etcd_deployment_type == 'docker' -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%} + {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%} source_url: "{{ etcd_download_url }}" url: "{{ etcd_download_url }}" unarchive: true owner: "etcd" mode: "0755" - container: "{{ etcd_deployment_type == 'docker' }}" + container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}" repo: "{{ etcd_image_repo }}" tag: "{{ etcd_image_tag }}" hyperkube: diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install_docker.yml similarity index 74% rename from roles/etcd/tasks/install.yml rename to roles/etcd/tasks/install_docker.yml index 0ed3f4154..f87caeb4c 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install_docker.yml @@ -1,17 +1,6 @@ --- -- name: Install | Copy etcd binary from downloaddir - command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd" - when: etcd_deployment_type == "host" - register: etcd_copy - changed_when: false - -- name: Install | Copy etcdctl binary from downloaddir - command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl" - when: etcd_deployment_type == "host" - changed_when: false - #Plan A: no docker-py deps -- name: Install | Copy etcdctl binary from container +- name: Install | Copy etcdctl binary from docker container command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml new file mode 100644 index 000000000..6f588a2f0 --- /dev/null +++ b/roles/etcd/tasks/install_host.yml @@ -0,0 +1,9 @@ +--- +- name: Install | Copy etcd binary from downloaddir + command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd" + register: etcd_copy + changed_when: false + +- name: Install | Copy etcdctl binary from downloaddir + command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl" + changed_when: false diff --git a/roles/etcd/tasks/install_rkt.yml b/roles/etcd/tasks/install_rkt.yml new file mode 100644 index 000000000..85f875383 --- /dev/null +++ b/roles/etcd/tasks/install_rkt.yml @@ -0,0 +1,26 @@ +--- +- name: Trust etcd container + command: >- + /usr/bin/rkt trust + --skip-fingerprint-review + --root + https://quay.io/aci-signing-key + register: etcd_rkt_trust_result + until: etcd_rkt_trust_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + +- name: Install | Copy etcdctl binary from rkt container + command: >- + /usr/bin/rkt run + --volume=bin-dir,kind=host,source={{ bin_dir}},readOnly=false + --mount=volume=bin-dir,target=/host/bin + {{ etcd_image_repo }}:{{ etcd_image_tag }} + --name=etcdctl-binarycopy + --exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl + register: etcd_task_result + until: etcd_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index d6320619c..cdd634517 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -5,7 +5,7 @@ tags: [etcd-secrets, facts] - include: gen_certs.yml tags: etcd-secrets -- include: install.yml +- include: "install_{{ etcd_deployment_type }}.yml" when: is_etcd_master tags: upgrade - include: set_cluster_health.yml diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2 new file mode 100644 index 000000000..eb26bc473 --- /dev/null +++ b/roles/etcd/templates/etcd-rkt.service.j2 @@ -0,0 +1,29 @@ +[Unit] +Description=etcd rkt wrapper +Documentation=https://github.com/coreos/etcd +Wants=network.target + +[Service] +Restart=on-failure +RestartSec=10s +TimeoutStartSec=0 +LimitNOFILE=40000 + +ExecStart=/usr/bin/rkt run \ +--uuid-file-save=/var/run/etcd.uuid \ +--volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ +--mount=volume=etc-ssl-certs,target=/etc/ssl/certs \ +--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \ +--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \ +--volume=var-lib-etcd,kind=host,source=/var/lib/etcd,readOnly=false \ +--mount=volume=var-lib-etcd,target=/var/lib/etcd \ +--set-env-file=/etc/etcd.env \ +--stage1-from-dir=stage1-fly.aci \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +--name={{ etcd_member_name | default("etcd") }} + +ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid +ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/etcd.uuid + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index c9061725a..bfe4a8cc8 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -1,8 +1,31 @@ --- +- name: Trust kubelet container + command: >- + /usr/bin/rkt trust + --skip-fingerprint-review + --root + {{ item }} + register: kubelet_rkt_trust_result + until: kubelet_rkt_trust_result.rc == 0 + with_items: + - "https://quay.io/aci-signing-key" + - "https://coreos.com/dist/pubkeys/aci-pubkeys.gpg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + when: kubelet_deployment_type == "rkt" + +- name: create kubelet working directory + file: + state: directory + path: /var/lib/kubelet + when: kubelet_deployment_type == "rkt" + - name: install | Write kubelet systemd init file - template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes + template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes" notify: restart kubelet - name: install | Install kubelet launch script template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes notify: restart kubelet + when: kubelet_deployment_type == "docker" diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.docker.service.j2 similarity index 100% rename from roles/kubernetes/node/templates/kubelet.service.j2 rename to roles/kubernetes/node/templates/kubelet.docker.service.j2 diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 new file mode 100644 index 000000000..89a9f01dc --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -0,0 +1,58 @@ +[Unit] +Description=Kubernetes Kubelet Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} +After=calico-node.service +Wants=network.target calico-node.service +{% else %} +Wants=network.target +{% endif %} + +[Service] +Restart=on-failure +RestartSec=10s +TimeoutStartSec=0 +LimitNOFILE=40000 + +ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid +ExecStartPre=-/bin/mkdir -p /var/lib/kubelet + +EnvironmentFile={{kube_config_dir}}/kubelet.env +# stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts +ExecStart=/usr/bin/rkt run \ + --volume var-log,kind=host,source=/var/log \ + --volume dns,kind=host,source=/etc/resolv.conf \ + --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ + --volume run,kind=host,source=/run,readOnly=false \ + --mount volume=var-log,target=/var/log \ + --mount volume=dns,target=/etc/resolv.conf \ + --mount volume=etc-kubernetes,target={{ kube_config_dir }} \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --mount volume=run,target=/run \ + --stage1-from-dir=stage1-fly.aci \ + {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \ + --uuid-file-save=/var/run/kubelet.uuid \ + --debug --exec=/kubelet -- \ + $KUBE_LOGTOSTDERR \ + $KUBE_LOG_LEVEL \ + $KUBELET_API_SERVER \ + $KUBELET_ADDRESS \ + $KUBELET_PORT \ + $KUBELET_HOSTNAME \ + $KUBE_ALLOW_PRIV \ + $KUBELET_ARGS \ + $DOCKER_SOCKET \ + $KUBELET_REGISTER_NODE \ + $KUBELET_NETWORK_PLUGIN + +ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet.uuid + +[Install] +WantedBy=multi-user.target diff --git a/roles/rkt/defaults/main.yml b/roles/rkt/defaults/main.yml new file mode 100644 index 000000000..6794429a5 --- /dev/null +++ b/roles/rkt/defaults/main.yml @@ -0,0 +1,6 @@ +--- + +rkt_version: 1.12.0 +rkt_pkg_version: "{{ rkt_version }}-1" +rkt_download_src: https://github.com/coreos/rkt +rkt_download_url: "{{ rkt_download_src }}/releases/download/v{{ rkt_version }}" diff --git a/roles/rkt/tasks/install.yml b/roles/rkt/tasks/install.yml new file mode 100644 index 000000000..41823bf54 --- /dev/null +++ b/roles/rkt/tasks/install.yml @@ -0,0 +1,35 @@ +--- +- name: gather os specific variables for rkt + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + tags: facts + +- name: install rkt pkg on ubuntu + apt: + deb: "{{ rkt_download_url }}/{{ rkt_pkg_name }}" + state: present + register: rkt_task_result + until: rkt_task_result|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: ansible_os_family == "Debian" + +- name: install rkt pkg on centos + yum: + pkg: "{{ rkt_download_url }}/{{ rkt_pkg_name }}" + state: present + register: rkt_task_result + until: rkt_task_result|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: ansible_os_family == "RedHat" diff --git a/roles/rkt/tasks/main.yml b/roles/rkt/tasks/main.yml new file mode 100644 index 000000000..9d87123bb --- /dev/null +++ b/roles/rkt/tasks/main.yml @@ -0,0 +1,4 @@ +--- + +- name: Install rkt + include: install.yml diff --git a/roles/rkt/vars/debian.yml b/roles/rkt/vars/debian.yml new file mode 100644 index 000000000..9cfffe5c9 --- /dev/null +++ b/roles/rkt/vars/debian.yml @@ -0,0 +1,2 @@ +--- +rkt_pkg_name: "rkt_{{ rkt_pkg_version }}_amd64.deb" diff --git a/roles/rkt/vars/fedora.yml b/roles/rkt/vars/fedora.yml new file mode 100644 index 000000000..13149e8fb --- /dev/null +++ b/roles/rkt/vars/fedora.yml @@ -0,0 +1,2 @@ +--- +rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm" diff --git a/roles/rkt/vars/redhat.yml b/roles/rkt/vars/redhat.yml new file mode 100644 index 000000000..13149e8fb --- /dev/null +++ b/roles/rkt/vars/redhat.yml @@ -0,0 +1,2 @@ +--- +rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"