From c6e5be91e9921851b4b5764fcb71755765111871 Mon Sep 17 00:00:00 2001 From: Hans Feldt <2808287+hafe@users.noreply.github.com> Date: Sat, 1 Aug 2020 09:33:48 +0200 Subject: [PATCH] crio: align template crio.conf with upstream (#6432) * log level by default increased to 'info' * cgroup manager by default set to 'systemd' * stream port (used by kubelet) bound to 127.0.0.1 for security reasons * metrics can be enabled and port specified --- .../container-engine/cri-o/defaults/main.yml | 15 +- .../cri-o/templates/crio.conf.j2 | 198 +++++++++++++----- 2 files changed, 158 insertions(+), 55 deletions(-) diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index ce17424c2..03325a527 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -1,10 +1,17 @@ --- -crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('cgroupfs') }}" - -crio_seccomp_profile: "" -crio_runc_path: "/usr/bin/runc" +crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}" crio_conmon: "/usr/bin/conmon" +crio_enable_metrics: false +crio_log_level: "info" +crio_metrics_port: "9090" +crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}" +crio_runc_path: "/usr/bin/runc" +crio_seccomp_profile: "" +crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}" +crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}" +crio_storage_driver: "overlay2" +crio_stream_port: "10010" crio_required_version: "{{ kube_version | regex_replace('^v(?P\\d+).(?P\\d+).(?P\\d+)$', '\\g.\\g') }}" diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index a27d746d4..c5e2cf89a 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -1,10 +1,14 @@ - # The CRI-O configuration file specifies all of the available configuration # options and command-line flags for the crio(8) OCI Kubernetes Container Runtime # daemon, but in a TOML format that can be more easily modified and versioned. # # Please refer to crio.conf(5) for details of all configuration options. +# CRI-O supports partial configuration reload during runtime, which can be +# done by sending SIGHUP to the running process. Currently supported options +# are explicitly mentioned with: 'This option supports live configuration +# reload'. + # CRI-O reads its storage defaults from the containers-storage.conf(5) file # located at /etc/containers/storage.conf. Modify this storage configuration if # you want to change the system's defaults. If you want to modify storage just @@ -20,19 +24,26 @@ # Storage driver used to manage the storage of images and containers. Please # refer to containers-storage.conf(5) to see all available storage drivers. -storage_driver = "overlay2" +storage_driver = "{{ crio_storage_driver }}" # List to pass options to the storage driver. Please refer to # containers-storage.conf(5) to see all available storage options. #storage_option = [ #] -# If set to false, in-memory locking will be used instead of file-based locking. -file_locking = true +# The default log directory where all logs will go unless directly specified by +# the kubelet. The log directory specified must be an absolute directory. +log_dir = "/var/log/crio/pods" -# Path to the lock file. -file_locking_path = "/run/crio.lock" +# Location for CRI-O to lay down the temporary version file. +# It is used to check if crio wipe should wipe containers, which should +# always happen on a node reboot +version_file = "/var/run/crio/version" +# Location for CRI-O to lay down the persistent version file. +# It is used to check if crio wipe should wipe images, which should +# only happen when CRI-O has been upgraded +version_file_persist = "/var/lib/crio/version" # The crio.api table contains settings for the kubelet/gRPC interface. [crio.api] @@ -41,10 +52,11 @@ file_locking_path = "/run/crio.lock" listen = "/var/run/crio/crio.sock" # IP address on which the stream server will listen. -stream_address = "" +stream_address = "127.0.0.1" -# The port on which the stream server will listen. -stream_port = "10010" +# The port on which the stream server will listen. If the port is set to "0", then +# CRI-O will allocate a random free port number. +stream_port = "{{ crio_stream_port }}" # Enable encrypted TLS transport of the stream server. stream_enable_tls = false @@ -55,11 +67,11 @@ stream_enable_tls = false stream_tls_cert = "" # Path to the key file used to serve the encrypted stream. This file can -# change, and CRI-O will automatically pick up the changes within 5 minutes. +# change and CRI-O will automatically pick up the changes within 5 minutes. stream_tls_key = "" # Path to the x509 CA(s) file used to verify and authenticate client -# communication with the encrypted stream. This file can change, and CRI-O will +# communication with the encrypted stream. This file can change and CRI-O will # automatically pick up the changes within 5 minutes. stream_tls_ca = "" @@ -78,7 +90,6 @@ grpc_max_recv_msg_size = 16777216 # "nofile=1024:2048" # If nothing is set here, settings will be inherited from the CRI-O daemon #default_ulimits = [ -# "nofile=65536:65536", #] # default_runtime is the _name_ of the OCI runtime to be used as the default. @@ -88,28 +99,46 @@ default_runtime = "runc" # If true, the runtime will not use pivot_root, but instead use MS_MOVE. no_pivot = false +# decryption_keys_path is the path where the keys required for +# image decryption are stored. This option supports live configuration reload. +decryption_keys_path = "/etc/crio/keys/" + # Path to the conmon binary, used for monitoring the OCI runtime. +# Will be searched for using $PATH if empty. conmon = "{{ crio_conmon }}" +# Cgroup setting for conmon +conmon_cgroup = "system.slice" + # Environment variable list for the conmon process, used for passing necessary # environment variables to conmon or the runtime. conmon_env = [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", ] +# Additional environment variables to set for all the +# containers. These are overridden if set in the +# container image spec or in the container runtime configuration. +default_env = [ +] + # If true, SELinux will be used for pod separation on the host. -selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }} +selinux = {{ crio_selinux }} # Path to the seccomp.json profile which is used as the default seccomp profile -# for the runtime. -seccomp_profile = "{{crio_seccomp_profile}}" +# for the runtime. If not specified, then the internal default seccomp profile +# will be used. This option supports live configuration reload. +seccomp_profile = "{{ crio_seccomp_profile }}" # Used to change the name of the default AppArmor profile of CRI-O. The default -# profile name is "crio-default-" followed by the version string of CRI-O. +# profile name is "crio-default". This profile only takes effect if the user +# does not specify a profile via the Kubernetes Pod's metadata annotation. If +# the profile is set to "unconfined", then this equals to disabling AppArmor. +# This option supports live configuration reload. # apparmor_profile = "crio-default" # Cgroup management implementation used for the runtime. -cgroup_manager = "{{crio_cgroup_manager}}" +cgroup_manager = "{{ crio_cgroup_manager }}" # List of default capabilities for containers. If it is empty or commented out, # only the capabilities defined in the containers json file by the user/kube @@ -140,16 +169,15 @@ default_sysctls = [ additional_devices = [ ] -# Path to OCI hooks directories for automatically executed hooks. +# Path to OCI hooks directories for automatically executed hooks. If one of the +# directories does not exist, then CRI-O will automatically skip them. hooks_dir = [ + "/usr/share/containers/oci/hooks.d", ] # List of default mounts for each container. **Deprecated:** this option will # be removed in future versions in favor of default_mounts_file. default_mounts = [ -{% if ansible_os_family == "RedHat" %} - "/usr/share/rhel/secrets:/run/secrets", -{% endif %} ] # Path to the file specifying the defaults mounts for each container. The @@ -185,12 +213,20 @@ container_exits_dir = "/var/run/crio/exits" # Path to directory for container attach sockets. container_attach_socket_dir = "/var/run/crio" +# The prefix to use for the source of the bind mounts. +bind_mount_prefix = "" + # If set to true, all containers will run in read-only mode. read_only = false # Changes the verbosity of the logs based on the level it is set to. Options -# are fatal, panic, error, warn, info, and debug. -log_level = "error" +# are fatal, panic, error, warn, info, debug and trace. This option supports +# live configuration reload. +log_level = "{{ crio_log_level }}" + +# Filter the log messages by the provided regular expression. +# This option supports live configuration reload. +log_filter = "" # The UID mappings for the user namespace of each container. A range is # specified in the form containerUID:HostUID:Size. Multiple ranges must be @@ -203,20 +239,65 @@ uid_mappings = "" gid_mappings = "" # The minimal amount of time in seconds to wait before issuing a timeout -# regarding the proper termination of the container. -ctr_stop_timeout = 0 +# regarding the proper termination of the container. The lowest possible +# value is 30s, whereas lower values are not considered by CRI-O. +ctr_stop_timeout = 30 + +# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below. +# manage_network_ns_lifecycle = false + +# manage_ns_lifecycle determines whether we pin and remove namespaces +# and manage their lifecycle +manage_ns_lifecycle = false + +# The directory where the state of the managed namespaces gets tracked. +# Only used when manage_ns_lifecycle is true. +namespaces_dir = "/var/run" - # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. - # The runtime to use is picked based on the runtime_handler provided by the CRI. - # If no runtime_handler is provided, the runtime will be picked based on the level - # of trust of the workload. +# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle +pinns_path = "" + +# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. +# The runtime to use is picked based on the runtime_handler provided by the CRI. +# If no runtime_handler is provided, the runtime will be picked based on the level +# of trust of the workload. Each entry in the table should follow the format: +# +#[crio.runtime.runtimes.runtime-handler] +# runtime_path = "/path/to/the/executable" +# runtime_type = "oci" +# runtime_root = "/path/to/the/root" +# +# Where: +# - runtime-handler: name used to identify the runtime +# - runtime_path (optional, string): absolute path to the runtime executable in +# the host filesystem. If omitted, the runtime-handler identifier should match +# the runtime executable name, and the runtime executable should be placed +# in $PATH. +# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If +# omitted, an "oci" runtime is assumed. +# - runtime_root (optional, string): root directory for storage of containers +# state. - [crio.runtime.runtimes.runc] - runtime_path = "{{ crio_runc_path }}" - runtime_type = "oci" +[crio.runtime.runtimes.runc] +runtime_path = "{{ crio_runc_path }}" +runtime_type = "oci" +runtime_root = "/run/runc" +# Kata Containers is an OCI runtime, where containers are run inside lightweight +# VMs. Kata provides additional isolation towards the host, minimizing the host attack +# surface and mitigating the consequences of containers breakout. + +# Kata Containers with the default configured VMM +#[crio.runtime.runtimes.kata-runtime] + +# Kata Containers with the QEMU VMM +#[crio.runtime.runtimes.kata-qemu] + +# Kata Containers with the Firecracker VMM +#[crio.runtime.runtimes.kata-fc] + # The crio.image table contains settings pertaining to the management of OCI images. # # CRI-O reads its configured registries defaults from the system wide @@ -229,14 +310,23 @@ ctr_stop_timeout = 0 # Default transport for pulling images from a remote container storage. default_transport = "docker://" +# The path to a file containing credentials necessary for pulling images from +# secure registries. The file is similar to that of /var/lib/kubelet/config.json +global_auth_file = "" + # The image used to instantiate infra containers. -pause_image = "docker://{{kube_image_repo}}/pause:3.1" +# This option supports live configuration reload. +pause_image = "{{ crio_pause_image }}" -# If not empty, the path to a docker/config.json-like file containing credentials -# necessary for pulling the image specified by pause_image above. +# The path to a file containing credentials specific for pulling the pause_image from +# above. The file is similar to that of /var/lib/kubelet/config.json +# This option supports live configuration reload. pause_image_auth_file = "" # The command to run to have a container stay in the paused state. +# When explicitly set to "", it will fallback to the entrypoint and command +# specified in the pause image. When commented out, it will fallback to the +# default: "/pause". This option supports live configuration reload. pause_command = "/pause" # Path to the file which decides what sort of policy we use when deciding @@ -244,11 +334,12 @@ pause_command = "/pause" # this option be used, as the default behavior of using the system-wide default # policy (i.e., /etc/containers/policy.json) is most often preferred. Please # refer to containers-policy.json(5) for more details. -{% if ansible_os_family == "ClearLinux" %} -signature_policy = "/usr/share/defaults/crio/policy.json" -{% else %} -signature_policy = "" -{% endif %} +signature_policy = "{{ crio_signature_policy }}" + +# List of registries to skip TLS verification for pulling images. Please +# consider configuring the registries via /etc/containers/registries.conf before +# changing them here. +#insecure_registries = "[]" # Controls how image volumes are handled. The valid values are mkdir, bind and # ignore; the latter will ignore volumes entirely. @@ -259,27 +350,32 @@ image_volumes = "mkdir" # compatibility reasons. Depending on your workload and usecase you may add more # registries (e.g., "quay.io", "registry.fedoraproject.org", # "registry.opensuse.org", etc.). -registries = [ - "docker.io" -] -# "registry.fedoraproject.org", -# "quay.io", -# "registry.centos.org", -#] +#registries = [ +# ] -insecure_registries = [ - "{{ kube_service_addresses }}" -] # The crio.network table containers settings pertaining to the management of # CNI plugins. [crio.network] +# The default CNI network name to be selected. If not set or "", then +# CRI-O will pick-up the first one found in network_dir. +# cni_default_network = "" + # Path to the directory where CNI configuration files are located. network_dir = "/etc/cni/net.d/" # Paths to directories where CNI plugin binaries are located. plugin_dirs = [ + "/opt/cni/bin", "/usr/libexec/cni", - "/opt/cni/bin/", ] + +# A necessary configuration for Prometheus based metrics retrieval +[crio.metrics] + +# Globally enable or disable metrics support. +enable_metrics = {{ crio_enable_metrics | bool | lower }} + +# The port on which the metrics server will listen. +metrics_port = {{ crio_metrics_port }}