You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

344 lines
12 KiB

  1. #!/usr/bin/env python3
  2. # Licensed under the Apache License, Version 2.0 (the "License");
  3. # you may not use this file except in compliance with the License.
  4. # You may obtain a copy of the License at
  5. #
  6. # http://www.apache.org/licenses/LICENSE-2.0
  7. #
  8. # Unless required by applicable law or agreed to in writing, software
  9. # distributed under the License is distributed on an "AS IS" BASIS,
  10. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  11. # implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. #
  15. # Usage: inventory.py ip1 [ip2 ...]
  16. # Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
  17. #
  18. # Advanced usage:
  19. # Add another host after initial creation: inventory.py 10.10.1.5
  20. # Delete a host: inventory.py -10.10.1.3
  21. # Delete a host by id: inventory.py -node1
  22. #
  23. # Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
  24. # YAML file should be in the following format:
  25. # group1:
  26. # host1:
  27. # ip: X.X.X.X
  28. # var: val
  29. # group2:
  30. # host2:
  31. # ip: X.X.X.X
  32. from collections import OrderedDict
  33. try:
  34. import configparser
  35. except ImportError:
  36. import ConfigParser as configparser
  37. import os
  38. import re
  39. import sys
  40. ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
  41. 'calico-rr', 'vault']
  42. PROTECTED_NAMES = ROLES
  43. AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
  44. _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
  45. '0': False, 'no': False, 'false': False, 'off': False}
  46. def get_var_as_bool(name, default):
  47. value = os.environ.get(name, '')
  48. return _boolean_states.get(value.lower(), default)
  49. # Configurable as shell vars start
  50. CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.ini")
  51. # Reconfigures cluster distribution at scale
  52. SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
  53. MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
  54. DEBUG = get_var_as_bool("DEBUG", True)
  55. HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
  56. # Configurable as shell vars end
  57. class KubesprayInventory(object):
  58. def __init__(self, changed_hosts=None, config_file=None):
  59. self.config = configparser.ConfigParser(allow_no_value=True,
  60. delimiters=('\t', ' '))
  61. self.config_file = config_file
  62. if self.config_file:
  63. self.config.read(self.config_file)
  64. if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
  65. self.parse_command(changed_hosts[0], changed_hosts[1:])
  66. sys.exit(0)
  67. self.ensure_required_groups(ROLES)
  68. if changed_hosts:
  69. self.hosts = self.build_hostnames(changed_hosts)
  70. self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
  71. self.set_all(self.hosts)
  72. self.set_k8s_cluster()
  73. self.set_etcd(list(self.hosts.keys())[:3])
  74. if len(self.hosts) >= SCALE_THRESHOLD:
  75. self.set_kube_master(list(self.hosts.keys())[3:5])
  76. else:
  77. self.set_kube_master(list(self.hosts.keys())[:2])
  78. self.set_kube_node(self.hosts.keys())
  79. if len(self.hosts) >= SCALE_THRESHOLD:
  80. self.set_calico_rr(list(self.hosts.keys())[:3])
  81. else: # Show help if no options
  82. self.show_help()
  83. sys.exit(0)
  84. self.write_config(self.config_file)
  85. def write_config(self, config_file):
  86. if config_file:
  87. with open(config_file, 'w') as f:
  88. self.config.write(f)
  89. else:
  90. print("WARNING: Unable to save config. Make sure you set "
  91. "CONFIG_FILE env var.")
  92. def debug(self, msg):
  93. if DEBUG:
  94. print("DEBUG: {0}".format(msg))
  95. def get_ip_from_opts(self, optstring):
  96. opts = optstring.split(' ')
  97. for opt in opts:
  98. if '=' not in opt:
  99. continue
  100. k, v = opt.split('=')
  101. if k == "ip":
  102. return v
  103. raise ValueError("IP parameter not found in options")
  104. def ensure_required_groups(self, groups):
  105. for group in groups:
  106. try:
  107. self.debug("Adding group {0}".format(group))
  108. self.config.add_section(group)
  109. except configparser.DuplicateSectionError:
  110. pass
  111. def get_host_id(self, host):
  112. '''Returns integer host ID (without padding) from a given hostname.'''
  113. try:
  114. short_hostname = host.split('.')[0]
  115. return int(re.findall("\d+$", short_hostname)[-1])
  116. except IndexError:
  117. raise ValueError("Host name must end in an integer")
  118. def build_hostnames(self, changed_hosts):
  119. existing_hosts = OrderedDict()
  120. highest_host_id = 0
  121. try:
  122. for host, opts in self.config.items('all'):
  123. existing_hosts[host] = opts
  124. host_id = self.get_host_id(host)
  125. if host_id > highest_host_id:
  126. highest_host_id = host_id
  127. except configparser.NoSectionError:
  128. pass
  129. # FIXME(mattymo): Fix condition where delete then add reuses highest id
  130. next_host_id = highest_host_id + 1
  131. all_hosts = existing_hosts.copy()
  132. for host in changed_hosts:
  133. if host[0] == "-":
  134. realhost = host[1:]
  135. if self.exists_hostname(all_hosts, realhost):
  136. self.debug("Marked {0} for deletion.".format(realhost))
  137. all_hosts.pop(realhost)
  138. elif self.exists_ip(all_hosts, realhost):
  139. self.debug("Marked {0} for deletion.".format(realhost))
  140. self.delete_host_by_ip(all_hosts, realhost)
  141. elif host[0].isdigit():
  142. if self.exists_hostname(all_hosts, host):
  143. self.debug("Skipping existing host {0}.".format(host))
  144. continue
  145. elif self.exists_ip(all_hosts, host):
  146. self.debug("Skipping existing host {0}.".format(host))
  147. continue
  148. next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
  149. next_host_id += 1
  150. all_hosts[next_host] = "ansible_host={0} ip={1}".format(
  151. host, host)
  152. elif host[0].isalpha():
  153. raise Exception("Adding hosts by hostname is not supported.")
  154. return all_hosts
  155. def exists_hostname(self, existing_hosts, hostname):
  156. return hostname in existing_hosts.keys()
  157. def exists_ip(self, existing_hosts, ip):
  158. for host_opts in existing_hosts.values():
  159. if ip == self.get_ip_from_opts(host_opts):
  160. return True
  161. return False
  162. def delete_host_by_ip(self, existing_hosts, ip):
  163. for hostname, host_opts in existing_hosts.items():
  164. if ip == self.get_ip_from_opts(host_opts):
  165. del existing_hosts[hostname]
  166. return
  167. raise ValueError("Unable to find host by IP: {0}".format(ip))
  168. def purge_invalid_hosts(self, hostnames, protected_names=[]):
  169. for role in self.config.sections():
  170. for host, _ in self.config.items(role):
  171. if host not in hostnames and host not in protected_names:
  172. self.debug("Host {0} removed from role {1}".format(host,
  173. role))
  174. self.config.remove_option(role, host)
  175. def add_host_to_group(self, group, host, opts=""):
  176. self.debug("adding host {0} to group {1}".format(host, group))
  177. self.config.set(group, host, opts)
  178. def set_kube_master(self, hosts):
  179. for host in hosts:
  180. self.add_host_to_group('kube-master', host)
  181. def set_all(self, hosts):
  182. for host, opts in hosts.items():
  183. self.add_host_to_group('all', host, opts)
  184. def set_k8s_cluster(self):
  185. self.add_host_to_group('k8s-cluster:children', 'kube-node')
  186. self.add_host_to_group('k8s-cluster:children', 'kube-master')
  187. def set_calico_rr(self, hosts):
  188. for host in hosts:
  189. if host in self.config.items('kube-master'):
  190. self.debug("Not adding {0} to calico-rr group because it "
  191. "conflicts with kube-master group".format(host))
  192. continue
  193. if host in self.config.items('kube-node'):
  194. self.debug("Not adding {0} to calico-rr group because it "
  195. "conflicts with kube-node group".format(host))
  196. continue
  197. self.add_host_to_group('calico-rr', host)
  198. def set_kube_node(self, hosts):
  199. for host in hosts:
  200. if len(self.config['all']) >= SCALE_THRESHOLD:
  201. if self.config.has_option('etcd', host):
  202. self.debug("Not adding {0} to kube-node group because of "
  203. "scale deployment and host is in etcd "
  204. "group.".format(host))
  205. continue
  206. if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD:
  207. if self.config.has_option('kube-master', host):
  208. self.debug("Not adding {0} to kube-node group because of "
  209. "scale deployment and host is in kube-master "
  210. "group.".format(host))
  211. continue
  212. self.add_host_to_group('kube-node', host)
  213. def set_etcd(self, hosts):
  214. for host in hosts:
  215. self.add_host_to_group('etcd', host)
  216. self.add_host_to_group('vault', host)
  217. def load_file(self, files=None):
  218. '''Directly loads JSON, or YAML file to inventory.'''
  219. if not files:
  220. raise Exception("No input file specified.")
  221. import json
  222. import yaml
  223. for filename in list(files):
  224. # Try JSON, then YAML
  225. try:
  226. with open(filename, 'r') as f:
  227. data = json.load(f)
  228. except ValueError:
  229. try:
  230. with open(filename, 'r') as f:
  231. data = yaml.load(f)
  232. print("yaml")
  233. except ValueError:
  234. raise Exception("Cannot read %s as JSON, YAML, or CSV",
  235. filename)
  236. self.ensure_required_groups(ROLES)
  237. self.set_k8s_cluster()
  238. for group, hosts in data.items():
  239. self.ensure_required_groups([group])
  240. for host, opts in hosts.items():
  241. optstring = "ansible_host={0} ip={0}".format(opts['ip'])
  242. for key, val in opts.items():
  243. if key == "ip":
  244. continue
  245. optstring += " {0}={1}".format(key, val)
  246. self.add_host_to_group('all', host, optstring)
  247. self.add_host_to_group(group, host)
  248. self.write_config(self.config_file)
  249. def parse_command(self, command, args=None):
  250. if command == 'help':
  251. self.show_help()
  252. elif command == 'print_cfg':
  253. self.print_config()
  254. elif command == 'print_ips':
  255. self.print_ips()
  256. elif command == 'load':
  257. self.load_file(args)
  258. else:
  259. raise Exception("Invalid command specified.")
  260. def show_help(self):
  261. help_text = '''Usage: inventory.py ip1 [ip2 ...]
  262. Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
  263. Available commands:
  264. help - Display this message
  265. print_cfg - Write inventory file to stdout
  266. print_ips - Write a space-delimited list of IPs from "all" group
  267. Advanced usage:
  268. Add another host after initial creation: inventory.py 10.10.1.5
  269. Delete a host: inventory.py -10.10.1.3
  270. Delete a host by id: inventory.py -node1
  271. Configurable env vars:
  272. DEBUG Enable debug printing. Default: True
  273. CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.ini
  274. HOST_PREFIX Host prefix for generated hosts. Default: node
  275. SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50
  276. MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
  277. '''
  278. print(help_text)
  279. def print_config(self):
  280. self.config.write(sys.stdout)
  281. def print_ips(self):
  282. ips = []
  283. for host, opts in self.config.items('all'):
  284. ips.append(self.get_ip_from_opts(opts))
  285. print(' '.join(ips))
  286. def main(argv=None):
  287. if not argv:
  288. argv = sys.argv[1:]
  289. KubesprayInventory(argv, CONFIG_FILE)
  290. if __name__ == "__main__":
  291. sys.exit(main())