From 39a43411c5bf9486927aef66bdb56235bf60e1f6 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Fri, 13 Oct 2023 15:40:13 +0200 Subject: [PATCH] Moving some methods into mixin module lib/cr_tf/handler/read.py --- lib/cr_tf/handler/__init__.py | 226 +---------------------------- lib/cr_tf/handler/read.py | 266 ++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 223 deletions(-) create mode 100644 lib/cr_tf/handler/read.py diff --git a/lib/cr_tf/handler/__init__.py b/lib/cr_tf/handler/__init__.py index e7b13be..7d27098 100644 --- a/lib/cr_tf/handler/__init__.py +++ b/lib/cr_tf/handler/__init__.py @@ -30,7 +30,6 @@ from operator import attrgetter # Third party modules import pytz -import yaml import six from fb_tools.common import pp, to_bool, to_str, RE_DOT_AT_END @@ -44,19 +43,18 @@ from fb_vmware.connect import VsphereConnection # Own modules from .first import CrTfHandlerFirstMixin +from .read import CrTfHandlerReadMixin from .. import MIN_VERSION_TERRAFORM, MAX_VERSION_TERRAFORM from .. import MIN_VERSION_VSPHERE_PROVIDER from ..errors import AbortExecution -from ..terraform.vm import TerraformVm - # from ..tools import password_input from ..xlate import XLATOR -__version__ = '3.9.2' +__version__ = '3.9.3' LOG = logging.getLogger(__name__) _ = XLATOR.gettext @@ -64,7 +62,7 @@ ngettext = XLATOR.ngettext # ============================================================================= -class CreateTerraformHandler(BaseHandler, CrTfHandlerFirstMixin): +class CreateTerraformHandler(BaseHandler, CrTfHandlerFirstMixin, CrTfHandlerReadMixin): """ A handler class for creating the terraform environment """ @@ -421,25 +419,6 @@ class CreateTerraformHandler(BaseHandler, CrTfHandlerFirstMixin): print() - # -------------------------------------------------------------------------· - def exec_read_yaml(self, yaml_file): - - if self.stop_at_step == 'read-yaml': - self.incr_verbosity() - - self.read_yaml_data(yaml_file) - self.eval_yaml_data() - if self.eval_errors: - msg = ngettext( - "Found one error in evaluation of YAML data of {f!r}.", - "Found {n} errors in evaluation of YAML data of {f!r}.", - self.eval_errors).format(n=self.eval_errors, f=str(yaml_file)) - raise ExpectedHandlerError(msg) - - LOG.info(_("Finished step {!r}.").format('read-yaml')) - if self.stop_at_step == 'read-yaml': - raise AbortExecution('read-yaml') - # -------------------------------------------------------------------------· def exec_collect_folders(self, yaml_file): @@ -900,205 +879,6 @@ class CreateTerraformHandler(BaseHandler, CrTfHandlerFirstMixin): if self.stop_at_step == 'ensure-vmw-folders': raise AbortExecution('ensure-vmw-folders') - # -------------------------------------------------------------------------· - def read_yaml_data(self, yaml_file): - - LOG.info(_("Reading YAML file {!r} ...").format(str(yaml_file))) - - open_opts = {} - if six.PY3 and self.config.encoding: - open_opts['encoding'] = self.config.encoding - open_opts['errors'] = 'surrogateescape' - - try: - with open(str(yaml_file), 'r', **open_opts) as fh: - self.yaml_data = yaml.full_load(fh) - except yaml.YAMLError as e: - msg = _("Error in YAML file {f!r}: {e}.").format( - f=str(yaml_file), e=e) - if hasattr(e, 'problem_mark'): - mark = e.problem_mark - msg += " " + _("Error position: {li}:{c}").format( - li=mark.line + 1, c=mark.column + 1) - raise ExpectedHandlerError(msg) - - if self.verbose > 2: - LOG.debug(_("Read data from YAML file:") + "\n" + pp(self.yaml_data)) - - if not isinstance(self.yaml_data, dict): - msg = _( - "Data read from YAML file {f!r} are not a dictionary, " - "but a {c} object instead.").format( - f=str(yaml_file), c=self.yaml_data.__class__.__name__) - raise ExpectedHandlerError(msg) - - for key in self.yaml_data.keys(): - if key.lower() == 'simulate': - self.simulate = to_bool(self.yaml_data[key]) - - # -------------------------------------------------------------------------· - def eval_yaml_data(self): - - self.vm_names = [] - - # Searching for default VM definition - LOG.debug(_("Searching for default VM definition ...")) - for key in self.yaml_data.keys(): - - if self.re_default.match(key): - vm = self._eval_tpl_vm(name='Default VM', vm_def=self.yaml_data[key]) - if vm: - self.default_vm = vm - - # Searching for VM definitions - LOG.debug(_("Searching for VM definitions ...")) - for key in self.yaml_data.keys(): - if self.re_vm_key.match(key): - for vm_def in self.yaml_data[key]: - vm = self._eval_vm(vm_def, template_vm=self.default_vm) - if vm: - self.vms.append(vm) - - # Searching for groups - for key in self.yaml_data.keys(): - if self.re_group.match(key): - self._eval_vm_groups(self.yaml_data[key], template_vm=self.default_vm, depth=1) - - if self.verbose > 2: - vm_list = [] - for vm in self.vms: - vm_list.append(vm.as_dict()) - LOG.debug(_("Evaluated VMs:") + "\n" + pp(vm_list)) - - # -------------------------------------------------------------------------· - def _eval_tpl_vm(self, name, vm_def, template_vm=None): - - try: - vm = TerraformVm.from_def( - vm_def, name=name, is_template=True, template_vm=template_vm, appname=self.appname, - verbose=self.verbose, base_dir=self.base_dir, simulate=self.simulate, - force=self.force, terminal_has_colors=self.terminal_has_colors) - except Exception as e: - if self.verbose > 2: - self.handle_error(str(e), e.__class__.__name__, True) - else: - LOG.error(_("{c} in evaluating template VM: {e}").format( - c=e.__class__.__name__, e=e)) - self.eval_errors += 1 - return None - - if self.verbose > 2: - LOG.debug(_( - "Defined Terraform Template VM {n!r}:").format( - n=vm.name) + "\n" + pp(vm.as_dict())) - - return vm - - # -------------------------------------------------------------------------· - def _eval_vm(self, vm_def, template_vm=None): - - try: - vm = TerraformVm.from_def( - vm_def, is_template=False, template_vm=template_vm, appname=self.appname, - verbose=self.verbose, base_dir=self.base_dir, simulate=self.simulate, - force=self.force, terminal_has_colors=self.terminal_has_colors) - except Exception as e: - if self.verbose > 2: - self.handle_error(str(e), e.__class__.__name__, True) - else: - LOG.error(_("{c} in evaluating VM: {e}").format(c=e.__class__.__name__, e=e)) - self.eval_errors += 1 - return None - - if self.verbose > 3: - LOG.debug(_( - "Defined Terraform-VM {n!r}:").format(n=vm.name) + "\n" + pp(vm.as_dict())) - - if vm.name in self.vm_names: - LOG.error(_("VM {!r} is already defined.").format(vm.name)) - self.eval_errors += 1 - return None - - return vm - - # -------------------------------------------------------------------------· - def _eval_vm_groups(self, groups_def, template_vm=None, depth=1): - - if not isinstance(groups_def, list): - msg = _("Group definition list is not a list:") + "\n" + pp(groups_def) - LOG.error(msg) - self.eval_errors += 1 - return - - if depth >= self.max_groups_depth: - LOG.warn(_("Maximum recursion depth for VM groups of {} reached.").format(depth)) - return - - if self.verbose > 2: - LOG.debug(_("Evaluating group list:") + "\n" + pp(groups_def)) - if self.verbose > 3: - LOG.debug(_("Used template: {!r}").format(template_vm)) - - for group_def in groups_def: - self._eval_vm_group(group_def, template_vm=template_vm, depth=depth) - - # -------------------------------------------------------------------------· - def _eval_vm_group(self, group_def, template_vm=None, depth=1): - - if not isinstance(group_def, dict): - msg = _("VM definition is not a dictionary:") + "\n" + pp(group_def) - LOG.error(msg) - self.eval_errors += 1 - return - - group_template = template_vm - group_name = None - - # Searching for the group name ..." - for key in group_def.keys(): - if self.re_group_name.match(key) and str(group_def[key]).strip(): - group_name = str(group_def[key]).strip() - - if not group_name: - LOG.error(_("No group name defined.")) - return - - # Searching for group default VM definition - LOG.debug(_("Searching for group default VM definition in group {!r} ...").format( - group_name)) - for key in group_def.keys(): - - if self.re_default.match(key): - vm_name = 'Default VM group {!r}'.format(group_name) - vm = self._eval_tpl_vm( - name=vm_name, vm_def=group_def[key], template_vm=template_vm) - if vm: - group_template = vm - break - - n = None - if group_template: - n = group_template.name - LOG.debug(_("Used template for creating VMs in group {g!r}: {n!r}").format( - g=group_name, n=n)) - if self.verbose > 3: - LOG.debug(_("Used template structure:") + "\n" + pp(group_template.as_dict())) - - # Searching for VM definitions - LOG.debug(_("Searching for VM definitions in group {!r} ...").format(group_name)) - for key in group_def.keys(): - if self.re_vm_key.match(key): - for vm_def in group_def[key]: - vm = self._eval_vm(vm_def, template_vm=group_template) - if vm: - self.vms.append(vm) - - # Searching for nested groups - for key in group_def.keys(): - if self.re_group.match(key): - self._eval_vm_groups( - group_def[key], template_vm=group_template, depth=depth + 1) - # -------------------------------------------------------------------------· def explore_vsphere_templates(self): diff --git a/lib/cr_tf/handler/read.py b/lib/cr_tf/handler/read.py new file mode 100644 index 0000000..3cf9ff4 --- /dev/null +++ b/lib/cr_tf/handler/read.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@contact: frank.brehm@pixelpark.com +@copyright: © 2023 by Frank Brehm, Berlin +@summary: A mixin module for the handler for methods for reading and evaluating YAML files. +""" +from __future__ import absolute_import, print_function + +# Standard module +import logging + +# Third party modules +import yaml +import six + +from fb_tools.common import pp, to_bool +from fb_tools.errors import ExpectedHandlerError + +# Own modules +from ..errors import AbortExecution + +from ..terraform.vm import TerraformVm + +from ..xlate import XLATOR + +__version__ = '0.1.0' +LOG = logging.getLogger(__name__) + +_ = XLATOR.gettext +ngettext = XLATOR.ngettext + + +# ============================================================================= +class CrTfHandlerReadMixin(): + """A mixin module for the handler module for reading and evaluation.""" + + # -------------------------------------------------------------------------· + def exec_read_yaml(self, yaml_file): + + if self.stop_at_step == 'read-yaml': + self.incr_verbosity() + + self.read_yaml_data(yaml_file) + self.eval_yaml_data() + if self.eval_errors: + msg = ngettext( + "Found one error in evaluation of YAML data of {f!r}.", + "Found {n} errors in evaluation of YAML data of {f!r}.", + self.eval_errors).format(n=self.eval_errors, f=str(yaml_file)) + raise ExpectedHandlerError(msg) + + LOG.info(_("Finished step {!r}.").format('read-yaml')) + if self.stop_at_step == 'read-yaml': + raise AbortExecution('read-yaml') + + # -------------------------------------------------------------------------· + def read_yaml_data(self, yaml_file): + + LOG.info(_("Reading YAML file {!r} ...").format(str(yaml_file))) + + open_opts = {} + if six.PY3 and self.config.encoding: + open_opts['encoding'] = self.config.encoding + open_opts['errors'] = 'surrogateescape' + + try: + with open(str(yaml_file), 'r', **open_opts) as fh: + self.yaml_data = yaml.full_load(fh) + except yaml.YAMLError as e: + msg = _("Error in YAML file {f!r}: {e}.").format( + f=str(yaml_file), e=e) + if hasattr(e, 'problem_mark'): + mark = e.problem_mark + msg += " " + _("Error position: {li}:{c}").format( + li=mark.line + 1, c=mark.column + 1) + raise ExpectedHandlerError(msg) + + if self.verbose > 2: + LOG.debug(_("Read data from YAML file:") + "\n" + pp(self.yaml_data)) + + if not isinstance(self.yaml_data, dict): + msg = _( + "Data read from YAML file {f!r} are not a dictionary, " + "but a {c} object instead.").format( + f=str(yaml_file), c=self.yaml_data.__class__.__name__) + raise ExpectedHandlerError(msg) + + for key in self.yaml_data.keys(): + if key.lower() == 'simulate': + self.simulate = to_bool(self.yaml_data[key]) + + # -------------------------------------------------------------------------· + def eval_yaml_data(self): + + self.vm_names = [] + + # Searching for default VM definition + LOG.debug(_("Searching for default VM definition ...")) + for key in self.yaml_data.keys(): + + if self.re_default.match(key): + vm = self._eval_tpl_vm(name='Default VM', vm_def=self.yaml_data[key]) + if vm: + self.default_vm = vm + + # Searching for VM definitions + LOG.debug(_("Searching for VM definitions ...")) + for key in self.yaml_data.keys(): + if self.re_vm_key.match(key): + for vm_def in self.yaml_data[key]: + vm = self._eval_vm(vm_def, template_vm=self.default_vm) + if vm: + self.vms.append(vm) + + # Searching for groups + for key in self.yaml_data.keys(): + if self.re_group.match(key): + self._eval_vm_groups(self.yaml_data[key], template_vm=self.default_vm, depth=1) + + if self.verbose > 2: + vm_list = [] + for vm in self.vms: + vm_list.append(vm.as_dict()) + LOG.debug(_("Evaluated VMs:") + "\n" + pp(vm_list)) + + # -------------------------------------------------------------------------· + def _eval_tpl_vm(self, name, vm_def, template_vm=None): + + try: + vm = TerraformVm.from_def( + vm_def, name=name, is_template=True, template_vm=template_vm, appname=self.appname, + verbose=self.verbose, base_dir=self.base_dir, simulate=self.simulate, + force=self.force, terminal_has_colors=self.terminal_has_colors) + except Exception as e: + if self.verbose > 2: + self.handle_error(str(e), e.__class__.__name__, True) + else: + LOG.error(_("{c} in evaluating template VM: {e}").format( + c=e.__class__.__name__, e=e)) + self.eval_errors += 1 + return None + + if self.verbose > 2: + LOG.debug(_( + "Defined Terraform Template VM {n!r}:").format( + n=vm.name) + "\n" + pp(vm.as_dict())) + + return vm + + # -------------------------------------------------------------------------· + def _eval_vm(self, vm_def, template_vm=None): + + try: + vm = TerraformVm.from_def( + vm_def, is_template=False, template_vm=template_vm, appname=self.appname, + verbose=self.verbose, base_dir=self.base_dir, simulate=self.simulate, + force=self.force, terminal_has_colors=self.terminal_has_colors) + except Exception as e: + if self.verbose > 2: + self.handle_error(str(e), e.__class__.__name__, True) + else: + LOG.error(_("{c} in evaluating VM: {e}").format(c=e.__class__.__name__, e=e)) + self.eval_errors += 1 + return None + + if self.verbose > 3: + LOG.debug(_( + "Defined Terraform-VM {n!r}:").format(n=vm.name) + "\n" + pp(vm.as_dict())) + + if vm.name in self.vm_names: + LOG.error(_("VM {!r} is already defined.").format(vm.name)) + self.eval_errors += 1 + return None + + return vm + + # -------------------------------------------------------------------------· + def _eval_vm_groups(self, groups_def, template_vm=None, depth=1): + + if not isinstance(groups_def, list): + msg = _("Group definition list is not a list:") + "\n" + pp(groups_def) + LOG.error(msg) + self.eval_errors += 1 + return + + if depth >= self.max_groups_depth: + LOG.warn(_("Maximum recursion depth for VM groups of {} reached.").format(depth)) + return + + if self.verbose > 2: + LOG.debug(_("Evaluating group list:") + "\n" + pp(groups_def)) + if self.verbose > 3: + LOG.debug(_("Used template: {!r}").format(template_vm)) + + for group_def in groups_def: + self._eval_vm_group(group_def, template_vm=template_vm, depth=depth) + + # -------------------------------------------------------------------------· + def _eval_vm_group(self, group_def, template_vm=None, depth=1): + + if not isinstance(group_def, dict): + msg = _("VM definition is not a dictionary:") + "\n" + pp(group_def) + LOG.error(msg) + self.eval_errors += 1 + return + + group_template = template_vm + group_name = None + + # Searching for the group name ..." + for key in group_def.keys(): + if self.re_group_name.match(key) and str(group_def[key]).strip(): + group_name = str(group_def[key]).strip() + + if not group_name: + LOG.error(_("No group name defined.")) + return + + # Searching for group default VM definition + LOG.debug(_("Searching for group default VM definition in group {!r} ...").format( + group_name)) + for key in group_def.keys(): + + if self.re_default.match(key): + vm_name = 'Default VM group {!r}'.format(group_name) + vm = self._eval_tpl_vm( + name=vm_name, vm_def=group_def[key], template_vm=template_vm) + if vm: + group_template = vm + break + + n = None + if group_template: + n = group_template.name + LOG.debug(_("Used template for creating VMs in group {g!r}: {n!r}").format( + g=group_name, n=n)) + if self.verbose > 3: + LOG.debug(_("Used template structure:") + "\n" + pp(group_template.as_dict())) + + # Searching for VM definitions + LOG.debug(_("Searching for VM definitions in group {!r} ...").format(group_name)) + for key in group_def.keys(): + if self.re_vm_key.match(key): + for vm_def in group_def[key]: + vm = self._eval_vm(vm_def, template_vm=group_template) + if vm: + self.vms.append(vm) + + # Searching for nested groups + for key in group_def.keys(): + if self.re_group.match(key): + self._eval_vm_groups( + group_def[key], template_vm=group_template, depth=depth + 1) + + +# ============================================================================= + +if __name__ == "__main__": + + pass + +# ============================================================================= + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 list -- 2.39.5