From 065c8146d0f1e2e30e9d77f156c4a59f316955ee Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Fri, 13 Oct 2023 16:21:02 +0200 Subject: [PATCH] Moving some methods into mixin module lib/cr_tf/handler/vmware.py --- lib/cr_tf/handler/__init__.py | 905 +------------------------------- lib/cr_tf/handler/vmware.py | 946 ++++++++++++++++++++++++++++++++++ 2 files changed, 950 insertions(+), 901 deletions(-) create mode 100644 lib/cr_tf/handler/vmware.py diff --git a/lib/cr_tf/handler/__init__.py b/lib/cr_tf/handler/__init__.py index 5f177cb..878d57e 100644 --- a/lib/cr_tf/handler/__init__.py +++ b/lib/cr_tf/handler/__init__.py @@ -16,7 +16,6 @@ import shutil import stat import textwrap import copy -import sys from pathlib import Path @@ -24,8 +23,6 @@ from subprocess import PIPE from distutils.version import LooseVersion -from operator import attrgetter - # Third party modules import pytz import six @@ -35,14 +32,11 @@ from fb_tools.errors import HandlerError, ExpectedHandlerError from fb_tools.handling_obj import HandlingObject, CalledProcessError from fb_tools.handler import BaseHandler -from fb_vmware.errors import VSphereExpectedError -from fb_vmware.config import VSPhereConfigInfo -from fb_vmware.connect import VsphereConnection - # Own modules from .dns import CrTfHandlerDnsMixin from .first import CrTfHandlerFirstMixin from .read import CrTfHandlerReadMixin +from .vmware import CrTfHandlerVmwMixin from .. import MIN_VERSION_TERRAFORM, MAX_VERSION_TERRAFORM from .. import MIN_VERSION_VSPHERE_PROVIDER @@ -53,7 +47,7 @@ from ..errors import AbortExecution from ..xlate import XLATOR -__version__ = '3.9.4' +__version__ = '3.9.5' LOG = logging.getLogger(__name__) _ = XLATOR.gettext @@ -62,7 +56,8 @@ ngettext = XLATOR.ngettext # ============================================================================= class CreateTerraformHandler( - BaseHandler, CrTfHandlerFirstMixin, CrTfHandlerReadMixin, CrTfHandlerDnsMixin): + BaseHandler, CrTfHandlerFirstMixin, CrTfHandlerReadMixin, CrTfHandlerDnsMixin, + CrTfHandlerVmwMixin): """ A handler class for creating the terraform environment """ @@ -419,375 +414,6 @@ class CreateTerraformHandler( print() - # -------------------------------------------------------------------------· - def exec_collect_folders(self, yaml_file): - - if self.stop_at_step == 'collect-folders': - self.incr_verbosity() - - LOG.info(_("Collecting all VMWare and local folders ...")) - LOG.info(_("Get vSphere datacenter ...")) - for vname in self.vsphere: - self.vsphere[vname].get_datacenter() - - LOG.debug(_("Collecting vSphere folders.")) - self.vsphere_folders = [] - for vm in self.vms: - if vm.folder: - if vm.folder not in self.vsphere_folders: - self.vsphere_folders.append(vm.folder) - self.vsphere_folders.sort(key=str.lower) - LOG.debug(_("Collected vSphere folders:") + "\n" + pp(self.vsphere_folders)) - - # Set project name and directory - yfile = Path(yaml_file) - yfile_base = yfile.name - yfile_dir = yfile.parent.resolve() - (yfile_stem, yfile_ext) = os.path.splitext(yfile_base) - self.project_name = yfile_stem - LOG.info(_("Project name is {!r}.").format(str(self.project_name))) - self.project_dir = yfile_dir / yfile_stem - LOG.info(_("Project directory is: {!r}.").format(str(self.project_dir))) - - # Evaluating root terraform directory - if not self.is_venv: - i = 4 - cdir = copy.copy(self.project_dir).parent - while i > 0: - git_dir = cdir / '.git' - if git_dir.is_dir(): - self._terraform_root_dir = cdir - break - i -= 1 - if cdir == cdir.parent: - break - cdir = cdir.parent - if not self._terraform_root_dir: - msg = _("Did not found root terraform directory above {!r}.").format( - str(self.project_dir)) - LOG.warn(msg) - - LOG.info(_("Full project name: {!r}").format(self.full_project_name)) - - LOG.info(_("Finished step {!r}.").format('collect-folders')) - if self.stop_at_step == 'collect-folders': - raise AbortExecution('collect-folders') - - # -------------------------------------------------------------------------· - def init_vspheres(self, yaml_file): - - if self.stop_at_step == 'vmw-init': - self.incr_verbosity() - - # Test for multiple VSphere references - found_vspheres = [] - for vm in self.vms: - vname = vm.vsphere - if vname not in found_vspheres: - found_vspheres.append(vname) - if len(found_vspheres) > 1: - yaml_file_rel = os.path.relpath(str(yaml_file), os.getcwd()) - msg = _("There is only one, unique VSPhere definition allowed in a project file.") - msg += '\n' - msg += _("In {f!r} were found {nr} different VSPhere definitions:").format( - f=yaml_file_rel, nr=len(found_vspheres)) - for vname in sorted(found_vspheres, key=str.lower): - msg += '\n * {!r}'.format(vname) - raise ExpectedHandlerError(msg) - - self._init_vspheres() - - LOG.info(_("Finished step {!r}.").format('vmw-init')) - if self.stop_at_step == 'vmw-init': - raise AbortExecution('vmw-init') - - # -------------------------------------------------------------------------· - def _init_vspheres(self): - - for vm in self.vms: - if vm.vsphere in self.vsphere: - continue - vname = vm.vsphere - if vname not in self.config.vsphere: - msg = _("VSPhere {!r} not defined in configuration.").format(vname) - raise ExpectedHandlerError(msg) - - if not self.vsphere_user and self.config.vsphere[vname].user: - self.vsphere_user = self.config.vsphere[vname].user - if not self.vsphere_password and self.config.vsphere[vname].password: - self.vsphere_password = self.config.vsphere[vname].password - - try: - params = { - 'appname': self.appname, - 'verbose': self.verbose, - 'base_dir': self.base_dir, - 'simulate': self.simulate, - 'force': self.force, - 'terminal_has_colors': self.terminal_has_colors, - 'initialized': True, - } - show_params = copy.copy(params) - - connect_info = VSPhereConfigInfo( - appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, - host=self.config.vsphere[vname].host, port=self.config.vsphere[vname].port, - dc=self.config.vsphere[vname].dc, user=self.vsphere_user, - password=self.vsphere_password, initialized=True) - - params['connect_info'] = connect_info - show_params['connect_info'] = connect_info.as_dict() - - if self.verbose > 1: - if self.verbose < 5: - show_params['connect_info']['password'] = '******' - msg = _("Initialising a {}-object with params:").format('VsphereConnection') - msg += '\n' + pp(show_params) - LOG.debug(msg) - - vsphere = VsphereConnection(**params) - self.vsphere[vname] = vsphere - - except VSphereExpectedError as e: - raise ExpectedHandlerError(str(e)) - - # -------------------------------------------------------------------------· - def test_vsphere_handlers(self): - - if self.stop_at_step == 'vmw-test': - self.incr_verbosity() - - for vname in self.vsphere.keys(): - - try: - - vsphere = self.vsphere[vname] - - vsphere.get_about() - if self.verbose > 2: - msg = _("Created {}-object:").format('VsphereConnection') - msg += '\n' + pp(vsphere.as_dict()) - LOG.debug(msg) - - except VSphereExpectedError as e: - raise ExpectedHandlerError(str(e)) - - LOG.info(_("Finished step {!r}.").format('vmw-test')) - if self.stop_at_step == 'vmw-test': - raise AbortExecution('vmw-test') - - # -------------------------------------------------------------------------· - def assign_default_vmw_values(self): - """Assigning not defined templates and clusters of VMs by their - appropriate default values.""" - - LOG.debug(_( - "Assigning not defined templates and clusters of VMs by their " - "appropriate default values.")) - - for vm in self.vms: - - if not vm.cluster: - cl = self.config.vsphere[vm.vsphere].cluster - if self.verbose > 1: - LOG.debug(_("Setting cluster of {n!r} to {c!r} ...").format( - n=vm.name, c=cl)) - vm.cluster = cl - - if not vm.vm_template: - tpl = self.config.vsphere[vm.vsphere].template_name - if self.verbose > 1: - LOG.debug(_("Setting template of {n!r} to {t!r} ...").format( - n=vm.name, t=tpl)) - vm.vm_template = tpl - - # -------------------------------------------------------------------------· - def exec_vmw_clusters(self): - - if self.stop_at_step == 'vmw-clusters': - self.incr_verbosity() - - for vname in self.vsphere: - LOG.debug(_("Searching for clusters in VSPhere {!r} ...").format(vname)) - self.vsphere[vname].get_clusters() - - LOG.info(_("Finished step {!r}.").format('vmw-clusters')) - if self.stop_at_step == 'vmw-clusters': - raise AbortExecution('vmw-clusters') - - # -------------------------------------------------------------------------· - def exec_vmw_datastores(self): - - if self.stop_at_step == 'vmw-datastores': - self.incr_verbosity() - - nr_total = 0 - - for vname in self.vsphere: - LOG.debug(_("Searching for datastores in VSPhere {!r} ...").format(vname)) - self.vsphere[vname].get_datastores() - nr_total += len(self.vsphere[vname].datastores.keys()) - - if nr_total: - msg = ngettext("Found one datastore.", "Found {n} datastores.", nr_total) - LOG.debug(msg.format(n=nr_total)) - else: - LOG.error(_("No VSPhere datastores found.")) - - LOG.info(_("Finished step {!r}.").format('vmw-datastores')) - if self.stop_at_step == 'vmw-datastores': - raise AbortExecution('vmw-datastores') - - # -------------------------------------------------------------------------· - def exec_vmw_ds_clusters(self): - - nr_total = 0 - - if self.stop_at_step == 'vmw-ds-clusters': - self.incr_verbosity() - - for vname in self.vsphere: - LOG.debug(_("Searching for datastore clusters in VSPhere {!r} ...").format(vname)) - self.vsphere[vname].get_ds_clusters() - nr_total += len(self.vsphere[vname].ds_clusters.keys()) - - if nr_total: - msg = ngettext( - "Found one datastore cluster.", - "Found {n} datastore clusters.", - nr_total) - LOG.debug(msg.format(n=nr_total)) - else: - LOG.warn(_("No VSPhere datastore clusters found.")) - - LOG.info(_("Finished step {!r}.").format('vmw-ds-clusters')) - if self.stop_at_step == 'vmw-ds-clusters': - raise AbortExecution('vmw-ds-clusters') - - # -------------------------------------------------------------------------· - def exec_vmw_networks(self): - - if self.stop_at_step == 'vmw-networks': - self.incr_verbosity() - - for vname in self.vsphere: - LOG.debug(_("Searching for networks in VSPhere {!r} ...").format(vname)) - self.vsphere[vname].get_networks() - if self.eval_errors: - msg = ngettext( - "Found one error in exploring vSphere {v!r} resources.", - "Found {n} errors in exploring vSphere {v!r} resources.", - self.eval_errors).format(n=self.eval_errors, v=vname) - raise ExpectedHandlerError(msg) - - LOG.info(_("Finished step {!r}.").format('vmw-networks')) - if self.stop_at_step == 'vmw-networks': - raise AbortExecution('vmw-networks') - - # -------------------------------------------------------------------------· - def exec_vmw_templates(self): - - if self.stop_at_step == 'vmw-templates': - self.incr_verbosity() - - self.explore_vsphere_templates() - if self.eval_errors: - msg = ngettext( - "Found one error in exploring vSphere templates.", - "Found {n} errors in exploring vSphere templates.", - self.eval_errors).format(n=self.eval_errors) - raise ExpectedHandlerError(msg) - - LOG.info(_("Finished step {!r}.").format('vmw-templates')) - if self.stop_at_step == 'vmw-templates': - raise AbortExecution('vmw-templates') - - # -------------------------------------------------------------------------· - def exec_validate_yaml(self): - - if self.stop_at_step == 'validate-yaml': - self.incr_verbosity() - - print() - LOG.info(_("Validating information from YAML file ...")) - - self.validate_clusters() - if self.eval_errors: - msg = ngettext( - "Found one error in validating vSphere computing clusters.", - "Found {n} errors in validating vSphere computing clusters.", - self.eval_errors).format(n=self.eval_errors) - raise ExpectedHandlerError(msg) - - self.get_all_vms() - self.validate_vms() - - LOG.info(_("Finished step {!r}.").format('validate-yaml')) - if self.stop_at_step == 'validate-yaml': - raise AbortExecution('validate-yaml') - - # -------------------------------------------------------------------------· - def get_all_vms(self): - - LOG.info(_("Got a list of all VMs and templates ...")) - self.all_vms = {} - re_vm = re.compile(r'.*') - - for vs_name in self.vsphere: - - if vs_name not in self.all_vms: - self.all_vms[vs_name] = {} - - vm_list = self.vsphere[vs_name].get_vms(re_vm, name_only=True) - for vm_tuple in vm_list: - vm_name = vm_tuple[0] - vm_path = vm_tuple[1] - if vm_name in self.all_vms[vs_name]: - self.all_vms[vs_name][vm_name].append(vm_path) - else: - self.all_vms[vs_name][vm_name] = [vm_path] - - if self.verbose > 2: - msg = _("All existing VMs and templates:") - msg += '\n' + pp(self.all_vms) - LOG.debug(msg) - - # -------------------------------------------------------------------------· - def exec_validate_storage(self): - - if self.stop_at_step == 'validate-storage': - self.incr_verbosity() - - self.validate_storages() - if self.eval_errors: - msg = ngettext( - "Found one error in validating VM storages.", - "Found {n} errors in validating VM storages.", - self.eval_errors).format(n=self.eval_errors) - raise ExpectedHandlerError(msg) - - LOG.info(_("Finished step {!r}.").format('validate-storage')) - if self.stop_at_step == 'validate-storage': - raise AbortExecution('validate-storage') - - # -------------------------------------------------------------------------· - def exec_validate_iface(self): - - if self.stop_at_step == 'validate-iface': - self.incr_verbosity() - - self.validate_interfaces() - if self.eval_errors: - msg = ngettext( - "Found one error in validating VM interfaces.", - "Found {n} errors in validating VM interfaces.", - self.eval_errors).format(n=self.eval_errors) - raise ExpectedHandlerError(msg) - - LOG.info(_("Finished step {!r}.").format('validate-iface')) - if self.stop_at_step == 'validate-iface': - raise AbortExecution('validate-iface') - # -------------------------------------------------------------------------· def exec_project_dir(self): @@ -813,529 +439,6 @@ class CreateTerraformHandler( if self.stop_at_step == 'tf-files': raise AbortExecution('tf-files') - # -------------------------------------------------------------------------· - def exec_vsphere_folders(self): - - if self.stop_at_step == 'ensure-vmw-folders': - self.incr_verbosity() - - self.ensure_vsphere_folders() - - LOG.info(_("Finished step {!r}.").format('ensure-vmw-folders')) - if self.stop_at_step == 'ensure-vmw-folders': - raise AbortExecution('ensure-vmw-folders') - - # -------------------------------------------------------------------------· - def explore_vsphere_templates(self): - - LOG.info(_("Exploring all vSphere templates ...")) - - for vname in self.vsphere: - - if vname not in self.vsphere_templates: - self.vsphere_templates[vname] = {} - - self.config.vsphere[vname].used_templates = [] - - for vm in self.vms: - template_name = vm.vm_template - if template_name: - if template_name not in self.config.vsphere[vname].used_templates: - self.config.vsphere[vname].used_templates.append(template_name) - else: - LOG.error(_("VM {!r} has not template defined.").format(vm.name)) - self.eval_errors += 1 - - msg = _("All {} VSPhere templates to explore:").format(vname) - msg += "\n" + pp(self.config.vsphere[vname].used_templates) - LOG.debug(msg) - - for template_name in self.config.vsphere[vname].used_templates: - - if template_name in self.vsphere_templates[vname]: - continue - - LOG.debug(_("Searching for template {t!r} in VSPhere {v!r} ...").format( - t=template_name, v=vname)) - re_vm = re.compile(r'^' + re.escape(template_name) + r'$', re.IGNORECASE) - vm_list = self.vsphere[vname].get_vms(re_vm, as_obj=True, stop_at_found=True) - if vm_list: - vm = vm_list[0] - tname = vm.name.lower() - if tname not in self.vsphere_templates[vname]: - self.vsphere_templates[vname][template_name] = vm - else: - LOG.error(_("Template {t!r} not found in VSPhere {v!r}.").format( - t=template_name, v=vname)) - self.eval_errors += 1 - - if self.verbose > 2: - msg = _("All explored vSphere templates:") - out_dict = {} - for vname in self.vsphere_templates: - out_dict[vname] = {} - for tname in self.vsphere_templates[vname]: - out_dict[vname][tname] = self.vsphere_templates[vname][tname].as_dict() - msg += "\n" + pp(out_dict) - LOG.debug(msg) - - # -------------------------------------------------------------------------· - def validate_clusters(self): - - print() - LOG.info(_("Validating existence of computing clusters of the VMs.")) - - clusters = {} - - for vm in self.vms: - - vname = vm.vsphere - if vname not in clusters: - clusters[vname] = {} - - if vm.cluster in clusters: - clusters[vname][vm.cluster].append(vm.name) - else: - clusters[vname][vm.cluster] = [vm.name] - - for vname in clusters.keys(): - for cluster in clusters[vname].keys(): - - vms = clusters[vname][cluster] - - cl = str(cluster) - LOG.debug(_( - "Checking existence of computing cluster {c!r} in VSPhere {v!r} ...").format( - c=cl, v=vname)) - - vsphere = self.vsphere[vname] - vmw_cluster = vsphere.get_cluster_by_name(cl) - if vmw_cluster: - if self.verbose > 1: - LOG.debug(_( - "Found computing cluster {cl!r} in VSPhere {v!r} (defined for VMs " - "{vms}).").format(cl=vmw_cluster.name, v=vname, vms=pp(vms))) - else: - LOG.error(_( - "Computing cluster {cl!r} (defined for VMs {vms}) in VSPhere {v!r} not " - "found.").format(cl=cl, vms=pp(vms), v=vname)) - self.eval_errors += 1 - - # -------------------------------------------------------------------------· - def validate_vms(self): - - print() - LOG.info(_("Validating existence of VMs in VMWare.")) - vms2perform = [] - - for vm in sorted(self.vms, key=attrgetter('tf_name')): - - print(" * {} ".format(vm.fqdn), end='', flush=True) - if self.verbose: - print() - vs_name = vm.vsphere - vsphere = self.vsphere[vs_name] - - vm_paths = None - if vs_name in self.all_vms: - if vm.fqdn in self.all_vms[vs_name]: - vm_paths = self.all_vms[vs_name][vm.fqdn] - - if vm_paths: - msg = _('[{m}] - VM is already existing in VSphere {v!r}, path {p!r}.').format( - m=self.colored('Existing', 'YELLOW'), v=vs_name, p=pp(vm_paths)) - print(msg, end='', flush=True) - if self.verbose: - print() - - vm_info = vsphere.get_vm(vm.fqdn, vsphere_name=vs_name, as_obj=True) - if self.verbose > 2: - LOG.debug(_("VM info:") + "\n" + pp(vm_info.as_dict(bare=True))) - ds = vm_info.config_path_storage - LOG.debug(_("Datastore of VM {vm!r}: {ds!r}.").format(vm=vm.name, ds=ds)) - vm.datastore = ds - vm.already_existing = True - self.existing_vms.append(vm_info) - - else: - - print('[{}] '.format(self.colored('OK', 'GREEN')), end='', flush=True) - vm.already_existing = False - - vms2perform.append(vm) - print() - - self.vms = vms2perform - - print() - - if not len(self.vms): - print() - print(self.colored('*' * 60, ('BOLD', 'RED')), file=sys.stderr) - print(self.colored('* ' + _('CAUTION!'), ('BOLD', 'RED')), file=sys.stderr) - print(self.colored('*' * 60, ('BOLD', 'RED')), file=sys.stderr) - print() - print( - self.colored(_('Did not found any VM to deploy!'), ('BOLD', 'RED')), - file=sys.stderr) - print() - raise ExpectedHandlerError(_("No VMs to deploy")) - - # -------------------------------------------------------------------------· - def validate_storages(self): - - self._validate_ds_clusters() - self._validate_datastores() - - if self.verbose: - if self.used_dc_clusters: - out_lines = [] - for vs_name in self.used_dc_clusters: - for cluster in self.used_dc_clusters[vs_name]: - out_lines.append(' * VSphere {v!r}: {c}'.format( - v=vs_name, c=cluster)) - out = '\n'.join(out_lines) - LOG.debug(_("Used datastore clusters:") + "\n" + out) - else: - LOG.debug(_("No datastore clusters are used.")) - if self.used_datastores: - out_lines = [] - for vs_name in self.used_datastores: - for ds in self.used_datastores[vs_name]: - out_lines.append(' * VSphere {v!r}: {ds}'.format(v=vs_name, ds=ds)) - out = '\n'.join(out_lines) - LOG.debug(_("Used datastors:") + "\n" + out) - else: - LOG.debug(_("No datastores are used.")) - - # -------------------------------------------------------------------------· - def _validate_ds_clusters(self): - - LOG.info(_("Validating given datastore clusters of VMs ...")) - - for vm in self.vms: - - if not vm.ds_cluster: - continue - - self._validate_dscluster_vm(vm) - - # -------------------------------------------------------------------------· - def _validate_dscluster_vm(self, vm): - - needed_gb = 0.0 - if not vm.already_existing: - for unit_number in vm.disks.keys(): - disk = vm.disks[unit_number] - needed_gb += disk.size_gb - - vs_name = vm.vsphere - vsphere = self.vsphere[vs_name] - - found = False - for cluster_name in vsphere.ds_clusters.keys(): - if cluster_name.lower() == vm.ds_cluster.lower(): - if self.verbose > 2: - LOG.debug(_( - "Found datastore cluster {c!r} in VSphere {v!r} for VM {n!r}.").format( - n=vm.name, v=vs_name, c=vm.ds_cluster)) - if vm.ds_cluster != cluster_name: - LOG.debug(_("Setting datastore cluster for VM {n!r} to {c!r} ...").format( - n=vm.name, c=cluster_name)) - vm.ds_cluster = cluster_name - ds_cluster = vsphere.ds_clusters[cluster_name] - if self.verbose > 2: - LOG.debug(_( - "Free space of cluster {c!r} in VSphere {v!r} before provisioning: " - "{a:0.1f} GiB.").format( - c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb)) - if ds_cluster.avail_space_gb < needed_gb: - LOG.error(_( - "Datastore cluster {d!r} in VSphere {v!r} has not sufficient space for " - "storage of VM {vm!r} (needed {n:0.1f} GiB, available {a:0.1f} " - "GiB).").format( - d=cluster_name, v=vs_name, vm=vm.name, n=needed_gb, - a=ds_cluster.avail_space_gb)) - self.eval_errors += 1 - else: - ds_cluster.calculated_usage += needed_gb - if self.verbose > 1: - LOG.debug(_( - "Free space in cluster {c!r} in VSphere {v!r} after provisioning: " - "{a:0.1f} GiB.").format( - c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb)) - found = True - if vs_name not in self.used_dc_clusters: - self.used_dc_clusters[vs_name] = [] - if cluster_name not in self.used_dc_clusters[vs_name]: - self.used_dc_clusters[vs_name].append(cluster_name) - break - - if not found: - LOG.error(_("Datastore cluster {c!r} of VM {n!r} not found in VSphere {v!r}.").format( - n=vm.name, c=vm.ds_cluster, v=vs_name)) - self.eval_errors += 1 - - # -------------------------------------------------------------------------· - def _validate_datastores(self): - - LOG.info(_("Validating given datastores of VMs and assign failing ...")) - - for vm in self.vms: - - if vm.ds_cluster: - if vm.datastore: - LOG.debug(_("Removing defined datastore {d!r} for VM {n!r} ...").format( - d=vm.datastore, n=vm.name)) - vm.datastore = None - continue - - self._validate_ds_vm(vm) - - # -------------------------------------------------------------------------· - def _validate_ds_vm(self, vm): - - needed_gb = 0.0 - if not vm.already_existing: - for unit_number in vm.disks.keys(): - disk = vm.disks[unit_number] - needed_gb += disk.size_gb - - vs_name = vm.vsphere - vsphere = self.vsphere[vs_name] - - vm_cluster = None - for cluster in vsphere.clusters: - if cluster.name.lower() == vm.cluster.lower(): - vm_cluster = cluster - break - if not vm_cluster: - msg = _("Did not found cluster object {c!r} for VM {n!r}.").format( - c=vm.cluster, n=vm.name) - raise HandlerError(msg) - - if vm.datastore: - found = False - found_ds_name = None - for ds_name in vsphere.datastores: - if ds_name.lower() == vm.datastore.lower(): - if self.verbose > 2: - LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format( - n=vm.name, d=vm.datastore, v=vs_name)) - if ds_name not in vm_cluster.datastores: - LOG.warn(_("Datastore {d!r} not available in cluster {c!r}.").format( - d=ds_name, c=vm.cluster)) - break - if vm.datastore != ds_name: - LOG.debug(_("Setting datastore for VM {n!r} to {d!r} ...").format( - n=vm.name, d=ds_name)) - vm.datastore = ds_name - ds = vsphere.datastores[ds_name] - if ds.avail_space_gb < needed_gb: - LOG.error(_( - "Datastore {d!r} has not sufficient space for storage of VM " - "{v!r} (needed {n:0.1f} GiB, available {a:0.1f} GiB).").format( - d=ds_name, v=vm.name, n=needed_gb, a=ds.avail_space_gb)) - self.eval_errors += 1 - else: - ds.calculated_usage += needed_gb - found = True - found_ds_name = ds_name - break - if not found: - LOG.error(_("Datastore {d!r} of VM {n!r} not found in VSPhere {v!r}.").format( - n=vm.name, d=vm.datastore, v=vs_name)) - self.eval_errors += 1 - if vs_name not in self.used_datastores: - self.used_datastores[vs_name] = [] - if found_ds_name not in self.used_datastores[vs_name]: - self.used_datastores[vs_name].append(found_ds_name) - return - - ds_name = vsphere.datastores.find_ds( - needed_gb, vm.ds_type, use_ds=copy.copy(vm_cluster.datastores), no_k8s=True) - if ds_name: - LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format( - d=ds_name, n=vm.name, v=vs_name)) - vm.datastore = ds_name - if vs_name not in self.used_datastores: - self.used_datastores[vs_name] = [] - if ds_name not in self.used_datastores[vs_name]: - self.used_datastores[vs_name].append(ds_name) - else: - self.eval_errors += 1 - - # -------------------------------------------------------------------------· - def validate_interfaces(self): - - LOG.info(_("Validating interfaces of VMs and assign networks ...")) - for vm in self.vms: - self._validate_interfaces_vm(vm) - - if self.verbose > 2: - LOG.debug(_("Validated FQDNs:") + "\n" + pp(self.fqdns)) - LOG.debug(_("Validated Addresses:") + "\n" + pp(self.addresses)) - - if self.verbose: - - lines = [] - for vs_name in self.used_networks: - for nw in self.used_networks[vs_name]: - lines.append(' * VSphere {v!r}: {n}'.format( - v=vs_name, n=nw)) - out = '\n'.join(lines) - LOG.debug(_("Used networks:") + "\n" + out) - - lines = [] - for pair in self.dns_mapping['forward']: - line = ' * {n!r} => {a!r}'.format(n=pair[0], a=str(pair[1])) - lines.append(line) - LOG.debug(_("Used forward DNS entries:") + "\n" + '\n'.join(lines)) - - lines = [] - for pair in self.dns_mapping['reverse']: - line = ' * {a!r} => {n!r}'.format(n=pair[1], a=str(pair[0])) - lines.append(line) - LOG.debug(_("Used reverse DNS entries:") + "\n" + '\n'.join(lines)) - - # -------------------------------------------------------------------------· - def _validate_interfaces_vm(self, vm): - - vs_name = vm.vsphere - LOG.debug(_("Checking interfaces of VM {n!r} in VSPhere {v!r} ...").format( - n=vm.name, v=vs_name)) - - if not vm.interfaces: - LOG.error(_("No interfaces defined for VM {!r}.").format(vm.name)) - self.eval_errors += 1 - return - - vsphere = self.vsphere[vs_name] - - vm_cluster = None - for cluster in vsphere.clusters: - if cluster.name.lower() == vm.cluster.lower(): - vm_cluster = cluster - break - if not vm_cluster: - msg = _("Did not found cluster object {c!r} for VM {n!r}.").format( - c=vm.cluster, n=vm.name) - raise HandlerError(msg) - - i = -1 - for iface in vm.interfaces: - i += 1 - self._validate_interface_of_vm( - vm_name=vm.name, iface=iface, vs_name=vs_name, vm_cluster=vm_cluster, i=i) - - # -------------------------------------------------------------------------· - def _validate_interface_of_vm(self, vm_name, iface, vs_name, vm_cluster, i=0): - - vsphere = self.vsphere[vs_name] - - if self.verbose > 1: - LOG.debug(_("Checking interface {i} of VM {n!r} ...").format( - i=i, n=vm_name)) - - if not iface.address: - LOG.error(_("Interface {i} of VM {n!r} has no defined address.").format( - i=i, n=vm_name)) - self.eval_errors += 1 - return - - if not iface.fqdn: - LOG.error(_("Interface {i} of VM {n!r} has no defined FQDN.").format( - i=i, n=vm_name)) - self.eval_errors += 1 - return - - if iface.fqdn in self.fqdns: - LOG.error(_( - "FQDN {f!r} already defined for VM {va!r}({ia}) should be set " - "for interface {ib} of {vb!r}.").format( - f=iface.fqdn, va=self.fqdns[iface.fqdn][0], ia=self.fqdns[iface.fqdn][1], - ib=i, vb=vm_name)) - self.eval_errors += 1 - return - - self.fqdns[iface.fqdn] = (vm_name, i) - - if iface.address_v4: - if iface.address_v4 in self.addresses: - LOG.error(_( - "IPv4 address {a} already defined for VM {va!r}({ia}) should be set " - "for interface {ib} of {vb!r}.").format( - a=iface.address_v4, va=self.fqdns[iface.fqdn][0], - ia=self.fqdns[iface.fqdn][1], ib=i, vb=vm_name)) - self.eval_errors += 1 - return - self.addresses[iface.address_v4] = (vm_name, i) - pair = (iface.fqdn, iface.address_v4) - self.dns_mapping['forward'].append(pair) - pair = (iface.address_v4, iface.fqdn) - self.dns_mapping['reverse'].append(pair) - - if iface.address_v6: - if iface.address_v6 in self.addresses: - LOG.error(_( - "IPv6 address {a} already defined for VM {va!r}({ia}) should be set " - "for interface {ib} of {vb!r}.").format( - a=iface.address_v6, va=self.fqdns[iface.fqdn][0], - ia=self.fqdns[iface.fqdn][1], ib=i, vb=vm_name)) - self.eval_errors += 1 - return - self.addresses[iface.address_v6] = (vm_name, i) - pair = (iface.fqdn, iface.address_v6) - self.dns_mapping['forward'].append(pair) - pair = (iface.address_v6, iface.fqdn) - self.dns_mapping['reverse'].append(pair) - - network = iface.network - if network: - if network not in vsphere.networks: - LOG.error(_( - "Could not find network {n!r} for VM {v!r}, interface {i}.").format( - n=network, v=vm_name, i=i)) - self.eval_errors += 1 - return - else: - network = vsphere.networks.get_network_for_ip( - iface.address_v4, iface.address_v6) - if not network: - self.eval_errors += 1 - return - iface.network = network - LOG.debug(_("Found network {n!r} for interface {i} of VM {v!r}.").format( - n=network, i=i, v=vm_name)) - - if network not in vm_cluster.networks: - LOG.error(_( - "Network {n!r} for interface {i} of VM {v!r} not available in " - "cluster {c!r}.").format(n=network, v=vm_name, i=i, c=vm_cluster.name)) - self.eval_errors += 1 - return - LOG.debug(_("Network {n!r} is available in cluster {c!r}.").format( - n=network, c=vm_cluster.name)) - - net = vsphere.networks[network] - if not iface.gateway: - LOG.debug(_("Setting gateway of interface {i} of VM {v!r} to {g}.").format( - i=i, v=vm_name, g=net.gateway)) - iface.gateway = net.gateway - - if net.network: - if net.network.version == 4: - if iface.netmask_v4 is None: - iface.netmask_v4 = net.network.prefixlen - else: - if iface.netmask_v6 is None: - iface.netmask_v6 = net.network.prefixlen - - if vs_name not in self.used_networks: - self.used_networks[vs_name] = [] - if network not in self.used_networks[vs_name]: - self.used_networks[vs_name].append(network) - # -------------------------------------------------------------------------· def get_tf_name_network(self, net_name, *args): diff --git a/lib/cr_tf/handler/vmware.py b/lib/cr_tf/handler/vmware.py new file mode 100644 index 0000000..78bd283 --- /dev/null +++ b/lib/cr_tf/handler/vmware.py @@ -0,0 +1,946 @@ +#!/usr/bin/env pythonV +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@contact: frank.brehm@pixelpark.com +@copyright: © 2023 by Frank Brehm, Berlin +@summary: A mixin module for the handler for methods for interacting with VMware/VSphere. +""" +from __future__ import absolute_import, print_function + +# Standard module +import copy +import logging +import os +import re +import sys + +from pathlib import Path + +from operator import attrgetter + +# Third party modules +from fb_tools.common import pp +from fb_tools.errors import HandlerError, ExpectedHandlerError +from fb_vmware.errors import VSphereExpectedError +from fb_vmware.config import VSPhereConfigInfo +from fb_vmware.connect import VsphereConnection + +# Own modules +from ..errors import AbortExecution + +from ..xlate import XLATOR + +__version__ = '0.1.0' +LOG = logging.getLogger(__name__) + +_ = XLATOR.gettext +ngettext = XLATOR.ngettext + + +# ============================================================================= +class CrTfHandlerVmwMixin(): + """A mixin module for the handler module for interacting with VMware/VSphere..""" + + # -------------------------------------------------------------------------· + def exec_collect_folders(self, yaml_file): + + if self.stop_at_step == 'collect-folders': + self.incr_verbosity() + + LOG.info(_("Collecting all VMWare and local folders ...")) + LOG.info(_("Get vSphere datacenter ...")) + for vname in self.vsphere: + self.vsphere[vname].get_datacenter() + + LOG.debug(_("Collecting vSphere folders.")) + self.vsphere_folders = [] + for vm in self.vms: + if vm.folder: + if vm.folder not in self.vsphere_folders: + self.vsphere_folders.append(vm.folder) + self.vsphere_folders.sort(key=str.lower) + LOG.debug(_("Collected vSphere folders:") + "\n" + pp(self.vsphere_folders)) + + # Set project name and directory + yfile = Path(yaml_file) + yfile_base = yfile.name + yfile_dir = yfile.parent.resolve() + (yfile_stem, yfile_ext) = os.path.splitext(yfile_base) + self.project_name = yfile_stem + LOG.info(_("Project name is {!r}.").format(str(self.project_name))) + self.project_dir = yfile_dir / yfile_stem + LOG.info(_("Project directory is: {!r}.").format(str(self.project_dir))) + + # Evaluating root terraform directory + if not self.is_venv: + i = 4 + cdir = copy.copy(self.project_dir).parent + while i > 0: + git_dir = cdir / '.git' + if git_dir.is_dir(): + self._terraform_root_dir = cdir + break + i -= 1 + if cdir == cdir.parent: + break + cdir = cdir.parent + if not self._terraform_root_dir: + msg = _("Did not found root terraform directory above {!r}.").format( + str(self.project_dir)) + LOG.warn(msg) + + LOG.info(_("Full project name: {!r}").format(self.full_project_name)) + + LOG.info(_("Finished step {!r}.").format('collect-folders')) + if self.stop_at_step == 'collect-folders': + raise AbortExecution('collect-folders') + + # -------------------------------------------------------------------------· + def init_vspheres(self, yaml_file): + + if self.stop_at_step == 'vmw-init': + self.incr_verbosity() + + # Test for multiple VSphere references + found_vspheres = [] + for vm in self.vms: + vname = vm.vsphere + if vname not in found_vspheres: + found_vspheres.append(vname) + if len(found_vspheres) > 1: + yaml_file_rel = os.path.relpath(str(yaml_file), os.getcwd()) + msg = _("There is only one, unique VSPhere definition allowed in a project file.") + msg += '\n' + msg += _("In {f!r} were found {nr} different VSPhere definitions:").format( + f=yaml_file_rel, nr=len(found_vspheres)) + for vname in sorted(found_vspheres, key=str.lower): + msg += '\n * {!r}'.format(vname) + raise ExpectedHandlerError(msg) + + self._init_vspheres() + + LOG.info(_("Finished step {!r}.").format('vmw-init')) + if self.stop_at_step == 'vmw-init': + raise AbortExecution('vmw-init') + + # -------------------------------------------------------------------------· + def _init_vspheres(self): + + for vm in self.vms: + if vm.vsphere in self.vsphere: + continue + vname = vm.vsphere + if vname not in self.config.vsphere: + msg = _("VSPhere {!r} not defined in configuration.").format(vname) + raise ExpectedHandlerError(msg) + + if not self.vsphere_user and self.config.vsphere[vname].user: + self.vsphere_user = self.config.vsphere[vname].user + if not self.vsphere_password and self.config.vsphere[vname].password: + self.vsphere_password = self.config.vsphere[vname].password + + try: + params = { + 'appname': self.appname, + 'verbose': self.verbose, + 'base_dir': self.base_dir, + 'simulate': self.simulate, + 'force': self.force, + 'terminal_has_colors': self.terminal_has_colors, + 'initialized': True, + } + show_params = copy.copy(params) + + connect_info = VSPhereConfigInfo( + appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, + host=self.config.vsphere[vname].host, port=self.config.vsphere[vname].port, + dc=self.config.vsphere[vname].dc, user=self.vsphere_user, + password=self.vsphere_password, initialized=True) + + params['connect_info'] = connect_info + show_params['connect_info'] = connect_info.as_dict() + + if self.verbose > 1: + if self.verbose < 5: + show_params['connect_info']['password'] = '******' + msg = _("Initialising a {}-object with params:").format('VsphereConnection') + msg += '\n' + pp(show_params) + LOG.debug(msg) + + vsphere = VsphereConnection(**params) + self.vsphere[vname] = vsphere + + except VSphereExpectedError as e: + raise ExpectedHandlerError(str(e)) + + # -------------------------------------------------------------------------· + def test_vsphere_handlers(self): + + if self.stop_at_step == 'vmw-test': + self.incr_verbosity() + + for vname in self.vsphere.keys(): + + try: + + vsphere = self.vsphere[vname] + + vsphere.get_about() + if self.verbose > 2: + msg = _("Created {}-object:").format('VsphereConnection') + msg += '\n' + pp(vsphere.as_dict()) + LOG.debug(msg) + + except VSphereExpectedError as e: + raise ExpectedHandlerError(str(e)) + + LOG.info(_("Finished step {!r}.").format('vmw-test')) + if self.stop_at_step == 'vmw-test': + raise AbortExecution('vmw-test') + + # -------------------------------------------------------------------------· + def assign_default_vmw_values(self): + """Assigning not defined templates and clusters of VMs by their + appropriate default values.""" + + LOG.debug(_( + "Assigning not defined templates and clusters of VMs by their " + "appropriate default values.")) + + for vm in self.vms: + + if not vm.cluster: + cl = self.config.vsphere[vm.vsphere].cluster + if self.verbose > 1: + LOG.debug(_("Setting cluster of {n!r} to {c!r} ...").format( + n=vm.name, c=cl)) + vm.cluster = cl + + if not vm.vm_template: + tpl = self.config.vsphere[vm.vsphere].template_name + if self.verbose > 1: + LOG.debug(_("Setting template of {n!r} to {t!r} ...").format( + n=vm.name, t=tpl)) + vm.vm_template = tpl + + # -------------------------------------------------------------------------· + def exec_vmw_clusters(self): + + if self.stop_at_step == 'vmw-clusters': + self.incr_verbosity() + + for vname in self.vsphere: + LOG.debug(_("Searching for clusters in VSPhere {!r} ...").format(vname)) + self.vsphere[vname].get_clusters() + + LOG.info(_("Finished step {!r}.").format('vmw-clusters')) + if self.stop_at_step == 'vmw-clusters': + raise AbortExecution('vmw-clusters') + + # -------------------------------------------------------------------------· + def exec_vmw_datastores(self): + + if self.stop_at_step == 'vmw-datastores': + self.incr_verbosity() + + nr_total = 0 + + for vname in self.vsphere: + LOG.debug(_("Searching for datastores in VSPhere {!r} ...").format(vname)) + self.vsphere[vname].get_datastores() + nr_total += len(self.vsphere[vname].datastores.keys()) + + if nr_total: + msg = ngettext("Found one datastore.", "Found {n} datastores.", nr_total) + LOG.debug(msg.format(n=nr_total)) + else: + LOG.error(_("No VSPhere datastores found.")) + + LOG.info(_("Finished step {!r}.").format('vmw-datastores')) + if self.stop_at_step == 'vmw-datastores': + raise AbortExecution('vmw-datastores') + + # -------------------------------------------------------------------------· + def exec_vmw_ds_clusters(self): + + nr_total = 0 + + if self.stop_at_step == 'vmw-ds-clusters': + self.incr_verbosity() + + for vname in self.vsphere: + LOG.debug(_("Searching for datastore clusters in VSPhere {!r} ...").format(vname)) + self.vsphere[vname].get_ds_clusters() + nr_total += len(self.vsphere[vname].ds_clusters.keys()) + + if nr_total: + msg = ngettext( + "Found one datastore cluster.", + "Found {n} datastore clusters.", + nr_total) + LOG.debug(msg.format(n=nr_total)) + else: + LOG.warn(_("No VSPhere datastore clusters found.")) + + LOG.info(_("Finished step {!r}.").format('vmw-ds-clusters')) + if self.stop_at_step == 'vmw-ds-clusters': + raise AbortExecution('vmw-ds-clusters') + + # -------------------------------------------------------------------------· + def exec_vmw_networks(self): + + if self.stop_at_step == 'vmw-networks': + self.incr_verbosity() + + for vname in self.vsphere: + LOG.debug(_("Searching for networks in VSPhere {!r} ...").format(vname)) + self.vsphere[vname].get_networks() + if self.eval_errors: + msg = ngettext( + "Found one error in exploring vSphere {v!r} resources.", + "Found {n} errors in exploring vSphere {v!r} resources.", + self.eval_errors).format(n=self.eval_errors, v=vname) + raise ExpectedHandlerError(msg) + + LOG.info(_("Finished step {!r}.").format('vmw-networks')) + if self.stop_at_step == 'vmw-networks': + raise AbortExecution('vmw-networks') + + # -------------------------------------------------------------------------· + def exec_vmw_templates(self): + + if self.stop_at_step == 'vmw-templates': + self.incr_verbosity() + + self.explore_vsphere_templates() + if self.eval_errors: + msg = ngettext( + "Found one error in exploring vSphere templates.", + "Found {n} errors in exploring vSphere templates.", + self.eval_errors).format(n=self.eval_errors) + raise ExpectedHandlerError(msg) + + LOG.info(_("Finished step {!r}.").format('vmw-templates')) + if self.stop_at_step == 'vmw-templates': + raise AbortExecution('vmw-templates') + + # -------------------------------------------------------------------------· + def exec_validate_yaml(self): + + if self.stop_at_step == 'validate-yaml': + self.incr_verbosity() + + print() + LOG.info(_("Validating information from YAML file ...")) + + self.validate_clusters() + if self.eval_errors: + msg = ngettext( + "Found one error in validating vSphere computing clusters.", + "Found {n} errors in validating vSphere computing clusters.", + self.eval_errors).format(n=self.eval_errors) + raise ExpectedHandlerError(msg) + + self.get_all_vms() + self.validate_vms() + + LOG.info(_("Finished step {!r}.").format('validate-yaml')) + if self.stop_at_step == 'validate-yaml': + raise AbortExecution('validate-yaml') + + # -------------------------------------------------------------------------· + def get_all_vms(self): + + LOG.info(_("Got a list of all VMs and templates ...")) + self.all_vms = {} + re_vm = re.compile(r'.*') + + for vs_name in self.vsphere: + + if vs_name not in self.all_vms: + self.all_vms[vs_name] = {} + + vm_list = self.vsphere[vs_name].get_vms(re_vm, name_only=True) + for vm_tuple in vm_list: + vm_name = vm_tuple[0] + vm_path = vm_tuple[1] + if vm_name in self.all_vms[vs_name]: + self.all_vms[vs_name][vm_name].append(vm_path) + else: + self.all_vms[vs_name][vm_name] = [vm_path] + + if self.verbose > 2: + msg = _("All existing VMs and templates:") + msg += '\n' + pp(self.all_vms) + LOG.debug(msg) + + # -------------------------------------------------------------------------· + def exec_validate_storage(self): + + if self.stop_at_step == 'validate-storage': + self.incr_verbosity() + + self.validate_storages() + if self.eval_errors: + msg = ngettext( + "Found one error in validating VM storages.", + "Found {n} errors in validating VM storages.", + self.eval_errors).format(n=self.eval_errors) + raise ExpectedHandlerError(msg) + + LOG.info(_("Finished step {!r}.").format('validate-storage')) + if self.stop_at_step == 'validate-storage': + raise AbortExecution('validate-storage') + + # -------------------------------------------------------------------------· + def exec_validate_iface(self): + + if self.stop_at_step == 'validate-iface': + self.incr_verbosity() + + self.validate_interfaces() + if self.eval_errors: + msg = ngettext( + "Found one error in validating VM interfaces.", + "Found {n} errors in validating VM interfaces.", + self.eval_errors).format(n=self.eval_errors) + raise ExpectedHandlerError(msg) + + LOG.info(_("Finished step {!r}.").format('validate-iface')) + if self.stop_at_step == 'validate-iface': + raise AbortExecution('validate-iface') + + # -------------------------------------------------------------------------· + def exec_vsphere_folders(self): + + if self.stop_at_step == 'ensure-vmw-folders': + self.incr_verbosity() + + self.ensure_vsphere_folders() + + LOG.info(_("Finished step {!r}.").format('ensure-vmw-folders')) + if self.stop_at_step == 'ensure-vmw-folders': + raise AbortExecution('ensure-vmw-folders') + + # -------------------------------------------------------------------------· + def explore_vsphere_templates(self): + + LOG.info(_("Exploring all vSphere templates ...")) + + for vname in self.vsphere: + + if vname not in self.vsphere_templates: + self.vsphere_templates[vname] = {} + + self.config.vsphere[vname].used_templates = [] + + for vm in self.vms: + template_name = vm.vm_template + if template_name: + if template_name not in self.config.vsphere[vname].used_templates: + self.config.vsphere[vname].used_templates.append(template_name) + else: + LOG.error(_("VM {!r} has not template defined.").format(vm.name)) + self.eval_errors += 1 + + msg = _("All {} VSPhere templates to explore:").format(vname) + msg += "\n" + pp(self.config.vsphere[vname].used_templates) + LOG.debug(msg) + + for template_name in self.config.vsphere[vname].used_templates: + + if template_name in self.vsphere_templates[vname]: + continue + + LOG.debug(_("Searching for template {t!r} in VSPhere {v!r} ...").format( + t=template_name, v=vname)) + re_vm = re.compile(r'^' + re.escape(template_name) + r'$', re.IGNORECASE) + vm_list = self.vsphere[vname].get_vms(re_vm, as_obj=True, stop_at_found=True) + if vm_list: + vm = vm_list[0] + tname = vm.name.lower() + if tname not in self.vsphere_templates[vname]: + self.vsphere_templates[vname][template_name] = vm + else: + LOG.error(_("Template {t!r} not found in VSPhere {v!r}.").format( + t=template_name, v=vname)) + self.eval_errors += 1 + + if self.verbose > 2: + msg = _("All explored vSphere templates:") + out_dict = {} + for vname in self.vsphere_templates: + out_dict[vname] = {} + for tname in self.vsphere_templates[vname]: + out_dict[vname][tname] = self.vsphere_templates[vname][tname].as_dict() + msg += "\n" + pp(out_dict) + LOG.debug(msg) + + # -------------------------------------------------------------------------· + def validate_clusters(self): + + print() + LOG.info(_("Validating existence of computing clusters of the VMs.")) + + clusters = {} + + for vm in self.vms: + + vname = vm.vsphere + if vname not in clusters: + clusters[vname] = {} + + if vm.cluster in clusters: + clusters[vname][vm.cluster].append(vm.name) + else: + clusters[vname][vm.cluster] = [vm.name] + + for vname in clusters.keys(): + for cluster in clusters[vname].keys(): + + vms = clusters[vname][cluster] + + cl = str(cluster) + LOG.debug(_( + "Checking existence of computing cluster {c!r} in VSPhere {v!r} ...").format( + c=cl, v=vname)) + + vsphere = self.vsphere[vname] + vmw_cluster = vsphere.get_cluster_by_name(cl) + if vmw_cluster: + if self.verbose > 1: + LOG.debug(_( + "Found computing cluster {cl!r} in VSPhere {v!r} (defined for VMs " + "{vms}).").format(cl=vmw_cluster.name, v=vname, vms=pp(vms))) + else: + LOG.error(_( + "Computing cluster {cl!r} (defined for VMs {vms}) in VSPhere {v!r} not " + "found.").format(cl=cl, vms=pp(vms), v=vname)) + self.eval_errors += 1 + + # -------------------------------------------------------------------------· + def validate_vms(self): + + print() + LOG.info(_("Validating existence of VMs in VMWare.")) + vms2perform = [] + + for vm in sorted(self.vms, key=attrgetter('tf_name')): + + print(" * {} ".format(vm.fqdn), end='', flush=True) + if self.verbose: + print() + vs_name = vm.vsphere + vsphere = self.vsphere[vs_name] + + vm_paths = None + if vs_name in self.all_vms: + if vm.fqdn in self.all_vms[vs_name]: + vm_paths = self.all_vms[vs_name][vm.fqdn] + + if vm_paths: + msg = _('[{m}] - VM is already existing in VSphere {v!r}, path {p!r}.').format( + m=self.colored('Existing', 'YELLOW'), v=vs_name, p=pp(vm_paths)) + print(msg, end='', flush=True) + if self.verbose: + print() + + vm_info = vsphere.get_vm(vm.fqdn, vsphere_name=vs_name, as_obj=True) + if self.verbose > 2: + LOG.debug(_("VM info:") + "\n" + pp(vm_info.as_dict(bare=True))) + ds = vm_info.config_path_storage + LOG.debug(_("Datastore of VM {vm!r}: {ds!r}.").format(vm=vm.name, ds=ds)) + vm.datastore = ds + vm.already_existing = True + self.existing_vms.append(vm_info) + + else: + + print('[{}] '.format(self.colored('OK', 'GREEN')), end='', flush=True) + vm.already_existing = False + + vms2perform.append(vm) + print() + + self.vms = vms2perform + + print() + + if not len(self.vms): + print() + print(self.colored('*' * 60, ('BOLD', 'RED')), file=sys.stderr) + print(self.colored('* ' + _('CAUTION!'), ('BOLD', 'RED')), file=sys.stderr) + print(self.colored('*' * 60, ('BOLD', 'RED')), file=sys.stderr) + print() + print( + self.colored(_('Did not found any VM to deploy!'), ('BOLD', 'RED')), + file=sys.stderr) + print() + raise ExpectedHandlerError(_("No VMs to deploy")) + + # -------------------------------------------------------------------------· + def validate_storages(self): + + self._validate_ds_clusters() + self._validate_datastores() + + if self.verbose: + if self.used_dc_clusters: + out_lines = [] + for vs_name in self.used_dc_clusters: + for cluster in self.used_dc_clusters[vs_name]: + out_lines.append(' * VSphere {v!r}: {c}'.format( + v=vs_name, c=cluster)) + out = '\n'.join(out_lines) + LOG.debug(_("Used datastore clusters:") + "\n" + out) + else: + LOG.debug(_("No datastore clusters are used.")) + if self.used_datastores: + out_lines = [] + for vs_name in self.used_datastores: + for ds in self.used_datastores[vs_name]: + out_lines.append(' * VSphere {v!r}: {ds}'.format(v=vs_name, ds=ds)) + out = '\n'.join(out_lines) + LOG.debug(_("Used datastors:") + "\n" + out) + else: + LOG.debug(_("No datastores are used.")) + + # -------------------------------------------------------------------------· + def _validate_ds_clusters(self): + + LOG.info(_("Validating given datastore clusters of VMs ...")) + + for vm in self.vms: + + if not vm.ds_cluster: + continue + + self._validate_dscluster_vm(vm) + + # -------------------------------------------------------------------------· + def _validate_dscluster_vm(self, vm): + + needed_gb = 0.0 + if not vm.already_existing: + for unit_number in vm.disks.keys(): + disk = vm.disks[unit_number] + needed_gb += disk.size_gb + + vs_name = vm.vsphere + vsphere = self.vsphere[vs_name] + + found = False + for cluster_name in vsphere.ds_clusters.keys(): + if cluster_name.lower() == vm.ds_cluster.lower(): + if self.verbose > 2: + LOG.debug(_( + "Found datastore cluster {c!r} in VSphere {v!r} for VM {n!r}.").format( + n=vm.name, v=vs_name, c=vm.ds_cluster)) + if vm.ds_cluster != cluster_name: + LOG.debug(_("Setting datastore cluster for VM {n!r} to {c!r} ...").format( + n=vm.name, c=cluster_name)) + vm.ds_cluster = cluster_name + ds_cluster = vsphere.ds_clusters[cluster_name] + if self.verbose > 2: + LOG.debug(_( + "Free space of cluster {c!r} in VSphere {v!r} before provisioning: " + "{a:0.1f} GiB.").format( + c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb)) + if ds_cluster.avail_space_gb < needed_gb: + LOG.error(_( + "Datastore cluster {d!r} in VSphere {v!r} has not sufficient space for " + "storage of VM {vm!r} (needed {n:0.1f} GiB, available {a:0.1f} " + "GiB).").format( + d=cluster_name, v=vs_name, vm=vm.name, n=needed_gb, + a=ds_cluster.avail_space_gb)) + self.eval_errors += 1 + else: + ds_cluster.calculated_usage += needed_gb + if self.verbose > 1: + LOG.debug(_( + "Free space in cluster {c!r} in VSphere {v!r} after provisioning: " + "{a:0.1f} GiB.").format( + c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb)) + found = True + if vs_name not in self.used_dc_clusters: + self.used_dc_clusters[vs_name] = [] + if cluster_name not in self.used_dc_clusters[vs_name]: + self.used_dc_clusters[vs_name].append(cluster_name) + break + + if not found: + LOG.error(_("Datastore cluster {c!r} of VM {n!r} not found in VSphere {v!r}.").format( + n=vm.name, c=vm.ds_cluster, v=vs_name)) + self.eval_errors += 1 + + # -------------------------------------------------------------------------· + def _validate_datastores(self): + + LOG.info(_("Validating given datastores of VMs and assign failing ...")) + + for vm in self.vms: + + if vm.ds_cluster: + if vm.datastore: + LOG.debug(_("Removing defined datastore {d!r} for VM {n!r} ...").format( + d=vm.datastore, n=vm.name)) + vm.datastore = None + continue + + self._validate_ds_vm(vm) + + # -------------------------------------------------------------------------· + def _validate_ds_vm(self, vm): + + needed_gb = 0.0 + if not vm.already_existing: + for unit_number in vm.disks.keys(): + disk = vm.disks[unit_number] + needed_gb += disk.size_gb + + vs_name = vm.vsphere + vsphere = self.vsphere[vs_name] + + vm_cluster = None + for cluster in vsphere.clusters: + if cluster.name.lower() == vm.cluster.lower(): + vm_cluster = cluster + break + if not vm_cluster: + msg = _("Did not found cluster object {c!r} for VM {n!r}.").format( + c=vm.cluster, n=vm.name) + raise HandlerError(msg) + + if vm.datastore: + found = False + found_ds_name = None + for ds_name in vsphere.datastores: + if ds_name.lower() == vm.datastore.lower(): + if self.verbose > 2: + LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format( + n=vm.name, d=vm.datastore, v=vs_name)) + if ds_name not in vm_cluster.datastores: + LOG.warn(_("Datastore {d!r} not available in cluster {c!r}.").format( + d=ds_name, c=vm.cluster)) + break + if vm.datastore != ds_name: + LOG.debug(_("Setting datastore for VM {n!r} to {d!r} ...").format( + n=vm.name, d=ds_name)) + vm.datastore = ds_name + ds = vsphere.datastores[ds_name] + if ds.avail_space_gb < needed_gb: + LOG.error(_( + "Datastore {d!r} has not sufficient space for storage of VM " + "{v!r} (needed {n:0.1f} GiB, available {a:0.1f} GiB).").format( + d=ds_name, v=vm.name, n=needed_gb, a=ds.avail_space_gb)) + self.eval_errors += 1 + else: + ds.calculated_usage += needed_gb + found = True + found_ds_name = ds_name + break + if not found: + LOG.error(_("Datastore {d!r} of VM {n!r} not found in VSPhere {v!r}.").format( + n=vm.name, d=vm.datastore, v=vs_name)) + self.eval_errors += 1 + if vs_name not in self.used_datastores: + self.used_datastores[vs_name] = [] + if found_ds_name not in self.used_datastores[vs_name]: + self.used_datastores[vs_name].append(found_ds_name) + return + + ds_name = vsphere.datastores.find_ds( + needed_gb, vm.ds_type, use_ds=copy.copy(vm_cluster.datastores), no_k8s=True) + if ds_name: + LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format( + d=ds_name, n=vm.name, v=vs_name)) + vm.datastore = ds_name + if vs_name not in self.used_datastores: + self.used_datastores[vs_name] = [] + if ds_name not in self.used_datastores[vs_name]: + self.used_datastores[vs_name].append(ds_name) + else: + self.eval_errors += 1 + + # -------------------------------------------------------------------------· + def validate_interfaces(self): + + LOG.info(_("Validating interfaces of VMs and assign networks ...")) + for vm in self.vms: + self._validate_interfaces_vm(vm) + + if self.verbose > 2: + LOG.debug(_("Validated FQDNs:") + "\n" + pp(self.fqdns)) + LOG.debug(_("Validated Addresses:") + "\n" + pp(self.addresses)) + + if self.verbose: + + lines = [] + for vs_name in self.used_networks: + for nw in self.used_networks[vs_name]: + lines.append(' * VSphere {v!r}: {n}'.format( + v=vs_name, n=nw)) + out = '\n'.join(lines) + LOG.debug(_("Used networks:") + "\n" + out) + + lines = [] + for pair in self.dns_mapping['forward']: + line = ' * {n!r} => {a!r}'.format(n=pair[0], a=str(pair[1])) + lines.append(line) + LOG.debug(_("Used forward DNS entries:") + "\n" + '\n'.join(lines)) + + lines = [] + for pair in self.dns_mapping['reverse']: + line = ' * {a!r} => {n!r}'.format(n=pair[1], a=str(pair[0])) + lines.append(line) + LOG.debug(_("Used reverse DNS entries:") + "\n" + '\n'.join(lines)) + + # -------------------------------------------------------------------------· + def _validate_interfaces_vm(self, vm): + + vs_name = vm.vsphere + LOG.debug(_("Checking interfaces of VM {n!r} in VSPhere {v!r} ...").format( + n=vm.name, v=vs_name)) + + if not vm.interfaces: + LOG.error(_("No interfaces defined for VM {!r}.").format(vm.name)) + self.eval_errors += 1 + return + + vsphere = self.vsphere[vs_name] + + vm_cluster = None + for cluster in vsphere.clusters: + if cluster.name.lower() == vm.cluster.lower(): + vm_cluster = cluster + break + if not vm_cluster: + msg = _("Did not found cluster object {c!r} for VM {n!r}.").format( + c=vm.cluster, n=vm.name) + raise HandlerError(msg) + + i = -1 + for iface in vm.interfaces: + i += 1 + self._validate_interface_of_vm( + vm_name=vm.name, iface=iface, vs_name=vs_name, vm_cluster=vm_cluster, i=i) + + # -------------------------------------------------------------------------· + def _validate_interface_of_vm(self, vm_name, iface, vs_name, vm_cluster, i=0): + + vsphere = self.vsphere[vs_name] + + if self.verbose > 1: + LOG.debug(_("Checking interface {i} of VM {n!r} ...").format( + i=i, n=vm_name)) + + if not iface.address: + LOG.error(_("Interface {i} of VM {n!r} has no defined address.").format( + i=i, n=vm_name)) + self.eval_errors += 1 + return + + if not iface.fqdn: + LOG.error(_("Interface {i} of VM {n!r} has no defined FQDN.").format( + i=i, n=vm_name)) + self.eval_errors += 1 + return + + if iface.fqdn in self.fqdns: + LOG.error(_( + "FQDN {f!r} already defined for VM {va!r}({ia}) should be set " + "for interface {ib} of {vb!r}.").format( + f=iface.fqdn, va=self.fqdns[iface.fqdn][0], ia=self.fqdns[iface.fqdn][1], + ib=i, vb=vm_name)) + self.eval_errors += 1 + return + + self.fqdns[iface.fqdn] = (vm_name, i) + + if iface.address_v4: + if iface.address_v4 in self.addresses: + LOG.error(_( + "IPv4 address {a} already defined for VM {va!r}({ia}) should be set " + "for interface {ib} of {vb!r}.").format( + a=iface.address_v4, va=self.fqdns[iface.fqdn][0], + ia=self.fqdns[iface.fqdn][1], ib=i, vb=vm_name)) + self.eval_errors += 1 + return + self.addresses[iface.address_v4] = (vm_name, i) + pair = (iface.fqdn, iface.address_v4) + self.dns_mapping['forward'].append(pair) + pair = (iface.address_v4, iface.fqdn) + self.dns_mapping['reverse'].append(pair) + + if iface.address_v6: + if iface.address_v6 in self.addresses: + LOG.error(_( + "IPv6 address {a} already defined for VM {va!r}({ia}) should be set " + "for interface {ib} of {vb!r}.").format( + a=iface.address_v6, va=self.fqdns[iface.fqdn][0], + ia=self.fqdns[iface.fqdn][1], ib=i, vb=vm_name)) + self.eval_errors += 1 + return + self.addresses[iface.address_v6] = (vm_name, i) + pair = (iface.fqdn, iface.address_v6) + self.dns_mapping['forward'].append(pair) + pair = (iface.address_v6, iface.fqdn) + self.dns_mapping['reverse'].append(pair) + + network = iface.network + if network: + if network not in vsphere.networks: + LOG.error(_( + "Could not find network {n!r} for VM {v!r}, interface {i}.").format( + n=network, v=vm_name, i=i)) + self.eval_errors += 1 + return + else: + network = vsphere.networks.get_network_for_ip( + iface.address_v4, iface.address_v6) + if not network: + self.eval_errors += 1 + return + iface.network = network + LOG.debug(_("Found network {n!r} for interface {i} of VM {v!r}.").format( + n=network, i=i, v=vm_name)) + + if network not in vm_cluster.networks: + LOG.error(_( + "Network {n!r} for interface {i} of VM {v!r} not available in " + "cluster {c!r}.").format(n=network, v=vm_name, i=i, c=vm_cluster.name)) + self.eval_errors += 1 + return + LOG.debug(_("Network {n!r} is available in cluster {c!r}.").format( + n=network, c=vm_cluster.name)) + + net = vsphere.networks[network] + if not iface.gateway: + LOG.debug(_("Setting gateway of interface {i} of VM {v!r} to {g}.").format( + i=i, v=vm_name, g=net.gateway)) + iface.gateway = net.gateway + + if net.network: + if net.network.version == 4: + if iface.netmask_v4 is None: + iface.netmask_v4 = net.network.prefixlen + else: + if iface.netmask_v6 is None: + iface.netmask_v6 = net.network.prefixlen + + if vs_name not in self.used_networks: + self.used_networks[vs_name] = [] + if network not in self.used_networks[vs_name]: + self.used_networks[vs_name].append(network) + + +# ============================================================================= + +if __name__ == "__main__": + + pass + +# ============================================================================= + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 list -- 2.39.5