From: Frank Brehm Date: Mon, 23 Nov 2020 17:21:10 +0000 (+0100) Subject: Mangling source entries after reading X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=f91c09a136175fa1148f176fdaedf7d217574f14;p=pixelpark%2Fldap-migration.git Mangling source entries after reading --- diff --git a/lib/ldap_migration/__init__.py b/lib/ldap_migration/__init__.py index c5ca22a..56b126a 100644 --- a/lib/ldap_migration/__init__.py +++ b/lib/ldap_migration/__init__.py @@ -17,6 +17,8 @@ import time import re import argparse +from numbers import Number + # 3rd party modules from ldap3 import Server, Connection, ALL, DSA, IP_V4_PREFERRED, SAFE_SYNC @@ -29,7 +31,7 @@ from ldap3.core.exceptions import LDAPException # Own modules from fb_tools.colored import ColoredFormatter from fb_tools.errors import IoTimeoutError -from fb_tools.common import pp, is_sequence +from fb_tools.common import pp, is_sequence, human2mbytes from fb_tools.app import BaseApplication, DirectoryOptionAction from fb_tools.config import CfgFileOptionAction from fb_tools.errors import FbAppError @@ -38,7 +40,7 @@ from .config import LDAPMigrationConfiguration from .idict import CaseInsensitiveDict from .istringset import CaseInsensitiveStringSet -__version__ = '0.6.7' +__version__ = '0.6.8' LOG = logging.getLogger(__name__) CFG_BASENAME = 'ldap-migration.ini' @@ -55,6 +57,12 @@ class CommonLDAPMigrationError(FbAppError, LDAPException): pass +# ============================================================================= +class ReadLDAPItemError(CommonLDAPMigrationError): + """Error class in cae, a LDAP item could not be read.""" + pass + + # ============================================================================= class NonNegativeItegerOptionAction(argparse.Action): @@ -496,6 +504,99 @@ class LDAPMigrationApplication(BaseApplication): LOG.info("Disconnecting from source server {!r} ...".format(self.config.src_server)) self.src_server = None + # ------------------------------------------------------------------------- + def get_source_item(self, src_dn, tgt_dn, with_acl=False): + """Reading a single LDAP item.""" + + # ReadLDAPItemError + src_entry = None + + sfilter = '(objectClass=*)' + src_attrs = [ALL_ATTRIBUTES] + if with_acl: + src_attrs = ['aci', ALL_ATTRIBUTES] + + if self.verbose > 2: + msg = "Trying to get source LDAP item {!r} ...".format(tgt_dn) + LOG.debug(msg) + + src_status, src_result, src_response, _ = self.source.search( + search_base=tgt_dn, search_scope=BASE, search_filter=sfilter, + attributes=src_attrs, time_limit=self.config.timeout) + + if not src_status: + msg = "Error retrieving source LDAP item {dn!r}: {res}".format( + dn=tgt_dn, res=src_result) + raise ReadLDAPItemError(msg) + + src_entry = src_response[0] + + if self.verbose > 3: + LOG.debug("Result of searching for source DN {dn!r}:\n{res}".format( + dn=tgt_dn, res=pp(src_result))) + if self.verbose > 4: + LOG.debug("Response of searching for source DN {dn!r}:\n{res}".format( + dn=tgt_dn, res=pp(src_entry))) + + entry = { + 'attributes': {}, + 'dn': src_entry['dn'], + 'raw_attributes': copy.copy(src_entry['raw_attributes']), + 'raw_dn': src_entry['raw_dn'], + 'type':src_entry['type'], + } + + for attribute in src_entry['attributes'].keys(): + key = self.attribute_types.get_key(attribute, strict=False) + if not key: + if self.verbose > 1: + msg = ( + "Attribute {attr!r} of item {dn!r} not found in the list of available " + "attribute types.").format(attr=attribute, dn=src_dn) + LOG.warn(msg) + continue + + src_val = src_entry['attributes'][attribute] + val = 0 + + if attribute in self.integer_attribute_types: + + if is_sequence(src_val): + val = [] + for value in src_val: + if isinstance(value, Number): + val.append(value) + continue + try: + mbytes = human2mbytes(value, as_float=True) + val.append(int(mbytes * 1024 * 1024)) + except ValueError as e: + msg = "Invalid value in attribute {attr!r} of item {dn!r}: {e}" + msg = msg.format(attr=attribute, dn=src_dn, e=e) + raise ReadLDAPItemError(msg) + else: + if isinstance(src_val, Number): + val = src_val + try: + mbytes = human2mbytes(src_val, as_float=True) + val = int(mbytes * 1024 * 1024) + except ValueError as e: + msg = "Invalid value in attribute {attr!r} of item {dn!r}: {e}" + msg = msg.format(attr=attribute, dn=src_dn, e=e) + raise ReadLDAPItemError(msg) + + if self.verbose > 1: + msg = "Migrated integer value: {old!r} => {new!r}.".format( + old=src_val, new=val) + LOG.warn(msg) + + else: + val = copy.copy(src_val) + + entry['attributes'][key] = val + + return entry + # ------------------------------------------------------------------------- def discover_target_schema(self): @@ -844,9 +945,11 @@ class LDAPMigrationApplication(BaseApplication): print() LOG.info("Migrating all entries from source to target LDAP cluster.") - self.migrate_structural_entries() + if not self.migrate_structural_entries(): + return False print() + return True # ------------------------------------------------------------------------- def migrate_structural_entries(self): @@ -858,7 +961,12 @@ class LDAPMigrationApplication(BaseApplication): self.count_added = 0 self.count_modified = 0 - self._migrate_entries(self.struct_dns, is_root=True, with_acl=False) + try: + self._migrate_entries(self.struct_dns, is_root=True, with_acl=False) + except ReadLDAPItemError as e: + msg = "Abort migration: " + str(e) + LOG.error(msg) + return False print() total = self.count_unchanged + self.count_added + self.count_modified @@ -868,6 +976,7 @@ class LDAPMigrationApplication(BaseApplication): to=total, ad=self.count_added, mo=self.count_modified, un=self.count_unchanged) LOG.info(msg) + return True # ------------------------------------------------------------------------- def generate_target_entry(self, src_entry, src_dn, tgt_dn): @@ -994,82 +1103,60 @@ class LDAPMigrationApplication(BaseApplication): src_entry = None tgt_entry = None - sfilter = '(objectClass=*)' - src_attrs = [ALL_ATTRIBUTES] - if with_acl: - src_attrs = ['aci', ALL_ATTRIBUTES] - tgt_attrs = [ALL_ATTRIBUTES] tgt_dn = self.mangle_dn(src_dn) + src_entry = self.get_source_item(src_dn, tgt_dn, with_acl=with_acl) + + tgt_attrs = [ALL_ATTRIBUTES] - src_status, src_result, src_response, _ = self.source.search( + if self.verbose > 1: + LOG.debug("Searching for target DN {dn!r}.".format(dn=tgt_dn)) + tgt_status, tgt_result, tgt_response, _ = self.target.search( search_base=tgt_dn, search_scope=BASE, search_filter=sfilter, - get_operational_attributes=True, attributes=src_attrs, + get_operational_attributes=with_acl, attributes=tgt_attrs, time_limit=self.config.timeout) - if src_status: - - src_entry = src_response[0] - + target_entry = None + if tgt_status: + target_entry = tgt_response[0] if self.verbose > 2: - LOG.debug("Result of searching for source DN {dn!r}:\n{res}".format( - dn=src_dn, res=pp(src_result))) + LOG.debug("Result of searching for target DN {dn!r}:\n{res}".format( + dn=tgt_dn, res=pp(tgt_result))) if self.verbose > 2: - LOG.debug("Response of searching for source DN {dn!r}:\n{res}".format( - dn=src_dn, res=pp(src_entry))) - - if self.verbose > 1: - LOG.debug("Searching for target DN {dn!r}.".format(dn=tgt_dn)) - tgt_status, tgt_result, tgt_response, _ = self.target.search( - search_base=tgt_dn, search_scope=BASE, search_filter=sfilter, - get_operational_attributes=with_acl, attributes=tgt_attrs, - time_limit=self.config.timeout) - - target_entry = None - if tgt_status: - target_entry = tgt_response[0] - if self.verbose > 2: - LOG.debug("Result of searching for target DN {dn!r}:\n{res}".format( - dn=tgt_dn, res=pp(tgt_result))) - if self.verbose > 2: - LOG.debug("Response of searching for target DN {dn!r}:\n{res}".format( - dn=tgt_dn, res=pp(target_entry))) - changes = self.generate_modify_data(src_entry, target_entry, src_dn, tgt_dn) - if changes: - if self.verbose: - LOG.info("Updating target entry {!r} ...".format(tgt_dn)) - if self.verbose > 2: - msg = "Changes on target entry {tdn!r}:\n{ch}".format( - tdn=tgt_dn, ch=pp(changes)) - LOG.debug(msg) - self.count_modified += 1 - if not self.simulate: - self.target.modify(tgt_dn, changes) - if wait: - time.sleep(wait) - - else: - if self.verbose > 2: - LOG.debug("Target DN {dn!r} not found.".format(dn=tgt_dn)) - (tgt_obj_classes, tgt_entry) = self.generate_target_entry(src_entry, src_dn, tgt_dn) + LOG.debug("Response of searching for target DN {dn!r}:\n{res}".format( + dn=tgt_dn, res=pp(target_entry))) + changes = self.generate_modify_data(src_entry, target_entry, src_dn, tgt_dn) + if changes: if self.verbose: - LOG.info("Creating target entry {!r} ...".format(tgt_dn)) + LOG.info("Updating target entry {!r} ...".format(tgt_dn)) if self.verbose > 2: - msg = "Generated entry for target DN {dn!r}:\n" - msg += "object classes: {oc}\n" - msg += "entry: {en}" - msg = msg.format(dn=tgt_dn, oc=tgt_obj_classes, en=tgt_entry) + msg = "Changes on target entry {tdn!r}:\n{ch}".format( + tdn=tgt_dn, ch=pp(changes)) LOG.debug(msg) - self.count_added += 1 + self.count_modified += 1 if not self.simulate: - self.target.add(tgt_dn, object_class=tgt_obj_classes, attributes=tgt_entry) + self.target.modify(tgt_dn, changes) if wait: time.sleep(wait) else: - msg = "Did not found source entry with DN {!r} (WTF?).".format(src_dn) - LOG.error(msg) + if self.verbose > 2: + LOG.debug("Target DN {dn!r} not found.".format(dn=tgt_dn)) + (tgt_obj_classes, tgt_entry) = self.generate_target_entry(src_entry, src_dn, tgt_dn) + if self.verbose: + LOG.info("Creating target entry {!r} ...".format(tgt_dn)) + if self.verbose > 2: + msg = "Generated entry for target DN {dn!r}:\n" + msg += "object classes: {oc}\n" + msg += "entry: {en}" + msg = msg.format(dn=tgt_dn, oc=tgt_obj_classes, en=tgt_entry) + LOG.debug(msg) + self.count_added += 1 + if not self.simulate: + self.target.add(tgt_dn, object_class=tgt_obj_classes, attributes=tgt_entry) + if wait: + time.sleep(wait) for key in cur_hash['childs'].keys(): self._migrate_entries(cur_hash['childs'][key], is_root=False, with_acl=with_acl)