From: Frank Brehm Date: Mon, 14 Dec 2020 17:05:19 +0000 (+0100) Subject: Cleaning up attributes X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=b8af299828e0dc694dd55e1903bb949d266e19ec;p=pixelpark%2Fldap-migration.git Cleaning up attributes --- diff --git a/lib/ldap_migration/__init__.py b/lib/ldap_migration/__init__.py index eb4f6c6..4392b78 100644 --- a/lib/ldap_migration/__init__.py +++ b/lib/ldap_migration/__init__.py @@ -22,6 +22,7 @@ from numbers import Number # 3rd party modules +import six import pytz from tzlocal import get_localzone @@ -37,7 +38,7 @@ from ldap3.core.exceptions import LDAPException # Own modules from fb_tools.colored import ColoredFormatter from fb_tools.errors import IoTimeoutError -from fb_tools.common import pp, is_sequence, human2mbytes, to_bool +from fb_tools.common import pp, is_sequence, human2mbytes, to_bool, compare_ldap_values from fb_tools.app import BaseApplication, DirectoryOptionAction from fb_tools.config import CfgFileOptionAction from fb_tools.errors import FbAppError @@ -46,7 +47,7 @@ from .config import LDAPMigrationConfiguration from .idict import CaseInsensitiveDict from .istringset import CaseInsensitiveStringSet -__version__ = '0.8.11' +__version__ = '0.8.12' LOG = logging.getLogger(__name__) CFG_BASENAME = 'ldap-migration.ini' @@ -219,6 +220,11 @@ class LDAPMigrationApplication(BaseApplication): self.migrated_entries = CaseInsensitiveDict() self.integer_attribute_types = CaseInsensitiveStringSet([]) self.src_items_not_found = CaseInsensitiveStringSet([]) + self.unknown_attributetypes = CaseInsensitiveStringSet([]) + self.unknown_objectclasses = CaseInsensitiveStringSet([]) + self.dn_attr_types = CaseInsensitiveStringSet([]) + self.pure_binary_attr_types = CaseInsensitiveStringSet([]) + super(LDAPMigrationApplication, self).__init__( appname=appname, verbose=verbose, version=version, base_dir=base_dir, @@ -271,6 +277,8 @@ class LDAPMigrationApplication(BaseApplication): res['struct_dns'] = self.struct_dns.as_dict(short=short) res['group_entries'] = self.group_entries.as_dict(short=short) res['integer_attribute_types'] = self.integer_attribute_types.as_list() + res['pure_binary_attr_types'] = self.pure_binary_attr_types.as_list() + res['dn_attr_types'] = self.dn_attr_types.as_list() return res @@ -643,6 +651,7 @@ class LDAPMigrationApplication(BaseApplication): "Attribute {attr!r} of item {dn!r} not found in the list of available " "attribute types.").format(attr=attribute, dn=src_dn) LOG.warn(msg) + self.unknown_attributetypes.add(attribute) continue src_val = src_entry['attributes'][attribute] @@ -651,10 +660,19 @@ class LDAPMigrationApplication(BaseApplication): if attribute in self.integer_attribute_types: val = self._get_integer_value(src_val) - if self.verbose > 1: + if self.verbose > 2: msg = "Migrated integer value: {old!r} => {new!r}.".format( old=src_val, new=val) - LOG.warn(msg) + LOG.debug(msg) + + elif attribute in self.dn_attr_types: + val = [] + if is_sequence(src_val): + for old_val in src_val: + new_val = self.mangle_dn(old_val) + val.append(new_val) + else: + val = self.mangle_dn(src_val) else: val = copy.copy(src_val) @@ -759,6 +777,8 @@ class LDAPMigrationApplication(BaseApplication): self.discover_target_object_classes() self.discover_target_attribute_types() self.discover_integer_attribute_types() + self.discover_dn_attribute_types() + self.discover_pure_binary_attribute_types() # ------------------------------------------------------------------------- def discover_target_object_classes(self): @@ -899,6 +919,36 @@ class LDAPMigrationApplication(BaseApplication): LOG.debug("Discovered Integer AttributeTypes:\n" + pp( self.integer_attribute_types.as_list())) + # ------------------------------------------------------------------------- + def discover_dn_attribute_types(self): + + for key in self.attribute_types.keys(): + attribute_type = self.attribute_types[key] + if not attribute_type['syntax']: + continue + if attribute_type['syntax'] == '1.3.6.1.4.1.1466.115.121.1.12': + self.dn_attr_types.add(key) + + if self.verbose > 2: + LOG.debug("Discovered DN AttributeTypes:\n" + pp( + self.dn_attr_types.as_list())) + + # ------------------------------------------------------------------------- + def discover_pure_binary_attribute_types(self): + + for key in self.attribute_types.keys(): + attribute_type = self.attribute_types[key] + if not attribute_type['syntax']: + continue + if attribute_type['syntax'] in ( + '1.3.6.1.4.1.1466.115.121.1.5', + '1.3.6.1.4.1.1466.115.121.1.28'): + self.pure_binary_attr_types.add(key) + + if self.verbose > 2: + LOG.debug("Discovered Pure Binary AttributeTypes:\n" + pp( + self.pure_binary_attr_types.as_list())) + # ------------------------------------------------------------------------- def check_log_dir(self): """Checking existence of logging-dir and creating it, if necessary.""" @@ -1248,6 +1298,7 @@ class LDAPMigrationApplication(BaseApplication): msg += "on target LDAP server." msg = msg.format(oc=src_oc_name, dn=src_dn) LOG.debug(msg) + self.unknown_objectclasses.add(src_oc_name) continue tgt_oc_name = self.object_classes.get_key(src_oc_name) used_classes.add(tgt_oc_name) @@ -1332,7 +1383,12 @@ class LDAPMigrationApplication(BaseApplication): do_replace = False if tgt_at_name in tgt_attributes: cur_tgt_value = tgt_attributes[tgt_at_name] - if self.compare_values(src_value, cur_tgt_value): + both_equal = False + if tgt_at_name in self.pure_binary_attr_types: + both_equal = self.compare_binary_values(src_value, cur_tgt_value) + else: + both_equal = compare_ldap_values(src_value, cur_tgt_value) + if both_equal: if self.verbose > 3: msg = ( "Attribute {atr!r} of source DN {sdn!r} is equal to " @@ -1434,11 +1490,24 @@ class LDAPMigrationApplication(BaseApplication): if tgt_entry: - changes = self.generate_modify_data(src_entry, tgt_entry, src_dn, tgt_dn) + try: + changes = self.generate_modify_data(src_entry, tgt_entry, src_dn, tgt_dn) + except UnicodeDecodeError as e: + msg = "Source attributes:\n{}".format(pp(src_entry['attributes'])) + LOG.debug(msg) + msg = "Current target attributes:\n{}".format(pp(tgt_entry['attributes'])) + LOG.debug(msg) + msg = "UnicodeDecodeError on generating changes for source DN {s!r}: {e}".format( + s=src_dn, e=e) + raise WriteLDAPItemError(msg) if changes: if self.verbose: LOG.info("Updating target entry {!r} ...".format(tgt_dn)) if self.verbose > 2: + msg = "Source attributes:\n{}".format(pp(src_entry['attributes'])) + LOG.debug(msg) + msg = "Current target attributes:\n{}".format(pp(tgt_entry['attributes'])) + LOG.debug(msg) msg = "Changes on target entry {tdn!r}:\n{ch}".format( tdn=tgt_dn, ch=pp(changes)) LOG.debug(msg) @@ -1504,40 +1573,38 @@ class LDAPMigrationApplication(BaseApplication): print(line, file=fh, flush=True) # ------------------------------------------------------------------------- - def compare_values(self, first, second): + def compare_binary_values(self, first, second): if is_sequence(first) and not is_sequence(second): if len(first) == 1: - value = first[0] - if str(value).lower() == str(second).lower(): + if first[0] == second: return True return False if is_sequence(second) and not is_sequence(first): if len(second) == 1: - value = second[0] - if str(value).lower() == str(first).lower(): + if first == second[0]: return True return False if not is_sequence(first): # second is also not an array at this point - if str(first).lower() == str(second).lower(): + if first == second: return True return False # Both parameters are arays first_array = [] - for val in first: - first_array.append(str(val).lower()) - first_array.sort() + for val in sorted(first): + first_array.append(val) + second_array = [] - for val in second: - second_array.append(str(val).lower()) - second_array.sort() + for val in sorted(second): + second_array.append(val) if first_array == second_array: return True + return False # ------------------------------------------------------------------------- @@ -1565,6 +1632,19 @@ class LDAPMigrationApplication(BaseApplication): print(self.colored('###################', 'AQUA')) for dn in self.src_items_not_found: print(self.colored(' * {!r}'.format(dn), 'AQUA')) + if self.verbose: + if len(self.unknown_attributetypes): + print() + print(self.colored('Unknown and not migrated Attribute Types:', 'AQUA')) + print(self.colored('#########################################', 'AQUA')) + for at in self.unknown_attributetypes: + print(self.colored(' * {!r}'.format(at), 'AQUA')) + if len(self.unknown_objectclasses): + print() + print(self.colored('Unknown and not migrated Object Classes:', 'AQUA')) + print(self.colored('########################################', 'AQUA')) + for oc in self.unknown_objectclasses: + print(self.colored(' * {!r}'.format(oc), 'AQUA')) finally: self.disconnect()