From: Frank Brehm Date: Wed, 18 Nov 2020 16:40:25 +0000 (+0100) Subject: First migration of structural entries successful X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=04748166dfe995d283ac35cbec19236214a20756;p=pixelpark%2Fldap-migration.git First migration of structural entries successful --- diff --git a/lib/ldap_migration/__init__.py b/lib/ldap_migration/__init__.py index 108f2b8..75f7a81 100644 --- a/lib/ldap_migration/__init__.py +++ b/lib/ldap_migration/__init__.py @@ -37,7 +37,7 @@ from fb_tools.errors import FbAppError from .config import LDAPMigrationConfiguration from .idict import CaseInsensitiveDict -__version__ = '0.6.2' +__version__ = '0.6.3' LOG = logging.getLogger(__name__) CFG_BASENAME = 'ldap-migration.ini' @@ -68,7 +68,7 @@ class NonNegativeItegerOptionAction(argparse.Action): # ============================================================================= -class LimitedItegerOptionAction(argparse.Action): +class LimitedIntegerOptionAction(argparse.Action): # ------------------------------------------------------------------------- def __init__(self, option_strings, min_val=None, max_val=None, *args, **kwargs): @@ -76,7 +76,7 @@ class LimitedItegerOptionAction(argparse.Action): self._min_val = min_val self._max_val = max_val - super(LimitedItegerOptionAction, self).__init__( + super(LimitedIntegerOptionAction, self).__init__( option_strings=option_strings, *args, **kwargs) # ------------------------------------------------------------------------- @@ -105,6 +105,44 @@ class LimitedItegerOptionAction(argparse.Action): setattr(namespace, self.dest, val) +# ============================================================================= +class LimitedFloatOptionAction(argparse.Action): + + # ------------------------------------------------------------------------- + def __init__(self, option_strings, min_val=None, max_val=None, *args, **kwargs): + + self._min_val = min_val + self._max_val = max_val + + super(LimitedFloatOptionAction, self).__init__( + option_strings=option_strings, *args, **kwargs) + + # ------------------------------------------------------------------------- + def __call__(self, parser, namespace, value, option_string=None): + + val = 0 + try: + val = float(value) + except Exception as e: + msg = "Got a {c} for converting {v!r} into a float value: {e}".format( + c=e.__class__.__name__, v=value, e=e) + raise argparse.ArgumentError(self, msg) + + if self._min_val is not None: + if val < self._min_val: + msg = "The option must be greater or equal to {m} (given: {v}).".format( + m=self._min_val, v=val) + raise argparse.ArgumentError(self, msg) + + if self._max_val is not None: + if val > self._max_val: + msg = "The option must be less or equal to {m} (given: {v}).".format( + m=self._max_val, v=val) + raise argparse.ArgumentError(self, msg) + + setattr(namespace, self.dest, val) + + # ============================================================================= class LDAPMigrationApplication(BaseApplication): """ @@ -199,12 +237,21 @@ class LDAPMigrationApplication(BaseApplication): app_group.add_argument( '-T', '--timeout', dest='timeout', type=int, metavar='SECONDS', - action=LimitedItegerOptionAction, + action=LimitedIntegerOptionAction, min_val=1, max_val=LDAPMigrationConfiguration.max_timeout, help="The timeout in seconds for LDAP operations (default: {}).".format( LDAPMigrationConfiguration.default_timeout), ) + app_group.add_argument( + '-W', '--wait', '--wait-affter-write', dest='wait', type=float, metavar='SECONDS', + action=LimitedFloatOptionAction, min_val=0, + help=( + "Number of seconds to wait after each write operation. Given as a " + "float value, and if set to zero, there is no waiting after a write. " + "(default: {:.1f})").format(LDAPMigrationConfiguration.default_wait_after_write), + ) + app_group.add_argument( '-L', '--limit', dest='limit', type=int, metavar='NUMBER', action=NonNegativeItegerOptionAction, @@ -325,17 +372,26 @@ class LDAPMigrationApplication(BaseApplication): self.arg_parser.print_usage(sys.stdout) self.exit(1) - if self.args.limit: - if self.simulate: - self.limit = self.args.limit - else: - LOG.warn( - "Limiting the number of entries to migrate is valid only in " - "siulation mode.") + if self.args.wait is not None: + try: + self.config.wait_after_write = self.args.wait + except (ValueError, KeyError) as e: + msg = "Invalid value {!r} for wait-fafter-write:".format(self.args.wait) + ' ' + str(e) + LOG.error(msg) print() self.arg_parser.print_usage(sys.stdout) self.exit(1) + if self.args.limit: + self.limit = self.args.limit + print() + LOG.warn( + "Limiting the number of entries for migration to {} entries.".format(self.limit)) + if not self.simulate: + print() + LOG.warn("Limition should only be done in simulation mode.") + print() + self.initialized = True # ------------------------------------------------------------------------- @@ -765,13 +821,15 @@ class LDAPMigrationApplication(BaseApplication): print() LOG.info("Migrating all entries from source to target LDAP cluster.") - print() self.migrate_structural_entries() + print() + # ------------------------------------------------------------------------- def migrate_structural_entries(self): + print() LOG.info("Migrating all structural from source to target LDAP cluster.") self._migrate_entries(self.struct_dns, is_root=True, with_acl=False) @@ -881,19 +939,18 @@ class LDAPMigrationApplication(BaseApplication): changes[tgt_at_name] = [(MODIFY_ADD, src_value)] if changes.keys(): - if self.verbose > 2: - msg = "Changes on target entry {tdn!r}:\n{ch}".format(tdn=tgt_dn, ch=pp(changes)) - LOG.debug(msg) return changes - if self.verbose > 2: + if self.verbose: msg = "No changes on target entry {tdn!r} necessary.".format(tdn=tgt_dn) - LOG.debug(msg) + LOG.info(msg) return None # ------------------------------------------------------------------------- def _migrate_entries(self, cur_hash, is_root=False, with_acl=False): + wait = self.config.wait_after_write + if not is_root: src_dn = cur_hash['dn'] @@ -942,17 +999,34 @@ class LDAPMigrationApplication(BaseApplication): LOG.debug("Response of searching for target DN {dn!r}:\n{res}".format( dn=tgt_dn, res=pp(target_entry))) changes = self.generate_modify_data(src_entry, target_entry, src_dn, tgt_dn) + if changes: + if self.verbose: + LOG.info("Updating target entry {!r} ...".format(tgt_dn)) + if self.verbose > 2: + msg = "Changes on target entry {tdn!r}:\n{ch}".format( + tdn=tgt_dn, ch=pp(changes)) + LOG.debug(msg) + if not self.simulate: + self.target.modify(tgt_dn, changes) + if wait: + time.sleep(wait) else: if self.verbose > 2: LOG.debug("Target DN {dn!r} not found.".format(dn=tgt_dn)) (tgt_obj_classes, tgt_entry) = self.generate_target_entry(src_entry, src_dn, tgt_dn) + if self.verbose: + LOG.info("Creating target entry {!r} ...".format(tgt_dn)) if self.verbose > 2: msg = "Generated entry for target DN {dn!r}:\n" msg += "object classes: {oc}\n" msg += "entry: {en}" msg = msg.format(dn=tgt_dn, oc=tgt_obj_classes, en=tgt_entry) LOG.debug(msg) + if not self.simulate: + self.target.add(tgt_dn, object_class=tgt_obj_classes, attributes=tgt_entry) + if wait: + time.sleep(wait) else: msg = "Did not found source entry with DN {!r} (WTF?).".format(src_dn) @@ -965,9 +1039,17 @@ class LDAPMigrationApplication(BaseApplication): def compare_values(self, first, second): if is_sequence(first) and not is_sequence(second): + if len(first) == 1: + value = first[0] + if value.lower() == second.lower(): + return True return False if is_sequence(second) and not is_sequence(first): + if len(second) == 1: + value = second[0] + if value.lower() == first.lower(): + return True return False if not is_sequence(first): diff --git a/lib/ldap_migration/config.py b/lib/ldap_migration/config.py index fbaf235..2014ef7 100644 --- a/lib/ldap_migration/config.py +++ b/lib/ldap_migration/config.py @@ -21,7 +21,7 @@ from fb_tools.common import to_bool from fb_tools.config import ConfigError, BaseConfiguration -__version__ = '0.2.2' +__version__ = '0.3.1' LOG = logging.getLogger(__name__) @@ -56,6 +56,7 @@ class LDAPMigrationConfiguration(BaseConfiguration): max_tcp_port = (2**16 - 1) default_xlations_definitions_base = 'xlations.yaml' max_timeout = 3600 + default_wait_after_write = 0.1 re_bind_dn = re.compile(r'^\s*bind[_-]?dn\s*$', re.IGNORECASE) re_bind_pw = re.compile(r'^\s*bind[_-]?(?:pw|passwd|passsword)\s*$', re.IGNORECASE) @@ -83,6 +84,7 @@ class LDAPMigrationConfiguration(BaseConfiguration): self.xlations_definitions_base = self.default_xlations_definitions_base self.xlations_definitions_file = None self.xlations_definitions = {} + self._wait_after_write = self.default_wait_after_write super(LDAPMigrationConfiguration, self).__init__( appname=appname, verbose=verbose, version=version, base_dir=base_dir, @@ -152,6 +154,20 @@ class LDAPMigrationConfiguration(BaseConfiguration): raise ValueError(msg) self._timeout = val + # ------------------------------------------------------------------------- + @property + def wait_after_write(self): + """The TCP port number of the source LDAP host.""" + return self._wait_after_write + + @wait_after_write.setter + def wait_after_write(self, value): + val = float(value) + err_msg = "Invalid duration {} seconds for waiting after each write operation." + if val <= 0: + raise ValueError(err_msg.format(val)) + self._wait_after_write = val + # ------------------------------------------------------------------------- def as_dict(self, short=True): """ @@ -183,6 +199,7 @@ class LDAPMigrationConfiguration(BaseConfiguration): res['timeout'] = self.timeout res['src_port'] = self.src_port res['tgt_port'] = self.tgt_port + res['wait_after_write'] = self.wait_after_write return res @@ -304,6 +321,8 @@ class LDAPMigrationConfiguration(BaseConfiguration): if self.verbose > 1: LOG.debug("Checking config section {!r} ...".format(section_name)) + re_wait = re.compile(r'^\s*wait[_-]?after[_-]?write\s*$', re.IGNORECASE) + for (key, value) in config.items(section_name): if key.lower() == 'suffix' and value.strip(): @@ -318,6 +337,14 @@ class LDAPMigrationConfiguration(BaseConfiguration): LOG.error(msg) continue + if re_wait.match(key): + try: + self.wait_after_write = value + except (ValueError, KeyError) as e: + msg = "Invalid value {!r} for wait_after_write:".format(value) + ' ' + str(e) + LOG.error(msg) + continue + LOG.warning(( "Unknown configuration option {o!r} with value {v!r} in " "section {s!r}.").format(o=key, v=value, s=section_name))