From: Frank Brehm Date: Fri, 14 Jul 2017 12:53:45 +0000 (+0200) Subject: Adding evaluation of configuration in pp_lib/import_pdnsdata.py X-Git-Tag: 0.1.2~161 X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=377bc8bd179095c8c41fe4555fe2ca55df8b1247;p=pixelpark%2Fadmin-tools.git Adding evaluation of configuration in pp_lib/import_pdnsdata.py --- diff --git a/pp_lib/cfg_app.py b/pp_lib/cfg_app.py index 6d357ca..b70aedd 100644 --- a/pp_lib/cfg_app.py +++ b/pp_lib/cfg_app.py @@ -49,7 +49,7 @@ from .mailaddress import MailAddress from .app import PpApplication -__version__ = '0.6.2' +__version__ = '0.6.3' LOG = logging.getLogger(__name__) VALID_MAIL_METHODS = ('smtp', 'sendmail') @@ -113,6 +113,7 @@ class PpConfigApplication(PpApplication): self.mail_method = 'smtp' self.mail_server = self.default_mail_server self.smtp_port = 25 + self._config_has_errors = None super(PpConfigApplication, self).__init__( appname=appname, verbose=verbose, version=version, base_dir=base_dir, @@ -188,6 +189,19 @@ class PpConfigApplication(PpApplication): else: self._cfg_encoding = codec.name + # ----------------------------------------------------------- + @property + def config_has_errors(self): + """A flag, showing, that there are errors in configuration.""" + return self._config_has_errors + + @config_has_errors.setter + def config_has_errors(self, value): + if value is None: + self._config_has_errors = None + else: + self._config_has_errors = to_bool(value) + # ----------------------------------------------------------- @property def cfg_dir(self): @@ -210,6 +224,7 @@ class PpConfigApplication(PpApplication): res['need_config_file'] = self.need_config_file res['cfg_encoding'] = self.cfg_encoding res['cfg_dir'] = self.cfg_dir + res['config_has_errors'] = self.config_has_errors return res @@ -468,6 +483,7 @@ class PpConfigApplication(PpApplication): if self.verbose > 2: LOG.debug("Reading config files with character set {!r} ...".format( self.cfg_encoding)) + self._config_has_errors = None open_opts = {} if six.PY3 and self.cfg_encoding: @@ -621,6 +637,13 @@ class PpConfigApplication(PpApplication): self._perform_mail_cmdline_options() + if self.config_has_errors: + LOG.error("There are errors in configuration.") + self.exit(1) + else: + LOG.debug("There are no errors in configuration.") + self.config_has_errors = False + # ------------------------------------------------------------------------- def _perform_mail_cmdline_options(self): diff --git a/pp_lib/import_pdnsdata.py b/pp_lib/import_pdnsdata.py index a1b9982..a9e920d 100644 --- a/pp_lib/import_pdnsdata.py +++ b/pp_lib/import_pdnsdata.py @@ -17,6 +17,7 @@ import re import pwd import textwrap import traceback +import socket # Third party modules import six @@ -29,7 +30,7 @@ from .common import pp from .cfg_app import PpCfgAppError, PpConfigApplication -__version__ = '0.5.2' +__version__ = '0.5.3' LOG = logging.getLogger(__name__) # ============================================================================= @@ -43,18 +44,16 @@ class ImportPdnsdataApp(PpConfigApplication): """ # Source DB data - src_db_host = 'mysql-pp06.pixelpark.com' - src_db_port = 3306 - src_db_schema = 'pdns' - src_db_user = 'pdns' - src_db_pass = 'ohtior4KaFei' + default_src_db_host = 'mysql-pp06.pixelpark.com' + default_src_db_port = 3306 + default_src_db_schema = 'pdns' + default_src_db_user = 'pdns' # Target DB data - tgt_db_host = 'systemshare.pixelpark.com' - tgt_db_port = 5432 - tgt_db_schema = 'pdns' - tgt_db_user = 'pdns' - tgt_db_pass = 'oo?fah7gai7X' + default_tgt_db_host = 'systemshare.pixelpark.com' + default_tgt_db_port = 5432 + default_tgt_db_schema = 'pdns' + default_tgt_db_user = 'pdns' # ------------------------------------------------------------------------- def __init__(self, appname=None, version=__version__): @@ -66,6 +65,18 @@ class ImportPdnsdataApp(PpConfigApplication): self.default_mail_recipients = ['frank.brehm@pixelpark.com'] + self.src_db_host = self.default_src_db_host + self.src_db_port = self.default_src_db_port + self.src_db_schema = self.default_src_db_schema + self.src_db_user = self.default_src_db_user + self.src_db_pass = None + + self.tgt_db_host = self.default_tgt_db_host + self.tgt_db_port = self.default_tgt_db_port + self.tgt_db_schema = self.default_tgt_db_schema + self.tgt_db_user = self.default_tgt_db_user + self.tgt_db_pass = None + self.src_connection = None self.tgt_connection = None @@ -115,24 +126,174 @@ class ImportPdnsdataApp(PpConfigApplication): }, } + # ------------------------------------------------------------------------- + def perform_config(self): + + super(ImportPdnsdataApp, self).perform_config() + + for section_name in self.cfg.keys(): + + if self.verbose > 2: + LOG.debug("Checking config section {!r} ...".format(section_name)) + section = self.cfg[section_name] + + if section_name.lower() in ('src_db', 'src_db', 'srcdb', 'source', 'src'): + self.do_src_db_cfg(section_name, section) + + if section_name.lower() in ('tgt_db', 'tgt_db', 'tgtdb', 'target', 'tgt'): + self.do_tgt_db_cfg(section_name, section) + + # ------------------------------------------------------------------------- + def do_src_db_cfg(self, section_name, section): + + if self.verbose > 2: + LOG.debug("Evaluating config section {n!r}:\n{s}".format( + n=section_name, s=pp(section))) + + if 'host' in section: + host = section['host'].lower().strip() + if not host: + LOG.error('Invalid source hostname {!r} in configuration section {!r} found.'.format( + section['host'], section_name)) + else: + try: + _ = socket.getaddrinfo(host, 3306, proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + msg = 'Invalid source hostname {!r} in configuration section {!r}: {}'.format( + section['host'], section_name, e) + LOG.error(msg) + self.config_has_errors = True + else: + self.src_db_host = host + + if 'port' in section: + try: + port = int(section['port']) + if port <= 0: + raise ValueError("port number may not be negative.") + elif port >= (2 ** 16): + raise ValueError("port number must be less than {}".format((2 ** 16))) + except (ValueError, TypeError) as e: + msg = 'Invalid source port number {!r} in configuration section {!r}: {}'.format( + section['port'],section_name, e) + LOG.error(msg) + self.config_has_errors = True + else: + self.src_db_port = port + + if 'schema' in section: + schema = section['schema'].lower().strip() + if not schema: + LOG.error('Invalid source database name {!r} in configuration section {!r} found.'.format( + section['schema'], section_name)) + else: + self.src_db_schema = schema + + if 'user' in section: + user = section['user'].lower().strip() + if not schema: + LOG.error('Invalid source database user {!r} in configuration section {!r} found.'.format( + section['user'], section_name)) + self.config_has_errors = True + else: + self.src_db_user = user + + if 'password' in section: + self.src_db_pass = section['password'] + + # ------------------------------------------------------------------------- + def do_tgt_db_cfg(self, section_name, section): + + if self.verbose > 2: + LOG.debug("Evaluating config section {n!r}:\n{s}".format( + n=section_name, s=pp(section))) + + if 'host' in section: + host = section['host'].lower().strip() + if not host: + LOG.error('Invalid target hostname {!r} in configuration section {!r} found.'.format( + section['host'], section_name)) + else: + try: + _ = socket.getaddrinfo(host, 3306, proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + msg = 'Invalid target hostname {!r} in configuration section {!r}: {}'.format( + section['host'], section_name, e) + LOG.error(msg) + self.config_has_errors = True + else: + self.tgt_db_host = host + + if 'port' in section: + try: + port = int(section['port']) + if port <= 0: + raise ValueError("port number may not be negative.") + elif port >= (2 ** 16): + raise ValueError("port number must be less than {}".format((2 ** 16))) + except (ValueError, TypeError) as e: + msg = 'Invalid target port number {!r} in configuration section {!r}: {}'.format( + section['port'],section_name, e) + LOG.error(msg) + self.config_has_errors = True + else: + self.tgt_db_port = port + + if 'schema' in section: + schema = section['schema'].lower().strip() + if not schema: + LOG.error('Invalid target database name {!r} in configuration section {!r} found.'.format( + section['schema'], section_name)) + else: + self.tgt_db_schema = schema + + if 'user' in section: + user = section['user'].lower().strip() + if not schema: + LOG.error('Invalid target database user {!r} in configuration section {!r} found.'.format( + section['user'], section_name)) + self.config_has_errors = True + else: + self.tgt_db_user = user + + if 'password' in section: + self.tgt_db_pass = section['password'] + # ------------------------------------------------------------------------- def pre_run(self): + result = None + + LOG.debug("Connecting to source MySQL database on {}@{}:{}/{} ...".format( + self.src_db_user, self.src_db_host, self.src_db_port, self.src_db_schema)) try: self.src_connection = pymysql.connect( host=self.src_db_host, + port=self.src_db_port, db=self.src_db_schema, user=self.src_db_user, password=self.src_db_pass, charset='utf8', cursorclass=pymysql.cursors.DictCursor ) - except ValueError as e: + + sql = 'SHOW VARIABLES LIKE "version"' + if self.verbose > 1: + LOG.debug("SQL: {}".format(sql)) + with self.src_connection.cursor() as cursor: + cursor.execute(sql) + result = cursor.fetchone() + if self.verbose > 2: + LOG.debug("Got version info:\n{}".format(pp(result))) + LOG.info("Source database is MySQL version {!r}.".format(result['Value'])) + + except (pymysql.err.OperationalError) as e: LOG.error("Could not connect to source database ({}): {}".format( e.__class__.__name__, e)) - traceback.print_exc() self.exit(6) + LOG.debug("Connecting to target PostgreSQL database on {}@{}:{}/{} ...".format( + self.tgt_db_user, self.tgt_db_host, self.tgt_db_port, self.tgt_db_schema)) try: self.tgt_connection = psycopg2.connect( host=self.tgt_db_host, @@ -141,10 +302,20 @@ class ImportPdnsdataApp(PpConfigApplication): user=self.tgt_db_user, password=self.tgt_db_pass, ) - except ValueError as e: + + sql = 'SHOW server_version' + if self.verbose > 1: + LOG.debug("SQL: {}".format(sql)) + with self.tgt_connection.cursor() as cursor: + cursor.execute(sql) + result = cursor.fetchone() + if self.verbose > 2: + LOG.debug("Got version info:\n{}".format(pp(result))) + LOG.info("Target database is PostgreSQL version {!r}.".format(result[0])) + + except psycopg2.OperationalError as e: LOG.error("Could not connect to target database ({}): {}".format( e.__class__.__name__, e)) - traceback.print_exc() self.exit(7) # -------------------------------------------------------------------------