From 4f883d551534d92a58c6875d958c10a4b396e3c5 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Tue, 11 Jul 2017 11:16:01 +0200 Subject: [PATCH] More verbse output of number of source datasets --- pp_lib/import_pdnsdata.py | 142 +++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 48 deletions(-) diff --git a/pp_lib/import_pdnsdata.py b/pp_lib/import_pdnsdata.py index 2e5a047..f49bfd1 100644 --- a/pp_lib/import_pdnsdata.py +++ b/pp_lib/import_pdnsdata.py @@ -29,7 +29,7 @@ from .common import pp from .cfg_app import PpCfgAppError, PpConfigApplication -__version__ = '0.4.1' +__version__ = '0.4.2' LOG = logging.getLogger(__name__) # ============================================================================= @@ -74,6 +74,45 @@ class ImportPdnsdataApp(PpConfigApplication): cfg_stems='import-pdnsdata' ) + self.nr = { + 'cryptokeys': { + 'has_domain': True, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + 'domainmetadata': { + 'has_domain': True, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + 'domains': { + 'has_domain': False, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + 'records': { + 'has_domain': True, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + 'supermasters': { + 'has_domain': False, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + 'tsigkeys': { + 'has_domain': True, + 'total': 0, + 'valid': 0, + 'invalid': 0, + }, + } + # ------------------------------------------------------------------------- def pre_run(self): @@ -121,64 +160,71 @@ class ImportPdnsdataApp(PpConfigApplication): # ------------------------------------------------------------------------- def get_src_info(self): + LOG.debug("Retreiving number of source datasets ...") + result = None - nr_cryptokeys = 0 - nr_domainmetadata = 0 - nr_domains = 0 - nr_records = 0 - nr_supermasters = 0 - nr_tsigkeys = 0 + max_tblname_len = 1 + for table in self.nr.keys(): + if len(table) > max_tblname_len: + max_tblname_len = len(table) + max_tblname_len += 1 + tpl = "Found {{:<{}}} {{:>8}}".format(max_tblname_len) + if self.verbose > 2: + LOG.debug("Output template: {!r}".format(tpl)) with self.src_connection.cursor() as cursor: - # Retrieve number of domains - sql = "SELECT COUNT(*) AS count_domains FROM domains" - cursor.execute(sql) - result = cursor.fetchone() - nr_domains = int(result['count_domains']) - - # Retrieve number of cryptokeys - sql = "SELECT COUNT(*) AS count_cryptokeys FROM cryptokeys" - cursor.execute(sql) - result = cursor.fetchone() - nr_cryptokeys = int(result['count_cryptokeys']) - - # Retrieve number of domainmetadata - sql = "SELECT COUNT(*) AS count_domainmetadata FROM domainmetadata" - cursor.execute(sql) - result = cursor.fetchone() - nr_domainmetadata = int(result['count_domainmetadata']) - - # Retrieve number of records - sql = "SELECT COUNT(*) AS count_records FROM records" - cursor.execute(sql) - result = cursor.fetchone() - nr_records = int(result['count_records']) - - # Retrieve number of supermasters - sql = "SELECT COUNT(*) AS count_supermasters FROM supermasters" - cursor.execute(sql) - result = cursor.fetchone() - nr_supermasters = int(result['count_supermasters']) - - # Retrieve number of tsigkeys - sql = "SELECT COUNT(*) AS count_tsigkeys FROM tsigkeys" - cursor.execute(sql) - result = cursor.fetchone() - nr_tsigkeys = int(result['count_tsigkeys']) + for table in sorted(self.nr.keys()): + has_domain = self.nr[table].get('has_domain', False) + count_total = 0 + count_valid = 0 + count_invalid = 0 + sql = "SELECT COUNT(*) AS count_rows FROM {}".format(table) + if self.verbose > 1: + LOG.debug("SQL: {}".format(sql)) + cursor.execute(sql) + result = cursor.fetchone() + count_total = int(result['count_rows']) + self.nr[table]['total'] = count_total + self.nr[table]['valid'] = 0 + self.nr[table]['invalid'] = 0 + + if count_total and has_domain: + + sql = textwrap.dedent('''\ + SELECT COUNT(*) AS count_rows + FROM {} + WHERE domain_id NOT IN ( + SELECT id FROM domains) + ''').strip().format(table) + + if self.verbose > 1: + LOG.debug("SQL: {}".format(sql)) + cursor.execute(sql) + result = cursor.fetchone() + count_invalid = int(result['count_rows']) + if count_invalid: + count_valid = count_total - count_invalid + self.nr[table]['valid'] = count_valid + self.nr[table]['invalid'] = count_invalid title = "Number of rows in current PowerDNS database" print() print(title) print(('=' * len(title))) - print("Found cryptokeys: {:>8}".format(nr_cryptokeys)) - print("Found domainmetadata: {:>8}".format(nr_domainmetadata)) - print("Found domains: {:>8}".format(nr_domains)) - print("Found records: {:>8}".format(nr_records)) - print("Found supermasters: {:>8}".format(nr_supermasters)) - print("Found tsigkeys: {:>8}".format(nr_tsigkeys)) + + for table in sorted(self.nr.keys()): + has_domain = self.nr[table].get('has_domain', False) + msg = tpl.format(table, self.nr[table]['total']) + if has_domain: + if self.nr[table]['invalid']: + msg += " ({} valid, {} invalid)".format( + self.nr[table]['valid'], self.nr[table]['invalid']) + else: + msg += " (all valid)" + print(msg) print() # ------------------------------------------------------------------------- -- 2.39.5