From 366e938c90cf95e60de900a5352472c81702943c Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Fri, 24 Aug 2018 11:36:09 +0200 Subject: [PATCH] Reorganized get-forge-modules --- lib/webhooks/__init__.py | 2 +- lib/webhooks/common.py | 21 +++- lib/webhooks/get_forge_modules.py | 70 +++--------- lib/webhooks/module_info.py | 178 ++++++++++++++++++++++++++++-- 4 files changed, 205 insertions(+), 66 deletions(-) diff --git a/lib/webhooks/__init__.py b/lib/webhooks/__init__.py index fb088a9..16503b7 100644 --- a/lib/webhooks/__init__.py +++ b/lib/webhooks/__init__.py @@ -1,6 +1,6 @@ #!/bin/env python3 # -*- coding: utf-8 -*- -__version__ = '0.9.4' +__version__ = '0.9.5' # vim: ts=4 et list diff --git a/lib/webhooks/common.py b/lib/webhooks/common.py index 4e17c40..369199b 100644 --- a/lib/webhooks/common.py +++ b/lib/webhooks/common.py @@ -15,10 +15,14 @@ import re import pprint import platform import locale +import collections + +# Third party modules +import six # Own modules -__version__ = '0.3.1' +__version__ = '0.3.2' LOG = logging.getLogger(__name__) RE_YES = re.compile(r'^\s*(?:y(?:es)?|true)\s*$', re.IGNORECASE) @@ -187,6 +191,21 @@ def to_bool(value): return bool(value) +# ============================================================================= +def is_sequence(arg): + + if six.PY3: + if not isinstance(arg, collections.Sequence): + return False + else: + if not isinstance(arg, collections.Sequence): + return False + + if hasattr(arg, "strip"): + return False + + return True + # ============================================================================= def caller_search_path(): """ diff --git a/lib/webhooks/get_forge_modules.py b/lib/webhooks/get_forge_modules.py index 7b3e9d3..cc7e927 100644 --- a/lib/webhooks/get_forge_modules.py +++ b/lib/webhooks/get_forge_modules.py @@ -156,6 +156,10 @@ class GetForgeModulesApp(BaseHookApp): self.check_data_dir() self.init_puppet_environments() self.collect_local_modules() + self.get_forge_information() + + if self.verbose > 2: + LOG.debug("Found modules:\n{}".format(pp(self.modules))) print() dt = datetime.datetime.now().isoformat(' ') @@ -171,22 +175,17 @@ class GetForgeModulesApp(BaseHookApp): self.read_puppetfile(env) self.read_metadata_files(env) - self.get_upstream_versions() - - if self.verbose > 2: - LOG.debug("Found modules:\n{}".format(pp(self.modules))) - # ------------------------------------------------------------------------- - def get_upstream_versions(self): + def get_forge_information(self): print() msg = "Collecting module information from upstream ..." self.print_out(msg) for full_name in self.modules.keys(): - self.get_upstream_version(full_name) + self.get_forge_module_info(full_name) if not self.verbose: - if self.modules[full_name].version_upstream: + if self.modules[full_name].forge_avail: print('.', end='', flush=True) else: print('~', end='', flush=True) @@ -194,59 +193,18 @@ class GetForgeModulesApp(BaseHookApp): print() # ------------------------------------------------------------------------- - def get_upstream_version(self, full_name): + def get_forge_module_info(self, full_name): module_info = self.modules[full_name] - url = "{url}/{name}".format(url=self.forge_uri, name=full_name) - - LOG.debug( - "Trying to get current version of module {} from Puppet forge.".format(full_name)) - if self.verbose > 1: - LOG.debug("URL to request: {}".format(url)) - - session = requests.Session() - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - response = session.request('GET', url, timeout=self.http_timeout) - if w: - warn_class = w[-1].category.__name__ - warn_msg = '{}: {}'.format( - warn_class, w[-1].message) - if warn_class == 'SubjectAltNameWarning': - LOG.debug(warn_msg) - else: - LOG.warn(warn_msg) - - LOG.debug("Got status code: {}.".format(response.status_code)) - if not response.ok: - LOG.debug("Did not found module {} on Puppet forge.".format(full_name)) - return - if not response.text: - LOG.warn("No output for URL {!r}".format(url)) - return None - if self.verbose > 2: - msg = "Output:\n{}".format(response.text) - LOG.debug(msg) + module_info_forge = ModuleInfo.get_from_forge( + full_name, forge_uri=self.forge_uri, http_timeout=self.http_timeout, + appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, ) - version = None - - js_info = response.json() - if 'current_release' in js_info and js_info['current_release']: - if 'version' in js_info['current_release']: - version = js_info['current_release']['version'] - else: - msg = "Did not found version of current_release of module {}.".format(full_name) - LOG.warn(msg) + if module_info_forge: + module_info.merge_in(module_info_forge) else: - msg = "Did not found current_release of module {}.".format(full_name) - LOG.warn(msg) - - LOG.debug("Current version of module {name} is {version}.".format( - name=full_name, version=version)) - - if version: - module_info._version_upstream = version + module_info.forge_avail = False # ------------------------------------------------------------------------- def read_metadata_files(self, env): diff --git a/lib/webhooks/module_info.py b/lib/webhooks/module_info.py index 9cb4f07..343821e 100644 --- a/lib/webhooks/module_info.py +++ b/lib/webhooks/module_info.py @@ -12,17 +12,20 @@ from __future__ import absolute_import import os import logging import re +import copy +import warnings # Third party modules import six +import requests # Own modules -from .common import pp, to_str +from .common import pp, to_str, to_bool, is_sequence from .obj import BaseObjectError from .obj import BaseObject -__version__ = '0.3.0' +__version__ = '0.4.0' LOG = logging.getLogger(__name__) @@ -59,10 +62,14 @@ class ModuleInfo(BaseObject): self._name = None self._vendor = None self._full_name_orig = None - self._version_upstream = None + self._forge_version = None self.local_versions = {} self.expected_versions = {} self.repo = None + self._forge_avail = None + self.forge_releases = None + self._forge_homepage_url = None + self._forge_source = None super(ModuleInfo, self).__init__( appname=appname, verbose=verbose, version=version, @@ -115,9 +122,51 @@ class ModuleInfo(BaseObject): # ------------------------------------------------------------------------- @property - def version_upstream(self): + def forge_version(self): """The current version number of the module from upstream.""" - return self._version_upstream + return self._forge_version + + @forge_version.setter + def forge_version(self, value): + if value is None: + self._forge_version = None + return + val = to_str(value).strip() + if val == '': + val = None + self._forge_version = val + + # ------------------------------------------------------------------------- + @property + def forge_homepage_url(self): + """The URL of the module home in Puppet forge.""" + return self._forge_homepage_url + + @forge_homepage_url.setter + def forge_homepage_url(self, value): + if value is None: + self._forge_homepage_url = None + return + val = to_str(value).strip() + if val == '': + val = None + self._forge_homepage_url = val + + # ------------------------------------------------------------------------- + @property + def forge_source(self): + """The source (URL) of the module from Puppet forge.""" + return self._forge_source + + @forge_source.setter + def forge_source(self, value): + if value is None: + self._forge_source = None + return + val = to_str(value).strip() + if val == '': + val = None + self._forge_source = val # ------------------------------------------------------------------------- @property @@ -160,6 +209,19 @@ class ModuleInfo(BaseObject): else: self._full_name_orig = None + # ------------------------------------------------------------------------- + @property + def forge_avail(self): + """Is this module available per upstream? None -> not checked.""" + return self._forge_avail + + @forge_avail.setter + def forge_avail(self, value): + if value is None: + self._forge_avail = None + return + self._forge_avail = to_bool(value) + # ------------------------------------------------------------------------- def __repr__(self): return str(self) @@ -179,7 +241,10 @@ class ModuleInfo(BaseObject): res['vendor'] = self.vendor res['full_name'] = self.full_name res['full_name_orig'] = self.full_name_orig - res['version_upstream'] = self.version_upstream + res['forge_version'] = self.forge_version + res['forge_avail'] = self.forge_avail + res['forge_homepage_url'] = self.forge_homepage_url + res['forge_source'] = self.forge_source return res @@ -191,8 +256,20 @@ class ModuleInfo(BaseObject): "Parameter {p!r} is not of class ModuleInfoTypeError, but of class {c} " " instead.").format(p=other, c=other.__class__.__name__)) - if other.version_upstream and not self.version_upstream: - self._version_upstream = other.version_upstream + if other.forge_avail is not None and self.forge_avail is None: + self.forge_avail = other.forge_avail + + if other.forge_version and not self.forge_version: + self.forge_version = other.forge_version + + if other.forge_homepage_url and not self.forge_homepage_url: + self.forge_homepage_url = other.forge_homepage_url + + if other.forge_source and not self.forge_source: + self.forge_source = other.forge_source + + if other.forge_releases is not None and self.forge_releases is None: + self.forge_releases = copy.copy(other.forge_releases) if other.repo and not self.repo: self.repo = other.repo @@ -319,6 +396,91 @@ class ModuleInfo(BaseObject): return defs + # ------------------------------------------------------------------------- + @classmethod + def get_from_forge( + cls, full_name, forge_uri, http_timeout=30, appname=None, verbose=0, base_dir=None): + + url = "{url}/{name}".format(url=forge_uri, name=full_name) + module_info = None + + LOG.debug("Trying to get module {m!r} from Puppet forge {u!r} ...".format( + m=full_name, u=url)) + + session = requests.Session() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + response = session.request('GET', url, timeout=http_timeout) + if w: + warn_class = w[-1].category.__name__ + warn_msg = '{}: {}'.format( + warn_class, w[-1].message) + if warn_class == 'SubjectAltNameWarning': + LOG.debug(warn_msg) + else: + LOG.warn(warn_msg) + + LOG.debug("Got status code: {}.".format(response.status_code)) + if not response.ok: + LOG.debug("Did not found module {} on Puppet forge.".format(full_name)) + return None + + if not response.text: + LOG.warn("No output for URL {!r}".format(url)) + return None + if verbose > 3: + msg = "Output:\n{}".format(response.text) + LOG.debug(msg) + + try: + module_info = cls( + appname=appname, verbose=verbose, base_dir=base_dir, + full_name=full_name, + ) + except ModuleInfoError as e: + LOG.warn("{c}: {e}".format(c=e.__class__.__name__, e=e)) + return None + + version = None + source = None + module_info.forge_avail = True + + js_info = response.json() + if 'current_release' in js_info and js_info['current_release']: + if 'version' in js_info['current_release']: + version = js_info['current_release']['version'] + else: + msg = "Did not found version of current_release of module {}.".format(full_name) + LOG.warn(msg) + if 'metadata' in js_info['current_release'] and js_info['current_release']['metadata']: + if ('source' in js_info['current_release']['metadata'] and + js_info['current_release']['metadata']['source']): + source = str(js_info['current_release']['metadata']['source']).strip() + if not source: + LOG.warn("Did not found source information of module {}.".format(full_name)) + else: + msg = "Did not found current_release of module {}.".format(full_name) + LOG.warn(msg) + + LOG.debug("Current version of module {name} is {version}.".format( + name=full_name, version=version)) + + if version: + module_info.forge_version = version + if source: + module_info.forge_source = source + + if 'homepage_url' in js_info and js_info['homepage_url']: + module_info.forge_homepage_url = js_info['homepage_url'] + + module_info.forge_releases = [] + if 'releases' in js_info and is_sequence(js_info['releases']): + for release in js_info['releases']: + if 'version' in release and release['version']: + module_info.forge_releases.append(release['version'].strip()) + + return module_info + # ============================================================================= if __name__ == "__main__": -- 2.39.5