Finish reading Puppetfile and metadata.json

diff --git a/lib/webhooks/__init__.py b/lib/webhooks/__init__.py
index f94029c3b4a68d8816d0f06c1b5c7ed608433d30..a425805d0111b2acd9995291ad14664be2569291 100644 (file)
--- a/lib/webhooks/__init__.py
+++ b/lib/webhooks/__init__.py
@@ -1,6 +1,6 @@
 #!/bin/env python3
 # -*- coding: utf-8 -*-
 
-__version__ = '0.9.2'
+__version__ = '0.9.3'
 
 # vim: ts=4 et list

diff --git a/lib/webhooks/get_forge_modules.py b/lib/webhooks/get_forge_modules.py
index 1d39f0a3fbd22e5aa1ac26f2ddab614e5dafabcf..98821e188439255d2976ac6e59aeb2538cc93e90 100644 (file)
--- a/lib/webhooks/get_forge_modules.py
+++ b/lib/webhooks/get_forge_modules.py
@@ -161,6 +161,68 @@ class GetForgeModulesApp(BaseHookApp):
 
         for env in self.environments:
             self.read_puppetfile(env)
+            self.read_metadata_files(env)
+
+        if self.verbose > 2:
+            LOG.debug("Found modules:\n{}".format(pp(self.modules)))
+
+    # -------------------------------------------------------------------------
+    def read_metadata_files(self, env):
+
+        pattern = os.path.join(self.puppet_root_env_dir, env, 'modules', '*')
+        if self.verbose > 2:
+            LOG.debug("Globbing pattern for module directories: {!r}".format(pattern))
+
+        for module_dir in glob.glob(pattern):
+            module_info = self.get_meta_module_info(module_dir, env)
+            if module_info:
+                full_name = module_info.full_name
+                if full_name in self.modules:
+                    self.modules[full_name].merge_in(module_info)
+                else:
+                    self.modules[full_name] = module_info
+
+    # -------------------------------------------------------------------------
+    def get_meta_module_info(self, module_dir, env):
+
+        if self.verbose > 2:
+            LOG.debug("Get module information from {!r}.".format(module_dir))
+
+        if not os.path.exists(module_dir):
+            LOG.warn("Directory {!r} does not exists.".format(module_dir))
+            return None
+
+        if not os.path.isdir(module_dir):
+            LOG.warn("Path {!r} is not a directory".format(module_dir))
+            return None
+
+        metadata_file = os.path.join(module_dir, 'metadata.json')
+        if not os.path.exists(metadata_file):
+            LOG.warn("Metadatafile {!r} does not exists.".format(metadata_file))
+            return None
+        if not os.path.isfile(metadata_file):
+            LOG.warn("Metadatafile {!r} is not a regular file.".format(metadata_file))
+            return None
+        if not os.access(metadata_file, os.R_OK):
+            LOG.warn("Metadatafile {!r} is readable.".format(metadata_file))
+            return None
+        if self.verbose > 2:
+            LOG.debug("Reading and evaluating {!r}.".format(metadata_file))
+
+        json_data = None
+        try:
+            with open(metadata_file, 'r', **self.open_args) as fh:
+                json_data = json.load(fh)
+        except ValueError as e:
+            LOG.warn( "Could not interprete {f!r} as a regular JSON file: {e}".format(
+                f=metadata_file, e=e))
+            return None
+        if not json_data:
+            LOG.warn("Did not found any data in  {!r}.".format(metadata_file))
+            return None
+
+        return ModuleInfo.init_from_json(
+            json_data, env, appname=self.appname, verbose=self.verbose, base_dir=self.base_dir)
 
     # -------------------------------------------------------------------------
     def read_puppetfile(self, env):
@@ -196,11 +258,17 @@ class GetForgeModulesApp(BaseHookApp):
                 if self.re_comma_at_end.search(line):
                     continue
 
-                if self.verbose > 2:
+                if self.verbose > 3:
                     LOG.debug("Evaluating line {!r}...".format(prev_line))
                 module_info = ModuleInfo.init_from_puppetfile_line(
                     appname=self.appname, verbose=self.verbose, base_dir=self.base_dir,
                     line=prev_line, env=env)
+                if module_info:
+                    full_name = module_info.full_name
+                    if full_name in self.modules:
+                        self.modules[full_name].merge_in(module_info)
+                    else:
+                        self.modules[full_name] = module_info
 
                 prev_line = ''
 
@@ -210,6 +278,12 @@ class GetForgeModulesApp(BaseHookApp):
                 module_info = ModuleInfo.init_from_puppetfile_line(
                     appname=self.appname, verbose=self.verbose, base_dir=self.base_dir,
                     line=prev_line, env=env)
+                if module_info:
+                    full_name = module_info.full_name
+                    if full_name in self.modules:
+                        self.modules[full_name].merge_in(module_info)
+                    else:
+                        self.modules[full_name] = module_info
 
     # -------------------------------------------------------------------------
     def init_puppet_environments(self):

diff --git a/lib/webhooks/module_info.py b/lib/webhooks/module_info.py
index 88395e9cc8bab46fb4c697ae3637cbf064f32a44..9cb4f07d10eb63d1ca2b07c66ab44aa2f4575d44 100644 (file)
--- a/lib/webhooks/module_info.py
+++ b/lib/webhooks/module_info.py
@@ -22,7 +22,7 @@ from .common import pp, to_str
 from .obj import BaseObjectError
 from .obj import BaseObject
 
-__version__ = '0.2.0'
+__version__ = '0.3.0'
 
 LOG = logging.getLogger(__name__)
 
@@ -33,6 +33,12 @@ class ModuleInfoError(BaseObjectError):
     pass
 
 
+# =============================================================================
+class ModuleInfoTypeError(ModuleInfoError, TypeError):
+
+    pass
+
+
 # =============================================================================
 class ModuleInfo(BaseObject):
     """Class for encapsulating information about a Puppet module."""
@@ -42,7 +48,7 @@ class ModuleInfo(BaseObject):
     re_pf_line_version = re.compile(r"^\s*'([^']+)'")
     re_def_token = re.compile(r'^\s*(?:,\s*)?([^,]+)(?:\s*,|$)')
     re_empty = re.compile(r'^\s*(?:,\s*)?$')
-    re_key_val_pair = re.compile(r"^\s*:?([a-z]+)\s*=>\s*'([^']+)'\s*$", re.IGNORECASE)
+    re_key_val_pair = re.compile(r'^\s*:?([a-z]+)\s*=>\s*[\'"]([^\'"]+)[\'"]\s*$', re.IGNORECASE)
     re_v_at_start = re.compile(r"^v", re.IGNORECASE)
 
     # -------------------------------------------------------------------------
@@ -107,6 +113,12 @@ class ModuleInfo(BaseObject):
             self._vendor = _vendor
             self._full_name_orig = self.full_name
 
+    # -------------------------------------------------------------------------
+    @property
+    def version_upstream(self):
+        """The current version number of the module from upstream."""
+        return self._version_upstream
+
     # -------------------------------------------------------------------------
     @property
     def name(self):
@@ -148,6 +160,10 @@ class ModuleInfo(BaseObject):
         else:
             self._full_name_orig = None
 
+    # -------------------------------------------------------------------------
+    def __repr__(self):
+        return str(self)
+
     # -------------------------------------------------------------------------
     def as_dict(self, short=True):
         """
@@ -163,9 +179,57 @@ class ModuleInfo(BaseObject):
         res['vendor'] = self.vendor
         res['full_name'] = self.full_name
         res['full_name_orig'] = self.full_name_orig
+        res['version_upstream'] = self.version_upstream
 
         return res
 
+    # -------------------------------------------------------------------------
+    def merge_in(self, other):
+
+        if not isinstance(other, ModuleInfo):
+            raise ModuleInfoTypeError((
+                "Parameter {p!r} is not of class ModuleInfoTypeError, but of class {c} "
+                " instead.").format(p=other, c=other.__class__.__name__))
+
+        if other.version_upstream and not self.version_upstream:
+            self._version_upstream = other.version_upstream
+
+        if other.repo and not self.repo:
+            self.repo = other.repo
+
+        for env in other.local_versions.keys():
+            if env not in self.local_versions:
+                self.local_versions[env] = other.local_versions[env]
+
+        for env in other.expected_versions.keys():
+            if env not in self.expected_versions:
+                self.expected_versions[env] = other.expected_versions[env]
+
+    # -------------------------------------------------------------------------
+    @classmethod
+    def init_from_json(
+        cls, json_data, env, appname=None, verbose=0, base_dir=None):
+
+        if 'name' not in json_data:
+            LOG.warn("Did not found module name in json.")
+            return None
+
+        module_info = None
+
+        try:
+            module_info = cls(
+                appname=appname, verbose=verbose, base_dir=base_dir,
+                full_name=json_data['name'],
+            )
+        except ModuleInfoError as e:
+            LOG.warn("{c}: {e}".format(c=e.__class__.__name__, e=e))
+            return None
+
+        if 'version' in json_data:
+            module_info.local_versions[env] = json_data['version'].strip()
+
+        return module_info
+
     # -------------------------------------------------------------------------
     @classmethod
     def init_from_puppetfile_line(