From: Frank Brehm Date: Thu, 26 Jan 2017 11:00:19 +0000 (+0100) Subject: Executing first underlying actions X-Git-Tag: 0.8.4~50 X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=884f14418d2889781d4cdb59919574f191351d02;p=pixelpark%2Fpuppetmaster-webhooks.git Executing first underlying actions --- diff --git a/deploy-new.py b/deploy-new.py index f701bd5..32317a3 100755 --- a/deploy-new.py +++ b/deploy-new.py @@ -4,33 +4,29 @@ # Standard modules import os import sys - +import logging # own modules: basedir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(basedir, 'lib') -#print("libdir: {!r}".format(libdir)) - sys.path.insert(0, libdir) import webhooks -#from webhooks.errors import MigrationAppError - from webhooks.deploy import WebhookDeployApp MY_APPNAME = 'deploy' +LOG = logging.getLogger(MY_APPNAME) app = WebhookDeployApp(appname=MY_APPNAME) -if app.verbose > 2 and 'REQUEST_METHOD' not in os.environ: - print("%s object:\n%s" % (app.__class__.__name__, app)) +if app.verbose > 2: + LOG.debug("{c} object:\n{o}".format(c=app.__class__.__name__, o=app)) app() sys.exit(0) - # vim: ts=4 et diff --git a/deploy.yaml b/deploy.yaml index b84e10a..2179b9c 100644 --- a/deploy.yaml +++ b/deploy.yaml @@ -1,7 +1,9 @@ --- verbose: 0 +do_sudo: true log_dir: '/var/log/webhooks' default_email: 'frank@brehm-online.com' +default_parent_dir: '/www/data' ignore_projects: mail_cc_addresses: - 'webmaster@pixelpark.com' @@ -15,6 +17,12 @@ projects: puppet_modules: namespace: 'puppet' parent_dir: '/www/data/puppet-environment' + postfix_config: + namespace: 'ppadmin' + name: 'postfix_config' + parent_dir: '/www/data' + workdir: 'postfix_config' + branch: 'master' mail_bodies: special_chars: "Received special characters in module name" diff --git a/lib/webhooks/common.py b/lib/webhooks/common.py index 41abed7..75b2b04 100644 --- a/lib/webhooks/common.py +++ b/lib/webhooks/common.py @@ -19,7 +19,10 @@ import platform __version__ = '0.2.0' -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) +RE_YES = re.compile(r'^\s*(?:y(?:es)?|true)\s*$', re.IGNORECASE) +RE_NO = re.compile(r'^\s*(?:no?|false|off)\s*$', re.IGNORECASE) + # ============================================================================= def pp(value, indent=4, width=99, depth=None): @@ -140,6 +143,47 @@ def to_str(obj, encoding='utf-8'): +# ============================================================================= +def to_bool(value): + """ + Converter from string to boolean values (e.g. from configurations) + """ + + if not value: + return False + + if isinstance(value, bool): + return value + + try: + v_int = int(value) + except ValueError: + pass + except TypeError: + pass + else: + if v_int == 0: + return False + else: + return True + + v_str = '' + if isinstance(value, str): + v_str = value + elif isinstance(value, bytes): + v_str = value.decode('utf-8') + else: + v_str = str(value) + + if RE_YES.search(v_str): + return True + + if RE_NO.search(v_str): + return False + + return bool(value) + + # ============================================================================= if __name__ == "__main__": diff --git a/lib/webhooks/deploy.py b/lib/webhooks/deploy.py index cc0edc0..66a3581 100644 --- a/lib/webhooks/deploy.py +++ b/lib/webhooks/deploy.py @@ -16,6 +16,9 @@ import textwrap import datetime import json import traceback +import copy +import pipes +import subprocess # Third party modules import yaml @@ -23,12 +26,13 @@ import yaml # Own modules import webhooks -from webhooks.common import pp, to_bytes +from webhooks.common import pp, to_bytes, to_str, to_bool __version__ = webhooks.__version__ LOG = logging.getLogger(__name__) DEFAULT_EMAIL = 'frank.brehm@pixelpark.com' DEFAULT_SENDER = 'Puppetmaster <{}>'.format(DEFAULT_EMAIL) +DEFAULT_PARENT_DIR = '/www/data' # ============================================================================= @@ -40,6 +44,8 @@ class WebhookDeployApp(object): cgi_bin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) base_dir = os.path.dirname(cgi_bin_dir) + special_chars_re = re.compile(r'[^a-z0-9_\-]', re.IGNORECASE) + mail_bodies = { 'special_chars': "Received special characters in module name", 'no_branch_dir': "Branch folder does not exist", @@ -81,9 +87,12 @@ class WebhookDeployApp(object): self.namespace = None self.name = None self.full_name = None + self.git_ssh_url = None + self.do_sudo = True self.ignore_projects = [] + self.default_parent_dir = '/www/data' self.default_email = DEFAULT_EMAIL self.mail_to_addresses = [] self.mail_cc_addresses = [ @@ -94,6 +103,18 @@ class WebhookDeployApp(object): self._log_directory = os.sep + os.path.join('var', 'log', 'webhooks') + self.projects = { + 'hiera': { + 'namespace': 'puppet', + 'name': 'hiera', + 'parent_dir': '/www/data/puppet-hiera' + }, + 'puppet_modules': { + 'namespace': 'puppet', + 'parent_dir': '/www/data/puppet-environment' + } + } + self.read_config() self.init_logging() @@ -210,12 +231,20 @@ class WebhookDeployApp(object): if 'verbose' in config: self.verbose = config['verbose'] + if 'do_sudo' in config: + self.do_sudo = to_bool(config['do_sudo']) + if 'log_dir' in config and config['log_dir']: self._log_directory = config['log_dir'] if 'default_email' in config and config['default_email']: self.default_email = config['default_email'] + if 'default_parent_dir' in config and config['default_parent_dir']: + pdir = config['default_parent_dir'] + if os.path.isabs(pdir): + self.default_parent_dir = pdir + if 'ignore_projects' in config: if config['ignore_projects'] is None: self.ignore_projects = [] @@ -240,6 +269,43 @@ class WebhookDeployApp(object): for key in config['mail_bodies'].keys(): self.mail_bodies[key] = config['mail_bodies'][key] + if 'projects' in config and isinstance(config['projects'], dict): + self.config_projects(config['projects']) + + # ------------------------------------------------------------------------- + def config_projects(self, project_cfg): + + for project_key in project_cfg.keys(): + if project_key == 'hiera': + cfg = project_cfg['hiera'] + if 'namespace' in cfg and cfg['namespace'].strip(): + self.projects['hiera']['namespace'] = cfg['namespace'].strip() + if 'name' in cfg and cfg['name'].strip(): + self.projects['hiera']['name'] = cfg['name'].strip() + if 'parent_dir' in cfg and cfg['parent_dir']: + self.projects['hiera']['parent_dir'] = cfg['parent_dir'] + continue + if project_key == 'puppet_modules': + cfg = project_cfg['puppet_modules'] + if 'namespace' in cfg and cfg['namespace'].strip(): + self.projects['puppet_modules']['namespace'] = cfg['namespace'].strip() + if 'parent_dir' in cfg and cfg['parent_dir']: + self.projects['puppet_modules']['parent_dir'] = cfg['parent_dir'] + continue + if project_key not in self.projects: + self.projects[project_key] = {} + cfg = project_cfg[project_key] + if 'namespace' in cfg and cfg['namespace'].strip(): + self.projects[project_key]['namespace'] = cfg['namespace'].strip() + if 'name' in cfg and cfg['name'].strip(): + self.projects[project_key]['name'] = cfg['name'].strip() + if 'parent_dir' in cfg and cfg['parent_dir']: + self.projects[project_key]['parent_dir'] = cfg['parent_dir'] + if 'workdir' in cfg and cfg['workdir']: + self.projects[project_key]['workdir'] = cfg['workdir'] + if 'branch' in cfg and cfg['branch'].strip(): + self.projects[project_key]['branch'] = cfg['branch'].strip() + # ------------------------------------------------------------------------- def init_logging(self): """ @@ -390,8 +456,148 @@ class WebhookDeployApp(object): self.name = self.json_data['project']['name'] self.full_name = self.json_data['project']['path_with_namespace'] + if self.full_name in self.ignore_projects or self.name in self.ignore_projects: + LOG.info("Ignoring project {!r}.".format(self.full_name)) + return True + + if self.special_chars_re.search(self.name): + LOG.error(("Project {!r}: " + self.mail_bodies['special_chars']).format( + self.full_name)) + return True + + committers = [] + timeformat = '%Y-%m-%dT%H:%M:%S%z' + for commit in self.json_data['commits']: + ts = commit['timestamp'].split('+') + ts_str = ts[0] + if len(ts) > 1: + ts_str += '+' + ts[1].replace(':', '').ljust(4, '0') + else: + ts_str += '+0000' + timestamp = datetime.datetime.strptime(ts_str, timeformat) + email = commit['author']['email'] + committers.append((timestamp, email)) + committers.sort() + if self.verbose > 1: + LOG.debug("Got committers: {}".format(pp(committers))) + + self.mail_to_addresses.append(committers[-1][1]) + + if 'git_ssh_url' in self.json_data['project']: + self.git_ssh_url = self.json_data['project']['git_ssh_url'] + else: + self.git_ssh_url = 'git@git.pixelpark.com:{ns}/{n}.git'.format( + ns=self.namespace, n=self.name) + + if self.full_name == 'puppet/hiera': + return self.deploy_hiera() + + if self.namespace == 'puppet': + return self.deploy_puppet_modules() + + for project_key in self.projects.keys(): + if project_key == 'hiera' or project_key == 'puppet_modules': + continue + cfg = copy.copy(self.projects[project_key]) + if 'namespace' not in cfg: + LOG.debug("No namespace defined in project {n!r}:\n{p]".format( + n=project_key, p=pp(cfg))) + continue + ns = cfg['namespace'] + pname = project_key + if 'name' in cfg: + pname = cfg['name'] + else: + cfg['name'] = project_key + full_name = ns + '/' + pname + + LOG.debug("Checking for {!r} ...".format(full_name)) + + if self.full_name == full_name: + return self.deploy(cfg) + + LOG.error("Could not find a definition for project {!r}.".format(self.full_name)) + + return True + + # ------------------------------------------------------------------------- + def deploy_hiera(self): + + LOG.info("Deploying Hiera working directory ...") + + return True + + # ------------------------------------------------------------------------- + def deploy_puppet_modules(self): + + LOG.info("Deploying puppet modules working directory ...") + return True + # ------------------------------------------------------------------------- + def deploy(self, cfg): + + ns = cfg['namespace'] + pname = cfg['name'] + full_name = ns + '/' + pname + parent_dir = self.default_parent_dir + if 'parent_dir' in cfg and cfg['parent_dir']: + parent_dir = cfg['parent_dir'] + workdir = pname + if 'workdir' in cfg and cfg['workdir']: + workdir = cfg['workdir'] + full_path = os.path.join(parent_dir, workdir) + branch = None + if 'branch' in cfg and cfg['branch']: + branch = cfg['branch'] + + LOG.info("Deploying working directory {f!r} for project {p!r} ...".format( + f=full_path, p=full_name)) + + if not os.access(parent_dir, os.F_OK): + LOG.error("Parent directory {d!r} for project {p!r} does not exists.".format( + d=parent_dir, p=full_name)) + return True + + if not os.path.isdir(parent_dir): + LOG.error(( + "Path for parent directory {d!r} for project {p!r} " + "is not a directory.").format(d=parent_dir, p=full_name)) + return True + + return self.ensure_workingdir(parent_dir, workdir, branch) + + # ------------------------------------------------------------------------- + def ensure_workingdir(self, parent_dir, workdir, branch=None): + + os.chdir(parent_dir) + cmd = [] + if self.do_sudo: + cmd = ['sudo', '-n'] + if os.access(workdir, os.F_OK): + os.chdir(workdir) + cmd += ['git', 'pull'] + else: + cmd += ['git', 'clone', self.git_ssh_url, workdir] + if branch: + cmd += ['-b', branch] + if self.verbose > 2: + LOG.debug("Cmd: {}".format(pp(cmd))) + cmd_str = ' '.join(map(lambda x: pipes.quote(x), cmd)) + LOG.debug("Executing: {}".format(cmd_str)) + + git = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdoutdata, stderrdata) = git.communicate() + ret_val = git.wait() + + LOG.debug("Return value: {}".format(ret_val)) + if stdoutdata: + LOG.debug("Output:\n{}".format(to_str(stdoutdata))) + else: + LOG.debug("No output.") + if stderrdata: + LOG.warn("Error messages on '{c}': {e}".format(c=cmd_str, e=to_str(stderrdata))) + # ============================================================================= if __name__ == "__main__":