From: Frank Brehm Date: Wed, 15 Feb 2017 15:39:56 +0000 (+0100) Subject: Changed inheritance of class WebhookDeployApp from object to BaseHookApp X-Git-Tag: 0.8.4~21 X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=2dd1ff3f63c3c7b68ef5bcd4392433a7d29b486c;p=pixelpark%2Fpuppetmaster-webhooks.git Changed inheritance of class WebhookDeployApp from object to BaseHookApp --- diff --git a/lib/webhooks/__init__.py b/lib/webhooks/__init__.py index 4d4ba82..7e33420 100644 --- a/lib/webhooks/__init__.py +++ b/lib/webhooks/__init__.py @@ -1,6 +1,6 @@ #!/bin/env python3 # -*- coding: utf-8 -*- -__version__ = '0.4.8' +__version__ = '0.5.1' # vim: ts=4 et list diff --git a/lib/webhooks/base_app.py b/lib/webhooks/base_app.py index 56e0be1..a8f8a7e 100644 --- a/lib/webhooks/base_app.py +++ b/lib/webhooks/base_app.py @@ -322,7 +322,7 @@ class BaseHookApp(object): LOG.debug("Command {!r} is not executable.".format(p)) if cmd_abs: - LOG.debug("Found {c!r} in {p!r}.".format(c=cmd, p=self.r10k_bin)) + LOG.debug("Found {c!r} in {p!r}.".format(c=cmd, p=cmd_abs)) else: LOG.error("Command {!r} not found.".format(cmd)) diff --git a/lib/webhooks/deploy.py b/lib/webhooks/deploy.py index 37b1b2d..1f9527c 100644 --- a/lib/webhooks/deploy.py +++ b/lib/webhooks/deploy.py @@ -30,157 +30,39 @@ import webhooks from webhooks.common import pp, to_bytes, to_str, to_bool +from webhooks.base_app import BaseHookApp + __version__ = webhooks.__version__ LOG = logging.getLogger(__name__) -DEFAULT_EMAIL = 'frank.brehm@pixelpark.com' -DEFAULT_SENDER = 'Puppetmaster <{}>'.format(DEFAULT_EMAIL) + DEFAULT_PARENT_DIR = '/www/data' # ============================================================================= -class WebhookDeployApp(object): +class WebhookDeployApp(BaseHookApp): """ Class for the application objects. """ - cgi_bin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - base_dir = os.path.dirname(cgi_bin_dir) - - special_chars_re = re.compile(r'[^a-z0-9_\-]', re.IGNORECASE) - dev_re = re.compile(r'^dev') - - mail_bodies = { - 'special_chars': "Received special characters in module name", - 'no_branch_dir': "Branch folder does not exist", - 'no_modules_dir': "Modules folder does not exist", - 'git_access_denied': "Access to remote repository was denied", - } - # ------------------------------------------------------------------------- - def __init__(self, appname=None, version=__version__): + def __init__(self, appname=None, verbose=0, version=__version__): """Constructor.""" - self._appname = None - """ - @ivar: name of the current running application - @type: str - """ - if appname: - v = str(appname).strip() - if v: - self._appname = v - if not self._appname: - self._appname = os.path.basename(sys.argv[0]) - - self._version = version - """ - @ivar: version string of the current object or application - @type: str - """ - - self._verbose = 1 - """ - @ivar: verbosity level (0 - 9) - @type: int - """ - - self.data = None - self.json_data = None - self.ref = None - self.namespace = None - self.name = None - self.full_name = None - self.git_ssh_url = None - self.do_sudo = True - - self.ignore_projects = [] - - self.error_data = [] - self.smtp_server = 'smtp.pixelpark.com' - self.smtp_port = 25 - - self.default_parent_dir = '/www/data' - self.default_email = DEFAULT_EMAIL - self.mail_to_addresses = [] - self.mail_cc_addresses = [ - 'webmaster@pixelpark.com', - self.default_email, - ] - self.sender_address = DEFAULT_SENDER - - self._log_directory = os.sep + os.path.join('var', 'log', 'webhooks') + self.description = textwrap.dedent('''\ + Receives push events as JSON-Data and synchronizes + the local repository. + ''').strip() self.projects = { 'hiera': { 'namespace': 'puppet', 'name': 'hiera', 'parent_dir': '/www/data/puppet-hiera' - }, - 'puppet_modules': { - 'namespace': 'puppet', - 'parent_dir': '/www/data/puppet-environment' } } - self.read_config() - self.init_logging() - - # ----------------------------------------------------------- - @property - def appname(self): - """The name of the current running application.""" - return self._appname - - @appname.setter - def appname(self, value): - if value: - v = str(value).strip() - if v: - self._appname = v - - # ----------------------------------------------------------- - @property - def version(self): - """The version string of the current object or application.""" - return self._version - - # ----------------------------------------------------------- - @property - def verbose(self): - """The verbosity level.""" - return getattr(self, '_verbose', 0) - - @verbose.setter - def verbose(self, value): - v = int(value) - if v >= 0: - self._verbose = v - else: - LOG.warn("Wrong verbose level %r, must be >= 0", value) - - # ----------------------------------------------------------- - @property - def log_directory(self): - """The directory containing the logfiles of this application.""" - return self._log_directory - - # ----------------------------------------------------------- - @property - def logfile(self): - """The logfile of this application.""" - return os.path.join(self.log_directory, self.appname + '.log') - - # ------------------------------------------------------------------------- - def __str__(self): - """ - Typecasting function for translating object structure - into a string - - @return: structure as string - @rtype: str - """ - - return pp(self.as_dict()) + super(WebhookDeployApp, self).__init__( + appname=appname, verbose=verbose, version=version) # ------------------------------------------------------------------------- def as_dict(self): @@ -191,105 +73,15 @@ class WebhookDeployApp(object): @rtype: dict """ - res = self.__dict__ - res = {} - for key in self.__dict__: - if key.startswith('_') and not key.startswith('__'): - continue - res[key] = self.__dict__[key] - res['__class_name__'] = self.__class__.__name__ - res['appname'] = self.appname - res['verbose'] = self.verbose - res['base_dir'] = self.base_dir - res['cgi_bin_dir'] = self.cgi_bin_dir - res['log_directory'] = self.log_directory - res['logfile'] = self.logfile - res['mail_bodies'] = self.mail_bodies + res = super(WebhookDeployApp, self).as_dict() return res - # ------------------------------------------------------------------------- - def read_config(self): - """Reading configuration from different YAML files.""" - - yaml_files = [] - # ./deploy.yaml - yaml_files.append(os.path.join(self.base_dir, self.appname + '.yaml')) - # /etc/pixelpark/deploy.yaml - yaml_files.append(os.sep + os.path.join('etc', 'pixelpark', self.appname + '.yaml')) - - for yaml_file in yaml_files: - self.read_from_yaml(yaml_file) # ------------------------------------------------------------------------- - def read_from_yaml(self, yaml_file): - """Reading configuration from given YAML file.""" - - # LOG.debug("Trying to read config from {!r} ...".format(yaml_file)) - if not os.access(yaml_file, os.F_OK): - # LOG.debug("File {!r} does not exists.".format(yaml_file)) - return - # LOG.debug("Reading config from {!r} ...".format(yaml_file)) - config = {} - with open(yaml_file, 'rb') as fh: - config = yaml.load(fh.read()) - # LOG.debug("Read config:\n{}".format(pp(config))) - - if 'verbose' in config: - self.verbose = config['verbose'] - - if 'do_sudo' in config: - self.do_sudo = to_bool(config['do_sudo']) - - if 'log_dir' in config and config['log_dir']: - self._log_directory = config['log_dir'] - - if 'default_email' in config and config['default_email']: - self.default_email = config['default_email'] - - if 'smtp_server' in config and config['smtp_server'].strip(): - self.smtp_server = config['smtp_server'].strip() - - if 'smtp_port' in config and config['smtp_port']: - msg = "Invalid port {p!r} for SMTP in {f!r} found.".format( - p=config['smtp_port'], f=yaml_file) - try: - port = int(config['smtp_port']) - if port > 0 and port < 2**16: - self.smtp_port = port - else: - self.error_data.append(msg) - except ValueError: - self.error_data.append(msg) - - if 'default_parent_dir' in config and config['default_parent_dir']: - pdir = config['default_parent_dir'] - if os.path.isabs(pdir): - self.default_parent_dir = pdir - - if 'ignore_projects' in config: - if config['ignore_projects'] is None: - self.ignore_projects = [] - elif isinstance(config['ignore_projects'], str): - if config['ignore_projects']: - self.ignore_projects = [config['ignore_projects']] - elif isinstance(config['ignore_projects'], list): - self.ignore_projects = config['ignore_projects'] - - if 'mail_cc_addresses' in config: - if config['mail_cc_addresses'] is None: - self.mail_cc_addresses = [] - elif isinstance(config['mail_cc_addresses'], str): - if config['mail_cc_addresses']: - self.mail_cc_addresses = [config['mail_cc_addresses']] - else: - self.mail_cc_addresses = [] - elif isinstance(config['mail_cc_addresses'], list): - self.mail_cc_addresses = config['mail_cc_addresses'] + def evaluate_config(self, config, yaml_file): - if 'mail_bodies' in config and isinstance(config['mail_bodies'], list): - for key in config['mail_bodies'].keys(): - self.mail_bodies[key] = config['mail_bodies'][key] + super(WebhookDeployApp, self).evaluate_config(config, yaml_file) if 'projects' in config and isinstance(config['projects'], dict): self.config_projects(config['projects']) @@ -307,13 +99,6 @@ class WebhookDeployApp(object): if 'parent_dir' in cfg and cfg['parent_dir']: self.projects['hiera']['parent_dir'] = cfg['parent_dir'] continue - if project_key == 'puppet_modules': - cfg = project_cfg['puppet_modules'] - if 'namespace' in cfg and cfg['namespace'].strip(): - self.projects['puppet_modules']['namespace'] = cfg['namespace'].strip() - if 'parent_dir' in cfg and cfg['parent_dir']: - self.projects['puppet_modules']['parent_dir'] = cfg['parent_dir'] - continue if project_key not in self.projects: self.projects[project_key] = {} cfg = project_cfg[project_key] @@ -329,72 +114,8 @@ class WebhookDeployApp(object): self.projects[project_key]['branch'] = cfg['branch'].strip() # ------------------------------------------------------------------------- - def init_logging(self): - """ - Initialize the logger object. - It creates a colored loghandler with all output to STDERR. - Maybe overridden in descendant classes. - - @return: None - """ - - root_log = logging.getLogger() - root_log.setLevel(logging.INFO) - if self.verbose: - root_log.setLevel(logging.DEBUG) - - # create formatter - format_str = '' - if 'REQUEST_METHOD' in os.environ or self.verbose > 1: - format_str = '[%(asctime)s]: ' - format_str += self.appname + ': ' - if self.verbose: - if self.verbose > 1: - format_str += '%(name)s(%(lineno)d) %(funcName)s() ' - else: - format_str += '%(name)s ' - format_str += '%(levelname)s - %(message)s' - formatter = logging.Formatter(format_str) - - if 'REQUEST_METHOD' in os.environ: - - #sys.stderr.write("Trying to open logfile {!r} ...\n".format(self.logfile)) - # we are in a CGI environment - if os.path.isdir(self.log_directory) and os.access(self.log_directory, os.W_OK): - lh_file = logging.FileHandler( - self.logfile, mode='a', encoding='utf-8', delay=True) - if self.verbose: - lh_file.setLevel(logging.DEBUG) - else: - lh_file.setLevel(logging.INFO) - lh_file.setFormatter(formatter) - root_log.addHandler(lh_file) - - else: - # create log handler for console output - lh_console = logging.StreamHandler(sys.stderr) - if self.verbose: - lh_console.setLevel(logging.DEBUG) - else: - lh_console.setLevel(logging.INFO) - lh_console.setFormatter(formatter) - - root_log.addHandler(lh_console) - - return - - # ------------------------------------------------------------------------- - def print_out(self, *objects, sep=' ', end='\n', file=sys.stdout.buffer, flush=True): - - file.write(to_bytes(sep.join(map(lambda x: str(x), objects)))) - if end: - file.write(to_bytes(end)) - if flush: - file.flush() - - # ------------------------------------------------------------------------- - def __call__(self): - """Helper method to make the resulting object callable.""" + def run(self): + """Main routine.""" """ Sample data: @@ -451,150 +172,6 @@ class WebhookDeployApp(object): 'user_name': 'Frank Brehm'} """ - self.print_out("Content-Type: text/plain;charset=utf-8\n") - self.print_out("Python CGI läuft.\n") - - if self.verbose > 1: - LOG.debug("Base directory: {!r}".format(self.base_dir)) - - self.data = sys.stdin.read() - try: - self.json_data = json.loads(self.data) - except Exception as e: - msg = "Got a {n} reading input data as JSON: {e}".format(n=e.__class__.__name__, e=e) - msg += "\nInput data: {!r}".format(self.data) - LOG.error(msg) - self.error_data.append(msg) - else: - - if self.verbose > 1: - LOG.debug("Got JSON data:\n{}".format(pp(self.json_data))) - - try: - self.perform() - except Exception as e: - msg = "Got a {n} performing the deploy: {e}".format(n=e.__class__.__name__, e=e) - msg += "\n\nTraceback:\n{}".format(traceback.format_exc()) - self.error_data.append(msg) - LOG.error(msg) - finally: - if self.full_name: - self.send_error_msgs(self.full_name) - else: - self.send_error_msgs() - LOG.info("Finished.") - sys.exit(0) - - # ------------------------------------------------------------------------- - def send_error_msgs(self, project='undefined'): - - if not self.error_data: - return - - msg = EmailMessage() - - s = '' - if len(self.error_data) > 1: - s = 's' - - body = 'Error{s} while processing {p!r} project:\n\n'.format( - s=s, p=project) - body += '\n\n'.join(self.error_data) - body += '\n\nCheers\nPuppetmaster' - msg.set_content(body) - msg.set_charset('utf-8') - - msg['Subject'] = 'Puppetmaster deploy error{s} for project {p!r}'.format( - s=s, p=project) - msg['From'] = self.sender_address - to_addresses = '' - if self.mail_to_addresses: - to_addresses = ', '.join(self.mail_to_addresses) - else: - to_addresses = self.default_email - msg['To'] = to_addresses - if self.mail_cc_addresses: - msg['CC'] = ', '.join(self.mail_cc_addresses) - - msg['X-Mailer'] = "Puppetmaster deploy script v.{}".format(__version__) - - if self.verbose: - LOG.info("Sending the following mail to {r!r} via {s}:{p}:\n{m}".format( - r=to_addresses, m=msg.as_string(unixfrom=True), - s=self.smtp_server, p=self.smtp_port)) - else: - LOG.info("Sending a mail to {r!r} via {s}:{p}:\n{e}".format( - r=to_addresses, e=pp(self.error_data), - s=self.smtp_server, p=self.smtp_port)) - - server = smtplib.SMTP(self.smtp_server, self.smtp_port) - if 'REQUEST_METHOD' not in os.environ: - if self.verbose > 2: - server.set_debuglevel(2) - elif self.verbose > 1: - server.set_debuglevel(1) - server.starttls() - result = server.send_message(msg) - server.quit() - - if not result.keys(): - LOG.debug("Susseccful sent message to {r!r} via {s}:{p}.".format( - r=to_addresses, s=self.smtp_server, p=self.smtp_port)) - else: - LOG.error(( - "Errors on sending error message for project " - "{pr!r} to {r!r} via {s}:{p}:\n{e}").format( - r=to_addresses, s=self.smtp_server, p=self.smtp_port, - pr=project, e=pp(result))) - - return - - # ------------------------------------------------------------------------- - def perform(self): - '''Performing the stuff...''' - - self.ref = self.json_data['ref'].split('/')[-1] - self.namespace = self.json_data['project']['namespace'] - self.name = self.json_data['project']['name'] - self.full_name = self.json_data['project']['path_with_namespace'] - - if self.full_name in self.ignore_projects or self.name in self.ignore_projects: - LOG.info("Ignoring project {!r}.".format(self.full_name)) - return True - - if self.special_chars_re.search(self.name): - msg = "Project {!r}: ".format(self.full_name) + self.mail_bodies['special_chars'] - LOG.error(msg) - self.error_data.append(msg) - return True - - committers = [] - timeformat = '%Y-%m-%dT%H:%M:%S%z' - for commit in self.json_data['commits']: - ts = commit['timestamp'].split('+') - ts_str = ts[0] - if len(ts) > 1: - ts_str += '+' + ts[1].replace(':', '').ljust(4, '0') - else: - ts_str += '+0000' - timestamp = datetime.datetime.strptime(ts_str, timeformat) - email = commit['author']['email'] - committers.append((timestamp, email)) - - if committers: - committers.sort() - if self.verbose > 1: - LOG.debug("Got committers: {}".format(pp(committers))) - self.mail_to_addresses.append(committers[-1][1]) - else: - LOG.debug("No committers found to append a mail address.") - - if 'git_ssh_url' in self.json_data['project']: - self.git_ssh_url = self.json_data['project']['git_ssh_url'] - else: - self.git_ssh_url = 'git@git.pixelpark.com:{ns}/{n}.git'.format( - ns=self.namespace, n=self.name) - if self.full_name == 'puppet/hiera': cfg = copy.copy(self.projects['hiera']) return self.deploy_hiera(cfg) diff --git a/lib/webhooks/r10k.py b/lib/webhooks/r10k.py index aa9e566..98ed864 100644 --- a/lib/webhooks/r10k.py +++ b/lib/webhooks/r10k.py @@ -45,8 +45,7 @@ class R10kHookApp(BaseHookApp): self.r10k_bin = None self.description = textwrap.dedent('''\ - Receives push events as JSON-Data and synchronizes - the local repository and deploys it with r10k. + Receives push events as JSON-Data and deploys it with r10k. ''').strip() self.locale = 'de_DE.utf8'