from webhooks.common import pp, to_bytes, to_str, to_bool
+from webhooks.base_app import BaseHookApp
+
__version__ = webhooks.__version__
LOG = logging.getLogger(__name__)
-DEFAULT_EMAIL = 'frank.brehm@pixelpark.com'
-DEFAULT_SENDER = 'Puppetmaster <{}>'.format(DEFAULT_EMAIL)
+
DEFAULT_PARENT_DIR = '/www/data'
# =============================================================================
-class WebhookDeployApp(object):
+class WebhookDeployApp(BaseHookApp):
"""
Class for the application objects.
"""
- cgi_bin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
- base_dir = os.path.dirname(cgi_bin_dir)
-
- special_chars_re = re.compile(r'[^a-z0-9_\-]', re.IGNORECASE)
- dev_re = re.compile(r'^dev')
-
- mail_bodies = {
- 'special_chars': "Received special characters in module name",
- 'no_branch_dir': "Branch folder does not exist",
- 'no_modules_dir': "Modules folder does not exist",
- 'git_access_denied': "Access to remote repository was denied",
- }
-
# -------------------------------------------------------------------------
- def __init__(self, appname=None, version=__version__):
+ def __init__(self, appname=None, verbose=0, version=__version__):
"""Constructor."""
- self._appname = None
- """
- @ivar: name of the current running application
- @type: str
- """
- if appname:
- v = str(appname).strip()
- if v:
- self._appname = v
- if not self._appname:
- self._appname = os.path.basename(sys.argv[0])
-
- self._version = version
- """
- @ivar: version string of the current object or application
- @type: str
- """
-
- self._verbose = 1
- """
- @ivar: verbosity level (0 - 9)
- @type: int
- """
-
- self.data = None
- self.json_data = None
- self.ref = None
- self.namespace = None
- self.name = None
- self.full_name = None
- self.git_ssh_url = None
- self.do_sudo = True
-
- self.ignore_projects = []
-
- self.error_data = []
- self.smtp_server = 'smtp.pixelpark.com'
- self.smtp_port = 25
-
- self.default_parent_dir = '/www/data'
- self.default_email = DEFAULT_EMAIL
- self.mail_to_addresses = []
- self.mail_cc_addresses = [
- 'webmaster@pixelpark.com',
- self.default_email,
- ]
- self.sender_address = DEFAULT_SENDER
-
- self._log_directory = os.sep + os.path.join('var', 'log', 'webhooks')
+ self.description = textwrap.dedent('''\
+ Receives push events as JSON-Data and synchronizes
+ the local repository.
+ ''').strip()
self.projects = {
'hiera': {
'namespace': 'puppet',
'name': 'hiera',
'parent_dir': '/www/data/puppet-hiera'
- },
- 'puppet_modules': {
- 'namespace': 'puppet',
- 'parent_dir': '/www/data/puppet-environment'
}
}
- self.read_config()
- self.init_logging()
-
- # -----------------------------------------------------------
- @property
- def appname(self):
- """The name of the current running application."""
- return self._appname
-
- @appname.setter
- def appname(self, value):
- if value:
- v = str(value).strip()
- if v:
- self._appname = v
-
- # -----------------------------------------------------------
- @property
- def version(self):
- """The version string of the current object or application."""
- return self._version
-
- # -----------------------------------------------------------
- @property
- def verbose(self):
- """The verbosity level."""
- return getattr(self, '_verbose', 0)
-
- @verbose.setter
- def verbose(self, value):
- v = int(value)
- if v >= 0:
- self._verbose = v
- else:
- LOG.warn("Wrong verbose level %r, must be >= 0", value)
-
- # -----------------------------------------------------------
- @property
- def log_directory(self):
- """The directory containing the logfiles of this application."""
- return self._log_directory
-
- # -----------------------------------------------------------
- @property
- def logfile(self):
- """The logfile of this application."""
- return os.path.join(self.log_directory, self.appname + '.log')
-
- # -------------------------------------------------------------------------
- def __str__(self):
- """
- Typecasting function for translating object structure
- into a string
-
- @return: structure as string
- @rtype: str
- """
-
- return pp(self.as_dict())
+ super(WebhookDeployApp, self).__init__(
+ appname=appname, verbose=verbose, version=version)
# -------------------------------------------------------------------------
def as_dict(self):
@rtype: dict
"""
- res = self.__dict__
- res = {}
- for key in self.__dict__:
- if key.startswith('_') and not key.startswith('__'):
- continue
- res[key] = self.__dict__[key]
- res['__class_name__'] = self.__class__.__name__
- res['appname'] = self.appname
- res['verbose'] = self.verbose
- res['base_dir'] = self.base_dir
- res['cgi_bin_dir'] = self.cgi_bin_dir
- res['log_directory'] = self.log_directory
- res['logfile'] = self.logfile
- res['mail_bodies'] = self.mail_bodies
+ res = super(WebhookDeployApp, self).as_dict()
return res
- # -------------------------------------------------------------------------
- def read_config(self):
- """Reading configuration from different YAML files."""
-
- yaml_files = []
- # ./deploy.yaml
- yaml_files.append(os.path.join(self.base_dir, self.appname + '.yaml'))
- # /etc/pixelpark/deploy.yaml
- yaml_files.append(os.sep + os.path.join('etc', 'pixelpark', self.appname + '.yaml'))
-
- for yaml_file in yaml_files:
- self.read_from_yaml(yaml_file)
# -------------------------------------------------------------------------
- def read_from_yaml(self, yaml_file):
- """Reading configuration from given YAML file."""
-
- # LOG.debug("Trying to read config from {!r} ...".format(yaml_file))
- if not os.access(yaml_file, os.F_OK):
- # LOG.debug("File {!r} does not exists.".format(yaml_file))
- return
- # LOG.debug("Reading config from {!r} ...".format(yaml_file))
- config = {}
- with open(yaml_file, 'rb') as fh:
- config = yaml.load(fh.read())
- # LOG.debug("Read config:\n{}".format(pp(config)))
-
- if 'verbose' in config:
- self.verbose = config['verbose']
-
- if 'do_sudo' in config:
- self.do_sudo = to_bool(config['do_sudo'])
-
- if 'log_dir' in config and config['log_dir']:
- self._log_directory = config['log_dir']
-
- if 'default_email' in config and config['default_email']:
- self.default_email = config['default_email']
-
- if 'smtp_server' in config and config['smtp_server'].strip():
- self.smtp_server = config['smtp_server'].strip()
-
- if 'smtp_port' in config and config['smtp_port']:
- msg = "Invalid port {p!r} for SMTP in {f!r} found.".format(
- p=config['smtp_port'], f=yaml_file)
- try:
- port = int(config['smtp_port'])
- if port > 0 and port < 2**16:
- self.smtp_port = port
- else:
- self.error_data.append(msg)
- except ValueError:
- self.error_data.append(msg)
-
- if 'default_parent_dir' in config and config['default_parent_dir']:
- pdir = config['default_parent_dir']
- if os.path.isabs(pdir):
- self.default_parent_dir = pdir
-
- if 'ignore_projects' in config:
- if config['ignore_projects'] is None:
- self.ignore_projects = []
- elif isinstance(config['ignore_projects'], str):
- if config['ignore_projects']:
- self.ignore_projects = [config['ignore_projects']]
- elif isinstance(config['ignore_projects'], list):
- self.ignore_projects = config['ignore_projects']
-
- if 'mail_cc_addresses' in config:
- if config['mail_cc_addresses'] is None:
- self.mail_cc_addresses = []
- elif isinstance(config['mail_cc_addresses'], str):
- if config['mail_cc_addresses']:
- self.mail_cc_addresses = [config['mail_cc_addresses']]
- else:
- self.mail_cc_addresses = []
- elif isinstance(config['mail_cc_addresses'], list):
- self.mail_cc_addresses = config['mail_cc_addresses']
+ def evaluate_config(self, config, yaml_file):
- if 'mail_bodies' in config and isinstance(config['mail_bodies'], list):
- for key in config['mail_bodies'].keys():
- self.mail_bodies[key] = config['mail_bodies'][key]
+ super(WebhookDeployApp, self).evaluate_config(config, yaml_file)
if 'projects' in config and isinstance(config['projects'], dict):
self.config_projects(config['projects'])
if 'parent_dir' in cfg and cfg['parent_dir']:
self.projects['hiera']['parent_dir'] = cfg['parent_dir']
continue
- if project_key == 'puppet_modules':
- cfg = project_cfg['puppet_modules']
- if 'namespace' in cfg and cfg['namespace'].strip():
- self.projects['puppet_modules']['namespace'] = cfg['namespace'].strip()
- if 'parent_dir' in cfg and cfg['parent_dir']:
- self.projects['puppet_modules']['parent_dir'] = cfg['parent_dir']
- continue
if project_key not in self.projects:
self.projects[project_key] = {}
cfg = project_cfg[project_key]
self.projects[project_key]['branch'] = cfg['branch'].strip()
# -------------------------------------------------------------------------
- def init_logging(self):
- """
- Initialize the logger object.
- It creates a colored loghandler with all output to STDERR.
- Maybe overridden in descendant classes.
-
- @return: None
- """
-
- root_log = logging.getLogger()
- root_log.setLevel(logging.INFO)
- if self.verbose:
- root_log.setLevel(logging.DEBUG)
-
- # create formatter
- format_str = ''
- if 'REQUEST_METHOD' in os.environ or self.verbose > 1:
- format_str = '[%(asctime)s]: '
- format_str += self.appname + ': '
- if self.verbose:
- if self.verbose > 1:
- format_str += '%(name)s(%(lineno)d) %(funcName)s() '
- else:
- format_str += '%(name)s '
- format_str += '%(levelname)s - %(message)s'
- formatter = logging.Formatter(format_str)
-
- if 'REQUEST_METHOD' in os.environ:
-
- #sys.stderr.write("Trying to open logfile {!r} ...\n".format(self.logfile))
- # we are in a CGI environment
- if os.path.isdir(self.log_directory) and os.access(self.log_directory, os.W_OK):
- lh_file = logging.FileHandler(
- self.logfile, mode='a', encoding='utf-8', delay=True)
- if self.verbose:
- lh_file.setLevel(logging.DEBUG)
- else:
- lh_file.setLevel(logging.INFO)
- lh_file.setFormatter(formatter)
- root_log.addHandler(lh_file)
-
- else:
- # create log handler for console output
- lh_console = logging.StreamHandler(sys.stderr)
- if self.verbose:
- lh_console.setLevel(logging.DEBUG)
- else:
- lh_console.setLevel(logging.INFO)
- lh_console.setFormatter(formatter)
-
- root_log.addHandler(lh_console)
-
- return
-
- # -------------------------------------------------------------------------
- def print_out(self, *objects, sep=' ', end='\n', file=sys.stdout.buffer, flush=True):
-
- file.write(to_bytes(sep.join(map(lambda x: str(x), objects))))
- if end:
- file.write(to_bytes(end))
- if flush:
- file.flush()
-
- # -------------------------------------------------------------------------
- def __call__(self):
- """Helper method to make the resulting object callable."""
+ def run(self):
+ """Main routine."""
"""
Sample data:
'user_name': 'Frank Brehm'}
"""
- self.print_out("Content-Type: text/plain;charset=utf-8\n")
- self.print_out("Python CGI läuft.\n")
-
- if self.verbose > 1:
- LOG.debug("Base directory: {!r}".format(self.base_dir))
-
- self.data = sys.stdin.read()
- try:
- self.json_data = json.loads(self.data)
- except Exception as e:
- msg = "Got a {n} reading input data as JSON: {e}".format(n=e.__class__.__name__, e=e)
- msg += "\nInput data: {!r}".format(self.data)
- LOG.error(msg)
- self.error_data.append(msg)
- else:
-
- if self.verbose > 1:
- LOG.debug("Got JSON data:\n{}".format(pp(self.json_data)))
-
- try:
- self.perform()
- except Exception as e:
- msg = "Got a {n} performing the deploy: {e}".format(n=e.__class__.__name__, e=e)
- msg += "\n\nTraceback:\n{}".format(traceback.format_exc())
- self.error_data.append(msg)
- LOG.error(msg)
- finally:
- if self.full_name:
- self.send_error_msgs(self.full_name)
- else:
- self.send_error_msgs()
- LOG.info("Finished.")
- sys.exit(0)
-
- # -------------------------------------------------------------------------
- def send_error_msgs(self, project='undefined'):
-
- if not self.error_data:
- return
-
- msg = EmailMessage()
-
- s = ''
- if len(self.error_data) > 1:
- s = 's'
-
- body = 'Error{s} while processing {p!r} project:\n\n'.format(
- s=s, p=project)
- body += '\n\n'.join(self.error_data)
- body += '\n\nCheers\nPuppetmaster'
- msg.set_content(body)
- msg.set_charset('utf-8')
-
- msg['Subject'] = 'Puppetmaster deploy error{s} for project {p!r}'.format(
- s=s, p=project)
- msg['From'] = self.sender_address
- to_addresses = ''
- if self.mail_to_addresses:
- to_addresses = ', '.join(self.mail_to_addresses)
- else:
- to_addresses = self.default_email
- msg['To'] = to_addresses
- if self.mail_cc_addresses:
- msg['CC'] = ', '.join(self.mail_cc_addresses)
-
- msg['X-Mailer'] = "Puppetmaster deploy script v.{}".format(__version__)
-
- if self.verbose:
- LOG.info("Sending the following mail to {r!r} via {s}:{p}:\n{m}".format(
- r=to_addresses, m=msg.as_string(unixfrom=True),
- s=self.smtp_server, p=self.smtp_port))
- else:
- LOG.info("Sending a mail to {r!r} via {s}:{p}:\n{e}".format(
- r=to_addresses, e=pp(self.error_data),
- s=self.smtp_server, p=self.smtp_port))
-
- server = smtplib.SMTP(self.smtp_server, self.smtp_port)
- if 'REQUEST_METHOD' not in os.environ:
- if self.verbose > 2:
- server.set_debuglevel(2)
- elif self.verbose > 1:
- server.set_debuglevel(1)
- server.starttls()
- result = server.send_message(msg)
- server.quit()
-
- if not result.keys():
- LOG.debug("Susseccful sent message to {r!r} via {s}:{p}.".format(
- r=to_addresses, s=self.smtp_server, p=self.smtp_port))
- else:
- LOG.error((
- "Errors on sending error message for project "
- "{pr!r} to {r!r} via {s}:{p}:\n{e}").format(
- r=to_addresses, s=self.smtp_server, p=self.smtp_port,
- pr=project, e=pp(result)))
-
- return
-
- # -------------------------------------------------------------------------
- def perform(self):
- '''Performing the stuff...'''
-
- self.ref = self.json_data['ref'].split('/')[-1]
- self.namespace = self.json_data['project']['namespace']
- self.name = self.json_data['project']['name']
- self.full_name = self.json_data['project']['path_with_namespace']
-
- if self.full_name in self.ignore_projects or self.name in self.ignore_projects:
- LOG.info("Ignoring project {!r}.".format(self.full_name))
- return True
-
- if self.special_chars_re.search(self.name):
- msg = "Project {!r}: ".format(self.full_name) + self.mail_bodies['special_chars']
- LOG.error(msg)
- self.error_data.append(msg)
- return True
-
- committers = []
- timeformat = '%Y-%m-%dT%H:%M:%S%z'
- for commit in self.json_data['commits']:
- ts = commit['timestamp'].split('+')
- ts_str = ts[0]
- if len(ts) > 1:
- ts_str += '+' + ts[1].replace(':', '').ljust(4, '0')
- else:
- ts_str += '+0000'
- timestamp = datetime.datetime.strptime(ts_str, timeformat)
- email = commit['author']['email']
- committers.append((timestamp, email))
-
- if committers:
- committers.sort()
- if self.verbose > 1:
- LOG.debug("Got committers: {}".format(pp(committers)))
- self.mail_to_addresses.append(committers[-1][1])
- else:
- LOG.debug("No committers found to append a mail address.")
-
- if 'git_ssh_url' in self.json_data['project']:
- self.git_ssh_url = self.json_data['project']['git_ssh_url']
- else:
- self.git_ssh_url = 'git@git.pixelpark.com:{ns}/{n}.git'.format(
- ns=self.namespace, n=self.name)
-
if self.full_name == 'puppet/hiera':
cfg = copy.copy(self.projects['hiera'])
return self.deploy_hiera(cfg)
projects / pixelpark / puppetmaster-webhooks.git / commitdiff