From 594290a05cb2043fdebbae4ecc8ab8533eda68b9 Mon Sep 17 00:00:00 2001 From: Robin Wittler Date: Mon, 1 Aug 2011 15:26:40 +0200 Subject: [PATCH] initial commit --- lib/apt_handler.py | 97 +++++++++++++++++++++ testgit_build.py | 212 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 lib/apt_handler.py create mode 100644 testgit_build.py diff --git a/lib/apt_handler.py b/lib/apt_handler.py new file mode 100644 index 0000000..a00d4e3 --- /dev/null +++ b/lib/apt_handler.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import apt +import difflib + +def get_apt_cache(): + cache = apt.cache.Cache() + cache.update() + return cache + +def guess_packages_by_name(name, cache=None, cache_update=False, + count=5, cutoff=0.6): + cache = cache or apt.cache.Cache() + if cache_update: + cache.update() + result = difflib.get_close_matches( + name, + cache.keys(), + count, + cutoff + ) + return map(lambda x: get_package_by_name(x, cache=cache), result) + +def get_package_by_name(name, cache=None, cache_update=False): + cache = cache or apt.cache.Cache() + if cache_update: + cache.update() + try: + return cache[name] + except KeyError: + return None + +def get_providing_packages_by_name(name, cache=None, cache_update=False): + cache = cache or apt.cache.Cache() + if cache_update: + cache.update() + return cache.get_providing_packages(name) + +def find_linux_headers_packages(cache=None, cache_update=False): + return filter(lambda x: x.name.startswith('linux-headers-2.6.') and + 'vserver' not in x.name and + 'xen' not in x.name and + 'openvz' not in x.name, + get_providing_packages_by_name( + 'linux-headers', + cache=cache, + cache_update=cache_update + ) + ) + +def find_linux_image_packages(cache=None, cache_update=False): + return filter(lambda x: x.name.startswith('linux-image-2.6.') and + 'vserver' not in x.name and + 'xen' not in x.name and + 'openvz' not in x.name, + get_providing_packages_by_name( + 'linux-image', + cache=cache, + cache_update=cache_update + + ) + ) + +def find_linux_source_packages(cache=None, cache_update=False): + return filter(lambda x: x.name.startswith('linux-source-2.6.') and + 'vserver' not in x.name and + 'xen' not in x.name and + 'openvz' not in x.name, + get_providing_packages_by_name( + 'linux-source', + cache=cache, + cache_update=cache_update + ) + ) + +def find_gcc_compiler_packages(cache=None, cache_update=False): + return filter(lambda x: x.name.startswith('gcc-') and + 'vserver' not in x.name and + 'xen' not in x.name and + 'openvz' not in x.name, + get_providing_packages_by_name( + 'c-compiler', + cache=cache, + cache_update=cache_update + ) + ) + +def find_iptables_packages(cache=None, cache_update=False): + return [get_package_by_name('iptables', cache=cache, + cache_update=cache_update)] + +def find_ebtables_packages(cache=None, cache_update=False): + return [get_package_by_name('ebtables', cache=cache, + cache_update=cache_update)] + + diff --git a/testgit_build.py b/testgit_build.py new file mode 100644 index 0000000..d3b211f --- /dev/null +++ b/testgit_build.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import git +import atexit +import shutil +import subprocess +import logging +import platform +import smtplib +from glob import glob +from logging import Formatter +from ftplib import FTP +from multiprocessing import cpu_count +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, Column, Integer, String, MetaData, Sequence +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm.exc import NoResultFound + +GIT = '/usr/bin/git' +MAKE_KPKG = '/usr/bin/make-kpkg' +DEFAULT_PARALLEL_JOBS = cpu_count() + 1 + +BUILD_ARCH_MAP = { + 'x86_64': 'amd64', + 'i386': '686' +} + +BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686') + +CWD = os.environ.get('WORKSPACE') +BUILD_NUMBER = os.environ.get('BUILD_NUMBER') +BUILD_ID = os.environ.get('BUILD_ID') +BUILD_URL = os.environ.get('BUILD_URL') + +GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') +GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH) +GIT_OLD_ID = os.environ.get('GIT_OLD_ID') +GIT_NEW_ID = os.environ.get('GIT_NEW_ID') +GIT_BRANCH_NAME = os.environ.get('GIT_BRANCH_NAME') +GIT_REMOTE_BRANCH_NAME = os.path.join('origin', '%s' %(GIT_BRANCH_NAME)) +GIT_TARGET_WORKSPACE = os.path.join( + CWD, + '%s-build%s' %(BUILD_ID, BUILD_NUMBER) +) + +GIT_TARGET_DIR = os.path.join( + GIT_TARGET_WORKSPACE, + os.path.basename(GIT_REPO_PATH) +) + +GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL') +SMTP_SERVER = 'roma.profitbricks.localdomain' +SMTP_SUBJECT = 'Build for branch %s, buildnumber %s was %s' +SMTP_TEXT = ( + 'Build for branch %s, buildnumber %s was %s. ' + + 'Take a close look at: ' + BUILD_URL +) +SMTP_BUILD_SUCCESS = 'SUCCESSFULL' +SMTP_BUILD_ERROR = 'NOT SUCCESSFULL' +SMTP_FROM = '%s@profitbricks.com' %(pwd.getpwuid(os.geteuid()).pw_name) + +logger = logging.getLogger(sys.argv[0]) +logger.setLevel(logging.DEBUG) +stream_handler = logging.StreamHandler() +stream_handler.setLevel(logging.DEBUG) +formatter = Formatter('%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s') +stream_handler.setFormatter(formatter) +logger.addHandler(stream_handler) + +def send_email(result): + smtp = smtplib.SMTP(SMTP_SERVER) + msg = ( + 'From: %s\n' %(SMTP_FROM) + + 'To: %s\n' %(GIT_COMMITTER_EMAIL) + + 'Subject: %s\n' %(SMTP_SUBJECT %(GIT_BRANCH_NAME, BUILD_NUMBER, + result)) + + '%s\n' %(SMTP_TEXT %(GIT_BRANCH_NAME, BUILD_NUMBER, result)) + ) + smtp.sendmail(SMTP_FROM, GIT_COMMITTER_EMAIL, msg) + smtp.quit() + +def git_clone_remote_repository(url, destination): + if os.path.exists(destination): + logger.debug('%s allready exists' %(destination)) + if os.path.isdir(destination): + shutil.rmtree(destination) + else: + os.unlink(destination) + logger.debug('%s deleted' %(destination)) + + cmd = [GIT, 'clone', '%s' %(url), '%s' %(destination)] + cmdobj = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + env={'':''}, + cwd=CWD, + close_fds=True + ) + + logger.info('begin to clone git repo from %s' %(url)) + logger.debug( + 'calling »%s« for cloning git repo' %(' '.join(cmd)) + ) + ret = cmdobj.wait() + + if ret: + logger.error('%s returned with %s' %(' '.join(cmd), ret)) + logger.error('Error was: %s' %(cmdobj.stderr.readlines())) + return False + logger.debug('repository %s checked out into %s' %(url, destination)) + return True + +def git_checkout_branch(): + cmd = [GIT, 'checkout', '-b', GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME] + + cmdobj = subprocess.Popen( + cmd, + shell=False, + close_fds=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env={'':''}, + cwd=GIT_TARGET_DIR + ) + + logger.info( + 'checking out local branch %s from remote branch %s' + %(GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME) + ) + + logger.debug( + 'calling »%s« for checkout' %(' '.join(cmd)) + ) + + ret = cmdobj.wait() + if ret: + logger.error('%s returned with %s' %(' '.join(cmd), ret)) + logger.error('Error was: %s' %(cmdobj.stderr.readlines())) + return False + logger.info( + 'local branch %s successfully checked out.' %(GIT_BRANCH_NAME) + ) + return True + +def read_file(path): + try: + fh = open(path, 'r', 1) + except: + raise + else: + result = dict(enumerate(fh)) + fh.close() + return result + +def remove_git_target_workspace(): + try: + shutil.rmtree(GIT_TARGET_WORKSPACE) + except IOError, error: + if error.errno == 2: + pass + else: + raise + logger.info('deleted %s' %(GIT_TARGET_WORKSPACE)) + +def exit_ok(): + send_email(SMTP_BUILD_SUCCESS) + sys.exit(0) + +def exit_error(): + send_email(SMTP_BUILD_ERROR) + sys.exit(1) + +if __name__ == '__main__': + if git_clone_remote_repository(GIT_REPO_PATH, GIT_TARGET_DIR): + logger.info('git clone was successfull') + else: + logger.info('git clone was not successfull') + exit_error() + atexit.register(remove_git_target_workspace) + if not git_checkout_branch(): + sys.exit(1) + logger.debug('changing dir to %s' %(GIT_TARGET_DIR)) + os.chdir(GIT_TARGET_DIR) + + #ftp = FTP( + # 'alexandria.profitbricks.localdomain', + # 'debian-uploader', + # 'vae6tooZe1ec' + #) +# + #logger.info('Log in on %s successfull' %(ftp.host)) +# + #ftp.cwd('squeeze') + #for package in glob( + # os.path.join(GIT_TARGET_WORKSPACE, '*.deb') + #): + # fh = open(package, 'rb', 1) + # ftp.storbinary( + # 'STOR %s' %(os.path.basename(package)), + # fh + # ) + # fh.close() + # logger.info('Successfully uploaded %s' %(package)) + #ftp.quit() + exit_ok() -- 2.39.5