From: Holger Levsen Date: Wed, 14 Sep 2011 08:06:57 +0000 (+0200) Subject: add build.py (renamed to kernel_build.py) and inpbuilder.py (renamed to X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=3ee150d9974e2ef4c6b674d8f4b2a2d54793853a;p=profitbricks%2Fjenkins-build-scripts.git add build.py (renamed to kernel_build.py) and inpbuilder.py (renamed to kernel_inpbuilder.py) from old svn repo --- diff --git a/kernel_build.py b/kernel_build.py new file mode 100644 index 0000000..a3e4a7b --- /dev/null +++ b/kernel_build.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import git +import atexit +import shutil +import subprocess +import logging +import platform +import smtplib +from glob import glob +from logging import Formatter +from ftplib import FTP +from multiprocessing import cpu_count +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, Column, Integer, String, MetaData, Sequence +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm.exc import NoResultFound + +GIT = '/usr/bin/git' +MAKE_KPKG = '/usr/bin/make-kpkg' +DEFAULT_PARALLEL_JOBS = cpu_count() + 1 + +BUILD_ARCH_MAP = { + 'x86_64': 'amd64', + 'i386': '686' +} + +BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686') + +CWD = os.environ.get('WORKSPACE') +BUILD_NUMBER = os.environ.get('BUILD_NUMBER') +BUILD_ID = os.environ.get('BUILD_ID') +BUILD_URL = os.environ.get('BUILD_URL') + +GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') +GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH) +GIT_OLD_ID = os.environ.get('GIT_OLD_ID') +GIT_NEW_ID = os.environ.get('GIT_NEW_ID') +GIT_BRANCH_NAME = os.environ.get('GIT_BRANCH_NAME') +GIT_REMOTE_BRANCH_NAME = os.path.join('origin', '%s' %(GIT_BRANCH_NAME)) +GIT_TARGET_WORKSPACE = os.path.join( + CWD, + '%s-build%s' %(BUILD_ID, BUILD_NUMBER) +) + +GIT_TARGET_DIR = os.path.join( + GIT_TARGET_WORKSPACE, + os.path.basename(GIT_REPO_PATH) +) + +GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL') + +KERNEL_CONFIG_PATH = os.path.join(GIT_TARGET_DIR, '.config') +CREG_KERNEL_CONFIG_DELIM = re.compile(ur'\s*=\s*') +PERSISTENCE_FILE = os.path.join(CWD, '..', '.persistence') +SMTP_SERVER = 'roma.profitbricks.localdomain' +SMTP_SUBJECT = 'Kernel build for branch %s, buildnumber %s was %s' +SMTP_TEXT = ( + 'Kernel build for branch %s, buildnumber %s was %s. ' + + 'Take a close look at: ' + BUILD_URL +) +SMTP_BUILD_SUCCESS = 'SUCCESSFULL' +SMTP_BUILD_ERROR = 'NOT SUCCESSFULL' +SMTP_FROM = 'hudson@profitbricks.com' +PBUILDER = '/usr/sbin/pbuilder' +SUDO = '/usr/bin/sudo' + +logger = logging.getLogger(sys.argv[0]) +logger.setLevel(logging.DEBUG) +stream_handler = logging.StreamHandler() +stream_handler.setLevel(logging.DEBUG) +formatter = Formatter('%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s') +stream_handler.setFormatter(formatter) +logger.addHandler(stream_handler) + +Base = declarative_base() + +engine = create_engine('sqlite:///%s' %(PERSISTENCE_FILE)) +Session = sessionmaker(bind=engine) + +def send_email(result): + smtp = smtplib.SMTP(SMTP_SERVER) + msg = ( + 'From: %s\n' %(SMTP_FROM) + + 'To: %s\n' %(GIT_COMMITTER_EMAIL) + + 'Subject: %s\n' %(SMTP_SUBJECT %(GIT_BRANCH_NAME, BUILD_NUMBER, + result)) + + '%s\n' %(SMTP_TEXT %(GIT_BRANCH_NAME, BUILD_NUMBER, result)) + ) + smtp.sendmail(SMTP_FROM, GIT_COMMITTER_EMAIL, msg) + smtp.quit() + +class KernelVersion(Base): + __tablename__ = 'branch_kernel_version' + id = Column(Integer, Sequence('branch_id'), primary_key=True) + branch_name = Column(String(255), unique=True, nullable=False) + last_version = Column(Integer, nullable=False) + + def __init__(self, branch_name, last_version): + self.branch_name = branch_name + self.last_version = last_version + + def __repr__(self): + return '' %( + self.branch_name, self.last_version + ) + +Base.metadata.create_all(engine) + +def get_last_kernel_revision_obj(): + session = Session() + logger.debug('Getting kernel revision from persistence') + try: + answ = session.query(KernelVersion).filter( + KernelVersion.branch_name == GIT_BRANCH_NAME).one() + except NoResultFound: + session.add(KernelVersion(GIT_BRANCH_NAME, 0)) + session.commit() + answ = session.query(KernelVersion).filter( + KernelVersion.branch_name == GIT_BRANCH_NAME).one() + finally: + session.close() + + logger.info( + 'Got this Kernel revision for branch %s: %s' + %(GIT_BRANCH_NAME, answ) + ) + return answ + +def update_kernel_revision_obj(kernel_version_obj, new_revision): + session = Session() + kernel_version_obj.last_version = new_revision + session.add(kernel_version_obj) + try: + session.commit() + except Exception, error: + logger.error('Some error happend while commiting new revision') + logger.exception(error) + return False + else: + return True + finally: + session.close() + + +def build_kernel_with_pbuilder(revision, pbuilder_script, dist='stable'): + pbuilder_script = os.path.abspath(pbuilder_script) + cmd = [ + SUDO, + PBUILDER, + '--execute', + '--hookdir', '""', + '--', + pbuilder_script, + '--cwd', '%s' %('/tmp'), + '--build-number', '%s' %(BUILD_NUMBER), + '--build-id', '%s' %(BUILD_ID), + '--build-url', '%s' %(BUILD_URL), + '--git-repo-path', '%s' %(GIT_REPO_PATH), + '--git-repo-name', '%s' %(GIT_REPO_NAME), + '--git-old-id', '%s' %(GIT_OLD_ID), + '--git-new-id', '%s' %(GIT_NEW_ID), + '--git-branch-name', '%s' %(GIT_BRANCH_NAME), + '--revision', '%s' %(revision), + '--dist', '%s' %(dist), + ] + + os.environ.update({'DIST': '%s' %(dist)}) + + cmdobj = subprocess.Popen( + cmd, + shell=False, + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ + ) + + logger.info('calling "%s" ...', ' '.join(cmd)) +# ret = cmdobj.wait() + ret = os.system('%s' %(' '.join(cmd))) + if ret: +# errormsg = 'stdout: %s\n\n\nstderr: %s' %(cmdobj.stdout.read(), +# cmdobj.stderr.read()) +# if not errormsg: +# errormsg = None + logger.error( + '"%s" returned non-zero (exitcode was: %s).', + ' '.join(cmd), + ret, + ) + return False + +# message = cmdobj.stdout.read() +# if not message: +# message = None + logger.info( + '"%s" returned zero.', ' '.join(cmd)) + return True + +def read_file(path): + try: + fh = open(path, 'r', 1) + except: + raise + else: + result = dict(enumerate(fh)) + fh.close() + return result + +def remove_git_target_workspace(): + try: + shutil.rmtree(GIT_TARGET_WORKSPACE) + except IOError, error: + logger.debug('Got exception with error code: %s', error.errno) + if error.errno == 2: + pass + else: + raise + logger.info('deleted %s' %(GIT_TARGET_WORKSPACE)) + +def exit_ok(): + send_email(SMTP_BUILD_SUCCESS) + sys.exit(0) + +def exit_error(): + send_email(SMTP_BUILD_ERROR) + sys.exit(1) + +if __name__ == '__main__': + logger.debug('running with this enviroment: %s', os.environ) + atexit.register(remove_git_target_workspace) + kernel_revision_obj = get_last_kernel_revision_obj() + new_version = kernel_revision_obj.last_version + 1 + if not update_kernel_revision_obj(kernel_revision_obj, new_version): + logger.info( + 'Could not update persistence version to %s for %s' + %(new_version, GIT_BRANCH_NAME) + ) + exit_error() + + if not build_kernel_with_pbuilder( + new_version, 'kernel_inpbuilder.py', + dist='stable'): + exit_error() + exit_ok() diff --git a/kernel_inpbuilder.py b/kernel_inpbuilder.py new file mode 100644 index 0000000..a39c18b --- /dev/null +++ b/kernel_inpbuilder.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import git +import atexit +import shutil +import subprocess +import logging +import platform +import smtplib +import optparse +from glob import glob +from logging import Formatter +#from ftplib import FTP +from multiprocessing import cpu_count + +__version__ = '0.0.1' + +GIT = '/usr/bin/git' +MAKE_KPKG = '/usr/bin/make-kpkg' +APT_GET = '/usr/bin/apt-get' + +DEFAULT_PARALLEL_JOBS = cpu_count() + 1 + +BUILD_ARCH_MAP = { + 'x86_64': 'amd64', + 'i386': '686' +} + +BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686') +CREG_KERNEL_CONFIG_DELIM = re.compile(ur'\s*=\s*') +ERROR = 1 +OK = 0 + +def getopts(): + usage = '%prog [options]' + parser = optparse.OptionParser( + usage=usage, + version='%prog ' + __version__ + ) + + parser.add_option( + '--cwd', + dest='cwd', + default='/tmp', + metavar='CWD', + help='Set the current working dir. Default: %default' + ) + + parser.add_option( + '--build-number', + dest='build_number', + default=None, + metavar='BUILD_NUMBER', + help='Set the Build number. Default: %default' + ) + + parser.add_option( + '--build-id', + dest='build_id', + default=None, + metavar='BUILD_ID', + help='Set the Build Id. Default: %default' + ) + + parser.add_option( + '--build-url', + dest='build_url', + default=None, + metavar='BUILD_URL', + help='Set the Build Url. Default: %default' + ) + + parser.add_option( + '--git-repo-path', + dest='git_repo_path', + default=None, + metavar='GIT_REPO_PATH', + help='Set the Git Repo Path. Default: %default' + ) + + parser.add_option( + '--git-repo-name', + dest='git_repo_name', + default=None, + metavar='GIT_REPO_NAME', + help='Set the Git Repo Name. Default: %default' + ) + + parser.add_option( + '--git-old-id', + dest='git_old_id', + default=None, + metavar='GIT_OLD_ID', + help='Set the Git Old Id. Default: %default' + ) + + parser.add_option( + '--git-new-id', + dest='git_new_id', + default=None, + metavar='GIT_NEW_ID', + help='Set the Git New Id. Default: %default' + ) + + parser.add_option( + '--git-branch-name', + dest='git_branch_name', + default=None, + metavar='GIT_BRANCH_NAME', + help='Set the Name of the branch. Default: %default' + ) + + parser.add_option( + '--revision', + dest='revision', + default=None, + metavar='REVISION', + help='Set the Revision. Default: %default' + ) + + parser.add_option( + '--dist', + dest='dist', + default='stable', + metavar='DIST', + help='Set the distribution. Default: %default' + ) + + return parser.parse_args() + +def git_clone_remote_repository(url, destination): + if os.path.exists(destination): + logger.debug('%s allready exists' %(destination)) + if os.path.isdir(destination): + shutil.rmtree(destination) + else: + os.unlink(destination) + logger.debug('%s deleted' %(destination)) + + cmd = [GIT, 'clone', '%s' %(url), '%s' %(destination)] + cmdobj = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + env={'':''}, + cwd=CWD, + close_fds=True + ) + + logger.info('begin to clone git repo from %s' %(url)) + logger.debug( + 'calling »%s« for cloning git repo' %(' '.join(cmd)) + ) + ret = cmdobj.wait() + + if ret: + logger.error('%s returned with %s' %(' '.join(cmd), ret)) + logger.error('Error was: %s' %(cmdobj.stderr.readlines())) + return False + logger.debug('repository %s checked out into %s' %(url, destination)) + return True + +def git_checkout_branch(): + cmd = [GIT, 'checkout', '-b', GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME] + + cmdobj = subprocess.Popen( + cmd, + shell=False, + close_fds=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env={'':''}, + cwd=GIT_TARGET_DIR + ) + + logger.info( + 'checking out local branch %s from remote branch %s' + %(GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME) + ) + + logger.debug( + 'calling »%s« for checkout' %(' '.join(cmd)) + ) + + ret = cmdobj.wait() + if ret: + logger.error('%s returned with %s' %(' '.join(cmd), ret)) + logger.error('Error was: %s' %(cmdobj.stderr.readlines())) + return False + logger.info( + 'local branch %s successfully checked out.' %(GIT_BRANCH_NAME) + ) + return True + +def build_kernel(revision, parallel_jobs='auto', distcc=False): + if parallel_jobs == 'auto': + parallel_jobs = DEFAULT_PARALLEL_JOBS + else: + parallel_jobs = int(parallel_jobs) + + cmd = [MAKE_KPKG, '-j', '%s' %(parallel_jobs), '--arch', + '%s' %(BUILD_ARCH), '--rootcmd', 'fakeroot', '--revision', + '%s' %(revision), '--initrd', '--arch-in-name', 'kernel_debug', + 'kernel_image', 'kernel_source', 'kernel_headers', 'modules'] + + logger.info('start compile process') + + logger.debug( + 'calling "%s" for compiling the kernel' %(' '.join(cmd)) + ) + + os.putenv('LOCALVERSION', '') + os.environ['LOCALVERSION'] = '' + if distcc: + os.environ['MAKEFLAGS'] = 'CC=distcc' + os.putenv('MAKEFLAGS', 'CC=distcc') + + ret = os.system('%s' %(' '.join(cmd))) + + if ret: + logger.error('%s returned with %s' %(' '.join(cmd), ret)) + return False + logger.info( + 'kernel successfully compiled.' + ) + return True + +def read_file(path): + try: + fh = open(path, 'r', 1) + except: + raise + else: + result = dict(enumerate(fh)) + fh.close() + return result + +def add_local_version_to_config(kernel_build_revision): + if not os.path.exists(KERNEL_CONFIG_PATH): + raise Exception( + 'No config file found at %s' %(KERNEL_CONFIG_PATH) + ) + kernel_config = read_file(KERNEL_CONFIG_PATH) + localversion = '-%s-%s' %(GIT_BRANCH_NAME, BUILD_ARCH) + for lino, line in kernel_config.iteritems(): + try: + key, value = CREG_KERNEL_CONFIG_DELIM.split(line) + except ValueError: + # found a comment line or something else + # we dont want. + continue + if key == 'CONFIG_LOCALVERSION': + value = localversion + kernel_config.update(((lino, '%s="%s"\n' %(key, value)),)) + break + else: + # no CONFIG_LOCALVERSION in kernel config + lino += 1 + kernel_config.setdefault( + lino, + 'CONFIG_LOCALVERSION="%s"\n' %(localversion) + ) + + try: + fh = open(KERNEL_CONFIG_PATH, 'w', 1) + except: + raise + else: + for lino, line in sorted( + kernel_config.iteritems(), + key=lambda x: x[0] + ): + fh.write(line) + fh.close() + return localversion + return False + +def remove_git_target_workspace(): + try: + shutil.rmtree(GIT_TARGET_WORKSPACE) + except IOError, error: + if error.errno == 2: + pass + else: + raise + logger.info('deleted %s' %(GIT_TARGET_WORKSPACE)) + +def exit(retcode): + logger.info('Exit with %s', retcode) + sys.exit(retcode) + + +def has_extra_modules(): + return os.path.exists(EXTRA_MODULES_PATH) + +def install_extra_modules(): + cmd = [APT_GET, 'update'] + subprocess.check_call(cmd) + + fh = open(EXTRA_MODULES_PATH, 'r', 1) + extra_modules = map(lambda x: x.rstrip(), fh) + fh.close() + + if extra_modules: + cmd = [APT_GET, 'install'] + cmd += extra_modules + subprocess.check_call(cmd) + return True + + +if __name__ == '__main__': + logger = logging.getLogger(sys.argv[0]) + logger.setLevel(logging.DEBUG) + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(logging.DEBUG) + formatter = Formatter('%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s') + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + + options, args = getopts() + logger.debug('running with this enviroment: %s', os.environ) + + CWD = os.environ.get('WORKSPACE') or options.cwd + if not CWD: + logger.info('No CWD given - using "/tmp" as CWD') + CWD = '/tmp' + + BUILD_NUMBER = os.environ.get('BUILD_NUMBER') or options.build_number + if not BUILD_NUMBER: + logger.error('No build number given') + exit(ERROR) + + BUILD_ID = os.environ.get('BUILD_ID') or options.build_id + if not BUILD_ID: + logger.error('No build id given') + exit(ERROR) + + # FIXME: Do we realy need BUILD_URL here? + BUILD_URL = os.environ.get('BUILD_URL') or options.build_url + if not BUILD_URL: + logger.error('No build url given') + exit(ERROR) + + GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') or options.git_repo_path + if not GIT_REPO_PATH: + logger.error('No git repo path given') + exit(ERROR) + + GIT_OLD_ID = os.environ.get('GIT_OLD_ID') or options.git_old_id + if not GIT_OLD_ID: + logger.error('No git old id given') + exit(ERROR) + + GIT_NEW_ID = os.environ.get('GIT_NEW_ID') or options.git_new_id + if not GIT_NEW_ID: + logger.error('No git new id given') + exit(ERROR) + + GIT_BRANCH_NAME = ( + os.environ.get('GIT_BRANCH_NAME') or options.git_branch_name + ) + if not GIT_BRANCH_NAME: + logger.error('No git branch name given') + exit(ERROR) + + REVISION = os.environ.get('REVISION') or options.revision + if not REVISION: + logger.error('No revision given') + exit(ERROR) + + GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH) + GIT_REMOTE_BRANCH_NAME = os.path.join('origin', '%s' %(GIT_BRANCH_NAME)) + GIT_TARGET_WORKSPACE = os.path.join( + CWD, + '%s-build%s' %(BUILD_ID, BUILD_NUMBER) + ) + + GIT_TARGET_DIR = os.path.join( + GIT_TARGET_WORKSPACE, + os.path.basename(GIT_REPO_PATH) + ) + + GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL') + + KERNEL_CONFIG_PATH = os.path.join(GIT_TARGET_DIR, '.config') + + EXTRA_MODULES_PATH = os.path.join(GIT_TARGET_DIR, '.modules') + + if git_clone_remote_repository(GIT_REPO_PATH, GIT_TARGET_DIR): + logger.info('git clone was successfull') + else: + logger.info('git clone was not successfull') + exit(ERROR) + #atexit.register(remove_git_target_workspace) + if not git_checkout_branch(): + exit(ERROR) + #kernel_revision_obj = get_last_kernel_revision_obj() + #new_version = kernel_revision_obj.last_version + 1 + new_version = REVISION + try: + add_local_version_to_config(new_version) + except Exception, error: + logger.exception(error) + exit(ERROR) + else: + logger.info( + 'updated CONFIG_LOCALVERSION in %s to %s' + %(KERNEL_CONFIG_PATH, new_version) + ) +# if not update_kernel_revision_obj(kernel_revision_obj, new_version): +# logger.info( +# 'Could not update persistence version to %s for %s' +# %(new_version, GIT_BRANCH_NAME) +# ) +# exit(ERROR) + + logger.debug('changing dir to %s' %(GIT_TARGET_DIR)) + os.chdir(GIT_TARGET_DIR) + + if has_extra_modules(): + if not install_extra_modules(): + logger.error('Error during installation of extra kernel modules') + exit(ERROR) + + if not build_kernel(new_version): + exit(ERROR) + else: + #ftp = FTP( + # 'alexandria.profitbricks.localdomain', + # 'debian-uploader', + # 'vae6tooZe1ec' + #) + + #logger.info('FTP Login on %s successfull' %(ftp.host)) + + #ftp.cwd('squeeze') + #for package in glob( + # os.path.join(GIT_TARGET_WORKSPACE, '*.deb') + #): + # fh = open(package, 'rb', 1) + # ftp.storbinary( + # 'STOR %s' %(os.path.basename(package)), + # fh + # ) + # fh.close() + # logger.info('Successfully uploaded %s' %(package)) + #ftp.quit() + logger.info('Build successfull') + logger.info('dirlist: %s' %(os.listdir(GIT_TARGET_WORKSPACE))) + exit(OK)