From: Robin Wittler Date: Tue, 9 Aug 2011 13:52:22 +0000 (+0200) Subject: moved testgit_build.py to build.py X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=04b5f59d69d90dd2a34bd601d0637b4c378bcbe3;p=profitbricks%2Fjenkins-build-scripts.git moved testgit_build.py to build.py --- diff --git a/build.py b/build.py new file mode 100755 index 0000000..22d678b --- /dev/null +++ b/build.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import pwd +import sys +import git +import errno +import atexit +import shutil +import logging +import smtplib +import platform +import subprocess +from glob import glob +from ftplib import FTP +from lib import git_helper +from logging import Formatter +from lib import git_buildpackage +from multiprocessing import cpu_count +from ConfigParser import SafeConfigParser + +GIT = '/usr/bin/git' +BIN_RM = '/bin/rm' +BIN_SUDO = '/usr/bin/sudo' +BIN_DPUT = '/usr/bin/dput' +MAKE_KPKG = '/usr/bin/make-kpkg' +DEFAULT_PARALLEL_JOBS = cpu_count() + 1 + +BUILD_ARCH_MAP = { + 'x86_64': 'amd64', + 'i386': '686' +} + +BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686') + +CWD = os.environ.get('WORKSPACE') +BUILD_NUMBER = os.environ.get('BUILD_NUMBER') +BUILD_ID = os.environ.get('BUILD_ID') +BUILD_URL = os.environ.get('BUILD_URL') + +GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') +GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH) +GIT_OLD_ID = os.environ.get('GIT_OLD_ID') +GIT_NEW_ID = os.environ.get('GIT_NEW_ID') +GIT_UPSTREAM_BRANCH = os.environ.get('GIT_UPSTREAM_BRANCH') +GIT_DEBIAN_BRANCH = os.environ.get('GIT_DEBIAN_BRANCH') +GIT_DEBIAN_REMOTE_BRANCH = os.path.join('origin', '%s' %(GIT_DEBIAN_BRANCH)) +GIT_TARGET_WORKSPACE = os.path.join( + CWD, + '%s-build%s' %(BUILD_ID, BUILD_NUMBER) +) + +if GIT_UPSTREAM_BRANCH == 'NONE': + raise Exception('You must give a GIT_UPSTREAM_BRANCH') + +if GIT_DEBIAN_BRANCH == 'NONE': + GIT_DEBIAN_BRANCH = GIT_UPSTREAM_BRANCH + +GIT_TARGET_DIR = os.path.join( + GIT_TARGET_WORKSPACE, + os.path.basename(GIT_REPO_PATH) +) + +GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL') +SMTP_SERVER = 'roma.profitbricks.localdomain' +SMTP_SUBJECT = 'Build for branch %s, buildnumber %s was %s' +SMTP_TEXT = ( + 'Build for branch %s, buildnumber %s was %s. ' + + 'Take a close look at: ' + BUILD_URL +) +SMTP_BUILD_SUCCESS = 'SUCCESSFULL' +SMTP_BUILD_ERROR = 'NOT SUCCESSFULL' +SMTP_FROM = '%s@profitbricks.com' %(pwd.getpwuid(os.geteuid()).pw_name) + +CRE_URI = re.compile(r'(?P\w+)\:\/\/(?P[a-zA-Z0-9\-\_\.\+]+)\/(?P.*)$') +DPUT_URI = CRE_URI.match(GIT_REPO_PATH) +if not DPUT_URI: + raise Exception( + 'could not build DPUT_URI from "%s"' + %(GIT_REPO_PATH) + ) +DPUT_OPTIONS = { + 'fqdn': DPUT_URI.groupdict().get('url'), + 'method': 'scp', + 'login': 'jenkins', + 'incoming': '/srv/debian-repository/incoming', + 'allow_unsigned_uploads': 0 + } +DPUT_CF = os.path.join(GIT_TARGET_WORKSPACE, 'dput.cf') + +log_format = '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s' +formatter = Formatter(log_format) +logging.basicConfig( + format=log_format, + level=logging.DEBUG +) +logger = logging.getLogger(__file__) +logger.setLevel(logging.DEBUG) + +def send_email(result): + smtp = smtplib.SMTP(SMTP_SERVER) + msg = ( + 'From: %s\n' %(SMTP_FROM) + + 'To: %s\n' %(GIT_COMMITTER_EMAIL) + + 'Subject: %s\n' %(SMTP_SUBJECT %(GIT_UPSTREAM_BRANCH, BUILD_NUMBER, + result)) + + '%s\n' %(SMTP_TEXT %(GIT_UPSTREAM_BRANCH, BUILD_NUMBER, result)) + ) + smtp.sendmail(SMTP_FROM, GIT_COMMITTER_EMAIL, msg) + smtp.quit() + +def read_file(path): + try: + fh = open(path, 'r', 1) + except: + raise + else: + result = dict(enumerate(fh)) + fh.close() + return result + +def dput_package_upload(changes_path): + cmd = [BIN_DPUT, '-c', '%s' %(DPUT_CF), '%s' %(changes_path)] + logger.debug( + 'Trying to execute: "%s"' + %(cmd) + ) + cmd_obj = subprocess.Popen( + cmd, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + cwd=os.path.dirname(DPUT_CF) + ) + + ret = cmd_obj.wait() + + if ret: + errormsg = cmd_obj.stderr.read() + if not errormsg: + errormsg = cmd_obj.stdout.read() + if not errormsg: + errormsg = None + message = ( + '"%s" returned non-zero (returned with: %s). Error Msg was: %s' + %(cmd, ret, errormsg) + ) + logger.debug(message) + raise Exception(message) + + output = cmd_obj.stdout.read() or None + message = ( + '"%s" returned zero. Output was: %s' + %(cmd, ret, output) + ) + logger.debug(message) + return True + +def create_dput_cfg(): + fh = open(DPUT_CF, 'w') + config = SafeConfigParser() + config.add_section('origin') + for option, value in DPUT_OPTIONS.iteritems(): + config.set('origin', '%s' %(option), '%s' %(value)) + config.write(fh) + fh.close() + return True + + +def remove_git_target_workspace(): + try: + cmd = [BIN_SUDO, BIN_RM, '-rvf', GIT_TARGET_WORKSPACE] + cmdobj = subprocess.Popen( + cmd, + shell=False, + cwd='/', + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={'':''} + ) + + logger.debug( + 'Trying to call "%s" to delete "%s"' + %(cmd, GIT_TARGET_WORKSPACE) + ) + + ret = cmdobj.wait() + if ret: + stderr_msg = cmdobj.stderr.read() + stdout_msg = cmdobj.stdout.read() + _str = ( + 'Cmd "%s" returned non-zero (exitcode: %s). ' + %(cmd, ret) + + 'Output was: stdout="%s", stderr="%s"' + %(stdout_msg, stderr_msg) + ) + logger.debug(_str) + raise Exception(_str) + except Exception, error: + logger.exception(error) + raise + else: + logger.info('deleted %s' %(GIT_TARGET_WORKSPACE)) + return cmdobj + +def exit_ok(): + send_email(SMTP_BUILD_SUCCESS) + sys.exit(0) + +def exit_error(): + send_email(SMTP_BUILD_ERROR) + sys.exit(1) + +if __name__ == '__main__': + logger.debug( + 'Initialised with Enviroment: %s' + %( + ', '.join( + map( + lambda x: '%s => %s' %(x[0], x[1]), + os.environ.iteritems() + ) + ) + ) + ) + logging.getLogger('lib.git_helper').setLevel(logging.DEBUG) + if git_helper.git_clone_remote_repository(GIT_REPO_PATH, GIT_TARGET_DIR): + logger.info('git clone was successfull') + else: + logger.info('git clone was not successfull') + exit_error() + + atexit.register(remove_git_target_workspace) + os.chdir(GIT_TARGET_DIR) + + if GIT_UPSTREAM_BRANCH != GIT_DEBIAN_BRANCH: + git_helper.git_new_branch_from( + GIT_DEBIAN_BRANCH, + os.path.join('origin', GIT_DEBIAN_BRANCH) + ) + + repo = git.repo.Repo() + + if repo.active_branch != 'master' and GIT_DEBIAN_BRANCH != 'master': + git_helper.git_checkout_branch(GIT_DEBIAN_BRANCH) + + if not GIT_COMMITTER_EMAIL: + for commit in repo.commits(): + if commit.id == GIT_NEW_ID: + GIT_COMMITTER_EMAIL = commit.committer.email + logger.debug( + 'Found "%s" in commit-id "%s" at "%s"' + %(GIT_COMMITTER_EMAIL, commit.id, repo.active_branch) + ) + else: + raise Exception('No git_committer_email found') + + gbp = git_buildpackage.GitBuildPackage( + upstream_branch=GIT_UPSTREAM_BRANCH, + debian_branch=GIT_DEBIAN_BRANCH, + dist='squeeze', + arch='amd64' + ) + + logger.info('starting git-buildpackage') + ret = gbp.build() + logger.debug( + 'This is the output of git-buildpackage: \n%s' + %(ret) + ) + #create_dput_cfg() + #dput_package_upload() + + exit_ok() + +#class GitBuildPackage(object): +# def __init__(self, upstream_branch=None, +# debian_branch=None, dist=None, arch=None) + + #ftp = FTP( + # 'alexandria.profitbricks.localdomain', + # 'debian-uploader', + # 'vae6tooZe1ec' + #) +# + #logger.info('Log in on %s successfull' %(ftp.host)) +# + #ftp.cwd('squeeze') + #for package in glob( + # os.path.join(GIT_TARGET_WORKSPACE, '*.deb') + #): + # fh = open(package, 'rb', 1) + # ftp.storbinary( + # 'STOR %s' %(os.path.basename(package)), + # fh + # ) + # fh.close() + # logger.info('Successfully uploaded %s' %(package)) + #ftp.quit() + exit_ok() diff --git a/testgit_build.py b/testgit_build.py deleted file mode 100755 index 22d678b..0000000 --- a/testgit_build.py +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import re -import pwd -import sys -import git -import errno -import atexit -import shutil -import logging -import smtplib -import platform -import subprocess -from glob import glob -from ftplib import FTP -from lib import git_helper -from logging import Formatter -from lib import git_buildpackage -from multiprocessing import cpu_count -from ConfigParser import SafeConfigParser - -GIT = '/usr/bin/git' -BIN_RM = '/bin/rm' -BIN_SUDO = '/usr/bin/sudo' -BIN_DPUT = '/usr/bin/dput' -MAKE_KPKG = '/usr/bin/make-kpkg' -DEFAULT_PARALLEL_JOBS = cpu_count() + 1 - -BUILD_ARCH_MAP = { - 'x86_64': 'amd64', - 'i386': '686' -} - -BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686') - -CWD = os.environ.get('WORKSPACE') -BUILD_NUMBER = os.environ.get('BUILD_NUMBER') -BUILD_ID = os.environ.get('BUILD_ID') -BUILD_URL = os.environ.get('BUILD_URL') - -GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') -GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH) -GIT_OLD_ID = os.environ.get('GIT_OLD_ID') -GIT_NEW_ID = os.environ.get('GIT_NEW_ID') -GIT_UPSTREAM_BRANCH = os.environ.get('GIT_UPSTREAM_BRANCH') -GIT_DEBIAN_BRANCH = os.environ.get('GIT_DEBIAN_BRANCH') -GIT_DEBIAN_REMOTE_BRANCH = os.path.join('origin', '%s' %(GIT_DEBIAN_BRANCH)) -GIT_TARGET_WORKSPACE = os.path.join( - CWD, - '%s-build%s' %(BUILD_ID, BUILD_NUMBER) -) - -if GIT_UPSTREAM_BRANCH == 'NONE': - raise Exception('You must give a GIT_UPSTREAM_BRANCH') - -if GIT_DEBIAN_BRANCH == 'NONE': - GIT_DEBIAN_BRANCH = GIT_UPSTREAM_BRANCH - -GIT_TARGET_DIR = os.path.join( - GIT_TARGET_WORKSPACE, - os.path.basename(GIT_REPO_PATH) -) - -GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL') -SMTP_SERVER = 'roma.profitbricks.localdomain' -SMTP_SUBJECT = 'Build for branch %s, buildnumber %s was %s' -SMTP_TEXT = ( - 'Build for branch %s, buildnumber %s was %s. ' + - 'Take a close look at: ' + BUILD_URL -) -SMTP_BUILD_SUCCESS = 'SUCCESSFULL' -SMTP_BUILD_ERROR = 'NOT SUCCESSFULL' -SMTP_FROM = '%s@profitbricks.com' %(pwd.getpwuid(os.geteuid()).pw_name) - -CRE_URI = re.compile(r'(?P\w+)\:\/\/(?P[a-zA-Z0-9\-\_\.\+]+)\/(?P.*)$') -DPUT_URI = CRE_URI.match(GIT_REPO_PATH) -if not DPUT_URI: - raise Exception( - 'could not build DPUT_URI from "%s"' - %(GIT_REPO_PATH) - ) -DPUT_OPTIONS = { - 'fqdn': DPUT_URI.groupdict().get('url'), - 'method': 'scp', - 'login': 'jenkins', - 'incoming': '/srv/debian-repository/incoming', - 'allow_unsigned_uploads': 0 - } -DPUT_CF = os.path.join(GIT_TARGET_WORKSPACE, 'dput.cf') - -log_format = '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s' -formatter = Formatter(log_format) -logging.basicConfig( - format=log_format, - level=logging.DEBUG -) -logger = logging.getLogger(__file__) -logger.setLevel(logging.DEBUG) - -def send_email(result): - smtp = smtplib.SMTP(SMTP_SERVER) - msg = ( - 'From: %s\n' %(SMTP_FROM) + - 'To: %s\n' %(GIT_COMMITTER_EMAIL) + - 'Subject: %s\n' %(SMTP_SUBJECT %(GIT_UPSTREAM_BRANCH, BUILD_NUMBER, - result)) + - '%s\n' %(SMTP_TEXT %(GIT_UPSTREAM_BRANCH, BUILD_NUMBER, result)) - ) - smtp.sendmail(SMTP_FROM, GIT_COMMITTER_EMAIL, msg) - smtp.quit() - -def read_file(path): - try: - fh = open(path, 'r', 1) - except: - raise - else: - result = dict(enumerate(fh)) - fh.close() - return result - -def dput_package_upload(changes_path): - cmd = [BIN_DPUT, '-c', '%s' %(DPUT_CF), '%s' %(changes_path)] - logger.debug( - 'Trying to execute: "%s"' - %(cmd) - ) - cmd_obj = subprocess.Popen( - cmd, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True, - cwd=os.path.dirname(DPUT_CF) - ) - - ret = cmd_obj.wait() - - if ret: - errormsg = cmd_obj.stderr.read() - if not errormsg: - errormsg = cmd_obj.stdout.read() - if not errormsg: - errormsg = None - message = ( - '"%s" returned non-zero (returned with: %s). Error Msg was: %s' - %(cmd, ret, errormsg) - ) - logger.debug(message) - raise Exception(message) - - output = cmd_obj.stdout.read() or None - message = ( - '"%s" returned zero. Output was: %s' - %(cmd, ret, output) - ) - logger.debug(message) - return True - -def create_dput_cfg(): - fh = open(DPUT_CF, 'w') - config = SafeConfigParser() - config.add_section('origin') - for option, value in DPUT_OPTIONS.iteritems(): - config.set('origin', '%s' %(option), '%s' %(value)) - config.write(fh) - fh.close() - return True - - -def remove_git_target_workspace(): - try: - cmd = [BIN_SUDO, BIN_RM, '-rvf', GIT_TARGET_WORKSPACE] - cmdobj = subprocess.Popen( - cmd, - shell=False, - cwd='/', - close_fds=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={'':''} - ) - - logger.debug( - 'Trying to call "%s" to delete "%s"' - %(cmd, GIT_TARGET_WORKSPACE) - ) - - ret = cmdobj.wait() - if ret: - stderr_msg = cmdobj.stderr.read() - stdout_msg = cmdobj.stdout.read() - _str = ( - 'Cmd "%s" returned non-zero (exitcode: %s). ' - %(cmd, ret) + - 'Output was: stdout="%s", stderr="%s"' - %(stdout_msg, stderr_msg) - ) - logger.debug(_str) - raise Exception(_str) - except Exception, error: - logger.exception(error) - raise - else: - logger.info('deleted %s' %(GIT_TARGET_WORKSPACE)) - return cmdobj - -def exit_ok(): - send_email(SMTP_BUILD_SUCCESS) - sys.exit(0) - -def exit_error(): - send_email(SMTP_BUILD_ERROR) - sys.exit(1) - -if __name__ == '__main__': - logger.debug( - 'Initialised with Enviroment: %s' - %( - ', '.join( - map( - lambda x: '%s => %s' %(x[0], x[1]), - os.environ.iteritems() - ) - ) - ) - ) - logging.getLogger('lib.git_helper').setLevel(logging.DEBUG) - if git_helper.git_clone_remote_repository(GIT_REPO_PATH, GIT_TARGET_DIR): - logger.info('git clone was successfull') - else: - logger.info('git clone was not successfull') - exit_error() - - atexit.register(remove_git_target_workspace) - os.chdir(GIT_TARGET_DIR) - - if GIT_UPSTREAM_BRANCH != GIT_DEBIAN_BRANCH: - git_helper.git_new_branch_from( - GIT_DEBIAN_BRANCH, - os.path.join('origin', GIT_DEBIAN_BRANCH) - ) - - repo = git.repo.Repo() - - if repo.active_branch != 'master' and GIT_DEBIAN_BRANCH != 'master': - git_helper.git_checkout_branch(GIT_DEBIAN_BRANCH) - - if not GIT_COMMITTER_EMAIL: - for commit in repo.commits(): - if commit.id == GIT_NEW_ID: - GIT_COMMITTER_EMAIL = commit.committer.email - logger.debug( - 'Found "%s" in commit-id "%s" at "%s"' - %(GIT_COMMITTER_EMAIL, commit.id, repo.active_branch) - ) - else: - raise Exception('No git_committer_email found') - - gbp = git_buildpackage.GitBuildPackage( - upstream_branch=GIT_UPSTREAM_BRANCH, - debian_branch=GIT_DEBIAN_BRANCH, - dist='squeeze', - arch='amd64' - ) - - logger.info('starting git-buildpackage') - ret = gbp.build() - logger.debug( - 'This is the output of git-buildpackage: \n%s' - %(ret) - ) - #create_dput_cfg() - #dput_package_upload() - - exit_ok() - -#class GitBuildPackage(object): -# def __init__(self, upstream_branch=None, -# debian_branch=None, dist=None, arch=None) - - #ftp = FTP( - # 'alexandria.profitbricks.localdomain', - # 'debian-uploader', - # 'vae6tooZe1ec' - #) -# - #logger.info('Log in on %s successfull' %(ftp.host)) -# - #ftp.cwd('squeeze') - #for package in glob( - # os.path.join(GIT_TARGET_WORKSPACE, '*.deb') - #): - # fh = open(package, 'rb', 1) - # ftp.storbinary( - # 'STOR %s' %(os.path.basename(package)), - # fh - # ) - # fh.close() - # logger.info('Successfully uploaded %s' %(package)) - #ftp.quit() - exit_ok()