From: Mathias Klette Date: Thu, 2 May 2013 17:00:52 +0000 (+0200) Subject: common_code, debian_build: cleanup #5 X-Git-Url: https://git.uhu-banane.de/?a=commitdiff_plain;h=1d7adbe12abf141c0aa3ca3eb23d32df32bca58c;p=profitbricks%2Fjenkins-build-scripts.git common_code, debian_build: cleanup #5 - added CWD to default ENV - remove unused variables - facilitate git submodule more - change the way of tag handling - beautifying some parts of the code --- diff --git a/common_code.py b/common_code.py index 23f49d7..bb57bb0 100755 --- a/common_code.py +++ b/common_code.py @@ -46,6 +46,7 @@ def exit_error(): ENV = os.environ ENV.setdefault('ARGV',' '.join(sys.argv)) +ENV.setdefault('CWD',os.getcwd()) BIN_DPUT = '/usr/bin/dput' BIN_FIGLET = '/usr/bin/figlet-figlet' diff --git a/debian_build.py b/debian_build.py index 5ac743f..29cf1af 100755 --- a/debian_build.py +++ b/debian_build.py @@ -38,13 +38,10 @@ from lib import git_helper # from common_code logger = logger_init(__file__) -# jenkins parameters -GIT_REPO_PATH = ENV['GIT_REPO_PATH'] -GIT_REPO_NAME = os.path.basename(ENV['GIT_REPO_PATH']) -GIT_BRANCH_NAME = ENV['GIT_BRANCH_NAME'] -GIT_COMMITTER_EMAIL = ENV['GIT_COMMITTER_EMAIL'] +# jenkins environment - parameters ENV.setdefault('NO_UPLOAD','') -NO_UPLOAD = ENV['NO_UPLOAD'] +ENV.setdefault('GIT_BRANCH_NAME',ENV['GIT_BRANCH']) +GIT_REPO_PATH = ENV['GIT_REPO_PATH'] # local constants BUILD_START = datetime.datetime.now() @@ -67,7 +64,7 @@ STABLE_DISTRIBUTIONS = ( dput_obj = dput.Dput() dput_obj.config = os.path.join(ENV['WORKSPACE'], '..', 'dput.cf') dput_obj.section = 'profitbricks' -if GIT_BRANCH_NAME.startswith('feature/') or GIT_BRANCH_NAME.startswith('poc/') or GIT_BRANCH_NAME.startswith('bugfix/'): +if ENV['GIT_BRANCH_NAME'].startswith('feature/') or ENV['GIT_BRANCH_NAME'].startswith('poc/') or ENV['GIT_BRANCH_NAME'].startswith('bugfix/'): dput_obj.contents = { 'fqdn': 'alexandria.pb.local', 'method': 'scp', @@ -122,83 +119,36 @@ def getopts(): return parser.parse_args() if __name__ == '__main__': - logger.debug('Called in: %s' % os.getcwd()) - logger.debug('Called with: %s' % " ".join(sys.argv)) - logger.debug( - 'Environment: \n%s' - %( - ' '.join( - map( - lambda x: ' %s="%s" \n' %(x[0], x[1]), - os.environ.iteritems() - ) - ) - ) - ) + logger.debug('Environment: %s' %(ENV)) + # FIXME: not used right now, shall it be used? options, args = getopts() - # FIXME: this is just needed for gbp and can go away eventually - if GIT_BRANCH_NAME: - GIT_UPSTREAM_BRANCH = GIT_BRANCH_NAME - GIT_DEBIAN_BRANCH = GIT_BRANCH_NAME - logging.debug( - 'setting GIT_UPSTREAM_BRANCH and GIT_DEBIAN_BRANCH to %s', - GIT_UPSTREAM_BRANCH - ) - else: - logger.error('Could not determine GIT_UPSTREAM_BRANCH') - exit_error() - - repo = git.Repo() + gitrepo = git.Repo('.') # reset local repository first - localname = GIT_BRANCH_NAME - remotename = os.path.join('origin', GIT_BRANCH_NAME) + localname = ENV['GIT_BRANCH_NAME'] + remotename = os.path.join('origin', ENV['GIT_BRANCH_NAME']) logger.info('Reset and clean repository.') - gitcmd = git.Git('.') - logger.debug( - 'git fetch origin --force --prune:\n%s' - %( gitcmd.fetch('--force','--prune').strip() ) - ) - logger.debug( - 'git reset --hard HEAD:\n%s' - %(gitcmd.reset('--hard','HEAD').strip()) - ) - logger.debug( - 'sudo git clean -fdx:\n%s' - %(gitcmd.execute(['sudo','git','clean','-fdx']).strip()) - ) - logger.debug( - 'git branch -D %s\n%s' - %(localname,gitcmd.branch('-D','%s' %(localname),with_exceptions=False).strip()) - ) + gitrepo.git.fetch('--force','--prune').strip() + gitrepo.git.reset('--hard','HEAD').strip() + gitrepo.git.execute('sudo','git','clean','-fdx').strip() + gitrepo.git.branch('-D','%s' %(localname),with_exceptions=False).strip() try: logger.info('Checkout branch %s.' %(remotename)) - logger.debug( - 'git checkout -b %s %s:\n%s' - %( - localname, - remotename, - gitcmd.checkout('-b','%s' %(localname),'%s' %(remotename).strip()) - ) - ) - except Exception, exception: - raise Exception('Failure while checking out Git clone:\n%s' %(exception)) + gitrepo.git.checkout('-b','%s' %(localname),'%s' %(remotename)).strip() + except Exception as error: + raise Exception('Failure while checking out Git clone: ',exc_info=error) else: - logger.debug('Status is now:\n%s' %( gitcmd.status().strip() )) - - if GIT_BRANCH_NAME == 'master' and ENV['JOB_NAME'] in AUTO_CHANGELOG_REPONAMES: - cmd = ['/usr/bin/dch', '-i', 'Released by jenkins.'] - subprocess.check_call(cmd) - cmd = ['/usr/bin/git-dch', '-R', '-a', '--spawn-editor=none'] - subprocess.check_call(cmd) - cmd = ['/usr/bin/git', 'commit', '-a', '-s', '-m', 'Released by jenkins'] - subprocess.check_call(cmd) - cmd = ['/usr/bin/git', 'push', 'origin', 'master'] - subprocess.check_call(cmd) + logger.debug('Status is now:\n%s' %(gitrepo.git.status().strip())) + + if ENV['GIT_BRANCH_NAME'] == 'master' and ENV['JOB_NAME'] in AUTO_CHANGELOG_REPONAMES: + subprocess.check_call(['/usr/bin/dch', '-i', 'Released by jenkins.']) + subprocess.check_call(['/usr/bin/git-dch', '-R', '-a', '--spawn-editor=none']) + gitrepo.git.commit('-a','-s','-m','Released by jenkins') + gitrepo.git.push('origin', 'master') logger.info('Changelog incremented by jenkins using debian_build.py!') # cleanup existing *orig.tar.gz @@ -288,7 +238,7 @@ if __name__ == '__main__': exit_error() # enforce correct distribution in debian/changelog for master and hotfix branches - if (GIT_BRANCH_NAME == 'master' or GIT_BRANCH_NAME.startswith('hotfix/')) and distribution not in STABLE_DISTRIBUTIONS: + if (ENV['GIT_BRANCH_NAME'] == 'master' or ENV['GIT_BRANCH_NAME'].startswith('hotfix/')) and distribution not in STABLE_DISTRIBUTIONS: message = 'Distribution %s in debian/changelog not listed in %s' %( distribution, STABLE_DISTRIBUTIONS, @@ -300,20 +250,20 @@ if __name__ == '__main__': # get the current commit id current_commit = git_helper.git_get_commit_id() - if GIT_BRANCH_NAME == 'develop' or GIT_BRANCH_NAME == 'pre-staging' or GIT_BRANCH_NAME.startswith('feature/') or GIT_BRANCH_NAME.startswith('poc/') or GIT_BRANCH_NAME.startswith('bugfix/') or GIT_BRANCH_NAME.startswith('release/'): + if ENV['GIT_BRANCH_NAME'] == 'develop' or ENV['GIT_BRANCH_NAME'] == 'pre-staging' or ENV['GIT_BRANCH_NAME'].startswith('feature/') or ENV['GIT_BRANCH_NAME'].startswith('poc/') or ENV['GIT_BRANCH_NAME'].startswith('bugfix/') or ENV['GIT_BRANCH_NAME'].startswith('release/'): daily_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - if GIT_BRANCH_NAME.startswith('release/'): + if ENV['GIT_BRANCH_NAME'].startswith('release/'): changelog_distro = 'staging' # use shorter date because we can daily_date = datetime.datetime.now().strftime('%Y%m%d') version = '%s~rc%s+%s+%s' %(version, daily_date, ENV['BUILD_NUMBER'], current_commit[0:7]) - elif GIT_BRANCH_NAME.startswith('feature/') or GIT_BRANCH_NAME.startswith('poc/') or GIT_BRANCH_NAME.startswith('bugfix/'): - changelog_distro ='dev-'+ GIT_BRANCH_NAME.replace("/","-") + elif ENV['GIT_BRANCH_NAME'].startswith('feature/') or ENV['GIT_BRANCH_NAME'].startswith('poc/') or ENV['GIT_BRANCH_NAME'].startswith('bugfix/'): + changelog_distro ='dev-'+ ENV['GIT_BRANCH_NAME'].replace("/","-") version = '%s~experimental%s+%s+%s' %(version, daily_date, ENV['BUILD_NUMBER'], current_commit[0:7]) else: #FIXME: enable this once Florian acks. #if GIT_REPO_PATH.startswith('/srv/git/dev/'): - # if GIT_BRANCH_NAME == 'pre-staging': + # if ENV['GIT_BRANCH_NAME'] == 'pre-staging': # version = '%s~develop%s+%s+%s' %(version, daily_date, BUILD_NUMBER, current_commit[0:7]) # changelog_distro = 'pre-staging' # else: @@ -328,30 +278,30 @@ if __name__ == '__main__': new_log = 'Generated by jenkins build of %s' % current_commit cmd = [ - '/usr/bin/dch', - '--newversion', - '%s' %(version), - '--force-bad-version', - '--distribution', - '%s' %(changelog_distro), - '--force-distribution', - '--preserve', - '--no-auto-nmu', - '--', - '%s' %(''.join(new_log)) - ] + '/usr/bin/dch', + '--newversion', + '%s' %(version), + '--force-bad-version', + '--distribution', + '%s' %(changelog_distro), + '--force-distribution', + '--preserve', + '--no-auto-nmu', + '--', + '%s' %(''.join(new_log)), + ] logger.debug('Trying to call: %s' %(' '.join(cmd))) dch = subprocess.Popen( - cmd, - shell=False, - close_fds=True, - stdin=subprocess.PIPE, - stdout=sys.stdout, - stderr=sys.stderr, - cwd=os.getcwd() - ) + cmd, + shell=False, + close_fds=True, + stdin=subprocess.PIPE, + stdout=sys.stdout, + stderr=sys.stderr, + cwd=os.getcwd() + ) dch.stdin.write('\n') ret = dch.wait() @@ -369,10 +319,8 @@ if __name__ == '__main__': # we need to commit here else git-buildpackage will use the existing debian/changelog... # TODO: Later we should investigate why "--ignore-new" does not work! - cmd = ['/usr/bin/git', 'add', '-A'] - subprocess.check_call(cmd) - cmd = ['/usr/bin/git', 'commit', '-a', '-m', 'add new changelog entry'] - subprocess.check_call(cmd) + gitrepo.git.add('-A') + gitrepo.git.commit('-a', '-m', 'add new changelog entry') # let me see the first two changelog entries: line_counter = 0 @@ -382,32 +330,30 @@ if __name__ == '__main__': if line_counter <= 2: print('debian/changelog: %s' %(line.rstrip('\n'))) - # let's test if we can tag this build, so we can fail early if we cannot tag it later - if GIT_BRANCH_NAME == 'master' or GIT_BRANCH_NAME.startswith('hotfix/'): + # fail early if we already found the tag we would create upon a successfull build, + # except the already existing tag uses exactly the same commit as we are triggered with + if ENV['GIT_BRANCH_NAME'] == 'master' or ENV['GIT_BRANCH_NAME'].startswith('hotfix/'): + create_tag = True logger.debug('version: %s' %(version)) # replace valid debian version chars that are invalid for git tagging version_tag = version.replace('~','_') version_tag = version_tag.replace(':',',') logger.debug('version_tag: %s' %(version_tag)) - cmd = ['/usr/bin/git', 'tag', version_tag] - try: - subprocess.check_call(cmd) - except: - logger.error('Could not tag repository with "%s".' % version_tag) - exit_error() - cmd = ['/usr/bin/git', 'tag', '-d', version_tag] - try: - subprocess.check_call(cmd) - except: - logger.error('Could not delete tag "%s" from repository.' % version_tag) - exit_error() - logger.info('Tagged as "%s" woks, tag does not exist yet.' % version_tag) + + remote_tag = [tag for tag in gitrepo.tags if tag.name == version_tag] + if len(remote_tag) > 0: + if remote_tag[0].commit.id == ENV['GIT_COMMIT']: + logger.info('Tag was already created for this commit.') + create_tag = False + else: + logger.error('Tag was already created for another commit.') + exit_error() # set pb_suite which GitBuildPackage() will turn into PB_SUITE # if we wrote a changelog entry, use that one - if GIT_BRANCH_NAME.startswith('feature/') or GIT_BRANCH_NAME.startswith('poc/') or GIT_BRANCH_NAME.startswith('bugfix/') or GIT_BRANCH_NAME == 'develop' or GIT_BRANCH_NAME == 'pre-staging' or GIT_BRANCH_NAME.startswith('release/'): + if ENV['GIT_BRANCH_NAME'].startswith('feature/') or ENV['GIT_BRANCH_NAME'].startswith('poc/') or ENV['GIT_BRANCH_NAME'].startswith('bugfix/') or ENV['GIT_BRANCH_NAME'] == 'develop' or ENV['GIT_BRANCH_NAME'] == 'pre-staging' or ENV['GIT_BRANCH_NAME'].startswith('release/'): pb_suite=changelog_distro - elif GIT_BRANCH_NAME.startswith('hotfix/') or GIT_BRANCH_NAME == 'master': + elif ENV['GIT_BRANCH_NAME'].startswith('hotfix/') or ENV['GIT_BRANCH_NAME'] == 'master': # just take suite from changelog for these branches pb_suite='production-proposed-updates' else: @@ -420,8 +366,8 @@ if __name__ == '__main__': figlet('version: %s' %(version)) gbp = git_buildpackage.GitBuildPackage( - upstream_branch=GIT_UPSTREAM_BRANCH, - debian_branch=GIT_DEBIAN_BRANCH, + upstream_branch=ENV['GIT_BRANCH'], + debian_branch=ENV['GIT_BRANCH'], dist='squeeze', arch='amd64', pb_suite=pb_suite, @@ -431,13 +377,12 @@ if __name__ == '__main__': logger.info('used to start git-buildpackage here...') ret = gbp.build() # remove last commit (the one where we added the changelog entry) - cmd = ['/usr/bin/git', 'reset', '--soft', 'HEAD~1'] - subprocess.check_call(cmd) + gitrepo.git.reset('--soft', 'HEAD~1') # now handle gpb result if ret: logger.error( - 'git-buildpackage returned non-zero. exitcode was: %s' % ret - ) + 'git-buildpackage returned non-zero. exitcode was: %s' % ret + ) exit_error() else: logger.debug('git-buildpackage executed successfully') @@ -446,33 +391,30 @@ if __name__ == '__main__': # # if reports.tgz exists untar it to workspace and delete it. # reports.tgz is generated by /root/.pbuilder/hooks.d/B01-test - cmd = ['/bin/tar', 'xzvf', os.path.join(ENV['WORKSPACE'], '../build-area/result/reports.tgz'), '-C', ENV['WORKSPACE']] try: - subprocess.check_call(cmd) - cmd = ['/usr/bin/sudo' , '/bin/rm', os.path.join(ENV['WORKSPACE'], '../build-area/result/reports.tgz')] - subprocess.check_call(cmd) + reports_file = os.path.join( + ENV['WORKSPACE'], + '../build-area/result/reports.tgz', + ) + subprocess.check_call(['/bin/tar', 'xzvf', '-C', ENV['WORKSPACE'], reports_file]) + subprocess.check_call(['/usr/bin/sudo' , '/bin/rm', reports_file]) except: pass # build was succesful, now let's tag it - if GIT_BRANCH_NAME == 'master' or GIT_BRANCH_NAME.startswith('hotfix/'): - logger.debug('version: %s' %(version)) - # replace valid debian version chars that are invalid for git tagging - version_tag = version.replace('~','_') - version_tag = version_tag.replace(':',',') - logger.debug('version_tag: %s' %(version_tag)) - cmd = ['/usr/bin/git', 'tag', version_tag] + if create_tag: try: - subprocess.check_call(cmd) - except: - logger.error('Could not tag repository with "%s".' % version_tag) + gitrepo.git.tag(version_tag) + except Exception as error: + logger.error('Could not create tag: ', exc_info=error) exit_error() - cmd = ['/usr/bin/git', 'push', 'origin', version_tag] + try: - subprocess.check_call(cmd) - except: - logger.error('Could not push tag "%s" to repository.' % version_tag) + gitrepo.git.push('origin', version_tag) + except Exception as error: + logger.error('Could not push tag: ', exc_info=error) exit_error() + logger.info('Tagged as "%s".' % version_tag) # now upload (at least, try to...) @@ -487,31 +429,41 @@ if __name__ == '__main__': logger.info('%s' % (fh.read())) fh.close() # upload (if NO_UPLOAD is not set) - if NO_UPLOAD not in ('true', 'True'): + if ENV['NO_UPLOAD'] not in ('true', 'True'): dput_obj.upload(changes_file) else: - logger.debug('value of NO_UPLOAD: %s' % NO_UPLOAD) + logger.debug('value of NO_UPLOAD: %s' % ENV['NO_UPLOAD']) logger.info('dist: %s' %(pb_suite)) figlet('dist: %s' %(pb_suite)) logger.info('version: %s' %(version)) figlet('version: %s' %(version)) except Exception, error: - figlet('upload to reprepro failed:') + figlet('upload failed:') logger.exception(error) exit_error() + BUILD_END = datetime.datetime.now() + # cidb wise, we only care about builds from master, hotfix + develop package_instances=[] - if GIT_BRANCH_NAME == 'master' or GIT_BRANCH_NAME == 'develop' or GIT_BRANCH_NAME.startswith('hotfix/'): + if ENV['GIT_BRANCH_NAME'] == 'master' or ENV['GIT_BRANCH_NAME'] == 'develop' or ENV['GIT_BRANCH_NAME'].startswith('hotfix/'): try: - package_instances = add_package_instances("profitbricks", ENV['JOB_NAME'], ENV['BUILD_NUMBER'], changes_file, version, BUILD_START, BUILD_END) + package_instances = add_package_instances( + "profitbricks", + ENV['JOB_NAME'], + ENV['BUILD_NUMBER'], + changes_file, + version, + BUILD_START, + BUILD_END, + ) except Exception as error: figlet('CIDB problem:') logger.error("package instance not added to DB", exc_info=error) exit_error() # only trigger liveboots automatically for builds from master+hotfix/* branches - if len(package_instances) > 0 and ( GIT_BRANCH_NAME == 'master' or GIT_BRANCH_NAME.startswith('hotfix/')): + if len(package_instances) > 0 and ( ENV['GIT_BRANCH_NAME'] == 'master' or ENV['GIT_BRANCH_NAME'].startswith('hotfix/')): try: add_liveboot_request(package_instances) except: