From 8868362d14087004544dd5677e1fa242eef200f8 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Thu, 27 Sep 2012 13:54:11 +0200 Subject: [PATCH] work in progress: debian package importer --- .gitignore | 3 ++ db_add.py | 123 ++++++++++++++++++++++++++++++++++++++++++ debian_build.py | 105 +----------------------------------- debian_packages2db.py | 45 ++++++++++++++++ debian_repos2db.sh | 20 +++++++ 5 files changed, 192 insertions(+), 104 deletions(-) create mode 100755 db_add.py create mode 100755 debian_packages2db.py create mode 100755 debian_repos2db.sh diff --git a/.gitignore b/.gitignore index 9fa524a..3220327 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ .*\.swp +*/*\.pyc +*\.pyc +Packages* diff --git a/db_add.py b/db_add.py new file mode 100755 index 0000000..b9ce111 --- /dev/null +++ b/db_add.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import pwd +import sys +import errno +import logging +import datetime +import subprocess +import time +import fileinput +import psycopg2 +from cidb import * +from logging import Formatter + + +def db_add_job(con, name): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO jenkins_job(name) VALUES('%s') RETURNING id" % (name)) + except psycopg2.DatabaseError as e: + # record already exist + cur.execute("ROLLBACK TO a") # have to rollback after failed command + cur.execute("SELECT id FROM jenkins_job WHERE name='%s'" % (name)) + #logger.debug("INSERT INTO jenkins_job(name) VALUES('%s')" % (name)) + return cur.fetchone()[0] + +def db_add_build(con, number, db_id): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id)) + except psycopg2.DatabaseError as e: + cur.execute("ROLLBACK TO a") # have to rollback after failed command + #logger.debug("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id)) + return cur.fetchone()[0] + +def db_add_build_result(con, build_id, start, end): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true')) + except psycopg2.DatabaseError as e: + cur.execute("ROLLBACK TO a") # have to rollback after failed command + #logger.debug("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true')) + return cur.fetchone()[0] + +def db_add_package(con, package): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package)) + except psycopg2.DatabaseError as e: + # record already exist + cur.execute("ROLLBACK TO a") # have to rollback after failed command + cur.execute("SELECT id FROM deb_package WHERE name='%s'" % (package)) + #logger.debug("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package)) + return cur.fetchone()[0] + +def db_add_package_instance(con, package_id, origin_id, build_result_id, version): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) + except psycopg2.DatabaseError as e: + # record already exist - this MUST NOT HAPPEN + cur.execute("ROLLBACK TO a") # have to rollback after failed command + logger_error("FAILED: INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) + exit_error() + return cur.fetchone()[0] + +def db_add_origin(con, origin): + cur = con.cursor() + cur.execute("SAVEPOINT a") + try: + cur.execute("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin)) + except psycopg2.DatabaseError as e: + # record already exist + cur.execute("ROLLBACK TO a") # have to rollback after failed command + cur.execute("SELECT id FROM deb_package_origin WHERE origin='%s'" % (origin)) + #logger.debug("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin)) + return cur.fetchone()[0] + +def add_package_instance(origin, job_name, build_number, changes_file, version, start, end): + con = db_connect() + #logger.debug("save %s to database" %(job_name)) + db_origin_id = db_add_origin(con, origin) + #logger.debug("INSERT success origin %s" % origin) + db_job_id = db_add_job(con, job_name) + #logger.debug("INSERT success job %s" % job_name) + # maybe FIXME: we could add the build already once its started, but for package builds we dont wanna add failures to the db at all + db_build_id = db_add_build (con, build_number, db_job_id) + db_build_result_id = db_add_build_result (con, db_build_id, start, end) + #logger.debug("INSERT success build number %s" % build_number) + #logger.debug("processing %s" %(changes_file)) + cmd = ["dcmd", '--deb', changes_file] + cmdobj = subprocess.Popen( + cmd, + shell=False, + cwd='/', + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env={'':''} + ) + #logger.debug('Calling "%s":' %(' '.join(cmd))) + ret = cmdobj.wait() + for line in cmdobj.stdout.readlines(): + package = os.path.basename(line.strip()).split('_')[0] + #logger.debug('package: %s' %(package)) + db_package_id = db_add_package (con, package) + #logger.debug("INSERT success package %s" % package) + db_package_instance_id = db_add_package_instance (con, db_package_id, db_origin_id, db_build_result_id, version) + logger.debug("INSERT success package version %s %s" %( package,version)) + #logger.debug('Cmd returned with status %d' %(cmdobj.returncode)) + con.commit() + logger.info("CIDB update OK.") + return db_package_instance_id + + diff --git a/debian_build.py b/debian_build.py index 71a95cb..63eb8ce 100755 --- a/debian_build.py +++ b/debian_build.py @@ -20,6 +20,7 @@ import urllib import fileinput import psycopg2 from add_liveboot_request import add_liveboot_request +from db_add import * from cidb import * from glob import glob from ftplib import FTP @@ -117,110 +118,6 @@ def read_file(path): fh.close() return result -def db_add_job(con, name): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO jenkins_job(name) VALUES('%s') RETURNING id" % (name)) - except psycopg2.DatabaseError as e: - # record already exist - cur.execute("ROLLBACK TO a") # have to rollback after failed command - cur.execute("SELECT id FROM jenkins_job WHERE name='%s'" % (name)) - #logger.debug("INSERT INTO jenkins_job(name) VALUES('%s')" % (name)) - return cur.fetchone()[0] - -def db_add_build(con, number, db_id): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id)) - except psycopg2.DatabaseError as e: - cur.execute("ROLLBACK TO a") # have to rollback after failed command - #logger.debug("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id)) - return cur.fetchone()[0] - -def db_add_build_result(con, build_id, start, end): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true')) - except psycopg2.DatabaseError as e: - cur.execute("ROLLBACK TO a") # have to rollback after failed command - #logger.debug("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true')) - return cur.fetchone()[0] - -def db_add_package(con, package): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package)) - except psycopg2.DatabaseError as e: - # record already exist - cur.execute("ROLLBACK TO a") # have to rollback after failed command - cur.execute("SELECT id FROM deb_package WHERE name='%s'" % (package)) - #logger.debug("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package)) - return cur.fetchone()[0] - -def db_add_package_instance(con, package_id, origin_id, build_result_id, version): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) - except psycopg2.DatabaseError as e: - # record already exist - this MUST NOT HAPPEN - cur.execute("ROLLBACK TO a") # have to rollback after failed command - logger_error("FAILED: INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) - exit_error() - return cur.fetchone()[0] - -def db_add_origin(con, origin): - cur = con.cursor() - cur.execute("SAVEPOINT a") - try: - cur.execute("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin)) - except psycopg2.DatabaseError as e: - # record already exist - cur.execute("ROLLBACK TO a") # have to rollback after failed command - cur.execute("SELECT id FROM deb_package_origin WHERE origin='%s'" % (origin)) - #logger.debug("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin)) - return cur.fetchone()[0] - -def add_package_instance(origin, job_name, build_number, changes_file, version, start, end): - con = db_connect() - #logger.debug("save %s to database" %(job_name)) - db_origin_id = db_add_origin(con, origin) - #logger.debug("INSERT success origin %s" % origin) - db_job_id = db_add_job(con, job_name) - #logger.debug("INSERT success job %s" % job_name) - # maybe FIXME: we could add the build already once its started, but for package builds we dont wanna add failures to the db at all - db_build_id = db_add_build (con, build_number, db_job_id) - db_build_result_id = db_add_build_result (con, db_build_id, start, end) - #logger.debug("INSERT success build number %s" % build_number) - #logger.debug("processing %s" %(changes_file)) - cmd = ["dcmd", '--deb', changes_file] - cmdobj = subprocess.Popen( - cmd, - shell=False, - cwd='/', - close_fds=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env={'':''} - ) - #logger.debug('Calling "%s":' %(' '.join(cmd))) - ret = cmdobj.wait() - for line in cmdobj.stdout.readlines(): - package = os.path.basename(line.strip()).split('_')[0] - #logger.debug('package: %s' %(package)) - db_package_id = db_add_package (con, package) - #logger.debug("INSERT success package %s" % package) - db_package_instance_id = db_add_package_instance (con, db_package_id, db_origin_id, db_build_result_id, version) - logger.debug("INSERT success package version %s %s" %( package,version)) - #logger.debug('Cmd returned with status %d' %(cmdobj.returncode)) - con.commit() - logger.info("CIDB update OK.") - return db_package_instance_id - def dput_package_upload(changes_path): try: cmd = [BIN_DPUT, '-c', '%s' %(DPUT_CF), '--no-upload-log', 'profitbricks', '%s' %(changes_path)] diff --git a/debian_packages2db.py b/debian_packages2db.py new file mode 100755 index 0000000..e46994a --- /dev/null +++ b/debian_packages2db.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import datetime +import platform +import subprocess +import time +import fileinput +import string +from db_add import * + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("usage: %s $packages_file") + sys.exit(1) + packages_file = sys.argv[1] + + cmd = ['grep-dctrl', '-n', '-s', 'Package,Version', '', packages_file] + print cmd + grep_dctrl = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + close_fds=True, + shell=False, + ) + ret = grep_dctrl.wait() + if ret: + print '%s was not successfull, return code was %s ' % (' '.join(cmd), ret) + sys.exit(1) + + pkg_name = ' ' + while pkg_name: + pkg_name = grep_dctrl.stdout.readline() + pkg_name = pkg_name.strip() + if pkg_name == '': + break + version = grep_dctrl.stdout.readline() + version = version.strip() + empty = grep_dctrl.stdout.readline() + print 'would add %s %s' % (pkg_name,version) + diff --git a/debian_repos2db.sh b/debian_repos2db.sh new file mode 100755 index 0000000..db6ff9a --- /dev/null +++ b/debian_repos2db.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +REPO_SUITES[0]="production-approved-updates production-proposed-updates production pre-staging" +REPO_SECTIONS[0]="main contrib non-free" +REPO_BASE_URL[0]="http://alexandria/profitbricks-repository/dists/SUITE/SECTION/binary-amd64/" + +REPO_SUITES[1]="squeeze profitbricks-backports profitbricks-backports-proposed-updates profitbricks-backports-pre-staging" +REPO_SECTIONS[1]="main contrib non-free" +REPO_BASE_URL[1]="http://alexandria.profitbricks.localdomain/debian-mirror/dists/SUITE/SECTION/binary-amd64/" + +for RUN in 0 1 ; do + for SUITE in ${REPO_SUITES[$RUN]} ; do + for SECTION in ${REPO_SECTIONS[$RUN]} ; do + URL=$(echo ${REPO_BASE_URL[$RUN]} | sed -s "s#SUITE#$SUITE#" | sed -s "s#SECTION#$SECTION#")/Packages + wget $URL -o /dev/null + [ -s Packages ] && ./debian_packages2db.py Packages + rm Packages + done + done +done -- 2.39.5