import sys import protocol import socket try: import cPickle as pickle except ImportError: import pickle from time import sleep import os import errno import subprocess from subprocess import CalledProcessError import re import string from string import atoi from traceback import print_exc, format_exc import portage try: import portage._sets as psets except ImportError: import portage.sets as psets from util import WritableObject, flatten_deps from common.exceptions import ChrootPreparationException import config from logger import log, init_logging class Tinderbox(object): NOMERGE_PKGS=['sys-apps/portage'] def __init__(self): self.hostname = config.MATCHBOX_HOST self.port = config.MATCHBOX_PORT self.sock = None self.settings = portage.config(clone=portage.settings) self.trees = portage.create_trees() self.settings["PORTAGE_VERBOSE"]="1" self.settings.backup_changes("PORTAGE_VERBOSE") self.setconf = psets.SetConfig([], self.settings, self.trees) init_logging('/tmp') def start_tinderbox(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TODO load settings for contacting matchbox self.sock.connect((self.hostname,self.port)) while 1: # TODO error/exception checking msg = protocol.GetNextPackage() msg_pickled = pickle.dumps(msg) self.sock.sendall(msg_pickled) reply = self.sock.recv(1024) reply_unpickled = pickle.loads(reply) if type(reply_unpickled) is protocol.GetNextPackageReply: gnp = reply_unpickled print "going to compile: %s\nuse flags: %s" %\ (gnp.package_name,gnp.use_flags) package = Package(gnp.package_name, gnp.version, gnp.use_flags) sleep(5) try: self.emerge_package(package) except Exception, e: log.error("Fatal error when emerging package %s, see backtrace" % package.name) log.error(format_exc()) else: print "Unknown reply: %s" % reply_unpickled def emerge_package(self, package): log.debug("emerge_package starting for %s" % package.name) settings = self.settings porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi if not package.version: # we are compiling ALL versions of package allversions = portdb.xmatch('match-all', package.name) else: # we were told exact version to compile allversions = ["%s-%s" % (package.name, package.version)] for pkg in allversions: ebuild = portdb.findname(pkg) archs = portdb.aux_get(pkg, ["KEYWORDS"])[0] archs = archs.split() if settings["ARCH"] not in archs and "~%s" % settings["ARCH"] not in archs: log.warning("Was asked to compile %s but it doesn't have our arch in KEYWORDS" % pkg) continue deps = portdb.aux_get(pkg,["DEPEND"]) deps = portage.dep.paren_reduce(deps[0]) settings.setcpv(pkg, mydb=portdb) use_enabled = set(settings["PORTAGE_USE"].split()) iuse = set(settings["IUSE"].split()) # only count deps enabled by USE flags use_deps = portage.dep.use_reduce(deps, list(use_enabled & iuse)) use_deps = self._normalize_dependencies(use_deps) use_deps = flatten_deps(use_deps) log.debug("calling create_dep_groups for pkg %s use_deps %s" % (pkg, use_deps)) dep_groups = self.create_dep_groups(use_deps) # prepare chroot & fork & do work try: subprocess.check_call([config.MK_CHROOT_SCRIPT,"-s",config.STAGE_TARBALL, config.BASE_CHROOT, config.WORK_CHROOT]) except CalledProcessError, cpe: raise ChrootPreparationException("Chroot preparation for %s failed with error code: %d" % (pkg, cpe.returncode)) except OSError, ose: raise ChrootPreparationException("Chroot preparation for %s failed with error(%d): %s\n\ Check your settings" % (pkg, ose.errno, ose.strerror)) self.child_pid = os.fork() childpid = self.child_pid if 0 == childpid: # we are the child! try: # setup logging! os.chroot(config.WORK_CHROOT) os.chdir("/") init_logging(config.CHROOT_LOGS) pkgname, pkgver, pkgrev = portage.pkgsplit(pkg) if pkgrev is "r0": package.version = pkgver else: package.version = "%s-%s" % (pkgver, pkgrev) package.reinit() self._emerge_package_subprocess(pkg, ebuild, dep_groups, package) sys.exit(0) except Exception, e: print_exc() print "Something went really bad and we need logging, stat!" log.error("Unrecoverable error in tinderbox slave, see backtrace for possible solutions:") log.error(format_exc()) sys.exit(1) (retpid, status) = os.waitpid(childpid, 0) if 0 != status: # something went really wrong, grab all the info we can print "Something went really bad and we need logging, stat!" log.error("Emerge of package %s failed with error code: %d" % (pkg, status)) try: package_infos = self._load_info('package_infos') except IOError, e: log.error("Loading package_infos file failed, something has gone wrong in subprocess apparently") package.attachments["error"] = "Loading package_infos failed, package was not compiled with any dependency combination" package.version = 'all' package_infos = [package.get_info()] msg = protocol.AddPackageInfo(package_infos) self.sock.sendall(pickle.dumps(msg)) #TODO make binpkg def _emerge_package_subprocess(self, pkg, ebuild, dep_groups, package): # We are chrooted inside WORK_CHROOT remember! porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi vartree = self.trees[portage.root]["vartree"] package_infos = [] settings = self.settings for group in dep_groups: dep_failed = None deps_processed = [] for dep in group: try: # this will need to change since it's only a quick hack so that # we don't have to do dep resolution ourselves import _emerge as emerge dep_emergepid = os.fork() # we need to run emerge_main() in child process since a lot of stuff in there # likes to call sys.exit() and we don't want that do we? if 0 == dep_emergepid: try: extra_use = dep[0] if extra_use: os.environ["USE"]=" ".join(extra_use) os.environ["FEATURES"] = "-strict" sys.argv = ["emerge","--verbose", "--usepkg", "--buildpkg" ,"=%s" % dep[1]] exit_code = emerge.emerge_main() sys.exit(exit_code) except Exception, e: print_exc() log.error(format_exc()) sys.exit(1) ret = os.waitpid(dep_emergepid, 0) if 0 != ret[1]: raise Exception("emerge_main() failed with error code %d" % ret[1]) except Exception, e: log.error(format_exc()) log.error("Unable to merge dependency %s for package %s" % (dep, pkg)) dep_failed = dep[1] deps_processed.append(dep) settings.setcpv(dep[1], mydb=portdb) dep_use_enabled = set(settings["PORTAGE_USE"].split()) dep_iuse = set(settings["IUSE"].split()) dep_name, dep_ver, dep_rev = portage.pkgsplit(dep[1]) real_use_enabled = list(dep_use_enabled & dep_iuse) if dep[0]: for useflag in dep[0]: if useflag.startswith('-'): if 0 is not l.count(useflag[1:]): real_use_enabled.remove(useflag[1:]) elif 0 == real_use_enabled.count(useflag): real_use_enabled.append(useflag) if dep_rev is'r0': dep_ver_full = dep_ver else: dep_ver_full = "%s-%s" % (dep_ver, dep_rev) dep_pkg = Package(dep_name, dep_ver_full, real_use_enabled) if dep_failed == dep[1]: build_dir = self.get_build_dir(dep_failed) self._add_attachment(dep_pkg, "%s/temp/build.log" % build_dir) self._add_attachment(dep_pkg, "%s/temp/environment" % build_dir) package_infos.append(dep_pkg.get_info()) if dep_failed: log.error("Unable to emerge package %s with deps %s" % (pkg, group)) # TODO unmerge succeeded deps self._add_attachment(package, "/var/log/emerge.log") self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS) package.depends = [x[1] for x in group] continue settings.setcpv(pkg, mydb=portdb) ret = portage.doebuild(ebuild, "merge", portage.root, settings, debug = False, tree="porttree") if 0 != ret: # error installing, grab logs self._add_attachment(package, "%s/build.log" % settings["T"]) self._add_attachment(package, "%s/environment" % settings["T"]) self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS) self._add_attachment(package, "/var/log/emerge.log") package.depends = [x[1] for x in deps_processed] package_infos.append(package.get_info()) for dep in group: if dep[1] in self.setconf.getSetAtoms("system"): pass dep_cat, dep_pv = portage.catsplit(dep[1]) ret = portage.unmerge(dep_cat, dep_pv, portage.root, settings, True, vartree=vartree) if 0 != ret: log.error("Unable to unmerge dep %s" % dep) pkg_cat, pkg_pv = portage.catsplit(pkg) ret = portage.unmerge(pkg_cat, pkg_pv, portage.root, settings, True, vartree=vartree) if ret != 0: log.error("Unable to unmerge package %s" % dep) self._save_info('package_infos', package_infos) def _add_attachment(self, pkg, path): try: attfile = open(path,"r") except IOError, (errno, strerror): print "Unable to read file %s: %d: %s" % (path, errno, strerror) log.warning("Unable to read file (%s) %d: %s" % (path, errno, strerror)) pkg.attachments[os.path.basename(path)] = "Unable to read file %s" % path else: pkg.attachments[os.path.basename(path)] = attfile.read() attfile.close() del attfile def _save_info(self, key, data): """ Save data inside CHROOT_LOGS directory, under name 'key'. This function is called from within _emerge_package_subprocess to save data for parent process @param key: key used to identify data inside CHROOT_LOGS directory @type key: string @param data: data to be saved @rtype: None """ outfile = open("%s/%s" % (config.CHROOT_LOGS, key), "w") pickle.dump(data, outfile) outfile.close() def _load_info(self, key): """ Load data from CHROOT_LOGS directory (under WORK_CHROOT) with filename 'key' @param key: key used to identify data inside directory @type key: string @rtype: depends on key @returns: data loaded from filename, usually blob """ infile = open("%s/%s/%s" % (config.WORK_CHROOT, config.CHROOT_LOGS, key),"r") return pickle.load(infile) def get_emerge_info(self): """ @rtype: string @returns: emerge --info output """ infoout = WritableObject() sys.stdout = infoout # emerge.action_info(self.settings, self.trees, [], []) # REDO withouth emerge.action_info ret = sys.stdout.content sys.stdout = sys.__stdout__ return string.join(infoout.content,sep='') def _normalize_dependencies(self, deps): """ Normalizes dependencies by replacing '||' by with alternative @param deps: depedencies (list of package atoms) @type deps: List @rtype: List @returns: Normalized list of package atoms with replaced || groups Example: >>> from tinderbox import Tinderbox as tb >>> t = tb() >>> t._normalize_dependencies(['||', ['=virtual/jdk-1.5*', '=virtual/jdk-1.4*'], 'app-arch/unzip']) ['=virtual/jdk-1.5*', 'app-arch/unzip'] >>> """ log.debug("normalize_dependencies called with deps: %s" % deps) new_deps = [] for i in range(len(deps)): if '||' == deps[i]: next = deps[i+1] if type(next) is list and len(next) > i: deps[i+1] = next[0] elif type(deps[i]) == list: new_deps.extend(self._normalize_dependencies(deps[i])) else: new_deps.append(deps[i]) return new_deps def create_dep_groups(self, deps): """ Create dependency groups from package dependencies Every valid version of dep (according to input spec) has to be at least in one output dependency group @param deps: dependencies of package (atoms) @type deps: List @rtype: List of Lists @returns: List of dependency groups (list of dependency versions) together with special use flags needed Example: Input: ['=dev-libs/glib-2*', '>=net-fs/samba-3.0.0', 'x11-libs/libX11','dev-util/subversion[-dso]'] Output: [[(None,'net-fs/samba-3.2.11'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.2-r1'), (['-dso'],'dev-util/subversion-1.5.5'], [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.3-r1'), (['-dso'],'dev-util/subversion-1.5.5'], [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.20.3'),(None, 'x11-libs/libX11-1.1.3'), (['-dso'],'dev-util/subversion-1.5.5'], """ log.debug("create_dep_groups called with deps %s" % deps) result = None porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi deps_expanded = [] max_dep_versions = 0 for dep in deps: if dep[0].startswith('!') or dep in self.NOMERGE_PKGS: continue dep_useflag = list(portage.dep.dep_getusedeps(dep)) if 0 == len(dep_useflag): dep_useflag = None if not portage.dep.isvalidatom(dep): log.error("%s is not valid atom in %s" % (dep, str(deps))) continue depversions = portdb.xmatch('match-all',dep) depversions = [(dep_useflag,x) for x in depversions] deps_expanded.append(depversions) if len(depversions) > max_dep_versions: max_dep_versions = len(depversions) for dep in deps_expanded: while len(dep) != 0 and len(dep) < max_dep_versions: dep.append(dep[0]) result = [] for i in range(max_dep_versions): group = [] for dep in deps_expanded: if 0 == len(dep): continue group.append(dep.pop()) result.append(group) return result def get_build_dir(self, cpv): settings = self.settings bldprefix = settings["PORTAGE_TMPDIR"]+"/portage" return os.path.join(bldprefix, cpv) class Package(object): def __init__(self, name, version, use_flags): """ @param name: category/name of given package (excluding version/release) @type name: string @param version: version part of CPV e.g. 1.2.0-r4 @type version: string @param use_flags: list of enabled use flags for given package @type use_flags: list """ self.name = name self.version = version self.use_flags = use_flags self.reinit() def reinit(self): self.content = None self.attachments = {} self.depends = None def get_info(self): """Returns protocol.PackageInfo with information about this package @returns: PackageInfo for sending across network @rtype: protocol.PackageInfo """ pi = protocol.PackageInfo() pi.name = self.name pi.version = self.version pi.use_flags = self.use_flags if self.content is None: self.content = self.get_package_contents() pi.content = self.content pi.attachments = self.attachments if 0 < len(self.attachments): pi.error = 1 pi.depends = self.depends pi.profile = self._get_current_profile() return pi def _get_current_profile(self): ret = None try: profpath = os.readlink("%s/etc/make.profile" % portage.root) ret = profpath[profpath.rindex("profiles")+len("profiles")+1:] except Exception, e: print e return ret def get_package_contents(self): """Returns package contents as dict with paths as keys data values are tuples of information (such as hash and size) @returns: package contents (or {} if it's not installed) @rtype: dict """ vartree = portage.db[portage.root]["vartree"] cpv = "%s-%s" % (self.name, self.version) if not vartree.dbapi.cpv_exists(cpv): # maybe raise exception? instead return {} cat, pkg = portage.catsplit(cpv) dblink = portage.dblink(cat, pkg, portage.root, vartree.settings, treetype="vartree", vartree=vartree) return dblink.getcontents()