path: root/src/tinderbox/__init__.py

import sys
import protocol
import socket
try:
    import cPickle as pickle
except ImportError:
    import pickle
from time import sleep
import os
import errno
import subprocess
from subprocess import CalledProcessError
import re
import string
from string import atoi
from traceback import print_exc, format_exc

import portage
try:
    import portage._sets as psets
except ImportError:
    import portage.sets as psets


from util import WritableObject, flatten_deps
from common.exceptions import ChrootPreparationException
import config
from logger import log, init_logging




class Tinderbox(object):

    NOMERGE_PKGS=['sys-apps/portage']

    def __init__(self):
        self.hostname = config.MATCHBOX_HOST
        self.port = config.MATCHBOX_PORT
        self.sock = None

        self.settings = portage.config(clone=portage.settings)
        self.trees = portage.create_trees()
        self.settings["PORTAGE_VERBOSE"]="1"
        self.settings.backup_changes("PORTAGE_VERBOSE")
        self.setconf = psets.SetConfig([], self.settings, self.trees)
        init_logging('/tmp')


    def start_tinderbox(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # TODO load settings for contacting matchbox 
        self.sock.connect((self.hostname,self.port))

        while 1:
            # TODO error/exception checking
            msg = protocol.GetNextPackage()
            msg_pickled = pickle.dumps(msg)
            self.sock.sendall(msg_pickled)
            reply = self.sock.recv(1024)
            reply_unpickled = pickle.loads(reply)

            if type(reply_unpickled) is protocol.GetNextPackageReply:
                gnp = reply_unpickled
                print "going to compile: %s\nuse flags: %s" %\
                (gnp.package_name,gnp.use_flags)
                package = Package(gnp.package_name, gnp.version, gnp.use_flags)
                sleep(5)
                try:
                    self.emerge_package(package)
                except Exception, e:
                    log.error("Fatal error when emerging package %s, see backtrace" % package.name)
                    log.error(format_exc())
            else:
                print "Unknown reply: %s" % reply_unpickled



    def emerge_package(self, package):
        log.debug("emerge_package starting for %s" % package.name)
        settings = self.settings

        porttree = self.trees[portage.root]['porttree']
        portdb = porttree.dbapi
        if not package.version:
            # we are compiling ALL versions of package
            allversions = portdb.xmatch('match-all', package.name)
        else:
            # we were told exact version to compile
            allversions = ["%s-%s" % (package.name, package.version)]

        for pkg in allversions:
            ebuild = portdb.findname(pkg)
            archs = portdb.aux_get(pkg, ["KEYWORDS"])[0]
            archs = archs.split()
            if settings["ARCH"] not in archs and "~%s" % settings["ARCH"] not in archs:
                log.warning("Was asked to compile %s but it doesn't have our arch in KEYWORDS" % pkg)
                continue
            deps = portdb.aux_get(pkg,["DEPEND"])

            deps = portage.dep.paren_reduce(deps[0])

            settings.setcpv(pkg, mydb=portdb)
            use_enabled = set(settings["PORTAGE_USE"].split())
            iuse = set(settings["IUSE"].split())

            # only count deps enabled by USE flags
            use_deps = portage.dep.use_reduce(deps, list(use_enabled & iuse))
            use_deps = self._normalize_dependencies(use_deps)

            use_deps = flatten_deps(use_deps)

            log.debug("calling create_dep_groups for pkg %s use_deps %s" % (pkg, use_deps))
            dep_groups = self.create_dep_groups(use_deps)

            # prepare chroot & fork & do work
            try:
                subprocess.check_call([config.MK_CHROOT_SCRIPT,"-s",config.STAGE_TARBALL,
                    config.BASE_CHROOT, config.WORK_CHROOT])
            except CalledProcessError, cpe:
                raise ChrootPreparationException("Chroot preparation for %s failed with error code: %d" 
                        % (pkg, cpe.returncode))
            except OSError, ose:
                raise ChrootPreparationException("Chroot preparation for %s failed with error(%d): %s\n\
                        Check your settings"
                        % (pkg, ose.errno, ose.strerror))

            self.child_pid = os.fork()
            childpid = self.child_pid
            if 0 == childpid:
                # we are the child!
                try:
                    # setup logging!
                    os.chroot(config.WORK_CHROOT)
                    os.chdir("/")
                    init_logging(config.CHROOT_LOGS)
                    pkgname, pkgver, pkgrev = portage.pkgsplit(pkg)

                    if pkgrev is "r0":
                        package.version = pkgver
                    else:
                        package.version = "%s-%s" % (pkgver, pkgrev)
                    package.reinit()
                    self._emerge_package_subprocess(pkg, ebuild, dep_groups, package)

                    sys.exit(0)
                except Exception, e:
                    print_exc()
                    print "Something went really bad and we need logging, stat!"
                    log.error("Unrecoverable error in tinderbox slave, see backtrace for possible solutions:")
                    log.error(format_exc())
                    sys.exit(1)


            (retpid, status) = os.waitpid(childpid, 0)
            if 0 != status:
                # something went really wrong, grab all the info we can
                print "Something went really bad and we need logging, stat!"
                log.error("Emerge of package %s failed with error code: %d" % (pkg, status))

            try:
                package_infos = self._load_info('package_infos')
            except IOError, e:
                log.error("Loading package_infos file failed, something has gone wrong in subprocess apparently")
                package.attachments["error"] = "Loading package_infos failed, package was not compiled with any dependency combination"
                package.version = 'all'
                package_infos = [package.get_info()]

            msg = protocol.AddPackageInfo(package_infos)
            self.sock.sendall(pickle.dumps(msg))
        #TODO make binpkg


    def _emerge_package_subprocess(self, pkg, ebuild, dep_groups, package):
        # We are chrooted inside WORK_CHROOT remember!
        porttree = self.trees[portage.root]['porttree']
        portdb = porttree.dbapi
        vartree =  self.trees[portage.root]["vartree"]

        package_infos = []

        settings = self.settings
        for group in dep_groups:
            dep_failed = None
            deps_processed = []
            for dep in group:

                try:
                    # this will need to change since it's only a quick hack so that
                    # we don't have to do dep resolution ourselves
                    import _emerge as emerge
                    dep_emergepid = os.fork()
                    # we need to run emerge_main() in child process since a lot of stuff in there
                    # likes to call sys.exit() and we don't want that do we?
                    if 0 == dep_emergepid:
                        try:
                            extra_use = dep[0]
                            if extra_use:
                                os.environ["USE"]=" ".join(extra_use)
                            os.environ["FEATURES"] = "-strict"
                            sys.argv = ["emerge","--verbose", "--usepkg", "--buildpkg" ,"=%s" % dep[1]]
                            exit_code =  emerge.emerge_main()
                            sys.exit(exit_code)
                        except Exception, e:
                            print_exc()
                            log.error(format_exc())
                            sys.exit(1)
                    ret = os.waitpid(dep_emergepid, 0)
                    if 0 != ret[1]:
                        raise Exception("emerge_main() failed with error code %d" % ret[1])
                except Exception, e:
                    log.error(format_exc())
                    log.error("Unable to merge dependency %s for package %s" % (dep, pkg))
                    dep_failed = dep[1]

                deps_processed.append(dep)

                settings.setcpv(dep[1], mydb=portdb)
                dep_use_enabled = set(settings["PORTAGE_USE"].split())
                dep_iuse = set(settings["IUSE"].split())
                dep_name, dep_ver, dep_rev = portage.pkgsplit(dep[1])

                real_use_enabled = list(dep_use_enabled & dep_iuse)
                if dep[0]:
                    for useflag in dep[0]:
                        if useflag.startswith('-'):
                            if 0 is not l.count(useflag[1:]):
                                real_use_enabled.remove(useflag[1:])
                        elif 0 == real_use_enabled.count(useflag):
                            real_use_enabled.append(useflag)

                if dep_rev is'r0':
                    dep_ver_full = dep_ver
                else:
                    dep_ver_full = "%s-%s" % (dep_ver, dep_rev)
                dep_pkg = Package(dep_name, dep_ver_full, real_use_enabled)
                if dep_failed == dep[1]:
                    build_dir = self.get_build_dir(dep_failed)
                    self._add_attachment(dep_pkg, "%s/temp/build.log" % build_dir)
                    self._add_attachment(dep_pkg, "%s/temp/environment" % build_dir)
                package_infos.append(dep_pkg.get_info())


            if dep_failed:
                log.error("Unable to emerge package %s with deps %s" % (pkg, group))
                # TODO unmerge succeeded deps
                self._add_attachment(package, "/var/log/emerge.log")
                self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS)
                package.depends = [x[1] for x in group]
                continue
            settings.setcpv(pkg, mydb=portdb)

            ret = portage.doebuild(ebuild, "merge", portage.root, settings, debug = False, tree="porttree")
            if 0 != ret:
                # error installing, grab logs
                self._add_attachment(package, "%s/build.log" % settings["T"])
                self._add_attachment(package, "%s/environment" % settings["T"])
                self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS)
                self._add_attachment(package, "/var/log/emerge.log")



            package.depends = [x[1] for x in deps_processed]
            package_infos.append(package.get_info())

            for dep in group:
                if dep[1] in self.setconf.getSetAtoms("system"):
                    pass
                dep_cat, dep_pv = portage.catsplit(dep[1])

                ret = portage.unmerge(dep_cat, dep_pv, portage.root, settings, True, vartree=vartree)
                if 0 != ret:
                    log.error("Unable to unmerge dep %s" % dep)

            pkg_cat, pkg_pv = portage.catsplit(pkg)
            ret = portage.unmerge(pkg_cat, pkg_pv, portage.root, settings, True, vartree=vartree)
            if ret != 0:
                log.error("Unable to unmerge package %s" % dep)

        self._save_info('package_infos', package_infos)

    def _add_attachment(self, pkg, path):
        try:
            attfile = open(path,"r")
        except IOError, (errno, strerror):
            print "Unable to read file %s: %d: %s" % (path, errno, strerror)
            log.warning("Unable to read file (%s) %d: %s" % (path, errno, strerror))
            pkg.attachments[os.path.basename(path)] = "Unable to read file %s" % path
        else:
            pkg.attachments[os.path.basename(path)] = attfile.read()
            attfile.close()
            del attfile

    def _save_info(self, key, data):
        """
        Save data inside CHROOT_LOGS directory, under name 'key'. This
        function is called from within _emerge_package_subprocess to
        save data for parent process


        @param key: key used to identify data inside CHROOT_LOGS directory
        @type key: string
        @param data: data to be saved
        @rtype: None
        """
        outfile = open("%s/%s" % (config.CHROOT_LOGS, key), "w")
        pickle.dump(data, outfile)
        outfile.close()

    def _load_info(self, key):
        """
        Load data from CHROOT_LOGS directory (under WORK_CHROOT) with filename 'key'


        @param key: key used to identify data inside directory
        @type key: string
        @rtype: depends on key
        @returns: data loaded from filename, usually blob
        """
        infile = open("%s/%s/%s" % (config.WORK_CHROOT, config.CHROOT_LOGS,
                                    key),"r")
        return pickle.load(infile)



    def get_emerge_info(self):
        """
        @rtype: string
        @returns: emerge --info output

        """
        infoout = WritableObject()
        sys.stdout = infoout

        # emerge.action_info(self.settings, self.trees, [], [])
        # REDO withouth emerge.action_info
        ret = sys.stdout.content
        sys.stdout = sys.__stdout__

        return  string.join(infoout.content,sep='')

    def _normalize_dependencies(self, deps):
        """
        Normalizes dependencies by replacing '||' by with alternative

        @param deps: depedencies (list of package atoms)
        @type deps: List
        @rtype: List
        @returns: Normalized list of package atoms with replaced || groups

        Example:
        >>> from tinderbox import Tinderbox as tb
        >>> t = tb()
        >>> t._normalize_dependencies(['||', ['=virtual/jdk-1.5*', '=virtual/jdk-1.4*'], 'app-arch/unzip'])
        ['=virtual/jdk-1.5*', 'app-arch/unzip']
        >>>
        """
        log.debug("normalize_dependencies called with deps: %s" % deps)
        new_deps = []
        for i in range(len(deps)):
            if '||' == deps[i]:
                next = deps[i+1]
                if type(next) is list and len(next) > i:
                    deps[i+1] = next[0]
            elif type(deps[i]) == list:
                new_deps.extend(self._normalize_dependencies(deps[i]))
            else:
                new_deps.append(deps[i])
        return new_deps

    def create_dep_groups(self, deps):
        """
        Create dependency groups from package dependencies

        Every valid version of dep (according to input spec) has to be at least in 
        one output dependency group

        @param deps: dependencies of package (atoms)
        @type deps: List
        @rtype: List of Lists
        @returns: List of dependency groups (list of dependency versions) together with special use flags needed

        Example: 
        Input: ['=dev-libs/glib-2*', '>=net-fs/samba-3.0.0', 'x11-libs/libX11','dev-util/subversion[-dso]']
        Output:
        [[(None,'net-fs/samba-3.2.11'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.2-r1'), (['-dso'],'dev-util/subversion-1.5.5'],
         [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.3-r1'), (['-dso'],'dev-util/subversion-1.5.5'],
         [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.20.3'),(None, 'x11-libs/libX11-1.1.3'), (['-dso'],'dev-util/subversion-1.5.5'],
        """
        log.debug("create_dep_groups called with deps %s" % deps)
        result = None
        porttree = self.trees[portage.root]['porttree']
        portdb = porttree.dbapi
        deps_expanded = []
        max_dep_versions = 0
        for dep in deps:
            if dep[0].startswith('!') or dep in self.NOMERGE_PKGS:
                continue
            dep_useflag = list(portage.dep.dep_getusedeps(dep))
            if 0 == len(dep_useflag):
                dep_useflag = None
            if not portage.dep.isvalidatom(dep):
                log.error("%s is not valid atom in %s" % (dep, str(deps)))
                continue
            depversions = portdb.xmatch('match-all',dep)
            depversions = [(dep_useflag,x) for x in depversions]
            deps_expanded.append(depversions)
            if len(depversions) > max_dep_versions:
                max_dep_versions = len(depversions)

        for dep in deps_expanded:
            while len(dep) != 0 and len(dep) < max_dep_versions:
                dep.append(dep[0])

        result = []
        for i in range(max_dep_versions):
            group = []
            for dep in deps_expanded:
                if 0 == len(dep):
                    continue
                group.append(dep.pop())
            result.append(group)
        return result

    def get_build_dir(self, cpv):
        settings = self.settings
        bldprefix = settings["PORTAGE_TMPDIR"]+"/portage"
        return os.path.join(bldprefix, cpv)




class Package(object):

    def __init__(self, name, version, use_flags):
        """
        @param name: category/name of given package (excluding version/release)
        @type name: string
        @param version: version part of CPV e.g. 1.2.0-r4
        @type version: string
        @param use_flags: list of enabled use flags for given package
        @type use_flags: list

        """
        self.name = name
        self.version = version
        self.use_flags = use_flags
        self.reinit()

    def reinit(self):
        self.content = None
        self.attachments = {}
        self.depends = None


    def get_info(self):
        """Returns protocol.PackageInfo with information about this package

        @returns: PackageInfo for sending across network
        @rtype: protocol.PackageInfo
        """
        pi = protocol.PackageInfo()
        pi.name = self.name
        pi.version = self.version
        pi.use_flags = self.use_flags
        if self.content is None:
            self.content = self.get_package_contents()
        pi.content = self.content
        pi.attachments = self.attachments
        if 0 < len(self.attachments):
            pi.error = 1
        pi.depends = self.depends
        pi.profile = self._get_current_profile()
        return pi

    def _get_current_profile(self):
        ret = None
        try:
            profpath = os.readlink("%s/etc/make.profile" % portage.root)
            ret = profpath[profpath.rindex("profiles")+len("profiles")+1:]
        except Exception, e:
            print e
        return ret


    def get_package_contents(self):
        """Returns package contents as dict with paths as keys

        data values are tuples of information (such as hash and size)

        @returns: package contents (or {} if it's not installed)
        @rtype: dict
        """
        vartree =  portage.db[portage.root]["vartree"]
        cpv = "%s-%s" % (self.name, self.version)
        if not vartree.dbapi.cpv_exists(cpv):
            # maybe raise exception? instead
            return {}

        cat, pkg = portage.catsplit(cpv)
        dblink = portage.dblink(cat, pkg, portage.root, vartree.settings, 
                    treetype="vartree", vartree=vartree)

        return dblink.getcontents()