From 2afa2439d20402b7cb7af0ca12edd5bb0c269bdc Mon Sep 17 00:00:00 2001 From: André Erdmann Date: Tue, 19 Jun 2012 20:16:01 +0200 Subject: roverlay, 2012-06-19 * added Manifest creation to the overlay module * code moved/adjusted for R package -> overlay conversion * removed unnecessary code in ebuild module new file: roverlay/ebuild/__init__.py renamed: roverlay/portage/ebuild.py -> roverlay/ebuild/construction.py new file: roverlay/ebuild/creation.py renamed: roverlay/portage/manifest.py -> roverlay/manifest/__init__.py renamed: roverlay/portage/manifesthelpers.py -> roverlay/manifest/helpers.py renamed: roverlay/portage/metadata/creation.py -> roverlay/metadata/__init__.py renamed: roverlay/portage/metadata/abstractnodes.py -> roverlay/metadata/abstractnodes.py renamed: roverlay/portage/metadata/nodes.py -> roverlay/metadata/nodes.py renamed: roverlay/portage/overlay/root.py -> roverlay/overlay/__init__.py renamed: roverlay/portage/overlay/category.py -> roverlay/overlay/category.py renamed: roverlay/portage/overlay/package.py -> roverlay/overlay/package.py renamed: roverlay/portage/packageinfo.py -> roverlay/packageinfo.py deleted: roverlay/portage/__init__.py deleted: roverlay/portage/ebuildcreator.py deleted: roverlay/portage/ebuildjob.py deleted: roverlay/portage/metadata/__init__.py deleted: roverlay/portage/overlay/__init__.py new file: roverlay/static/__init__.py new file: roverlay/static/depres.py --- roverlay/ebuild/__init__.py | 38 +++ roverlay/ebuild/construction.py | 328 +++++++++++++++++++++ roverlay/ebuild/creation.py | 203 +++++++++++++ roverlay/manifest/__init__.py | 29 ++ roverlay/manifest/helpers.py | 207 +++++++++++++ roverlay/metadata/__init__.py | 90 ++++++ roverlay/metadata/abstractnodes.py | 284 ++++++++++++++++++ roverlay/metadata/nodes.py | 193 ++++++++++++ roverlay/overlay/__init__.py | 306 +++++++++++++++++++ roverlay/overlay/category.py | 105 +++++++ roverlay/overlay/package.py | 290 ++++++++++++++++++ roverlay/packageinfo.py | 254 ++++++++++++++++ roverlay/portage/__init__.py | 0 roverlay/portage/ebuild.py | 456 ----------------------------- roverlay/portage/ebuildcreator.py | 184 ------------ roverlay/portage/ebuildjob.py | 333 --------------------- roverlay/portage/manifest.py | 32 -- roverlay/portage/manifesthelpers.py | 207 ------------- roverlay/portage/metadata/__init__.py | 0 roverlay/portage/metadata/abstractnodes.py | 284 ------------------ roverlay/portage/metadata/creation.py | 89 ------ roverlay/portage/metadata/nodes.py | 193 ------------ roverlay/portage/overlay/__init__.py | 0 roverlay/portage/overlay/category.py | 85 ------ roverlay/portage/overlay/package.py | 219 -------------- roverlay/portage/overlay/root.py | 302 ------------------- roverlay/portage/packageinfo.py | 245 ---------------- roverlay/static/__init__.py | 5 + roverlay/static/depres.py | 29 ++ 29 files changed, 2361 insertions(+), 2629 deletions(-) create mode 100644 roverlay/ebuild/__init__.py create mode 100644 roverlay/ebuild/construction.py create mode 100644 roverlay/ebuild/creation.py create mode 100644 roverlay/manifest/__init__.py create mode 100644 roverlay/manifest/helpers.py create mode 100644 roverlay/metadata/__init__.py create mode 100644 roverlay/metadata/abstractnodes.py create mode 100644 roverlay/metadata/nodes.py create mode 100644 roverlay/overlay/__init__.py create mode 100644 roverlay/overlay/category.py create mode 100644 roverlay/overlay/package.py create mode 100644 roverlay/packageinfo.py delete mode 100644 roverlay/portage/__init__.py delete mode 100644 roverlay/portage/ebuild.py delete mode 100644 roverlay/portage/ebuildcreator.py delete mode 100644 roverlay/portage/ebuildjob.py delete mode 100644 roverlay/portage/manifest.py delete mode 100644 roverlay/portage/manifesthelpers.py delete mode 100644 roverlay/portage/metadata/__init__.py delete mode 100644 roverlay/portage/metadata/abstractnodes.py delete mode 100644 roverlay/portage/metadata/creation.py delete mode 100644 roverlay/portage/metadata/nodes.py delete mode 100644 roverlay/portage/overlay/__init__.py delete mode 100644 roverlay/portage/overlay/category.py delete mode 100644 roverlay/portage/overlay/package.py delete mode 100644 roverlay/portage/overlay/root.py delete mode 100644 roverlay/portage/packageinfo.py create mode 100644 roverlay/static/__init__.py create mode 100644 roverlay/static/depres.py (limited to 'roverlay') diff --git a/roverlay/ebuild/__init__.py b/roverlay/ebuild/__init__.py new file mode 100644 index 0000000..259a352 --- /dev/null +++ b/roverlay/ebuild/__init__.py @@ -0,0 +1,38 @@ +# R Overlay -- ebuild creation, ebuild class +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +class Ebuild ( object ): + + def __init__ ( self, content, header=None ): + """Initializes an Ebuild that has text content and optionally a + header (text, too). + + arguments: + * content -- + * header -- + """ + self.content = content + self.header = header + # --- end of __init__ (...) --- + + def write ( self, fh, header=None ): + """Write the ebuild into a file-like object. + + arguments: + * fh -- file handle + """ + if not self.content: + raise Exception ( "ebuild is empty!" ) + + if header is None: + if not self.header is None: + fh.write ( self.header ) + fh.write ( '\n' ) + else: + fh.write ( header ) + fh.write ( '\n' ) + + fh.write ( self.content ) + fh.write ( '\n' ) + # --- end of write_fh (...) --- diff --git a/roverlay/ebuild/construction.py b/roverlay/ebuild/construction.py new file mode 100644 index 0000000..ed5bca5 --- /dev/null +++ b/roverlay/ebuild/construction.py @@ -0,0 +1,328 @@ +# R Overlay -- ebuild creation, ebuild class +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import copy + +import roverlay.config + +from roverlay.util import shorten_str +from roverlay.ebuild import Ebuild + + +class EbuildConstruction ( object ): + """Class that helps to create Ebuild objects.""" + + EBUILD_INDENT = roverlay.config.get ( 'EBUILD.indent', '\t' ) + + ADD_REMAP = { + # pkg vs package + 'package_name' : 'pkg_name', + 'package_version' : 'pkg_version', + 'package_revision' : 'pkg_revision', + # TITLE is in DESCRIPTION + 'TITLE' : 'DESCRIPTION', + + # TODO: remove these entries by fixing ebuildcreator/ebuildjob + 'DEPENDS' : 'DEPEND', + 'RDEPENDS' : 'RDEPEND', + 'RSUGGESTS' : 'R_SUGGESTS', + } + + def __init__ ( self, logger ): + """Initializes an EbuildConstruction object. + + arguments: + * logger -- + """ + self.logger = logger + + self.has_rsuggests = False + + # elements in data are either a str or a list of str + self._data = dict () + # --- end of __init__ (...) --- + + def get_ebuild ( self ): + """Creates and returns an Ebuild.""" + lines = '\n'.join ( self._make_ebuild_lines() ) + return Ebuild ( lines, header=None ) + # --- end of get_ebuild (...) --- + + def add ( self, key, value, append=True ): + """Adds data. + + arguments: + * key -- identifier of the data (e.g. DEPEND). + May be remapped (e.g. merging 'Title' and 'Description') + or even refused here. + * value -- + * append -- whether to append values or overwrite existing ones, + defaults to True. + + returns: None (implicit) + """ + if self._data is None: + # -- todo + raise Exception ("Ebuild is readonly.") + + _key = Ebuild.ADD_REMAP [key] if key in Ebuild.ADD_REMAP else key + + if _key is None: + self.logger.debug ( "add (%s, %s): filtered key.", key, value ) + else: + if append and _key in self._data: + if not isinstance ( self._data [_key], list ): + self._data [_key] = [ self._data [_key] ] + + if isinstance ( value, list ): + self._data [_key].extend ( value ) + else: + self._data [_key].append ( value ) + + else: + self._data [_key] = value + + # --- end of add (...) --- + + def _make_ebuild_lines ( self ): + """Creates text lines for the Ebuild. + It assumes that enough data to do this are available. + Exceptions (KeyError, NameError, ...) are passed if that's not the case. + """ + + def get_dep_and_use(): + """Creates values for the DEPEND, RDEPEND, IUSE and, if possible, + R_SUGGESTS variables and returns them as dict { VARNAME -> VALUE }. + """ + + # have suggests if they're set and not empty + self.has_rsuggests = bool ( + 'R_SUGGESTS' in self._data and self._data ['R_SUGGESTS'] + ) + + # set defaults: inherit eclass + include depend in rdepend + # TODO: is ${DEPEND:-},... necessary? + ret = dict ( + DEPEND = [ '${DEPEND:-}' ], + # assuming that the eclass includes it's DEPEND in RDEPEND + RDEPEND = [ '${RDEPEND:-}' ], + IUSE = [ '${IUSE:-}' ], + ) + + for kw in ( x for x in ( 'DEPEND', 'RDEPEND' ) if x in self._data ): + if isinstance ( self._data [kw], list ): + ret [kw].extend ( self._data [kw] ) + else: + ret [kw].append ( self._data [kw] ) + + + if self.has_rsuggests: + ret ['R_SUGGESTS'] = self._data ['R_SUGGESTS'] + + # +R_SUGGESTS, -R_SUGGESTS? + ret ['IUSE'].append ( 'R_suggests' ) + # do these braces help or confuse? TODO FIXME + ret ['RDEPEND'].append ( '( R_suggests ? ${R_SUGGESTS} )' ) + + return ret + + # --- end of get_dep_and_use () --- + + def make_var ( + varname, + value=None, oneline_list=True, indent_list=True, indent_level=0 + ): + """Creates a = statement for ebuilds. + + arguments: + * varname -- name of the variable + * value -- value of the variable. + This has to be either None (the default), str, + or list of str. + * oneline_list -- if value is a list: controls whether its components + should be put into one line (True) or multiple. + Defaults to True. + * indent_list -- if value is a list and not oneline_list: + controls whether each value line should be + indentend (by indent_level + 1) or not ("by 0"). + Defaults to True. + * indent_level -- current indentation level, defaults to 0 + + """ + + # assumption: value is either None, + # scalar with str representation or list of str + var_value = None + + if not value: + var_value = "" + + elif isinstance ( value, list ): + if oneline_list: + var_value = ' '.join ( value ) + elif indent_list: + var_value = ( + '\n' + (indent_level + 1) * Ebuild.EBUILD_INDENT + ).join ( value ) + else: + '\n'.join ( value ) + + else: + var_value = str ( value ) + + + # (TODO) + # fixing ebuild var values here + + # cut DESCRIPTION line if too long + if varname == 'DESCRIPTION': + var_value = shorten_str ( var_value, 45, '... (see metadata)' ) + + + ret ='%s%s="%s"' % ( + indent_level * Ebuild.EBUILD_INDENT, + varname, + var_value + ) + + # (TODO) + # fixing ebuild var lines here + + return ret + + # --- end of make_var (...) --- + + def remove_newlines ( line_list ): + """Removes leading, ending and repeated blank lines in line_list. + + arguments: + * line_list -- + + returns: filtered lines + + TODO: check if a filter function could be used for this + """ + lines = [] + line = None + last_line_empty = False + + for line in line_list: + line = line.rstrip() + # re.sub \n{2,} \n :: FIXME? + + if line: + last_line_empty = False + elif not last_line_empty: + last_line_empty = True + else: + continue + + lines.append ( line ) + + # remove last line if empty + ##if last_line_empty: (?) + if len ( lines ) and not lines [-1]: + del lines [-1] + + return lines + + # --- end of remove_newlines (...) --- + + def add_easyvar ( + ebuild_content, varname, + value_key=None, add_newline=False + ): + """Adds a 'simple' variable to the ebuild lines. + This means that it can directly be taken from self._data [value_key]. + This method assumes that value_key exists in self._data, + any exceptions (KeyError) will be passed. + + arguments: + * ebuild_content -- list of ebuild text lines, will be modified + directly, so copy it before calling addvar if + you need the original list. + * varname -- name of the variable. + Nothing happens if this is None. + * value_key -- key of the value, + defaults to varname if it is None + * add_newline -- adds a newline after the var statement, + defaults to False + + returns: given+modified list (ebuild_content) + """ + + if not varname is None: + if value_key is None: + ebuild_content.append ( + make_var ( varname, self._data [varname] ) + ) + else: + ebuild_content.append ( + make_var ( varname, self._data [value_key] ) + ) + + if add_newline: + ebuild_content.append ( "" ) + + return ebuild_content + + # --- end of add_easyvar (...) --- + + # -- actual start of _make_ebuild_lines (...) -- + try: + ebuild_lines = [] + + if 'ebuild_header' in self._data: + ebuild_lines = copy.copy ( self._data ['ebuild_header'] ) + ebuild_lines.append ( "" ) + + add_easyvar ( ebuild_lines, "PKG_FILE" ) + if 'PKG_ORIGIN' in self._data: + add_easyvar ( ebuild_lines, "PKG_ORIGIN", None, False ) + + ebuild_lines.append ( "" ) + + # TODO/FIXME: this makes DESCRIPTION mandatory, maybe check with + # >if 'DESCRIPTION' in self._data< + add_easyvar ( ebuild_lines, "DESCRIPTION" ) + + add_easyvar ( ebuild_lines, "SRC_URI", add_newline=True ) + + # FIXME/TODO: LICENSE? + + dep_and_use = get_dep_and_use () + + # check that IUSE has more than one element, + # don't write IUSE="${IUSE:-}" etc. + if len ( dep_and_use ['IUSE'] ) > 1: + ebuild_lines.append ( + make_var ( "IUSE", dep_and_use ['IUSE'], True ) + ) + + if 'R_SUGGESTS' in dep_and_use: + ebuild_lines.append ( + make_var ( "R_SUGGESTS", dep_and_use ['R_SUGGESTS'], False ) + ) + + # see IUSE + if len ( dep_and_use ['DEPEND'] ) > 1: + ebuild_lines.append ( + make_var ( "DEPEND", dep_and_use ['DEPEND'], False ) + ) + + # see IUSE + if len ( dep_and_use ['RDEPEND'] ) > 1: + ebuild_lines.append ( + make_var ( "RDEPEND", dep_and_use ['RDEPEND'], False ) + ) + + del dep_and_use + return remove_newlines ( ebuild_lines ) + + except ( ValueError, KeyError, NameError ) as err: + self.logger.exception ( err ) + self.logger.error ( "Cannot create ebuild text lines." ) + return None + + # --- end of make_ebuild_lines (...) --- diff --git a/roverlay/ebuild/creation.py b/roverlay/ebuild/creation.py new file mode 100644 index 0000000..eed54b2 --- /dev/null +++ b/roverlay/ebuild/creation.py @@ -0,0 +1,203 @@ +# R Overlay -- ebuild creation, "job" module +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import logging + +import roverlay.static.depres + +from roverlay.ebuild.construction import EbuildConstruction +from roverlay.rpackage.descriptionreader import DescriptionReader + +# move this to const / config +DEPENDENCY_FIELDS = { + 'R_SUGGESTS' : [ 'Suggests' ], + 'DEPENDS' : [ 'Depends', 'Imports' ], + 'RDEPENDS' : [ 'LinkingTo', 'SystemRequirements' ] +} + +LOGGER = logging.getLogger ( 'EbuildCreation' ) + +class EbuildCreation ( object ): + + def __init__ ( self, package_info, depres_channel_spawner=None ): + + self.logger = LOGGER.getChild ( package_info ['name'] ) + self.package_info = package_info + + if depres_channel_spawner is None: + self.request_resolver = roverlay.static.depres.get_ebuild_channel + else: + self.request_resolver = depres_channel_spawner + + # > 0 busy/working; 0 == done,success; < 0 done,fail + self.status = 1 + + self.package_info.set_readonly() + # --- end of __init__ (...) --- + + def done() : return self.status < 1 + def busy() : return self.status > 0 + def success() : return self.status == 0 + def fail() : return self.status < 0 + + + def _resolve_dependencies ( self, ebuilder ): + if self.request_resolver is None: + self.logger.warning ( + "Cannot resolve dependencies, no resolver available!" + ) + return True + + res = None + # -- end pre func block -- + + def init_channels(): + # collect dep strings and initialize resolver channels + desc = self.package_info ['desc_data'] + channels = dict() + + def get_resolver ( dependency_type ): + if dependency_type not in channels: + channels [dependency_type] = self.request_resolver ( + dependency_type, + self.logger + ) + return channels [dependency_type] + # --- end of get_resolver (...) --- + + dep_type = desc_field = None + + for dep_type in DEPENDENCY_FIELDS: + resolver = None + + for desc_field in DEPENDENCY_FIELDS [dep_type]: + if desc_field in desc: + if not resolver: + resolver = get_resolver ( dep_type ) + + if isinstance ( desc [desc_field], str ): + resolver.add_dependency ( desc [desc_field] ) + elif hasattr ( desc [desc_field], '__iter__' ): + resolver.add_dependencies ( desc [desc_field] ) + else: + logger.warning ( + "Cannot add dependency '%s'." % desc [desc_field] + ) + # -- if desc_field + # -- for desc_field + # -- for dep_type + return channels + # --- end of init_resolvers (...) --- + + def try_resolve(): + for r in res.values(): + if r.satisfy_request() is None: + return False + return True + # --- end of try_resolve (...) --- + + # TODO + # replace try_resolve with + # False in ( r.satisfy_request() for r in res.values() ) + # ? + res = init_channels() + if not res: return True + success = False + + + if try_resolve(): + for dep_type, resolver in res.items(): + deplist = list ( filter ( None, resolver.collect_dependencies() ) ) + + if deplist is None: + ## FIXME: false positive: "empty" channel + raise Exception ( + 'dep_resolver is broken: ' + 'lookup() returns None but satisfy_request() says ok.' + ) + elif hasattr ( deplist, '__iter__' ): + # add dependencies in no_append/override mode + self.logger.debug ( "adding %s to %s", deplist, dep_type ) + ebuilder.add ( dep_type, deplist, False ) + else: + raise Exception ( "dep_resolver is broken: iterable expected!" ) + # -- for dep_type,.. + + success = True + + # tell the dep resolver channels that we're done + for r in res.values(): r.close() + return success + # --- end of resolve_dependencies (...) --- + + def _make_ebuild ( self ): + desc = self.package_info ['desc_data'] + if desc is None: + self.logger ( + 'desc empty- cannot create an ebuild for this package.' + ) + return None + + ebuilder = EbuildConstruction ( self.logger ) + + have_desc = False + + if 'Title' in desc: + ebuilder.add ( 'DESCRIPTION', desc ['Title'] ) + have_desc = True + + if 'Description' in desc: + ebuilder.add ( + 'DESCRIPTION', + ( '// ' if have_description else '' ) + desc ['Description'] + ) + + + ebuilder.add ( 'SRC_URI', self.package_info ['package_url'] ) + + if self._resolve_dependencies(): + return ( ebuilder.get_ebuild(), ebuilder.has_rsuggests ) + + return None + # --- end of _make_ebuild (...) --- + + def run ( self ): + if self.status < 1: + raise Exception ( "Cannot run again." ) + + try: + if self.package_info.get ( 'desc_data', + fallback_value=None, do_fallback=True ) is None: + + logging.warning ( 'Reading description data now.' ) + reader = DescriptionReader ( + self.package_info, + logger=self.logger, + read_now=True + ) + self.package_info.set_writeable() + self.package_info.update ( + desc_data=reader.get_desc ( run_if_unset=False ) + ) + del reader + # -- if + + self.package_info.set_readonly() + + ebuild_info = self._make_ebuild() + if ebuild_info is None: + self.status = -1 + else: + self.package_info.set_writeable() + self.package_info.update ( + ebuild=ebuild_info [0], + suggests=ebuild_info [1] + ) + self.package_info.set_readonly() + self.status = 0 + except Exception as e: + # log this and set status to fail + self.status = -10 + self.logger.exception ( e ) + # --- end of run (...) --- diff --git a/roverlay/manifest/__init__.py b/roverlay/manifest/__init__.py new file mode 100644 index 0000000..b95a855 --- /dev/null +++ b/roverlay/manifest/__init__.py @@ -0,0 +1,29 @@ +# R Overlay -- Manifest creation for ebuilds +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +import logging + +from roverlay.manifest import helpers + +_MANIFEST_IMPLEMENTATION = helpers.ExternalManifestCreation + + +def create_manifest ( package_info, nofail=False ): + """Creates a Manifest for package_info, using the <> implementation + available. + + current implementation: ExternalManifestCreation (using ebuild(1)) + + arguments: + * package_info -- + * nofail -- catch exceptions and return False + """ + try: + return _MANIFEST_IMPLEMENTATION.do ( package_info ) + except Exception as e: + logging.exception ( e ) + if nofail: + return False + else: + raise +# --- end of create_manifest (...) --- diff --git a/roverlay/manifest/helpers.py b/roverlay/manifest/helpers.py new file mode 100644 index 0000000..e7a54eb --- /dev/null +++ b/roverlay/manifest/helpers.py @@ -0,0 +1,207 @@ +# R Overlay -- Manifest creation for ebuilds +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +# TODO (in future): could use portage api directly, namely +# '/usr/lib/portage/pym/portage/package/ebuild/doebuild.py' +# instead of using '/usr/bin/ebuild' + +import os +import re +import copy +import logging +import subprocess + +from roverlay import config, util + +class _ManifestCreation ( object ): + """This is the base class for Manifest file creation.""" + + static_instance = None + + def __init__ ( self ): + self.logger = logging.getLogger ( 'ManifestCreation' ) + # --- end of __init__ (...) --- + + def create_for ( self, package_info ): + """Creates a Manifest file for the ebuild of the given package_info.""" + raise Exception ( "method stub" ) + # --- end of create_for (...) --- + + @classmethod + def do ( cls, package_info ): + """Class/static access to Manifest creation.""" + if cls.static_instance is None: + cls.static_instance = cls() + + return cls.static_instance.create_for ( package_info ) + # --- end of do (...) --- + + +class ExternalManifestCreation ( _ManifestCreation ): + """This class implements Manifest creation using the low level ebuild + interface, ebuild(1), which is called in a filtered environment. + """ + + def __init__ ( self ): + super ( ExternalManifestCreation, self ) . __init__ () + self.manifest_env = ManifestEnv ( filter_env=True ) + # ebuild , where target is: + self.ebuild_tgt = config.get ( 'TOOLS.EBUILD.target', 'manifest' ) + self.ebuild_prog = config.get ( 'TOOLS.EBUILD.prog', '/usr/bin/ebuild' ) + + # --- end of __init__ (...) --- + + def create_for ( self, package_info ): + """See ManifestCreation.create_for. + Calls ebuild, returns True on success else False. + + raises: *passes Exceptions from failed config lookups + """ + + my_env = self.manifest_env [ package_info ['distdir'] ] + + ebuild_file = package_info ['ebuild_file'] + + ebuild_call = subprocess.Popen ( + ( + self.ebuild_prog, + ebuild_file, + self.ebuild_tgt + ), + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=my_env + ) + + output = ebuild_call.communicate() + # necessary? (probably not, FIXME/TODO) + ebuild_call.wait() + + # log stdout? + #for line in util.pipe_lines ( output [0] ): + # LOGGER.debug ( line ) + #for line in util.pipe_lines ( output [0] ): print ( line ) + + # log stderr + for line in util.pipe_lines ( output [1], use_filter=True ): + self.logger.warning ( line ) + + if ebuild_call.returncode == 0: + return True + else: + self.logger.error ( "Couldn't create Manifest for %s!" % ebuild_file ) + return False + # --- end of create_for (...) --- + + +class ManifestEnv ( object ): + """per-repo environment container for Manifest creation using ebuild.""" + + def __init__ ( self, filter_env=True ): + """Initializes a ManifestEnv. + + arguments: + * filter_env -- if True: start with an empty env and copy vars + from os.environ selectively + else : start with os.environ as env + """ + self.filter_env = filter_env + self._manenv = dict() + self.logger = logging.getLogger ( 'ManifestEnv' ) + self._common_env = None + # --- end of __init__ (...) --- + + def get_env ( self, repo_dir ): + """Returns an env dict for repo_dir. + + arguments: + * repo_dir -- + """ + if not repo_dir in self._manenv: + repo_env = self._get_common_manifest_env() + repo_env ['DISTDIR'] = repo_dir + self._manenv [repo_dir] = repo_env + + return self._manenv [repo_dir] + # --- end of get_env (...) --- + + # x = ManifestEnv(); env = x [] etc. + __getitem__ = get_env + # --- + + def _get_common_manifest_env ( self, noret=False ): + """Creates an environment suitable for an + "ebuild digest|manifest" call (or uses an already existing env). + Returns a shallow copy of this env which can then be locally modified + (setting DISTDIR). + + arguments: + * noret -- do not return copied env if True + """ + + if self._common_env is None: + + if self.filter_env: + + # selectively import os.environ + # FIXME: keep EBUILD_DEFAULT_OPTS? + our_env = util.keepenv ( + ( 'PATH', '' ), + 'LANG', + 'PWD', + 'EBUILD_DEFAULT_OPTS' + ) + else: + # copy os.environ + our_env = dict ( os.environ ) + + # -- common env part + + # set FEATURES + # * digest -- needed? (works without it) + # * assume-digests -- + # * unknown-features-warn -- should FEATURES ever change + # + # * noauto -- should prevent ebuild from adding additional actions, + # it still tries to download source packages, which is just wrong + # here 'cause it is expected that the R package file exists when + # calling this function, so FETCHCOMMAND/RESUMECOMMAND will be set + # to /bin/true if possible. + # + # * distlocks -- disabled if FETCHCOMMAND/RESUMECOMMAND set to no-op + # + our_env ['FEATURES'] = \ + "noauto digest assume-digests unknown-features-warn" + + # try to prevent src fetching + fetch_nop = util.sysnop ( + nop_returns_success=True, + format_str="%s \${DISTDIR} \${FILE} \${URI}" + ) + + if not fetch_nop is None: + self.logger.debug ( + "%s disables/replaces FETCHCOMMAND,RESUMECOMMAND." + % fetch_nop [0] + ) + our_env ['FETCHCOMMAND'] = fetch_nop [1] + our_env ['RESUMECOMMAND'] = fetch_nop [1] + our_env ['FEATURES'] += " -distlocks" + + + + # set PORDIR_OVERLAY + our_env ['PORTDIR_OVERLAY'] = config.get_or_fail ( + [ 'OVERLAY', 'dir' ] + ) + + self._common_env = our_env + # -- end if + if noret: + return None + else: + return copy.copy ( self._common_env ) + # --- end of _get_common_manifest_env (...) --- diff --git a/roverlay/metadata/__init__.py b/roverlay/metadata/__init__.py new file mode 100644 index 0000000..82821bb --- /dev/null +++ b/roverlay/metadata/__init__.py @@ -0,0 +1,90 @@ +# R Overlay -- ebuild creation, metadata creation +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import roverlay.config + +from roverlay.metadata import nodes + +class MetadataJob ( object ): + """R package description data -> metadata.xml interface.""" + + def __init__ ( self, logger ): + """Initializes a MetadataJob. + + arguments: + (((* package_info -- reserved for future usage))) + * logger -- parent logger to use + """ + self.logger = logger.getChild ( 'metadata' ) + self._metadata = nodes.MetadataRoot() + # reserved for future usage ("dominant ebuilds": when ebuildjobs + # share one metadata instance etc.) + self.package_info = None + self.filename = 'metadata.xml' + # --- end of __init__ (...) --- + + def update ( self, package_info ): + """Updates the metadata using the given description data. + + It's expected that this method is called when Ebuild creation is done. + + arguments: + * desc_data -- description data read from R package + * package_info -- reserved for future usage + + returns: None (implicit) + """ + desc_data = package_info ['desc_data'] + + mref = self._metadata + + max_textline_width = roverlay.config.get ( 'METADATA.linewidth', 65 ) + + have_desc = False + + if 'Title' in desc_data: + mref.add ( nodes.DescriptionNode ( + desc_data ['Title'], + is_long=have_desc, + linewidth=max_textline_width + ) ) + have_desc = True + + if 'Description' in desc_data: + # passing have_desc for DescriptionNode's is_long parameter redirects + # the second description info into + mref.add ( nodes.DescriptionNode ( + desc_data ['Description'], + is_long=have_desc, + linewidth=max_textline_width + ) ) + have_desc = True + + mref.add_useflag ( 'byte-compile', 'enable byte-compiling' ) + + if package_info ['has_suggests']: + mref.add_useflag ( 'R_suggests', 'install optional dependencies' ) + + # --- end of update (...) --- + + # used in some test scripts + update_metadata = update + + def write ( self, _file ): + """Writes the metadata into a file. + + arguments: + * _file -- file to write, either a file handle or string in which case + a file object will be created + + returns: True if writing succeeds, else False + + raises: Exception if no metadata to write + """ + if self._metadata.empty(): + raise Exception ( "not enough metadata to write!" ) + #return False + else: + return self._metadata.write_file ( _file ) + # --- end of write (...) --- diff --git a/roverlay/metadata/abstractnodes.py b/roverlay/metadata/abstractnodes.py new file mode 100644 index 0000000..50c9f56 --- /dev/null +++ b/roverlay/metadata/abstractnodes.py @@ -0,0 +1,284 @@ +# R Overlay -- ebuild creation, basic metadata nodes +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +INDENT = '\t' + +get_indent = lambda k : k * INDENT + +# -- "abstract" metadata nodes -- +class _MetadataBasicNode ( object ): + """ + This is the most basic metadata node that should never be used directly. + """ + + def __init__ ( self, name, flags ): + """Initializes a _MetadataBasicNode. + + arguments: + * name -- name of this node, e.g. 'flag' + * flags -- flags of this node, e.g. 'name=byte-compile' + """ + self.name = name + self.flags = flags + # priority is used to sort nodes (e.g. longdescription after description) + self.priority = 1000 + self._enabled = True + self._set_level ( 0 ) + # --- end of __init__ (...) --- + + def active ( self ): + """Returns True if this node is active.""" + return self._enabled + # --- end of active (...) --- + + def _set_level ( self, _level, has_indent=True ): + """Sets the level (depth) of this node. + + arguments: + * _level -- level to set + * has_indent -- if this node should indent its subnodes (or text) + """ + self.level = _level + if has_indent: + self.indent = get_indent ( self.level ) + else: + self.indent = '' + # --- end of _set_level (...) --- + + def set_flag ( self, flagname, flagvalue=None ): + """Sets the specified flag. + + arguments: + * flagname -- flag name + * flagvalue -- flag value, defaults to None->'' + """ + self.flags [flagname] = '' if flagvalue is None else flagvalue + # --- end of set_flag (...) --- + + def _flaglist ( self ): + """Returns a "flagname=flagvalue" list.""" + return [ '%s="%s"' % ftup for ftup in self.flags.items() ] + # --- end of _flaglist (...) --- + + + def _flagstr ( self ): + """Returns the string representation of this node's flags, including + a leading whitespace char. + """ + if self.flags is None: + return '' + + ret = ' '.join ( self._flaglist() ) + if ret: + # don't forget the leading space + return ' ' + ret + else: + return '' + # --- end of _flagstr (...) --- + + def _do_verify ( self ): + """Verifies this node. + Does nothing if self._verify is not implemented, else dies on error. + """ + if hasattr ( self, '_verify' ) and not self._verify(): + # todo, verify could return ( Status, ErrorMessages ) etc. + raise Exception ( "verification failed for a metadata node." ) + # --- end of _do_verify (...) --- + + # not using __str__, make (recursive) node calls explicit + def to_str ( self ): + """Returns a string representing this node.""" + self._do_verify() + + return "%s<%s%s>" % ( + self.indent, + self.name, + self._flagstr(), + self.name + ) + # --- end of to_str (...) --- + + +class MetadataNode ( _MetadataBasicNode ): + """A _MetadataBasicNode with child nodes.""" + + def __init__ ( self, name, flags=dict() ): + super ( MetadataNode, self ) . __init__ ( name, flags ) + self.nodes = list() + # --- end of __init__ (...) --- + + def add ( self, node ): + """Adds a child node to this node. Fixes/sets the level of this node. + + arguments: + * node -- + """ + node._set_level ( self.level + 1 ) + self.nodes.append ( node ) + # --- end of add (...) --- + + # copy add to _add_node + _add_node = add + + def _sort_nodes ( self ): + """Sorts the child nodes of this node.""" + self.nodes.sort ( key=lambda node : node.priority ) + # --- end of _sort_nodes (...) --- + + def _nodelist ( self ): + """Returns a list of strings representing the child nodes.""" + return list ( + filter ( + None, + [ node.to_str() for node in self.nodes if node.active() ] + ), + ) + # --- end of _nodelist (...) --- + + def _nodestr ( self ): + """Returns a string representing all child nodes.""" + self._sort_nodes() + # todo filter only None? + node_repr = self._nodelist() + if len ( node_repr ): + # add newlines before/after and indent after node_repr! + return "\n%s\n%s" % ( '\n'.join ( node_repr ), self.indent ) + else: + return '' + # --- end of _nodestr (...) --- + + def to_str ( self ): + """Returns a string representing this node and all of its child nodes.""" + self._do_verify() + return "%s<%s%s>%s" % ( + self.indent, + self.name, + self._flagstr(), + self._nodestr(), + self.name + ) + + +class MetadataNodeOrdered ( MetadataNode ): + """A metadata node whose nodes have to be interpreted as an already + ordered string.""" + # could derive MetadataNode from this + def _sort_nodes ( self ): return + + +class MetadataLeaf ( _MetadataBasicNode ): + """A metadata node that has no child nodes, only a string.""" + + def __init__ ( self, name, flags=dict(), value=None ): + self._text_wrapper = None + super ( MetadataLeaf, self ) . __init__ ( name, flags ) + + self.value = "" if value is None else value + self.print_node_name = True + self.value_format = 0 + + def _set_level ( self, _level, has_indent=True ): + super ( MetadataLeaf, self ) . _set_level ( _level, has_indent ) + + # indenting multi-line text by one level more than self.indent + self.text_indent = self.indent + INDENT + + # update text wrapper if existent + if not self._text_wrapper is None: + self._text_wrapper.subsequent_indent = self.text_indent + + def _value_str ( self ): + """Returns the value string. Derived classes may override this.""" + #if self.value_format == ?: format value ~ + return str ( self.value ) + # --- end of _value_str (...) --- + + def _pretty_value_str ( self ): + """Returns a formatted value string (max line length etc.). + Not used here, but subclasses can use it by simply writing + '_value_str = MetadataLeaf._pretty_value_str' in the class body. + """ + # FIXME/TODO: could move this func to util + if not self.value: return "" + + if self._text_wrapper is None: + self._text_wrapper = textwrap.TextWrapper ( + initial_indent='', + subsequent_indent=self.text_indent, + width=self.linewidth if hasattr ( self, 'linewidth' ) else 50 + ) + + val_lines = self._text_wrapper.wrap ( self.value ) + if len ( val_lines ) < 1: + # why? + return "" + elif len ( val_lines ) == 1: + # single line, no indent/newline + return val_lines [0] + else: + # add newline before/after, add indent after + val_lines [0] = '\n' + self.text_indent + val_lines [0] + val_lines.append ( self.indent ) + return '\n'.join ( val_lines ) + # --- end of _pretty_value_str (...) --- + + def to_str ( self ): + self._do_verify() + if self.print_node_name: + return "%s<%s%s>%s" % ( + self.indent, + self.name, + self._flagstr(), + self._value_str(), + self.name + ) + else: + # not very useful, but allows to insert strings as nodes + return self.indent + self._value_str() + # --- end of to_str (...) --- + +class MetadataNodeNamedAccess ( MetadataNode ): + """A metadata node that offers key-based (dictionary) access to some of + its nodes.""" + + def __init__ ( self, name, flags=dict() ): + super ( MetadataNodeNamedAccess, self ) . __init__ ( name, flags ) + # the access dict + self.node_dict = dict() + + def add ( self, node, with_dict_entry=True, fail_if_existent=True ): + """Adds a child node. + + arguments: + * node -- node to add + * with_dict_entry -- add node to the access dict, defaults to True + * fail_if_existent -- fail if node's name already in the access dict, + defaults to True + """ + + super ( MetadataNodeNamedAccess, self ) . add ( node ) + if with_dict_entry: + if fail_if_existent and node.name in self.node_dict: + raise Exception ( "key exists." ) + else: + self.node_dict [node.name] = node + # --- end of add (...) --- + + def get ( self, node_name ): + """Returns node by name. + + arguments: + * node_name + + raises: KeyError if node_name not in the access dict + """ + return self.node_dict [node_name] + # --- end of get (...) --- + + def has_named ( self, node_name ): + """Returns True if node_name in the access dict else False.""" + return node_name in self.node_dict + # --- end of has_named (...) --- diff --git a/roverlay/metadata/nodes.py b/roverlay/metadata/nodes.py new file mode 100644 index 0000000..f35f8da --- /dev/null +++ b/roverlay/metadata/nodes.py @@ -0,0 +1,193 @@ +# R Overlay -- ebuild creation, concrete metadata nodes +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# import abstract nodes +from roverlay.metadata.abstractnodes import \ + MetadataNode, MetadataNodeNamedAccess, MetadataLeaf + + +class MetadataRoot ( MetadataNodeNamedAccess ): + """This is the metadata root which represents a metadata file. + Intended usage is metadata file creation. + """ + + # the common metadata.xml header + HEADER = '\n'.join ( [ + '', + '' + ] ) + + + def __init__ ( self ): + super ( MetadataRoot, self ) . __init__ ( 'pkgmetadata' ) + self.priority = 0 + # --- end of __init__ (...) --- + + def empty ( self ): + """Returns True if this node has no child nodes.""" + #return 0 == len ( self.nodes ) or \ + # True in ( node.empty() for node in self.nodes ) + return 0 == len ( self.nodes ) + # --- end of empty (...) --- + + def add_useflag ( self, flag_name, flag_description ): + """Adds a USE Flag to the metadata. + A UseFlagListNode 'use' will be created if required and a new UseFlagNode + will then be created an added to 'use'. + + arguments: + * flag_name -- see UseFlagNode.__init__ + * flag_description -- see UseFlagNode.__init__ + + returns: the created UseFlagNode for further editing + """ + if not self.has_named ( 'use' ): + # passing fail_if_existent, this node shouldn't be used in parallel + self.add ( + UseFlagListNode(), + with_dict_entry=True, fail_if_existent=True + ) + + node = self.get ( 'use' ) + use_node = UseFlagNode ( flag_name, flag_description ) + node.add ( use_node ) + + return use_node + # --- end of add_useflag (...) --- + + def write_file ( self, _file ): + """Writes the metadata to a file. + + arguments: + * _file -- either a File object or a string + + returns: success True/False + + raises: *passes IOError + """ + to_write = self.to_str() + + own_fh = False + fh = None + success = False + + newline = '\n' + + try: + if isinstance ( _file, str ): + own_fh = True + fh = open ( _file, 'w' ) + else: + fh = _file + + + fh.write ( MetadataRoot.HEADER ) + fh.write ( newline ) + fh.write ( to_write ) + fh.write ( newline ) + + success = True + + except IOError: + # log this TODO + pass + finally: + if own_fh and fh: fh.close() + + return success + # --- end of write_file (...) --- + + +class DescriptionNode ( MetadataLeaf ): + """A description (, ) node.""" + + def __init__ ( self, description, is_long=False, linewidth=None ): + """Initializes a DescriptionNode. + + arguments: + * description -- description text + * is_long -- if this is a longdescription or a description node + * linewidth -- max text line width, TODO/FIXME: is this ignored? + """ + super ( DescriptionNode, self ) . __init__ ( + 'longdescription' if is_long else 'description', + value=description, + ) + # self.value_format = "break lines after 80c, ..." + + if not linewidth is None and linewidth > 0: + self.linewidth = linewidth + + self.priority = 150 if is_long else 149 + # --- end of __init__ (...) --- + + # using value formatting + _value_str = MetadataLeaf._pretty_value_str + + +class UseFlagNode ( MetadataLeaf ): + """A USE Flag node, this flag does...""" + def __init__ ( self, flag_name, flag_description ): + """Initializes an USE Flag node. + + arguments: + * flag_name -- name of the use flag + * flag_description -- flag description + """ + super ( UseFlagNode, self ) . __init__ ( + 'flag', + flags=dict ( name = flag_name ), + value=flag_description, + ) + # priority shouldn't be used for this node + self.priority = -1 + # --- end of __init__ (...) --- + + +class UseFlagListNode ( MetadataNode ): + """A USE Flag list node, ....""" + + def __init__ ( self, flags=dict() ): + """Initializes an USE Flag list node. + + arguments: + * flags -- optional + """ + super ( UseFlagListNode, self ) . __init__ ( 'use', flags=flags ) + self.priority = 850 + # --- end of __init__ (...) --- + + def active ( self ): + """The UseFlag list is only active if it is enabled and at least + one UseFlag child node is active. + """ + # generator should stop after first True + # todo/fixme: could use super ( UseFlagListNode, self ).active() instead + # of self._enabled + return True in ( node.active() for node in self.nodes ) and self._enabled + # --- end of active (...) --- + + def _sort_nodes ( self ): + """UseFlags are sorted by lowercase flag name, not priority.""" + self.nodes.sort ( key=lambda node : node.flags ['name'].lower() ) + # --- end of _sort_nodes (...) --- + + def add ( self, node ): + """Adds a child node only if it is a UseFlagNode. + + arguments: + * node -- + """ + if isinstance ( node, UseFlagNode ): + super ( UseFlagListNode, self ) . add ( node ) + else: + raise Exception ( "UseFlagListNode accepts UseFlagNodes only." ) + # --- end of add (...) --- + + +class NopNode ( MetadataNode ): + """This node is meant for testing only.""" + def __init__ ( self ): + super ( NopNode, self ) . __init__ ( 'nop', flags=dict() ) + # --- end of __init__ (...) --- diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py new file mode 100644 index 0000000..081f0b8 --- /dev/null +++ b/roverlay/overlay/__init__.py @@ -0,0 +1,306 @@ +# R Overlay -- +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import threading +import logging +import shutil +import os + +from roverlay import config, util + +from roverlay.overlay.category import Category + +DEFAULT_USE_DESC = '\n'.join ( [ + 'byte-compile - enable byte compiling', + 'R_suggests - install recommended packages' +] ) + +class Overlay ( object ): + + def __init__ ( + self, + name=None, + logger=None, + directory=None, + default_category=None, + eclass_files=None + ): + + # not setting any default values here (currently) + + if name is None: + self.name = config.get_or_fail ( 'OVERLAY.name' ) + else: + self.name = name + + if logger is None: + #self.logger = logging.getLogger ( self.name ) + self.logger = logging.getLogger ( 'overlay' ) + else: + #self.logger = logger.getChild ( self.name ) + self.logger = logger.getChild ( 'overlay' ) + + if directory is None: + self.physical_location = config.get_or_fail ( 'OVERLAY.dir' ) + else: + self.physical_location = directory + + if default_category is None: + self.default_category = config.get_or_fail ( 'OVERLAY.category' ) + else: + self.default_category = default_category + + self.eclass_files = eclass_files + + # + self._profiles_dir = os.path.join ( self.physical_location, 'profiles' ) + + self._catlock = threading.Lock() + self._categories = dict() + # --- end of __init__ (...) --- + + def _get_category ( self, category ): + """Returns a reference to the given category. Creates it if necessary. + + arguments: + * category -- category identifier as string + """ + if not category in self._categories: + self._catlock.acquire() + if not category in self._categories: + self._categories [category] = Category ( + category, + self.logger, + None if self.physical_location is None else \ + os.path.join ( self.physical_location, category ) + ) + self._catlock.release() + + return self._categories [category] + # --- end of _get_category (...) --- + + def add ( self, package_info, category=None ): + """Adds a package to this overlay. + + arguments: + * package_info -- PackageInfo of the package to add + * category -- category where the pkg should be put in, defaults to + self.default_category + + returns: None (implicit) + """ + self._get_category ( + self.default_category if category is None else category + ) . add ( package_info ) + # --- end of add (...) --- + + def show ( self, **show_kw ): + """Presents the ebuilds/metadata stored in this overlay. + + arguments: + * **show_kw -- keywords for package.PackageDir.show(...) + + returns: None (implicit) + """ + for cat in self._categories.values(): cat.show ( **show_kw ) + # --- end of show (...) --- + + def write ( self, **write_kw ): + """Writes the overlay to its physical location (filesystem), including + metadata and Manifest files. + TODO include Manifest generation in package.py + + arguments: + * **write_kw -- keywords for package.PackageDir.write(...) + + returns: None (implicit) + + raises: !! TODO + + TODO/FIXME/DOC: This is not thread-safe, it's expected to be called + when ebuild creation is done. + """ + # writing profiles/ here, rewriting categories/ later + self._init_overlay ( reimport_eclass=True, make_profiles_dir=True ) + + for cat in self._categories.values(): + if cat.physical_location and not cat.empty(): + util.dodir ( cat.physical_location ) + cat.write() + + self._write_categories ( only_active=True ) + # --- end of write (...) --- + + def write_incremental ( self, **write_kw ): + """Writes all ebuilds that have been added since the last + write_incremental call. + TODO: + * This could be useful to save some mem by removing already written + package infos. + * This has to be thread safe + """ + raise Exception ( "method stub" ) + # --- end of write_incremental (...) --- + + def generate_metadata ( self, **metadata_kw ): + """Tells the overlay's categories to create metadata. + You don't have to call this before write()/show() unless you want to use + special metadata options. + + arguments: + * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...) + + returns: None (implicit) + """ + for cat in self._categories.values(): + cat.generate_metadata ( **metadata_kw ) + # --- end of generate_metadata (...) --- + + def generate_manifest ( self, **manifest_kw ): + """Generates Manifest files for all ebuilds in this overlay that exist + physically/in filesystem. + Manifest files are automatically created when calling write(). + + arguments: + * **manifest_kw -- see PackageDir.generate_manifest(...) + + returns: None (implicit) + """ + for cat in self._categories.values(): + cat.generate_manifest ( **manifest_kw ) + # --- end of generate_manifest (...) --- + + def _write_profiles_dir ( self, only_active_categories=True ): + """Creates and updates the profiles/ dir. + + arguments: + * only_active_categories -- if True: do not list categories without + ebuilds in profiles/categories + """ + # profiles/ + util.dodir ( self._profiles_dir ) + self._write_repo_name() + self._write_categories ( only_active=only_active_categories ) + self._write_usedesc() + # --- end of _write_profiles_dir (...) --- + + def _write_profiles_file ( self, filename, to_write ): + """Writes a file in profiles/. + + arguments: + * filename -- name of the file to write (including file extension) + * to_write -- string to write (don't forget newline at the end) + """ + fh = None + try: + fh = open ( os.path.join ( self._profiles_dir, filename ), 'w' ) + if to_write: + # else touch file + fh.write ( to_write ) + except IOError as e: + self.logger.exception ( e ) + raise + finally: + if fh: fh.close() + # --- end of _write_profiles_file (...) --- + + def _write_repo_name ( self ): + """Writes profiles/repo_name.""" + self._write_profiles_file ( 'repo_name', self.name + '\n' ) + # --- end of _write_repo_name (...) --- + + def _write_categories ( self, only_active=True ): + """Writes profiles/categories. + + arguments: + * only_active -- exclude categories without ebuilds + """ + cats = None + if only_active: + cats = [ + name for name, category + in self._categories.items() if not category.empty() + ] + else: + cats = list ( self._categories.keys() ) + + if cats: + self._write_profiles_file ( + 'categories', + '\n'.join ( cats ) + '\n' + ) + # --- end of _write_categories (...) --- + + def _write_usedesc ( self ): + """Writes profiles/use.desc.""" + # TODO: config entry + use_desc = config.get ( + 'OVERLAY.use_desc', + fallback_value=DEFAULT_USE_DESC + ) + if use_desc: + self._write_profiles_file ( 'use.desc', use_desc + '\n' ) + # --- end of _write_usedesc (...) --- + + def _init_overlay ( self, reimport_eclass=False, make_profiles_dir=False ): + """Initializes the overlay at its physical/filesystem location. + + arguments: + * reimport_eclass -- whether to copy existing eclass files + again (True) or not + * make_profiles_dir -- if True: create the profiles/ dir now + + raises: + * Exception if no physical location assigned + * passes IOError,... + """ + if self.physical_location is None: + raise Exception ( "no directory assigned." ) + + try: + root = self.physical_location + # mkdir overlay root + os.makedirs ( root, exist_ok=True ) # raises? + + if self.eclass_files: + # import eclass files + eclass_dir = os.path.join ( root, 'eclass' ) + try: + util.dodir ( eclass_dir ) + + for eclass in self.eclass_files: + src = eclass + dest = None + if isinstance ( eclass, str ): + dest = os.path.basename ( eclass ) + else: + # list-like specification ( src, destname ) + src = eclass [0] + dest = eclass [1] + + if reimport_eclass or not os.path.isfile ( dest ): + shutil.copyfile ( src, dest ) + + + except Exception as e: + #self.logger.exception ( e ) TODO try-catch blocks + self.logger.critical ( "Cannot import eclass files!" ) + raise + + # -- eclass + if make_profiles_dir: + self._write_profiles_dir ( only_active_categories=False ) + + except IOError as e: + + self.logger.exception ( e ) + self.logger.critical ( "^failed to init overlay" ) + raise + + + + + + + diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py new file mode 100644 index 0000000..768f296 --- /dev/null +++ b/roverlay/overlay/category.py @@ -0,0 +1,105 @@ +# R Overlay -- +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import threading +import os.path + +from roverlay.overlay.package import PackageDir + +import roverlay.util + +class Category ( object ): + + def __init__ ( self, name, logger, directory ): + """Initializes a overlay/portage category (such as 'app-text', 'sci-R'). + + arguments: + * name -- name of the category + * logger -- parent logger + * directory -- filesystem location + """ + self.logger = logger.getChild ( name ) + self.name = name + self._lock = threading.RLock() + self._subdirs = dict() + self.physical_location = directory + # --- end of __init__ (...) --- + + def empty ( self ): + """Returns True if this category contains 0 ebuilds.""" + return \ + len ( self._subdirs ) == 0 or \ + not False in ( d.empty() for d in self._subdirs ) + # --- end of empty (...) --- + + def add ( self, package_info ): + """Adds a package to this category. + + arguments: + * package_info -- + + returns: None (implicit) + """ + # TODO make keys available + pkg_name = package_info ['name'] + + if not pkg_name in self._content: + self._lock.acquire() + if not pkg_name in self._content: + self._content [pkg_name] = PackageDir ( + pkg_name, + self.logger, + None if self.physical_location is None else \ + os.path.join ( self.physical_location, pkg_name ) + ) + self._lock.release() + + self._content [pkg_name].add ( package_info ) + # --- end of add (...) --- + + def generate_metadata ( self, **metadata_kw ): + """Generates metadata for all packages in this category. + Metadata are automatically generated when calling write(). + + arguments: + * **metadata_kw -- see PackageDir.generate_metadata(...) + + returns: None (implicit) + """ + for package in self._subdirs.values(): + package.generate_metadata ( **metadata_kw ) + # --- end of generate_metadata (...) --- + + def generate_manifest ( self, **manifest_kw ): + """Generates Manifest files for all packages in this category. + Manifest files are automatically created when calling write(). + + arguments: + * **manifest_kw -- see PackageDir.generate_manifest(...) + + returns: None (implicit) + """ + for package in self._subdirs.values(): + package.generate_manifest ( **manifest_kw ) + # --- end of generate_manifest (...) --- + + def show ( self, **show_kw ): + """Prints this category (its ebuild and metadata files). + + returns: None (implicit) + """ + for package in self._subdirs.values(): + package.show ( **show_kw ) + # --- end of show (...) --- + + def write ( self, **write_kw ): + """Writes this category to its filesystem location. + + returns: None (implicit) + """ + for package in self._subdirs.values(): + if package.physical_location and not package.empty(): + roverlay.util.dodir ( package.physical_location ) + package.write ( **write_kw ) + # --- end of write (...) --- diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py new file mode 100644 index 0000000..d43ec1f --- /dev/null +++ b/roverlay/overlay/package.py @@ -0,0 +1,290 @@ +# R Overlay -- +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import threading +import os.path +import sys + +from roverlay.metadata import MetadataJob + +SUPPRESS_EXCEPTIONS = True + +class PackageDir ( object ): + + def __init__ ( self, name, logger, directory ): + """Initializes a PackageDir which contains ebuilds, metadata and + a Manifest file. + + arguments: + * name -- name of the directory (${PN} in ebuilds) + * logger -- parent logger + * directory -- filesystem location of this PackageDir + """ + self.logger = logger.getChild ( name ) + self._lock = threading.RLock() + self._packages = dict() + self._metadata = None + self.physical_location = directory + # --- end of __init__ (...) --- + + def empty ( self ): + """Returns True if no ebuilds stored, else False.""" + return len ( self._packages ) == 0 + # --- end of empty (...) --- + + def _get_metadata_filepath ( self ): + """Returns the path to the metadata file.""" + return os.path.join ( + '??' if self.physical_location is None else self.physical_location, + self._metadata.filename + ) + # --- end of _get_metadata_filepath (...) --- + + def _get_ebuild_filepath ( self, pvr ): + """Returns the path to the ebuild file. + + arguments: + * pvr -- version number with the revision (${PVR} in ebuilds) + """ + filename = "%s-%s.ebuild" % ( self.name, pvr ) + return os.path.join ( + '??' if self.physical_location is None else self.physical_location, + filename + ) + # --- end of _get_ebuild_filepath (...) --- + + def write ( self ): + """Writes this directory to its (existent!) filesystem location. + + returns: None (implicit) + + raises: !! TODO + """ + if self.physical_location is None: + raise Exception ( "cannot write - no directory assigned!" ) + + self._lock.acquire() + self._regen_metadata() + + # mkdir not required here, overlay.Category does this + + # write ebuilds + for ver, p_info in self._packages.items(): + fh = None + try: + efile = self._get_ebuild_filepath ( ver ) + ebuild = p_info.ebuild + + fh = open ( efile, 'w' ) + ebuild.write ( fh ) + if fh: fh.close() + + # adjust owner/perm? TODO + # chmod 0644 or 0444 + # chown 250.250 + + # this marks the package as 'written to fs' + # TODO update PackageInfo + p_info ['ebuild_filepath'] = efile + + self.logger.info ( "Wrote ebuild %s." % efile ) + except IOError as e: + if fh: fh.close() + self.logger.error ( "Couldn't write ebuild %s." % efile ) + self.logger.exception ( e ) + + # write metadata + fh = None + try: + mfile = self._get_metadata_filepath() + + fh = open ( mfile, 'w' ) + self._metadata.write ( fh ) + if fh: fh.close() + + except IOError as e: + if fh: fh.close() + self.logger.error ( "Failed to write metadata at %s." % mfile ) + self.logger.exception ( e ) + + self.generate_manifest() + + self._lock.release() + # --- end of write (...) --- + + def show ( self, stream=sys.stderr ): + """Prints this dir (the ebuilds and the metadata) into a stream. + + arguments: + * stream -- stream to use, defaults to sys.stderr + + returns: None (implicit) + + raises: !! TODO + """ + self._lock.acquire() + self._regen_metadata() + + + for ver, p_info in self._packages.items(): + efile = self._get_ebuild_filepath ( ver ) + ebuild = p_info.ebuild + + stream.write ( "[BEGIN ebuild %s]\n" % efile ) + ebuild.write ( stream ) + stream.write ( "[END ebuild %s]\n" % efile ) + + mfile = self._get_metadata_filepath() + + stream.write ( "[BEGIN %s]\n" % mfile ) + self._metadata.write ( stream ) + stream.write ( "[END %s]\n" % mfile ) + + + self._lock.release() + # --- end of show (...) --- + + def _latest_package ( self, pkg_filter=None, use_lock=False ): + """Returns the package info with the highest version number. + + arguments: + * pkg_filter -- either None or a callable, + None: do not filter packages + else: ignore package if it does not pass the filter + * use_lock -- if True: hold lock while searching + """ + first = True + retver = None + retpkg = None + + if use_lock: self._lock.acquire() + for p in self._packages.values(): + if pkg_filter is None or pkg_filter ( p ): + newver = p ['version'] + if first or newver > retver: + retver = newver + retpkg = p + first = False + + if use_lock: self._lock.release() + return retpkg + # --- end of _latest_package (...) --- + + def add ( self, package_info ): + """Adds a package to this PackageDir. + + arguments: + * package_info -- + + returns: success as bool + + raises: Exception when ebuild already exists. + """ + # !! p info key TODO + shortver = package_info ['ebuild_verstr'] + + def already_exists ( release=False ): + if filename in self._packages: + + if release: self._lock.release() + + msg = "'%s-%s.ebuild' already exists, cannot add it!" % ( + self.name, shortver + ) + if SUPPRESS_EXCEPTIONS: + logger.warning ( msg ) + else: + raise Exception ( msg ) + + return True + else: + return False + # --- end of already_exists (...) --- + + if already_exists ( release=False ): return False + self._lock.acquire() + if already_exists ( release=True ): return False + + self._packages [shortver] = package_info + + self._lock.release() + return True + # --- end of add (...) --- + + def _regen_metadata ( self ): + """Regenerates the metadata.""" + self.generate_metadata ( + skip_if_existent=True, + use_all_packages=False, + use_old_metadata=False + ) + # --- end of _regen_metadata (...) --- + + def generate_metadata ( + self, + skip_if_existent=False, use_all_packages=False, use_old_metadata=False + ): + """Creates metadata for this package. + + arguments: + * skip_if_existent -- do not create if metadata already exist + * use_all_packages -- TODO + * use_old_metadata -- TODO + """ + if use_old_metadata or use_all_packages: + raise Exception ( "using >1 package for metadata.xml is TODO!" ) + + if skip_if_existent and not self._metadata is None: + return + + self._lock.acquire() + + if self._metadata is None or not use_old_metadata: + del self._metadata + self._metadata = MetadataJob ( self.logger ) + + if use_all_packages: + for p_info in self._packages: + self._metadata.update ( p_info ) + else: + self._metadata.update ( _latest_package() ) + + self._lock.release() + # --- end of generate_metadata (...) --- + + def generate_manifest ( self ): + """Generates the Manifest file for this package. + + expects: called in self.write(), after writing metadata/ebuilds + + returns: None (implicit) + + raises: !! TODO + * Exception if not physical + """ + if self.physical_location is None: + raise Exception ( "no directory assigned." ) + + # it should be sufficient to call create_manifest for one ebuild, + # choosing the latest one here that exists in self.physical_location. + # + # metadata.xml's full path cannot be used for manifest creation here + # 'cause DISTDIR would be unknown + # + pkg_info_for_manifest = _latest_package ( + pkg_filter=lambda pkg : not pkg ['ebuild_filepath'] is None, + use_lock=True + ) + + if pkg_info_for_manifest is None: + # ? FIXME + raise Exception ( + "No ebuild written so far! I really don't know what do to!" + ) + else: + # TODO: manifest creation interface is single threaded, + # may want to 'fix' this later + manifest.create_manifest ( pkg_info_for_manifest, nofail=False ) + + # --- end of generate_manifest (...) --- diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py new file mode 100644 index 0000000..7a77ca0 --- /dev/null +++ b/roverlay/packageinfo.py @@ -0,0 +1,254 @@ +# R Overlay -- package info class +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import re +import os.path +import logging +import threading + +from roverlay import config, util + +LOGGER = logging.getLogger ( 'PackageInfo' ) + +VIRTUAL_KEYS = dict ( + DISTDIR = frozenset ( ( 'distdir', 'pkg_distdir' ) ), + # removing this key + #EBUILD_FILE = frozenset ( ( 'ebuild_file', 'efile' ) ), + HAS_SUGGESTS = frozenset ( ( 'has_suggests', 'has_rsuggests' ) ), + SRC_URI = frozenset ( ( 'src_uri', 'package_url', 'url' ) ), + ALWAYS_FALLBACK = frozenset ( ( 'ebuild_filepath' ) ), +) + + +class PackageInfo ( object ): + """PackageInfo offers easy, subscriptable (['sth']) access to package + information, whether stored or calculated. + """ + + def __init__ ( self, **initial_info ): + """Initializes a PackageInfo. + + arguments: + * **initial_info -- passed to update ( **kw ) + """ + self._info = dict() + self.readonly = False + self._update_lock = threading.RLock() + + self.update ( **initial_info ) + # --- end of __init__ (...) --- + + def set_readonly ( self, immediate=False, final=False ): + """Makes the package info readonly. + + arguments: + * immediate -- do not acquire lock, set readonly directly, + defaults to False + * final -- if set and True: make this decision final + """ + self._set_mode ( True, immediate, final ) + # --- end of set_readonly (...) --- + + def set_writeable ( self, immediate=False ): + """Makes the package info writeable. + + arguments: + * immediate -- do not acquire lock, set writeable directly, + defaults to False + """ + self._set_mode ( False, immediate ) + # --- end of set_writeable (...) --- + + def _set_mode ( self, readonly_val, immediate=False, final=False ): + """Sets readonly to True/False. + + arguments: + * readonly_val -- new value for readonly + * immediate -- do not acquire lock + * final -- only if readonly_val is True: make this decision final, + + raises: Exception if self.readonly is a constant (_readonly_final is set) + """ + if hasattr ( self, '_readonly_final' ): + raise Exception ( "cannot modify readonly - it's a constant." ) + elif immediate: + self.readonly = readonly_val + if final and readonly_val: + self._readonly_final = True + elif not self.readonly is readonly_val: + self._update_lock.acquire() + self.readonly = readonly_val + if final and readonly_val: + self._readonly_final = True + self._update_lock.release() + # --- end of _set_mode (...) --- + + def _writelock_acquire ( self ): + """Acquires the lock required for adding new information. + + raises: Exception if readonly (writing not allowed) + """ + if self.readonly or hasattr ( self, '_readonly_final' ): + raise Exception ( "package info is readonly!" ) + + self._update_lock.acquire() + + if self.readonly or hasattr ( self, '_readonly_final' ): + self._update_lock.release() + raise Exception ( "package info is readonly!" ) + + return True + # --- end of _writelock_acquire (...) --- + + def get ( self, key, fallback_value=None, do_fallback=False ): + """Returns the value specified by key. + The value is either calculated or taken from dict self._info. + + arguments: + * key -- + * fallback_value -- fallback value if key not found / unknown + * do_fallback -- if True: return fallback_value, else raise KeyError + + raises: KeyError + """ + key_low = key.lower() + + # 'virtual' keys - calculate result + if key_low in VIRTUAL_KEYS ['DISTDIR']: + if 'package_dir' in self._info: + return self._info ['package_dir'] + + elif 'origin' in self._info: + return util.get_distdir ( self._info ['origin'] ) + + elif key_low in VIRTUAL_KEYS ['HAS_SUGGESTS']: + if 'has_suggests' in self._info: + return self._info ['has_suggests'] + + else: + return False + + elif key_low in VIRTUAL_KEYS ['SRC_URI']: + # comment from ebuildjob: + ## origin is todo (sync module knows the package origin) + ## could calculate SRC_URI in the eclass depending on origin + # comment from ebuild: + ## calculate SRC_URI using self._data ['origin'], + ## either here or in eclass + return "**packageinfo needs information from sync module!" + + # normal keys + if key in self._info: + return self._info [key] + + elif key_low in self._info: + return self._info [key_low] + + elif do_fallback or key_low in VIRTUAL_KEYS ['ALWAYS_FALLBACK']: + return fallback_value + else: + raise KeyError ( key ) + # --- end of get (...) --- + + def __getitem__ ( self, key ): + return self.get ( key, do_fallback=False ) + # --- end of __getitem__ (...) --- + + def __setitem__ ( self, key, value ): + """Sets an item. + + arguments: + * key -- + * value -- + + raises: Exception when readonly + """ + self._writelock_acquire() + self._info [key] = value + self._update_lock.release() + # --- end of __setitem__ (...) --- + + def update ( self, **info ): + """Uses **info to update the package info data. + + arguments: + * **info -- + + raises: Exception when readonly + """ + if len ( info ) == 0 : + # nothing to do + return + + self._writelock_acquire() + + if 'filepath' in info: + self._use_filepath ( info ['filepath'] ) + + if 'ebuild' in info: + self._use_ebuild ( info ['ebuild'] ) + + if 'desc_data' in info + self ['desc_data'] = info ['desc_data'] + elif 'desc' in info: + self ['desc_data'] = info ['desc'] + + + if 'suggests' in info: + self ['has_suggests'] = info ['suggests'] + + + self._update_lock.release() + # --- end of update (**kw) --- + + def _use_filepath ( self, filepath ): + """auxiliary method for update(**kw) + + arguments: + * filepath -- + """ + + package_file = os.path.basename ( filepath ) + + # remove .tar.gz .tar.bz2 etc. + filename = re.sub ( + config.get ( 'R_PACKAGE.suffix_regex' ) + '$', + '', + package_file + ) + + package_name, sepa, package_version = filename.partition ( + config.get ( 'R_PACKAGE.name_ver_separator', '_' ) + ) + + if not sepa: + # file name unexpected, tarball extraction will (probably) fail + LOGGER.error ( "unexpected file name '%s'." % filename ) + raise Exception ( "cannot use file '%s'." % filename ) + return + + + self ['filepath'] = filepath + self ['package_file'] = package_file + self ['package_dir' ] = os.path.dirname ( filepath ) + self ['filename'] = filename + self ['package_name'] = package_name + self ['package_version'] = package_version + # --- end of _use_filepath (...) --- + + def _use_ebuild ( self, ebuild ): + """auxiliary method for update(**kw) + + arguments: + * ebuild -- + """ + self ['ebuild'] = ebuild + # set status to ready for overlay + + # this does no longer work FIXME + self ['has_suggests'] = ebuild.has_rsuggests + # todo move Ebuild funcs to here + self ['ebuild_dir'] = ebuild.suggest_dir_name() + self ['ebuild_name'] = ebuild.suggest_name() + # --- end of _use_ebuild (...) --- diff --git a/roverlay/portage/__init__.py b/roverlay/portage/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/roverlay/portage/ebuild.py b/roverlay/portage/ebuild.py deleted file mode 100644 index a435f5c..0000000 --- a/roverlay/portage/ebuild.py +++ /dev/null @@ -1,456 +0,0 @@ -# R Overlay -- ebuild creation, ebuild class -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import copy - -import roverlay.config - -from roverlay.util import shorten_str - - -class Ebuild ( object ): - EBUILD_INDENT = roverlay.config.get ( 'EBUILD.indent', '\t' ) - - ADD_REMAP = { - # pkg vs package - 'package_name' : 'pkg_name', - 'package_version' : 'pkg_version', - 'package_revision' : 'pkg_revision', - # TITLE is in DESCRIPTION - 'TITLE' : 'DESCRIPTION', - - # TODO: remove these entries by fixing ebuildcreator/ebuildjob - 'DEPENDS' : 'DEPEND', - 'RDEPENDS' : 'RDEPEND', - 'RSUGGESTS' : 'R_SUGGESTS', - } - - def __init__ ( self, logger ): - """Initializes an Ebuild. - This is an abstraction layer between the verified + calculated data - and the ebuild data, which can be written into a file / stdout / stdin. - Most functions here assume that everything is fine when it reaches them. - - arguments: - * logger -- logger for this Ebuild - """ - - self.logger = logger - - # elements in ebuild_data are either a str or a list of str - self._data = dict () - self._ebuild_lines = None - self._ebuild_name = None - self.has_rsuggests = False - - # --- end of __init__ (...) --- - - def cleanup ( self ): - """Removes stored data if ebuild_lines have already been calculated. - This saves some memory but makes this Ebuild read-only. - """ - if self._ebuild_lines: - # determine the ebuild name first - self._ebuild_name = self.suggest_name() - del self._data - self._data = None - - # --- end of cleanup (...) --- - - def prepare ( self, force_update=False, cleanup_after=False ): - """Tells this Ebuild to create ebuild lines. - - arguments: - * force_update -- create ebuild lines if they exist; defaults to False - and ignored if this Ebuild has been cleaned up - * cleanup_after -- run cleanup() after successful creation - - Returns True if ebuild_lines have been created or if they exist and - an update has not been enforced. Else returns False. - """ - if self._ebuild_lines and not force_update: - return True - elif self._data: - self._ebuild_lines = self._make_ebuild_lines() - if self._ebuild_lines: - if cleanup_after: self.cleanup() - return True - elif self._ebuild_lines: - # self._data is None - return True - - return False - - # --- end of prepare (...) --- - - def has_ebuild ( self ): - """Returns True if this object has ebuild text lines else False.""" - return bool ( self._ebuild_lines ) - # --- end of has_ebuild (...) --- - - def add ( self, key, value, append=True ): - """Adds data to this Ebuild. - - arguments: - * key -- identifier of the data (e.g. DEPEND). - May be remapped (e.g. merging 'Title' and 'Description') - or even refused here - * value -- - * append -- whether to append values or overwrite existing ones, - defaults to True. - - raises: Exception when ebuild data are readonly - """ - if self._data is None: - # -- todo - raise Exception ("Ebuild is readonly.") - - _key = Ebuild.ADD_REMAP [key] if key in Ebuild.ADD_REMAP else key - - if _key is None: - self.logger.debug ( "add (%s, %s): filtered key.", key, value ) - else: - if append and _key in self._data: - if not isinstance ( self._data [_key], list ): - self._data [_key] = [ self._data [_key] ] - - if isinstance ( value, list ): - self._data [_key].extend ( value ) - else: - self._data [_key].append ( value ) - - else: - self._data [_key] = value - - # --- end of add (...) --- - - def write ( self, file_to_write ): - """Writes an ebuild file. - - arguments: - * file_to_write -- path to the file that should be written - """ - # prepare ebuild lines and open file handle after that - if self.prepare ( False, False ): - try: - fh = open ( file_to_write, 'w' ) - self.show ( fh ) - fh.close() - del fh - except IOError as err: - self.logger.exception ( err ) - raise - - else: - self.logger.warning ( - 'Cannot write ebuild - it\'s empty! ' - '(check with has_ebuild() before calling this method.)' - ) - - # --- end of write (...) --- - - def show ( self, file_handle ): - """Prints the ebuild content into a file_handle. - - arguments: - file_handle -- object that has a writelines ( list ) method, e.g. file. - - Returns True if writing was successful, else False. - """ - if self.prepare ( False, False ): - lines = [ line + "\n" for line in self._ebuild_lines ] - file_handle.writelines ( lines ) - del lines - return True - else: - return False - - # --- end of show (...) --- - - def suggest_dir_name ( self ): - """Suggests a direcory name for this Ebuild.""" - if self._data is None: - return self._ebuild_name.partition ( '-' ) [0] - elif 'pkg_name' in self._data: - return self._data ['pkg_name'] - else: - return self.suggest_name().partition ( '-' ) [0] - # --- end of suggest_dir_name (...) --- - - def suggest_name ( self, fallback_name='' ): - """Suggests a file name for the ebuild. This is calculated using - pkg_name/version/revision. Returns a fallback_name if this is not - possible. - - arguments: - fallback_name -- name to return if no suggestion available, - defaults to empty string - """ - - if self._ebuild_name: - return self._ebuild_name - elif not self._data is None and 'pkg_name' in self._data: - name_components = [ self._data ['pkg_name'] ] - - if 'pkg_version' in self._data: - name_components.append ( self._data ['pkg_version'] ) - else: - # default ver - name_components.append ( '1.0' ) - - if 'pkg_revision' in self._data: - rev = self._data ['pkg_revision'] - - # omit rev == 0 and invalid revisions - if isinstance ( rev, int ) and rev > 0: - name_components.append ( 'r%i' % rev ) - - return '-'.join ( name_components ) - - else: - return fallback_name - - # --- end of suggest_name (...) --- - - def _make_ebuild_lines ( self ): - """Creates text lines for this Ebuild. - It assumes that enough data to do this are available. - Exceptions (KeyError, NameError, ...) are passed if that's not the case. - """ - - def get_dep_and_use(): - """Creates values for the DEPEND, RDEPEND, IUSE and, if possible, - R_SUGGESTS variables and returns them as dict { VARNAME -> VALUE }. - """ - - # have suggests if they're set and not empty - self.has_rsuggests = bool ( - 'R_SUGGESTS' in self._data and self._data ['R_SUGGESTS'] - ) - - # set defaults: inherit eclass + include depend in rdepend - # TODO: is ${DEPEND:-},... necessary? - ret = dict ( - DEPEND = [ '${DEPEND:-}' ], - # assuming that the eclass includes it's DEPEND in RDEPEND - RDEPEND = [ '${RDEPEND:-}' ], - IUSE = [ '${IUSE:-}' ], - ) - - for kw in ( x for x in ( 'DEPEND', 'RDEPEND' ) if x in self._data ): - if isinstance ( self._data [kw], list ): - ret [kw].extend ( self._data [kw] ) - else: - ret [kw].append ( self._data [kw] ) - - - if self.has_rsuggests: - ret ['R_SUGGESTS'] = self._data ['R_SUGGESTS'] - - # +R_SUGGESTS, -R_SUGGESTS? - ret ['IUSE'].append ( 'R_suggests' ) - # do these braces help or confuse? TODO FIXME - ret ['RDEPEND'].append ( '( R_suggests ? ${R_SUGGESTS} )' ) - - return ret - - # --- end of get_dep_and_use () --- - - def make_var ( - varname, - value=None, oneline_list=True, indent_list=True, indent_level=0 - ): - """Creates a = statement for ebuilds. - - arguments: - * varname -- name of the variable - * value -- value of the variable. - This has to be either None (the default), str, - or list of str. - * oneline_list -- if value is a list: controls whether its components - should be put into one line (True) or multiple. - Defaults to True. - * indent_list -- if value is a list and not oneline_list: - controls whether each value line should be - indentend (by indent_level + 1) or not ("by 0"). - Defaults to True. - * indent_level -- current indentation level, defaults to 0 - - """ - - # assumption: value is either None, - # scalar with str representation or list of str - var_value = None - - if not value: - var_value = "" - - elif isinstance ( value, list ): - if oneline_list: - var_value = ' '.join ( value ) - elif indent_list: - var_value = ( - '\n' + (indent_level + 1) * Ebuild.EBUILD_INDENT - ).join ( value ) - else: - '\n'.join ( value ) - - else: - var_value = str ( value ) - - - # (TODO) - # fixing ebuild var values here - - # cut DESCRIPTION line if too long - if varname == 'DESCRIPTION': - var_value = shorten_str ( var_value, 45, '... (see metadata)' ) - - - ret ='%s%s="%s"' % ( - indent_level * Ebuild.EBUILD_INDENT, - varname, - var_value - ) - - # (TODO) - # fixing ebuild var lines here - - return ret - - # --- end of make_var (...) --- - - def remove_newlines ( line_list ): - """Removes leading, ending and repeated blank lines in line_list. - - arguments: - * line_list -- - - returns: filtered lines - - TODO: check if a filter function could be used for this - """ - lines = [] - line = None - last_line_empty = False - - for line in line_list: - line = line.rstrip() - # re.sub \n{2,} \n :: FIXME? - - if line: - last_line_empty = False - elif not last_line_empty: - last_line_empty = True - else: - continue - - lines.append ( line ) - - # remove last line if empty - ##if last_line_empty: (?) - if len ( lines ) and not lines [-1]: - del lines [-1] - - return lines - - # --- end of remove_newlines (...) --- - - def add_easyvar ( - ebuild_content, varname, - value_key=None, add_newline=False - ): - """Adds a 'simple' variable to the ebuild lines. - This means that it can directly be taken from self._data [value_key]. - This method assumes that value_key exists in self._data, - any exceptions (KeyError) will be passed. - - arguments: - * ebuild_content -- list of ebuild text lines, will be modified - directly, so copy it before calling addvar if - you need the original list. - * varname -- name of the variable. - Nothing happens if this is None. - * value_key -- key of the value, - defaults to varname if it is None - * add_newline -- adds a newline after the var statement, - defaults to False - - returns: given+modified list (ebuild_content) - """ - - if not varname is None: - if value_key is None: - ebuild_content.append ( - make_var ( varname, self._data [varname] ) - ) - else: - ebuild_content.append ( - make_var ( varname, self._data [value_key] ) - ) - - if add_newline: - ebuild_content.append ( "" ) - - return ebuild_content - - # --- end of add_easyvar (...) --- - - # -- actual start of _make_ebuild_lines (...) -- - try: - ebuild_lines = [] - - if 'ebuild_header' in self._data: - ebuild_lines = copy.copy ( self._data ['ebuild_header'] ) - ebuild_lines.append ( "" ) - - add_easyvar ( ebuild_lines, "PKG_FILE" ) - if 'PKG_ORIGIN' in self._data: - add_easyvar ( ebuild_lines, "PKG_ORIGIN", None, False ) - - ebuild_lines.append ( "" ) - - # TODO/FIXME: this makes DESCRIPTION mandatory, maybe check with - # >if 'DESCRIPTION' in self._data< - add_easyvar ( ebuild_lines, "DESCRIPTION" ) - - add_easyvar ( ebuild_lines, "SRC_URI", add_newline=True ) - - # FIXME/TODO: LICENSE? - - dep_and_use = get_dep_and_use () - - # check that IUSE has more than one element, - # don't write IUSE="${IUSE:-}" etc. - if len ( dep_and_use ['IUSE'] ) > 1: - ebuild_lines.append ( - make_var ( "IUSE", dep_and_use ['IUSE'], True ) - ) - - if 'R_SUGGESTS' in dep_and_use: - ebuild_lines.append ( - make_var ( "R_SUGGESTS", dep_and_use ['R_SUGGESTS'], False ) - ) - - # see IUSE - if len ( dep_and_use ['DEPEND'] ) > 1: - ebuild_lines.append ( - make_var ( "DEPEND", dep_and_use ['DEPEND'], False ) - ) - - # see IUSE - if len ( dep_and_use ['RDEPEND'] ) > 1: - ebuild_lines.append ( - make_var ( "RDEPEND", dep_and_use ['RDEPEND'], False ) - ) - - del dep_and_use - return remove_newlines ( ebuild_lines ) - - except ( ValueError, KeyError, NameError ) as err: - self.logger.exception ( err ) - self.logger.error ( "Cannot create ebuild text lines." ) - return None - - # --- end of make_ebuild_lines (...) --- diff --git a/roverlay/portage/ebuildcreator.py b/roverlay/portage/ebuildcreator.py deleted file mode 100644 index aee45e1..0000000 --- a/roverlay/portage/ebuildcreator.py +++ /dev/null @@ -1,184 +0,0 @@ -# R Overlay -- ebuild creation, "master" module -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import time -import logging -import threading - -try: - import queue -except ImportError: - # python2 - import Queue as queue - - -from roverlay import config -from roverlay.depres import depresolver -from roverlay.depres.channels import EbuildJobChannel -from roverlay.portage.ebuildjob import EbuildJob - -class EbuildCreator ( object ): - - NUMTHREADS = config.get ( 'EBUILD.jobcount', 0 ) - - def __init__ ( self ): - """Initializes an EbuildCreator. This is an Object that controls the - R package -> ebuild creation. It continuously creates EbuildJobs for - every R package added. - """ - self.ebuild_headers = dict () - self.depresolve_main = depresolver.DependencyResolver () - - self.ebuild_jobs = queue.Queue() - self.ebuild_jobs_done = list() - - self.runlock = threading.Lock() - self._threads = None - - self.logger = logging.getLogger ( 'EbuildCreator' ) - - # this topic to change FIXME/TODO - # metadata [] = MetadataJob instance - self.metadata = dict() - - # --- end of init (...) --- - - def add_package ( self, package_file ): - """Adds an R package to the EbuildCreator, which means that an EbuildJob - will be created for it. Returns the EbuildJob, which is also stored - in the job queue. - - arguments: - * package_file -- path R package file - """ - new_job = EbuildJob ( package_file, self.get_resolver_channel ) - - self.ebuild_jobs.put ( new_job ) - - return new_job - - # --- end of add_package (...) --- - - def get_resolver_channel ( self, name=None, logger=None ): - """Returns a communication channel to the dependency resolver. - - arguments: - readonly -- whether the channel is listen-only (no write methods) or not - defaults to True - """ - return self.depresolve_main.register_channel ( - EbuildJobChannel ( name=name, logger=logger ) - ) - - # --- end of get_resolver_channel (...) --- - - def close ( self ): - self.depresolve_main.close() - # --- end of close (...) --- - - def _thread_run_ebuilds ( self ): - - while not self.ebuild_jobs.empty(): - try: - job = self.ebuild_jobs.get_nowait() - except queue.Empty: - # queue is empty, done - return - - job.run() - self.ebuild_jobs_done.append ( job ) - - # --- end of _thread_run_ebuilds (...) --- - - def start ( self ): - """Tells all EbuildJobs to run.""" - - if not self.runlock.acquire ( False ): - # already running - return True - - start = time.time() - - jobcount = EbuildCreator.NUMTHREADS - - if jobcount < 1: - ( self.logger.warning if jobcount < 0 else self.logger.debug ) ( - "Running in sequential mode." - ) - self._thread_run_ebuilds() - else: - self.logger.warning ( - "Running in concurrent mode with %i jobs." % jobcount - ) - self._threads = [ - threading.Thread ( target = self._thread_run_ebuilds ) - for n in range ( jobcount ) - ] - - for t in self._threads: t.start() - for t in self._threads: t.join() - - del self._threads - self._threads = None - - - stop = time.time() - self.logger.info ( 'done after %f seconds' % ( stop - start ) ) - - # make metadata, topic to change... FIXME/TODO - for ejob in self.ebuild_jobs_done: - if ejob.get_ebuild() is None: continue - - edir = ejob.package_info ['ebuild_dir'] - if not edir in self.metadata: - self.metadata [edir] = ejob.feed_metadata ( create=True ) - else: - ejob.feed_metadata ( metadata=self.metadata [edir] ) - - self.runlock.release() - - # --- end of start (...) --- - - def collect_ebuilds ( self ): - """Returns all ebuilds. (They may not be ready / TODO)""" - ebuilds = [ job.get_ebuild() for job in self.ebuild_jobs_done ] - return filter ( None, ebuilds ) - - # --- end of collect_ebuilds (...) --- - - def get_ebuild_header ( self, ebuild_header_file=None ): - """Reads and returns the content of an ebuild header file. - This is a normal file that can be included in ebuilds. - Every header file will only be read on first access, its content will - be stored in a dict that is shared among all EbuildCreator instances. - - arguments: - * ebuild_header_file -- path to the header file; defaults to none which - means that nothing will be read and an empty list - is returned. - """ - - if ebuild_header_file is None: - # nothing to read - return [] - - elif ebuild_header_file in self.ebuild_headers: - # previously read - return self.ebuild_headers [ebuild_header_file] - - else: - # read file - try: - fh = open ( ebuild_header_file, 'r' ) - lines = fh.readlines() - fh.close() - self.ebuild_headers [ebuild_header_file] = lines - del fh - return lines - - except IOError as err: - # todo - raise - - # --- end of get_ebuild_header (...) --- diff --git a/roverlay/portage/ebuildjob.py b/roverlay/portage/ebuildjob.py deleted file mode 100644 index 6ad2738..0000000 --- a/roverlay/portage/ebuildjob.py +++ /dev/null @@ -1,333 +0,0 @@ -# R Overlay -- ebuild creation, "job" module -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import logging -import re - -from roverlay import config, util -from roverlay.portage.ebuild import Ebuild -from roverlay.portage.packageinfo import PackageInfo -from roverlay.portage.metadata.creation import MetadataJob -from roverlay.rpackage.descriptionreader import DescriptionReader - - -class EbuildJob ( object ): - LOGGER = logging.getLogger ( 'EbuildJob' ) - - DEFAULT_EBUILD_HEADER = config.get ( 'EBUILD.default_header' ) - - # move this to const / config - DEPENDENCY_FIELDS = { - 'R_SUGGESTS' : [ 'Suggests' ], - 'DEPENDS' : [ 'Depends', 'Imports' ], - 'RDEPENDS' : [ 'LinkingTo', 'SystemRequirements' ] - } - - STATUS_LIST = [ 'INIT', 'BUSY', 'WAIT_RESOLVE', 'SUCCESS', 'FAIL' ] - - # status 'jump' control - # FAIL is always allowed, S -> S has to be explicitly allowed - STATUS_BRANCHMAP = dict ( - INIT = [ 'BUSY' ], - BUSY = [ 'BUSY', 'WAIT_RESOLVE', 'SUCCESS' ], - WAIT_RESOLVE = [ 'BUSY' ], - SUCCESS = [], - FAIL = [], - ) - - def __init__ ( self, package_file, depres_channel_spawner=None ): - """Initializes an EbuildJob, which creates an ebuild for an R package. - - arguments: - * package_info -- R package file info - * dep_resolver -- dependency resolver - """ - - """Note: - it is intended to run this job as thread, that's why it has its own - dep resolver 'communication channel', status codes etc. - """ - - self.package_info = PackageInfo ( filepath=package_file ) - self.package_info.set_readonly() - - try: - self.logger = EbuildJob.LOGGER.getChild ( - self.package_info ['filename'] - ) - except KeyError: - self.logger = EbuildJob.LOGGER.getChild ( '__undef__' ) - - self.description_reader = DescriptionReader ( - self.package_info, logger=self.logger - ) - - self.ebuild = None - - # only allow a function (at least callable) for self.get_resolver - if hasattr ( depres_channel_spawner, '__call__' ): - self.request_resolver = depres_channel_spawner - # _depres contains (almost) dependency resolution data/.., including - # communication channels and should only be modified in run() - self._depres = dict () - else: - self.request_resolver = None - - self.status = 'INIT' - - # --- end of __init__ (...) --- - - def feed_metadata ( self, create=False, metadata=None ): - """Feeds metadata, either a existing MetadataJob instance or a new one. - - arguments: - * create -- if True: create new metadata, "function mode", - requires bool (metadata) == False - * metadata -- if not None: metadata to update, "method mode", - requires bool (create) == False - - returns: created metadata if in function mode (create is True) else - None (implicit) - - raises: Exception if (create <=> metadata) - """ - if not create and not metadata: - raise Exception ( "either create or metadata" ) - elif create and metadata: - raise Exception ( "either create or metadata" ) - elif create: - metadata = MetadataJob ( - self.package_info, - self.logger.getChild ( 'metadata' ) - ) - - - metadata.update ( - self.description_reader.get_desc ( run_if_unset=False ), - self.package_info - ) - - if create: return metadata - # --- end of feed_metadata (...) --- - - def get_resolver ( self, dependency_type ): - # comment TODO - if not dependency_type in self._depres: - self._depres [dependency_type] = \ - self.request_resolver ( dependency_type, self.logger ) - - return self._depres [dependency_type] - # --- end of get_resolver (...) --- - - def get_ebuild ( self ): - """Returns the Ebuild that is created by this object. Note that you should - check the status with status ( $TODO::EBUILD_READY ) before trying to use - the Ebuild. - ##fixme: it is (should be?) guaranteed that self.ebuild is None unless the Ebuild is successfully created## - """ - return self.ebuild - - # --- end of get_ebuild (...) --- - - def get_status ( self, expected_status=None ): - """Returns the current status of this job or a bool that indicates - whether to current status matches the expected one. - - arguments: - * expected_status -- if not None: check if this job's state - is expected_status - """ - if not expected_status is None: - return bool ( self.status == expected_status ) - else: - return self.status - - # --- end of get_status (...) --- - - def done_success ( self ): - """Returns True if this has been successfully finished.""" - return self.get_status ( 'SUCCESS' ) - - # --- end of done_success (...) --- - - def run ( self ): - """Tells this EbuildJob to run. - This means that it reads the package file, resolves dependencies - using its resolver and creates an Ebuild object that is ready - to be written into a file. - """ - - # TODO move hardcoded entries to config/const - - try: - # enforcing BRANCHMAP status control: set status or return - if not self._set_status ( 'BUSY', True ): return - - desc = self.description_reader.get_desc ( True ) - if desc is None: - self._set_status ( 'FAIL' ) - self.logger.info ( 'Cannot create an ebuild for this package.' ) - - - ebuild = Ebuild ( self.logger.getChild ( "Ebuild" ) ) - - ebuild.add ( 'pkg_name', self.package_info ['package_name'] ) - - # TODO move regex to config/const - ebuild.add ( - 'pkg_version', - re.sub ( '[-]{1,}', '.', self.package_info ['package_version'] ) - ) - ebuild.add ( 'PKG_FILE', self.package_info ['package_file'] ) - - - have_description = False - - if 'Title' in desc: - ebuild.add ( 'DESCRIPTION', desc ['Title'] ) - have_description = True - - if 'Description' in desc: - ebuild.add ( - 'DESCRIPTION', - ( '// ' if have_description else '' ) + desc ['Description'] - ) - - del have_description - - - ebuild.add ( 'SRC_URI', self.package_info ['package_url'] ) - - ## default ebuild header, could use some const here (eclass name,..) - # TODO use a single string as ebuild header instead of joining it - # for every ebuild - ebuild.add ( - 'ebuild_header', - EbuildJob.DEFAULT_EBUILD_HEADER, - False - ) - - if not self.request_resolver is None: - # dependency resolution is enabled - - dep_type = desc_field = None - - - for dep_type in EbuildJob.DEPENDENCY_FIELDS: - - resolver = None - - for desc_field in EbuildJob.DEPENDENCY_FIELDS [dep_type]: - - if desc_field in desc: - if not resolver: - resolver = self.get_resolver ( dep_type ) - - if isinstance ( desc [desc_field], list ): - resolver.add_dependencies ( desc [desc_field] ) - - else: - resolver.add_depency ( desc [desc_field] ) - - - # lazy depres: wait until done and stop if any resolver channel - # returns None (which implies failure) - # wait for depres and store results - resolved = True - - if not self._set_status ( 'WAIT_RESOLVE' ): return - - for resolver in self._depres.values(): - if resolver.satisfy_request() is None: - resolved = False - break - - if not self._set_status ( 'BUSY' ): return - - if not resolved: - # ebuild is not creatable, - # set status to FAIL and close dep resolvers - self.logger.info ( - "Failed to resolve dependencies for this package." - ) - for r in self._depres.values(): r.close () - self._set_status ( 'FAIL' ) - return - else: - # add deps to the ebuild - for dep_type, resolver in self._depres.items(): - # python3 requires list ( filter ( ... ) ) - deplist = list ( - filter ( None, resolver.collect_dependencies () ) - ) - - if deplist is None: - ## FIXME: false positive: "empty" channel - raise Exception ( - 'dep_resolver is broken: ' - 'lookup() returns None but satisfy_request() says ok.' - ) - - elif isinstance ( deplist, ( list, set ) ): - # add dependencies in no_append/override mode - self.logger.debug ( "adding %s to %s", deplist, dep_type ) - ebuild.add ( dep_type, deplist, False ) - - else: - raise Exception ( - "dep_resolver is broken: list or set expected!" - ) - # --- end for - - # tell the dep resolver channels that we're done - for r in self._depres.values(): r.close () - - # --- end dep resolution - - - ## finalize self.ebuild: forced text creation + make it readonly - if ebuild.prepare ( True, True ): - self.ebuild = ebuild - - # update package info - self.package_info.set_writeable() - self.package_info.update ( ebuild=ebuild ) - self.package_info.set_readonly() - - - - except Exception as e: - # any exception means failure - self._set_status ( 'FAIL' ) - self.logger.exception ( e ) - raise - - # --- end of run (...) --- - - def _set_status ( self, new_status, ignore_invalid=False ): - """Changes the status of this job. May refuse to do that - if invalid change requested (e.g. 'FAIL' -> 'SUCCESS'). - - arguments: - new_status -- - """ - - if new_status == 'FAIL': - # always allowed - self.logger.info ( "Entering status '%s'.", new_status ) - self.status = new_status - return True - - if new_status and new_status in EbuildJob.STATUS_LIST: - # check if jumping from self.status to new_status is allowed - if new_status in EbuildJob.STATUS_BRANCHMAP [self.status]: - self.logger.debug ( "Entering status '%s'.", new_status ) - self.status = new_status - return True - - # default return - self.logger.error ( "Cannot enter status '%s'.", new_status ) - return False - - # --- end of _set_status (...) --- diff --git a/roverlay/portage/manifest.py b/roverlay/portage/manifest.py deleted file mode 100644 index 79e241c..0000000 --- a/roverlay/portage/manifest.py +++ /dev/null @@ -1,32 +0,0 @@ -# R Overlay -- Manifest creation for ebuilds -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - - -import logging - -import roverlay.portage.manifesthelpers - -_MANIFEST_IMPLEMENTATION = \ - roverlay.portage.manifesthelpers.ExternalManifestCreation - - -def create_manifest ( package_info, nofail=False ): - """Creates a Manifest for package_info, using the <> implementation - available. - - current implementation: ExternalManifestCreation (using ebuild(1)) - - arguments: - * package_info -- - * nofail -- catch exceptions and return False - """ - try: - return _MANIFEST_IMPLEMENTATION.do ( package_info ) - except Exception as e: - logging.exception ( e ) - if nofail: - return False - else: - raise -# --- end of create_manifest (...) --- diff --git a/roverlay/portage/manifesthelpers.py b/roverlay/portage/manifesthelpers.py deleted file mode 100644 index e7a54eb..0000000 --- a/roverlay/portage/manifesthelpers.py +++ /dev/null @@ -1,207 +0,0 @@ -# R Overlay -- Manifest creation for ebuilds -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - - -# TODO (in future): could use portage api directly, namely -# '/usr/lib/portage/pym/portage/package/ebuild/doebuild.py' -# instead of using '/usr/bin/ebuild' - -import os -import re -import copy -import logging -import subprocess - -from roverlay import config, util - -class _ManifestCreation ( object ): - """This is the base class for Manifest file creation.""" - - static_instance = None - - def __init__ ( self ): - self.logger = logging.getLogger ( 'ManifestCreation' ) - # --- end of __init__ (...) --- - - def create_for ( self, package_info ): - """Creates a Manifest file for the ebuild of the given package_info.""" - raise Exception ( "method stub" ) - # --- end of create_for (...) --- - - @classmethod - def do ( cls, package_info ): - """Class/static access to Manifest creation.""" - if cls.static_instance is None: - cls.static_instance = cls() - - return cls.static_instance.create_for ( package_info ) - # --- end of do (...) --- - - -class ExternalManifestCreation ( _ManifestCreation ): - """This class implements Manifest creation using the low level ebuild - interface, ebuild(1), which is called in a filtered environment. - """ - - def __init__ ( self ): - super ( ExternalManifestCreation, self ) . __init__ () - self.manifest_env = ManifestEnv ( filter_env=True ) - # ebuild , where target is: - self.ebuild_tgt = config.get ( 'TOOLS.EBUILD.target', 'manifest' ) - self.ebuild_prog = config.get ( 'TOOLS.EBUILD.prog', '/usr/bin/ebuild' ) - - # --- end of __init__ (...) --- - - def create_for ( self, package_info ): - """See ManifestCreation.create_for. - Calls ebuild, returns True on success else False. - - raises: *passes Exceptions from failed config lookups - """ - - my_env = self.manifest_env [ package_info ['distdir'] ] - - ebuild_file = package_info ['ebuild_file'] - - ebuild_call = subprocess.Popen ( - ( - self.ebuild_prog, - ebuild_file, - self.ebuild_tgt - ), - stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=my_env - ) - - output = ebuild_call.communicate() - # necessary? (probably not, FIXME/TODO) - ebuild_call.wait() - - # log stdout? - #for line in util.pipe_lines ( output [0] ): - # LOGGER.debug ( line ) - #for line in util.pipe_lines ( output [0] ): print ( line ) - - # log stderr - for line in util.pipe_lines ( output [1], use_filter=True ): - self.logger.warning ( line ) - - if ebuild_call.returncode == 0: - return True - else: - self.logger.error ( "Couldn't create Manifest for %s!" % ebuild_file ) - return False - # --- end of create_for (...) --- - - -class ManifestEnv ( object ): - """per-repo environment container for Manifest creation using ebuild.""" - - def __init__ ( self, filter_env=True ): - """Initializes a ManifestEnv. - - arguments: - * filter_env -- if True: start with an empty env and copy vars - from os.environ selectively - else : start with os.environ as env - """ - self.filter_env = filter_env - self._manenv = dict() - self.logger = logging.getLogger ( 'ManifestEnv' ) - self._common_env = None - # --- end of __init__ (...) --- - - def get_env ( self, repo_dir ): - """Returns an env dict for repo_dir. - - arguments: - * repo_dir -- - """ - if not repo_dir in self._manenv: - repo_env = self._get_common_manifest_env() - repo_env ['DISTDIR'] = repo_dir - self._manenv [repo_dir] = repo_env - - return self._manenv [repo_dir] - # --- end of get_env (...) --- - - # x = ManifestEnv(); env = x [] etc. - __getitem__ = get_env - # --- - - def _get_common_manifest_env ( self, noret=False ): - """Creates an environment suitable for an - "ebuild digest|manifest" call (or uses an already existing env). - Returns a shallow copy of this env which can then be locally modified - (setting DISTDIR). - - arguments: - * noret -- do not return copied env if True - """ - - if self._common_env is None: - - if self.filter_env: - - # selectively import os.environ - # FIXME: keep EBUILD_DEFAULT_OPTS? - our_env = util.keepenv ( - ( 'PATH', '' ), - 'LANG', - 'PWD', - 'EBUILD_DEFAULT_OPTS' - ) - else: - # copy os.environ - our_env = dict ( os.environ ) - - # -- common env part - - # set FEATURES - # * digest -- needed? (works without it) - # * assume-digests -- - # * unknown-features-warn -- should FEATURES ever change - # - # * noauto -- should prevent ebuild from adding additional actions, - # it still tries to download source packages, which is just wrong - # here 'cause it is expected that the R package file exists when - # calling this function, so FETCHCOMMAND/RESUMECOMMAND will be set - # to /bin/true if possible. - # - # * distlocks -- disabled if FETCHCOMMAND/RESUMECOMMAND set to no-op - # - our_env ['FEATURES'] = \ - "noauto digest assume-digests unknown-features-warn" - - # try to prevent src fetching - fetch_nop = util.sysnop ( - nop_returns_success=True, - format_str="%s \${DISTDIR} \${FILE} \${URI}" - ) - - if not fetch_nop is None: - self.logger.debug ( - "%s disables/replaces FETCHCOMMAND,RESUMECOMMAND." - % fetch_nop [0] - ) - our_env ['FETCHCOMMAND'] = fetch_nop [1] - our_env ['RESUMECOMMAND'] = fetch_nop [1] - our_env ['FEATURES'] += " -distlocks" - - - - # set PORDIR_OVERLAY - our_env ['PORTDIR_OVERLAY'] = config.get_or_fail ( - [ 'OVERLAY', 'dir' ] - ) - - self._common_env = our_env - # -- end if - if noret: - return None - else: - return copy.copy ( self._common_env ) - # --- end of _get_common_manifest_env (...) --- diff --git a/roverlay/portage/metadata/__init__.py b/roverlay/portage/metadata/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/roverlay/portage/metadata/abstractnodes.py b/roverlay/portage/metadata/abstractnodes.py deleted file mode 100644 index 50c9f56..0000000 --- a/roverlay/portage/metadata/abstractnodes.py +++ /dev/null @@ -1,284 +0,0 @@ -# R Overlay -- ebuild creation, basic metadata nodes -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import textwrap - -INDENT = '\t' - -get_indent = lambda k : k * INDENT - -# -- "abstract" metadata nodes -- -class _MetadataBasicNode ( object ): - """ - This is the most basic metadata node that should never be used directly. - """ - - def __init__ ( self, name, flags ): - """Initializes a _MetadataBasicNode. - - arguments: - * name -- name of this node, e.g. 'flag' - * flags -- flags of this node, e.g. 'name=byte-compile' - """ - self.name = name - self.flags = flags - # priority is used to sort nodes (e.g. longdescription after description) - self.priority = 1000 - self._enabled = True - self._set_level ( 0 ) - # --- end of __init__ (...) --- - - def active ( self ): - """Returns True if this node is active.""" - return self._enabled - # --- end of active (...) --- - - def _set_level ( self, _level, has_indent=True ): - """Sets the level (depth) of this node. - - arguments: - * _level -- level to set - * has_indent -- if this node should indent its subnodes (or text) - """ - self.level = _level - if has_indent: - self.indent = get_indent ( self.level ) - else: - self.indent = '' - # --- end of _set_level (...) --- - - def set_flag ( self, flagname, flagvalue=None ): - """Sets the specified flag. - - arguments: - * flagname -- flag name - * flagvalue -- flag value, defaults to None->'' - """ - self.flags [flagname] = '' if flagvalue is None else flagvalue - # --- end of set_flag (...) --- - - def _flaglist ( self ): - """Returns a "flagname=flagvalue" list.""" - return [ '%s="%s"' % ftup for ftup in self.flags.items() ] - # --- end of _flaglist (...) --- - - - def _flagstr ( self ): - """Returns the string representation of this node's flags, including - a leading whitespace char. - """ - if self.flags is None: - return '' - - ret = ' '.join ( self._flaglist() ) - if ret: - # don't forget the leading space - return ' ' + ret - else: - return '' - # --- end of _flagstr (...) --- - - def _do_verify ( self ): - """Verifies this node. - Does nothing if self._verify is not implemented, else dies on error. - """ - if hasattr ( self, '_verify' ) and not self._verify(): - # todo, verify could return ( Status, ErrorMessages ) etc. - raise Exception ( "verification failed for a metadata node." ) - # --- end of _do_verify (...) --- - - # not using __str__, make (recursive) node calls explicit - def to_str ( self ): - """Returns a string representing this node.""" - self._do_verify() - - return "%s<%s%s>" % ( - self.indent, - self.name, - self._flagstr(), - self.name - ) - # --- end of to_str (...) --- - - -class MetadataNode ( _MetadataBasicNode ): - """A _MetadataBasicNode with child nodes.""" - - def __init__ ( self, name, flags=dict() ): - super ( MetadataNode, self ) . __init__ ( name, flags ) - self.nodes = list() - # --- end of __init__ (...) --- - - def add ( self, node ): - """Adds a child node to this node. Fixes/sets the level of this node. - - arguments: - * node -- - """ - node._set_level ( self.level + 1 ) - self.nodes.append ( node ) - # --- end of add (...) --- - - # copy add to _add_node - _add_node = add - - def _sort_nodes ( self ): - """Sorts the child nodes of this node.""" - self.nodes.sort ( key=lambda node : node.priority ) - # --- end of _sort_nodes (...) --- - - def _nodelist ( self ): - """Returns a list of strings representing the child nodes.""" - return list ( - filter ( - None, - [ node.to_str() for node in self.nodes if node.active() ] - ), - ) - # --- end of _nodelist (...) --- - - def _nodestr ( self ): - """Returns a string representing all child nodes.""" - self._sort_nodes() - # todo filter only None? - node_repr = self._nodelist() - if len ( node_repr ): - # add newlines before/after and indent after node_repr! - return "\n%s\n%s" % ( '\n'.join ( node_repr ), self.indent ) - else: - return '' - # --- end of _nodestr (...) --- - - def to_str ( self ): - """Returns a string representing this node and all of its child nodes.""" - self._do_verify() - return "%s<%s%s>%s" % ( - self.indent, - self.name, - self._flagstr(), - self._nodestr(), - self.name - ) - - -class MetadataNodeOrdered ( MetadataNode ): - """A metadata node whose nodes have to be interpreted as an already - ordered string.""" - # could derive MetadataNode from this - def _sort_nodes ( self ): return - - -class MetadataLeaf ( _MetadataBasicNode ): - """A metadata node that has no child nodes, only a string.""" - - def __init__ ( self, name, flags=dict(), value=None ): - self._text_wrapper = None - super ( MetadataLeaf, self ) . __init__ ( name, flags ) - - self.value = "" if value is None else value - self.print_node_name = True - self.value_format = 0 - - def _set_level ( self, _level, has_indent=True ): - super ( MetadataLeaf, self ) . _set_level ( _level, has_indent ) - - # indenting multi-line text by one level more than self.indent - self.text_indent = self.indent + INDENT - - # update text wrapper if existent - if not self._text_wrapper is None: - self._text_wrapper.subsequent_indent = self.text_indent - - def _value_str ( self ): - """Returns the value string. Derived classes may override this.""" - #if self.value_format == ?: format value ~ - return str ( self.value ) - # --- end of _value_str (...) --- - - def _pretty_value_str ( self ): - """Returns a formatted value string (max line length etc.). - Not used here, but subclasses can use it by simply writing - '_value_str = MetadataLeaf._pretty_value_str' in the class body. - """ - # FIXME/TODO: could move this func to util - if not self.value: return "" - - if self._text_wrapper is None: - self._text_wrapper = textwrap.TextWrapper ( - initial_indent='', - subsequent_indent=self.text_indent, - width=self.linewidth if hasattr ( self, 'linewidth' ) else 50 - ) - - val_lines = self._text_wrapper.wrap ( self.value ) - if len ( val_lines ) < 1: - # why? - return "" - elif len ( val_lines ) == 1: - # single line, no indent/newline - return val_lines [0] - else: - # add newline before/after, add indent after - val_lines [0] = '\n' + self.text_indent + val_lines [0] - val_lines.append ( self.indent ) - return '\n'.join ( val_lines ) - # --- end of _pretty_value_str (...) --- - - def to_str ( self ): - self._do_verify() - if self.print_node_name: - return "%s<%s%s>%s" % ( - self.indent, - self.name, - self._flagstr(), - self._value_str(), - self.name - ) - else: - # not very useful, but allows to insert strings as nodes - return self.indent + self._value_str() - # --- end of to_str (...) --- - -class MetadataNodeNamedAccess ( MetadataNode ): - """A metadata node that offers key-based (dictionary) access to some of - its nodes.""" - - def __init__ ( self, name, flags=dict() ): - super ( MetadataNodeNamedAccess, self ) . __init__ ( name, flags ) - # the access dict - self.node_dict = dict() - - def add ( self, node, with_dict_entry=True, fail_if_existent=True ): - """Adds a child node. - - arguments: - * node -- node to add - * with_dict_entry -- add node to the access dict, defaults to True - * fail_if_existent -- fail if node's name already in the access dict, - defaults to True - """ - - super ( MetadataNodeNamedAccess, self ) . add ( node ) - if with_dict_entry: - if fail_if_existent and node.name in self.node_dict: - raise Exception ( "key exists." ) - else: - self.node_dict [node.name] = node - # --- end of add (...) --- - - def get ( self, node_name ): - """Returns node by name. - - arguments: - * node_name - - raises: KeyError if node_name not in the access dict - """ - return self.node_dict [node_name] - # --- end of get (...) --- - - def has_named ( self, node_name ): - """Returns True if node_name in the access dict else False.""" - return node_name in self.node_dict - # --- end of has_named (...) --- diff --git a/roverlay/portage/metadata/creation.py b/roverlay/portage/metadata/creation.py deleted file mode 100644 index d2f133c..0000000 --- a/roverlay/portage/metadata/creation.py +++ /dev/null @@ -1,89 +0,0 @@ -# R Overlay -- ebuild creation, metadata creation -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import roverlay.config - -from roverlay.portage.metadata import nodes - -class MetadataJob ( object ): - """R package description data -> metadata.xml interface.""" - - def __init__ ( self, package_info, logger ): - """Initializes a MetadataJob. - - arguments: - * package_info -- reserved for future usage - * logger -- logger to use (this instance won't call getChild) - """ - self.logger = logger - self._metadata = nodes.MetadataRoot() - # reserved for future usage ("dominant ebuilds": when ebuildjobs - # share one metadata instance etc.) - self.package_info = None - # --- end of __init__ (...) --- - - def update ( self, desc_data, package_info ): - """Updates the metadata using the given description data. - - It's expected that this method is called when Ebuild creation is done. - - arguments: - * desc_data -- description data read from R package - * package_info -- reserved for future usage - - returns: None (implicit) - """ - pass - - mref = self._metadata - - max_textline_width = roverlay.config.get ( 'METADATA.linewidth', 65 ) - - have_desc = False - - if 'Title' in desc_data: - mref.add ( nodes.DescriptionNode ( - desc_data ['Title'], - is_long=have_desc, - linewidth=max_textline_width - ) ) - have_desc = True - - if 'Description' in desc_data: - # passing have_desc for DescriptionNode's is_long parameter redirects - # the second description info into - mref.add ( nodes.DescriptionNode ( - desc_data ['Description'], - is_long=have_desc, - linewidth=max_textline_width - ) ) - have_desc = True - - mref.add_useflag ( 'byte-compile', 'enable byte-compiling' ) - - if package_info ['has_suggests']: - mref.add_useflag ( 'R_suggests', 'install optional dependencies' ) - - # --- end of update (...) --- - - # used in some test scripts - update_metadata = update - - def write ( self, _file ): - """Writes the metadata into a file. - - arguments: - * _file -- file to write, either a file handle or string in which case - a file object will be created - - returns: True if writing succeeds, else False - - raises: Exception if no metadata to write - """ - if self._metadata.empty(): - raise Exception ( "not enough metadata to write!" ) - #return False - else: - return self._metadata.write_file ( _file ) - # --- end of write (...) --- diff --git a/roverlay/portage/metadata/nodes.py b/roverlay/portage/metadata/nodes.py deleted file mode 100644 index 36bf22d..0000000 --- a/roverlay/portage/metadata/nodes.py +++ /dev/null @@ -1,193 +0,0 @@ -# R Overlay -- ebuild creation, concrete metadata nodes -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -# import abstract nodes -from roverlay.portage.metadata.abstractnodes import \ - MetadataNode, MetadataNodeNamedAccess, MetadataLeaf - - -class MetadataRoot ( MetadataNodeNamedAccess ): - """This is the metadata root which represents a metadata file. - Intended usage is metadata file creation. - """ - - # the common metadata.xml header - HEADER = '\n'.join ( [ - '', - '' - ] ) - - - def __init__ ( self ): - super ( MetadataRoot, self ) . __init__ ( 'pkgmetadata' ) - self.priority = 0 - # --- end of __init__ (...) --- - - def empty ( self ): - """Returns True if this node has no child nodes.""" - #return 0 == len ( self.nodes ) or \ - # True in ( node.empty() for node in self.nodes ) - return 0 == len ( self.nodes ) - # --- end of empty (...) --- - - def add_useflag ( self, flag_name, flag_description ): - """Adds a USE Flag to the metadata. - A UseFlagListNode 'use' will be created if required and a new UseFlagNode - will then be created an added to 'use'. - - arguments: - * flag_name -- see UseFlagNode.__init__ - * flag_description -- see UseFlagNode.__init__ - - returns: the created UseFlagNode for further editing - """ - if not self.has_named ( 'use' ): - # passing fail_if_existent, this node shouldn't be used in parallel - self.add ( - UseFlagListNode(), - with_dict_entry=True, fail_if_existent=True - ) - - node = self.get ( 'use' ) - use_node = UseFlagNode ( flag_name, flag_description ) - node.add ( use_node ) - - return use_node - # --- end of add_useflag (...) --- - - def write_file ( self, _file ): - """Writes the metadata to a file. - - arguments: - * _file -- either a File object or a string - - returns: success True/False - - raises: *passes IOError - """ - to_write = self.to_str() - - own_fh = False - fh = None - success = False - - newline = '\n' - - try: - if isinstance ( _file, str ): - own_fh = True - fh = open ( _file, 'w' ) - else: - fh = _file - - - fh.write ( MetadataRoot.HEADER ) - fh.write ( newline ) - fh.write ( to_write ) - fh.write ( newline ) - - success = True - - except IOError: - # log this TODO - pass - finally: - if own_fh and fh: fh.close() - - return success - # --- end of write_file (...) --- - - -class DescriptionNode ( MetadataLeaf ): - """A description (, ) node.""" - - def __init__ ( self, description, is_long=False, linewidth=None ): - """Initializes a DescriptionNode. - - arguments: - * description -- description text - * is_long -- if this is a longdescription or a description node - * linewidth -- max text line width, TODO/FIXME: is this ignored? - """ - super ( DescriptionNode, self ) . __init__ ( - 'longdescription' if is_long else 'description', - value=description, - ) - # self.value_format = "break lines after 80c, ..." - - if not linewidth is None and linewidth > 0: - self.linewidth = linewidth - - self.priority = 150 if is_long else 149 - # --- end of __init__ (...) --- - - # using value formatting - _value_str = MetadataLeaf._pretty_value_str - - -class UseFlagNode ( MetadataLeaf ): - """A USE Flag node, this flag does...""" - def __init__ ( self, flag_name, flag_description ): - """Initializes an USE Flag node. - - arguments: - * flag_name -- name of the use flag - * flag_description -- flag description - """ - super ( UseFlagNode, self ) . __init__ ( - 'flag', - flags=dict ( name = flag_name ), - value=flag_description, - ) - # priority shouldn't be used for this node - self.priority = -1 - # --- end of __init__ (...) --- - - -class UseFlagListNode ( MetadataNode ): - """A USE Flag list node, ....""" - - def __init__ ( self, flags=dict() ): - """Initializes an USE Flag list node. - - arguments: - * flags -- optional - """ - super ( UseFlagListNode, self ) . __init__ ( 'use', flags=flags ) - self.priority = 850 - # --- end of __init__ (...) --- - - def active ( self ): - """The UseFlag list is only active if it is enabled and at least - one UseFlag child node is active. - """ - # generator should stop after first True - # todo/fixme: could use super ( UseFlagListNode, self ).active() instead - # of self._enabled - return True in ( node.active() for node in self.nodes ) and self._enabled - # --- end of active (...) --- - - def _sort_nodes ( self ): - """UseFlags are sorted by lowercase flag name, not priority.""" - self.nodes.sort ( key=lambda node : node.flags ['name'].lower() ) - # --- end of _sort_nodes (...) --- - - def add ( self, node ): - """Adds a child node only if it is a UseFlagNode. - - arguments: - * node -- - """ - if isinstance ( node, UseFlagNode ): - super ( UseFlagListNode, self ) . add ( node ) - else: - raise Exception ( "UseFlagListNode accepts UseFlagNodes only." ) - # --- end of add (...) --- - - -class NopNode ( MetadataNode ): - """This node is meant for testing only.""" - def __init__ ( self ): - super ( NopNode, self ) . __init__ ( 'nop', flags=dict() ) - # --- end of __init__ (...) --- diff --git a/roverlay/portage/overlay/__init__.py b/roverlay/portage/overlay/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/roverlay/portage/overlay/category.py b/roverlay/portage/overlay/category.py deleted file mode 100644 index ff497f7..0000000 --- a/roverlay/portage/overlay/category.py +++ /dev/null @@ -1,85 +0,0 @@ -# R Overlay -- -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import threading -import os.path - - -from roverlay.portage.overlay.package import PackageDir - -import roverlay.util - -class Category ( object ): - - def __init__ ( self, name, logger, directory ): - self.logger = logger.getChild ( name ) - self.name = name - self._lock = threading.RLock() - self._subdirs = dict() - self.physical_location = directory - # --- end of __init__ (...) --- - - def empty ( self ): - return \ - len ( self._subdirs ) == 0 or \ - not False in ( d.empty() for d in self._subdirs ) - # --- end of empty (...) --- - - def set_fs_location ( self, directory ): - self._lock.acquire() - self.physical_location = directory - - if not directory is None: - for pkg_name, pkg in self._subdirs.items(): - pkg.set_fs_location ( - os.path.join ( directory, pkg_name ) - ) - - self._lock.release() - # --- end of set_fs_location (...) --- - - def add ( self, package_info ): - # TODO make keys available - pkg_name = package_info ['name'] - - if not pkg_name in self._content: - self._lock.acquire() - if not pkg_name in self._content: - self._content [pkg_name] = PackageDir ( - pkg_name, - self.logger, - None if self.physical_location is None else \ - os.path.join ( self.physical_location, pkg_name ) - ) - self._lock.release() - - self._content [pkg_name].add ( package_info ) - # --- end of add (...) --- - - def generate_metadata ( self, **metadata_kw ): - for package in self._subdirs.values(): - package.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - - def show ( self, **show_kw ): - for package in self._subdirs.values(): - package.show ( **show_kw ) - # --- end of show (...) --- - - def write ( self ): - for package in self._subdirs.values(): - if package.physical_location and not package.empty(): - roverlay.util.dodir ( package.physical_location ) - package.write() - - # --- end of write (...) --- - - def ls ( self ): - return frozenset ( - ( os.path.join ( n, p.ls() ) for n, p in self._subdirs.items() ) - ) - - def __str__ ( self ): return '\n'.join ( self.ls() ) - - diff --git a/roverlay/portage/overlay/package.py b/roverlay/portage/overlay/package.py deleted file mode 100644 index 9fc0112..0000000 --- a/roverlay/portage/overlay/package.py +++ /dev/null @@ -1,219 +0,0 @@ -# R Overlay -- -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import threading -import os.path -import sys - -from roverlay.portage.metadata.creation import MetadataJob - -SUPPRESS_EXCEPTIONS = True - -class PackageDir ( object ): - - # TODO: do Manifest creation here - - def __init__ ( self, name, logger, directory ): - self.logger = logger.getChild ( name ) - # Lock or RLock? (TODO) - self._lock = threading.RLock() - self._packages = dict() - self._metadata = None - self.physical_location = directory - # --- end of __init__ (...) --- - - def empty ( self ): - return len ( self._packages ) == 0 - - def set_fs_location ( self, directory ): - self.physical_location = directory - - def _get_metadata_filepath ( self ): - return os.path.join ( - '??' if self.physical_location is None else self.physical_location, - self._metadata.filename - ) - # --- end of _get_metadata_filepath (...) --- - - def _get_ebuild_filepath ( self, pvr ): - filename = "%s-%s.ebuild" % ( self.name, pvr ) - return os.path.join ( - '??' if self.physical_location is None else self.physical_location, - filename - ) - # --- end of _get_ebuild_filepath (...) --- - - def write ( self ): - if self.physical_location is None: - raise Exception ( "cannot write - no directory assigned!" ) - - self._lock.acquire() - self._regen_metadata() - - # mkdir not required here, overlay.Category does this - - for ver, p_info in self._packages.items(): - fh = None - try: - efile = self._get_ebuild_filepath ( ver ) - ebuild = p_info.ebuild - - fh = open ( efile, 'w' ) - ebuild.write ( fh ) - - # adjust owner/perm? TODO - # chmod 0644 or 0444 - # chown 250.250 - - self.logger.info ( "Wrote ebuild %s." % efile ) - except IOError as e: - self.logger.error ( "Couldn't write ebuild %s." % efile ) - self.logger.exception ( e ) - - finally: - if fh: fh.close() - fh = None - - fh = None - try: - mfile = self._get_metadata_filepath() - - fh = open ( mfile, 'w' ) - self._metadata.write ( fh ) - - except IOError as e: - self.logger.error ( "Failed to write metadata at %s." % mfile ) - self.logger.exception ( e ) - finally: - if fh: fh.close() - del fh - - # !! TODO write Manifest here - - self._lock.release() - # --- end of write (...) --- - - def show ( self, stream=sys.stderr ): - self._lock.acquire() - self._regen_metadata() - - - for ver, p_info in self._packages.items(): - efile = self._get_ebuild_filepath ( ver ) - ebuild = p_info.ebuild - - stream.write ( "[BEGIN ebuild %s]\n" % efile ) - ebuild.write ( stream ) - stream.write ( "[END ebuild %s]\n" % efile ) - - mfile = self._get_metadata_filepath() - - stream.write ( "[BEGIN %s]\n" % mfile ) - self._metadata.write ( stream ) - stream.write ( "[END %s]\n" % mfile ) - - - self._lock.release() - - def _latest_package ( self ): - """Returns the package info with the highest version number.""" - first = True - retver = None - retpkg = None - for p in self._packages.values(): - newver = p ['version'] - if first or newver > retver: - retver = newver - retpkg = p - first = False - - return retpkg - # --- end of _latest_package (...) --- - - def add ( self, package_info ): - # !! p info key TODO - shortver = package_info ['ebuild_verstr'] - - def already_exists ( release=False ): - if filename in self._packages: - - if release: self._lock.release() - - msg = "'%s-%s.ebuild' already exists, cannot add it!" % ( - self.name, shortver - ) - if SUPPRESS_EXCEPTIONS: - logger.warning ( msg ) - else: - raise Exception ( msg ) - - return True - else: - return False - # --- end of already_exists (...) --- - - if already_exists ( release=False ): return False - self._lock.acquire() - if already_exists ( release=True ): return False - - self._packages [shortver] = package_info - - self._lock.release() - # --- end of add (...) --- - - def _regen_metadata ( self ): - self.generate_metadata ( - skip_if_existent=True, - use_all_packages=True, - use_old_metadata=False - ) - - def generate_metadata ( - self, - skip_if_existent=False, use_all_packages=False, use_old_metadata=False - ): - if skip_if_existent and not self._metadata is None: - return - - self._lock.acquire() - - if not use_old_metadata or self._metadata is None: - del self._metadata - self._metadata = MetadataJob ( self.logger ) - - if use_all_packages: - for p_info in self._packages: - self._metadata.update ( p_info ) - else: - self._metadata.update ( _latest_package() ) - - - self._lock.release() - - def _flist ( self ): - files = list() - if not self._metadata is None: - files.append ( self._metadata.filename ) - - for ver in self._packages: - files.append ( "%s-%s.ebuild" % ( self.name, ver ) ) - - return files - # --- end of _flist (...) --- - - def ls ( self ): - return frozenset ( self._flist() ) - # --- end of ls (...) --- - - def lslong ( self ): - return frozenset ( ( os.path.join ( - '??' if self.physical_location is None else self.physical_location, - f - ) for f in self._flist() ) ) - # --- end of lslong (...) --- - - def __str__ ( self ): return '\n'.join ( self.ls() ) - - - diff --git a/roverlay/portage/overlay/root.py b/roverlay/portage/overlay/root.py deleted file mode 100644 index 086cab4..0000000 --- a/roverlay/portage/overlay/root.py +++ /dev/null @@ -1,302 +0,0 @@ -# R Overlay -- -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import threading -import logging -import shutil -import os - -from roverlay import config, util - -from roverlay.portage.overlay.category import Category - -DEFAULT_USE_DESC = '\n'.join ( [ - 'byte-compile - enable byte compiling', - 'R_suggests - install recommended packages' -] ) - -class Overlay ( object ): - - def __init__ ( - self, - name=None, - logger=None, - directory=None, - default_category=None, - eclass_files=None - ): - - # not setting any default values here (currently) - - if name is None: - self.name = config.get_or_fail ( 'OVERLAY.name' ) - else: - self.name = name - - if logger is None: - #self.logger = logging.getLogger ( self.name ) - self.logger = logging.getLogger ( 'overlay' ) - else: - #self.logger = logger.getChild ( self.name ) - self.logger = logger.getChild ( 'overlay' ) - - if directory is None: - self.physical_location = config.get_or_fail ( 'OVERLAY.dir' ) - else: - self.physical_location = directory - - if default_category is None: - self.default_category = config.get_or_fail ( 'OVERLAY.category' ) - else: - self.default_category = default_category - - self.eclass_files = eclass_files - - # - self._profiles_dir = os.path.join ( self.physical_location, 'profiles' ) - - self._catlock = threading.Lock() - self._categories = dict() - # --- end of __init__ (...) --- - - def _get_category ( self, category ): - """Returns a reference to the given category. Creates it if necessary. - - arguments: - * category -- category identifier as string - """ - if not category in self._categories: - self._catlock.acquire() - if not category in self._categories: - self._categories [category] = Category ( - category, - self.logger, - None if self.physical_location is None else \ - os.path.join ( self.physical_location, category ) - ) - self._catlock.release() - - return self._categories [category] - # --- end of _get_category (...) --- - - def add ( self, package_info, category=None ): - """Adds a package to this overlay. - - arguments: - * package_info -- PackageInfo of the package to add - * category -- category where the pkg should be put in, defaults to - self.default_category - - returns: None (implicit) - """ - self._get_category ( - self.default_category if category is None else category - ) . add ( package_info ) - # --- end of add (...) --- - - def ls ( self ): - """Returns a set of ebuilds/metadata stored in this overlay.""" - return frozenset ( - ( os.path.join ( n, c.ls() ) for n, c in self._categories.items() ) - ) - # --- end of ls (...) --- - - def __str__ ( self ): return '\n'.join ( self.ls() ) - # --- end of __str__ --- - - def show ( self, **show_kw ): - """Presents the ebuilds/metadata stored in this overlay. - - arguments: - * **show_kw -- keywords for package.PackageDir.show(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): cat.show ( **show_kw ) - # --- end of show (...) --- - - def write ( self, **write_kw ): - """Writes the overlay to its physical location (filesystem), including - metadata and Manifest files. - TODO include Manifest generation in package.py - - arguments: - * **write_kw -- keywords for package.PackageDir.write(...) - - returns: None (implicit) - - raises: !! TODO - - TODO/FIXME/DOC: This is not thread-safe, it's expected to be called - when ebuild creation is done. - """ - # writing profiles/ here, rewriting categories/ later - self._init_overlay ( reimport_eclass=True, make_profiles_dir=True ) - - for cat in self._categories.values(): - if cat.physical_location and not cat.empty(): - util.dodir ( cat.physical_location ) - cat.write() - - self._write_categories ( only_active=True ) - # --- end of write (...) --- - - def write_incremental ( self, **write_kw ): - """Writes all ebuilds that have been added since the last - write_incremental call. - TODO: - * This could be useful to save some mem by removing already written - package infos. - * This has to be thread safe - """ - raise Exception ( "method stub" ) - # --- end of write_incremental (...) --- - - def generate_metadata ( self, **metadata_kw ): - """Tells the overlay's categories to create metadata. - You don't have to call this before write()/show() unless you want to use - special metadata options. - - arguments: - * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): - cat.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - - def _write_profiles_dir ( self, only_active_categories=True ): - """Creates and updates the profiles/ dir. - - arguments: - * only_active_categories -- if True: do not list categories without - ebuilds in profiles/categories - """ - # profiles/ - util.dodir ( self._profiles_dir ) - self._write_repo_name() - self._write_categories ( only_active=only_active_categories ) - self._write_usedesc() - # --- end of _write_profiles_dir (...) --- - - def _write_profiles_file ( self, filename, to_write ): - """Writes a file in profiles/. - - arguments: - * filename -- name of the file to write (including file extension) - * to_write -- string to write (don't forget newline at the end) - """ - fh = None - try: - fh = open ( os.path.join ( self._profiles_dir, filename ), 'w' ) - if to_write: - # else touch file - fh.write ( to_write ) - except IOError as e: - self.logger.exception ( e ) - raise - finally: - if fh: fh.close() - # --- end of _write_profiles_file (...) --- - - def _write_repo_name ( self ): - """Writes profiles/repo_name.""" - self._write_profiles_file ( 'repo_name', self.name + '\n' ) - # --- end of _write_repo_name (...) --- - - def _write_categories ( self, only_active=True ): - """Writes profiles/categories. - - arguments: - * only_active -- exclude categories without ebuilds - """ - cats = None - if only_active: - cats = [ - name for name, category - in self._categories.items() if not category.empty() - ] - else: - cats = list ( self._categories.keys() ) - - if cats: - self._write_profiles_file ( - 'categories', - '\n'.join ( cats ) + '\n' - ) - # --- end of _write_categories (...) --- - - def _write_usedesc ( self ): - """Writes profiles/use.desc.""" - # TODO: config entry - use_desc = config.get ( - 'OVERLAY.use_desc', - fallback_value=DEFAULT_USE_DESC - ) - if use_desc: - self._write_profiles_file ( 'use.desc', use_desc + '\n' ) - # --- end of _write_usedesc (...) --- - - def _init_overlay ( self, reimport_eclass=False, make_profiles_dir=False ): - """Initializes the overlay at its physical/filesystem location. - - arguments: - * reimport_eclass -- whether to copy existing eclass files - again (True) or not - * make_profiles_dir -- if True: create the profiles/ dir now - - raises: - * Exception if no physical location assigned - * passes IOError,... - """ - if self.physical_location is None: - raise Exception ( "no directory assigned." ) - - try: - root = self.physical_location - # mkdir overlay root - os.makedirs ( root, exist_ok=True ) # raises? - - if self.eclass_files: - # import eclass files - eclass_dir = os.path.join ( root, 'eclass' ) - try: - util.dodir ( eclass_dir ) - - for eclass in self.eclass_files: - src = eclass - dest = None - if isinstance ( eclass, str ): - dest = os.path.basename ( eclass ) - else: - # list-like specification ( src, destname ) - src = eclass [0] - dest = eclass [1] - - if reimport_eclass or not os.path.isfile ( dest ): - shutil.copyfile ( src, dest ) - - - except Exception as e: - #self.logger.exception ( e ) TODO try-catch blocks - self.logger.critical ( "Cannot import eclass files!" ) - raise - - # -- eclass - if make_profiles_dir: - self._write_profiles_dir ( only_active_categories=False ) - - except IOError as e: - - self.logger.exception ( e ) - self.logger.critical ( "^failed to init overlay" ) - raise - - - - - - - diff --git a/roverlay/portage/packageinfo.py b/roverlay/portage/packageinfo.py deleted file mode 100644 index 7ac4940..0000000 --- a/roverlay/portage/packageinfo.py +++ /dev/null @@ -1,245 +0,0 @@ -# R Overlay -- package info class -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import re -import os.path -import logging -import threading - -from roverlay import config, util - -LOGGER = logging.getLogger ( 'PackageInfo' ) - -VIRTUAL_KEYS = dict ( - DISTDIR = frozenset ( [ 'distdir', 'pkg_distdir' ] ), - EBUILD_FILE = frozenset ( [ 'ebuild_file', 'efile' ] ), - HAS_SUGGESTS = frozenset ( [ 'has_suggests', 'has_rsuggests' ] ), - SRC_URI = frozenset ( [ 'src_uri', 'package_url', 'url' ] ), -) - - -class PackageInfo ( object ): - """PackageInfo offers easy, subscriptable (['sth']) access to package - information, whether stored or calculated. - """ - - def __init__ ( self, **initial_info ): - """Initializes a PackageInfo. - - arguments: - * **initial_info -- passed to update ( **kw ) - """ - self._info = dict() - self.readonly = False - self._update_lock = threading.RLock() - - self.update ( **initial_info ) - # --- end of __init__ (...) --- - - def set_readonly ( self, immediate=False, final=False ): - """Makes the package info readonly. - - arguments: - * immediate -- do not acquire lock, set readonly directly, - defaults to False - * final -- if set and True: make this decision final - """ - self._set_mode ( True, immediate, final ) - # --- end of set_readonly (...) --- - - def set_writeable ( self, immediate=False ): - """Makes the package info writeable. - - arguments: - * immediate -- do not acquire lock, set writeable directly, - defaults to False - """ - self._set_mode ( False, immediate ) - # --- end of set_writeable (...) --- - - def _set_mode ( self, readonly_val, immediate=False, final=False ): - """Sets readonly to True/False. - - arguments: - * readonly_val -- new value for readonly - * immediate -- do not acquire lock - * final -- only if readonly_val is True: make this decision final, - - raises: Exception if self.readonly is a constant (_readonly_final is set) - """ - if hasattr ( self, '_readonly_final' ): - raise Exception ( "cannot modify readonly - it's a constant." ) - elif immediate: - self.readonly = readonly_val - if final and readonly_val: - self._readonly_final = True - elif not self.readonly is readonly_val: - self._update_lock.acquire() - self.readonly = readonly_val - if final and readonly_val: - self._readonly_final = True - self._update_lock.release() - # --- end of _set_mode (...) --- - - def _writelock_acquire ( self ): - """Acquires the lock required for adding new information. - - raises: Exception if readonly (writing not allowed) - """ - if self.readonly or hasattr ( self, '_readonly_final' ): - raise Exception ( "package info is readonly!" ) - - self._update_lock.acquire() - - if self.readonly or hasattr ( self, '_readonly_final' ): - self._update_lock.release() - raise Exception ( "package info is readonly!" ) - - return True - # --- end of _writelock_acquire (...) --- - - def get ( self, key, fallback_value=None, do_fallback=False ): - """Returns the value specified by key. - The value is either calculated or taken from dict self._info. - - arguments: - * key -- - * fallback_value -- fallback value if key not found / unknown - * do_fallback -- if True: return fallback_value, else raise KeyError - - raises: KeyError - """ - key_low = key.lower() - - # 'virtual' keys - calculate result - if key_low in VIRTUAL_KEYS ['DISTDIR']: - if 'package_dir' in self._info: - return self._info ['package_dir'] - - elif 'origin' in self._info: - return util.get_distdir ( self._info ['origin'] ) - - elif key_low in VIRTUAL_KEYS ['EBUILD_FILE']: - return os.path.join ( - config.get_or_fail ( [ 'OVERLAY', 'dir' ] ), - config.get_or_fail ( [ 'OVERLAY', 'category' ] ), - self ['ebuild_name'].partition ( '-' ) [0], - self ['ebuild_name'] + ".ebuild" - ) - - elif key_low in VIRTUAL_KEYS ['HAS_SUGGESTS']: - if key_low in self._info: - return self._info [key_low] - - else: - return False - - elif key_low in VIRTUAL_KEYS ['SRC_URI']: - # comment from ebuildjob: - ## origin is todo (sync module knows the package origin) - ## could calculate SRC_URI in the eclass depending on origin - # comment from ebuild: - ## calculate SRC_URI using self._data ['origin'], - ## either here or in eclass - return "**packageinfo needs information from sync module!" - - # normal keys - if key in self._info: - return self._info [key] - - elif key_low in self._info: - return self._info [key_low] - - elif do_fallback: - return fallback_value - else: - raise KeyError ( key ) - # --- end of get (...) --- - - def __getitem__ ( self, key ): - return self.get ( key, do_fallback=False ) - # --- end of __getitem__ (...) --- - - def __setitem__ ( self, key, value ): - """Sets an item. - - arguments: - * key -- - * value -- - - raises: Exception when readonly - """ - self._writelock_acquire() - self._info [key] = value - self._update_lock.release() - # --- end of __setitem__ (...) --- - - def update ( self, **info ): - """Uses **info to update the package info data. - - arguments: - * **info -- - - raises: Exception when readonly - """ - if len ( info ) == 0 : - # nothing to do - return - - self._writelock_acquire() - - if 'filepath' in info: - self._use_filepath ( info ['filepath'] ) - - if 'ebuild' in info: - self._use_ebuild ( info ['ebuild'] ) - - self._update_lock.release() - # --- end of update (**kw) --- - - def _use_filepath ( self, filepath ): - """auxiliary method for update(**kw) - - arguments: - * filepath -- - """ - - package_file = os.path.basename ( filepath ) - - # remove .tar.gz .tar.bz2 etc. - filename = re.sub ( - config.get ( 'R_PACKAGE.suffix_regex' ) + '$', - '', - package_file - ) - - package_name, sepa, package_version = filename.partition ( - config.get ( 'R_PACKAGE.name_ver_separator', '_' ) - ) - - if not sepa: - # file name unexpected, tarball extraction will (probably) fail - LOGGER.error ( "unexpected file name '%s'." % filename ) - raise Exception ( "cannot use file '%s'." % filename ) - return - - - self ['filepath'] = filepath - self ['package_file'] = package_file - self ['package_dir' ] = os.path.dirname ( filepath ) - self ['filename'] = filename - self ['package_name'] = package_name - self ['package_version'] = package_version - # --- end of _use_filepath (...) --- - - def _use_ebuild ( self, ebuild ): - """auxiliary method for update(**kw) - - arguments: - * ebuild -- - """ - self ['has_suggests'] = ebuild.has_rsuggests - # todo move Ebuild funcs to here - self ['ebuild_dir'] = ebuild.suggest_dir_name() - self ['ebuild_name'] = ebuild.suggest_name() diff --git a/roverlay/static/__init__.py b/roverlay/static/__init__.py new file mode 100644 index 0000000..13e7869 --- /dev/null +++ b/roverlay/static/__init__.py @@ -0,0 +1,5 @@ +# R Overlay -- not signleton but static access to certain objects +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# currently used for testing diff --git a/roverlay/static/depres.py b/roverlay/static/depres.py new file mode 100644 index 0000000..a351f51 --- /dev/null +++ b/roverlay/static/depres.py @@ -0,0 +1,29 @@ +# R Overlay -- dependency resolution, static resolver access +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from roverlay.depres.channels import EbuildJobChannel +from roverlay.depres.depresolver import DependencyResolver + +_RESOLVER = None + +def resolver(): + """Returns the resolver.""" + global _RESOLVER + if _RESOLVER is None: + _RESOLVER = DependencyResolver() + return _RESOLVER +# --- end of resolver (...) --- + +def get_ebuild_channel ( name=None, logger=None ): + """Returns a communication channel to the dependency resolver. + + arguments: + name -- + logger -- + """ + return resolver().register_channel ( + EbuildJobChannel ( name=name, logger=logger ) + ) + +# --- end of get_resolver_channel (...) --- -- cgit v1.2.3-65-gdbad