Move package laymandbtools back to layman.dbtools

2 files changed, 358 insertions, 0 deletions

diff --git a/layman/dbtools/gitoliteextractor.py b/layman/dbtools/gitoliteextractor.py
new file mode 100755
index 0000000..0dbc344
--- /dev/null
+++ b/layman/dbtools/gitoliteextractor.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+# Licensed under GPL 2 or later
+
+import sys
+from optparse import OptionParser
+USAGE = 'Usage: %prog  [--fixes-only]  conf/user.conf conf/proj.conf bar/repositories.xml [baz/extended.xml]'
+parser = OptionParser(usage=USAGE)
+parser.add_option('--fixes-only',
+                  dest = 'fixes_only',
+                  default = False,
+                  action = 'store_true',
+                  help = 'do not add entries that are missing completely')
+(opts, args) = parser.parse_args()
+if len(args) not in (3, 4):
+    parser.print_help()
+    sys.exit(1)
+gitolite_conf_locations = args[0:2]
+repositories_xml_location = args[2]
+extended_xml_location = (len(args) == 4) and args[3] or None
+
+import xml.etree.ElementTree as ET
+from ConfigParser import ConfigParser
+import re
+
+from layman.dbtools.sharedutils import * # local
+from layman.dbtools.gitoliteparser import RepoDatabase
+
+
+# From gitolite config
+# ..to repositories.xml
+repo_name_mapping = {
+    'ruby-overlay':'ruby',
+    'sci':'science',
+    'perl-overlay':'perl-experimental',
+    'xfce':'xfce-dev',
+    'gnome-perf':'leio-gnome-perf',
+    'flameeyes':'flameeyes-overlay',
+}
+
+
+def sort_as_in(elements, tag_order):
+    order_map = dict((v, i) for i, v in enumerate(tag_order))
+    deco = (t for t in enumerate(elements))
+    deco_sorted = sorted(deco, key=lambda (i, v): (order_map[v.tag], i))
+    return list(v for _, v in deco_sorted)
+
+class ChangeLog:
+    def __init__(self, gitolite_repo_name, is_new):
+        self.empty = True
+        self.gitolite_repo_name = gitolite_repo_name
+        self.is_new = is_new
+
+    def log(self, kind, details):
+        if self.empty:
+            if self.is_new:
+                print 'Repo "%s" missing completely' % self.gitolite_repo_name
+            else:
+                print 'Analyzing repo "%s":' % self.gitolite_repo_name
+            self.empty = False
+
+        if not self.is_new:
+            print '- Missing %s "%s"' % (kind, details)
+
+
+OWNER_REGEX = re.compile('^([^<]+) (?:\([^)]+\) )?<([^ ]+@[^ ]+)>$')
+NOT_AN_OVERLAY_MESSAGE = 'Skipping %s (not an overlay)'
+
+
+gitolite_conf = RepoDatabase()
+for filename in gitolite_conf_locations:
+    gitolite_conf.feed(filename)
+
+
+a = ET.parse(open(repositories_xml_location))
+repositories = a.getroot()
+overlays_gentoo_org_dict = dict([[e.find('name').text, e] for e in repositories])
+
+def oct_string_to_int(os):
+    l = len(os)
+    res = 0
+    for i, v in enumerate(os):
+        res = res + (8**(l-1-i)) * int(v)
+    return res
+
+assert oct_string_to_int('0713') == 0713
+assert oct_string_to_int('103') == 0103
+
+
+# GLOBAL_DIRMODE = oct_string_to_int(gitosis_conf.get('gitosis', 'dirmode'))
+# GLOBAL_GITWEB = gitosis_conf.getboolean('gitosis', 'gitweb')
+
+def is_public(section_name):
+    #local_dirmode = GLOBAL_DIRMODE
+    #if gitosis_conf.has_option(section_name, 'dirmode'):
+    #    local_dirmode = oct_string_to_int(gitosis_conf.get(section_name, 'dirmode'))
+
+    #local_gitweb = GLOBAL_GITWEB
+    #if gitosis_conf.has_option(section_name, 'gitweb'):
+    #    local_gitweb = gitosis_conf.getboolean(section_name, 'gitweb')
+
+    return True  # ((local_dirmode & 0005) == 0005) and local_gitweb
+
+
+for section_name in gitolite_conf.names():
+    if True:
+        _repo_base = section_name
+
+        try:
+            owner_part, gitolite_repo_name = _repo_base.split('/')
+        except (ValueError) as e:
+            # TODO print NOT_AN_OVERLAY_MESSAGE % gitolite_repo_name
+            continue
+
+        if owner_part == 'proj':
+            owner_type = "project"
+        elif owner_part in ('dev', 'user'):
+            owner_type = "person"
+        else:
+            # TODO print NOT_AN_OVERLAY_MESSAGE % gitolite_repo_name
+            continue
+
+        terms_status, is_overlay, dont_add_to_layman, owner_contact, _description = gitolite_conf.data(section_name)
+
+        if dont_add_to_layman:
+            continue
+
+        overlay_status_clear = False
+        if not is_overlay is None:
+            if not is_overlay:
+                continue
+            overlay_status_clear = True
+
+        if not overlay_status_clear \
+                and not gitolite_repo_name.endswith('overlay') \
+                and not _description.lower().endswith('overlay'):
+            continue
+
+        if not is_public(section_name):
+            # TODO print 'Skipping %s (not public)' % gitolite_repo_name
+            continue
+
+        repositores_xml_repo_name = repo_name_mapping.get(gitolite_repo_name, gitolite_repo_name)
+        is_new = repositores_xml_repo_name not in overlays_gentoo_org_dict
+        if is_new:
+            if opts.fixes_only:
+                continue
+            repo = ET.Element('repo')
+            repositories.append(repo)
+            name = ET.Element('name')
+            name.text = gitolite_repo_name
+            repo.append(name)
+        else:
+            repo = overlays_gentoo_org_dict[repositores_xml_repo_name]
+        log = ChangeLog(gitolite_repo_name, is_new)
+
+        if 'status' not in repo.attrib:
+            if owner_part == 'user':
+                repo.attrib['status'] = 'unofficial'
+            else:
+                repo.attrib['status'] = 'official'
+            log.log('attribute', 'status')
+
+        if 'quality' not in repo.attrib:
+            repo.attrib['quality'] = 'experimental'
+            log.log('attribute', 'quality')
+
+        homepage = repo.find('homepage')
+        if homepage == None:
+            homepage = ET.Element('homepage')
+            homepage.text = 'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=summary' % _repo_base
+            repo.append(homepage)
+            log.log('homepage', homepage.text)
+
+        description = repo.find('description')
+        if description == None:
+            description = ET.Element('description', lang='en')
+            description.text = _description
+            repo.append(description)
+            log.log('description', _description)
+
+
+        _owner = owner_contact
+        _owner_match = OWNER_REGEX.match(_owner)
+
+        owner = repo.find('owner')
+        if owner == None:
+            owner = ET.Element('owner', type=owner_type)
+            repo.append(owner)
+            log.log('owner', 'TODO')
+
+        owner_name = owner.find('name')
+        if owner_name == None:
+            owner_name = ET.Element('name')
+            owner_name.text = _owner_match.group(1)
+            log.log('owner name', owner_name.text)
+
+        owner_email = owner.find('email')
+        if owner_email == None:
+            owner_email = ET.Element('email')
+            owner_email.text = _owner_match.group(2)
+            log.log('owner email', owner_email.text)
+
+        owner[:] = [owner_email, owner_name]
+
+
+        _sources = set((source.attrib['type'], source.text) for source in repo.findall('source'))
+        source_uris = (
+            'git://git.overlays.gentoo.org/%s.git' % _repo_base,
+            'http://git.overlays.gentoo.org/gitroot/%s.git' % _repo_base,
+            'git+ssh://git@git.overlays.gentoo.org/%s.git' % _repo_base,
+        )
+        for uri in source_uris:
+            if ('git', uri) not in _sources and \
+                    ('git', uri[:-len('.git')]) not in _sources:
+                source = ET.Element('source', type='git')
+                source.text = uri
+                repo.append(source)
+                log.log('git source', uri)
+
+
+        _feeds = set(feed.text for feed in repo.findall('feed'))
+        feed_uris = (
+            'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=atom' % _repo_base,
+            'http://git.overlays.gentoo.org/gitweb/?p=%s.git;a=rss' % _repo_base,
+        )
+        for uri in feed_uris:
+            if uri not in _feeds:
+                feed = ET.Element('feed')
+                feed.text = uri
+                repo.append(feed)
+                log.log('feed', uri)
+
+        repo[:] = sort_as_in(repo[:], (
+                'name', 'description', 'longdescription',
+                'homepage', 'owner', 'source', 'feed'))
+
+        if is_new or not log.empty:
+            if extended_xml_location == None:
+                TERM_WIDTH = 67
+                print '-'*TERM_WIDTH
+                sys.stdout.write('  ')
+                indent(repo, 1)
+                repo.tail = '\n'
+                ET.ElementTree(repo).write(sys.stdout)
+                print '-'*TERM_WIDTH
+            print
+
+
+if extended_xml_location != None:
+    indent(repositories)
+    extended_xml = open(extended_xml_location, 'w')
+    extended_xml.write("""\
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- $Header$ -->
+<?xml-stylesheet href="/xsl/repositories.xsl" type="text/xsl"?>
+<!DOCTYPE repositories SYSTEM "/dtd/repositories.dtd">
+""")
+    a.write(extended_xml, encoding='utf-8')
+    extended_xml.close()
diff --git a/layman/dbtools/gitoliteparser.py b/layman/dbtools/gitoliteparser.py
new file mode 100644
index 0000000..d63173c
--- /dev/null
+++ b/layman/dbtools/gitoliteparser.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Gentoo Foundation
+# Written by Sebastian Pipping <sping@gentoo.org>
+#
+# Licensed under GPL v2 or later
+
+from __future__ import print_function
+import re
+
+
+_repo_line = re.compile('^repo ([^ ]+)')
+_terms_status_line = re.compile('^# gentoo-terms-status = (.+)$')
+_overlay_marker_line = re.compile('^# gentoo-is-overlay = (True|False)')
+_desc_line = re.compile('^([^ ]+) "(.+ <[^@]+@[^>]+>)" = "(.+)"')
+_dont_add_line = re.compile('^# gentoo-dont-add-to-layman = (.+)')
+
+
+def _parse_bool(text):
+    if text == 'False':
+        return False
+    else:
+        assert(text == 'True')
+        return True
+
+
+class RepoDatabase:
+    def __init__(self):
+        self._db = dict()
+
+    def _add(self, repo, terms_status, is_overlay, dont_add_reason):
+        if not repo or repo in self._db:
+            return
+        self._db[repo] = (terms_status, is_overlay, dont_add_reason, None, None)
+
+    def _describe(self, repo, contact, desc):
+        self._db[repo] = self._db[repo][0:3] + (contact, desc)
+
+    def names(self):
+        return self._db.keys()
+
+    def data(self, repo):
+        return self._db[repo]
+
+    def feed(self, conf_filename):
+        f = open(conf_filename, 'r')
+
+        repo = None
+        terms_status = None
+        is_overlay = None
+        dont_add_reason = None
+
+        desc_map = dict()
+        for l in f:
+            line = l.rstrip('\n').lstrip()
+            for matcher in (_repo_line, _terms_status_line, _overlay_marker_line, _desc_line, _dont_add_line):
+                m = matcher.search(line)
+                if m:
+                    if matcher is _repo_line:
+                        self._add(repo, terms_status, is_overlay, dont_add_reason)
+                        repo = m.group(1)
+                        terms_status = None
+                        is_overlay = None
+                        dont_add_reason = None
+
+                    elif matcher is _terms_status_line:
+                        terms_status = m.group(1)
+
+                    elif matcher is _overlay_marker_line:
+                        is_overlay = _parse_bool(m.group(1))
+
+                    elif matcher is _desc_line:
+                        desc_repo = m.group(1)
+                        desc_contact = m.group(2)
+                        desc_desc = m.group(3)
+                        desc_map[desc_repo] = (desc_contact, desc_desc)
+
+                    elif matcher is _dont_add_line:
+                        dont_add_reason = m.group(1)
+
+        self._add(repo, terms_status, is_overlay, dont_add_reason)
+
+        for desc_repo, (desc_contact, desc_desc) in desc_map.items():
+            self._describe(desc_repo, desc_contact, desc_desc)
+        f.close()
+
+    def _dump(self):
+        for repo, (terms_status, is_overlay, dont_add_reason, contact, desc) in sorted(self._db.items()):
+            print('repo %s' % repo)
+            if terms_status:
+                print('\t# gentoo-terms-status = %s' % terms_status)
+            if is_overlay:
+                print('\t# gentoo-is-overlay = %s' % str(is_overlay))
+            if dont_add_reason:
+                print('\t# gentoo-dont-add-to-layman = %s' % dont_add_reason)
+            if contact and desc:
+                print('\t%s "%s" = "%s"' % (repo, contact, desc))
+            print()