[g_sorcery/package_db] new DB syncing

13 files changed, 247 insertions, 62 deletions

diff --git a/docs/developer_instructions.html b/docs/developer_instructions.html
index 4cea319..4af5e35 100644
--- a/docs/developer_instructions.html
+++ b/docs/developer_instructions.html
@@ -581,7 +581,8 @@ backends). PackageDB class API should be used instead.</p>
 to add categories and packages and to do queries on them. Usually you do not want to customize this
 class.</p>
 <p>If you have a database that should be synced with another already generate database
-you can redifine URI to be used for syncing using <strong>get_real_db_uri</strong> method.</p>
+you can use <strong>sync</strong> method. Two sync methods are available
+currently: <strong>tgz</strong> and <strong>git</strong>.</p>
 <p>Note that before add any package you should add a category for it using <strong>add_category</strong>.
 Then packages can be added using <strong>add_package</strong>. PackageDB currently does not write changes
 automatically, so you should call <strong>write</strong> after changes are done. This is not relevant
diff --git a/docs/developer_instructions.rst b/docs/developer_instructions.rst
index f0667ea..f868d2f 100644
--- a/docs/developer_instructions.rst
+++ b/docs/developer_instructions.rst
@@ -242,7 +242,8 @@ to add categories and packages and to do queries on them. Usually you do not wan
 class.
 
 If you have a database that should be synced with another already generate database
-you can redifine URI to be used for syncing using **get_real_db_uri** method.
+you can use **sync** method. Two sync methods are available
+currently: **tgz** and **git**.
 
 Note that before add any package you should add a category for it using **add_category**.
 Then packages can be added using **add_package**. PackageDB currently does not write changes
diff --git a/docs/g-sorcery.8 b/docs/g-sorcery.8
index 6782c98..c2f5cbb 100644
--- a/docs/g-sorcery.8
+++ b/docs/g-sorcery.8
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH G-SORCERY 8 "2015-04-20" "0.2" "g-sorcery"
+.TH G-SORCERY 8 "2015-04-20" "0.2.1" "g-sorcery"
 .SH NAME
 g-sorcery \- manage overlays for 3rd party software providers
 .
diff --git a/docs/g-sorcery.8.rst b/docs/g-sorcery.8.rst
index 1e9af4e..2ace6b0 100644
--- a/docs/g-sorcery.8.rst
+++ b/docs/g-sorcery.8.rst
@@ -11,7 +11,7 @@ manage overlays for 3rd party software providers
 	 by Brian Dolbec. Integration with layman based on work of Auke Booij.
 :Date:   2015-04-20
 :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2
-:Version: 0.2
+:Version: 0.2.1
 :Manual section: 8
 :Manual group: g-sorcery
 
diff --git a/docs/g-sorcery.cfg.8 b/docs/g-sorcery.cfg.8
index ea97ada..f9c02ce 100644
--- a/docs/g-sorcery.cfg.8
+++ b/docs/g-sorcery.cfg.8
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH G-SORCERY.CFG 8 "2015-04-20" "0.2" "g-sorcery"
+.TH G-SORCERY.CFG 8 "2015-04-20" "0.2.1" "g-sorcery"
 .SH NAME
 g-sorcery.cfg \- custom settings for g-sorcery
 .
diff --git a/docs/g-sorcery.cfg.8.rst b/docs/g-sorcery.cfg.8.rst
index f9b5aac..bd85d08 100644
--- a/docs/g-sorcery.cfg.8.rst
+++ b/docs/g-sorcery.cfg.8.rst
@@ -11,7 +11,7 @@ custom settings for g-sorcery
 	 by Brian Dolbec.
 :Date:   2015-04-20
 :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2
-:Version: 0.2
+:Version: 0.2.1
 :Manual section: 8
 :Manual group: g-sorcery
 
diff --git a/g_sorcery/backend.py b/g_sorcery/backend.py
index e606348..809ff18 100644
--- a/g_sorcery/backend.py
+++ b/g_sorcery/backend.py
@@ -4,10 +4,10 @@
 """
     backend.py
     ~~~~~~~~~~
-    
+
     base class for backends
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -30,7 +30,7 @@ class Backend(object):
 
     Command format is as follows:
     g-backend [-o overlay_dir] [-r repository] command
-    
+
     where command is one of the following:
     sync
     list
@@ -38,11 +38,11 @@ class Backend(object):
     generate package_name
     generate-tree [-d --digest]
     install package_name [portage flags]
-    
+
     If no overlay directory is given the default one from backend config is used.
     """
-    
-    def __init__(self, package_db_generator_class, 
+
+    def __init__(self, package_db_generator_class,
                  ebuild_g_with_digest_class, ebuild_g_without_digest_class,
                  eclass_g_class, metadata_g_class,
                  package_db_class=PackageDB, sync_db=False):
@@ -74,7 +74,7 @@ class Backend(object):
         p_generate_tree = subparsers.add_parser('generate-tree')
         p_generate_tree.add_argument('-d', '--digest', action='store_true')
         p_generate_tree.set_defaults(func=self.generate_tree)
-        
+
         p_install = subparsers.add_parser('install')
         p_install.add_argument('pkgname')
         p_install.add_argument('pkgmanager_flags', nargs=argparse.REMAINDER)
@@ -150,7 +150,7 @@ class Backend(object):
 
         if repository:
             if not "repositories" in config:
-                self.logger.error("repository " + repository + 
+                self.logger.error("repository " + repository +
                     " specified, but there is no repositories entry in config")
                 return -1
             repositories = config["repositories"]
@@ -161,11 +161,15 @@ class Backend(object):
         else:
             self.logger.error('no repository given\n')
             return -1
-                
+
+        try:
+            sync_method = repository_config["sync_method"]
+        except KeyError:
+            sync_method = "tgz"
         if self.sync_db:
             pkg_db = self.package_db_generator(backend_path, repository,
                             common_config, repository_config, generate=False)
-            pkg_db.sync(repository_config["db_uri"])
+            pkg_db.sync(repository_config["db_uri"], repository_config=repository_config, sync_method=sync_method)
         else:
             pkg_db = self.package_db_generator(backend_path,
                             repository, common_config, repository_config)
@@ -227,7 +231,7 @@ class Backend(object):
         except Exception as e:
             self.logger.error('dependency solving failed: ' + str(e) + '\n')
             return -1
-        
+
         eclasses = []
         for package in dependencies:
             eclasses += pkg_db.get_package_description(package)['eclasses']
@@ -409,17 +413,17 @@ class Backend(object):
             try:
                 versions = package_db.list_package_versions(pkg.category,
                                                         pkg.package)
-                for version in versions:            
+                for version in versions:
                     solved_deps, unsolved_deps = self.solve_dependencies(package_db,
                                     Package(pkg.category, pkg.package, version),
                                     solved_deps, unsolved_deps)
             except InvalidKeyError:
                 # ignore non existing packages
                 continue
-        
+
         solved_deps.add(package)
         unsolved_deps.remove(package)
-        
+
         return (solved_deps, unsolved_deps)
 
 
@@ -449,7 +453,7 @@ class Backend(object):
         for pkgname in pkgnames:
             directory = os.path.join(overlay, pkgname)
             fast_manifest(directory)
-        
+
     def generate_tree(self, args, config, global_config):
         """
         Generate entire overlay.
@@ -509,7 +513,7 @@ class Backend(object):
         with open(os.path.join(overlay, 'metadata', 'layout.conf'), 'w') as f:
             f.write("repo-name = %s\n" % os.path.basename(overlay))
             f.write("masters = %s\n" % masters_overlays)
-        
+
         if args.digest:
             ebuild_g = self.ebuild_g_with_digest_class(pkg_db)
         else:
@@ -566,6 +570,13 @@ class Backend(object):
             self.fast_digest(overlay, pkgnames)
         overlays.write(overlays_info)
 
+        try:
+            clean_db = config["repositories"][args.repository]["clean_db"]
+        except KeyError:
+            clean_db = False
+        if clean_db:
+            pkg_db.clean()
+
     def install(self, args, config, global_config):
         """
         Install a package.
@@ -592,7 +603,7 @@ class Backend(object):
             package_manager_class = package_managers[package_manager]
         package_manager = package_manager_class()
         package_manager.install(args.pkgname, *args.pkgmanager_flags)
-        
+
     def __call__(self, args, config, global_config):
         """
         Execute a command
diff --git a/g_sorcery/git_syncer/__init__.py b/g_sorcery/git_syncer/__init__.py
new file mode 100644
index 0000000..4265cc3
--- /dev/null
+++ b/g_sorcery/git_syncer/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/g_sorcery/git_syncer/git_syncer.py b/g_sorcery/git_syncer/git_syncer.py
new file mode 100644
index 0000000..0f6c58e
--- /dev/null
+++ b/g_sorcery/git_syncer/git_syncer.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    git_syncer.py
+    ~~~~~~~~~~~~~
+
+    git sync helper
+
+    :copyright: (c) 2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import os
+
+from g_sorcery.compatibility import TemporaryDirectory
+
+from g_sorcery.exceptions import SyncError
+from g_sorcery.syncer import Syncer, SyncedData, TmpSyncedData
+
+
+class GITSyncer(Syncer):
+    """
+    Class used to sync with git repos.
+    """
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        if self.persistent_datadir is None:
+            tmp_dir = TemporaryDirectory()
+            path = os.path.join(tmp_dir.name, "remote")
+        else:
+            path = self.persistent_datadir
+        try:
+            branch = repository_config["branch"]
+        except KeyError:
+            branch = "master"
+
+        if os.path.exists(path):
+            #TODO: allow changing of remotes/branches
+            self.pull(path)
+        else:
+            self.clone(db_uri, branch, path)
+
+        if self.persistent_datadir is None:
+            return TmpSyncedData(path, tmp_dir)
+        else:
+            return SyncedData(path)
+
+
+    def clone(self, db_uri, branch, path):
+        if os.system("git clone --depth 1 --branch " + branch + " " + db_uri + " " + path):
+            raise SyncError("sync failed (clonning): " + db_uri)
+
+
+    def pull(self, path):
+        if os.system("cd " + path + " && git pull"):
+            raise SyncError("sync failed (pulling): " + path)
diff --git a/g_sorcery/package_db.py b/g_sorcery/package_db.py
index a88474d..8daa665 100644
--- a/g_sorcery/package_db.py
+++ b/g_sorcery/package_db.py
@@ -11,18 +11,18 @@
     :license: GPL-2, see LICENSE for more details.
 """
 
-import glob
 import os
 
 import portage
 
-from .compatibility import basestring, py2k, TemporaryDirectory
+from .compatibility import basestring, py2k
 
 from .db_layout import DBLayout, JSON_FILE_SUFFIX, SUPPORTED_DB_LAYOUTS, SUPPORTED_FILE_FORMATS
 from .exceptions import DBError, DBLayoutError, DBStructureError, InvalidKeyError, SyncError
-from .fileutils import FileJSON, load_remote_file, copy_all, wget
+from .fileutils import FileJSON, load_remote_file, copy_all
 from .g_collections import Package
 from .logger import Logger
+from .syncer import SUPPORTED_SYNCERS
 
 SUPPORTED_DB_STRUCTURES=[0, 1]
 
@@ -137,6 +137,7 @@ class PackageDB(object):
 
 
     def __init__(self, directory,
+                 persistent_datadir = None,
                  preferred_layout_version=1,
                  preferred_db_version=1,
                  preferred_category_format=JSON_FILE_SUFFIX):
@@ -157,6 +158,11 @@ class PackageDB(object):
 
         self.logger = Logger()
         self.directory = os.path.abspath(directory)
+
+        self.persistent_datadir = persistent_datadir
+        if self.persistent_datadir is not None:
+            self.persistent_datadir = os.path.abspath(self.persistent_datadir)
+
         self.preferred_layout_version = preferred_layout_version
         self.preferred_db_version = preferred_db_version
         self.preferred_category_format = preferred_category_format
@@ -176,24 +182,30 @@ class PackageDB(object):
         self.categories = {}
 
 
-    def sync(self, db_uri):
+    def sync(self, db_uri, repository_config = None, sync_method="tgz"):
         """
         Synchronize local database with remote database.
 
         Args:
             db_uri: URI for synchronization with remote database.
-        """
-        real_db_uri = self.get_real_db_uri(db_uri)
-        download_dir = TemporaryDirectory()
-        if wget(real_db_uri, download_dir.name):
-            raise SyncError('sync failed: ' + real_db_uri)
-
-        temp_dir = TemporaryDirectory()
-        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
-            self.logger.info("unpacking " + f_name)
-            os.system("tar -xvzf " + f_name + " -C " + temp_dir.name)
+            repository_config: repository config.
+            sync_method: sync method (tgz or git).
+        """
+        if repository_config is None:
+            repository_config = {}
+
+        try:
+            syncer_cls = SUPPORTED_SYNCERS[sync_method]
+        except KeyError:
+            raise SyncError('unsupported sync method: ' + sync_method)
+        if self.persistent_datadir is not None:
+            remotedb_dir = os.path.join(self.persistent_datadir, 'remote')
+        else:
+            remotedb_dir = None
+        syncer = syncer_cls(remotedb_dir)
+        synced_data = syncer.sync(db_uri, repository_config)
 
-        tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0])
+        tempdb_dir = synced_data.get_path()
         tempdb = PackageDB(tempdb_dir)
 
         tempdb.db_layout.check_manifest()
@@ -204,19 +216,7 @@ class PackageDB(object):
 
         self.db_layout.check_manifest()
 
-        del download_dir
-        del temp_dir
-
-
-    def get_real_db_uri(self, db_uri):
-        """
-        Convert self.db_uri to URI where remote database can be
-        fetched from.
-
-        Returns:
-            URI of remote database file.
-        """
-        return db_uri
+        del synced_data
 
 
     def clean(self):
@@ -538,10 +538,12 @@ class DBGenerator(object):
             Package database.
         """
         db_path = os.path.join(directory, repository, "db")
+        persistent_datadir = os.path.join(directory, repository, "persistent")
         pkg_db = self.package_db_class(db_path,
                                        preferred_layout_version=self.preferred_layout_version,
                                        preferred_db_version=self.preferred_db_version,
-                                       preferred_category_format=self.preferred_category_format)
+                                       preferred_category_format=self.preferred_category_format,
+                                       persistent_datadir=persistent_datadir)
 
         config_f = FileJSON(os.path.join(directory, repository),
                             "config.json", [])
diff --git a/g_sorcery/syncer.py b/g_sorcery/syncer.py
new file mode 100644
index 0000000..beeffd4
--- /dev/null
+++ b/g_sorcery/syncer.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    syncer.py
+    ~~~~~~~~~
+
+    sync helper
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import glob
+import os
+
+from .compatibility import TemporaryDirectory
+
+from .exceptions import SyncError
+from .fileutils import wget
+
+
+class SyncedData(object):
+    """
+    Synced data.
+
+    Directory with sync data is guaranted to exist only as long as this
+    object does.
+    """
+    def __init__(self, directory):
+        self.directory = os.path.abspath(directory)
+
+    def get_path(self):
+        return self.directory
+
+
+class TmpSyncedData(SyncedData):
+    """
+    Synced data that lives in a temporary directory.
+    """
+
+    def __init__(self, directory, tmpdirobj):
+        super(TmpSyncedData, self).__init__(directory)
+        self.tmpdirobj = tmpdirobj
+
+
+class Syncer(object):
+    """
+    Class used to sync data with remote source.
+    """
+
+    def __init__(self, persistent_datadir):
+        self.persistent_datadir = persistent_datadir
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        raise NotImplementedError
+
+
+class TGZSyncer(Syncer):
+    """
+    Class used to download and unpack tarballs.
+    """
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        download_dir = TemporaryDirectory()
+        if wget(db_uri, download_dir.name):
+            raise SyncError('sync failed: ' + db_uri)
+
+        tmp_dir = TemporaryDirectory()
+        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
+            if os.system("tar -xvzf " + f_name + " -C " + tmp_dir.name):
+                raise SyncError('sync failed (unpacking)')
+
+        tmp_path = os.path.join(tmp_dir.name, os.listdir(tmp_dir.name)[0])
+        del download_dir
+        return TmpSyncedData(tmp_path, tmp_dir)
+
+
+SUPPORTED_SYNCERS = {"tgz": TGZSyncer}
+
+# git_syncer module is optional, we should check if it is installed
+try:
+    from .git_syncer.git_syncer import GITSyncer
+    SUPPORTED_SYNCERS["git"] = GITSyncer
+
+except ImportError as e:
+    pass
diff --git a/setup.py b/setup.py
index 68ad410..b7345a6 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ import os
 
 from distutils.core import setup
 
-SELECTABLE = {'bson': 'file_bson'}
+SELECTABLE = {'bson': 'file_bson', 'git': 'git_syncer'}
 
 use_defaults = ' '.join(list(SELECTABLE))
 USE = os.environ.get("USE", use_defaults).split()
@@ -29,7 +29,7 @@ for mod in SELECTABLE:
         optional_modules.append('g_sorcery.%s' % SELECTABLE[mod])
 
 setup(name          = 'g-sorcery',
-      version       = '0.2',
+      version       = '0.2.1',
       description   = 'framework for automated ebuild generators',
       author        = 'Jauhien Piatlicki',
       author_email  = 'jauhien@gentoo.org',
diff --git a/tests/test_PackageDB.py b/tests/test_PackageDB.py
index 179cc32..152c605 100644
--- a/tests/test_PackageDB.py
+++ b/tests/test_PackageDB.py
@@ -34,17 +34,12 @@ except ImportError as e:
     pass
 
 
-class TestDB(PackageDB):
-    def get_real_db_uri(self, db_uri):
-        return db_uri + "/dummy.tar.gz"
-
-
 class TestPackageDB(BaseTest):
 
     def test_functionality(self):
         port = 8080
         for fmt in SUPPORTED_FILE_FORMATS:
-            sync_address = "127.0.0.1:" + str(port)
+            sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz"
             orig_tempdir = TemporaryDirectory()
             orig_path = os.path.join(orig_tempdir.name, "db")
             os.makedirs(orig_path)
@@ -70,7 +65,7 @@ class TestPackageDB(BaseTest):
             os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt)
             os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db")
 
-            test_db = TestDB(self.tempdir.name)
+            test_db = PackageDB(self.tempdir.name)
             self.assertRaises(SyncError, test_db.sync, sync_address)
 
             srv = Server(orig_tempdir.name, port=port)