diff options
Diffstat (limited to 'lib/portage/cache')
| -rw-r--r-- | lib/portage/cache/anydbm.py | 26 | ||||
| -rw-r--r-- | lib/portage/cache/cache_errors.py | 17 | ||||
| -rw-r--r-- | lib/portage/cache/ebuild_xattr.py | 17 | ||||
| -rw-r--r-- | lib/portage/cache/flat_hash.py | 22 | ||||
| -rw-r--r-- | lib/portage/cache/fs_template.py | 7 | ||||
| -rw-r--r-- | lib/portage/cache/index/IndexStreamIterator.py | 4 | ||||
| -rw-r--r-- | lib/portage/cache/index/meson.build | 9 | ||||
| -rw-r--r-- | lib/portage/cache/index/pkg_desc_index.py | 3 | ||||
| -rw-r--r-- | lib/portage/cache/mappings.py | 329 | ||||
| -rw-r--r-- | lib/portage/cache/meson.build | 20 | ||||
| -rw-r--r-- | lib/portage/cache/metadata.py | 20 | ||||
| -rw-r--r-- | lib/portage/cache/sql_template.py | 28 | ||||
| -rw-r--r-- | lib/portage/cache/sqlite.py | 44 | ||||
| -rw-r--r-- | lib/portage/cache/template.py | 41 | ||||
| -rw-r--r-- | lib/portage/cache/volatile.py | 3 |
15 files changed, 328 insertions, 262 deletions
diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py index ce0077e3f..ad7042ae4 100644 --- a/lib/portage/cache/anydbm.py +++ b/lib/portage/cache/anydbm.py @@ -1,4 +1,4 @@ -# Copyright 2005-2020 Gentoo Authors +# Copyright 2005-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) @@ -18,7 +18,6 @@ from portage.cache import cache_errors class database(fs_template.FsBased): - validation_chf = "md5" chf_types = ("md5", "mtime") @@ -27,7 +26,7 @@ class database(fs_template.FsBased): serialize_eclasses = False def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) + super().__init__(*args, **config) default_db = config.get("dbtype", "anydbm") if not default_db.startswith("."): @@ -50,12 +49,12 @@ class database(fs_template.FsBased): try: self._ensure_dirs() self._ensure_dirs(self._db_path) - except (OSError, IOError) as e: + except OSError as e: raise cache_errors.InitializationError(self.__class__, e) # try again if failed try: - if self.__db == None: + if self.__db is None: # dbm.open() will not work with bytes in python-3.1: # TypeError: can't concat bytes to str if gdbm is None: @@ -68,6 +67,21 @@ class database(fs_template.FsBased): raise cache_errors.InitializationError(self.__class__, e) self._ensure_access(self._db_path) + def __getstate__(self): + state = self.__dict__.copy() + # These attributes are not picklable, so they are automatically + # regenerated after unpickling. + state["_database__db"] = None + return state + + def __setstate__(self, state): + self.__dict__.update(state) + mode = "w" + if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"): + # Allow multiple concurrent writers (see bug #53607). + mode += "u" + self.__db = dbm.open(self._db_path, mode, self._perms) + def iteritems(self): # dbm doesn't implement items() for k in self.__db.keys(): @@ -90,7 +104,7 @@ class database(fs_template.FsBased): return cpv in self.__db def __del__(self): - if "__db" in self.__dict__ and self.__db != None: + if "__db" in self.__dict__ and self.__db is not None: self.__db.sync() self.__db.close() diff --git a/lib/portage/cache/cache_errors.py b/lib/portage/cache/cache_errors.py index 080029734..581c4eed3 100644 --- a/lib/portage/cache/cache_errors.py +++ b/lib/portage/cache/cache_errors.py @@ -12,10 +12,7 @@ class InitializationError(CacheError): self.error, self.class_name = error, class_name def __str__(self): - return "Creation of instance %s failed due to %s" % ( - self.class_name, - str(self.error), - ) + return f"Creation of instance {self.class_name} failed due to {str(self.error)}" class CacheCorruption(CacheError): @@ -23,7 +20,7 @@ class CacheCorruption(CacheError): self.key, self.ex = key, ex def __str__(self): - return "%s is corrupt: %s" % (self.key, str(self.ex)) + return f"{self.key} is corrupt: {str(self.ex)}" class GeneralCacheCorruption(CacheError): @@ -31,17 +28,17 @@ class GeneralCacheCorruption(CacheError): self.ex = ex def __str__(self): - return "corruption detected: %s" % str(self.ex) + return f"corruption detected: {str(self.ex)}" class InvalidRestriction(CacheError): def __init__(self, key, restriction, exception=None): - if exception == None: + if exception is None: exception = "" self.key, self.restriction, self.ex = key, restriction, ex def __str__(self): - return "%s:%s is not valid: %s" % (self.key, self.restriction, str(self.ex)) + return f"{self.key}:{self.restriction} is not valid: {str(self.ex)}" class ReadOnlyRestriction(CacheError): @@ -67,14 +64,14 @@ class StatCollision(CacheError): self.size = size def __str__(self): - return "%s has stat collision with size %s and mtime %s" % ( + return "{} has stat collision with size {} and mtime {}".format( self.key, self.size, self.mtime, ) def __repr__(self): - return "portage.cache.cache_errors.StatCollision(%s)" % ( + return "portage.cache.cache_errors.StatCollision({})".format( ", ".join( (repr(self.key), repr(self.filename), repr(self.mtime), repr(self.size)) ), diff --git a/lib/portage/cache/ebuild_xattr.py b/lib/portage/cache/ebuild_xattr.py index 587466589..86fd5dbf0 100644 --- a/lib/portage/cache/ebuild_xattr.py +++ b/lib/portage/cache/ebuild_xattr.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright: 2009-2020 Gentoo Authors # Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 @@ -21,11 +20,10 @@ class NoValueException(Exception): class database(fs_template.FsBased): - autocommits = True def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) + super().__init__(*args, **config) self.portdir = self.label self.ns = xattr.NS_USER + ".gentoo.cache" self.keys = set(self._known_keys) @@ -57,7 +55,7 @@ class database(fs_template.FsBased): while True: self.__set(path, "test_max", s) s += hundred - except IOError as e: + except OSError as e: # ext based give wrong errno # https://bugzilla.kernel.org/show_bug.cgi?id=12793 if e.errno in (errno.E2BIG, errno.ENOSPC): @@ -67,7 +65,7 @@ class database(fs_template.FsBased): try: self.__remove(path, "test_max") - except IOError as e: + except OSError as e: if e.errno != errno.ENODATA: raise @@ -88,7 +86,7 @@ class database(fs_template.FsBased): def __get(self, path, key, default=None): try: return xattr.get(path, key, namespace=self.ns) - except IOError as e: + except OSError as e: if not default is None and errno.ENODATA == e.errno: return default raise NoValueException() @@ -135,7 +133,7 @@ class database(fs_template.FsBased): parts += 1 # Only the first entry carries the number of parts - self.__set(path, key, "%s:%s" % (parts, s[0:max_len])) + self.__set(path, key, f"{parts}:{s[0:max_len]}") # Write out the rest for i in range(1, parts): @@ -143,7 +141,7 @@ class database(fs_template.FsBased): val = s[start : start + max_len] self.__set(path, key + str(i), val) else: - self.__set(path, key, "%s:%s" % (1, s)) + self.__set(path, key, f"{1}:{s}") def _delitem(self, cpv): pass # Will be gone with the ebuild @@ -152,7 +150,6 @@ class database(fs_template.FsBased): return os.path.exists(self.__get_path(cpv)) def __iter__(self): - for root, dirs, files in os.walk(self.portdir): for file in files: try: @@ -166,4 +163,4 @@ class database(fs_template.FsBased): pn_pv = file[:-7] path = os.path.join(root, file) if self.__has_cache(path): - yield "%s/%s/%s" % (cat, os.path.basename(root), file[:-7]) + yield f"{cat}/{os.path.basename(root)}/{file[:-7]}" diff --git a/lib/portage/cache/flat_hash.py b/lib/portage/cache/flat_hash.py index d3f4dad4c..cc0536276 100644 --- a/lib/portage/cache/flat_hash.py +++ b/lib/portage/cache/flat_hash.py @@ -5,7 +5,6 @@ from portage.cache import fs_template from portage.cache import cache_errors import errno -import io import stat import tempfile import os as _os @@ -17,17 +16,16 @@ from portage.versions import _pkg_str class database(fs_template.FsBased): - autocommits = True def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) + super().__init__(*args, **config) self.location = os.path.join( self.location, self.label.lstrip(os.path.sep).rstrip(os.path.sep) ) write_keys = set(self._known_keys) write_keys.add("_eclasses_") - write_keys.add("_%s_" % (self.validation_chf,)) + write_keys.add(f"_{self.validation_chf}_") self._write_keys = sorted(write_keys) if not self.readonly and not os.path.exists(self.location): self._ensure_dirs() @@ -36,9 +34,8 @@ class database(fs_template.FsBased): # Don't use os.path.join, for better performance. fp = self.location + _os.sep + cpv try: - with io.open( + with open( _unicode_encode(fp, encoding=_encodings["fs"], errors="strict"), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as myf: @@ -51,7 +48,7 @@ class database(fs_template.FsBased): # that uses mtime mangling. d["_mtime_"] = _os.fstat(myf.fileno())[stat.ST_MTIME] return d - except (IOError, OSError) as e: + except OSError as e: if e.errno != errno.ENOENT: raise cache_errors.CacheCorruption(cpv, e) raise KeyError(cpv, e) @@ -66,17 +63,17 @@ class database(fs_template.FsBased): def _setitem(self, cpv, values): try: fd, fp = tempfile.mkstemp(dir=self.location) - except EnvironmentError as e: + except OSError as e: raise cache_errors.CacheCorruption(cpv, e) - with io.open( + with open( fd, mode="w", encoding=_encodings["repo.content"], errors="backslashreplace" ) as myf: for k in self._write_keys: v = values.get(k) if not v: continue - myf.write("%s=%s\n" % (k, v)) + myf.write(f"{k}={v}\n") self._ensure_access(fp) @@ -85,7 +82,7 @@ class database(fs_template.FsBased): new_fp = os.path.join(self.location, cpv) try: os.rename(fp, new_fp) - except EnvironmentError as e: + except OSError as e: success = False try: if errno.ENOENT == e.errno: @@ -93,7 +90,7 @@ class database(fs_template.FsBased): self._ensure_dirs(cpv) os.rename(fp, new_fp) success = True - except EnvironmentError as e: + except OSError as e: raise cache_errors.CacheCorruption(cpv, e) else: raise cache_errors.CacheCorruption(cpv, e) @@ -150,7 +147,6 @@ class database(fs_template.FsBased): class md5_database(database): - validation_chf = "md5" store_eclass_paths = False diff --git a/lib/portage/cache/fs_template.py b/lib/portage/cache/fs_template.py index a3f803740..738bb5417 100644 --- a/lib/portage/cache/fs_template.py +++ b/lib/portage/cache/fs_template.py @@ -21,7 +21,6 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - for x, y in (("gid", -1), ("perms", 0o644)): if x in config: # Since Python 3.4, chown requires int type (no proxies). @@ -29,7 +28,7 @@ class FsBased(template.database): del config[x] else: setattr(self, "_" + x, y) - super(FsBased, self).__init__(*args, **config) + super().__init__(*args, **config) if self.label.startswith(os.path.sep): # normpath. @@ -43,7 +42,7 @@ class FsBased(template.database): if mtime != -1: mtime = int(mtime) os.utime(path, (mtime, mtime)) - except (PortageException, EnvironmentError): + except (PortageException, OSError): return False return True @@ -87,4 +86,4 @@ def gen_label(base, label): label = label.strip('"').strip("'") label = os.path.join(*(label.rstrip(os.path.sep).split(os.path.sep))) tail = os.path.split(label)[1] - return "%s-%X" % (tail, abs(label.__hash__())) + return f"{tail}-{abs(label.__hash__()):X}" diff --git a/lib/portage/cache/index/IndexStreamIterator.py b/lib/portage/cache/index/IndexStreamIterator.py index 78d7f0e5f..616aca0e3 100644 --- a/lib/portage/cache/index/IndexStreamIterator.py +++ b/lib/portage/cache/index/IndexStreamIterator.py @@ -4,20 +4,16 @@ class IndexStreamIterator: def __init__(self, f, parser): - self.parser = parser self._file = f def close(self): - if self._file is not None: self._file.close() self._file = None def __iter__(self): - try: - for line in self._file: node = self.parser(line) if node is not None: diff --git a/lib/portage/cache/index/meson.build b/lib/portage/cache/index/meson.build new file mode 100644 index 000000000..fdc427953 --- /dev/null +++ b/lib/portage/cache/index/meson.build @@ -0,0 +1,9 @@ +py.install_sources( + [ + 'IndexStreamIterator.py', + 'pkg_desc_index.py', + '__init__.py', + ], + subdir : 'portage/cache/index', + pure : not native_extensions +) diff --git a/lib/portage/cache/index/pkg_desc_index.py b/lib/portage/cache/index/pkg_desc_index.py index be81b9bb9..9c63c2559 100644 --- a/lib/portage/cache/index/pkg_desc_index.py +++ b/lib/portage/cache/index/pkg_desc_index.py @@ -33,11 +33,10 @@ class pkg_node(str): def pkg_desc_index_line_format(cp, pkgs, desc): - return "%s %s: %s\n" % (cp, " ".join(_pkg_str(cpv).version for cpv in pkgs), desc) + return f"{cp} {' '.join(_pkg_str(cpv).version for cpv in pkgs)}: {desc}\n" def pkg_desc_index_line_read(line, repo=None): - try: pkgs, desc = line.split(":", 1) except ValueError: diff --git a/lib/portage/cache/mappings.py b/lib/portage/cache/mappings.py index c0f2147a9..469f3dbc4 100644 --- a/lib/portage/cache/mappings.py +++ b/lib/portage/cache/mappings.py @@ -1,4 +1,4 @@ -# Copyright: 2005-2020 Gentoo Authors +# Copyright: 2005-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) @@ -69,7 +69,7 @@ class Mapping: class MutableMapping(Mapping): """ - A mutable vesion of the Mapping class. + A mutable version of the Mapping class. """ __slots__ = () @@ -146,7 +146,6 @@ class UserDict(MutableMapping): __slots__ = ("data",) def __init__(self, *args, **kwargs): - self.data = {} if len(args) > 1: @@ -250,7 +249,7 @@ class LazyLoad(Mapping): def __getitem__(self, key): if key in self.d: return self.d[key] - if self.pull != None: + if self.pull is not None: self.d.update(self.pull()) self.pull = None return self.d[key] @@ -264,7 +263,7 @@ class LazyLoad(Mapping): def __contains__(self, key): if key in self.d: return True - if self.pull != None: + if self.pull is not None: self.d.update(self.pull()) self.pull = None return key in self.d @@ -272,6 +271,175 @@ class LazyLoad(Mapping): keys = __iter__ +class _SlotDict: + """ + Base class for classes returned from slot_dict_class. + """ + + _prefix = "" + allowed_keys = frozenset() + __slots__ = ("__weakref__",) + + def __init__(self, *args, **kwargs): + if len(args) > 1: + raise TypeError( + "expected at most 1 positional argument, got " + repr(len(args)) + ) + + if args: + self.update(args[0]) + + if kwargs: + self.update(kwargs) + + def __reduce__(self): + return _PickledSlotDict, ( + self._prefix, + self.allowed_keys, + dict(self), + ) + + def __eq__(self, other): + return dict(self) == dict(other) + + def __iter__(self): + for k, v in self.iteritems(): + yield k + + def __len__(self): + l = 0 + for i in self.iteritems(): + l += 1 + return l + + def iteritems(self): + prefix = self._prefix + for k in self.allowed_keys: + try: + yield (k, getattr(self, prefix + k)) + except AttributeError: + pass + + def itervalues(self): + for k, v in self.iteritems(): + yield v + + def __delitem__(self, k): + try: + delattr(self, self._prefix + k) + except AttributeError: + raise KeyError(k) + + def __setitem__(self, k, v): + setattr(self, self._prefix + k, v) + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + + def update(self, *args, **kwargs): + if len(args) > 1: + raise TypeError( + "expected at most 1 positional argument, got " + repr(len(args)) + ) + other = None + if args: + other = args[0] + if other is None: + pass + elif hasattr(other, "iteritems"): + # Use getattr to avoid interference from 2to3. + for k, v in getattr(other, "iteritems")(): + self[k] = v + elif hasattr(other, "items"): + # Use getattr to avoid interference from 2to3. + for k, v in getattr(other, "items")(): + self[k] = v + elif hasattr(other, "keys"): + for k in other.keys(): + self[k] = other[k] + else: + for k, v in other: + self[k] = v + if kwargs: + self.update(kwargs) + + def __getitem__(self, k): + try: + return getattr(self, self._prefix + k) + except AttributeError: + raise KeyError(k) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, k): + return hasattr(self, self._prefix + k) + + def pop(self, key, *args): + if len(args) > 1: + raise TypeError( + "pop expected at most 2 arguments, got " + repr(1 + len(args)) + ) + try: + value = self[key] + except KeyError: + if args: + return args[0] + raise + del self[key] + return value + + def popitem(self): + try: + k, v = self.iteritems().next() + except StopIteration: + raise KeyError("container is empty") + del self[k] + return (k, v) + + def copy(self): + c = self.__class__() + c.update(self) + return c + + def clear(self): + for k in self.allowed_keys: + try: + delattr(self, self._prefix + k) + except AttributeError: + pass + + def __str__(self): + return str(dict(self.iteritems())) + + def __repr__(self): + return repr(dict(self.iteritems())) + + items = iteritems + keys = __iter__ + values = itervalues + + +class _PickledSlotDict(_SlotDict): + """ + Since LocalSlotDict instances are not directly picklable, this + class exists as a way to express pickled LocalSlotDict instances, + using a plain __dict__ instead of custom __slots__. + """ + + def __init__(self, prefix, allowed_keys, *args, **kwargs): + self._prefix = prefix + self.allowed_keys = allowed_keys + super().__init__(*args, **kwargs) + + _slot_dict_classes = weakref.WeakValueDictionary() @@ -294,152 +462,15 @@ def slot_dict_class(keys, prefix="_val_"): keys_set = keys else: keys_set = frozenset(keys) - v = _slot_dict_classes.get((keys_set, prefix)) + cache_key = (keys_set, prefix) + v = _slot_dict_classes.get(cache_key) if v is None: - class SlotDict: - + class LocalSlotDict(_SlotDict): allowed_keys = keys_set _prefix = prefix - __slots__ = ("__weakref__",) + tuple(prefix + k for k in allowed_keys) - - def __init__(self, *args, **kwargs): - - if len(args) > 1: - raise TypeError( - "expected at most 1 positional argument, got " + repr(len(args)) - ) - - if args: - self.update(args[0]) - - if kwargs: - self.update(kwargs) - - def __iter__(self): - for k, v in self.iteritems(): - yield k - - def __len__(self): - l = 0 - for i in self.iteritems(): - l += 1 - return l - - def iteritems(self): - prefix = self._prefix - for k in self.allowed_keys: - try: - yield (k, getattr(self, prefix + k)) - except AttributeError: - pass - - def itervalues(self): - for k, v in self.iteritems(): - yield v - - def __delitem__(self, k): - try: - delattr(self, self._prefix + k) - except AttributeError: - raise KeyError(k) - - def __setitem__(self, k, v): - setattr(self, self._prefix + k, v) - - def setdefault(self, key, default=None): - try: - return self[key] - except KeyError: - self[key] = default - return default - - def update(self, *args, **kwargs): - if len(args) > 1: - raise TypeError( - "expected at most 1 positional argument, got " + repr(len(args)) - ) - other = None - if args: - other = args[0] - if other is None: - pass - elif hasattr(other, "iteritems"): - # Use getattr to avoid interference from 2to3. - for k, v in getattr(other, "iteritems")(): - self[k] = v - elif hasattr(other, "items"): - # Use getattr to avoid interference from 2to3. - for k, v in getattr(other, "items")(): - self[k] = v - elif hasattr(other, "keys"): - for k in other.keys(): - self[k] = other[k] - else: - for k, v in other: - self[k] = v - if kwargs: - self.update(kwargs) - - def __getitem__(self, k): - try: - return getattr(self, self._prefix + k) - except AttributeError: - raise KeyError(k) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def __contains__(self, k): - return hasattr(self, self._prefix + k) - - def pop(self, key, *args): - if len(args) > 1: - raise TypeError( - "pop expected at most 2 arguments, got " + repr(1 + len(args)) - ) - try: - value = self[key] - except KeyError: - if args: - return args[0] - raise - del self[key] - return value - - def popitem(self): - try: - k, v = self.iteritems().next() - except StopIteration: - raise KeyError("container is empty") - del self[k] - return (k, v) - - def copy(self): - c = self.__class__() - c.update(self) - return c - - def clear(self): - for k in self.allowed_keys: - try: - delattr(self, self._prefix + k) - except AttributeError: - pass - - def __str__(self): - return str(dict(self.iteritems())) - - def __repr__(self): - return repr(dict(self.iteritems())) - - items = iteritems - keys = __iter__ - values = itervalues - - v = SlotDict - _slot_dict_classes[v.allowed_keys] = v + __slots__ = tuple(prefix + k for k in allowed_keys) + + v = LocalSlotDict + _slot_dict_classes[cache_key] = v return v diff --git a/lib/portage/cache/meson.build b/lib/portage/cache/meson.build new file mode 100644 index 000000000..5ebeda6bb --- /dev/null +++ b/lib/portage/cache/meson.build @@ -0,0 +1,20 @@ +py.install_sources( + [ + 'anydbm.py', + 'cache_errors.py', + 'ebuild_xattr.py', + 'flat_hash.py', + 'fs_template.py', + 'mappings.py', + 'metadata.py', + 'sqlite.py', + 'sql_template.py', + 'template.py', + 'volatile.py', + '__init__.py', + ], + subdir : 'portage/cache', + pure : not native_extensions +) + +subdir('index') diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py index 02d8385e0..791ad8344 100644 --- a/lib/portage/cache/metadata.py +++ b/lib/portage/cache/metadata.py @@ -53,7 +53,7 @@ class database(flat_hash.database): def __init__(self, location, *args, **config): loc = location - super(database, self).__init__(location, *args, **config) + super().__init__(location, *args, **config) self.location = os.path.join(loc, "metadata", "cache") self.ec = None self.raise_stat_collision = False @@ -83,9 +83,9 @@ class database(flat_hash.database): getter = attrgetter(self.validation_chf) try: ec_data = self.ec.get_eclass_data(d["INHERITED"].split()) - d["_eclasses_"] = dict( - (k, (v.eclass_dir, getter(v))) for k, v in ec_data.items() - ) + d["_eclasses_"] = { + k: (v.eclass_dir, getter(v)) for k, v in ec_data.items() + } except KeyError as e: # INHERITED contains a non-existent eclass. raise cache_errors.CacheCorruption(cpv, e) @@ -120,7 +120,7 @@ class database(flat_hash.database): _unicode_encode(new_fp, encoding=_encodings["fs"], errors="strict"), "rb", ) - except EnvironmentError: + except OSError: pass else: try: @@ -129,7 +129,7 @@ class database(flat_hash.database): existing_content = f.read() finally: f.close() - except EnvironmentError: + except OSError: pass else: existing_mtime = existing_st[stat.ST_MTIME] @@ -156,7 +156,7 @@ class database(flat_hash.database): myf = open( _unicode_encode(fp, encoding=_encodings["fs"], errors="strict"), "wb" ) - except EnvironmentError as e: + except OSError as e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) @@ -164,7 +164,7 @@ class database(flat_hash.database): _unicode_encode(fp, encoding=_encodings["fs"], errors="strict"), "wb", ) - except EnvironmentError as e: + except OSError as e: raise cache_errors.CacheCorruption(cpv, e) else: raise cache_errors.CacheCorruption(cpv, e) @@ -177,9 +177,9 @@ class database(flat_hash.database): try: os.rename(fp, new_fp) - except EnvironmentError as e: + except OSError as e: try: os.unlink(fp) - except EnvironmentError: + except OSError: pass raise cache_errors.CacheCorruption(cpv, e) diff --git a/lib/portage/cache/sql_template.py b/lib/portage/cache/sql_template.py index 99cd41a34..ec058ab15 100644 --- a/lib/portage/cache/sql_template.py +++ b/lib/portage/cache/sql_template.py @@ -13,7 +13,7 @@ class SQLDatabase(template.database): _BaseError must be an exception class that all Exceptions thrown from the derived RDBMS are derived from. - SCHEMA_INSERT_CPV_INTO_PACKAGE should be modified dependant on the RDBMS, as should SCHEMA_PACKAGE_CREATE- + SCHEMA_INSERT_CPV_INTO_PACKAGE should be modified dependent on the RDBMS, as should SCHEMA_PACKAGE_CREATE- basically you need to deal with creation of a unique pkgid. If the dbapi2 rdbms class has a method of recovering that id, then modify _insert_cpv to remove the extra select. @@ -27,7 +27,7 @@ class SQLDatabase(template.database): pkgid INTEGER PRIMARY KEY, label VARCHAR(255), cpv VARCHAR(255), UNIQUE(label, cpv))" % SCHEMA_PACKAGE_NAME ) - SCHEMA_PACKAGE_DROP = "DROP TABLE %s" % SCHEMA_PACKAGE_NAME + SCHEMA_PACKAGE_DROP = f"DROP TABLE {SCHEMA_PACKAGE_NAME}" SCHEMA_VALUES_NAME = "values_cache" SCHEMA_VALUES_CREATE = ( @@ -35,7 +35,7 @@ class SQLDatabase(template.database): key varchar(255), value text, UNIQUE(pkgid, key))" % (SCHEMA_VALUES_NAME, SCHEMA_PACKAGE_NAME) ) - SCHEMA_VALUES_DROP = "DROP TABLE %s" % SCHEMA_VALUES_NAME + SCHEMA_VALUES_DROP = f"DROP TABLE {SCHEMA_VALUES_NAME}" SCHEMA_INSERT_CPV_INTO_PACKAGE = ( "INSERT INTO %s (label, cpv) VALUES(%%s, %%s)" % SCHEMA_PACKAGE_NAME ) @@ -53,7 +53,7 @@ class SQLDatabase(template.database): """initialize the instance. derived classes shouldn't need to override this""" - super(SQLDatabase, self).__init__(location, label, auxdbkeys, *args, **config) + super().__init__(location, label, auxdbkeys, *args, **config) config.setdefault("host", "127.0.0.1") config.setdefault("autocommit", self.autocommits) @@ -69,14 +69,14 @@ class SQLDatabase(template.database): def _initdb_con(self, config): """ensure needed tables are in place. - If the derived class needs a different set of table creation commands, overload the approriate + If the derived class needs a different set of table creation commands, overload the appropriate SCHEMA_ attributes. If it needs additional execution beyond, override""" self._dbconnect(config) if not self._table_exists(self.SCHEMA_PACKAGE_NAME): if self.readonly: raise cache_errors.ReadOnlyRestriction( - "table %s doesn't exist" % self.SCHEMA_PACKAGE_NAME + f"table {self.SCHEMA_PACKAGE_NAME} doesn't exist" ) try: self.con.execute(self.SCHEMA_PACKAGE_CREATE) @@ -86,7 +86,7 @@ class SQLDatabase(template.database): if not self._table_exists(self.SCHEMA_VALUES_NAME): if self.readonly: raise cache_errors.ReadOnlyRestriction( - "table %s doesn't exist" % self.SCHEMA_VALUES_NAME + f"table {self.SCHEMA_VALUES_NAME} doesn't exist" ) try: self.con.execute(self.SCHEMA_VALUES_CREATE) @@ -122,7 +122,7 @@ class SQLDatabase(template.database): if len(rows) == 0: raise KeyError(cpv) - vals = dict([(k, "") for k in self._known_keys]) + vals = {k: "" for k in self._known_keys} vals.update(dict(rows)) return vals @@ -152,12 +152,11 @@ class SQLDatabase(template.database): def __del__(self): # just to be safe. - if "db" in self.__dict__ and self.db != None: + if "db" in self.__dict__ and self.db is not None: self.commit() self.db.close() def _setitem(self, cpv, values): - try: # insert. try: @@ -255,8 +254,7 @@ class SQLDatabase(template.database): try: self.con.execute( - "SELECT cpv FROM %s WHERE label=%s" - % (self.SCHEMA_PACKAGE_NAME, self.label) + f"SELECT cpv FROM {self.SCHEMA_PACKAGE_NAME} WHERE label={self.label}" ) except self._BaseError as e: raise cache_errors.GeneralCacheCorruption(e) @@ -278,7 +276,7 @@ class SQLDatabase(template.database): l = [] for x, y, v in self.con.fetchall(): if oldcpv != x: - if oldcpv != None: + if oldcpv is not None: d = dict(l) if "_eclasses_" in d: d["_eclasses_"] = reconstruct_eclasses(oldcpv, d["_eclasses_"]) @@ -288,7 +286,7 @@ class SQLDatabase(template.database): l.clear() oldcpv = x l.append((y, v)) - if oldcpv != None: + if oldcpv is not None: d = dict(l) if "_eclasses_" in d: d["_eclasses_"] = reconstruct_eclasses(oldcpv, d["_eclasses_"]) @@ -309,7 +307,7 @@ class SQLDatabase(template.database): v = v.replace("%", "\\%") v = v.replace(".*", "%") query_list.append( - "(key=%s AND value LIKE %s)" % (self._sfilter(k), self._sfilter(v)) + f"(key={self._sfilter(k)} AND value LIKE {self._sfilter(v)})" ) if len(query_list): diff --git a/lib/portage/cache/sqlite.py b/lib/portage/cache/sqlite.py index 23a775e65..77dc8bc41 100644 --- a/lib/portage/cache/sqlite.py +++ b/lib/portage/cache/sqlite.py @@ -1,4 +1,4 @@ -# Copyright 1999-2020 Gentoo Authors +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import collections @@ -14,7 +14,6 @@ from portage.localization import _ class database(fs_template.FsBased): - validation_chf = "md5" chf_types = ("md5", "mtime") @@ -30,11 +29,11 @@ class database(fs_template.FsBased): ) def __init__(self, *args, **config): - super(database, self).__init__(*args, **config) + super().__init__(*args, **config) self._import_sqlite() self._allowed_keys = ["_eclasses_"] self._allowed_keys.extend(self._known_keys) - self._allowed_keys.extend("_%s_" % k for k in self.chf_types) + self._allowed_keys.extend(f"_{k}_" for k in self.chf_types) self._allowed_keys_set = frozenset(self._allowed_keys) self._allowed_keys = sorted(self._allowed_keys_set) @@ -54,6 +53,19 @@ class database(fs_template.FsBased): self._config = config self._db_connection_info = None + def __getstate__(self): + state = self.__dict__.copy() + # These attributes are not picklable, so they are automatically + # regenerated after unpickling. + state["_db_module"] = None + state["_db_error"] = None + state["_db_connection_info"] = None + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._import_sqlite() + def _import_sqlite(self): # sqlite3 is optional with >=python-2.5 try: @@ -108,11 +120,11 @@ class database(fs_template.FsBased): connection, cursor, portage.getpid() ) self._db_cursor.execute( - "PRAGMA encoding = %s" % self._db_escape_string("UTF-8") + f"PRAGMA encoding = {self._db_escape_string('UTF-8')}" ) if not self.readonly and not self._ensure_access(self._dbpath): raise cache_errors.InitializationError( - self.__class__, "can't ensure perms on %s" % self._dbpath + self.__class__, f"can't ensure perms on {self._dbpath}" ) self._db_init_cache_size(config["cache_bytes"]) self._db_init_synchronous(config["synchronous"]) @@ -136,12 +148,10 @@ class database(fs_template.FsBased): "%s INTEGER PRIMARY KEY AUTOINCREMENT" % self._db_table["packages"]["package_id"] ) - table_parameters.append("%s TEXT" % self._db_table["packages"]["package_key"]) + table_parameters.append(f"{self._db_table['packages']['package_key']} TEXT") for k in self._allowed_keys: - table_parameters.append("%s TEXT" % k) - table_parameters.append( - "UNIQUE(%s)" % self._db_table["packages"]["package_key"] - ) + table_parameters.append(f"{k} TEXT") + table_parameters.append(f"UNIQUE({self._db_table['packages']['package_key']})") create_statement.append(",".join(table_parameters)) create_statement.append(")") @@ -163,13 +173,13 @@ class database(fs_template.FsBased): ) else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) - cursor.execute("DROP TABLE %s" % v["table_name"]) + cursor.execute(f"DROP TABLE {v['table_name']}") cursor.execute(v["create"]) else: cursor.execute(v["create"]) def _db_table_exists(self, table_name): - """return true/false dependant on a tbl existing""" + """return true/false dependent on a tbl existing""" cursor = self._db_cursor cursor.execute( 'SELECT name FROM sqlite_master WHERE type="table" AND name=%s' @@ -178,7 +188,7 @@ class database(fs_template.FsBased): return len(cursor.fetchall()) == 1 def _db_table_get_create(self, table_name): - """return true/false dependant on a tbl existing""" + """return true/false dependent on a tbl existing""" cursor = self._db_cursor cursor.execute( "SELECT sql FROM sqlite_master WHERE name=%s" @@ -202,7 +212,7 @@ class database(fs_template.FsBased): if m is None: return False, missing_keys - unique_constraints = set([self._db_table["packages"]["package_key"]]) + unique_constraints = {self._db_table["packages"]["package_key"]} missing_keys = set(self._allowed_keys) unique_re = re.compile(r"^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$") column_re = re.compile(r"^\s*(\w*)\s*TEXT\s*$") @@ -289,7 +299,7 @@ class database(fs_template.FsBased): def _setitem(self, cpv, values): update_statement = [] update_statement.append( - "REPLACE INTO %s" % self._db_table["packages"]["table_name"] + f"REPLACE INTO {self._db_table['packages']['table_name']}" ) update_statement.append("(") update_statement.append( @@ -309,7 +319,7 @@ class database(fs_template.FsBased): s = " ".join(update_statement) cursor.execute(s) except self._db_error as e: - writemsg("%s: %s\n" % (cpv, str(e))) + writemsg(f"{cpv}: {str(e)}\n") raise def commit(self): diff --git a/lib/portage/cache/template.py b/lib/portage/cache/template.py index 3677dfa74..9e69e368f 100644 --- a/lib/portage/cache/template.py +++ b/lib/portage/cache/template.py @@ -11,7 +11,7 @@ import operator class database: # this is for metadata/cache transfer. - # basically flags the cache needs be updated when transfered cache to cache. + # basically flags the cache needs be updated when transferred cache to cache. # leave this. complete_eclass_entries = True @@ -32,7 +32,7 @@ class database: def __getitem__(self, cpv): """set a cpv to values - This shouldn't be overriden in derived classes since it handles the __eclasses__ conversion. + This shouldn't be overridden in derived classes since it handles the __eclasses__ conversion. that said, if the class handles it, they can override it.""" if self.updates > self.sync_rate: self.commit() @@ -46,7 +46,7 @@ class database: if self.serialize_eclasses and "_eclasses_" in d: for chf_type in chf_types: - if "_%s_" % chf_type not in d: + if f"_{chf_type}_" not in d: # Skip the reconstruct_eclasses call, since it's # a waste of time if it contains a different chf_type # than the current one. In the past, it was possible @@ -77,7 +77,7 @@ class database: # those that egencache uses to avoid redundant writes. d.pop("INHERITED", None) - mtime_required = not any(d.get("_%s_" % x) for x in chf_types if x != "mtime") + mtime_required = not any(d.get(f"_{x}_") for x in chf_types if x != "mtime") mtime = d.get("_mtime_") if not mtime: @@ -89,14 +89,14 @@ class database: mtime = int(mtime) except ValueError: raise cache_errors.CacheCorruption( - cpv, "_mtime_ conversion to int failed: %s" % (mtime,) + cpv, f"_mtime_ conversion to int failed: {mtime}" ) d["_mtime_"] = mtime return d def _getitem(self, cpv): """get cpv's values. - override this in derived classess""" + override this in derived classes""" raise NotImplementedError @staticmethod @@ -111,16 +111,17 @@ class database: return extern_ec_dict chf_getter = operator.attrgetter(chf_type) if paths: - intern_ec_dict = dict( - (k, (v.eclass_dir, chf_getter(v))) for k, v in extern_ec_dict.items() - ) + intern_ec_dict = { + k: (v.eclass_dir, chf_getter(v)) for k, v in extern_ec_dict.items() + } else: - intern_ec_dict = dict((k, chf_getter(v)) for k, v in extern_ec_dict.items()) + intern_ec_dict = {k: chf_getter(v) for k, v in extern_ec_dict.items()} return intern_ec_dict def __setitem__(self, cpv, values): """set a cpv to values - This shouldn't be overriden in derived classes since it handles the readonly checks""" + This shouldn't be overridden in derived classes since it handles the readonly checks + """ if self.readonly: raise cache_errors.ReadOnlyRestriction() d = None @@ -156,7 +157,8 @@ class database: def __delitem__(self, cpv): """delete a key from the cache. - This shouldn't be overriden in derived classes since it handles the readonly checks""" + This shouldn't be overridden in derived classes since it handles the readonly checks + """ if self.readonly: raise cache_errors.ReadOnlyRestriction() if not self.autocommits: @@ -240,7 +242,7 @@ class database: return False def _validate_entry(self, chf_type, entry, ebuild_hash, eclass_db): - hash_key = "_%s_" % chf_type + hash_key = f"_{chf_type}_" try: entry_hash = entry[hash_key] except KeyError: @@ -311,12 +313,11 @@ def serialize_eclasses(eclass_dict, chf_type="mtime", paths=True): getter = operator.attrgetter(chf_type) if paths: return "\t".join( - "%s\t%s\t%s" % (k, v.eclass_dir, getter(v)) + f"{k}\t{v.eclass_dir}\t{getter(v)}" for k, v in sorted(eclass_dict.items(), key=_keysorter) ) return "\t".join( - "%s\t%s" % (k, getter(v)) - for k, v in sorted(eclass_dict.items(), key=_keysorter) + f"{k}\t{getter(v)}" for k, v in sorted(eclass_dict.items(), key=_keysorter) ) @@ -349,11 +350,11 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type="mtime", paths=True): if paths: if len(eclasses) % 3 != 0: raise cache_errors.CacheCorruption( - cpv, "_eclasses_ was of invalid len %i" % len(eclasses) + cpv, f"_eclasses_ was of invalid len {len(eclasses)}" ) elif len(eclasses) % 2 != 0: raise cache_errors.CacheCorruption( - cpv, "_eclasses_ was of invalid len %i" % len(eclasses) + cpv, f"_eclasses_ was of invalid len {len(eclasses)}" ) d = {} try: @@ -367,11 +368,11 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type="mtime", paths=True): d[name] = converter(val) except IndexError: raise cache_errors.CacheCorruption( - cpv, "_eclasses_ was of invalid len %i" % len(eclasses) + cpv, f"_eclasses_ was of invalid len {len(eclasses)}" ) except ValueError: raise cache_errors.CacheCorruption( - cpv, "_eclasses_ not valid for chf_type {}".format(chf_type) + cpv, f"_eclasses_ not valid for chf_type {chf_type}" ) del eclasses return d diff --git a/lib/portage/cache/volatile.py b/lib/portage/cache/volatile.py index 67afd20e7..6c0473b32 100644 --- a/lib/portage/cache/volatile.py +++ b/lib/portage/cache/volatile.py @@ -6,7 +6,6 @@ from portage.cache import template class database(template.database): - autocommits = True serialize_eclasses = False store_eclass_paths = False @@ -14,7 +13,7 @@ class database(template.database): def __init__(self, *args, **config): config.pop("gid", None) config.pop("perms", None) - super(database, self).__init__(*args, **config) + super().__init__(*args, **config) self._data = {} self._delitem = self._data.__delitem__ |