diff options
author | Sheng Yu <syu.os@protonmail.com> | 2022-09-01 10:44:55 -0400 |
---|---|---|
committer | Michał Górny <mgorny@gentoo.org> | 2022-09-09 12:15:58 +0200 |
commit | 1d94992a2df2b5cc963c26c7978a899dc642deb1 (patch) | |
tree | e3c658b33d73660832b5e568b91ae8b824fb9dc7 | |
parent | Add gpkg-sign tool to sign exist GPKG files. (diff) | |
download | portage-1d94992a.tar.gz portage-1d94992a.tar.bz2 portage-1d94992a.zip |
Move all files into basename/DATA structure
Signed-off-by: Sheng Yu <syu.os@protonmail.com>
Signed-off-by: Michał Górny <mgorny@gentoo.org>
-rw-r--r-- | lib/portage/binpkg.py | 8 | ||||
-rw-r--r-- | lib/portage/dbapi/bintree.py | 2 | ||||
-rw-r--r-- | lib/portage/gpkg.py | 201 | ||||
-rw-r--r-- | lib/portage/tests/gpkg/test_gpkg_checksum.py | 21 | ||||
-rw-r--r-- | lib/portage/tests/gpkg/test_gpkg_gpg.py | 28 |
5 files changed, 158 insertions, 102 deletions
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py index ed2fda827..f3f149b14 100644 --- a/lib/portage/binpkg.py +++ b/lib/portage/binpkg.py @@ -17,8 +17,8 @@ def get_binpkg_format(binpkg_path): try: with open(binpkg_path, "rb") as binpkg_file: - header = binpkg_file.read(6) - if header == b"gpkg-1": + header = binpkg_file.read(100) + if b"/gpkg-1\x00" in header: file_format = "gpkg" else: binpkg_file.seek(-16, 2) @@ -32,7 +32,9 @@ def get_binpkg_format(binpkg_path): if file_format is None: try: with tarfile.open(binpkg_path) as gpkg_tar: - if "gpkg-1" in gpkg_tar.getnames(): + if "gpkg-1" in [ + f.split("/", maxsplit=1)[-1] for f in gpkg_tar.getnames() + ]: file_format = "gpkg" except tarfile.TarError: pass diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 0857ff21a..814e6627c 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -744,7 +744,7 @@ class binarytree: mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, update_path) - mybinpkg.update_metadata(mydata, newcpv=mynewcpv) + mybinpkg.update_metadata(mydata, new_basename=mynewcpv) else: raise InvalidBinaryPackageFormat(binpkg_format) self.inject(mynewcpv, filename=update_path) diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 644ff412b..5f8e19341 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -764,10 +764,10 @@ class gpkg: https://www.gentoo.org/glep/glep-0078.html """ - def __init__(self, settings, base_name=None, gpkg_file=None): + def __init__(self, settings, basename=None, gpkg_file=None): """ gpkg class handle all gpkg operations for one package. - base_name is the package basename. + basename is the package basename. gpkg_file should be exists file path for read or will create. """ self.settings = settings @@ -778,10 +778,16 @@ class gpkg: self.gpkg_file = _unicode_decode( gpkg_file, encoding=_encodings["fs"], errors="strict" ) - self.base_name = base_name + + if basename is None: + self.basename = None + else: + self.basename = basename.split("/", maxsplit=1)[-1] + self.checksums = [] self.manifest_old = [] - signature_exist = None + self.signature_exist = None + self.prefix = None # Compression is the compression algorithm, if set to None will # not use compression. @@ -892,7 +898,9 @@ class gpkg: # Check gpkg and metadata with tarfile.open(mode="r", fileobj=container_file) as container: - if self.gpkg_version not in container.getnames(): + if self.gpkg_version not in ( + os.path.basename(f) for f in container.getnames() + ): raise InvalidBinaryPackageFormat("Invalid gpkg file.") metadata_tarinfo, metadata_comp = self._get_inner_tarinfo( @@ -985,7 +993,7 @@ class gpkg: ) # Long CPV - if len(self.base_name) >= 154: + if len(self.basename) >= 154: container_tar_format = tarfile.GNU_FORMAT # gpkg container @@ -994,9 +1002,14 @@ class gpkg: ) # gpkg version - gpkg_version_file = tarfile.TarInfo(self.gpkg_version) + gpkg_version_file = tarfile.TarInfo( + os.path.join(self.basename, self.gpkg_version) + ) gpkg_version_file.mtime = datetime.utcnow().timestamp() container.addfile(gpkg_version_file) + checksum_info = checksum_helper(self.settings) + checksum_info.finish() + self._record_checksum(checksum_info, gpkg_version_file) compression_cmd = self._get_compression_cmd() @@ -1059,16 +1072,19 @@ class gpkg: finally: image_tar.kill() - def update_metadata(self, metadata, newcpv=None): + def update_metadata(self, metadata, new_basename=None): """ Update metadata in the gpkg file. """ self._verify_binpkg() self.checksums = [] - oldcpv = None + old_basename = self.prefix - if newcpv: - oldcpv = self.base_name + if new_basename is None: + new_basename = old_basename + else: + new_basename = new_basename.split("/", maxsplit=1)[-1] + self.basename = new_basename with open(self.gpkg_file, "rb") as container: container_tar_format = self._get_tar_format(container) @@ -1081,19 +1097,19 @@ class gpkg: name=tmp_gpkg_file_name, mode="w", format=container_tar_format ) as container: # gpkg version - gpkg_version_file = tarfile.TarInfo(self.gpkg_version) + gpkg_version_file = tarfile.TarInfo( + os.path.join(new_basename, self.gpkg_version) + ) gpkg_version_file.mtime = datetime.utcnow().timestamp() container.addfile(gpkg_version_file) + checksum_info = checksum_helper(self.settings) + checksum_info.finish() + self._record_checksum(checksum_info, gpkg_version_file) compression_cmd = self._get_compression_cmd() # metadata - if newcpv: - self.base_name = newcpv - self._add_metadata(container, metadata, compression_cmd) - self.base_name = oldcpv - else: - self._add_metadata(container, metadata, compression_cmd) + self._add_metadata(container, metadata, compression_cmd) # reuse other stuffs with tarfile.open(self.gpkg_file, "r") as container_old: @@ -1101,15 +1117,17 @@ class gpkg: for m in manifest_old: file_name_old = m[1] + if os.path.basename(file_name_old) == self.gpkg_version: + continue if os.path.basename(file_name_old).startswith("metadata"): continue - old_data_tarinfo = container_old.getmember(file_name_old) + old_data_tarinfo = container_old.getmember( + os.path.join(self.prefix, file_name_old) + ) new_data_tarinfo = copy(old_data_tarinfo) - if newcpv: - m[1] = m[1].replace(oldcpv, newcpv, 1) - new_data_tarinfo.name = new_data_tarinfo.name.replace( - oldcpv, newcpv, 1 - ) + new_data_tarinfo.name = new_data_tarinfo.name.replace( + old_basename, new_basename, 1 + ) container.addfile( new_data_tarinfo, container_old.extractfile(old_data_tarinfo) ) @@ -1139,9 +1157,14 @@ class gpkg: name=tmp_gpkg_file_name, mode="w", format=container_tar_format ) as container: # gpkg version - gpkg_version_file = tarfile.TarInfo(self.gpkg_version) + gpkg_version_file = tarfile.TarInfo( + os.path.join(self.prefix, self.gpkg_version) + ) gpkg_version_file.mtime = datetime.utcnow().timestamp() container.addfile(gpkg_version_file) + checksum_info = checksum_helper(self.settings) + checksum_info.finish() + self._record_checksum(checksum_info, gpkg_version_file) # reuse other stuffs with tarfile.open(self.gpkg_file, "r") as container_old: @@ -1150,9 +1173,13 @@ class gpkg: for m in manifest_old: file_name_old = m[1] + if os.path.basename(file_name_old) == self.gpkg_version: + continue if os.path.basename(file_name_old).endswith(".sig"): continue - old_data_tarinfo = container_old.getmember(file_name_old) + old_data_tarinfo = container_old.getmember( + os.path.join(self.prefix, file_name_old) + ) new_data_tarinfo = copy(old_data_tarinfo) container.addfile( @@ -1261,7 +1288,7 @@ class gpkg: ) # Long CPV - if len(self.base_name) >= 154: + if len(self.basename) >= 154: container_tar_format = tarfile.GNU_FORMAT # GPKG container @@ -1270,9 +1297,14 @@ class gpkg: ) # GPKG version - gpkg_version_file = tarfile.TarInfo(self.gpkg_version) + gpkg_version_file = tarfile.TarInfo( + os.path.join(self.basename, self.gpkg_version) + ) gpkg_version_file.mtime = datetime.utcnow().timestamp() container.addfile(gpkg_version_file) + checksum_info = checksum_helper(self.settings) + checksum_info.finish() + self._record_checksum(checksum_info, gpkg_version_file) compression_cmd = self._get_compression_cmd() # Metadata @@ -1404,12 +1436,15 @@ class gpkg: Record checksum result for the given file. Replace old checksum if already exists. """ + + # Remove prefix directory from the filename + file_name = os.path.basename(tarinfo.name) + for c in self.checksums: - if c[1] == tarinfo.name: + if c[1] == file_name: self.checksums.remove(c) - break - checksum_record = ["DATA", tarinfo.name, str(tarinfo.size)] + checksum_record = ["DATA", file_name, str(tarinfo.size)] for c in checksum_info.libs: checksum_record.append(c) @@ -1436,7 +1471,14 @@ class gpkg: manifest.seek(0) manifest.write(checksum_info.gpg_output) - manifest_tarinfo = tarfile.TarInfo("Manifest") + if self.basename is not None: + basename = self.basename + elif self.prefix is not None: + basename = self.prefix + else: + raise InvalidBinaryPackageFormat("No basename or prefix specified") + + manifest_tarinfo = tarfile.TarInfo(os.path.join(basename, "Manifest")) manifest_tarinfo.size = manifest.tell() manifest_tarinfo.mtime = datetime.utcnow().timestamp() manifest.seek(0) @@ -1523,10 +1565,30 @@ class gpkg: f"Cannot read tar file: {self.gpkg_file}" ) - # Check gpkg header - if self.gpkg_version not in container_files: + # Check if gpkg version file exists in any place + if self.gpkg_version not in (os.path.basename(f) for f in container_files): raise InvalidBinaryPackageFormat(f"Invalid gpkg file: {self.gpkg_file}") + # Check how many layers are in the container + for f in container_files: + if f.startswith("/"): + raise InvalidBinaryPackageFormat( + f"gpkg file structure mismatch '{f}' in {self.gpkg_file}" + ) + if f.count("/") != 1: + raise InvalidBinaryPackageFormat( + f"gpkg file structure mismatch '{f}' in {self.gpkg_file}" + ) + + # Check if all directories are the same in the container + prefix = os.path.commonpath(container_files) + if not prefix: + raise InvalidBinaryPackageFormat( + f"gpkg file structure mismatch in {self.gpkg_file}, {str(container_files)}" + ) + + gpkg_version_file = os.path.join(prefix, self.gpkg_version) + # If any signature exists, we assume all files have signature. if any(f.endswith(".sig") for f in container_files): signature_exist = True @@ -1546,13 +1608,13 @@ class gpkg: # Add all files to check list unverified_files = container_files.copy() - unverified_files.remove(self.gpkg_version) # Check Manifest file - if "Manifest" not in unverified_files: + manifest_filename = os.path.join(prefix, "Manifest") + if manifest_filename not in unverified_files: raise MissingSignature(f"Manifest not found: {self.gpkg_file}") - manifest_file = container.extractfile("Manifest") + manifest_file = container.extractfile(manifest_filename) manifest_data = manifest_file.read() manifest_file.close() @@ -1574,9 +1636,9 @@ class gpkg: raise manifest_data = checksum_info.gpg_output - unverified_files.remove("Manifest") + unverified_files.remove(manifest_filename) else: - unverified_files.remove("Manifest") + unverified_files.remove(manifest_filename) # Load manifest and create manifest check list manifest = self._load_manifest(manifest_data.decode("UTF-8")) @@ -1592,7 +1654,7 @@ class gpkg: # Find current file manifest record manifest_record = None for m in manifest: - if m[1] == f: + if m[1] == os.path.basename(f): manifest_record = m if manifest_record is None: @@ -1625,6 +1687,9 @@ class gpkg: gpg_operation=checksum_helper.VERIFY, signature=signature, ) + elif f == gpkg_version_file: + # gpkg version file is not signed + checksum_info = checksum_helper(self.settings) else: raise MissingSignature( f"{f} signature not found in {self.gpkg_file}" @@ -1684,6 +1749,7 @@ class gpkg: # Save current Manifest for other operations. self.manifest_old = manifest.copy() self.signature_exist = signature_exist + self.prefix = prefix def _generate_metadata_from_dir(self, metadata_dir): """ @@ -2017,9 +2083,13 @@ class gpkg: else: raise InvalidCompressionMethod(self.compression) - data_tarinfo = tarfile.TarInfo( - os.path.join(self.base_name, file_name + ".tar" + ext) - ) + if self.basename: + basename = self.basename + elif self.prefix: + basename = self.prefix + else: + raise InvalidBinaryPackageFormat("No basename or prefix specified") + data_tarinfo = tarfile.TarInfo(os.path.join(basename, file_name + ".tar" + ext)) return data_tarinfo def _extract_filename_compression(self, file_name): @@ -2046,42 +2116,23 @@ class gpkg: if it fail, try any file that have same name as file_name, and return the first one. """ - if self.gpkg_version not in tar.getnames(): - raise InvalidBinaryPackageFormat("Invalid gpkg file.") + if self.gpkg_version not in (os.path.basename(f) for f in tar.getnames()): + raise InvalidBinaryPackageFormat(f"Invalid gpkg file") + + if self.basename and self.prefix and not self.prefix.startswith(self.basename): + writemsg( + colorize("WARN", f"Package basename mismatched, using {self.prefix}") + ) - # Try get file with correct basename - inner_tarinfo = None - if self.base_name is None: - base_name = "" - else: - base_name = self.base_name all_files = tar.getmembers() for f in all_files: - if os.path.dirname(f.name) == base_name: - try: - f_name, f_comp = self._extract_filename_compression(f.name) - except InvalidCompressionMethod: - continue - - if f_name == file_name: - return f, f_comp + try: + f_name, f_comp = self._extract_filename_compression(f.name) + except InvalidCompressionMethod: + continue - # If failed, try get any file name matched - if inner_tarinfo is None: - for f in all_files: - try: - f_name, f_comp = self._extract_filename_compression(f.name) - except InvalidCompressionMethod: - continue - if f_name == file_name: - if self.base_name is not None: - writemsg( - colorize( - "WARN", "Package basename mismatched, using " + f.name - ) - ) - self.base_name_alt = os.path.dirname(f.name) - return f, f_comp + if f_name == file_name: + return f, f_comp # Not found raise FileNotFound(f"File Not found: {file_name}") diff --git a/lib/portage/tests/gpkg/test_gpkg_checksum.py b/lib/portage/tests/gpkg/test_gpkg_checksum.py index 5ccc099e6..be4b08661 100644 --- a/lib/portage/tests/gpkg/test_gpkg_checksum.py +++ b/lib/portage/tests/gpkg/test_gpkg_checksum.py @@ -47,7 +47,7 @@ class test_gpkg_checksum_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name != binpkg_1.gpkg_version: + if f.name != os.path.join("test", binpkg_1.gpkg_version): tar_2.addfile(f, tar_1.extractfile(f)) binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar")) @@ -89,7 +89,7 @@ class test_gpkg_checksum_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name != "Manifest": + if f.name != os.path.join("test", "Manifest"): tar_2.addfile(f, tar_1.extractfile(f)) binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar")) @@ -174,7 +174,7 @@ class test_gpkg_checksum_case(TestCase): ) as tar_2: for f in tar_1.getmembers(): tar_2.addfile(f, tar_1.extractfile(f)) - data_tarinfo = tarfile.TarInfo("data2") + data_tarinfo = tarfile.TarInfo(os.path.join("test", "data2")) data_tarinfo.size = len(data) data2 = io.BytesIO(data) tar_2.addfile(data_tarinfo, data2) @@ -217,7 +217,7 @@ class test_gpkg_checksum_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name == "Manifest": + if f.name == os.path.join("test", "Manifest"): data = io.BytesIO(tar_1.extractfile(f).read()) data_view = data.getbuffer() data_view[-16:] = b"20a6d80ab0320fh9" @@ -306,7 +306,7 @@ class test_gpkg_checksum_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name == "Manifest": + if f.name == os.path.join("test", "Manifest"): manifest = tar_1.extractfile(f).read() data = io.BytesIO(manifest) data.seek(io.SEEK_END) @@ -356,13 +356,18 @@ class test_gpkg_checksum_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - tar_2.addfile(f, tar_1.extractfile(f)) - tar_2.addfile(f, tar_1.extractfile(f)) + if "image" in f.name: + data = tar_1.extractfile(f).read() + data = data + b"1234" + f.size = len(data) + tar_2.addfile(f, io.BytesIO(data)) + else: + tar_2.addfile(f, tar_1.extractfile(f)) binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar")) self.assertRaises( - InvalidBinaryPackageFormat, + DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test"), ) diff --git a/lib/portage/tests/gpkg/test_gpkg_gpg.py b/lib/portage/tests/gpkg/test_gpkg_gpg.py index 9a9b7ef23..442764d7e 100644 --- a/lib/portage/tests/gpkg/test_gpkg_gpg.py +++ b/lib/portage/tests/gpkg/test_gpkg_gpg.py @@ -46,7 +46,7 @@ class test_gpkg_gpg_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name == "Manifest": + if f.name == os.path.join("test", "Manifest"): manifest = tar_1.extractfile(f).read().decode("UTF-8") manifest = manifest.replace( "-----BEGIN PGP SIGNATURE-----", "" @@ -116,11 +116,19 @@ class test_gpkg_gpg_case(TestCase): playground.cleanup() def test_gpkg_ignore_signature(self): + gpg_test_path = os.environ["PORTAGE_GNUPGHOME"] + playground = ResolverPlayground( user_config={ "make.conf": ( 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-ignore-signature"', 'BINPKG_FORMAT="gpkg"', + f'BINPKG_GPG_SIGNING_BASE_COMMAND="flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --batch --no-tty --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]"', + 'BINPKG_GPG_SIGNING_DIGEST="SHA512"', + f'BINPKG_GPG_SIGNING_GPG_HOME="{gpg_test_path}"', + 'BINPKG_GPG_SIGNING_KEY="0x8812797DDF1DD192"', + 'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"', + f'BINPKG_GPG_VERIFY_GPG_HOME="{gpg_test_path}"', ), } ) @@ -140,17 +148,7 @@ class test_gpkg_gpg_case(TestCase): binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar")) binpkg_1.compress(orig_full_path, {}) - with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1: - with tarfile.open( - os.path.join(tmpdir, "test-2.gpkg.tar"), "w" - ) as tar_2: - for f in tar_1.getmembers(): - if f.name == "Manifest.sig": - pass - else: - tar_2.addfile(f, tar_1.extractfile(f)) - - binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar")) + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar")) binpkg_2.decompress(os.path.join(tmpdir, "test")) finally: shutil.rmtree(tmpdir) @@ -230,7 +228,7 @@ class test_gpkg_gpg_case(TestCase): os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name == "Manifest": + if f.name == os.path.join("test", "Manifest"): sig = b""" -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 @@ -281,7 +279,7 @@ qGAN3VUF+8EsdcsV781H0F86PANhyBgEYTGDrnItTGe3/vAPjCo= 'BINPKG_GPG_SIGNING_DIGEST="SHA512"', f'BINPKG_GPG_SIGNING_GPG_HOME="{gpg_test_path}"', 'BINPKG_GPG_SIGNING_KEY="0x8812797DDF1DD192"', - 'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 1 [PORTAGE_CONFIG] [SIGNATURE]"', + 'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"', f'BINPKG_GPG_VERIFY_GPG_HOME="{gpg_test_path}"', ), } @@ -341,7 +339,7 @@ qGAN3VUF+8EsdcsV781H0F86PANhyBgEYTGDrnItTGe3/vAPjCo= os.path.join(tmpdir, "test-2.gpkg.tar"), "w" ) as tar_2: for f in tar_1.getmembers(): - if f.name == "Manifest": + if f.name == os.path.join("test", "Manifest"): sig = b""" -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 |