diff options
author | Magnus Granberg <zorry@gentoo.org> | 2016-01-26 21:30:38 +0100 |
---|---|---|
committer | Magnus Granberg <zorry@gentoo.org> | 2016-01-26 21:30:38 +0100 |
commit | 85d70df173a81e8257172d39be1ae971be54422b (patch) | |
tree | 29b7971a05435f8c2f9b7785e77cba1be79df482 | |
parent | add support for logging part2 (diff) | |
download | tinderbox-cluster-85d70df1.tar.gz tinderbox-cluster-85d70df1.tar.bz2 tinderbox-cluster-85d70df1.zip |
check all ebuilds in package instead of only check Manifest
-rw-r--r-- | pym/tbc/build_log.py | 5 | ||||
-rw-r--r-- | pym/tbc/db_mapping.py | 1 | ||||
-rw-r--r-- | pym/tbc/package.py | 151 | ||||
-rw-r--r-- | pym/tbc/sqlquerys.py | 5 | ||||
-rw-r--r-- | pym/tbc/sync.py | 20 |
5 files changed, 80 insertions, 102 deletions
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py index a16cdc6..ec4aaa7 100644 --- a/pym/tbc/build_log.py +++ b/pym/tbc/build_log.py @@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \ get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \ add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \ - update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \ + update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \ add_repoman_log from tbc.log import write_log @@ -112,8 +112,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): if status: if ebuild_id_list is None: log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) - add_logs(session, log_msg, "info", config_id) - update_mtime_sql(session, build_dict['package_id'], 0) + add_logs(session, log_msg, "error", config_id) else: old_ebuild_id_list = [] for ebuild_id in ebuild_id_list: diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py index 2636bb6..d9fcf25 100644 --- a/pym/tbc/db_mapping.py +++ b/pym/tbc/db_mapping.py @@ -84,7 +84,6 @@ class Packages(Base): CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id')) Package = Column('package',String(150)) RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id')) - Mtime = Column('mtime', DateTime, default=0) Active = Column('active', Boolean, default=False) TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) __tablename__ = 'packages' diff --git a/pym/tbc/package.py b/pym/tbc/package.py index 65a0f40..68574cf 100644 --- a/pym/tbc/package.py +++ b/pym/tbc/package.py @@ -17,7 +17,7 @@ from tbc.build_log import check_repoman_full from tbc.log import write_log from tbc.sqlquerys import get_package_info, get_config_info, \ add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \ - get_package_metadata_sql, update_package_metadata, update_mtime_sql, \ + get_package_metadata_sql, update_package_metadata, \ get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \ get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \ get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package @@ -210,7 +210,7 @@ class tbc_package(object): package_metadataDict[package_id] = attDict return package_metadataDict - def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree): + def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list): # Use packageDict to update the db ebuild_id_list = add_new_ebuild_sql(self._session, packageDict) @@ -227,9 +227,6 @@ class tbc_package(object): # update package metadata update_package_metadata(self._session, package_metadataDict) - # update the cp manifest checksum - update_mtime_sql(self._session, package_id, manifest_mtime_tree) - # Get the best cpv for the configs and add it to config_cpv_listDict PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id) cp = CategoryInfo.Category + '/' + PackageInfo.Package @@ -244,18 +241,6 @@ class tbc_package(object): # Add the ebuild to the build jobs table if needed self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict) - def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree): - # Get the cp manifest file mtime. - try: - mtime = os.path.getmtime(pkgdir + "/Manifest") - except: - log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,) - write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree') - log_msg = "C %s:%s ... Fail." % (cp, repo) - write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree') - return False - return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0) - def add_new_package_db(self, cp, repo): # Add new categories package ebuild to tables package and ebuilds # C = Checking @@ -268,9 +253,6 @@ class tbc_package(object): mytree = [] mytree.append(repodir) pkgdir = repodir + "/" + cp # Get RepoDIR + cp - manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree) - if not manifest_mtime_tree: - return None package_id = add_new_package_sql(self._session, cp, repo) @@ -332,83 +314,78 @@ class tbc_package(object): mytree = [] mytree.append(repodir) - manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree) - if not manifest_mtime_tree: + + # Get the ebuild list for cp + old_ebuild_id_list = [] + ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) + if ebuild_list_tree == []: + log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) + write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db') + log_msg = "C %s:%s ... Fail." % (cp, repo) + write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db') return None - # if we NOT have the same mtime in the db update the package - if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime: + package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp) + packageDict ={} + new_ebuild_id_list = [] + package_updated = False + for cpv in sorted(ebuild_list_tree): - # U = Update - log_msg = "U %s:%s" % (cp, repo) - write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') + # split out ebuild version + ebuild_version_tree = portage.versions.cpv_getversion(cpv) - # Get the ebuild list for cp - old_ebuild_id_list = [] - ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) - if ebuild_list_tree == []: - log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) - write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db') - log_msg = "C %s:%s ... Fail." % (cp, repo) - write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db') - return None + # Get packageDict for cpv + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) + + # take package descriptions from the ebuilds + if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']: + package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree'] + + # Get the checksum of the ebuild in tree and db + ebuild_version_checksum_tree = packageDict[cpv]['checksum'] + checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree) + + # check if we have dupes of the checksum from db + if checksums_db is None: + ebuild_version_manifest_checksum_db = None + elif fail: + dupe_ebuild_id_list = [] + for checksum in checksums_db: + ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id) + for ebuild_id in ebuilds_id: + log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,) + write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db') + dupe_ebuild_id_list.append(ebuild_id) + add_old_ebuild(self._session, dupe_ebuild_id_list) + ebuild_version_manifest_checksum_db = None + else: + ebuild_version_manifest_checksum_db = checksums_db + + # Check if the checksum have change + if ebuild_version_manifest_checksum_db is None: + # N = New ebuild + log_msg = "N %s:%s" % (cpv, repo,) + write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') + packageDict[cpv]['new'] = True + package_updated = True + elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: + # U = Updated ebuild + log_msg = "U %s:%s" % (cpv, repo,) + write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') + package_updated = True + else: + # Remove cpv from packageDict and add ebuild to new ebuils list + del packageDict[cpv] + ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id) + new_ebuild_id_list.append(ebuild_id) + self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree) + if package_updated: # Check cp with repoman full status = check_repoman_full(self._session, pkgdir, package_id, self._config_id) if status: log_msg = "Repoman %s::%s ... Fail." % (cp, repo) write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db') - package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp) - packageDict ={} - new_ebuild_id_list = [] - for cpv in sorted(ebuild_list_tree): - - # split out ebuild version - ebuild_version_tree = portage.versions.cpv_getversion(cpv) - - # Get packageDict for cpv - packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) - - # take package descriptions from the ebuilds - if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']: - package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree'] - - # Get the checksum of the ebuild in tree and db - ebuild_version_checksum_tree = packageDict[cpv]['checksum'] - checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree) - - # check if we have dupes of the checksum from db - if checksums_db is None: - ebuild_version_manifest_checksum_db = None - elif fail: - dupe_ebuild_id_list = [] - for checksum in checksums_db: - ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id) - for ebuild_id in ebuilds_id: - log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,) - write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db') - dupe_ebuild_id_list.append(ebuild_id) - add_old_ebuild(self._session, dupe_ebuild_id_list) - ebuild_version_manifest_checksum_db = None - else: - ebuild_version_manifest_checksum_db = checksums_db - - # Check if the checksum have change - if ebuild_version_manifest_checksum_db is None: - # N = New ebuild - log_msg = "N %s:%s" % (cpv, repo,) - write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') - packageDict[cpv]['new'] = True - elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: - # U = Updated ebuild - log_msg = "U %s:%s" % (cpv, repo,) - write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') - else: - # Remove cpv from packageDict and add ebuild to new ebuils list - del packageDict[cpv] - ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id) - new_ebuild_id_list.append(ebuild_id) - self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree) log_msg = "C %s:%s ... Done." % (cp, repo) write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db') diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py index e43b8fc..4161461 100644 --- a/pym/tbc/sqlquerys.py +++ b/pym/tbc/sqlquerys.py @@ -515,11 +515,6 @@ def update_package_metadata(session, package_metadataDict): EmailInfo = update_email_info(session, email) PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k) -def update_mtime_sql(session, package_id, manifest_mtime_tree): - PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one() - PackagesInfo.Mtime = manifest_mtime_tree - session.commit() - def get_package_info_from_package_id(session, package_id): PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one() CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one() diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py index 1732f1f..2d22e74 100644 --- a/pym/tbc/sync.py +++ b/pym/tbc/sync.py @@ -70,9 +70,10 @@ def git_sync_main(session): except: pass - # check git diffs witch Manifests get updated and pass that to a dict + # check git diffs witch get updated and pass that to a dict # fetch and merge the repo repo_cp_dict = {} + search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',] for repo_dir in git_repos_list(myportdb): reponame = myportdb.getRepositoryName(repo_dir) repo = git.Repo(repo_dir) @@ -82,14 +83,21 @@ def git_sync_main(session): if not repouptodate: cp_list = [] attr = {} - # We check for Manifest changes and add the package to a list + # We check for dir changes and add the package to a list repo_diff = repo.git.diff('origin', '--name-only') - write_log(session, 'Git diff: %s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main') + write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main') for diff_line in repo_diff.splitlines(): - if re.search("Manifest$", diff_line): - cp = re.sub('/Manifest', '', diff_line) - cp_list.append(cp) + find_search = True + for search_line in search_list: + if re.search(search_line, diff_line): + find_search = False + if find_search: + splited_diff_line = re.split('/', diff_line) + cp = splited_diff_line[0] + '/' + splited_diff_line[1] + if not cp in cp_list: + cp_list.append(cp) attr['cp_list'] = cp_list + write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main') repo_cp_dict[reponame] = attr git_merge(repo, info_list[0]) else: |