From d7c4700b8cdb6b87bbfab61753e712cc082efa4e Mon Sep 17 00:00:00 2001 From: Arthur Zamarin Date: Mon, 29 Apr 2024 16:55:09 +0300 Subject: Integrate with release-monitoring - Collect outdated information from Anitya - Prefer it over repology - Add Anitya links to package overview sidebar - Show a little bit different outdated information block for anitya. Resolves: https://github.com/gentoo/soko/issues/13 Signed-off-by: Arthur Zamarin --- pkg/app/handler/packages/overview.templ | 14 ++- pkg/models/outdated.go | 8 ++ pkg/models/package.go | 5 + pkg/portage/anitya/anitya.go | 171 ++++++++++++++++++++++++++++++++ pkg/portage/repology/category.go | 58 +++++++++++ pkg/portage/repology/outdated.go | 49 +-------- soko.go | 17 +++- 7 files changed, 274 insertions(+), 48 deletions(-) create mode 100644 pkg/portage/anitya/anitya.go create mode 100644 pkg/portage/repology/category.go diff --git a/pkg/app/handler/packages/overview.templ b/pkg/app/handler/packages/overview.templ index f9e13af..cd6985c 100644 --- a/pkg/app/handler/packages/overview.templ +++ b/pkg/app/handler/packages/overview.templ @@ -129,7 +129,11 @@ templ overview(pkg *models.Package, userPreferences *models.UserPreferences) {
It seems that version { pkg.Outdated[0].NewestVersion } is available upstream, while the latest version in the Gentoo tree is { pkg.Outdated[0].GentooVersion }.
- You think this warning is false? Read more about it here. + if pkg.Outdated[0].Source == models.OutdatedSourceRepology { + You think this warning is false? Read more about it here. + } else { + This information is provided from Release-Monitoring, so fix association issues there. + } } } @@ -415,6 +419,14 @@ templ overview(pkg *models.Package, userPreferences *models.UserPreferences) { Repology + if pkg.AnityaInfo != nil { +
+ + + Release-Monitoring + +
+ }
diff --git a/pkg/models/outdated.go b/pkg/models/outdated.go index a760da3..e65fcb4 100644 --- a/pkg/models/outdated.go +++ b/pkg/models/outdated.go @@ -2,8 +2,16 @@ package models +type OutdatedSource string + +const ( + OutdatedSourceRepology OutdatedSource = "repology" + OutdatedSourceAnitya OutdatedSource = "anitya" +) + type OutdatedPackages struct { Atom string `pg:",pk"` GentooVersion string NewestVersion string + Source OutdatedSource } diff --git a/pkg/models/package.go b/pkg/models/package.go index f543891..659dcd8 100644 --- a/pkg/models/package.go +++ b/pkg/models/package.go @@ -15,6 +15,7 @@ type Package struct { Longdescription string Maintainers []*Maintainer Upstream Upstream + AnityaInfo *AnityaInfo Commits []*Commit `pg:"many2many:commit_to_packages,join_fk:commit_id"` PrecedingCommits int `pg:",use_zero"` PkgCheckResults []*PkgCheckResult `pg:",fk:atom,rel:has-many"` @@ -63,6 +64,10 @@ type RemoteId struct { Id string } +type AnityaInfo struct { + Project string +} + type packageDepMap struct { Version string Atom string diff --git a/pkg/portage/anitya/anitya.go b/pkg/portage/anitya/anitya.go new file mode 100644 index 0000000..25f91cd --- /dev/null +++ b/pkg/portage/anitya/anitya.go @@ -0,0 +1,171 @@ +package anitya + +import ( + "encoding/json" + "log/slog" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "soko/pkg/config" + "soko/pkg/database" + "soko/pkg/models" +) + +// API documentation: https://release-monitoring.org/static/docs/api.html#get--api-v2-packages- + +const itemsPerPage = 250 + +type ApiPackage struct { + Name string `json:"name"` + Project string `json:"project"` + StableVersion string `json:"stable_version"` + Version string `json:"version"` +} + +type ApiResponse struct { + Items []ApiPackage `json:"items"` + TotalItems int `json:"total_items"` +} + +func UpdateAnitya() { + anityaPackages, err := readAllResults() + if err != nil { + slog.Error("Failed fetching anitya data", slog.Any("err", err)) + return + } else if len(anityaPackages) == 0 { + slog.Error("No anitya packages found") + } + + packagesMap := make(map[string]int, len(anityaPackages)) + packages := make([]*models.Package, len(anityaPackages)) + for i, p := range anityaPackages { + packages[i] = &models.Package{Atom: p.Name} + packagesMap[p.Name] = i + } + + err = database.DBCon.Model(&packages).WherePK().Relation("Versions").Select() + if err != nil { + slog.Error("Failed fetching packages", slog.Any("err", err)) + return + } + + outdatedEntries := make([]*models.OutdatedPackages, 0, len(packages)) + +nextPackage: + for _, p := range packages { + anitya := anityaPackages[packagesMap[p.Atom]] + p.AnityaInfo = &models.AnityaInfo{ + Project: anitya.Project, + } + if len(p.Versions) == 0 { + continue + } + + latest := models.Version{Version: anitya.LatestVersion()} + currentLatest := p.Versions[0] + for _, v := range p.Versions { + if slices.Contains(v.Properties, "live") { + continue + } + if strings.HasPrefix(v.Version, latest.Version) || !latest.GreaterThan(*v) { + continue nextPackage + } + if v.GreaterThan(*currentLatest) { + currentLatest = v + } + } + outdatedEntries = append(outdatedEntries, &models.OutdatedPackages{ + Atom: p.Atom, + GentooVersion: currentLatest.Version, + NewestVersion: anitya.StableVersion, + Source: models.OutdatedSourceAnitya, + }) + } + _, err = database.DBCon.Model(&packages).Set("anitya_info = ?anitya_info").Update() + if err != nil { + slog.Error("Failed updating packages", slog.Any("err", err)) + return + } + slog.Info("Updated anitya information", slog.Int("count", len(packages))) + + _, _ = database.DBCon.Model((*models.OutdatedPackages)(nil)).Where("source = ?", models.OutdatedSourceAnitya).Delete() + res, err := database.DBCon.Model(&outdatedEntries).OnConflict("(atom) DO UPDATE").Insert() + if err != nil { + slog.Error("Error while inserting outdated packages", slog.Any("err", err)) + } else { + slog.Info("Inserted outdated packages", slog.Int("res", res.RowsAffected())) + } + + updateStatus() +} + +var client = http.Client{Timeout: 1 * time.Minute} + +func fetchResults(page int, params url.Values) (int, []ApiPackage, error) { + req, err := http.NewRequest("GET", "https://release-monitoring.org/api/v2/packages/?"+params.Encode(), nil) + if err != nil { + slog.Error("Failed creating request", slog.Int("page", page), slog.Any("err", err)) + return 0, nil, err + } + req.Header.Set("User-Agent", config.UserAgent()) + + resp, err := client.Do(req) + if err != nil { + slog.Error("Failed fetching anitya data", slog.Int("page", page), slog.Any("err", err)) + return 0, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + slog.Error("Failed fetching anitya data", slog.Int("page", page), slog.Int("status", resp.StatusCode)) + return 0, nil, nil + } + + var data ApiResponse + err = json.NewDecoder(resp.Body).Decode(&data) + return data.TotalItems, data.Items, err +} + +func readAllResults() (result []ApiPackage, err error) { + params := url.Values{ + "distribution": {"Gentoo"}, + "items_per_page": {strconv.Itoa(itemsPerPage)}, + } + totalPages := 1 + for page := 1; page <= totalPages; page++ { + slog.Info("Fetching anitya data", slog.Int("page", page)) + params.Set("page", strconv.Itoa(page)) + total, items, err := fetchResults(page, params) + if err != nil { + return nil, err + } + + if page == 1 { + totalPages = (total + itemsPerPage - 1) / itemsPerPage + result = make([]ApiPackage, 0, total) + } + result = append(result, items...) + } + return +} + +func (p *ApiPackage) LatestVersion() (result string) { + result = p.Version + if p.StableVersion != "" { + result = p.StableVersion + } + result, _, _ = strings.Cut(result, ".post") + return +} + +func updateStatus() { + database.DBCon.Model(&models.Application{ + Id: "anitya", + LastUpdate: time.Now(), + Version: config.Version(), + }).OnConflict("(id) DO UPDATE").Insert() +} diff --git a/pkg/portage/repology/category.go b/pkg/portage/repology/category.go new file mode 100644 index 0000000..41efe57 --- /dev/null +++ b/pkg/portage/repology/category.go @@ -0,0 +1,58 @@ +package repology + +import ( + "log/slog" + + "soko/pkg/database" + "soko/pkg/models" +) + +func UpdateCategoriesMetadata() { + var categoriesInfoArr []*models.CategoryPackagesInformation + err := database.DBCon.Model((*models.OutdatedPackages)(nil)). + ColumnExpr("SPLIT_PART(atom, '/', 1) as name"). + ColumnExpr("COUNT(*) as outdated"). + GroupExpr("SPLIT_PART(atom, '/', 1)"). + Select(&categoriesInfoArr) + if err != nil { + slog.Error("Failed collecting outdated stats", slog.Any("err", err)) + return + } + categoriesInfo := make(map[string]*models.CategoryPackagesInformation, len(categoriesInfoArr)) + for _, categoryInfo := range categoriesInfoArr { + if categoryInfo.Name != "" { + categoriesInfo[categoryInfo.Name] = categoryInfo + } + } + + var categories []*models.CategoryPackagesInformation + err = database.DBCon.Model(&categories).Column("name").Select() + if err != nil { + slog.Error("Failed fetching categories packages information", slog.Any("err", err)) + return + } else if len(categories) > 0 { + for _, category := range categories { + if info, found := categoriesInfo[category.Name]; found { + category.Outdated = info.Outdated + } else { + category.Outdated = 0 + } + delete(categoriesInfo, category.Name) + } + _, err = database.DBCon.Model(&categories).Set("outdated = ?outdated").Update() + if err != nil { + slog.Error("Failed updating categories packages information", slog.Any("err", err)) + } + categories = make([]*models.CategoryPackagesInformation, 0, len(categoriesInfo)) + } + + for _, catInfo := range categoriesInfo { + categories = append(categories, catInfo) + } + if len(categories) > 0 { + _, err = database.DBCon.Model(&categories).Insert() + if err != nil { + slog.Error("Failed inserting categories packages information", slog.Any("err", err)) + } + } +} diff --git a/pkg/portage/repology/outdated.go b/pkg/portage/repology/outdated.go index 93f9008..794697b 100644 --- a/pkg/portage/repology/outdated.go +++ b/pkg/portage/repology/outdated.go @@ -32,9 +32,6 @@ var clientRateLimiter = rate.NewLimiter(rate.Every(2*time.Second), 1) // UpdateOutdated will update the database table that contains all outdated gentoo versions func UpdateOutdated() { - database.Connect() - defer database.DBCon.Close() - // Get all outdated Versions outdated := newOutdatedCheck() for letter := 'a'; letter <= 'z'; letter++ { @@ -43,9 +40,9 @@ func UpdateOutdated() { // Update the database if len(outdated.outdatedVersions) > 0 { - database.TruncateTable((*models.OutdatedPackages)(nil)) + _, _ = database.DBCon.Model((*models.OutdatedPackages)(nil)).Where("source = ?", models.OutdatedSourceRepology).Delete() - res, err := database.DBCon.Model(&outdated.outdatedVersions).Insert() + res, err := database.DBCon.Model(&outdated.outdatedVersions).OnConflict("(atom) DO NOTHING").Insert() if err != nil { slog.Error("Error while inserting outdated packages", slog.Any("err", err)) } else { @@ -53,37 +50,6 @@ func UpdateOutdated() { } } - // Updated the outdated status of categories - var categories []*models.CategoryPackagesInformation - err := database.DBCon.Model(&categories).Column("name").Select() - if err != nil { - slog.Error("Failed fetching categories packages information", slog.Any("err", err)) - return - } else if len(categories) > 0 { - for _, category := range categories { - category.Outdated = outdated.outdatedCategories[category.Name] - delete(outdated.outdatedCategories, category.Name) - } - _, err = database.DBCon.Model(&categories).Set("outdated = ?outdated").Update() - if err != nil { - slog.Error("Failed updating categories packages information", slog.Any("err", err)) - } - categories = make([]*models.CategoryPackagesInformation, 0, len(outdated.outdatedCategories)) - } - - for category, count := range outdated.outdatedCategories { - categories = append(categories, &models.CategoryPackagesInformation{ - Name: category, - Outdated: count, - }) - } - if len(categories) > 0 { - _, err = database.DBCon.Model(&categories).Insert() - if err != nil { - slog.Error("Error while inserting categories packages information", slog.Any("err", err)) - } - } - updateStatus() } @@ -131,8 +97,7 @@ type outdatedCheck struct { blockedCategories map[string]struct{} atomRules map[string]*atomOutdatedRules - outdatedCategories map[string]int - outdatedVersions []*models.OutdatedPackages + outdatedVersions []*models.OutdatedPackages } func newOutdatedCheck() outdatedCheck { @@ -140,8 +105,6 @@ func newOutdatedCheck() outdatedCheck { blockedRepos: readBlockList("ignored-repositories"), blockedCategories: readBlockList("ignored-categories"), atomRules: buildAtomRules(), - - outdatedCategories: make(map[string]int), } } @@ -206,12 +169,8 @@ func (o *outdatedCheck) getOutdatedStartingWith(letter rune) { Atom: atom, GentooVersion: currentVersion[atom], NewestVersion: newestVersion, + Source: models.OutdatedSourceRepology, }) - - category, _, found := strings.Cut(atom, "/") - if found { - o.outdatedCategories[category]++ - } } } } diff --git a/soko.go b/soko.go index c38f660..9afe675 100644 --- a/soko.go +++ b/soko.go @@ -13,7 +13,9 @@ import ( "soko/pkg/app" "soko/pkg/config" + "soko/pkg/database" "soko/pkg/portage" + "soko/pkg/portage/anitya" "soko/pkg/portage/bugs" "soko/pkg/portage/dependencies" "soko/pkg/portage/github" @@ -58,8 +60,7 @@ func main() { portage.FullUpdate() } if *updateOutdatedPackages { - slog.Info("Updating the repology data") - repology.UpdateOutdated() + updateOutdated() } if *updatePkgcheckResults { slog.Info("Updating the qa-reports that is the pkgcheck data") @@ -96,6 +97,18 @@ func main() { } } +func updateOutdated() { + database.Connect() + defer database.DBCon.Close() + + slog.Info("Updating the anitya data") + anitya.UpdateAnitya() + slog.Info("Updating the repology data") + repology.UpdateOutdated() + + repology.UpdateCategoriesMetadata() +} + // initialize the loggers depending on whether // config.debug is set to true func initLoggers() { -- cgit v1.2.3-65-gdbad