Integrate with release-monitoring

- Collect outdated information from Anitya
- Prefer it over repology
- Add Anitya links to package overview sidebar
- Show a little bit different outdated information block for anitya.

Resolves: https://github.com/gentoo/soko/issues/13
Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>

7 files changed, 274 insertions, 48 deletions

diff --git a/pkg/app/handler/packages/overview.templ b/pkg/app/handler/packages/overview.templ
index f9e13af..cd6985c 100644
--- a/pkg/app/handler/packages/overview.templ
+++ b/pkg/app/handler/packages/overview.templ
@@ -129,7 +129,11 @@ templ overview(pkg *models.Package, userPreferences *models.UserPreferences) {
 						<br/>
 						It seems that version { pkg.Outdated[0].NewestVersion } is available upstream, while the latest version in the Gentoo tree is { pkg.Outdated[0].GentooVersion }.
 						<br/>
-						<small><i>You think this warning is false? Read more about it <a href="https://archives.gentoo.org/gentoo-dev/message/b793f4da5a5b5e20a063ea431500a820">here</a>.</i></small>
+						if pkg.Outdated[0].Source == models.OutdatedSourceRepology {
+							<small><i>You think this warning is false? Read more about it <a href="https://archives.gentoo.org/gentoo-dev/message/b793f4da5a5b5e20a063ea431500a820">here</a>.</i></small>
+						} else {
+							<small>This information is provided from <a href="https://release-monitoring.org">Release-Monitoring</a>, so fix association issues there.</small>
+						}
 					</div>
 				}
 			}
@@ -415,6 +419,14 @@ templ overview(pkg *models.Package, userPreferences *models.UserPreferences) {
 							Repology
 						</a>
 					</dd>
+					if pkg.AnityaInfo != nil {
+						<dd>
+							<span class="fa fa-fw fa-sort-numeric-desc"></span>
+							<a href={ templ.URL("https://release-monitoring.org/projects/search/?pattern=" + pkg.AnityaInfo.Project) } target="_blank">
+								Release-Monitoring
+							</a>
+						</dd>
+					}
 					<dd>
 						<span class="octicon octicon-git-pull-request opticon-resource-icon ml-1"></span>
 						<a href={ templ.URL("https://github.com/gentoo/gentoo/pulls?q=is%3Apr+is%3Aopen+in%3Atitle+" + pkg.Atom) } target="_blank">
diff --git a/pkg/models/outdated.go b/pkg/models/outdated.go
index a760da3..e65fcb4 100644
--- a/pkg/models/outdated.go
+++ b/pkg/models/outdated.go
@@ -2,8 +2,16 @@
 
 package models
 
+type OutdatedSource string
+
+const (
+	OutdatedSourceRepology OutdatedSource = "repology"
+	OutdatedSourceAnitya   OutdatedSource = "anitya"
+)
+
 type OutdatedPackages struct {
 	Atom          string `pg:",pk"`
 	GentooVersion string
 	NewestVersion string
+	Source        OutdatedSource
 }
diff --git a/pkg/models/package.go b/pkg/models/package.go
index f543891..659dcd8 100644
--- a/pkg/models/package.go
+++ b/pkg/models/package.go
@@ -15,6 +15,7 @@ type Package struct {
 	Longdescription     string
 	Maintainers         []*Maintainer
 	Upstream            Upstream
+	AnityaInfo          *AnityaInfo
 	Commits             []*Commit            `pg:"many2many:commit_to_packages,join_fk:commit_id"`
 	PrecedingCommits    int                  `pg:",use_zero"`
 	PkgCheckResults     []*PkgCheckResult    `pg:",fk:atom,rel:has-many"`
@@ -63,6 +64,10 @@ type RemoteId struct {
 	Id   string
 }
 
+type AnityaInfo struct {
+	Project string
+}
+
 type packageDepMap struct {
 	Version string
 	Atom    string
diff --git a/pkg/portage/anitya/anitya.go b/pkg/portage/anitya/anitya.go
new file mode 100644
index 0000000..25f91cd
--- /dev/null
+++ b/pkg/portage/anitya/anitya.go
@@ -0,0 +1,171 @@
+package anitya
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"soko/pkg/config"
+	"soko/pkg/database"
+	"soko/pkg/models"
+)
+
+// API documentation: https://release-monitoring.org/static/docs/api.html#get--api-v2-packages-
+
+const itemsPerPage = 250
+
+type ApiPackage struct {
+	Name          string `json:"name"`
+	Project       string `json:"project"`
+	StableVersion string `json:"stable_version"`
+	Version       string `json:"version"`
+}
+
+type ApiResponse struct {
+	Items      []ApiPackage `json:"items"`
+	TotalItems int          `json:"total_items"`
+}
+
+func UpdateAnitya() {
+	anityaPackages, err := readAllResults()
+	if err != nil {
+		slog.Error("Failed fetching anitya data", slog.Any("err", err))
+		return
+	} else if len(anityaPackages) == 0 {
+		slog.Error("No anitya packages found")
+	}
+
+	packagesMap := make(map[string]int, len(anityaPackages))
+	packages := make([]*models.Package, len(anityaPackages))
+	for i, p := range anityaPackages {
+		packages[i] = &models.Package{Atom: p.Name}
+		packagesMap[p.Name] = i
+	}
+
+	err = database.DBCon.Model(&packages).WherePK().Relation("Versions").Select()
+	if err != nil {
+		slog.Error("Failed fetching packages", slog.Any("err", err))
+		return
+	}
+
+	outdatedEntries := make([]*models.OutdatedPackages, 0, len(packages))
+
+nextPackage:
+	for _, p := range packages {
+		anitya := anityaPackages[packagesMap[p.Atom]]
+		p.AnityaInfo = &models.AnityaInfo{
+			Project: anitya.Project,
+		}
+		if len(p.Versions) == 0 {
+			continue
+		}
+
+		latest := models.Version{Version: anitya.LatestVersion()}
+		currentLatest := p.Versions[0]
+		for _, v := range p.Versions {
+			if slices.Contains(v.Properties, "live") {
+				continue
+			}
+			if strings.HasPrefix(v.Version, latest.Version) || !latest.GreaterThan(*v) {
+				continue nextPackage
+			}
+			if v.GreaterThan(*currentLatest) {
+				currentLatest = v
+			}
+		}
+		outdatedEntries = append(outdatedEntries, &models.OutdatedPackages{
+			Atom:          p.Atom,
+			GentooVersion: currentLatest.Version,
+			NewestVersion: anitya.StableVersion,
+			Source:        models.OutdatedSourceAnitya,
+		})
+	}
+	_, err = database.DBCon.Model(&packages).Set("anitya_info = ?anitya_info").Update()
+	if err != nil {
+		slog.Error("Failed updating packages", slog.Any("err", err))
+		return
+	}
+	slog.Info("Updated anitya information", slog.Int("count", len(packages)))
+
+	_, _ = database.DBCon.Model((*models.OutdatedPackages)(nil)).Where("source = ?", models.OutdatedSourceAnitya).Delete()
+	res, err := database.DBCon.Model(&outdatedEntries).OnConflict("(atom) DO UPDATE").Insert()
+	if err != nil {
+		slog.Error("Error while inserting outdated packages", slog.Any("err", err))
+	} else {
+		slog.Info("Inserted outdated packages", slog.Int("res", res.RowsAffected()))
+	}
+
+	updateStatus()
+}
+
+var client = http.Client{Timeout: 1 * time.Minute}
+
+func fetchResults(page int, params url.Values) (int, []ApiPackage, error) {
+	req, err := http.NewRequest("GET", "https://release-monitoring.org/api/v2/packages/?"+params.Encode(), nil)
+	if err != nil {
+		slog.Error("Failed creating request", slog.Int("page", page), slog.Any("err", err))
+		return 0, nil, err
+	}
+	req.Header.Set("User-Agent", config.UserAgent())
+
+	resp, err := client.Do(req)
+	if err != nil {
+		slog.Error("Failed fetching anitya data", slog.Int("page", page), slog.Any("err", err))
+		return 0, nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		slog.Error("Failed fetching anitya data", slog.Int("page", page), slog.Int("status", resp.StatusCode))
+		return 0, nil, nil
+	}
+
+	var data ApiResponse
+	err = json.NewDecoder(resp.Body).Decode(&data)
+	return data.TotalItems, data.Items, err
+}
+
+func readAllResults() (result []ApiPackage, err error) {
+	params := url.Values{
+		"distribution":   {"Gentoo"},
+		"items_per_page": {strconv.Itoa(itemsPerPage)},
+	}
+	totalPages := 1
+	for page := 1; page <= totalPages; page++ {
+		slog.Info("Fetching anitya data", slog.Int("page", page))
+		params.Set("page", strconv.Itoa(page))
+		total, items, err := fetchResults(page, params)
+		if err != nil {
+			return nil, err
+		}
+
+		if page == 1 {
+			totalPages = (total + itemsPerPage - 1) / itemsPerPage
+			result = make([]ApiPackage, 0, total)
+		}
+		result = append(result, items...)
+	}
+	return
+}
+
+func (p *ApiPackage) LatestVersion() (result string) {
+	result = p.Version
+	if p.StableVersion != "" {
+		result = p.StableVersion
+	}
+	result, _, _ = strings.Cut(result, ".post")
+	return
+}
+
+func updateStatus() {
+	database.DBCon.Model(&models.Application{
+		Id:         "anitya",
+		LastUpdate: time.Now(),
+		Version:    config.Version(),
+	}).OnConflict("(id) DO UPDATE").Insert()
+}
diff --git a/pkg/portage/repology/category.go b/pkg/portage/repology/category.go
new file mode 100644
index 0000000..41efe57
--- /dev/null
+++ b/pkg/portage/repology/category.go
@@ -0,0 +1,58 @@
+package repology
+
+import (
+	"log/slog"
+
+	"soko/pkg/database"
+	"soko/pkg/models"
+)
+
+func UpdateCategoriesMetadata() {
+	var categoriesInfoArr []*models.CategoryPackagesInformation
+	err := database.DBCon.Model((*models.OutdatedPackages)(nil)).
+		ColumnExpr("SPLIT_PART(atom, '/', 1) as name").
+		ColumnExpr("COUNT(*) as outdated").
+		GroupExpr("SPLIT_PART(atom, '/', 1)").
+		Select(&categoriesInfoArr)
+	if err != nil {
+		slog.Error("Failed collecting outdated stats", slog.Any("err", err))
+		return
+	}
+	categoriesInfo := make(map[string]*models.CategoryPackagesInformation, len(categoriesInfoArr))
+	for _, categoryInfo := range categoriesInfoArr {
+		if categoryInfo.Name != "" {
+			categoriesInfo[categoryInfo.Name] = categoryInfo
+		}
+	}
+
+	var categories []*models.CategoryPackagesInformation
+	err = database.DBCon.Model(&categories).Column("name").Select()
+	if err != nil {
+		slog.Error("Failed fetching categories packages information", slog.Any("err", err))
+		return
+	} else if len(categories) > 0 {
+		for _, category := range categories {
+			if info, found := categoriesInfo[category.Name]; found {
+				category.Outdated = info.Outdated
+			} else {
+				category.Outdated = 0
+			}
+			delete(categoriesInfo, category.Name)
+		}
+		_, err = database.DBCon.Model(&categories).Set("outdated = ?outdated").Update()
+		if err != nil {
+			slog.Error("Failed updating categories packages information", slog.Any("err", err))
+		}
+		categories = make([]*models.CategoryPackagesInformation, 0, len(categoriesInfo))
+	}
+
+	for _, catInfo := range categoriesInfo {
+		categories = append(categories, catInfo)
+	}
+	if len(categories) > 0 {
+		_, err = database.DBCon.Model(&categories).Insert()
+		if err != nil {
+			slog.Error("Failed inserting categories packages information", slog.Any("err", err))
+		}
+	}
+}
diff --git a/pkg/portage/repology/outdated.go b/pkg/portage/repology/outdated.go
index 93f9008..794697b 100644
--- a/pkg/portage/repology/outdated.go
+++ b/pkg/portage/repology/outdated.go
@@ -32,9 +32,6 @@ var clientRateLimiter = rate.NewLimiter(rate.Every(2*time.Second), 1)
 
 // UpdateOutdated will update the database table that contains all outdated gentoo versions
 func UpdateOutdated() {
-	database.Connect()
-	defer database.DBCon.Close()
-
 	// Get all outdated Versions
 	outdated := newOutdatedCheck()
 	for letter := 'a'; letter <= 'z'; letter++ {
@@ -43,9 +40,9 @@ func UpdateOutdated() {
 
 	// Update the database
 	if len(outdated.outdatedVersions) > 0 {
-		database.TruncateTable((*models.OutdatedPackages)(nil))
+		_, _ = database.DBCon.Model((*models.OutdatedPackages)(nil)).Where("source = ?", models.OutdatedSourceRepology).Delete()
 
-		res, err := database.DBCon.Model(&outdated.outdatedVersions).Insert()
+		res, err := database.DBCon.Model(&outdated.outdatedVersions).OnConflict("(atom) DO NOTHING").Insert()
 		if err != nil {
 			slog.Error("Error while inserting outdated packages", slog.Any("err", err))
 		} else {
@@ -53,37 +50,6 @@ func UpdateOutdated() {
 		}
 	}
 
-	// Updated the outdated status of categories
-	var categories []*models.CategoryPackagesInformation
-	err := database.DBCon.Model(&categories).Column("name").Select()
-	if err != nil {
-		slog.Error("Failed fetching categories packages information", slog.Any("err", err))
-		return
-	} else if len(categories) > 0 {
-		for _, category := range categories {
-			category.Outdated = outdated.outdatedCategories[category.Name]
-			delete(outdated.outdatedCategories, category.Name)
-		}
-		_, err = database.DBCon.Model(&categories).Set("outdated = ?outdated").Update()
-		if err != nil {
-			slog.Error("Failed updating categories packages information", slog.Any("err", err))
-		}
-		categories = make([]*models.CategoryPackagesInformation, 0, len(outdated.outdatedCategories))
-	}
-
-	for category, count := range outdated.outdatedCategories {
-		categories = append(categories, &models.CategoryPackagesInformation{
-			Name:     category,
-			Outdated: count,
-		})
-	}
-	if len(categories) > 0 {
-		_, err = database.DBCon.Model(&categories).Insert()
-		if err != nil {
-			slog.Error("Error while inserting categories packages information", slog.Any("err", err))
-		}
-	}
-
 	updateStatus()
 }
 
@@ -131,8 +97,7 @@ type outdatedCheck struct {
 	blockedCategories map[string]struct{}
 	atomRules         map[string]*atomOutdatedRules
 
-	outdatedCategories map[string]int
-	outdatedVersions   []*models.OutdatedPackages
+	outdatedVersions []*models.OutdatedPackages
 }
 
 func newOutdatedCheck() outdatedCheck {
@@ -140,8 +105,6 @@ func newOutdatedCheck() outdatedCheck {
 		blockedRepos:      readBlockList("ignored-repositories"),
 		blockedCategories: readBlockList("ignored-categories"),
 		atomRules:         buildAtomRules(),
-
-		outdatedCategories: make(map[string]int),
 	}
 }
 
@@ -206,12 +169,8 @@ func (o *outdatedCheck) getOutdatedStartingWith(letter rune) {
 					Atom:          atom,
 					GentooVersion: currentVersion[atom],
 					NewestVersion: newestVersion,
+					Source:        models.OutdatedSourceRepology,
 				})
-
-				category, _, found := strings.Cut(atom, "/")
-				if found {
-					o.outdatedCategories[category]++
-				}
 			}
 		}
 	}
diff --git a/soko.go b/soko.go
index c38f660..9afe675 100644
--- a/soko.go
+++ b/soko.go
@@ -13,7 +13,9 @@ import (
 
 	"soko/pkg/app"
 	"soko/pkg/config"
+	"soko/pkg/database"
 	"soko/pkg/portage"
+	"soko/pkg/portage/anitya"
 	"soko/pkg/portage/bugs"
 	"soko/pkg/portage/dependencies"
 	"soko/pkg/portage/github"
@@ -58,8 +60,7 @@ func main() {
 		portage.FullUpdate()
 	}
 	if *updateOutdatedPackages {
-		slog.Info("Updating the repology data")
-		repology.UpdateOutdated()
+		updateOutdated()
 	}
 	if *updatePkgcheckResults {
 		slog.Info("Updating the qa-reports that is the pkgcheck data")
@@ -96,6 +97,18 @@ func main() {
 	}
 }
 
+func updateOutdated() {
+	database.Connect()
+	defer database.DBCon.Close()
+
+	slog.Info("Updating the anitya data")
+	anitya.UpdateAnitya()
+	slog.Info("Updating the repology data")
+	repology.UpdateOutdated()
+
+	repology.UpdateCategoriesMetadata()
+}
+
 // initialize the loggers depending on whether
 // config.debug is set to true
 func initLoggers() {