From 98c0dabdd119b790c352f71f069d71c90f93f0d4 Mon Sep 17 00:00:00 2001 From: Arthur Zamarin Date: Sat, 16 Mar 2024 18:05:44 +0200 Subject: updater/repology: support versioned atoms & selected repositories Resolves: https://github.com/gentoo/soko/issues/11 Resolves: https://github.com/gentoo/soko/issues/12 Signed-off-by: Arthur Zamarin --- pkg/portage/repology/outdated.go | 176 +++++++++++++++++++++++++++++---------- 1 file changed, 132 insertions(+), 44 deletions(-) diff --git a/pkg/portage/repology/outdated.go b/pkg/portage/repology/outdated.go index fbdb280..aa8b6c9 100644 --- a/pkg/portage/repology/outdated.go +++ b/pkg/portage/repology/outdated.go @@ -1,12 +1,13 @@ package repology import ( - "bytes" + "bufio" "context" "encoding/json" "fmt" "log/slog" "net/http" + "regexp" "soko/pkg/config" "soko/pkg/database" "soko/pkg/models" @@ -24,7 +25,7 @@ type Package struct { Status string `json:"status"` } -type Packages map[string][]Package +type Packages = map[string][]Package var client = http.Client{Timeout: 1 * time.Minute} var clientRateLimiter = rate.NewLimiter(rate.Every(2*time.Second), 1) @@ -35,18 +36,21 @@ func UpdateOutdated() { defer database.DBCon.Close() // Get all outdated Versions - outdatedCategories := make(map[string]int) - var outdatedVersions []*models.OutdatedPackages - letters := "abcdefghijklmnopqrstuvwxyz" - for _, letter := range letters { - outdatedVersions = append(outdatedVersions, getOutdatedStartingWith(letter, outdatedCategories)...) + outdated := newOutdatedCheck() + for letter := 'a'; letter <= 'z'; letter++ { + outdated.getOutdatedStartingWith(letter) } // Update the database - if len(outdatedVersions) > 0 { + if len(outdated.outdatedVersions) > 0 { database.TruncateTable[models.OutdatedPackages]("atom") - database.DBCon.Model(&outdatedVersions).Insert() + res, err := database.DBCon.Model(&outdated.outdatedVersions).Insert() + if err != nil { + slog.Error("Error while inserting outdated packages", slog.Any("err", err)) + } else { + slog.Info("Inserted outdated packages", slog.Int("res", res.RowsAffected())) + } } // Updated the outdated status of categories @@ -57,17 +61,17 @@ func UpdateOutdated() { return } else if len(categories) > 0 { for _, category := range categories { - category.Outdated = outdatedCategories[category.Name] - delete(outdatedCategories, category.Name) + category.Outdated = outdated.outdatedCategories[category.Name] + delete(outdated.outdatedCategories, category.Name) } _, err = database.DBCon.Model(&categories).Set("outdated = ?outdated").Update() if err != nil { slog.Error("Failed updating categories packages information", slog.Any("err", err)) } - categories = make([]*models.CategoryPackagesInformation, 0, len(outdatedCategories)) + categories = make([]*models.CategoryPackagesInformation, 0, len(outdated.outdatedCategories)) } - for category, count := range outdatedCategories { + for category, count := range outdated.outdatedCategories { categories = append(categories, &models.CategoryPackagesInformation{ Name: category, Outdated: count, @@ -83,19 +87,71 @@ func UpdateOutdated() { updateStatus() } +type atomOutdatedRules struct { + ignore bool + ignoreVersions []string + ignoreRepos []string + selectedRepos []string +} + +func (a *atomOutdatedRules) isIgnored(version string, repo string) bool { + if a == nil { + return false + } else if a.ignore { + return true + } + + for _, v := range a.ignoreVersions { + if strings.HasPrefix(version, v) { + return true + } + } + for _, r := range a.ignoreRepos { + if strings.HasPrefix(repo, r) { + return true + } + } + if len(a.selectedRepos) > 0 { + found := false + for _, r := range a.selectedRepos { + if strings.HasPrefix(repo, r) { + found = true + break + } + } + if !found { + return true + } + } + return false +} + +type outdatedCheck struct { + blockedRepos map[string]struct{} + blockedCategories map[string]struct{} + atomRules map[string]*atomOutdatedRules + + outdatedCategories map[string]int + outdatedVersions []*models.OutdatedPackages +} + +func newOutdatedCheck() outdatedCheck { + return outdatedCheck{ + blockedRepos: readBlockList("ignored-repositories"), + blockedCategories: readBlockList("ignored-categories"), + atomRules: buildAtomRules(), + + outdatedCategories: make(map[string]int), + } +} + // getOutdatedStartingWith gets all outdated packages starting with the given letter -func getOutdatedStartingWith(letter rune, outdatedCategories map[string]int) []*models.OutdatedPackages { +func (o *outdatedCheck) getOutdatedStartingWith(letter rune) { repoPackages, err := parseRepologyData(letter) if err != nil { slog.Error("Error while fetching repology data", slog.String("letter", string(letter)), slog.Any("err", err)) - return nil } - blockedRepos := readBlockList("ignored-repositories") - blockedCategories := readBlockList("ignored-categories") - blockedPackages := readBlockList("ignored-packages") - - var outdatedVersions []*models.OutdatedPackages for packagename := range repoPackages { outdated := make(map[string]bool) currentVersion := make(map[string]string) @@ -114,9 +170,10 @@ func getOutdatedStartingWith(letter rune, outdatedCategories map[string]int) []* if v.Repo == "gentoo" { if v.Status == "newest" { outdated[v.VisibleName] = false - } else if v.Status == "outdated" && - !containsPrefix(blockedCategories, category) && - !containsPrefix(blockedPackages, v.VisibleName) { + } else if v.Status == "outdated" { + if contains(o.blockedCategories, category) || o.atomRules[v.VisibleName].isIgnored(v.Version, v.Repo) { + continue + } if _, found := outdated[v.VisibleName]; !found { outdated[v.VisibleName] = true } @@ -129,9 +186,9 @@ func getOutdatedStartingWith(letter rune, outdatedCategories map[string]int) []* currentVersion[v.VisibleName] = v.Version } } - } else if len(newestVersion) == 0 && v.Status == "newest" && !contains(blockedRepos, v.Repo) { + } else if len(newestVersion) == 0 && v.Status == "newest" && !contains(o.blockedRepos, v.Repo) { for atom := range gentooPackages { - if contains(blockedPackages, atom+"::"+v.Repo) { + if o.atomRules[atom].isIgnored(v.Version, v.Repo) { continue mainLoop } } @@ -145,7 +202,7 @@ func getOutdatedStartingWith(letter rune, outdatedCategories map[string]int) []* for atom, outdated := range outdated { if outdated && packagename[0] == byte(letter) { - outdatedVersions = append(outdatedVersions, &models.OutdatedPackages{ + o.outdatedVersions = append(o.outdatedVersions, &models.OutdatedPackages{ Atom: atom, GentooVersion: currentVersion[atom], NewestVersion: newestVersion, @@ -153,13 +210,11 @@ func getOutdatedStartingWith(letter rune, outdatedCategories map[string]int) []* category, _, found := strings.Cut(atom, "/") if found { - outdatedCategories[category]++ + o.outdatedCategories[category]++ } } } } - - return outdatedVersions } // parseRepologyData gets the json from given url and parses it @@ -197,15 +252,19 @@ func readBlockList(file string) map[string]struct{} { blocklist := make(map[string]struct{}) resp, err := client.Get("https://gitweb.gentoo.org/sites/soko-metadata.git/plain/repology/" + file) if err != nil { + slog.Error("Failed to fetch blacklist", slog.String("file", file), slog.Any("err", err)) return blocklist } defer resp.Body.Close() - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - rawBlocklist := buf.String() + if resp.StatusCode != http.StatusOK { + slog.Error("Failed to fetch blacklist", slog.String("file", file), slog.String("status", resp.Status)) + return blocklist + } - for _, line := range strings.Split(rawBlocklist, "\n") { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() if !strings.HasPrefix(line, "#") && strings.TrimSpace(line) != "" { blocklist[line] = struct{}{} } @@ -213,6 +272,46 @@ func readBlockList(file string) map[string]struct{} { return blocklist } +func buildAtomRules() map[string]*atomOutdatedRules { + var versionNumber = regexp.MustCompile(`-[0-9]`) + + blacklist := readBlockList("ignored-packages") + whitelist := readBlockList("selected-packages") + + atomRules := make(map[string]*atomOutdatedRules, len(blacklist)+len(whitelist)) + for line := range blacklist { + cpv, repo, hasRepo := strings.Cut(line, "::") + atom := versionNumber.Split(cpv, 2)[0] + rule, found := atomRules[atom] + if !found { + rule = &atomOutdatedRules{} + atomRules[atom] = rule + } + if hasRepo && repo != "" { + rule.ignoreRepos = append(rule.ignoreRepos, repo) + } else if atom != cpv { + rule.ignoreVersions = append(rule.ignoreVersions, strings.TrimPrefix(cpv, atom+"-")) + } else { + rule.ignore = true + } + } + + for line := range whitelist { + cpv, repo, hasRepo := strings.Cut(line, "::") + atom := versionNumber.Split(cpv, 2)[0] + rule, found := atomRules[atom] + if !found { + rule = &atomOutdatedRules{} + atomRules[atom] = rule + } + if hasRepo && repo != "" { + rule.selectedRepos = append(rule.selectedRepos, repo) + } + } + + return atomRules +} + // contains returns true if the given list includes // the given string. Otherwise false is returned. func contains(list map[string]struct{}, item string) bool { @@ -220,17 +319,6 @@ func contains(list map[string]struct{}, item string) bool { return found } -// contains returns true if the given string is a prefix -// of an item in the given list. Otherwise false is returned. -func containsPrefix(list map[string]struct{}, item string) bool { - for i := range list { - if strings.HasPrefix(i, item) { - return true - } - } - return false -} - func updateStatus() { database.DBCon.Model(&models.Application{ Id: "repology", -- cgit v1.2.3-65-gdbad