From 44b036f4c75ef78f8a1d68da3ff99d9b116d1fbb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 30 Oct 2026 23:03:50 -0400 Subject: [PATCH 1/3] follow moving mirrorlists --- repos/tests/test_mirror_sync.py | 64 +++++++++++++++++++++++++++++++++ repos/utils.py | 60 +++++++++++++++++++++++++------ 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/repos/tests/test_mirror_sync.py b/repos/tests/test_mirror_sync.py index ca32fdd9..dcacbac4 100644 --- a/repos/tests/test_mirror_sync.py +++ b/repos/tests/test_mirror_sync.py @@ -14,11 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from unittest.mock import patch + from django.test import TestCase, override_settings from arch.models import MachineArchitecture, PackageArchitecture from packages.models import Package, PackageName from repos.models import Mirror, MirrorPackage, Repository +from repos.utils import ( + add_mirrors_from_urls, check_for_mirrorlists, get_latest_mirrorlist_url, +) + +AL2023 = 'https://cdn.amazonlinux.com/al2023/core' @override_settings( @@ -139,3 +146,60 @@ def test_mirror_fail_disables_after_threshold(self): def test_mirror_str(self): """Test mirror string representation.""" self.assertIn(self.mirror.url, str(self.mirror)) + + +@override_settings(MAX_MIRRORS=2) +class MirrorlistTests(TestCase): + """Tests for adding mirrors from mirrorlists.""" + + def setUp(self): + arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='Amazon Linux 2023 repository', + arch=arch, + repotype=Repository.RPM, + ) + + def _urls(self): + return set(self.repo.mirror_set.values_list('url', flat=True)) + + def test_mirrors_limited_to_max(self): + """Test only MAX_MIRRORS mirrors are added from a list.""" + add_mirrors_from_urls(self.repo, [f'http://m{i}.example.com/repo' for i in range(5)]) + self.assertEqual(self.repo.mirror_set.count(), 2) + + def test_previous_mirrors_do_not_block_current(self): + """Test mirrors no longer in the list do not count towards the limit.""" + for guid in ('a' * 64, 'b' * 64): + Mirror.objects.create(repo=self.repo, url=f'{AL2023}/guids/{guid}/x86_64') + current = f'{AL2023}/guids/{"c" * 64}/x86_64' + add_mirrors_from_urls(self.repo, [current + '/']) + self.assertIn(current, self._urls()) + + def test_latest_mirrorlist_url(self): + """Test pinned AL2023 mirrorlists map to the moving latest one.""" + pinned = f'{AL2023}/mirrors/2023.12.20261102/x86_64/mirror.list' + self.assertEqual( + get_latest_mirrorlist_url(pinned), + f'{AL2023}/mirrors/latest/x86_64/mirror.list', + ) + regional = 'https://al2023-repos-us-east-1-de612dc2.s3.dualstack.us-east-1.amazonaws.com/core/mirrors' + self.assertEqual( + get_latest_mirrorlist_url(f'{regional}/2023.6.20241010/SRPMS/mirror.list'), + f'{regional}/latest/SRPMS/mirror.list', + ) + self.assertIsNone(get_latest_mirrorlist_url(f'{AL2023}/mirrors/latest/x86_64/mirror.list')) + self.assertIsNone(get_latest_mirrorlist_url('https://mirrors.example.com/mirrorlist?repo=x')) + + def test_pinned_mirrorlist_follows_latest(self): + """Test a repo with only a pinned mirrorlist also gets the latest release.""" + pinned = f'{AL2023}/mirrors/2023.6.20241010/x86_64/mirror.list' + latest = f'{AL2023}/mirrors/latest/x86_64/mirror.list' + old_guid = f'{AL2023}/guids/{"a" * 64}/x86_64' + new_guid = f'{AL2023}/guids/{"b" * 64}/x86_64' + Mirror.objects.create(repo=self.repo, url=pinned) + lists = {pinned: [old_guid + '/'], latest: [new_guid + '/']} + with patch('repos.utils.get_mirrorlist_urls', side_effect=lambda url: lists.get(url)): + check_for_mirrorlists(self.repo) + self.assertEqual(self._urls(), {pinned, latest, old_guid, new_guid}) + self.assertTrue(Mirror.objects.get(url=latest).mirrorlist) diff --git a/repos/utils.py b/repos/utils.py index 87bc2192..972697ee 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -169,33 +169,73 @@ def get_mirrorlist_urls(url): def add_mirrors_from_urls(repo, mirror_urls): - """ Creates mirrors from a list of mirror urls + """ Creates up to MAX_MIRRORS mirrors from a list of mirror urls. + Only mirrors in the list count towards the limit, so that mirrors + from a previous version of the list (e.g. a list that moves to a + new immutable repo on each release) do not block the current ones. """ + from repos.models import Mirror max_mirrors = get_max_mirrors() + urls = [] for mirror_url in mirror_urls: mirror_url = mirror_url.replace('$ARCH', repo.arch.name) mirror_url = mirror_url.replace('$basearch', repo.arch.name) - mirror_url = mirror_url.rstrip('/') - q = Q(mirrorlist=False, refresh=True, enabled=True) - existing = repo.mirror_set.filter(q).count() + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? + # that is what metalink urls return now + urls.append(mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) + q = Q(mirrorlist=False, refresh=True, enabled=True, url__in=urls) + existing = repo.mirror_set.filter(q).count() + for url in urls: if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' warning_message(text=text) break - from repos.models import Mirror - - # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? - # that is what metalink urls return now - m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) + m, c = Mirror.objects.get_or_create(repo=repo, url=url) if c: - text = f'Added Mirror - {mirror_url}' + existing += 1 + text = f'Added Mirror - {url}' info_message(text=text) +# mirrorlists pinned to a release, and the moving mirrorlist to also follow +LATEST_MIRRORLISTS = [ + # Amazon Linux 2023 locks releasever by default, so hosts report + # e.g. .../core/mirrors/2023.12.20261102/x86_64/mirror.list + ( + re.compile( + r'^(https?://[^/]*(?:amazonlinux\.com/al2023|al2023-repos-[^/]*)/[^/]+/mirrors/)' + r'(?!latest/)[^/]+(/.*mirror\.list)$' + ), + r'\1latest\2', + ), +] + + +def get_latest_mirrorlist_url(url): + """ Returns the moving mirrorlist url for a mirrorlist pinned to a release + """ + for regex, replacement in LATEST_MIRRORLISTS: + if regex.match(url): + return regex.sub(replacement, url) + + +def add_latest_mirrorlists(repo): + """ Adds the moving mirrorlist for any mirrorlists pinned to a release, + so that new releases are found even if all hosts are pinned + """ + from repos.models import Mirror + for mirror in repo.mirror_set.all(): + latest_url = get_latest_mirrorlist_url(mirror.url) + if latest_url and not Mirror.objects.filter(url=latest_url).exists(): + Mirror.objects.create(repo=repo, url=latest_url) + info_message(text=f'Added latest mirrorlist - {latest_url}') + + def check_for_mirrorlists(repo): """ Check if any of the mirrors are actually mirrorlists. Creates MAX_MIRRORS mirrors from list if so. """ + add_latest_mirrorlists(repo) for mirror in repo.mirror_set.all(): if not mirror.url.startswith(('http://', 'https://')): warning_message(text=f'Skipping non-http(s) mirror URL: {mirror.url}') From f192fb18d04c67349a987ebf82963e69173b1164 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Nov 2026 23:33:40 -0500 Subject: [PATCH 2/3] store metalink mirror urls without trailing slash --- repos/tests/test_mirror_sync.py | 7 +++++++ repos/utils.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/repos/tests/test_mirror_sync.py b/repos/tests/test_mirror_sync.py index dcacbac4..049c82a2 100644 --- a/repos/tests/test_mirror_sync.py +++ b/repos/tests/test_mirror_sync.py @@ -176,6 +176,13 @@ def test_previous_mirrors_do_not_block_current(self): add_mirrors_from_urls(self.repo, [current + '/']) self.assertIn(current, self._urls()) + def test_metalink_and_mirrorlist_urls_match(self): + """Test metalink repomd.xml urls are stored like mirrorlist base urls.""" + base = 'http://m.example.com/repo/os' + add_mirrors_from_urls(self.repo, [f'{base}/repodata/repomd.xml']) + add_mirrors_from_urls(self.repo, [f'{base}/']) + self.assertEqual(self._urls(), {base}) + def test_latest_mirrorlist_url(self): """Test pinned AL2023 mirrorlists map to the moving latest one.""" pinned = f'{AL2023}/mirrors/2023.12.20261102/x86_64/mirror.list' diff --git a/repos/utils.py b/repos/utils.py index 972697ee..de9c59d4 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -180,9 +180,9 @@ def add_mirrors_from_urls(repo, mirror_urls): for mirror_url in mirror_urls: mirror_url = mirror_url.replace('$ARCH', repo.arch.name) mirror_url = mirror_url.replace('$basearch', repo.arch.name) - # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? - # that is what metalink urls return now - urls.append(mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) + # metalinks return the full path to repomd.xml, mirrorlists the base url + mirror_url = mirror_url.rstrip('/').removesuffix('/repodata/repomd.xml') + urls.append(mirror_url.rstrip('/')) q = Q(mirrorlist=False, refresh=True, enabled=True, url__in=urls) existing = repo.mirror_set.filter(q).count() for url in urls: From 5b48227705291a9edfafd2fba808416125c33419 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 4 Nov 2026 00:03:52 -0500 Subject: [PATCH 3/3] follow moving mirrorlists on local mirrors and amazon linux 2 --- repos/tests/test_mirror_sync.py | 21 +++++++++++++++++++++ repos/utils.py | 24 ++++++++++++++---------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/repos/tests/test_mirror_sync.py b/repos/tests/test_mirror_sync.py index 049c82a2..a35fda9d 100644 --- a/repos/tests/test_mirror_sync.py +++ b/repos/tests/test_mirror_sync.py @@ -195,7 +195,18 @@ def test_latest_mirrorlist_url(self): get_latest_mirrorlist_url(f'{regional}/2023.6.20241010/SRPMS/mirror.list'), f'{regional}/latest/SRPMS/mirror.list', ) + local = 'http://mirror.example.com/al2027/core/mirrors' + self.assertEqual( + get_latest_mirrorlist_url(f'{local}/2027.0.20260803/aarch64/mirror.list'), + f'{local}/latest/aarch64/mirror.list', + ) + al2 = 'https://amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com/2/core' + self.assertEqual( + get_latest_mirrorlist_url(f'{al2}/2.0.20231101.0/x86_64/mirror.list'), + f'{al2}/latest/x86_64/mirror.list', + ) self.assertIsNone(get_latest_mirrorlist_url(f'{AL2023}/mirrors/latest/x86_64/mirror.list')) + self.assertIsNone(get_latest_mirrorlist_url(f'{al2}/latest/x86_64/mirror.list')) self.assertIsNone(get_latest_mirrorlist_url('https://mirrors.example.com/mirrorlist?repo=x')) def test_pinned_mirrorlist_follows_latest(self): @@ -210,3 +221,13 @@ def test_pinned_mirrorlist_follows_latest(self): check_for_mirrorlists(self.repo) self.assertEqual(self._urls(), {pinned, latest, old_guid, new_guid}) self.assertTrue(Mirror.objects.get(url=latest).mirrorlist) + + def test_missing_latest_mirrorlist_not_added(self): + """Test the latest mirrorlist is not added if it does not exist, e.g. on a local mirror.""" + pinned = 'http://mirror.example.com/al2023/core/mirrors/2023.6.20241010/x86_64/mirror.list' + guid = f'http://mirror.example.com/al2023/core/guids/{"a" * 64}/x86_64' + Mirror.objects.create(repo=self.repo, url=pinned) + lists = {pinned: [guid]} + with patch('repos.utils.get_mirrorlist_urls', side_effect=lambda url: lists.get(url)): + check_for_mirrorlists(self.repo) + self.assertEqual(self._urls(), {pinned, guid}) diff --git a/repos/utils.py b/repos/utils.py index de9c59d4..e5ff68ab 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -197,15 +197,14 @@ def add_mirrors_from_urls(repo, mirror_urls): info_message(text=text) -# mirrorlists pinned to a release, and the moving mirrorlist to also follow +# mirrorlists pinned to a release, and the moving mirrorlist to also follow. +# Matched on the path only, so local mirrors and proxies of these are covered. LATEST_MIRRORLISTS = [ - # Amazon Linux 2023 locks releasever by default, so hosts report + # Amazon Linux 2023 and later lock releasever by default, so hosts report # e.g. .../core/mirrors/2023.12.20261102/x86_64/mirror.list + # Amazon Linux 2 can be locked to e.g. .../2/core/2.0.20231101.0/x86_64/mirror.list ( - re.compile( - r'^(https?://[^/]*(?:amazonlinux\.com/al2023|al2023-repos-[^/]*)/[^/]+/mirrors/)' - r'(?!latest/)[^/]+(/.*mirror\.list)$' - ), + re.compile(r'^(https?://.+/)\d+(?:\.\d+){2,3}(/[^/]+/mirror\.list)$'), r'\1latest\2', ), ] @@ -221,14 +220,19 @@ def get_latest_mirrorlist_url(url): def add_latest_mirrorlists(repo): """ Adds the moving mirrorlist for any mirrorlists pinned to a release, - so that new releases are found even if all hosts are pinned + so that new releases are found even if all hosts are pinned. + Only added if it exists, as a local mirror may only carry pinned releases. """ from repos.models import Mirror for mirror in repo.mirror_set.all(): latest_url = get_latest_mirrorlist_url(mirror.url) - if latest_url and not Mirror.objects.filter(url=latest_url).exists(): - Mirror.objects.create(repo=repo, url=latest_url) - info_message(text=f'Added latest mirrorlist - {latest_url}') + if not latest_url or Mirror.objects.filter(url=latest_url).exists(): + continue + if not get_mirrorlist_urls(latest_url): + debug_message(text=f'No latest mirrorlist found at {latest_url}') + continue + Mirror.objects.create(repo=repo, url=latest_url) + info_message(text=f'Added latest mirrorlist - {latest_url}') def check_for_mirrorlists(repo):