follow moving mirrorlists

repos/tests/test_mirror_sync.py

@@ -14,11 +14,18 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
+from unittest.mock import patch
+
 from django.test import TestCase, override_settings
 
 from arch.models import MachineArchitecture, PackageArchitecture
 from packages.models import Package, PackageName
 from repos.models import Mirror, MirrorPackage, Repository
+from repos.utils import (
+    add_mirrors_from_urls, check_for_mirrorlists, get_latest_mirrorlist_url,
+)
+
+AL2023 = 'https://cdn.amazonlinux.com/al2023/core'
 
 
 @override_settings(
@@ -139,3 +146,88 @@ def test_mirror_fail_disables_after_threshold(self):
     def test_mirror_str(self):
         """Test mirror string representation."""
         self.assertIn(self.mirror.url, str(self.mirror))
+
+
+@override_settings(MAX_MIRRORS=2)
+class MirrorlistTests(TestCase):
+    """Tests for adding mirrors from mirrorlists."""
+
+    def setUp(self):
+        arch = MachineArchitecture.objects.create(name='x86_64')
+        self.repo = Repository.objects.create(
+            name='Amazon Linux 2023 repository',
+            arch=arch,
+            repotype=Repository.RPM,
+        )
+
+    def _urls(self):
+        return set(self.repo.mirror_set.values_list('url', flat=True))
+
+    def test_mirrors_limited_to_max(self):
+        """Test only MAX_MIRRORS mirrors are added from a list."""
+        add_mirrors_from_urls(self.repo, [f'http://m{i}.example.com/repo' for i in range(5)])
+        self.assertEqual(self.repo.mirror_set.count(), 2)
+
+    def test_previous_mirrors_do_not_block_current(self):
+        """Test mirrors no longer in the list do not count towards the limit."""
+        for guid in ('a' * 64, 'b' * 64):
+            Mirror.objects.create(repo=self.repo, url=f'{AL2023}/guids/{guid}/x86_64')
+        current = f'{AL2023}/guids/{"c" * 64}/x86_64'
+        add_mirrors_from_urls(self.repo, [current + '/'])
+        self.assertIn(current, self._urls())
+
+    def test_metalink_and_mirrorlist_urls_match(self):
+        """Test metalink repomd.xml urls are stored like mirrorlist base urls."""
+        base = 'http://m.example.com/repo/os'
+        add_mirrors_from_urls(self.repo, [f'{base}/repodata/repomd.xml'])
+        add_mirrors_from_urls(self.repo, [f'{base}/'])
+        self.assertEqual(self._urls(), {base})
+
+    def test_latest_mirrorlist_url(self):
+        """Test pinned AL2023 mirrorlists map to the moving latest one."""
+        pinned = f'{AL2023}/mirrors/2023.12.20261102/x86_64/mirror.list'
+        self.assertEqual(
+            get_latest_mirrorlist_url(pinned),
+            f'{AL2023}/mirrors/latest/x86_64/mirror.list',
+        )
+        regional = 'https://al2023-repos-us-east-1-de612dc2.s3.dualstack.us-east-1.amazonaws.com/core/mirrors'
+        self.assertEqual(
+            get_latest_mirrorlist_url(f'{regional}/2023.6.20241010/SRPMS/mirror.list'),
+            f'{regional}/latest/SRPMS/mirror.list',
+        )
+        local = 'http://mirror.example.com/al2027/core/mirrors'
+        self.assertEqual(
+            get_latest_mirrorlist_url(f'{local}/2027.0.20260803/aarch64/mirror.list'),
+            f'{local}/latest/aarch64/mirror.list',
+        )
+        al2 = 'https://amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com/2/core'
+        self.assertEqual(
+            get_latest_mirrorlist_url(f'{al2}/2.0.20231101.0/x86_64/mirror.list'),
+            f'{al2}/latest/x86_64/mirror.list',
+        )
+        self.assertIsNone(get_latest_mirrorlist_url(f'{AL2023}/mirrors/latest/x86_64/mirror.list'))
+        self.assertIsNone(get_latest_mirrorlist_url(f'{al2}/latest/x86_64/mirror.list'))
+        self.assertIsNone(get_latest_mirrorlist_url('https://mirrors.example.com/mirrorlist?repo=x'))
+
+    def test_pinned_mirrorlist_follows_latest(self):
+        """Test a repo with only a pinned mirrorlist also gets the latest release."""
+        pinned = f'{AL2023}/mirrors/2023.6.20241010/x86_64/mirror.list'
+        latest = f'{AL2023}/mirrors/latest/x86_64/mirror.list'
+        old_guid = f'{AL2023}/guids/{"a" * 64}/x86_64'
+        new_guid = f'{AL2023}/guids/{"b" * 64}/x86_64'
+        Mirror.objects.create(repo=self.repo, url=pinned)
+        lists = {pinned: [old_guid + '/'], latest: [new_guid + '/']}
+        with patch('repos.utils.get_mirrorlist_urls', side_effect=lambda url: lists.get(url)):
+            check_for_mirrorlists(self.repo)
+        self.assertEqual(self._urls(), {pinned, latest, old_guid, new_guid})
+        self.assertTrue(Mirror.objects.get(url=latest).mirrorlist)
+
+    def test_missing_latest_mirrorlist_not_added(self):
+        """Test the latest mirrorlist is not added if it does not exist, e.g. on a local mirror."""
+        pinned = 'http://mirror.example.com/al2023/core/mirrors/2023.6.20241010/x86_64/mirror.list'
+        guid = f'http://mirror.example.com/al2023/core/guids/{"a" * 64}/x86_64'
+        Mirror.objects.create(repo=self.repo, url=pinned)
+        lists = {pinned: [guid]}
+        with patch('repos.utils.get_mirrorlist_urls', side_effect=lambda url: lists.get(url)):
+            check_for_mirrorlists(self.repo)
+        self.assertEqual(self._urls(), {pinned, guid})

repos/utils.py

@@ -169,33 +169,77 @@ def get_mirrorlist_urls(url):
 
 
 def add_mirrors_from_urls(repo, mirror_urls):
-    """ Creates mirrors from a list of mirror urls
+    """ Creates up to MAX_MIRRORS mirrors from a list of mirror urls.
+        Only mirrors in the list count towards the limit, so that mirrors
+        from a previous version of the list (e.g. a list that moves to a
+        new immutable repo on each release) do not block the current ones.
     """
+    from repos.models import Mirror
     max_mirrors = get_max_mirrors()
+    urls = []
     for mirror_url in mirror_urls:
         mirror_url = mirror_url.replace('$ARCH', repo.arch.name)
         mirror_url = mirror_url.replace('$basearch', repo.arch.name)
-        mirror_url = mirror_url.rstrip('/')
-        q = Q(mirrorlist=False, refresh=True, enabled=True)
-        existing = repo.mirror_set.filter(q).count()
+        # metalinks return the full path to repomd.xml, mirrorlists the base url
+        mirror_url = mirror_url.rstrip('/').removesuffix('/repodata/repomd.xml')
+        urls.append(mirror_url.rstrip('/'))
+    q = Q(mirrorlist=False, refresh=True, enabled=True, url__in=urls)
+    existing = repo.mirror_set.filter(q).count()
+    for url in urls:
         if existing >= max_mirrors:
             text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more'
             warning_message(text=text)
             break
-        from repos.models import Mirror
-
-        # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml?
-        # that is what metalink urls return now
-        m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', ''))
+        m, c = Mirror.objects.get_or_create(repo=repo, url=url)
         if c:
-            text = f'Added Mirror - {mirror_url}'
+            existing += 1
+            text = f'Added Mirror - {url}'
             info_message(text=text)
 
 
+# mirrorlists pinned to a release, and the moving mirrorlist to also follow.
+# Matched on the path only, so local mirrors and proxies of these are covered.
+LATEST_MIRRORLISTS = [
+    # Amazon Linux 2023 and later lock releasever by default, so hosts report
+    # e.g. .../core/mirrors/2023.12.20261102/x86_64/mirror.list
+    # Amazon Linux 2 can be locked to e.g. .../2/core/2.0.20231101.0/x86_64/mirror.list
+    (
+        re.compile(r'^(https?://.+/)\d+(?:\.\d+){2,3}(/[^/]+/mirror\.list)$'),
+        r'\1latest\2',
+    ),
+]
+
+
+def get_latest_mirrorlist_url(url):
+    """ Returns the moving mirrorlist url for a mirrorlist pinned to a release
+    """
+    for regex, replacement in LATEST_MIRRORLISTS:
+        if regex.match(url):
+            return regex.sub(replacement, url)
+
+
+def add_latest_mirrorlists(repo):
+    """ Adds the moving mirrorlist for any mirrorlists pinned to a release,
+        so that new releases are found even if all hosts are pinned.
+        Only added if it exists, as a local mirror may only carry pinned releases.
+    """
+    from repos.models import Mirror
+    for mirror in repo.mirror_set.all():
+        latest_url = get_latest_mirrorlist_url(mirror.url)
+        if not latest_url or Mirror.objects.filter(url=latest_url).exists():
+            continue
+        if not get_mirrorlist_urls(latest_url):
+            debug_message(text=f'No latest mirrorlist found at {latest_url}')
+            continue
+        Mirror.objects.create(repo=repo, url=latest_url)
+        info_message(text=f'Added latest mirrorlist - {latest_url}')
+
+
 def check_for_mirrorlists(repo):
     """ Check if any of the mirrors are actually mirrorlists.
         Creates MAX_MIRRORS mirrors from list if so.
     """
+    add_latest_mirrorlists(repo)
     for mirror in repo.mirror_set.all():
         if not mirror.url.startswith(('http://', 'https://')):
             warning_message(text=f'Skipping non-http(s) mirror URL: {mirror.url}')