diff --git a/VERSION.txt b/VERSION.txt index 249b676ad..094cee5cc 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.21 +4.0.22 diff --git a/client/patchman-client b/client/patchman-client index cbec72688..420e600b4 100755 --- a/client/patchman-client +++ b/client/patchman-client @@ -592,6 +592,7 @@ get_repos() { mirror_file=$(echo "${mirror_repo}" | sed -e "s/.* mirror+file://g" | cut -d " " -f 1) if [ -f "${mirror_file}" ] ; then for url in $(cat ${mirror_file}) ; do + url=${url%/} dist_repo=$(echo "${mirror_repo}" | sed -e "s#mirror+file:${mirror_file}#${url}#g") dist_repos="${dist_repos}"$'\n'"${dist_repo}" done @@ -948,7 +949,7 @@ build_json_report() { # Convert tags to JSON array local tags_json="[]" if [ ! -z "${tags}" ] && [ "${tags}" != "Default" ] ; then - tags_json=$(echo "${tags}" | tr ',' '\n' | jq -R . | jq -s .) + tags_json=$(echo "${tags}" | jq -R 'if test(",") then split(",") else [splits("\\s+")] end | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))') fi # Convert reboot to boolean diff --git a/client/patchman-client.conf b/client/patchman-client.conf index 666dd54cc..483d9a3de 100644 --- a/client/patchman-client.conf +++ b/client/patchman-client.conf @@ -4,7 +4,7 @@ server=https://patchman.example.com # Options to curl curl_options="--insecure --connect-timeout 60 --max-time 300" -# Space delimited tags to send +# Comma or space delimited tags to send tags="Server" # Does the client output a report of the upload (e.g. for cronjob output) diff --git a/debian/changelog b/debian/changelog index bef9185f8..70754b5eb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,35 @@ +patchman (4.0.22-1) stable; urgency=medium + + [ dependabot[bot] ] + * Bump gitpython from 3.1.50 to 3.1.57 + * Bump gitpython from 3.1.57 to 3.1.58 + + [ John ] + * Fix NVD CVE 2.0 API: cveId -> cveIds query parameter + + [ Aman Maharjan ] + * skip deb kernel meta-packages in reboot check + + [ Matthew Howle ] + * Add PACKAGE_TYPES choices + + [ Marcus Furlong ] + * add missing report migration + + [ Gedminas, Marius ] + * Fix UnboundLocalError while recovering from an error + + [ Marcus Furlong ] + * fix legacy tagging migrations + * fix zstd repos without declared sizes + * fix tag handling to use commas or spaces + * make -a also update errata + * ensure update counts are refreshed + * strip trailing slash from apt mirror file urls + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Wed, 04 Nov 2026 04:38:08 +0000 + patchman (4.0.21-1) stable; urgency=medium [ dependabot[bot] ] diff --git a/errata/utils.py b/errata/utils.py index 7023e1fb3..c8fb6ec91 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from errata.models import Erratum +from hosts.utils import refresh_host_update_counts from packages.models import PackageUpdate from patchman.signals import pbar_start, pbar_update from util import fetch_concurrently, run_concurrently, tz_aware_datetime @@ -72,6 +73,7 @@ def mark_errata_security_updates(concurrent_processing=True, max_workers=25): for i, e in enumerate(Erratum.objects.all()): e.scan_for_security_updates() pbar_update.send(sender=None, index=i + 1) + refresh_host_update_counts() def scan_security_worker(pk): diff --git a/hosts/migrations/0014_drop_legacy_tagging.py b/hosts/migrations/0014_drop_legacy_tagging.py new file mode 100644 index 000000000..3b16a5325 --- /dev/null +++ b/hosts/migrations/0014_drop_legacy_tagging.py @@ -0,0 +1,34 @@ +from django.db import migrations + +LEGACY_TAGGING_TABLES = ['tagging_taggeditem', 'tagging_tag'] + + +def drop_legacy_tagging(apps, schema_editor): + # 0004 never dropped the django-tagging column/tables on 3.x upgrades + conn = schema_editor.connection + q = schema_editor.quote_name + with conn.cursor() as cursor: + columns = [c.name for c in conn.introspection.get_table_description(cursor, 'hosts_host')] + tables = conn.introspection.table_names(cursor) + + if 'tags' in columns: + if getattr(conn.features, 'can_alter_table_drop_column', True): + schema_editor.execute(f'ALTER TABLE {q("hosts_host")} DROP COLUMN {q("tags")}') + else: + # old sqlite: rebuilding the table from the model drops unknown columns + schema_editor._remake_table(apps.get_model('hosts', 'Host')) + + for table in LEGACY_TAGGING_TABLES: + if table in tables: + schema_editor.execute(f'DROP TABLE {q(table)}') + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0013_alter_hostrepo_unique_together_and_more'), + ] + + operations = [ + migrations.RunPython(drop_legacy_tagging, migrations.RunPython.noop), + ] diff --git a/hosts/models.py b/hosts/models.py index 62f54947b..7b3ff3541 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -633,6 +633,11 @@ def find_deb_kernel_updates(self, kernel_packages, repo_packages, hostrepos): if running_kernel_pkg: for package in kernel_packages: if package.name.name.startswith('linux-image-'): + # skip meta-packages (e.g. linux-image-generic): their version + # (5.15.0.198.171) always sorts above the kernel they pull in + # (5.15.0-198.208), so they would flag a reboot on every host + if self.get_deb_kernel_series(package.name.name) is None: + continue flavour = self.get_deb_kernel_flavour(package.name.name) if running_flavour is None or flavour == running_flavour: if running_kernel_pkg.compare_version(package) == -1: diff --git a/hosts/tests/test_migrations.py b/hosts/tests/test_migrations.py new file mode 100644 index 000000000..1b31dad11 --- /dev/null +++ b/hosts/tests/test_migrations.py @@ -0,0 +1,99 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import json +from importlib import import_module + +from django.apps import apps +from django.db import connection +from django.test import TransactionTestCase, override_settings + +from hosts.models import Host +from reports.models import Report + +migration = import_module('hosts.migrations.0014_drop_legacy_tagging') + + +def host_columns(): + with connection.cursor() as cursor: + return [c.name for c in connection.introspection.get_table_description(cursor, 'hosts_host')] + + +def table_names(): + with connection.cursor() as cursor: + return connection.introspection.table_names(cursor) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DropLegacyTaggingTests(TransactionTestCase): + """Tests for hosts migration 0014 (leftover django-tagging schema, #690).""" + + def setUp(self): + self.addCleanup(self.run_migration) + + def run_migration(self): + with connection.schema_editor() as schema_editor: + migration.drop_legacy_tagging(apps, schema_editor) + + def add_legacy_schema(self): + with connection.cursor() as cursor: + cursor.execute('ALTER TABLE hosts_host ADD COLUMN tags varchar(255) CHECK (tags IS NOT NULL)') + cursor.execute('CREATE TABLE tagging_tag (id integer PRIMARY KEY, name varchar(50))') + cursor.execute( + 'CREATE TABLE tagging_taggeditem ' + '(id integer PRIMARY KEY, tag_id integer REFERENCES tagging_tag (id))' + ) + + def create_report(self): + return Report.objects.create( + host='legacy.example.com', + domain='example.com', + report_ip='192.168.1.10', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + def test_legacy_column_does_not_crash_report_processing(self): + """Host creation fails on the legacy column; process() must not raise.""" + self.add_legacy_schema() + self.create_report().process(find_updates=False) + self.assertFalse(Host.objects.filter(hostname='legacy.example.com').exists()) + + def test_drops_legacy_column_and_tables(self): + """Leftover tags column and tagging tables are dropped; hosts can be created again.""" + self.add_legacy_schema() + self.run_migration() + self.assertNotIn('tags', host_columns()) + self.assertNotIn('tagging_tag', table_names()) + self.assertNotIn('tagging_taggeditem', table_names()) + self.create_report().process(find_updates=False) + self.assertTrue(Host.objects.filter(hostname='legacy.example.com').exists()) + + def test_noop_on_clean_schema(self): + """Fresh installs have nothing to drop.""" + before = host_columns() + self.run_migration() + self.assertEqual(host_columns(), before) diff --git a/hosts/tests/test_models.py b/hosts/tests/test_models.py index 72198ca6c..2a62b537c 100644 --- a/hosts/tests/test_models.py +++ b/hosts/tests/test_models.py @@ -543,6 +543,45 @@ def test_deb_latest_not_installed(self): self.assertEqual(update.oldpackage, self.img_51) self.assertEqual(update.newpackage, self.img_53) + def test_deb_reboot_required_newer_kernel_installed(self): + """DEB: newer kernel installed but not running. Reboot required.""" + host = self._create_host( + '6.8.0-51-generic', + [self.img_49, self.img_51, self.img_53], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertTrue(host.reboot_required) + + def test_deb_meta_package_does_not_require_reboot(self): + """DEB: running the latest kernel with linux-image-generic installed. + + The meta-package version (6.8.0.53.54) sorts above the kernel it + depends on (6.8.0-53.54) and must not flag a reboot. + """ + meta_name = PackageName.objects.create(name='linux-image-generic') + meta = Package.objects.create( + name=meta_name, arch=self.pkg_arch, epoch='', + version='6.8.0.53.54', release='', packagetype='D' + ) + host = self._create_host( + '6.8.0-53-generic', + [self.img_49, self.img_53, meta], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertFalse(host.reboot_required) + def test_deb_flavour_extraction(self): """Test get_deb_kernel_flavour helper.""" host = self._create_host('6.8.0-51-generic', [self.img_51]) diff --git a/hosts/tests/test_utils.py b/hosts/tests/test_utils.py new file mode 100644 index 000000000..068d5b7cd --- /dev/null +++ b/hosts/tests/test_utils.py @@ -0,0 +1,91 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from errata.models import Erratum +from errata.utils import mark_errata_security_updates +from hosts.models import Host +from hosts.utils import get_or_create_host +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName, PackageUpdate +from reports.models import Report + + +@patch('hosts.models.Host.check_rdns') +class GetOrCreateHostTagTests(TestCase): + + def setUp(self): + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.osvariant = OSVariant.objects.create( + name='Debian 13', osrelease=OSRelease.objects.create(name='Debian 13'), + ) + + def host_tags(self, protocol, tags): + report = Report.objects.create( + host='tags.example.com', report_ip='192.168.1.10', os='Debian 13', + kernel='6.12.57+deb13-amd64', arch='x86_64', protocol=protocol, tags=tags, + ) + host = get_or_create_host(report, self.arch, self.osvariant, self.domain) + return sorted(host.tags.names()) + + def test_protocol1_space_delimited(self, _): + self.assertEqual(self.host_tags('1', 'web prod'), ['prod', 'web']) + + def test_protocol1_comma_delimited(self, _): + self.assertEqual(self.host_tags('1', 'web, prod'), ['prod', 'web']) + + def test_protocol2_keeps_spaces_within_tags(self, _): + self.assertEqual(self.host_tags('2', 'my tag, web'), ['my tag', 'web']) + + def test_protocol2_empty_tags(self, _): + self.assertEqual(self.host_tags('2', ''), []) + + +class RefreshHostUpdateCountsTests(TestCase): + + def test_counts_refreshed_after_errata_security_marking(self): + os_release = OSRelease.objects.create(name='Debian 13') + host = Host.objects.create( + hostname='h.example.com', ipaddress='192.0.2.1', + osvariant=OSVariant.objects.create(name='Debian 13', osrelease=os_release), + kernel='6.12.0', arch=MachineArchitecture.objects.create(name='x86_64'), + domain=Domain.objects.create(name='example.com'), + lastreport=timezone.now(), + ) + pkg_arch = PackageArchitecture.objects.create(name='amd64') + name = PackageName.objects.create(name='curl') + old = Package.objects.create(name=name, arch=pkg_arch, epoch='', version='1', release='1', packagetype='D') + new = Package.objects.create(name=name, arch=pkg_arch, epoch='', version='2', release='1', packagetype='D') + update = PackageUpdate.objects.create(oldpackage=old, newpackage=new, security=False) + host.updates.add(update) + host.refresh_from_db() + self.assertEqual((host.sec_updates_count, host.bug_updates_count), (0, 1)) + + erratum = Erratum.objects.create( + name='DSA-0000-1', e_type='security', issue_date=timezone.now(), synopsis='curl', + ) + erratum.fixed_packages.add(new) + mark_errata_security_updates(concurrent_processing=False) + + host.refresh_from_db() + self.assertEqual((host.sec_updates_count, host.bug_updates_count), (1, 0)) diff --git a/hosts/utils.py b/hosts/utils.py index f8f89a15b..58a11fd38 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -18,7 +18,10 @@ from socket import gaierror, gethostbyaddr, herror from django.db import IntegrityError, transaction +from django.db.models import Count, OuterRef, Subquery +from django.db.models.functions import Coalesce from taggit.models import Tag +from taggit.utils import parse_tags from util import get_datetime_now from util.logging import error_message, info_message @@ -36,6 +39,36 @@ def update_rdns(host): host.save() +def refresh_host_update_counts(): + """ Recalculate cached security/bugfix update counts for all hosts. + Needed when PackageUpdate.security changes, as that does not fire + the Host.updates m2m_changed signal. + """ + from hosts.models import Host + through = Host.updates.through + + def count_updates(security): + counts = through.objects.filter( + host_id=OuterRef('pk'), + packageupdate__security=security, + ).values('host_id').annotate(c=Count('pk')).values('c') + return Coalesce(Subquery(counts), 0) + + Host.objects.update( + sec_updates_count=count_updates(True), + bug_updates_count=count_updates(False), + ) + + +def get_report_tags(report): + """ Protocol 2 sends a list (stored comma-joined), protocol 1 sends the + raw config string, which may be comma or space delimited + """ + if report.protocol == '2': + return [t.strip() for t in (report.tags or '').split(',') if t.strip()] + return parse_tags(report.tags or '') + + def get_or_create_host(report, arch, osvariant, domain): """ Get or create a host from from a report """ @@ -46,6 +79,7 @@ def get_or_create_host(report, arch, osvariant, domain): except herror: report.host = report.report_ip report.save() + host = None try: with transaction.atomic(): host, created = Host.objects.get_or_create( @@ -64,7 +98,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.osvariant = osvariant host.domain = domain host.lastreport = report.created - host.tags.set(report.tags.split(','), clear=True) + host.tags.set(get_report_tags(report), clear=True) if report.reboot == 'True': host.reboot_required = True else: diff --git a/packages/models.py b/packages/models.py index e3446c95d..0db1b5f80 100644 --- a/packages/models.py +++ b/packages/models.py @@ -183,7 +183,7 @@ class PackageString(models.Model): epoch = models.CharField(max_length=128, blank=True, null=True) release = models.CharField(max_length=128, blank=True, null=True) arch = models.CharField(max_length=128) - packagetype = models.CharField(max_length=1, blank=True, null=True) + packagetype = models.CharField(max_length=1, choices=Package.PACKAGE_TYPES, blank=True, null=True) category = models.CharField(max_length=128, blank=True, null=True) description = models.TextField(blank=True, null=True) url = models.URLField(max_length=255, blank=True, null=True) diff --git a/reports/migrations/0006_alter_report_options.py b/reports/migrations/0006_alter_report_options.py new file mode 100644 index 000000000..98091f162 --- /dev/null +++ b/reports/migrations/0006_alter_report_options.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.29 on 2026-10-30 02:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0005_alter_report_options'), + ] + + operations = [ + migrations.AlterModelOptions( + name='report', + options={'ordering': ['-created'], 'verbose_name': 'Report', 'verbose_name_plural': 'Reports'}, + ), + ] diff --git a/reports/models.py b/reports/models.py index 578e2c46e..0e2c482b6 100644 --- a/reports/models.py +++ b/reports/models.py @@ -201,6 +201,9 @@ def process(self, find_updates=True, verbose=False): osvariant = get_os(self.os, arch) domain = get_domain(self.domain) host = get_or_create_host(self, arch, osvariant, domain) + if host is None: + error_message(text=f'Error: could not get or create Host for report {self.id}') + return if verbose: info_message(text=f'Processing report {self.id} - {self.host}') diff --git a/requirements.txt b/requirements.txt index d75ebdec8..32ba69831 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ django-filter==25.1 humanize==4.12.1 version-utils==0.3.2 python-magic==0.4.27 -gitpython==3.1.50 +gitpython==3.1.58 tenacity==8.2.3 celery==5.4.0 redis==6.4.0 diff --git a/sbin/patchman b/sbin/patchman index 7040f228a..9de7b8867 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -428,7 +428,7 @@ def process_args(args): args.clean_updates = True args.dbcheck = True args.dns_checks = True - args.errata = True + args.update_errata = True if args.list_repos: list_repos(args.repo) return False diff --git a/security/models.py b/security/models.py index 9b7dd5b38..a2710b903 100644 --- a/security/models.py +++ b/security/models.py @@ -200,7 +200,7 @@ def parse_osv_dev_cve_data(self, cve_json): self.add_cvss_score(vector_string=score.get('score')) def fetch_nist_cve_data(self): - nist_cve_url = f'https://services.nvd.nist.gov/rest/json/cves/2.0?cveId={self.cve_id}' + nist_cve_url = f'https://services.nvd.nist.gov/rest/json/cves/2.0?cveIds={self.cve_id}' res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: diff --git a/util/__init__.py b/util/__init__.py index e823efeac..037b0d8a9 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -196,9 +196,11 @@ def unzstd(contents): """ unzstd contents in memory and return the data """ try: - if hasattr(zstd, 'decompress'): - return zstd.decompress(contents) - return zstd.ZstdDecompressor().stream_reader(contents).read() + if zstd.__name__ == 'zstandard': + # zstandard.decompress() fails without a content size in the frame + # header and stops after the first frame, so stream instead + return zstd.ZstdDecompressor().stream_reader(contents, read_across_frames=True).read() + return zstd.decompress(contents) except (zstd.ZstdError, Exception) as e: error_message(text=f'zstd: {e}') diff --git a/util/tests/test_utils.py b/util/tests/test_utils.py index 4205150f2..237353480 100644 --- a/util/tests/test_utils.py +++ b/util/tests/test_utils.py @@ -17,6 +17,7 @@ import gzip import hashlib from io import BytesIO +from unittest import skipIf from unittest.mock import MagicMock from django.test import TestCase, override_settings @@ -24,9 +25,14 @@ from util import ( Checksum, bunzip2, extract, get_checksum, get_md5, get_sha1, get_sha256, get_sha512, gunzip, has_setting_of_type, is_epoch_time, response_is_valid, - sanitize_filter_params, tz_aware_datetime, + sanitize_filter_params, tz_aware_datetime, unzstd, ) +try: + import zstandard +except ImportError: + zstandard = None + @override_settings( CELERY_TASK_ALWAYS_EAGER=True, @@ -138,6 +144,20 @@ def test_extract_gzip(self): result = extract(compressed, 'gz') self.assertEqual(result, original) + @skipIf(zstandard is None, 'zstandard not installed') + def test_unzstd_without_content_size(self): + """Test unzstd with a frame that does not declare its content size.""" + original = b'x' * 100000 + compressed = zstandard.ZstdCompressor(write_content_size=False).compress(original) + self.assertEqual(unzstd(compressed), original) + + @skipIf(zstandard is None, 'zstandard not installed') + def test_extract_zstd_multiple_frames(self): + """Test extract with zstd data made of multiple frames.""" + first, second = b'a' * 50000, b'b' * 50000 + compressed = zstandard.ZstdCompressor().compress(first) + zstandard.ZstdCompressor().compress(second) + self.assertEqual(extract(compressed, 'zst'), first + second) + def test_extract_unknown_format(self): """Test extract with unknown format returns original.""" data = b'unchanged data'