diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/nodescraper/plugins/inband/network/analyzer_args.py new file mode 100644 index 00000000..dae0e22a --- /dev/null +++ b/nodescraper/plugins/inband/network/analyzer_args.py @@ -0,0 +1,19 @@ +############################################################################### +# +# MIT License +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class NetworkAnalyzerArgs(AnalyzerArgs): + """Arguments for network and ethtool analysis.""" + + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected NIC.", + ) diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 90819e09..da58d8fa 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -24,20 +24,86 @@ # ############################################################################### import re +from typing import Any, Dict, Optional from nodescraper.base.regexanalyzer import RegexAnalyzer from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.models import TaskResult +from .analyzer_args import NetworkAnalyzerArgs from .networkdata import NetworkDataModel -class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, None]): +def _validate_firmware_policy(records: list[Dict[str, Any]], policies: Any) -> list[Dict[str, Any]]: + """Validate ethtool firmware locally against this analyzer's policies.""" + policy_list = [policies] if isinstance(policies, dict) else policies or [] + issues = [] + for record in records: + policy = next( + ( + item + for item in policy_list + if isinstance(item, dict) and _policy_matches(record, item) + ), + None, + ) + if policy is None: + continue + actual = _normalize_firmware_version(record.get("version")) + expected = _normalize_firmware_version(policy.get("expected_nic_firmware")) + if not actual: + reason = "firmware version is unavailable" + elif expected and not _firmware_matches(actual, expected): + reason = ( + f"actual {record.get('version')} != expected_nic_firmware " + f"{policy.get('expected_nic_firmware')}" + ) + else: + continue + issues.append({"reason": reason, "record": record, "policy": dict(policy)}) + return issues + + +def _policy_matches(record: Dict[str, Any], policy: Dict[str, Any]) -> bool: + match = policy.get("match", {}) + if not isinstance(match, dict): + return False + if not match: + match = {field: policy[field] for field in record if field in policy} + for field, expected in match.items(): + actual = record.get(str(field)) + if actual is None: + return False + expected_values = expected if isinstance(expected, list) else [expected] + if not any( + str(actual).strip().lower() == str(value).strip().lower() for value in expected_values + ): + return False + return True + + +def _normalize_firmware_version(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().strip("'\"").lower() + return normalized or None + + +def _firmware_matches(actual: str, expected_pattern: str) -> bool: + try: + return re.fullmatch(expected_pattern, actual, flags=re.IGNORECASE) is not None + except re.error: + return False + + +class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: + def analyze_data( + self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None + ) -> TaskResult: """Analyze ethtool -S statistics via RDMA-scoped vendor models. Args: @@ -47,6 +113,40 @@ def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ + args = args or NetworkAnalyzerArgs() + firmware_records = [] + for interface, info in data.ethtool_info.items(): + vendor = _vendor_for_driver(info.driver) + if vendor is None: + continue + firmware_records.append( + { + "identity": interface, + "version": info.firmware_version, + "source": "ethtool", + "vendor": vendor, + "driver": info.driver, + "interface": interface, + "pci_bdf": info.bus_info, + } + ) + firmware_policy_issues = _validate_firmware_policy( + firmware_records, + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="Network adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + if not data.ethtool_info and not data.ethtool_statistics: self.result.message = "No network devices found" self.result.status = ExecutionStatus.WARNING @@ -165,8 +265,25 @@ def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: elif vendor_warning: self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING + elif firmware_policy_issues: + self.result.message = "Network adapter firmware policy mismatch" + self.result.status = ExecutionStatus.WARNING else: self.result.message = "No network errors detected in statistics" self.result.status = ExecutionStatus.OK return self.result + + +def _vendor_for_driver(driver: Optional[str]) -> Optional[str]: + """Return a stable vendor label for common ethtool driver families.""" + if not driver: + return None + normalized = driver.lower() + if normalized.startswith(("bnxt", "bnx2")): + return "Broadcom" + if normalized.startswith(("mlx", "ib_")): + return "Mellanox" + if normalized.startswith(("ionic",)): + return "Pensando" + return None diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 9f109f89..27af097c 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -450,6 +450,24 @@ def _parse_ethtool(self, interface: str, output: str) -> EthtoolInfo: return ethtool_info + def _parse_ethtool_driver_info(self, info: EthtoolInfo, output: str) -> None: + """Merge ``ethtool -i`` identity and firmware fields into an interface model.""" + for line in output.splitlines(): + if ":" not in line: + continue + key, value = (part.strip() for part in line.split(":", 1)) + if not key or not value: + continue + info.settings[key] = value + normalized_key = key.lower().replace("-", "_").replace(" ", "_") + if normalized_key == "driver": + info.driver = value + elif normalized_key in {"bus_info", "bus"}: + info.bus_info = value + elif normalized_key in {"firmware_version", "firmware"}: + firmware_match = re.search(r"\d+(?:\.\d+){2,}", value) + info.firmware_version = firmware_match.group(0) if firmware_match else value + def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, str]: """Parse 'ethtool -S ' output into a key-value dictionary. @@ -522,6 +540,18 @@ def _collect_ethtool_info( priority=EventPriority.WARNING, ) + # ``ethtool `` does not include firmware identity on all + # drivers, so always retain the separate ``ethtool -i`` response. + res_driver = self._run_sut_cmd( + self.CMD_ETHTOOL_I_TEMPLATE.format(interface=iface.name), sudo=True + ) + if res_driver.exit_code == 0: + ethtool_info = ethtool_data.get( + iface.name, EthtoolInfo(interface=iface.name, raw_output="") + ) + self._parse_ethtool_driver_info(ethtool_info, res_driver.stdout) + ethtool_data[iface.name] = ethtool_info + return ethtool_data, skipped def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[EthtoolStatistics]: diff --git a/nodescraper/plugins/inband/network/network_plugin.py b/nodescraper/plugins/inband/network/network_plugin.py index 69e7b671..5d6d9822 100644 --- a/nodescraper/plugins/inband/network/network_plugin.py +++ b/nodescraper/plugins/inband/network/network_plugin.py @@ -25,13 +25,14 @@ ############################################################################### from nodescraper.base import InBandDataPlugin +from .analyzer_args import NetworkAnalyzerArgs from .collector_args import NetworkCollectorArgs from .network_analyzer import NetworkAnalyzer from .network_collector import NetworkCollector from .networkdata import NetworkDataModel -class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, None]): +class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, NetworkAnalyzerArgs]): """Plugin for collection of network configuration data""" DATA_MODEL = NetworkDataModel @@ -39,5 +40,6 @@ class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, Non COLLECTOR = NetworkCollector COLLECTOR_ARGS = NetworkCollectorArgs + ANALYZER_ARGS = NetworkAnalyzerArgs ANALYZER = NetworkAnalyzer diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index 86e8cb6f..bcd7fc95 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -97,6 +97,9 @@ class EthtoolInfo(BaseModel): interface: str # Interface name this info belongs to raw_output: str # Raw ethtool command output + driver: Optional[str] = None # Kernel driver from ethtool -i + bus_info: Optional[str] = None # PCI bus identity from ethtool -i + firmware_version: Optional[str] = None # Firmware version from ethtool -i settings: Dict[str, str] = Field(default_factory=dict) # Parsed key-value settings supported_link_modes: List[str] = Field(default_factory=list) # Supported link modes advertised_link_modes: List[str] = Field(default_factory=list) # Advertised link modes diff --git a/nodescraper/plugins/inband/nic/analyzer_args.py b/nodescraper/plugins/inband/nic/analyzer_args.py index 65214b76..cec45ac2 100644 --- a/nodescraper/plugins/inband/nic/analyzer_args.py +++ b/nodescraper/plugins/inband/nic/analyzer_args.py @@ -33,6 +33,10 @@ class NicAnalyzerArgs(AnalyzerArgs): """Analyzer args for niccli/nicctl data, with expected_values keyed by canonical command key.""" + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected NIC.", + ) expected_values: Optional[Dict[str, Dict[str, Any]]] = Field( default=None, description="Per-command expected checks keyed by canonical key (see command_to_canonical_key).", diff --git a/nodescraper/plugins/inband/nic/nic_analyzer.py b/nodescraper/plugins/inband/nic/nic_analyzer.py index 30543867..63dc4e16 100644 --- a/nodescraper/plugins/inband/nic/nic_analyzer.py +++ b/nodescraper/plugins/inband/nic/nic_analyzer.py @@ -103,6 +103,111 @@ def _normalize_tsa_map(d: Optional[Dict[Any, Any]]) -> Optional[Dict[int, str]]: return {int(k): str(v) for k, v in d.items()} +def _firmware_records(data: NicDataModel) -> list[Dict[str, Any]]: + """Build normalized records for Broadcom and Pensando NIC output.""" + records: list[Dict[str, Any]] = [] + devices = {device.device_num: device for device in data.broadcom_nic_devices} + for device_num, version in data.broadcom_nic_firmware.items(): + device = devices.get(device_num) + records.append( + { + "identity": str(device_num), + "version": version, + "source": data.broadcom_cli_type or "broadcom", + "vendor": "Broadcom", + "model": device.model if device else None, + "interface": device.interface_name if device else None, + "pci_bdf": device.pci_address if device else None, + } + ) + for device in data.broadcom_nic_devices: + if device.device_num not in data.broadcom_nic_firmware: + records.append( + { + "identity": str(device.device_num), + "version": None, + "source": data.broadcom_cli_type or "broadcom", + "vendor": "Broadcom", + "model": device.model, + "interface": device.interface_name, + "pci_bdf": device.pci_address, + } + ) + for firmware in data.pensando_nic_version_firmware: + records.append( + { + "identity": firmware.nic_id, + "version": firmware.firmware_a, + "source": "nicctl", + "vendor": "Pensando", + "pci_bdf": firmware.pcie_bdf, + } + ) + return records + + +def _validate_firmware_policy(records: list[Dict[str, Any]], policies: Any) -> list[Dict[str, Any]]: + """Validate NIC firmware locally against this analyzer's policies.""" + policy_list = [policies] if isinstance(policies, dict) else policies or [] + issues = [] + for record in records: + policy = next( + ( + item + for item in policy_list + if isinstance(item, dict) and _policy_matches(record, item) + ), + None, + ) + if policy is None: + continue + actual = _normalize_firmware_version(record.get("version")) + expected = _normalize_firmware_version(policy.get("expected_nic_firmware")) + if not actual: + reason = "firmware version is unavailable" + elif expected and not _firmware_matches(actual, expected): + reason = ( + f"actual {record.get('version')} != expected_nic_firmware " + f"{policy.get('expected_nic_firmware')}" + ) + else: + continue + issues.append({"reason": reason, "record": record, "policy": dict(policy)}) + return issues + + +def _policy_matches(record: Dict[str, Any], policy: Dict[str, Any]) -> bool: + match = policy.get("match", {}) + if not isinstance(match, dict): + return False + if not match: + match = {field: policy[field] for field in record if field in policy} + for field, expected in match.items(): + actual = record.get(str(field)) + if actual is None: + return False + expected_values = expected if isinstance(expected, list) else [expected] + if not any( + str(actual).strip().lower() == str(value).strip().lower() for value in expected_values + ): + return False + return True + + +def _normalize_firmware_version(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().strip("'\"").lower() + return normalized or None + + +def _firmware_matches(actual: str, expected_pattern: str) -> bool: + try: + return re.fullmatch(expected_pattern, actual, flags=re.IGNORECASE) is not None + except re.error: + return False + + class NicAnalyzer(DataAnalyzer[NicDataModel, NicAnalyzerArgs]): """Analyze niccli/nicctl data; checks Broadcom support_rdma, performance_profile (RoCE), pcie_relaxed_ordering (enabled), and getqos (expected QoS across adapters).""" @@ -115,11 +220,33 @@ def analyze_data( if args is None: args = NicAnalyzerArgs() + firmware_policy_issues = _validate_firmware_policy( + _firmware_records(data), + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="Network adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + has_firmware_data = bool( + data.broadcom_nic_devices + or data.broadcom_nic_firmware + or data.pensando_nic_version_firmware + ) + has_broadcom = bool(data.broadcom_nic_support_rdma) has_nicctl_logs = bool( data.nicctl_card_logs and any((c or "").strip() for c in data.nicctl_card_logs.values()) ) - if not has_broadcom and not has_nicctl_logs: + if not has_broadcom and not has_nicctl_logs and not has_firmware_data: self.result.message = "No Broadcom support_rdma or nicctl card log data to check" self.result.status = ExecutionStatus.OK return self.result @@ -296,12 +423,14 @@ def analyze_data( console_log=True, ) + any_firmware_mismatch = bool(firmware_policy_issues) if ( any_disabled or any_non_roce or any_relaxed_ordering_bad or any_qos_mismatch or any_nicctl_log_errors + or any_firmware_mismatch ): self.result.status = ExecutionStatus.WARNING parts = [] @@ -315,6 +444,8 @@ def analyze_data( parts.append("getqos") if any_nicctl_log_errors: parts.append("nicctl_card_logs") + if any_firmware_mismatch: + parts.append("firmware") self.result.message = f"Broadcom/nic check(s) failed: {' and/or '.join(parts)}" else: self.result.status = ExecutionStatus.OK diff --git a/nodescraper/plugins/inband/nic/nic_collector.py b/nodescraper/plugins/inband/nic/nic_collector.py index 27827bfe..605e249a 100644 --- a/nodescraper/plugins/inband/nic/nic_collector.py +++ b/nodescraper/plugins/inband/nic/nic_collector.py @@ -67,6 +67,21 @@ command_to_canonical_key, ) +_FIRMWARE_VERSION_RE = re.compile(r"\b\d+(?:\.\d+){2,}\b") + + +def _extract_firmware_version(output: str) -> Optional[str]: + """Extract the version from a firmware/package-version command response.""" + for line in output.splitlines(): + lowered = line.lower() + if not any(token in lowered for token in ("firmware", "fwpackage", "pkg_ver", "package")): + continue + matches = _FIRMWARE_VERSION_RE.findall(line) + if matches: + return matches[-1] + return None + + # niccli version threshold: legacy (<=233) vs new (>233) command syntax. NICCLI_VERSION_LEGACY_MAX = 233 # Commands use -dev/-getoption/getqos; for version > this use --dev/--getoption/qos --ets --show @@ -641,6 +656,7 @@ def collect_data( custom_commands = args.commands if args and args.commands else None results: dict[str, NicCommandResult] = {} + broadcom_firmware: Dict[int, str] = {} # Detect which Broadcom CLI is present (bcmcli takes priority over niccli). broadcom_cli = self._detect_broadcom_cli(args, results) @@ -779,6 +795,7 @@ def _bcmcli_cmd(cmd: str) -> str: ) # Populate broadcom_nic_* fields from bcmcli results keyed by device_id + broadcom_firmware = self._collect_broadcom_nic_firmware(results) broadcom_support_rdma: Dict[int, str] = {} broadcom_performance_profile: Dict[int, str] = {} broadcom_pcie_relaxed_ordering: Dict[int, str] = {} @@ -991,6 +1008,7 @@ def _bcmcli_cmd(cmd: str) -> str: broadcom_performance_profile, broadcom_pcie_relaxed_ordering, ) = self._collect_broadcom_nic_structured(results, niccli_version=niccli_version) + broadcom_firmware = self._collect_broadcom_nic_firmware(results) ( pensando_cards, pensando_dcqcn, @@ -1055,6 +1073,7 @@ def _truncate(s: str, max_len: int) -> str: broadcom_nic_support_rdma=broadcom_support_rdma, broadcom_nic_performance_profile=broadcom_performance_profile, broadcom_nic_pcie_relaxed_ordering=broadcom_pcie_relaxed_ordering, + broadcom_nic_firmware=broadcom_firmware, pensando_nic_cards=pensando_cards, pensando_nic_dcqcn=pensando_dcqcn, pensando_nic_environment=pensando_environment, @@ -1067,6 +1086,27 @@ def _truncate(s: str, max_len: int) -> str: pensando_nic_version_firmware=pensando_version_firmware, ) + def _collect_broadcom_nic_firmware( + self, results: Dict[str, NicCommandResult] + ) -> Dict[int, str]: + """Parse Broadcom bcmcli/niccli firmware commands into device records.""" + firmware: Dict[int, str] = {} + for command, result in results.items(): + lowered = command.lower() + if not ( + "fwmanager show fwpackage" in lowered + or "show --pkg_ver" in lowered + or "show -pkg_ver" in lowered + ): + continue + if not result.succeeded: + continue + device_match = re.search(r"(?:-d|--dev|-dev)\s+(\d+)", command) + version = _extract_firmware_version(result.stdout) + if device_match and version: + firmware[int(device_match.group(1))] = version + return firmware + def _resolve_bcmcli_bin_dir(self) -> str: res = self._run_sut_cmd("which bcmcli_show", sudo=False, log_artifact=False) if res.exit_code == 0 and (res.stdout or "").strip(): diff --git a/nodescraper/plugins/inband/nic/nic_data.py b/nodescraper/plugins/inband/nic/nic_data.py index 9720c4c1..e82172d5 100644 --- a/nodescraper/plugins/inband/nic/nic_data.py +++ b/nodescraper/plugins/inband/nic/nic_data.py @@ -381,6 +381,10 @@ class NicDataModel(DataModel): default_factory=dict, description="Per-device output of 'niccli -dev X nvm -getoption pcie_relaxed_ordering' (device_num -> raw stdout).", ) + broadcom_nic_firmware: Dict[int, str] = Field( + default_factory=dict, + description="Per-device firmware version parsed from bcmcli fwpackage or niccli pkg_ver output.", + ) pensando_nic_cards: List[PensandoNicCard] = Field(default_factory=list) pensando_nic_dcqcn: List[PensandoNicDcqcn] = Field(default_factory=list) pensando_nic_environment: List[PensandoNicEnvironment] = Field(default_factory=list) diff --git a/nodescraper/plugins/inband/rdma/analyzer_args.py b/nodescraper/plugins/inband/rdma/analyzer_args.py index 0507ed15..282ab9cf 100644 --- a/nodescraper/plugins/inband/rdma/analyzer_args.py +++ b/nodescraper/plugins/inband/rdma/analyzer_args.py @@ -33,6 +33,10 @@ class RdmaAnalyzerArgs(AnalyzerArgs): """Arguments for the RDMA analyzer.""" + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected RDMA NIC.", + ) exclusion_regex: Optional[list[str]] = Field( default=None, description="Regex patterns matched against an interface netdev; matching interfaces are skipped.", diff --git a/nodescraper/plugins/inband/rdma/rdma_analyzer.py b/nodescraper/plugins/inband/rdma/rdma_analyzer.py index ba48c20e..e4a5d9a1 100644 --- a/nodescraper/plugins/inband/rdma/rdma_analyzer.py +++ b/nodescraper/plugins/inband/rdma/rdma_analyzer.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import Optional +from typing import Any, Dict, Optional from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.interfaces import DataAnalyzer @@ -34,6 +34,68 @@ from .rdmadata import RdmaDataModel +def _validate_firmware_policy(records: list[Dict[str, Any]], policies: Any) -> list[Dict[str, Any]]: + """Validate RDMA firmware locally against this analyzer's policies.""" + policy_list = [policies] if isinstance(policies, dict) else policies or [] + issues = [] + for record in records: + policy = next( + ( + item + for item in policy_list + if isinstance(item, dict) and _policy_matches(record, item) + ), + None, + ) + if policy is None: + continue + actual = _normalize_firmware_version(record.get("version")) + expected = _normalize_firmware_version(policy.get("expected_nic_firmware")) + if not actual: + reason = "firmware version is unavailable" + elif expected and not _firmware_matches(actual, expected): + reason = ( + f"actual {record.get('version')} != expected_nic_firmware " + f"{policy.get('expected_nic_firmware')}" + ) + else: + continue + issues.append({"reason": reason, "record": record, "policy": dict(policy)}) + return issues + + +def _policy_matches(record: Dict[str, Any], policy: Dict[str, Any]) -> bool: + match = policy.get("match", {}) + if not isinstance(match, dict): + return False + if not match: + match = {field: policy[field] for field in record if field in policy} + for field, expected in match.items(): + actual = record.get(str(field)) + if actual is None: + return False + expected_values = expected if isinstance(expected, list) else [expected] + if not any( + str(actual).strip().lower() == str(value).strip().lower() for value in expected_values + ): + return False + return True + + +def _normalize_firmware_version(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().strip("'\"").lower() + return normalized or None + + +def _firmware_matches(actual: str, expected_pattern: str) -> bool: + try: + return re.fullmatch(expected_pattern, actual, flags=re.IGNORECASE) is not None + except re.error: + return False + + class RdmaAnalyzer(DataAnalyzer[RdmaDataModel, RdmaAnalyzerArgs]): """Check RDMA statistics for errors (RoCE and other RDMA error counters).""" @@ -54,14 +116,41 @@ def analyze_data( Returns: TaskResult with status OK if no errors, ERROR if any error counter > 0. """ - if not data.statistic_list: + if not args: + args = RdmaAnalyzerArgs() + + firmware_records = [ + { + "identity": device.device, + "version": device.firmware_version, + "source": "rdma", + "vendor": _vendor_for_rdma_device(device.device), + "driver": device.device, + } + for device in data.dev_list + ] + firmware_policy_issues = _validate_firmware_policy( + firmware_records, + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="RDMA adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + + if not data.statistic_list and not data.dev_list: self.result.message = "No RDMA devices found" self.result.status = ExecutionStatus.WARNING return self.result - if not args: - args = RdmaAnalyzerArgs() - compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] error_detected = False @@ -124,6 +213,9 @@ def analyze_data( if error_detected or critical_detected: self.result.message = "RDMA errors detected in statistics" self.result.status = ExecutionStatus.ERROR + elif firmware_policy_issues: + self.result.message = "RDMA adapter firmware policy mismatch" + self.result.status = ExecutionStatus.WARNING else: self.result.message = "No RDMA errors detected in statistics" self.result.status = ExecutionStatus.OK @@ -132,3 +224,14 @@ def analyze_data( self.result.message += f" ({skipped_count} skipped)" return self.result + + +def _vendor_for_rdma_device(device: str) -> Optional[str]: + normalized = device.lower() + if normalized.startswith(("mlx", "ib_")): + return "Mellanox" + if normalized.startswith(("bnxt", "bnx2")): + return "Broadcom" + if normalized.startswith("ionic"): + return "Pensando" + return None diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 90d68aa4..fedd625b 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -125,6 +125,9 @@ def _parse_rdma_dev(self, output: str) -> list[RdmaDevice]: device.node_type = parts[i + 1] i += 2 elif parts[i] == "fw" and i + 1 < len(parts): + device.firmware_version = parts[i + 1] + # Keep the legacy attribute for consumers of older + # serialized RDMA data models. device.attributes["fw_version"] = parts[i + 1] i += 2 elif parts[i] == "node_guid" and i + 1 < len(parts): diff --git a/nodescraper/plugins/inband/rdma/rdmadata.py b/nodescraper/plugins/inband/rdma/rdmadata.py index 2859d325..7207209f 100644 --- a/nodescraper/plugins/inband/rdma/rdmadata.py +++ b/nodescraper/plugins/inband/rdma/rdmadata.py @@ -407,6 +407,7 @@ class RdmaDevice(BaseModel): node_guid: Optional[str] = None sys_image_guid: Optional[str] = None state: Optional[str] = None + firmware_version: Optional[str] = None attributes: dict[str, str] = Field(default_factory=dict) diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 9796a173..ab1f66c8 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -26,6 +26,7 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.inband.network.analyzer_args import NetworkAnalyzerArgs from nodescraper.plugins.inband.network.ethtool_vendor import ( EthtoolStatistics, Thor2EthtoolStatistics, @@ -77,6 +78,89 @@ def test_empty_ethtool_info(network_analyzer): assert result.message == "No network devices found" +def test_ethtool_firmware_exact_match(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + bus_info="0000:01:00.0", + firmware_version="238.1.169.0", + ) + } + ) + args = NetworkAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +@pytest.mark.parametrize( + "firmware_version", + ["1.117.5-a-77", "28.35.1012", "238.1.168.0"], +) +def test_ethtool_firmware_regex_matches_supported_formats(network_analyzer, firmware_version): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + firmware_version=firmware_version, + ) + } + ) + args = NetworkAnalyzerArgs( + expected_nic_firmware=(r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$") + ) + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_ethtool_firmware_mismatch_is_reported(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + firmware_version="238.1.168.0", + ) + } + ) + args = NetworkAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + + +def test_ethtool_unsupported_driver_is_non_blocking(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="i40e", + firmware_version="1.3534.0", + ) + } + ) + result = network_analyzer.analyze_data( + model, NetworkAnalyzerArgs(expected_nic_firmware="238.1.168.0") + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + def test_rdma_ethtool_vendor_error_only(network_analyzer): """Vendor ethtool: error-tier counter raises ERROR.""" stat = EthtoolStatistics( diff --git a/test/unit/plugin/test_network_collector.py b/test/unit/plugin/test_network_collector.py index 909e1319..9a389d6e 100644 --- a/test/unit/plugin/test_network_collector.py +++ b/test/unit/plugin/test_network_collector.py @@ -464,6 +464,22 @@ def test_parse_ethtool_basic(collector): assert ethtool_info.raw_output == ETHTOOL_OUTPUT +def test_parse_ethtool_driver_info_normalizes_package_firmware(collector): + """Keep the device firmware version when ethtool reports package metadata too.""" + info = EthtoolInfo(interface="ethmock123", raw_output="") + + collector._parse_ethtool_driver_info( + info, + "driver: bnxt_en\n" + "firmware-version: 238.1.168.0/pkg 238.1.169.0\n" + "bus-info: 0000:01:00.0\n", + ) + + assert info.driver == "bnxt_en" + assert info.firmware_version == "238.1.168.0" + assert info.bus_info == "0000:01:00.0" + + def test_parse_ethtool_supported_link_modes(collector): """Test parsing supported link modes from ethtool output""" ethtool_info = collector._parse_ethtool("ethmock123", ETHTOOL_OUTPUT) diff --git a/test/unit/plugin/test_nic_analyzer.py b/test/unit/plugin/test_nic_analyzer.py new file mode 100644 index 00000000..3beaa12f --- /dev/null +++ b/test/unit/plugin/test_nic_analyzer.py @@ -0,0 +1,97 @@ +############################################################################### +# +# MIT License +# +############################################################################### +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.inband.nic.analyzer_args import NicAnalyzerArgs +from nodescraper.plugins.inband.nic.nic_analyzer import NicAnalyzer +from nodescraper.plugins.inband.nic.nic_data import ( + NicCliDevice, + NicDataModel, + PensandoNicVersionFirmware, +) + + +def test_broadcom_firmware_exact_match(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: "238.1.169.0"}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_broadcom_firmware_regex_matches_supported_formats(system_info): + pattern = r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$" + for firmware_version in ("1.117.5-a-77", "28.35.1012", "238.1.168.0"): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: firmware_version}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware=pattern), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_broadcom_firmware_mismatch_is_reported(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: "238.1.168.0"}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + + +def test_missing_broadcom_firmware_is_reported(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.WARNING + assert any("unavailable" in event.data["reason"] for event in result.events) + + +def test_pensando_firmware_is_validated_by_nic_analyzer(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + pensando_nic_version_firmware=[ + PensandoNicVersionFirmware( + nic_id="card0", + pcie_bdf="0000:01:00.0", + firmware_a="238.1.169.0", + ) + ] + ) + args = NicAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = analyzer.analyze_data(data, args) + + assert result.status == ExecutionStatus.OK + assert not result.events diff --git a/test/unit/plugin/test_rdma_analyzer.py b/test/unit/plugin/test_rdma_analyzer.py index 1a3804a5..ca1531e3 100644 --- a/test/unit/plugin/test_rdma_analyzer.py +++ b/test/unit/plugin/test_rdma_analyzer.py @@ -37,6 +37,7 @@ Cx7RdmaStatistics, PollaraRdmaStatistics, RdmaDataModel, + RdmaDevice, RdmaLink, RdmaStatistics, RdmaVendorStatistics, @@ -160,6 +161,70 @@ def test_empty_statistics(rdma_analyzer): assert result.message == "No RDMA devices found" +def test_roce_firmware_exact_match(rdma_analyzer): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="bnxt_re0", + transport="RoCE", + firmware_version="238.1.169.0", + ) + ] + ) + args = RdmaAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = rdma_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_roce_firmware_regex_matches_supported_formats(rdma_analyzer): + pattern = r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$" + for firmware_version in ("1.117.5-a-77", "28.35.1012", "238.1.168.0"): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="bnxt_re0", + transport="RoCE", + firmware_version=firmware_version, + ) + ] + ) + + result = rdma_analyzer.analyze_data( + model, + RdmaAnalyzerArgs(expected_nic_firmware=pattern), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_infini_band_firmware_mismatch_is_reported(rdma_analyzer): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="mlx5_0", + node_type="CA", + transport="InfiniBand", + firmware_version="32.40.1000", + ) + ] + ) + args = RdmaAnalyzerArgs(expected_nic_firmware="32.50.1000") + + result = rdma_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + firmware_event = next( + event for event in result.events if "firmware" in event.description.lower() + ) + assert firmware_event.data["policy"] == {"expected_nic_firmware": "32.50.1000"} + assert "expected_nic_firmware" in firmware_event.data["reason"] + + def test_multiple_interfaces_with_errors(rdma_analyzer, example_stat_dicts): stats_multi_errors = _build_stats(example_stat_dicts) stats_multi_errors[0].vendor_statistics.req_rx_pkt_seq_err = 15