From d5d6b640c2de8c5988403988e90173b665594385 Mon Sep 17 00:00:00 2001 From: graepaul_amdeng Date: Thu, 17 Sep 2026 13:34:17 -0700 Subject: [PATCH 1/2] Remove other possible large events --- nodescraper/plugins/inband/memory/memory_collector.py | 6 +++++- nodescraper/plugins/inband/memory/memorydata.py | 7 +++++++ nodescraper/plugins/inband/nic/nic_analyzer.py | 1 - nodescraper/plugins/inband/process/process_collector.py | 1 - nodescraper/plugins/inband/storage/storage_collector.py | 1 - 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nodescraper/plugins/inband/memory/memory_collector.py b/nodescraper/plugins/inband/memory/memory_collector.py index 35c279d9..e228562d 100644 --- a/nodescraper/plugins/inband/memory/memory_collector.py +++ b/nodescraper/plugins/inband/memory/memory_collector.py @@ -173,7 +173,11 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel] self._log_event( category=EventCategory.OS, description="Free and total memory read", - data=mem_data.model_dump(), + data={ + "mem_total": mem_total, + "mem_available": mem_available, + "mem_free": mem_free, + }, priority=EventPriority.INFO, ) if mem_available: diff --git a/nodescraper/plugins/inband/memory/memorydata.py b/nodescraper/plugins/inband/memory/memorydata.py index 9e9209bb..38f33605 100644 --- a/nodescraper/plugins/inband/memory/memorydata.py +++ b/nodescraper/plugins/inband/memory/memorydata.py @@ -89,3 +89,10 @@ class MemoryDataModel(DataModel): mem_available: Optional[str] = None lsmem_data: Optional[LsmemData] = None numa_topology: Optional[NumaTopology] = None + + def __str__(self) -> str: + return ( + f"MemoryDataModel(mem_free={self.mem_free}, mem_total={self.mem_total}, " + f"mem_available={self.mem_available}, lsmem_data={self.lsmem_data}, " + f"numa_topology={self.numa_topology})" + ) diff --git a/nodescraper/plugins/inband/nic/nic_analyzer.py b/nodescraper/plugins/inband/nic/nic_analyzer.py index 30543867..d6d324ee 100644 --- a/nodescraper/plugins/inband/nic/nic_analyzer.py +++ b/nodescraper/plugins/inband/nic/nic_analyzer.py @@ -233,7 +233,6 @@ def analyze_data( description=f"Broadcom device {device_num}: getqos does not match expected QoS: {'; '.join(mismatches)}", data={ "device_num": device_num, - "qos": qos.model_dump(), "mismatches": mismatches, }, priority=EventPriority.WARNING, diff --git a/nodescraper/plugins/inband/process/process_collector.py b/nodescraper/plugins/inband/process/process_collector.py index 72feb8c5..47dfc076 100644 --- a/nodescraper/plugins/inband/process/process_collector.py +++ b/nodescraper/plugins/inband/process/process_collector.py @@ -99,7 +99,6 @@ def collect_data( self._log_event( category="PROCESS_READ", description="Process data collected", - data=process_data.model_dump(), priority=EventPriority.INFO, ) self.result.message = "Process data collected" diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index e5373ebc..69b6d106 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -98,7 +98,6 @@ def collect_data( self._log_event( category="STORAGE_READ", description="Available storage read", - data=storage_model.model_dump(), priority=EventPriority.INFO, ) self.result.message = f"{len(storage_model.storage_data)} storage devices collected" From e9e5b1b66ba329011848b81a458c68edc924987a Mon Sep 17 00:00:00 2001 From: graepaul_amdeng Date: Thu, 17 Sep 2026 14:08:29 -0700 Subject: [PATCH 2/2] Make event generation 4x faster and make it more accurate --- nodescraper/models/event.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/nodescraper/models/event.py b/nodescraper/models/event.py index de2ecc64..7e3851db 100644 --- a/nodescraper/models/event.py +++ b/nodescraper/models/event.py @@ -31,24 +31,15 @@ from typing import Any, Optional, Union, cast from pydantic import BaseModel, Field, field_serializer, field_validator +from pydantic_core import to_jsonable_python from nodescraper.constants import DEFAULT_EVENT_REPORTER from nodescraper.enums import EventPriority def _data_to_json_safe(obj: Any) -> Any: - """Recursively convert event data to JSON-serializable form (e.g. exceptions -> str).""" - if isinstance(obj, BaseException): - return str(obj) - if isinstance(obj, dict): - return {k: _data_to_json_safe(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_data_to_json_safe(v) for v in obj] - if isinstance(obj, (str, int, float, bool, type(None))): - return obj - if isinstance(obj, (Enum, datetime.datetime, uuid.UUID)): - return str(obj) - return str(obj) + """Convert data to JSON-serializable form using Pydantic's fast Rust core.""" + return to_jsonable_python(obj, fallback=lambda x: str(x)) LOG_LEVEL_MAP = { @@ -178,7 +169,9 @@ def validate_data(cls, data: dict) -> dict: Returns: dict: data output """ - if len(str(data).encode("utf-8")) >= (1024 * 100): + # Use the same serialization that will happen during model_dump() + json_safe_data = _data_to_json_safe(data) + if len(str(json_safe_data).encode("utf-8")) >= (1024 * 100): raise ValueError("Data must be below 100KB in size") return data