Extend report lldp into a cabling check against NetBox

osism/utils/hardware/mapping.py

@@ -151,6 +151,41 @@ def fetch_interfaces(
     return interfaces
 
 
+def fetch_devices(
+    nb, device_ids: Iterable[int], chunk_size: int = 50
+) -> List[Dict[str, Any]]:
+    """Fetch devices by ID with few requests and return them as plain data."""
+    ids = sorted(set(device_ids))
+    devices: List[Dict[str, Any]] = []
+    for start in range(0, len(ids), chunk_size):
+        chunk = ids[start : start + chunk_size]
+        devices.extend(dict(record) for record in nb.dcim.devices.filter(id=chunk))
+    return devices
+
+
+def find_devices(
+    nb, names: Iterable[str], chunk_size: int = 50
+) -> List[Dict[str, Any]]:
+    """Find devices by name, ignoring case, and return them as plain data.
+
+    Each name is also tried without its domain, because a device can
+    announce itself fully qualified while NetBox holds the short name.
+    """
+    wanted: List[str] = []
+    for name in names:
+        for candidate in (name, name.split(".")[0]):
+            if candidate and candidate not in wanted:
+                wanted.append(candidate)
+
+    devices: Dict[Any, Dict[str, Any]] = {}
+    for start in range(0, len(wanted), chunk_size):
+        chunk = wanted[start : start + chunk_size]
+        for record in nb.dcim.devices.filter(name__ie=chunk):
+            device = dict(record)
+            devices.setdefault(device.get("id"), device)
+    return list(devices.values())
+
+
 def physical_interfaces(
     interfaces: Iterable[NetboxInterface], include_mgmt: bool = True
 ) -> List[NetboxInterface]:
@@ -217,6 +252,25 @@ def sonic_name_index(
     return index
 
 
+def sonic_alias_index(hwsku: Optional[str]) -> Dict[str, str]:
+    """Return ``{vendor alias: SONiC port name}`` of a HWSKU port config.
+
+    A switch that was not configured by OSISM advertises the vendor alias
+    of the port config file (``twentyFiveGigE1``) instead of the NetBox
+    name. Empty without a HWSKU or without its port config file.
+    """
+    if not hwsku:
+        return {}
+
+    from osism.tasks.conductor.sonic.interface import get_port_config
+
+    return {
+        fields["alias"]: name
+        for name, fields in get_port_config(hwsku).items()
+        if fields.get("alias")
+    }
+
+
 class _Matcher:
     """Keeps a NetBox interface from being matched to two ports."""
 

osism/utils/hardware/parse_lldp.py

@@ -10,10 +10,37 @@
 """
 
 import json
+import re
 from typing import Any, Dict, List, Optional
 
+from osism.utils.hardware.parse_sonic import table
+
 Neighbours = Dict[str, List[Dict[str, Any]]]
 
+# LLDP-MIB subtypes as stored in LLDP_ENTRY_TABLE, named as lldpctl does
+CHASSIS_ID_SUBTYPES = {
+    "1": "chassis-component",
+    "2": "ifalias",
+    "3": "port-component",
+    "4": "mac",
+    "5": "ip",
+    "6": "ifname",
+    "7": "local",
+}
+PORT_ID_SUBTYPES = {
+    "1": "ifalias",
+    "2": "port-component",
+    "3": "mac",
+    "4": "ip",
+    "5": "ifname",
+    "6": "agent-circuit-id",
+    "7": "local",
+}
+
+
+# Key of an LLDP_ENTRY_TABLE entry that names the neighbour after the port
+_ENTRY_KEY = re.compile(r"^(.+):((?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2})$")
+
 
 class LldpError(Exception):
     """LLDP data is not available on a device."""
@@ -142,3 +169,86 @@ def linux_neighbours(document: Dict[str, Any]) -> Neighbours:
         return parse_lldpctl(result.get("out") or "")
     except ValueError:
         raise LldpError("unparsable lldpctl output")
+
+
+def entry_port(key: str) -> str:
+    """Return the local port an ``LLDP_ENTRY_TABLE`` key belongs to.
+
+    The key is the port name (``Ethernet0``). Enterprise SONiC 4.5 appends
+    the source MAC address of the neighbour (``Ethernet0:aa:bb:cc:dd:ee:ff``)
+    and so keeps several neighbours per port.
+    """
+    match = _ENTRY_KEY.match(key)
+    return match.group(1) if match else key
+
+
+def parse_lldp_entry_table(entries: Dict[str, Dict[str, Any]]) -> Neighbours:
+    """Parse the entries of ``LLDP_ENTRY_TABLE`` (APPL_DB) of a SONiC switch.
+
+    The records have no ``age``. ``lldp_rem_time_mark`` is not one on every
+    release: Enterprise SONiC stores the uptime of ``lldpd`` at which the
+    neighbour was learned, so it is large for a neighbour that just came
+    up and small for one that has been there since boot.
+
+    Args:
+        entries: Fields per key without the table prefix, as
+            ``parse_sonic.table`` returns them
+    """
+    neighbours: Neighbours = {}
+    for key, fields in entries.items():
+        addresses = [
+            address.strip()
+            for address in str(fields.get("lldp_rem_man_addr") or "").split(",")
+            if address.strip()
+        ]
+        neighbours.setdefault(entry_port(key), []).append(
+            neighbour(
+                system_name=fields.get("lldp_rem_sys_name"),
+                chassis_id=fields.get("lldp_rem_chassis_id"),
+                chassis_id_type=CHASSIS_ID_SUBTYPES.get(
+                    str(fields.get("lldp_rem_chassis_id_subtype"))
+                ),
+                port_id=fields.get("lldp_rem_port_id"),
+                port_id_type=PORT_ID_SUBTYPES.get(
+                    str(fields.get("lldp_rem_port_id_subtype"))
+                ),
+                port_description=fields.get("lldp_rem_port_desc"),
+                management_addresses=addresses,
+            )
+        )
+    return neighbours
+
+
+def sonic_neighbours(document: Dict[str, Any]) -> Neighbours:
+    """Return the neighbours found in a SONiC collector document.
+
+    ``LLDP_ENTRY_TABLE`` is the primary source. Ports it does not list are
+    filled in from ``lldpctl``, when the collector ran it.
+
+    Raises:
+        LldpError: Neither source could be read
+    """
+    collected = "LLDP_ENTRY_TABLE" in ((document.get("db") or {}).get("APPL_DB") or {})
+    neighbours = parse_lldp_entry_table(table(document, "APPL_DB", "LLDP_ENTRY_TABLE"))
+
+    result = document.get("lldpctl")
+    if isinstance(result, dict) and result.get("rc") == 0:
+        try:
+            for name, found in parse_lldpctl(result.get("out") or "").items():
+                neighbours.setdefault(name, found)
+        except ValueError:
+            if not collected:
+                raise LldpError("unparsable lldpctl output")
+    elif not collected:
+        lines: List[str] = []
+        if isinstance(result, dict):
+            lines = (result.get("err") or "").strip().splitlines()
+        errors = document.get("errors") or {}
+        reason = (
+            (lines[-1] if lines else "")
+            or errors.get("APPL_DB LLDP_ENTRY_TABLE")
+            or "no LLDP data"
+        )
+        raise LldpError(f"LLDP neighbours not readable: {reason}")
+
+    return neighbours

osism/utils/hardware/remote_sonic.py

@@ -48,6 +48,9 @@
     "system": [
         ("STATE_DB", "EEPROM_INFO", "EEPROM_INFO|*", False),
     ],
+    "lldp": [
+        ("APPL_DB", "LLDP_ENTRY_TABLE", "LLDP_ENTRY_TABLE:*", False),
+    ],
 }
 
 
@@ -134,9 +137,27 @@ def extra_system(document, sys_net, runner):
     document["commands"] = commands
 
 
+def extra_lldp(document, sys_net, runner):
+    # lldpctl is the fallback for a switch without LLDP_ENTRY_TABLE, and the
+    # second source for the management port in case the table only lists
+    # front-panel ports. lldpd runs in the lldp container.
+    entries = (document["db"].get("APPL_DB") or {}).get("LLDP_ENTRY_TABLE") or {}
+    # Enterprise SONiC 4.5 appends the source MAC address of the neighbour
+    # to the key: LLDP_ENTRY_TABLE:eth0:aa:bb:cc:dd:ee:ff
+    key = "LLDP_ENTRY_TABLE:" + MGMT_PORT
+    if any(name == key or name.startswith(key + ":") for name in entries):
+        return
+    command = ["docker", "exec", "lldp", "lldpctl", "-f", "json"]
+    result = runner(command)
+    if result["rc"] != 0:
+        result = runner(["sudo", "-n"] + command)
+    document["lldpctl"] = result
+
+
 EXTRAS = {
     "ports": extra_ports,
     "system": extra_system,
+    "lldp": extra_lldp,
 }