feat(vmware): add vm_platform_from_annotation_relation config option

docs/source_vmware.md

@@ -103,7 +103,14 @@ First VM is filtered:
 Then all necessary VM data will be collected:<br>
 platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses
 
-Primary IPv4/6 will be determined by interface that provides the default route for this VM.
+Primary IPv6 will be determined by the interface that provides the default route for this VM.
+
+Primary IPv4 is determined by a tiered fallback, first match wins:
+1. an interface IP matches the DNS (A record) name of the VM (`vm_primary_ip4_by_dns_name`)
+2. interface with this IP provides the default route for this VM
+3. an interface IP sits on a VLAN listed in `vm_primary_ip4_fallback_vlans` (tried in order)
+
+An IP that falls within `vm_ip_permitted_overlapping_subnets` is never eligible for any tier.
 
 **Note:**<br>
 IP address information can only be extracted if guest tools are installed and running.

module/common/support.py

@@ -8,6 +8,8 @@
 #  repository or visit: <https://opensource.org/licenses/MIT>.
 
 import asyncio
+import socket
+from ipaddress import ip_address
 
 import aiodns
 
@@ -118,4 +120,92 @@ async def reverse_lookup(resolver, ip):
 
     return {ip: resolved_name}
 
+
+def perform_forward_lookups(names, dns_servers=None):
+    """
+    Perform DNS forward (A record) lookups for host names
+
+    Parameters
+    ----------
+    names: list
+        a list of host names to look up
+    dns_servers: list
+        a list of DNS servers to use to look up list of host names
+
+    Returns
+    -------
+    dict: of {"name": ["ip", ...]} for requested names, list will be empty if nothing was resolved
+    """
+
+    loop = asyncio.get_event_loop()
+
+    resolver = aiodns.DNSResolver(loop=loop)
+
+    if dns_servers is not None:
+        if isinstance(dns_servers, list):
+            log.debug2("using provided DNS servers to perform lookup: %s" % ", ".join(dns_servers))
+            resolver.nameservers = dns_servers
+        else:
+            log.error(f"List of provided DNS servers invalid: {dns_servers}")
+
+    queue = asyncio.gather(*(forward_lookup(resolver, name) for name in names))
+    results = loop.run_until_complete(queue)
+
+    # return dictionary instead of a list of dictionaries
+    return {k: v for x in results for k, v in x.items()}
+
+
+async def forward_lookup(resolver, name):
+    """
+    Perform actual forward lookup
+
+    Parameters
+    ----------
+    resolver: aiodns.DNSResolver
+        handler to DNS resolver
+    name: str
+        host name to look up
+
+    Returns
+    -------
+    dict: of {"name": ["ip", ...]} for requested name, list will be empty if nothing was resolved
+    """
+
+    valid_hostname_characters = "abcdefghijklmnopqrstuvwxyz0123456789-."
+
+    resolved_ips = list()
+    response = None
+
+    if name is None or len(f"{name}") == 0:
+        return dict()
+
+    # validate name to check if this is a valid host name before querying it
+    if not all([bool(str(c).lower() in valid_hostname_characters) for c in name]):
+        log.warning(f"Host name contains invalid characters, skipping A record lookup: {name}")
+        return {name: resolved_ips}
+
+    log.debug2(f"Requesting A record: {name}")
+
+    try:
+        # getaddrinfo (instead of the deprecated query()/gethostbyname()) also honors
+        # the resolver's search list and /etc/hosts, which helps when VM names are
+        # synced without their domain suffix (see 'strip_vm_domain_name')
+        response = await resolver.getaddrinfo(name, socket.AF_INET)
+    except aiodns.error.DNSError as err:
+        log.debug("Unable to find an A record for %s: %s", name, err.args[1])
+
+    for node in getattr(response, "nodes", None) or list():
+        node_address = (getattr(node, "addr", None) or (None,))[0]
+        if isinstance(node_address, bytes):
+            node_address = node_address.decode()
+        try:
+            resolved_ips.append(str(ip_address(node_address)))
+        except ValueError:
+            log.warning(f"A record for '{name}' returned an invalid IP address: {node_address}")
+
+    if len(resolved_ips) > 0:
+        log.debug2("A record(s) for %s: %s" % (name, ", ".join(resolved_ips)))
+
+    return {name: resolved_ips}
+
 # EOF

module/sources/common/source_base.py

@@ -16,6 +16,7 @@
 from module.netbox import *
 from module.common.logging import get_logger
 from module.common.misc import grab
+from module.common.support import normalize_mac_address
 
 log = get_logger()
 
@@ -310,6 +311,11 @@ def add_update_interface(self, interface_object, device_object, interface_data,
         if len(tagged_vlans) > 0:
             del interface_data["tagged_vlans"]
 
+        # a source (currently only vmware/connection.py, for acos/tmos/alteon data-plane
+        # interfaces) can mark an interface as fully owned elsewhere for IP-address purposes -
+        # skip both adding and removing IP assignments on it entirely
+        force_skip_ip_handling = interface_data.pop("_skip_ip_reconciliation", False) is True
+
         # get device tenant
         device_tenant = grab(device_object, "data.tenant")
 
@@ -363,9 +369,41 @@ def add_update_interface(self, interface_object, device_object, interface_data,
 
         # skip handling of IPs for VMs with not installed/running guest tools
         skip_ip_handling = False
-        if type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning":
+        if force_skip_ip_handling is True:
+            log.debug(f"Interface '{interface_object.get_display_name()}' is marked as fully owned elsewhere for "
+                      "IP-address purposes by its source; skipping IP handling for this interface")
+            skip_ip_handling = True
+        elif type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning":
             log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling")
             skip_ip_handling = True
+        elif type(device_object) == NBVM and len(grab(vmware_object, "guest.net", fallback=list())) == 0:
+            # guest tools report "running" but returned zero NICs for the whole VM -- this is a stale/
+            # incompatible guest tools install (seen on old TMOS releases), not a real "all interfaces lost
+            # their IP" event. Trusting it would tear down otherwise-valid NetBox IP-to-interface assignments
+            # every sync cycle. A real per-NIC IP removal still reports the NIC (with no IP), so that case is
+            # unaffected by this guard.
+            log.debug(f"VM '{device_object.name}' guest tools running but reported zero network interfaces; "
+                      f"skipping IP handling (stale/incompatible VMware Tools?)")
+            skip_ip_handling = True
+        elif type(device_object) == NBVM and interface_mac_address is not None:
+            # Same reasoning as the whole-VM guard above, but per-interface: on some guest-tools
+            # cycles an old/flaky TMOS install reports SOME interfaces in guest.net but omits one
+            # specific NIC's MAC entirely (confirmed live: bigip-ve-001's mgmt interface lost its
+            # IP even though guest.net wasn't totally empty that cycle) - the whole-VM guard above
+            # only catches a fully-empty guest.net, not this narrower case. A real per-NIC IP
+            # removal still reports the NIC's MAC (with an empty IP list); total absence of the MAC
+            # itself means guest tools simply didn't report on this NIC this cycle, not a genuine
+            # removal.
+            reported_macs = {
+                normalize_mac_address(grab(g, "macAddress"))
+                for g in grab(vmware_object, "guest.net", fallback=list())
+                if grab(g, "macAddress") is not None
+            }
+            if normalize_mac_address(interface_mac_address) not in reported_macs:
+                log.debug(f"VM '{device_object.name}' interface with MAC '{interface_mac_address}' not present "
+                          "in this cycle's guest.net at all; skipping IP handling for this interface "
+                          "(stale/incomplete VMware Tools report?)")
+                skip_ip_handling = True
 
         ip_address_objects = list()
         matching_ip_prefixes = list()
@@ -442,8 +480,26 @@ def add_update_interface(self, interface_object, device_object, interface_data,
             # try to find matching IP address object
             this_ip_object = None
             skip_this_ip = False
+
+            ip_is_overlapping = any(
+                ip_object.ip in subnet
+                for subnet in grab(self.settings, "vm_ip_permitted_overlapping_subnets", fallback=list())
+            )
+
+            if ip_is_overlapping:
+                for ip in self.inventory.get_all_items(NBIPAddress):
+                    ip_address_string = grab(ip, "data.address", fallback="")
+                    if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"):
+                        continue
+                    if ip.get_interface() == interface_object:
+                        this_ip_object = ip
+                        break
+
             for ip in self.inventory.get_all_items(NBIPAddress):
 
+                if ip_is_overlapping:
+                    continue
+
                 # check if address matches (without prefix length)
                 ip_address_string = grab(ip, "data.address", fallback="")
 
@@ -596,6 +652,14 @@ def add_update_interface(self, interface_object, device_object, interface_data,
         for current_ip in interface_object.get_ip_addresses():
 
             if skip_ip_handling is True:
+                if force_skip_ip_handling is True:
+                    # We're intentionally leaving this IP's assignment alone (a device-side
+                    # collector owns it), but still need to mark it as "seen" this cycle -
+                    # otherwise it stops being claimed by any source and netbox-sync's own
+                    # orphan-pruning (module/netbox/connection.py prune_data(), 30-day
+                    # default delay here) will tag it Orphaned and eventually delete it out
+                    # from under the collector that's actively managing it.
+                    current_ip.update(data={}, source=self)
                 continue
 
             if grab(current_ip, "data.role.value") == "anycast":

module/sources/vmware/config.py

@@ -8,7 +8,7 @@
 #  repository or visit: <https://opensource.org/licenses/MIT>.
 
 import re
-from ipaddress import ip_address
+from ipaddress import ip_address, ip_network
 
 from module.common.misc import quoted_split
 from module.config import source_config_section_name
@@ -82,6 +82,18 @@ def __init__(self):
 
             ConfigOption(**config_option_permitted_subnets_definition),
 
+            ConfigOption("vm_ip_permitted_overlapping_subnets",
+                         str,
+                         description="""\
+                         Define subnets where the same IP address may legitimately appear on
+                         multiple VM interfaces simultaneously — for example, isolated HA
+                         peer-to-peer links where the same /30 addressing is reused across
+                         many VM pairs. Supply a comma-separated list of prefixes in CIDR
+                         notation. When an IP falls within one of these subnets, netbox-sync
+                         creates a separate NetBox IP address object per interface rather than
+                         sharing a single object across VMs.""",
+                         config_example="10.99.99.0/24, 192.168.200.0/24"),
+
             ConfigOptionGroup(title="filter",
                               description="""filters can be used to include/exclude certain objects from importing
                               into NetBox. Include filters are checked first and exclude filters after.
@@ -166,6 +178,20 @@ def __init__(self):
                                                value: defines the desired NetBox platform name""",
                                              config_example="VMware ESXi 7.0.3 = VMware ESXi 7.0 Update 3o"),
                                 ConfigOption("vm_platform_relation", str, config_example="centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016"),
+                                ConfigOption("vm_platform_from_annotation_relation",
+                                             str,
+                                             description="""\
+                                             Override the platform of a VM based on the content of its vCenter
+                                             annotation (the Notes field, synced to the NetBox comments field).
+                                             Useful when vSphere misidentifies the guest OS — for example, F5
+                                             BIG-IP/BIG-IQ Virtual Edition VMs report as CentOS but their
+                                             annotation contains product-identifying text.
+                                             This is done with a comma separated key = value list.
+                                               key: regex matched anywhere in the annotation text (re.search,
+                                                    re.DOTALL — patterns span newlines automatically)
+                                               value: defines the desired NetBox platform name
+                                             Takes priority over vm_platform_relation when both match.""",
+                                             config_example="Virtual Edition.*F5 = TMOS"),
                                 ConfigOption("host_role_relation",
                                              str,
                                              description="""\
@@ -238,6 +264,27 @@ def __init__(self):
                                        as "when-undefined"
                          """,
                          default_value="when-undefined"),
+            ConfigOption("vm_primary_ip4_by_dns_name",
+                         bool,
+                         description="""\
+                         Resolve the VM's name (as it will be synced to NetBox) via DNS and, if
+                         the A record matches one of the IP addresses discovered on the VM's
+                         interfaces, prefer it as the primary IPv4 address. This is tried before
+                         the built-in default-gateway based detection and helps with appliances
+                         that route their default traffic out a data-plane interface instead of
+                         the management interface.
+                         """,
+                         default_value=False),
+            ConfigOption("vm_primary_ip4_fallback_vlans",
+                         str,
+                         description="""\
+                         Comma separated, ordered list of VLAN IDs used as a last resort to
+                         determine a VM's primary IPv4 address if neither its DNS name (see
+                         'vm_primary_ip4_by_dns_name' above) nor its default gateway could be
+                         used. The first VLAN in the list that has an IPv4 address on one of the
+                         VM's interfaces wins. Usually points at a management VLAN.
+                         """,
+                         config_example="1370"),
             ConfigOption("skip_vm_comments",
                          bool,
                          description="Do not sync notes from a VM in vCenter to the comments field on a VM in netbox",
@@ -469,6 +516,37 @@ def validate_options(self):
 
                 continue
 
+            if option.key == "vm_platform_from_annotation_relation":
+
+                relation_data = list()
+
+                for relation in quoted_split(option.value):
+
+                    object_name = relation.split("=")[0].strip()
+                    relation_name = relation.split("=")[1].strip()
+
+                    if len(object_name) == 0 or len(relation_name) == 0:
+                        log.error(f"Config option '{relation}' malformed got '{object_name}' for "
+                                  f"object name and '{relation_name}' for annotation platform name.")
+                        self.set_validation_failed()
+                        continue
+
+                    try:
+                        re_compiled = re.compile(object_name, re.DOTALL)
+                    except Exception as e:
+                        log.error(f"Problem parsing regular expression '{object_name}' for '{relation}': {e}")
+                        self.set_validation_failed()
+                        continue
+
+                    relation_data.append({
+                        "object_regex": re_compiled,
+                        "assigned_name": relation_name
+                    })
+
+                option.set_value(relation_data)
+
+                continue
+
             if "relation" in option.key and "vlan_group_relation" not in option.key:
 
                 relation_data = list()
@@ -477,8 +555,8 @@ def validate_options(self):
 
                 for relation in quoted_split(option.value):
 
-                    object_name = relation.split("=")[0].strip(' "')
-                    relation_name = relation.split("=")[1].strip(' "')
+                    object_name = relation.split("=")[0].strip()
+                    relation_name = relation.split("=")[1].strip()
 
                     if len(object_name) == 0 or len(relation_name) == 0:
                         log.error(f"Config option '{relation}' malformed got '{object_name}' for "
@@ -527,8 +605,14 @@ def validate_options(self):
             if option.key == "custom_dns_servers":
 
                 dns_name_lookup = self.get_option_by_name("dns_name_lookup")
+                vm_primary_ip4_by_dns_name = self.get_option_by_name("vm_primary_ip4_by_dns_name")
+
+                dns_lookup_needed = any([
+                    isinstance(dns_name_lookup, ConfigOption) and dns_name_lookup.value is True,
+                    isinstance(vm_primary_ip4_by_dns_name, ConfigOption) and vm_primary_ip4_by_dns_name.value is True
+                ])
 
-                if not isinstance(dns_name_lookup, ConfigOption) or dns_name_lookup.value is False:
+                if dns_lookup_needed is False:
                     continue
 
                 custom_dns_servers = quoted_split(option.value)
@@ -650,3 +734,28 @@ def validate_options(self):
                 self.set_validation_failed()
 
             permitted_subnets_option.set_value(permitted_subnets)
+
+        overlapping_subnets_option = self.get_option_by_name("vm_ip_permitted_overlapping_subnets")
+
+        if overlapping_subnets_option is not None and overlapping_subnets_option.value is not None:
+            subnet_list = [x.strip() for x in overlapping_subnets_option.value.split(",") if x.strip() != ""]
+            parsed_subnets = []
+            for subnet in subnet_list:
+                try:
+                    parsed_subnets.append(ip_network(subnet, strict=False))
+                except Exception as e:
+                    log.error(f"Problem parsing vm_ip_permitted_overlapping_subnets entry '{subnet}': {e}")
+                    self.set_validation_failed()
+            overlapping_subnets_option.set_value(parsed_subnets)
+
+        fallback_vlans_option = self.get_option_by_name("vm_primary_ip4_fallback_vlans")
+
+        if fallback_vlans_option is not None and fallback_vlans_option.value is not None:
+            parsed_vlan_ids = list()
+            for vlan_id in quoted_split(fallback_vlans_option.value) or list():
+                try:
+                    parsed_vlan_ids.append(int(vlan_id))
+                except ValueError:
+                    log.error(f"Problem parsing vm_primary_ip4_fallback_vlans entry '{vlan_id}', must be an integer")
+                    self.set_validation_failed()
+            fallback_vlans_option.set_value(parsed_vlan_ids)