diff --git a/docs/source_vmware.md b/docs/source_vmware.md
index 11f1d03e..d1039604 100644
--- a/docs/source_vmware.md
+++ b/docs/source_vmware.md
@@ -103,7 +103,14 @@ First VM is filtered:
Then all necessary VM data will be collected:
platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses
-Primary IPv4/6 will be determined by interface that provides the default route for this VM.
+Primary IPv6 will be determined by the interface that provides the default route for this VM.
+
+Primary IPv4 is determined by a tiered fallback, first match wins:
+1. an interface IP matches the DNS (A record) name of the VM (`vm_primary_ip4_by_dns_name`)
+2. interface with this IP provides the default route for this VM
+3. an interface IP sits on a VLAN listed in `vm_primary_ip4_fallback_vlans` (tried in order)
+
+An IP that falls within `vm_ip_permitted_overlapping_subnets` is never eligible for any tier.
**Note:**
IP address information can only be extracted if guest tools are installed and running.
diff --git a/module/common/support.py b/module/common/support.py
index 6fea6464..8d161ead 100644
--- a/module/common/support.py
+++ b/module/common/support.py
@@ -8,6 +8,8 @@
# repository or visit: .
import asyncio
+import socket
+from ipaddress import ip_address
import aiodns
@@ -118,4 +120,92 @@ async def reverse_lookup(resolver, ip):
return {ip: resolved_name}
+
+def perform_forward_lookups(names, dns_servers=None):
+ """
+ Perform DNS forward (A record) lookups for host names
+
+ Parameters
+ ----------
+ names: list
+ a list of host names to look up
+ dns_servers: list
+ a list of DNS servers to use to look up list of host names
+
+ Returns
+ -------
+ dict: of {"name": ["ip", ...]} for requested names, list will be empty if nothing was resolved
+ """
+
+ loop = asyncio.get_event_loop()
+
+ resolver = aiodns.DNSResolver(loop=loop)
+
+ if dns_servers is not None:
+ if isinstance(dns_servers, list):
+ log.debug2("using provided DNS servers to perform lookup: %s" % ", ".join(dns_servers))
+ resolver.nameservers = dns_servers
+ else:
+ log.error(f"List of provided DNS servers invalid: {dns_servers}")
+
+ queue = asyncio.gather(*(forward_lookup(resolver, name) for name in names))
+ results = loop.run_until_complete(queue)
+
+ # return dictionary instead of a list of dictionaries
+ return {k: v for x in results for k, v in x.items()}
+
+
+async def forward_lookup(resolver, name):
+ """
+ Perform actual forward lookup
+
+ Parameters
+ ----------
+ resolver: aiodns.DNSResolver
+ handler to DNS resolver
+ name: str
+ host name to look up
+
+ Returns
+ -------
+ dict: of {"name": ["ip", ...]} for requested name, list will be empty if nothing was resolved
+ """
+
+ valid_hostname_characters = "abcdefghijklmnopqrstuvwxyz0123456789-."
+
+ resolved_ips = list()
+ response = None
+
+ if name is None or len(f"{name}") == 0:
+ return dict()
+
+ # validate name to check if this is a valid host name before querying it
+ if not all([bool(str(c).lower() in valid_hostname_characters) for c in name]):
+ log.warning(f"Host name contains invalid characters, skipping A record lookup: {name}")
+ return {name: resolved_ips}
+
+ log.debug2(f"Requesting A record: {name}")
+
+ try:
+ # getaddrinfo (instead of the deprecated query()/gethostbyname()) also honors
+ # the resolver's search list and /etc/hosts, which helps when VM names are
+ # synced without their domain suffix (see 'strip_vm_domain_name')
+ response = await resolver.getaddrinfo(name, socket.AF_INET)
+ except aiodns.error.DNSError as err:
+ log.debug("Unable to find an A record for %s: %s", name, err.args[1])
+
+ for node in getattr(response, "nodes", None) or list():
+ node_address = (getattr(node, "addr", None) or (None,))[0]
+ if isinstance(node_address, bytes):
+ node_address = node_address.decode()
+ try:
+ resolved_ips.append(str(ip_address(node_address)))
+ except ValueError:
+ log.warning(f"A record for '{name}' returned an invalid IP address: {node_address}")
+
+ if len(resolved_ips) > 0:
+ log.debug2("A record(s) for %s: %s" % (name, ", ".join(resolved_ips)))
+
+ return {name: resolved_ips}
+
# EOF
diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py
index 4d20e9d2..8da1c794 100644
--- a/module/sources/common/source_base.py
+++ b/module/sources/common/source_base.py
@@ -16,6 +16,7 @@
from module.netbox import *
from module.common.logging import get_logger
from module.common.misc import grab
+from module.common.support import normalize_mac_address
log = get_logger()
@@ -310,6 +311,11 @@ def add_update_interface(self, interface_object, device_object, interface_data,
if len(tagged_vlans) > 0:
del interface_data["tagged_vlans"]
+ # a source (currently only vmware/connection.py, for acos/tmos/alteon data-plane
+ # interfaces) can mark an interface as fully owned elsewhere for IP-address purposes -
+ # skip both adding and removing IP assignments on it entirely
+ force_skip_ip_handling = interface_data.pop("_skip_ip_reconciliation", False) is True
+
# get device tenant
device_tenant = grab(device_object, "data.tenant")
@@ -363,9 +369,41 @@ def add_update_interface(self, interface_object, device_object, interface_data,
# skip handling of IPs for VMs with not installed/running guest tools
skip_ip_handling = False
- if type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning":
+ if force_skip_ip_handling is True:
+ log.debug(f"Interface '{interface_object.get_display_name()}' is marked as fully owned elsewhere for "
+ "IP-address purposes by its source; skipping IP handling for this interface")
+ skip_ip_handling = True
+ elif type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning":
log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling")
skip_ip_handling = True
+ elif type(device_object) == NBVM and len(grab(vmware_object, "guest.net", fallback=list())) == 0:
+ # guest tools report "running" but returned zero NICs for the whole VM -- this is a stale/
+ # incompatible guest tools install (seen on old TMOS releases), not a real "all interfaces lost
+ # their IP" event. Trusting it would tear down otherwise-valid NetBox IP-to-interface assignments
+ # every sync cycle. A real per-NIC IP removal still reports the NIC (with no IP), so that case is
+ # unaffected by this guard.
+ log.debug(f"VM '{device_object.name}' guest tools running but reported zero network interfaces; "
+ f"skipping IP handling (stale/incompatible VMware Tools?)")
+ skip_ip_handling = True
+ elif type(device_object) == NBVM and interface_mac_address is not None:
+ # Same reasoning as the whole-VM guard above, but per-interface: on some guest-tools
+ # cycles an old/flaky TMOS install reports SOME interfaces in guest.net but omits one
+ # specific NIC's MAC entirely (confirmed live: bigip-ve-001's mgmt interface lost its
+ # IP even though guest.net wasn't totally empty that cycle) - the whole-VM guard above
+ # only catches a fully-empty guest.net, not this narrower case. A real per-NIC IP
+ # removal still reports the NIC's MAC (with an empty IP list); total absence of the MAC
+ # itself means guest tools simply didn't report on this NIC this cycle, not a genuine
+ # removal.
+ reported_macs = {
+ normalize_mac_address(grab(g, "macAddress"))
+ for g in grab(vmware_object, "guest.net", fallback=list())
+ if grab(g, "macAddress") is not None
+ }
+ if normalize_mac_address(interface_mac_address) not in reported_macs:
+ log.debug(f"VM '{device_object.name}' interface with MAC '{interface_mac_address}' not present "
+ "in this cycle's guest.net at all; skipping IP handling for this interface "
+ "(stale/incomplete VMware Tools report?)")
+ skip_ip_handling = True
ip_address_objects = list()
matching_ip_prefixes = list()
@@ -442,8 +480,26 @@ def add_update_interface(self, interface_object, device_object, interface_data,
# try to find matching IP address object
this_ip_object = None
skip_this_ip = False
+
+ ip_is_overlapping = any(
+ ip_object.ip in subnet
+ for subnet in grab(self.settings, "vm_ip_permitted_overlapping_subnets", fallback=list())
+ )
+
+ if ip_is_overlapping:
+ for ip in self.inventory.get_all_items(NBIPAddress):
+ ip_address_string = grab(ip, "data.address", fallback="")
+ if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"):
+ continue
+ if ip.get_interface() == interface_object:
+ this_ip_object = ip
+ break
+
for ip in self.inventory.get_all_items(NBIPAddress):
+ if ip_is_overlapping:
+ continue
+
# check if address matches (without prefix length)
ip_address_string = grab(ip, "data.address", fallback="")
@@ -596,6 +652,14 @@ def add_update_interface(self, interface_object, device_object, interface_data,
for current_ip in interface_object.get_ip_addresses():
if skip_ip_handling is True:
+ if force_skip_ip_handling is True:
+ # We're intentionally leaving this IP's assignment alone (a device-side
+ # collector owns it), but still need to mark it as "seen" this cycle -
+ # otherwise it stops being claimed by any source and netbox-sync's own
+ # orphan-pruning (module/netbox/connection.py prune_data(), 30-day
+ # default delay here) will tag it Orphaned and eventually delete it out
+ # from under the collector that's actively managing it.
+ current_ip.update(data={}, source=self)
continue
if grab(current_ip, "data.role.value") == "anycast":
diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py
index e4d8e2bc..80ebcf57 100644
--- a/module/sources/vmware/config.py
+++ b/module/sources/vmware/config.py
@@ -8,7 +8,7 @@
# repository or visit: .
import re
-from ipaddress import ip_address
+from ipaddress import ip_address, ip_network
from module.common.misc import quoted_split
from module.config import source_config_section_name
@@ -82,6 +82,18 @@ def __init__(self):
ConfigOption(**config_option_permitted_subnets_definition),
+ ConfigOption("vm_ip_permitted_overlapping_subnets",
+ str,
+ description="""\
+ Define subnets where the same IP address may legitimately appear on
+ multiple VM interfaces simultaneously — for example, isolated HA
+ peer-to-peer links where the same /30 addressing is reused across
+ many VM pairs. Supply a comma-separated list of prefixes in CIDR
+ notation. When an IP falls within one of these subnets, netbox-sync
+ creates a separate NetBox IP address object per interface rather than
+ sharing a single object across VMs.""",
+ config_example="10.99.99.0/24, 192.168.200.0/24"),
+
ConfigOptionGroup(title="filter",
description="""filters can be used to include/exclude certain objects from importing
into NetBox. Include filters are checked first and exclude filters after.
@@ -166,6 +178,20 @@ def __init__(self):
value: defines the desired NetBox platform name""",
config_example="VMware ESXi 7.0.3 = VMware ESXi 7.0 Update 3o"),
ConfigOption("vm_platform_relation", str, config_example="centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016"),
+ ConfigOption("vm_platform_from_annotation_relation",
+ str,
+ description="""\
+ Override the platform of a VM based on the content of its vCenter
+ annotation (the Notes field, synced to the NetBox comments field).
+ Useful when vSphere misidentifies the guest OS — for example, F5
+ BIG-IP/BIG-IQ Virtual Edition VMs report as CentOS but their
+ annotation contains product-identifying text.
+ This is done with a comma separated key = value list.
+ key: regex matched anywhere in the annotation text (re.search,
+ re.DOTALL — patterns span newlines automatically)
+ value: defines the desired NetBox platform name
+ Takes priority over vm_platform_relation when both match.""",
+ config_example="Virtual Edition.*F5 = TMOS"),
ConfigOption("host_role_relation",
str,
description="""\
@@ -238,6 +264,27 @@ def __init__(self):
as "when-undefined"
""",
default_value="when-undefined"),
+ ConfigOption("vm_primary_ip4_by_dns_name",
+ bool,
+ description="""\
+ Resolve the VM's name (as it will be synced to NetBox) via DNS and, if
+ the A record matches one of the IP addresses discovered on the VM's
+ interfaces, prefer it as the primary IPv4 address. This is tried before
+ the built-in default-gateway based detection and helps with appliances
+ that route their default traffic out a data-plane interface instead of
+ the management interface.
+ """,
+ default_value=False),
+ ConfigOption("vm_primary_ip4_fallback_vlans",
+ str,
+ description="""\
+ Comma separated, ordered list of VLAN IDs used as a last resort to
+ determine a VM's primary IPv4 address if neither its DNS name (see
+ 'vm_primary_ip4_by_dns_name' above) nor its default gateway could be
+ used. The first VLAN in the list that has an IPv4 address on one of the
+ VM's interfaces wins. Usually points at a management VLAN.
+ """,
+ config_example="1370"),
ConfigOption("skip_vm_comments",
bool,
description="Do not sync notes from a VM in vCenter to the comments field on a VM in netbox",
@@ -469,6 +516,37 @@ def validate_options(self):
continue
+ if option.key == "vm_platform_from_annotation_relation":
+
+ relation_data = list()
+
+ for relation in quoted_split(option.value):
+
+ object_name = relation.split("=")[0].strip()
+ relation_name = relation.split("=")[1].strip()
+
+ if len(object_name) == 0 or len(relation_name) == 0:
+ log.error(f"Config option '{relation}' malformed got '{object_name}' for "
+ f"object name and '{relation_name}' for annotation platform name.")
+ self.set_validation_failed()
+ continue
+
+ try:
+ re_compiled = re.compile(object_name, re.DOTALL)
+ except Exception as e:
+ log.error(f"Problem parsing regular expression '{object_name}' for '{relation}': {e}")
+ self.set_validation_failed()
+ continue
+
+ relation_data.append({
+ "object_regex": re_compiled,
+ "assigned_name": relation_name
+ })
+
+ option.set_value(relation_data)
+
+ continue
+
if "relation" in option.key and "vlan_group_relation" not in option.key:
relation_data = list()
@@ -477,8 +555,8 @@ def validate_options(self):
for relation in quoted_split(option.value):
- object_name = relation.split("=")[0].strip(' "')
- relation_name = relation.split("=")[1].strip(' "')
+ object_name = relation.split("=")[0].strip()
+ relation_name = relation.split("=")[1].strip()
if len(object_name) == 0 or len(relation_name) == 0:
log.error(f"Config option '{relation}' malformed got '{object_name}' for "
@@ -527,8 +605,14 @@ def validate_options(self):
if option.key == "custom_dns_servers":
dns_name_lookup = self.get_option_by_name("dns_name_lookup")
+ vm_primary_ip4_by_dns_name = self.get_option_by_name("vm_primary_ip4_by_dns_name")
+
+ dns_lookup_needed = any([
+ isinstance(dns_name_lookup, ConfigOption) and dns_name_lookup.value is True,
+ isinstance(vm_primary_ip4_by_dns_name, ConfigOption) and vm_primary_ip4_by_dns_name.value is True
+ ])
- if not isinstance(dns_name_lookup, ConfigOption) or dns_name_lookup.value is False:
+ if dns_lookup_needed is False:
continue
custom_dns_servers = quoted_split(option.value)
@@ -650,3 +734,28 @@ def validate_options(self):
self.set_validation_failed()
permitted_subnets_option.set_value(permitted_subnets)
+
+ overlapping_subnets_option = self.get_option_by_name("vm_ip_permitted_overlapping_subnets")
+
+ if overlapping_subnets_option is not None and overlapping_subnets_option.value is not None:
+ subnet_list = [x.strip() for x in overlapping_subnets_option.value.split(",") if x.strip() != ""]
+ parsed_subnets = []
+ for subnet in subnet_list:
+ try:
+ parsed_subnets.append(ip_network(subnet, strict=False))
+ except Exception as e:
+ log.error(f"Problem parsing vm_ip_permitted_overlapping_subnets entry '{subnet}': {e}")
+ self.set_validation_failed()
+ overlapping_subnets_option.set_value(parsed_subnets)
+
+ fallback_vlans_option = self.get_option_by_name("vm_primary_ip4_fallback_vlans")
+
+ if fallback_vlans_option is not None and fallback_vlans_option.value is not None:
+ parsed_vlan_ids = list()
+ for vlan_id in quoted_split(fallback_vlans_option.value) or list():
+ try:
+ parsed_vlan_ids.append(int(vlan_id))
+ except ValueError:
+ log.error(f"Problem parsing vm_primary_ip4_fallback_vlans entry '{vlan_id}', must be an integer")
+ self.set_validation_failed()
+ fallback_vlans_option.set_value(parsed_vlan_ids)
diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py
index e63763c9..56fb325a 100644
--- a/module/sources/vmware/connection.py
+++ b/module/sources/vmware/connection.py
@@ -30,7 +30,7 @@
from module.sources.vmware.config import VMWareConfig
from module.common.logging import get_logger, DEBUG3
from module.common.misc import grab, dump, get_string_or_none, plural, quoted_split
-from module.common.support import normalize_mac_address
+from module.common.support import normalize_mac_address, perform_forward_lookups
from module.netbox.inventory import NetBoxInventory
from module.netbox import *
@@ -117,6 +117,12 @@ def __init__(self, name=None):
self.create_api_session()
+ # Derive VCSA metadata from the connected vCenter source. The source
+ # FQDN is the deterministic correlation key; these values must not be
+ # duplicated in settings.ini.
+ self.vcsa_source_fqdn = self._normalise_fqdn(self.settings.host_fqdn)
+ self.vcsa_version = get_string_or_none(grab(self.session, "about.version"))
+
self.init_successful = True
# instantiate source specific vars
@@ -134,6 +140,49 @@ def __init__(self, name=None):
self.parsing_vms_the_first_time = True
self.objects_to_reevaluate = list()
self.parsing_objects_to_reevaluate = False
+ # cache of resolved VM name -> [ip, ...] A records, populated once per sync run
+ self.vm_dns_lookup_cache = dict()
+
+ @staticmethod
+ def _normalise_fqdn(value):
+ if value is None:
+ return None
+ return str(value).strip().rstrip(".").lower()
+
+ def _is_vcsa_vm(self, vm_name):
+ """Match only the VCSA VM for this source; do not infer by platform."""
+ vm_fqdn = self._normalise_fqdn(vm_name)
+ if vm_fqdn is None or self.vcsa_source_fqdn is None:
+ return False
+ return vm_fqdn in {
+ self.vcsa_source_fqdn,
+ self.vcsa_source_fqdn.split(".", 1)[0]
+ }
+
+ def _get_vcsa_custom_fields(self, vm_name):
+ object_type = "virtualization.virtualmachine"
+ fields = {}
+
+ field = self.add_update_custom_field({
+ "name": "is_vcsa", "label": "IS_VCSA",
+ "object_types": [object_type], "type": "boolean",
+ "description": f"Whether this VM is the VCSA for source '{self.name}'"
+ })
+ is_vcsa = self._is_vcsa_vm(vm_name)
+ fields[grab(field, "data.name")] = is_vcsa
+
+ if not is_vcsa:
+ return fields
+
+ if self.vcsa_version is not None:
+ field = self.add_update_custom_field({
+ "name": "vcsa_version", "label": "VCSA_VERSION",
+ "object_types": [object_type], "type": "text",
+ "description": f"VCSA version reported by source '{self.name}'"
+ })
+ fields[grab(field, "data.name")] = self.vcsa_version
+
+ return fields
def create_sdk_session(self):
"""
@@ -387,6 +436,9 @@ def apply(self):
self.parsing_vms_the_first_time = False
log.debug("Iterating over all virtual machines a second time ")
+ if view_details.get("view_type") == vim.VirtualMachine:
+ self.resolve_vm_dns_names(view_objects)
+
for obj in view_objects:
if log.level == DEBUG3:
@@ -414,6 +466,63 @@ def apply(self):
self.update_basic_data()
+ def resolve_vm_dns_names(self, vm_objects):
+ """
+ Bulk-resolve A records for a list of VMs and populate 'vm_dns_lookup_cache' with the
+ results. Called once per vCenter view so 'get_vm_dns_ips' below never has to perform a
+ blocking per-VM lookup while VMs are being processed.
+
+ Only does anything if 'vm_primary_ip4_by_dns_name' is enabled.
+
+ Parameters
+ ----------
+ vm_objects: list of vim.VirtualMachine
+ VMs to resolve names for
+ """
+
+ if self.settings.vm_primary_ip4_by_dns_name is not True:
+ return
+
+ vm_names = list()
+ for vm_object in vm_objects or list():
+
+ vm_name = get_string_or_none(grab(vm_object, "name"))
+
+ if vm_name is not None and self.settings.strip_vm_domain_name is True:
+ vm_name = vm_name.split(".")[0]
+
+ if vm_name is not None and vm_name not in vm_names and vm_name not in self.vm_dns_lookup_cache:
+ vm_names.append(vm_name)
+
+ if len(vm_names) == 0:
+ return
+
+ log.debug(f"Resolving DNS A record{plural(len(vm_names))} for {len(vm_names)} VM name{plural(len(vm_names))}")
+
+ self.vm_dns_lookup_cache.update(perform_forward_lookups(vm_names, self.settings.custom_dns_servers))
+
+ def get_vm_dns_ips(self, vm_name):
+ """
+ Return the resolved IPv4 addresses for a VM name from 'vm_dns_lookup_cache', performing
+ the lookup on demand if it wasn't resolved by 'resolve_vm_dns_names' already (e.g. VMs
+ picked up via 'objects_to_reevaluate').
+
+ Parameters
+ ----------
+ vm_name: str
+ VM name to look up
+
+ Returns
+ -------
+ list: of resolved IPv4 addresses as strings, empty list if none were found
+ """
+
+ if vm_name not in self.vm_dns_lookup_cache:
+ self.vm_dns_lookup_cache.update(
+ perform_forward_lookups([vm_name], self.settings.custom_dns_servers))
+
+ return self.vm_dns_lookup_cache.get(vm_name) or list()
+
@staticmethod
def passes_filter(name, include_filter, exclude_filter):
"""
@@ -1235,6 +1344,50 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v
f"'{device_vm_object.get_display_name()}'")
device_vm_object.update(data={f"primary_ip{ip_version}": ip_object})
+ # some interfaces are not touched this run (e.g. a VM whose guest tools are running but
+ # not reporting any guest.net data, see 'skip_ip_handling' in source_base.py) so their
+ # IP address objects never end up in 'ip_address_objects' above and the assignment loop
+ # never sees them. Fall back to an existing NetBox IP address already assigned to one of
+ # this object's OWN interfaces that matches the requested primary IP, applying the same
+ # 'set_primary_ip' semantics as the loop above.
+ for ip_version, wanted_ip_interface in ((4, primary_ipv4_object), (6, primary_ipv6_object)):
+
+ if wanted_ip_interface is None:
+ continue
+
+ current_primary_ip_object = grab(device_vm_object, f"data.primary_ip{ip_version}")
+
+ existing_ip_object = None
+ for ip_object in self.inventory.get_all_items(NBIPAddress):
+
+ if ip_object.get_device_vm() is not device_vm_object:
+ continue
+
+ # noinspection PyBroadException
+ try:
+ if ip_interface(grab(ip_object, "data.address")) != wanted_ip_interface:
+ continue
+ except Exception:
+ continue
+
+ existing_ip_object = ip_object
+ break
+
+ # nothing found, or already set to the desired IP
+ if existing_ip_object is None or existing_ip_object is current_primary_ip_object:
+ continue
+
+ set_this_primary_ip = False
+ if self.settings.set_primary_ip == "always":
+ set_this_primary_ip = True
+ elif self.settings.set_primary_ip != "never" and current_primary_ip_object is None:
+ set_this_primary_ip = True
+
+ if set_this_primary_ip is True:
+ log.debug(f"Setting IP '{grab(existing_ip_object, 'data.address')}' as primary IPv{ip_version} "
+ f"for '{device_vm_object.get_display_name()}' (existing assignment untouched this run)")
+ device_vm_object.update(data={f"primary_ip{ip_version}": existing_ip_object})
+
return
def get_parent_object_by_class(self, obj, object_class_to_find):
@@ -2059,6 +2212,54 @@ def add_host(self, obj):
return
+ # NIC slot 1 is always the management interface on these platforms' standard OVA/vmx
+ # deployment layout; subsequent slots are sequential data-plane interfaces. Confirmed by
+ # direct MAC-address cross-check between vCenter's reported vNIC order and each platform's
+ # own interface API/CLI (aXAPI 'interface/management'+'interface/ethernet', iControl REST
+ # 'mgmt'+'1.N', AlteonOS REST 'hwMACAddress'+'PortInfoTable' Indx) across multiple A10
+ # vThunder, F5 BIG-IP VE, and Radware Alteon VA instances.
+ _NATIVE_VNIC_NAMES_BY_PLATFORM = {
+ "acos": ("management", "ethernet{}"),
+ "tmos": ("mgmt", "1.{}"),
+ "alteon adc": ("mgmt", "{}"),
+ }
+
+ def _uses_native_vnic_names(self, platform):
+ return str(platform or "").strip().lower() in self._NATIVE_VNIC_NAMES_BY_PLATFORM
+
+ def get_vnic_name(self, platform, int_label):
+ """
+ Return the NetBox interface name for a vNIC: the platform's own native interface
+ name (e.g. 'mgmt', '1.1') for platforms in _NATIVE_VNIC_NAMES_BY_PLATFORM, otherwise
+ the default 'vNIC N' vSphere-generic name.
+
+ Naming interfaces to match what the guest OS itself calls them lets other tools that
+ manage these VMs by their own device APIs (e.g. netbox-device-onboard.py) write to the
+ SAME interface object instead of creating a second, differently-named one and fighting
+ over which interface owns the shared MAC address on every sync cycle.
+
+ Parameters
+ ----------
+ platform: str, None
+ VM platform name as already resolved for vm_data["platform"] (e.g. "ACOS", "TMOS")
+ int_label: str
+ vSphere deviceInfo.label for this NIC (e.g. "Network adapter 3")
+
+ Returns
+ -------
+ str: interface name to use
+ """
+
+ slot_str = int_label.split(" ")[-1]
+ native_names = self._NATIVE_VNIC_NAMES_BY_PLATFORM.get(str(platform or "").strip().lower())
+
+ if native_names is not None and slot_str.isdigit():
+ mgmt_name, data_plane_format = native_names
+ slot_num = int(slot_str)
+ return mgmt_name if slot_num == 1 else data_plane_format.format(slot_num - 1)
+
+ return "vNIC {}".format(slot_str)
+
def add_virtual_machine(self, obj):
"""
Parse a vCenter VM add to NetBox once all data is gathered.
@@ -2075,7 +2276,10 @@ def add_virtual_machine(self, obj):
Then all necessary VM data will be collected.
platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses
- Primary IPv4/6 will be determined by interface that provides the default route for this VM
+ Primary IPv6 will be determined by interface that provides the default route for this VM.
+
+ Primary IPv4 is determined by 'get_vm_primary_ip4' using a tiered fallback (DNS name,
+ then default route, then a configured fallback VLAN).
Note:
IP address information can only be extracted if guest tools are installed and running.
@@ -2216,9 +2420,17 @@ def add_virtual_machine(self, obj):
hardware_devices = grab(obj, "config.hardware.device", fallback=list())
- annotation = None
- if self.settings.skip_vm_comments is False:
- annotation = get_string_or_none(grab(obj, "config.annotation"))
+ # always read annotation — needed for platform detection even when skip_vm_comments is True
+ annotation = get_string_or_none(grab(obj, "config.annotation"))
+
+ # override platform based on annotation content; takes priority over vm_platform_relation
+ if annotation is not None:
+ for relation in grab(self.settings, "vm_platform_from_annotation_relation", fallback=list()):
+ if relation.get("object_regex").search(annotation):
+ platform = relation.get("assigned_name")
+ log.debug2(f"Overriding VM platform to '{platform}' based on annotation content "
+ f"(pattern: '{relation.get('object_regex').pattern}')")
+ break
# assign vm_tenant_relation
tenant_name = self.get_object_relation(name, "vm_tenant_relation")
@@ -2272,7 +2484,7 @@ def add_virtual_machine(self, obj):
if platform is not None:
vm_data["platform"] = {"name": platform}
- if annotation is not None:
+ if annotation is not None and self.settings.skip_vm_comments is False:
vm_data["comments"] = annotation
if tenant_name is not None:
vm_data["tenant"] = {"name": tenant_name}
@@ -2281,6 +2493,9 @@ def add_virtual_machine(self, obj):
# add custom fields if present and configured
vm_custom_fields = self.get_object_custom_fields(obj)
+ # Source-derived VCSA metadata is authoritative for the matching
+ # appliance VM and is merged after generic vCenter attributes.
+ vm_custom_fields.update(self._get_vcsa_custom_fields(name))
if len(vm_custom_fields) > 0:
vm_data["custom_fields"] = vm_custom_fields
@@ -2427,10 +2642,15 @@ def add_virtual_machine(self, obj):
int_connected = grab(vm_device, "connectable.connected", fallback=False)
int_label = grab(vm_device, "deviceInfo.label", fallback="")
- int_name = "vNIC {}".format(int_label.split(" ")[-1])
+ int_name = self.get_vnic_name(platform, int_label)
int_full_name = int_name
- if int_network_name is not None:
+ # Native platform names (e.g. "mgmt", "1.1", "ethernet1") must stay verbatim so
+ # other tools that manage these VMs via the guest's own API/CLI (e.g.
+ # netbox-device-onboard.py) look up the exact same interface name — appending the
+ # VLAN/portgroup name here would make the two tools create two different interfaces
+ # for the same physical NIC.
+ if int_network_name is not None and not self._uses_native_vnic_names(platform):
int_full_name = f"{int_full_name} ({int_network_name})"
int_description = f"{int_label} ({device_class})"
@@ -2474,61 +2694,101 @@ def add_virtual_machine(self, obj):
nic_ips[int_full_name].append(int_ip_address)
- # check if primary gateways are in the subnet of this IP address
- # if it matches IP gets chosen as primary IP
- if vm_default_gateway_ip4 is not None and \
- vm_default_gateway_ip4 in ip_interface(int_ip_address).network and \
- vm_primary_ip4 is None:
-
- vm_primary_ip4 = int_ip_address
+ # IPv4 primary IP selection happens after this loop, in 'get_vm_primary_ip4',
+ # once all interfaces and their VLANs are known (see tiered fallback there)
+ # check if the default IPv6 gateway is in the subnet of this address
+ # if it matches, IP gets chosen as primary IPv6
if vm_default_gateway_ip6 is not None and \
vm_default_gateway_ip6 in ip_interface(int_ip_address).network and \
vm_primary_ip6 is None:
vm_primary_ip6 = int_ip_address
+ # For acos/tmos-platform interfaces, netbox-device-onboard.py's device-API
+ # collectors (aXAPI/iControl) manage this same interface object and are
+ # authoritative for it - every attribute below (enabled, description, mtu,
+ # mode, tagged_vlans, untagged_vlan) was, in turn, confirmed live to fight
+ # with whatever the device-side collector had just set, each on its own next
+ # 5-minute cron cycle (see PRs #5, #6, #7 for the individual field-by-field
+ # history). Rather than keep chasing one field at a time, netbox-sync writes
+ # ONLY identity for these platforms - name, mac_address (needed to create the
+ # interface and seed MAC-object/IP matching) - and leaves every other
+ # attribute to the device-side collector entirely, including on first
+ # creation (NetBox's own defaults apply until the device-side onboarder runs).
vm_nic_data = {
"name": unquote(int_full_name),
"virtual_machine": None,
"mac_address": int_mac,
- "description": unquote(int_description),
- "enabled": int_connected,
}
- if int_mtu is not None and self.settings.sync_vm_interface_mtu is True:
- vm_nic_data["mtu"] = int_mtu
- if int_mode is not None:
- vm_nic_data["mode"] = int_mode
-
- if int_network_vlan_ids is not None and int_mode != "tagged-all":
-
- if len(int_network_vlan_ids) == 1 and int_network_vlan_ids[0] != 0:
-
- vm_nic_data["untagged_vlan"] = {
- "name": unquote(int_network_name),
- "vid": int_network_vlan_ids[0],
- "site": {
- "name": site_name
- }
- }
- else:
- tagged_vlan_list = list()
- for int_network_vlan_id in int_network_vlan_ids:
-
- if int_network_vlan_id == 0:
- continue
-
- tagged_vlan_list.append({
- "name": unquote(f"{int_network_name}-{int_network_vlan_id}"),
- "vid": int_network_vlan_id,
+ # Same identity-only reasoning as above, extended to IP-address ownership: for
+ # these platforms' DATA-plane interfaces (not mgmt - its IP has always come from
+ # guest.net without conflict), netbox-device-onboard.py's device-API collectors
+ # are authoritative for self-IP-to-interface assignment too. guest.net reports
+ # these addresses against the real vNIC's MAC regardless of whether the device-
+ # side collector binds the address to that same physical interface (acos/alteon)
+ # or to a separate synthetic VLAN-interface object (tmos) - confirmed live to
+ # fight over ownership of the address every 5-minute cron cycle either way
+ # (alt01a: netbox-sync tore the address back off a real vNIC guest.net never
+ # reports it on; ltm01a: netbox-sync kept reparenting it from the synthetic VLAN
+ # interface back onto the real vNIC). _skip_ip_reconciliation is a private,
+ # popped-before-write flag (same smuggling pattern as untagged_vlan/tagged_vlans
+ # above) telling add_update_interface() to leave this interface's IP assignment
+ # (both additions and removals) alone entirely.
+ native_names = self._NATIVE_VNIC_NAMES_BY_PLATFORM.get(str(platform or "").strip().lower())
+ if native_names is not None and int_full_name != native_names[0]:
+ vm_nic_data["_skip_ip_reconciliation"] = True
+
+ if not self._uses_native_vnic_names(platform):
+ vm_nic_data["enabled"] = int_connected
+ vm_nic_data["description"] = unquote(int_description)
+
+ if int_mtu is not None and self.settings.sync_vm_interface_mtu is True:
+ vm_nic_data["mtu"] = int_mtu
+
+ # ACOS in this environment runs in routed mode with zero device-side VLAN
+ # awareness (confirmed live: aXAPI's network/vlan table is empty on every
+ # ACOS target onboarded so far) - the vSphere portgroup is the only
+ # available source of truth for which VLAN an ethernet port sits on, unlike
+ # tmos/alteon which do have real device-side VLAN data that
+ # netbox-device-onboard.py collects and manages instead. So mode/
+ # untagged_vlan/tagged_vlans stay netbox-sync-managed for acos specifically,
+ # even though its other attributes (enabled/description/mtu, handled above)
+ # remain identity-only/device-authoritative like the other two platforms.
+ if not self._uses_native_vnic_names(platform) or str(platform or "").strip().lower() == "acos":
+
+ if int_mode is not None:
+ vm_nic_data["mode"] = int_mode
+
+ if int_network_vlan_ids is not None and int_mode != "tagged-all":
+
+ if len(int_network_vlan_ids) == 1 and int_network_vlan_ids[0] != 0:
+
+ vm_nic_data["untagged_vlan"] = {
+ "name": unquote(int_network_name),
+ "vid": int_network_vlan_ids[0],
"site": {
"name": site_name
}
- })
+ }
+ else:
+ tagged_vlan_list = list()
+ for int_network_vlan_id in int_network_vlan_ids:
- if len(tagged_vlan_list) > 0:
- vm_nic_data["tagged_vlans"] = tagged_vlan_list
+ if int_network_vlan_id == 0:
+ continue
+
+ tagged_vlan_list.append({
+ "name": unquote(f"{int_network_name}-{int_network_vlan_id}"),
+ "vid": int_network_vlan_id,
+ "site": {
+ "name": site_name
+ }
+ })
+
+ if len(tagged_vlan_list) > 0:
+ vm_nic_data["tagged_vlans"] = tagged_vlan_list
nic_data[int_full_name] = vm_nic_data
@@ -2573,6 +2833,10 @@ def add_virtual_machine(self, obj):
nic_data[int_full_name] = vm_nic_data
+ # determine primary IPv4 address using the tiered fallback (DNS name, default route,
+ # configured fallback VLAN); see 'get_vm_primary_ip4' for details
+ vm_primary_ip4 = self.get_vm_primary_ip4(name, nic_data, nic_ips, vm_default_gateway_ip4)
+
# if VM has only one IPv6 on all interfaces, use it as primary IPv6 address
if vm_primary_ip6 is None or True:
all_ips = [y for xs in nic_ips.values() for y in xs]
@@ -2593,6 +2857,14 @@ def add_virtual_machine(self, obj):
f"VM '{name}', using it as primary IPv6.")
vm_primary_ip6 = potential_primary_ipv6_list[0]
+ # For acos/tmos VMs, primary IP selection is netbox-device-onboard.py's job (it knows
+ # which interface is actually reachable/managed) - passing our own gateway-subnet guess
+ # here would fight it every cycle under set_primary_ip=always, the same conflict class
+ # as the interface-level mtu/description fields above.
+ if self._uses_native_vnic_names(platform):
+ vm_primary_ip4 = None
+ vm_primary_ip6 = None
+
# add VM to inventory
self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=nic_data,
nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6,
@@ -2600,6 +2872,169 @@ def add_virtual_machine(self, obj):
return
+ def get_existing_vm_ipv4_candidates(self, vm_name, overlapping_subnets):
+ """
+ Return eligible IPv4 addresses NetBox already has assigned to this VM's interfaces
+ from a previous sync. Used only as tier-1 (DNS) candidates when vCenter's live data
+ for this run yields none at all (e.g. guest tools reporting but not populating
+ guest.net) -- see 'get_vm_primary_ip4'.
+
+ Parameters
+ ----------
+ vm_name: str
+ name of the VM as it will be synced to NetBox
+ overlapping_subnets: list
+ parsed 'vm_ip_permitted_overlapping_subnets' networks to exclude
+
+ Returns
+ -------
+ list: of (interface name, "ip/prefixlen", ip_interface) tuples
+ """
+
+ existing_vm_object = None
+ for vm_object in self.inventory.get_all_items(NBVM):
+ if grab(vm_object, "data.name") == vm_name:
+ existing_vm_object = vm_object
+ break
+
+ if existing_vm_object is None:
+ return list()
+
+ candidates = list()
+ for ip_object in self.inventory.get_all_items(NBIPAddress):
+
+ if ip_object.get_device_vm() is not existing_vm_object:
+ continue
+
+ # noinspection PyBroadException
+ try:
+ ip_interface_object = ip_interface(grab(ip_object, "data.address"))
+ except Exception:
+ continue
+
+ if ip_interface_object.version != 4:
+ continue
+
+ if any(ip_interface_object.ip in subnet for subnet in overlapping_subnets):
+ continue
+
+ interface_object = ip_object.get_interface()
+ int_name = grab(interface_object, "data.name", fallback="unknown interface")
+
+ candidates.append((int_name, str(ip_interface_object), ip_interface_object))
+
+ return candidates
+
+ def get_vm_primary_ip4(self, vm_name, nic_data, nic_ips, default_gateway_ip4):
+ """
+ Determine the primary IPv4 address for a VM using a tiered fallback. The first tier
+ that produces a match wins:
+
+ 1. an interface IP exactly matches an A record for 'vm_name'
+ (only tried if 'vm_primary_ip4_by_dns_name' is enabled). If vCenter reported no
+ interface IPs at all this run (e.g. guest tools running but not populating
+ guest.net), IPs NetBox already has assigned to this VM's interfaces from a
+ previous sync are used as candidates instead -- DNS resolving to one of them
+ each run is the independent, current confirmation that it's still valid. This
+ fallback is intentionally DNS-only: tiers 2/3 below have no such confirmation
+ for stale data and are not extended this way.
+ 2. an interface IP is in the same network as the VM's default gateway
+ (the pre-existing behavior, derived from guest.ipStack)
+ 3. an interface IP sits on a VLAN listed in 'vm_primary_ip4_fallback_vlans',
+ tried in the configured order (only tried if the option is set)
+
+ An IP that falls within 'vm_ip_permitted_overlapping_subnets' (deliberately shared
+ HA/heartbeat addresses, e.g. VRRP peer links) is never eligible for any tier, since such
+ an address is intentionally assigned to more than one VM and would otherwise get
+ reassigned as primary IP back and forth between those VMs on every sync run.
+
+ Parameters
+ ----------
+ vm_name: str
+ name of the VM as it will be synced to NetBox
+ nic_data: dict
+ interface data keyed by full interface name, as collected in 'add_virtual_machine'
+ nic_ips: dict
+ list of "ip/prefixlen" strings keyed by full interface name
+ default_gateway_ip4: IPv4Address
+ default IPv4 gateway reported by the VM, or None if none was found
+
+ Returns
+ -------
+ str: primary IPv4 address including prefix length, None if no candidate was found
+ """
+
+ overlapping_subnets = grab(self.settings, "vm_ip_permitted_overlapping_subnets", fallback=list())
+
+ # ordered list of (interface name, "ip/prefixlen", ip_interface) for eligible IPv4 addresses
+ ipv4_candidates = list()
+ for int_name, int_ip_list in nic_ips.items():
+ for int_ip in int_ip_list:
+ # noinspection PyBroadException
+ try:
+ ip_interface_object = ip_interface(int_ip)
+ except Exception:
+ continue
+
+ if ip_interface_object.version != 4:
+ continue
+
+ if any(ip_interface_object.ip in subnet for subnet in overlapping_subnets):
+ log.debug2(f"IP '{int_ip}' on interface '{int_name}' of VM '{vm_name}' is part of a "
+ "permitted overlapping subnet, excluding it from primary IPv4 selection")
+ continue
+
+ ipv4_candidates.append((int_name, int_ip, ip_interface_object))
+
+ # tier 1: VM name resolves to one of the discovered IPs, or to an existing NetBox
+ # assignment if vCenter reported no interface IPs at all this run
+ if self.settings.vm_primary_ip4_by_dns_name is True:
+
+ dns_candidates = ipv4_candidates
+ using_existing_assignments = False
+
+ if len(ipv4_candidates) == 0:
+ dns_candidates = self.get_existing_vm_ipv4_candidates(vm_name, overlapping_subnets)
+ using_existing_assignments = True
+
+ resolved_ips = self.get_vm_dns_ips(vm_name)
+ for int_name, int_ip, ip_interface_object in dns_candidates:
+ if str(ip_interface_object.ip) in resolved_ips:
+ match_source = "matches DNS A record, no live IPs reported by vCenter this run" \
+ if using_existing_assignments else "matches DNS A record"
+ log.debug(f"Using '{int_ip}' on interface '{int_name}' as primary IPv4 for VM "
+ f"'{vm_name}' ({match_source})")
+ return int_ip
+
+ if len(resolved_ips) > 0 and len(dns_candidates) > 0:
+ log.debug2(f"DNS A record(s) for '{vm_name}' ({', '.join(resolved_ips)}) don't match any "
+ "IPv4 address discovered on this VM")
+
+ if len(ipv4_candidates) == 0:
+ log.debug2(f"VM '{vm_name}' has no eligible IPv4 addresses, unable to determine "
+ "a primary IPv4 address")
+ return None
+
+ # tier 2: default gateway is in the same network as this IP
+ if default_gateway_ip4 is not None:
+ for int_name, int_ip, ip_interface_object in ipv4_candidates:
+ if default_gateway_ip4 in ip_interface_object.network:
+ log.debug(f"Using '{int_ip}' on interface '{int_name}' as primary IPv4 for VM "
+ f"'{vm_name}' (default gateway {default_gateway_ip4})")
+ return int_ip
+
+ # tier 3: static fallback to a configured VLAN, tried in the configured order
+ for fallback_vlan in self.settings.vm_primary_ip4_fallback_vlans or list():
+ for int_name, int_ip, ip_interface_object in ipv4_candidates:
+ if grab(nic_data, f"{int_name}|untagged_vlan|vid", separator="|") == fallback_vlan:
+ log.debug(f"Using '{int_ip}' on interface '{int_name}' as primary IPv4 for VM "
+ f"'{vm_name}' (fallback VLAN {fallback_vlan})")
+ return int_ip
+
+ log.debug2(f"Unable to determine a primary IPv4 address for VM '{vm_name}'")
+
+ return None
+
def update_basic_data(self):
"""
diff --git a/netbox-sync.py b/netbox-sync.py
index c85de915..1168f1bf 100755
--- a/netbox-sync.py
+++ b/netbox-sync.py
@@ -12,6 +12,11 @@
Sync objects from various sources to NetBox
"""
+import warnings
+# vmware-vapi-runtime 2.52.0 imports pkg_resources at runtime, which emits a
+# DeprecationWarning on setuptools >= 81. Suppress it until the vapi stack is
+# upgraded to 9.x where the import is replaced with importlib.resources.
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated", category=UserWarning)
from datetime import datetime
diff --git a/settings-example.ini b/settings-example.ini
index ec2d5f1f..2d682805 100644
--- a/settings-example.ini
+++ b/settings-example.ini
@@ -146,6 +146,14 @@ password = super-secret
; blocks a leading '!' has to be added
;permitted_subnets = 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16, fd00::/8, !10.23.42.0/24
+; Subnets where the same IP address may legitimately appear on multiple VM interfaces
+; simultaneously. A common use case is isolated HA peer-to-peer links where the same /30
+; addressing is reused across many VM pairs (the IPs are unique within each link VLAN but
+; overlap globally). Supply a comma-separated list of prefixes in CIDR notation. When an
+; IP falls within one of these subnets, netbox-sync creates a separate NetBox IP address
+; object per interface rather than sharing a single object and emitting a duplicate warning.
+;vm_ip_permitted_overlapping_subnets = 10.99.99.0/24
+
; filter options
; filters can be used to include/exclude certain objects from importing into NetBox.
@@ -217,6 +225,21 @@ password = super-secret
;host_platform_relation = VMware ESXi 7.0.3 = VMware ESXi 7.0 Update 3o
;vm_platform_relation = centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016
+; Override the platform of a VM based on the content of its vCenter annotation (the Notes
+; field, synced to the NetBox comments field). Useful when vSphere misidentifies the guest
+; OS for appliance-style VMs — for example, network or security virtual appliances that run
+; on a modified Linux base are reported by vSphere as the base distro (CentOS, etc.) but
+; carry product-identifying text in their annotation.
+; Patterns are matched anywhere in the annotation text (re.search) and span newlines
+; automatically (re.DOTALL), so multi-line annotations work without special flags.
+; Takes priority over vm_platform_relation when both would match.
+; This is done with a comma separated key = value list.
+; key: defines a regex matched against the full VM annotation content
+; value: defines the desired NetBox platform name
+;vm_platform_from_annotation_relation =
+; BIG-IP Local Traffic Manager Virtual Edition.*F5 = TMOS,
+; BIG-IQ Virtual Edition.*F5 = BIG-IQ
+
; Define the NetBox device role used for hosts. The default is
; set to "Server". This is done with a comma separated key = value list.
; key: defines host(s) name as regex
@@ -271,6 +294,21 @@ password = super-secret
; as "when-undefined"
;set_primary_ip = when-undefined
+; Resolve the VM's name (as it will be synced to NetBox) via DNS and, if
+; the A record matches one of the IP addresses discovered on the VM's
+; interfaces, prefer it as the primary IPv4 address. This is tried before
+; the built-in default-gateway based detection and helps with appliances
+; that route their default traffic out a data-plane interface instead of
+; the management interface.
+;vm_primary_ip4_by_dns_name = False
+
+; Comma separated, ordered list of VLAN IDs used as a last resort to
+; determine a VM's primary IPv4 address if neither its DNS name (see
+; 'vm_primary_ip4_by_dns_name' above) nor its default gateway could be
+; used. The first VLAN in the list that has an IPv4 address on one of the
+; VM's interfaces wins. Usually points at a management VLAN.
+;vm_primary_ip4_fallback_vlans = 1370
+
; Do not sync notes from a VM in vCenter to the comments field on a VM in netbox
;skip_vm_comments = False