gpu memory check

nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py

@@ -441,6 +441,73 @@ def check_amdsmi_metric_ecc_totals(self, amdsmi_metric_data: list[AmdSmiMetric])
                         console_log=True,
                     )
 
+    def check_gpu_memory(
+        self,
+        amdsmi_metric_data: list[AmdSmiMetric],
+        minimum_available_percent: float,
+    ) -> None:
+        """Check the minimum free VRAM percentage for each GPU."""
+        for metric in amdsmi_metric_data:
+            memory = metric.mem_usage
+            total_vram = memory.total_vram if memory is not None else None
+            free_vram = memory.free_vram if memory is not None else None
+            values = {
+                "gpu": metric.gpu,
+                "total_vram": total_vram.value if total_vram is not None else None,
+                "free_vram": free_vram.value if free_vram is not None else None,
+                "unit": total_vram.unit if total_vram is not None else None,
+                "minimum_available_percent": minimum_available_percent,
+            }
+
+            if total_vram is None or free_vram is None:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} VRAM availability is not available",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            try:
+                total_value = float(total_vram.value)
+                free_value = float(free_vram.value)
+            except (TypeError, ValueError):
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} VRAM availability is invalid",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            if total_value <= 0:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} total VRAM is invalid",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            available_percent = free_value / total_value * 100
+            if available_percent < minimum_available_percent:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=(
+                        f"GPU {metric.gpu} free VRAM is {available_percent:.2f}% "
+                        f"(minimum {minimum_available_percent:.2f}%)"
+                    ),
+                    priority=EventPriority.WARNING,
+                    data={
+                        **values,
+                        "available_percent": available_percent,
+                    },
+                    console_log=True,
+                )
+
     def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]):
         """Check ECC counts in all blocks for all GPUs
 
@@ -959,6 +1026,11 @@ def analyze_data(
                     args.l0_to_recovery_count_error_threshold,
                     args.l0_to_recovery_count_warning_threshold or 1,
                 )
+            if args.gpu_memory:
+                self.check_gpu_memory(
+                    data.metric,
+                    args.gpu_memory.minimum_available_percent,
+                )
             self.check_amdsmi_metric_ecc_totals(data.metric)
             self.check_amdsmi_metric_ecc(data.metric)
 

nodescraper/plugins/inband/amdsmi/analyzer_args.py

@@ -26,12 +26,22 @@
 from datetime import datetime
 from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from nodescraper.models import AnalyzerArgs
 from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel
 
 
+class GpuMemoryConfig(BaseModel):
+    """GPU VRAM availability threshold."""
+
+    minimum_available_percent: float = Field(
+        ge=0,
+        le=100,
+        description="Minimum free VRAM percentage required for each GPU.",
+    )
+
+
 class AmdSmiAnalyzerArgs(AnalyzerArgs):
     check_static_data: bool = Field(
         default=False,
@@ -63,6 +73,10 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs):
         default=None,
         description="Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).",
     )
+    gpu_memory: Optional[GpuMemoryConfig] = Field(
+        default=None,
+        description="Minimum free VRAM threshold to validate for each GPU.",
+    )
     l0_to_recovery_count_error_threshold: Optional[int] = Field(
         default=3,
         description="L0-to-recovery count above which an error is raised.",

test/unit/plugin/test_amdsmi_analyzer.py

@@ -882,10 +882,24 @@ def _minimal_amdsmi_metric(
     ecc: Optional[MetricEccTotals] = None,
     ecc_blocks: Optional[dict] = None,
     power_management: Optional[str] = None,
+    mem_usage: Optional[dict] = None,
 ) -> AmdSmiMetric:
     """Build minimal AmdSmiMetric for PCIe/ECC tests with all required fields present."""
     pcie_dict = pcie.model_dump() if pcie is not None else {k: None for k in _PCIE_KEYS}
     ecc_dict = ecc.model_dump() if ecc is not None else {k: None for k in _ECC_TOTALS_KEYS}
+    mem_usage_dict = {
+        "total_vram": None,
+        "used_vram": None,
+        "free_vram": None,
+        "total_visible_vram": None,
+        "used_visible_vram": None,
+        "free_visible_vram": None,
+        "total_gtt": None,
+        "used_gtt": None,
+        "free_gtt": None,
+    }
+    if mem_usage is not None:
+        mem_usage_dict.update(mem_usage)
     return AmdSmiMetric.model_validate(
         {
             "gpu": gpu,
@@ -917,17 +931,7 @@ def _minimal_amdsmi_metric(
             "perf_level": None,
             "xgmi_err": None,
             "energy": None,
-            "mem_usage": {
-                "total_vram": None,
-                "used_vram": None,
-                "free_vram": None,
-                "total_visible_vram": None,
-                "used_visible_vram": None,
-                "free_visible_vram": None,
-                "total_gtt": None,
-                "used_gtt": None,
-                "free_gtt": None,
-            },
+            "mem_usage": mem_usage_dict,
             "throttle": {},
         }
     )
@@ -1002,10 +1006,55 @@ def test_analyze_data_expected_power_management(mock_analyzer):
     assert not any("power_management mismatch" in e.description for e in result.events)
 
 
+def test_check_gpu_memory_meets_minimum(mock_analyzer):
+    """GPU VRAM availability at the configured minimum passes."""
+    analyzer = mock_analyzer
+    metrics = [
+        _minimal_amdsmi_metric(
+            0,
+            mem_usage={
+                "total_vram": {"value": 100, "unit": "B"},
+                "free_vram": {"value": 95, "unit": "B"},
+            },
+        )
+    ]
+
+    analyzer.check_gpu_memory(metrics, 95)
+
+    assert not analyzer.result.events
+
+
+def test_check_gpu_memory_below_minimum_logs_warning(mock_analyzer):
+    """GPU VRAM availability below the configured minimum logs a warning."""
+    analyzer = mock_analyzer
+    metrics = [
+        _minimal_amdsmi_metric(
+            1,
+            mem_usage={
+                "total_vram": {"value": 100, "unit": "B"},
+                "free_vram": {"value": 90, "unit": "B"},
+            },
+        )
+    ]
+
+    analyzer.check_gpu_memory(metrics, 95)
+
+    assert len(analyzer.result.events) == 1
+    event = analyzer.result.events[0]
+    assert event.priority == EventPriority.WARNING
+    assert "GPU 1 free VRAM is 90.00%" in event.description
+    assert event.data["available_percent"] == 90.0
+    assert event.data["minimum_available_percent"] == 95
+
+
 def test_amdsmi_analyzer_args_rejects_unknown_fields():
     """Plugin config must only use declared AmdSmiAnalyzerArgs fields."""
     from pydantic import ValidationError
 
+    args = AmdSmiAnalyzerArgs.model_validate({"gpu_memory": {"minimum_available_percent": 95}})
+    assert args.gpu_memory is not None
+    assert args.gpu_memory.minimum_available_percent == 95
+
     with pytest.raises(ValidationError):
         AmdSmiAnalyzerArgs.model_validate(
             {"expected_power_management": "DISABLED", "not_a_field": 1}