test(breakfix): verify tenant notification delivery (BFX05/BFX06)

docs/test-plan.yaml

@@ -2011,6 +2011,7 @@ domains:
                 labels:
                   - bare_metal
                   - breakfix
+                  - kubernetes
                 dependencies:
                   - LimitedEnv
                 req_id: BFX05
@@ -2024,6 +2025,7 @@ domains:
                 labels:
                   - bare_metal
                   - breakfix
+                  - kubernetes
                 dependencies:
                   - LimitedEnv
                 req_id: BFX06

isvctl/configs/providers/aws/config/bare_metal.yaml

@@ -265,6 +265,42 @@ commands:
           - "{{ '--skip' if steps.deploy_nim.skipped | default(false) else '' }}"
         timeout: 300
 
+      # BFX05/BFX06: publish through an ephemeral SNS topic, prove delivery
+      # through its SQS subscriber, and remove both resources.
+      - name: query_planned_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python3 ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "aws"
+          - "--event-type"
+          - "planned_maintenance"
+          - "--machine-id"
+          - "{{steps.launch_instance.instance_id}}"
+          - "--message"
+          - "Planned node maintenance notification validation"
+          - "--region"
+          - "{{region}}"
+        timeout: 120
+
+      - name: query_failure_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python3 ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "aws"
+          - "--event-type"
+          - "node_failure"
+          - "--machine-id"
+          - "{{steps.launch_instance.instance_id}}"
+          - "--message"
+          - "Immediate node failure notification validation"
+          - "--region"
+          - "{{region}}"
+        timeout: 120
+
       # Teardown bare-metal instance (boto3)
       # Set AWS_BM_SKIP_TEARDOWN=true to keep the instance alive
       - name: teardown

isvctl/configs/providers/aws/config/eks.yaml

@@ -206,6 +206,42 @@ commands:
           TF_AUTO_APPROVE: "true"
           NODE_POOL_STATE_FILE: "terraform-delete.tfstate"
 
+      # BFX05/BFX06: prove acknowledged tenant notification delivery through
+      # an ephemeral in-cluster HTTP receiver, then remove all probe resources.
+      - name: query_planned_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python3 ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "planned_maintenance"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Planned node maintenance notification validation"
+        requires_available_validations:
+          - K8sPlannedMaintenanceNotificationCheck
+        timeout: 300
+
+      - name: query_failure_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python3 ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "node_failure"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Immediate node failure notification validation"
+        requires_available_validations:
+          - K8sFailureNotificationCheck
+        timeout: 300
+
       - name: teardown
         phase: teardown
         command: "../scripts/eks/teardown.sh"

isvctl/configs/providers/minikube.yaml

@@ -34,6 +34,38 @@ commands:
         phase: setup
         command: "my-isv/scripts/k8s/setup_minikube.sh"
         timeout: 60
+      - name: query_planned_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "planned_maintenance"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Planned node maintenance notification validation"
+        requires_available_validations:
+          - K8sPlannedMaintenanceNotificationCheck
+        timeout: 300
+      - name: query_failure_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "node_failure"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Immediate node failure notification validation"
+        requires_available_validations:
+          - K8sFailureNotificationCheck
+        timeout: 300
       - name: teardown
         phase: teardown
         command: "my-isv/scripts/k8s/teardown_minikube.sh"

isvctl/configs/providers/my-isv/config/k8s.yaml

@@ -73,6 +73,40 @@ commands:
         args: ["--region", "{{region}}"]
         timeout: 600
 
+      - name: query_planned_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "planned_maintenance"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Planned node maintenance notification validation"
+        requires_available_validations:
+          - K8sPlannedMaintenanceNotificationCheck
+        timeout: 300
+
+      - name: query_failure_notifications
+        phase: test
+        continue_on_failure: true
+        command: "python ../../shared/breakfix/query_tenant_notification.py"
+        args:
+          - "--backend"
+          - "kubernetes"
+          - "--event-type"
+          - "node_failure"
+          - "--machine-id"
+          - "notification-probe-node"
+          - "--message"
+          - "Immediate node failure notification validation"
+        requires_available_validations:
+          - K8sFailureNotificationCheck
+        timeout: 300
+
 tests:
   description: "my-isv Kubernetes platform suite (real cluster required)"
   settings:

isvctl/configs/providers/my-isv/scripts/breakfix/query_failure_notifications.py

@@ -31,6 +31,10 @@ def main() -> int:
                 "type": "node_failure",
                 "message": "Node became unreachable; GPU fault detected",
                 "notified_at": "2026-06-24T12:00:00Z",
+                "failed_at": "2026-06-24T11:59:30Z",
+                "channel": "webhook",
+                "delivery_status": "delivered",
+                "delivery_id": "demo-failure-delivery-001",
             }
         ],
     )

isvctl/configs/providers/my-isv/scripts/breakfix/query_planned_notifications.py

@@ -31,6 +31,10 @@ def main() -> int:
                 "type": "planned_maintenance",
                 "message": "Scheduled firmware update (demo)",
                 "notified_at": "2026-06-24T12:00:00Z",
+                "scheduled_at": "2026-06-25T12:00:00Z",
+                "channel": "webhook",
+                "delivery_status": "delivered",
+                "delivery_id": "demo-planned-delivery-001",
             }
         ],
     )