alerting/dashbaording: added minimal monitoring setup for the plugin

.github/workflows/validate-prometheus-alerts.yaml

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and cobaltcore-dev contributors
+# SPDX-License-Identifier: Apache-2.0
+
+name: Validate Prometheus Alerts
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'charts/openstack-hypervisor-operator/alerts/*.yaml'
+      - 'charts/openstack-hypervisor-operator/alerts/*.yml'
+
+permissions:
+  contents: read
+
+env:
+  PROMTOOL_VERSION: 3.8.0
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: validate-prometheus-alerts-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate-alerts:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install promtool
+        run: |
+          set -euo pipefail
+          curl -sSfL "https://github.com/prometheus/prometheus/releases/download/v${PROMTOOL_VERSION}/prometheus-${PROMTOOL_VERSION}.linux-amd64.tar.gz" \
+            | tar xz --strip-components=1 "prometheus-${PROMTOOL_VERSION}.linux-amd64/promtool"
+          sudo install -m 0755 promtool /usr/local/bin/promtool
+          promtool --version
+
+      - name: Validate Prometheus alert rules
+        run: |
+          set -euo pipefail
+          shopt -s nullglob
+          files=(charts/openstack-hypervisor-operator/alerts/*.{yaml,yml})
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "No Prometheus rule files found."
+            exit 1
+          fi
+          promtool check rules "${files[@]}"

charts/openstack-hypervisor-operator/alerts/eviction.yaml

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors
+# SPDX-License-Identifier: Apache-2.0
+
+groups:
+- name: evictionLifecycle
+  rules:
+  - alert: EvictionFailed
+    expr: |
+      kube_customresource_eviction_condition{condition="Evicting", reason="Failed"} == 1
+    for: 5m
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Eviction {{ $labels.name }} has failed"
+      description: "The eviction {{ $labels.name }} for hypervisor {{ $labels.hypervisor }} has reached a terminal failure state. Manual intervention is required — check if the hypervisor exists in OpenStack."
+
+  - alert: EvictionMigrationFailing
+    expr: |
+      kube_customresource_eviction_condition{condition="MigratingInstance", reason="Failed"} == 1
+        and on (name)
+      kube_customresource_eviction_condition{condition="Evicting", reason="Running"} == 1
+    for: 1h
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Eviction {{ $labels.name }} has failing instance migrations for over 1 hour"
+      description: "The eviction {{ $labels.name }} has had MigratingInstance=Failed for more than 1 hour while still running. Instances may be in ERROR state, blocking eviction progress."
+
+  - alert: EvictionOutstandingRamHigh
+    expr: |
+      kube_customresource_eviction_outstanding_ram_mb > 0
+        and on (name)
+      kube_customresource_eviction_condition{condition="Evicting", reason="Running"} == 1
+    for: 6h
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Eviction {{ $labels.name }} has outstanding RAM for over 6 hours"
+      description: "The eviction {{ $labels.name }} has had {{ $value }}MB of outstanding RAM for more than 6 hours. Check for stuck live-migrations or instances that cannot be moved."

charts/openstack-hypervisor-operator/alerts/operator.yaml

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors
+# SPDX-License-Identifier: Apache-2.0
+
+groups:
+- name: hypervisorLifecycle
+  rules:
+  - alert: HypervisorOnboardingStuck
+    expr: |
+      kube_customresource_hypervisor_condition{condition="Onboarding"} == 1
+    for: 1h
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor {{ $labels.name }} onboarding stuck for over 1 hour"
+      description: "The hypervisor {{ $labels.name }} in zone {{ $labels.zone }} has been onboarding for more than 1 hour. Check nova registration, test VM status, or trait/aggregate sync."
+
+  - alert: HypervisorEvictionStuck
+    expr: |
+      kube_customresource_hypervisor_condition{condition="Evicting"} == 1
+    for: 4h
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor {{ $labels.name }} eviction running for over 4 hours"
+      description: "The hypervisor {{ $labels.name }} in zone {{ $labels.zone }} has had an active eviction for more than 4 hours. Check for stuck live-migrations or failed VMs."
+
+  - alert: HypervisorEvictedTooLong
+    expr: |
+      kube_customresource_hypervisor_evicted == 1
+        unless on (name)
+      kube_customresource_hypervisor_condition{condition="Offboarded"} == 1
+    for: 7d
+    labels:
+      severity: info
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor {{ $labels.name }} has been evicted for over 7 days"
+      description: "The hypervisor {{ $labels.name }} in zone {{ $labels.zone }} has been evicted for more than 7 days without being offboarded. Consider re-enabling or decommissioning."
+
+- name: hypervisorSync
+  rules:
+  - alert: HypervisorTraitSyncFailed
+    expr: |
+      kube_customresource_hypervisor_condition{condition="TraitsUpdated"} == 0
+        and on (name)
+      kube_customresource_hypervisor_condition{condition="Onboarding"} == 0
+    for: 30m
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor {{ $labels.name }} trait sync has been failing"
+      description: "The hypervisor {{ $labels.name }} in zone {{ $labels.zone }} has had TraitsUpdated=False for more than 30 minutes outside of onboarding. Check OpenStack Placement API connectivity."
+
+  - alert: HypervisorAggregateSyncFailed
+    expr: |
+      kube_customresource_hypervisor_condition{condition="AggregatesUpdated"} == 0
+        and on (name)
+      kube_customresource_hypervisor_condition{condition="Onboarding"} == 0
+        unless on (name)
+      kube_customresource_hypervisor_condition{condition="Evicting"} == 1
+    for: 30m
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor {{ $labels.name }} aggregate sync has been failing"
+      description: "The hypervisor {{ $labels.name }} in zone {{ $labels.zone }} has had AggregatesUpdated=False for more than 30 minutes outside of onboarding and eviction. Check OpenStack Nova API connectivity."
+
+- name: hypervisorOperatorHealth
+  rules:
+  - alert: HypervisorOperatorReconcileErrors
+    expr: |
+      rate(controller_runtime_reconcile_errors_total[5m]) > 0.01
+    for: 15m
+    labels:
+      severity: warning
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor operator controller {{ $labels.controller }} has persistent reconcile errors"
+      description: "The controller {{ $labels.controller }} has been producing sustained reconciliation errors for more than 15 minutes."
+
+  - alert: HypervisorOperatorDown
+    expr: |
+      up{job=~".*hypervisor-operator.*"} == 0
+    for: 5m
+    labels:
+      severity: critical
+      type: hypervisor_operator
+    annotations:
+      summary: "Hypervisor operator is down"
+      description: "The hypervisor operator metrics endpoint has been unreachable for more than 5 minutes."