Add comprehensive ArgoCD application monitoring and alerts

This implements monitoring for ALL services deployed via ArgoCD by leveraging ArgoCD's native Prometheus metrics instead of scraping individual services.

Changes:
- Created ArgoCD application alerts for health and sync status monitoring
  - Alert when applications are unhealthy (Degraded, Missing, Unknown, Suspended)
  - Alert when applications are out of sync for >10 minutes
  - Alert when sync operations are failing repeatedly
  - Alert when applications are stuck in Progressing state
  - Added recording rules for unhealthy/out-of-sync application counts

- Added radicale health monitoring via scrape config
  - Added radicale to additional-scrape-configs for direct health checks
  - Monitors radicale web interface availability

Benefits:
- Single monitoring solution for all 21 ArgoCD-managed applications
- Automatic monitoring for new applications added to ArgoCD
- Early detection of configuration drift and deployment issues
- Centralized alerting with actionable remediation steps

Monitored applications include: radicale, registry, alloy, grafana, loki,
prometheus, tempo, anki-sync-server, audiobookshelf, filebrowser, immich,
keybr, kobo-sync-server, miniflux, opodsync, and more.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

3 files changed, 118 insertions, 0 deletions

diff --git a/f3s/prometheus/additional-scrape-configs.yaml b/f3s/prometheus/additional-scrape-configs.yaml
index 0ea8d69..9406c7f 100644
--- a/f3s/prometheus/additional-scrape-configs.yaml
+++ b/f3s/prometheus/additional-scrape-configs.yaml
@@ -17,3 +17,17 @@
   static_configs:
     - targets:
       - 'pushgateway.monitoring.svc.cluster.local:9091'
+
+# Radicale CalDAV/CardDAV server health monitoring
+# Radicale doesn't expose Prometheus metrics, so we scrape the web interface
+# and rely on the 'up' metric to determine if the service is available
+- job_name: 'radicale'
+  metrics_path: '/.web/'
+  static_configs:
+    - targets:
+      - 'radicale-service.services.svc.cluster.local:80'
+      labels:
+        service: radicale
+        namespace: services
+  scrape_interval: 30s
+  scrape_timeout: 10s
diff --git a/f3s/prometheus/manifests/additional-scrape-configs-secret.yaml b/f3s/prometheus/manifests/additional-scrape-configs-secret.yaml
index a2d9534..4f599e8 100644
--- a/f3s/prometheus/manifests/additional-scrape-configs-secret.yaml
+++ b/f3s/prometheus/manifests/additional-scrape-configs-secret.yaml
@@ -27,3 +27,17 @@ stringData:
       static_configs:
         - targets:
           - 'pushgateway.monitoring.svc.cluster.local:9091'
+
+    # Radicale CalDAV/CardDAV server health monitoring
+    # Radicale doesn't expose Prometheus metrics, so we scrape the web interface
+    # and rely on the 'up' metric to determine if the service is available
+    - job_name: 'radicale'
+      metrics_path: '/.web/'
+      static_configs:
+        - targets:
+          - 'radicale-service.services.svc.cluster.local:80'
+          labels:
+            service: radicale
+            namespace: services
+      scrape_interval: 30s
+      scrape_timeout: 10s
diff --git a/f3s/prometheus/manifests/argocd-application-alerts.yaml b/f3s/prometheus/manifests/argocd-application-alerts.yaml
new file mode 100644
index 0000000..d843347
--- /dev/null
+++ b/f3s/prometheus/manifests/argocd-application-alerts.yaml
@@ -0,0 +1,90 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: argocd-application-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # ArgoCD application health and sync monitoring
+    - name: argocd-applications
+      interval: 30s
+      rules:
+        # Alert when any ArgoCD application is unhealthy
+        - alert: ArgoCDApplicationUnhealthy
+          expr: argocd_app_info{health_status!="Healthy"} == 1
+          for: 5m
+          labels:
+            severity: critical
+            component: argocd
+          annotations:
+            summary: "ArgoCD application {{ $labels.name }} is unhealthy"
+            description: "ArgoCD application {{ $labels.name }} in namespace {{ $labels.dest_namespace }} has health status: {{ $labels.health_status }}. The application has been unhealthy for more than 5 minutes."
+            impact: "Service {{ $labels.name }} may be degraded or unavailable. Users may experience service interruptions."
+            action: "Check application status with: argocd app get {{ $labels.name }} --core\nCheck pods: kubectl get pods -n {{ $labels.dest_namespace }}\nView logs: kubectl logs -n {{ $labels.dest_namespace }} -l app={{ $labels.name }} --tail=100"
+            dashboard: "https://argocd.yourdomain.com/applications/{{ $labels.name }}"
+
+        # Alert when any ArgoCD application is out of sync
+        - alert: ArgoCDApplicationOutOfSync
+          expr: argocd_app_info{sync_status!="Synced"} == 1
+          for: 10m
+          labels:
+            severity: warning
+            component: argocd
+          annotations:
+            summary: "ArgoCD application {{ $labels.name }} is out of sync"
+            description: "ArgoCD application {{ $labels.name }} has sync status: {{ $labels.sync_status }}. The application has been out of sync for more than 10 minutes."
+            impact: "Configuration drift detected. Running configuration may not match Git repository."
+            action: "Check sync status with: argocd app get {{ $labels.name }} --core\nManual sync: argocd app sync {{ $labels.name }} --core\nCheck diff: argocd app diff {{ $labels.name }} --core"
+
+        # Alert when application sync fails repeatedly
+        - alert: ArgoCDApplicationSyncFailing
+          expr: increase(argocd_app_sync_total{phase="Error"}[30m]) > 3
+          for: 5m
+          labels:
+            severity: critical
+            component: argocd
+          annotations:
+            summary: "ArgoCD application {{ $labels.name }} sync is failing repeatedly"
+            description: "ArgoCD application {{ $labels.name }} has failed to sync {{ $value }} times in the last 30 minutes."
+            impact: "Application cannot be updated. New changes are not being deployed."
+            action: "Check sync errors: argocd app get {{ $labels.name }} --core\nView detailed logs: kubectl logs -n cicd -l app.kubernetes.io/name=argocd-application-controller --tail=200 | grep {{ $labels.name }}"
+
+        # Alert when application has been progressing for too long
+        - alert: ArgoCDApplicationStuckProgressing
+          expr: argocd_app_info{health_status="Progressing"} == 1
+          for: 30m
+          labels:
+            severity: warning
+            component: argocd
+          annotations:
+            summary: "ArgoCD application {{ $labels.name }} stuck in Progressing state"
+            description: "ArgoCD application {{ $labels.name }} has been in Progressing state for more than 30 minutes."
+            impact: "Application deployment is taking longer than expected. New version may not be fully deployed."
+            action: "Check application status: argocd app get {{ $labels.name }} --core\nCheck pods: kubectl get pods -n {{ $labels.dest_namespace }} -w\nCheck events: kubectl get events -n {{ $labels.dest_namespace }} --sort-by='.lastTimestamp'"
+
+        # Alert for critical application failures (Degraded, Suspended, Missing, Unknown)
+        - alert: ArgoCDApplicationDegraded
+          expr: argocd_app_info{health_status=~"Degraded|Suspended|Missing|Unknown"} == 1
+          for: 5m
+          labels:
+            severity: critical
+            component: argocd
+          annotations:
+            summary: "ArgoCD application {{ $labels.name }} is degraded"
+            description: "ArgoCD application {{ $labels.name }} has critical health status: {{ $labels.health_status }}."
+            impact: "Application is in a critical state and requires immediate attention."
+            action: "Check application status: argocd app get {{ $labels.name }} --core\nReview ArgoCD logs: kubectl logs -n cicd -l app.kubernetes.io/name=argocd-server --tail=100"
+
+        # Recording rule: count of unhealthy applications
+        - record: argocd:applications:unhealthy:count
+          expr: count(argocd_app_info{health_status!="Healthy"})
+
+        # Recording rule: count of out-of-sync applications
+        - record: argocd:applications:outof_sync:count
+          expr: count(argocd_app_info{sync_status!="Synced"})
+
+        # Recording rule: total applications
+        - record: argocd:applications:total:count
+          expr: count(argocd_app_info)