Bunch of monitoring services added. Need to work through all the configuration next.

configs/prometheus/alerts.yml

@@ -91,3 +91,145 @@ groups:
         annotations:
           summary: "High API latency"
           description: "95th percentile latency is {{ $value }}s for {{ $labels.route }}."
+
+  # System health alerts
+  - name: system_alerts
+    interval: 30s
+    rules:
+      # NocoDB unreachable
+      - alert: NocoDBUnreachable
+        expr: up{job="nocodb"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "NocoDB database is unreachable"
+          description: "NocoDB has been unreachable for more than 2 minutes. All database operations will fail."
+
+      # Redis down
+      - alert: RedisDown
+        expr: redis_up == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Redis cache is down"
+          description: "Redis has been down for more than 1 minute. Caching and session management will fail."
+
+      # Disk space running low
+      - alert: DiskSpaceLow
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.15
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Disk space is running low"
+          description: "Only {{ $value | humanizePercentage }} disk space remaining on root filesystem."
+
+      # Disk space critical
+      - alert: DiskSpaceCritical
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "CRITICAL: Disk space nearly exhausted"
+          description: "Only {{ $value | humanizePercentage }} disk space remaining! System may fail soon."
+
+      # High CPU usage
+      - alert: HighCPUUsage
+        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage detected"
+          description: "CPU usage is {{ $value }}% on {{ $labels.instance }}."
+
+      # Container CPU throttling
+      - alert: ContainerCPUThrottling
+        expr: rate(container_cpu_cfs_throttled_seconds_total[5m]) > 0.5
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container is being CPU throttled"
+          description: "Container {{ $labels.name }} is experiencing CPU throttling."
+
+      # Container memory usage high
+      - alert: ContainerMemoryHigh
+        expr: (container_memory_usage_bytes / container_spec_memory_limit_bytes) > 0.90
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container memory usage is high"
+          description: "Container {{ $labels.name }} is using {{ $value | humanizePercentage }} of its memory limit."
+
+  # Infrastructure alerts
+  - name: infrastructure_alerts
+    interval: 30s
+    rules:
+      # Prometheus scrape failures
+      - alert: PrometheusScrapeFailures
+        expr: rate(prometheus_target_scrapes_failed_total[5m]) > 0.1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus scrape failures detected"
+          description: "Prometheus is failing to scrape {{ $labels.job }} target."
+
+      # Prometheus configuration reload failure
+      - alert: PrometheusConfigReloadFailed
+        expr: prometheus_config_last_reload_successful == 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus configuration reload failed"
+          description: "Prometheus failed to reload its configuration. Check prometheus logs."
+
+      # Alertmanager down
+      - alert: AlertmanagerDown
+        expr: up{job="alertmanager"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Alertmanager is down"
+          description: "Alertmanager has been down for 2 minutes. Alerts will not be delivered!"
+
+  # Security alerts
+  - name: security_alerts
+    interval: 15s
+    rules:
+      # Possible DDoS attack
+      - alert: PossibleDDoSAttack
+        expr: rate(influence_http_requests_total[1m]) > 1000
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Possible DDoS attack detected"
+          description: "Receiving {{ $value }} requests per second for 2 minutes. This may be a DDoS attack."
+
+      # Sustained high traffic
+      - alert: SustainedHighTraffic
+        expr: rate(influence_http_requests_total[5m]) > 500
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Sustained high traffic detected"
+          description: "Receiving {{ $value }} requests per second for 10 minutes. Monitor for performance issues."
+
+      # Too many 4xx errors
+      - alert: HighClientErrorRate
+        expr: rate(influence_http_requests_total{status_code=~"4.."}[5m]) > 5
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High rate of 4xx client errors"
+          description: "Receiving {{ $value }} client errors per second. Check for broken links or API misuse."

configs/prometheus/prometheus.yml

@@ -4,11 +4,11 @@ global:
   external_labels:
     monitor: 'changemaker-lite'
 
-# Alertmanager configuration (optional)
+# Alertmanager configuration
 alerting:
   alertmanagers:
     - static_configs:
-        - targets: []
+        - targets: ['alertmanager:9093']
 
 # Load rules once and periodically evaluate them
 rule_files:
@@ -31,24 +31,31 @@ scrape_configs:
     metrics_path: '/metrics'
     scrape_interval: 30s
 
-  # Redis Metrics (requires redis_exporter - optional)
-  # Uncomment and add redis_exporter service to enable
-  # - job_name: 'redis'
-  #   static_configs:
-  #     - targets: ['redis-exporter:9121']
+  # Redis Metrics
+  - job_name: 'redis'
+    static_configs:
+      - targets: ['redis-exporter:9121']
+    scrape_interval: 15s
 
-  # Listmonk Metrics (if available)
-  # - job_name: 'listmonk'
-  #   static_configs:
-  #     - targets: ['listmonk-app:9000']
-  #   metrics_path: '/metrics'
+  # cAdvisor - Docker container metrics
+  - job_name: 'cadvisor'
+    static_configs:
+      - targets: ['cadvisor:8080']
+    scrape_interval: 15s
+
+  # Node Exporter - System metrics
+  - job_name: 'node'
+    static_configs:
+      - targets: ['node-exporter:9100']
+    scrape_interval: 15s
 
   # Prometheus self-monitoring
   - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
 
-  # Docker container metrics (requires cAdvisor - optional)
-  # - job_name: 'cadvisor'
-  #   static_configs:
-  #     - targets: ['cadvisor:8080']
+  # Alertmanager monitoring
+  - job_name: 'alertmanager'
+    static_configs:
+      - targets: ['alertmanager:9093']
+    scrape_interval: 30s