Bunch of monitoring services added. Need to work through all the configuration next.

configs/alertmanager/alertmanager.yml

@@ -0,0 +1,100 @@
+global:
+  resolve_timeout: 5m
+  # SMTP configuration for email alerts (fallback)
+  # Using MailHog for development - update for production
+  smtp_from: 'alerts@changemaker.local'
+  smtp_smarthost: 'mailhog:1025'
+  smtp_auth_username: ''
+  smtp_auth_password: ''
+  smtp_require_tls: false
+
+# Templates for notification content
+templates:
+  - '/etc/alertmanager/*.tmpl'
+
+# Route alerts to appropriate receivers based on severity
+route:
+  group_by: ['alertname', 'cluster', 'service']
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 12h
+  receiver: 'default'
+
+  routes:
+    # Critical alerts go to both Gotify and email
+    - match:
+        severity: critical
+      receiver: 'critical-alerts'
+      group_wait: 0s
+      group_interval: 5m
+      repeat_interval: 4h
+
+    # Warning alerts go to Gotify only
+    - match:
+        severity: warning
+      receiver: 'warning-alerts'
+      group_wait: 30s
+      repeat_interval: 12h
+
+    # Info alerts (rate limiting, etc.) - Gotify with lower priority
+    - match:
+        severity: info
+      receiver: 'info-alerts'
+      repeat_interval: 24h
+
+# Alert receivers
+receivers:
+  # Default receiver (catches all unmatched)
+  # Note: Configure GOTIFY_APP_TOKEN in .env and update this file for Gotify to work
+  - name: 'default'
+    email_configs:
+      - to: 'admin@changemaker.local'
+        headers:
+          Subject: '[Changemaker] {{ .GroupLabels.alertname }}'
+
+  # Critical alerts - email only (configure Gotify token for push notifications)
+  - name: 'critical-alerts'
+    email_configs:
+      - to: 'admin@changemaker.local'
+        headers:
+          Subject: '🚨 CRITICAL Alert: {{ .GroupLabels.alertname }}'
+        html: |
+          <h2 style="color: #d32f2f;">Critical Alert Triggered</h2>
+          {{ range .Alerts }}
+          <p><strong>Alert:</strong> {{ .Labels.alertname }}</p>
+          <p><strong>Severity:</strong> {{ .Labels.severity }}</p>
+          <p><strong>Summary:</strong> {{ .Annotations.summary }}</p>
+          <p><strong>Description:</strong> {{ .Annotations.description }}</p>
+          <p><strong>Started:</strong> {{ .StartsAt }}</p>
+          <hr>
+          {{ end }}
+
+  # Warning alerts - email only
+  - name: 'warning-alerts'
+    email_configs:
+      - to: 'admin@changemaker.local'
+        headers:
+          Subject: '⚠️ Warning: {{ .GroupLabels.alertname }}'
+
+  # Info alerts - email only
+  - name: 'info-alerts'
+    email_configs:
+      - to: 'admin@changemaker.local'
+        headers:
+          Subject: 'ℹ️ Info: {{ .GroupLabels.alertname }}'
+
+# Inhibition rules (prevent spam)
+inhibit_rules:
+  # If a critical alert is firing, suppress related warnings
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: ['alertname', 'instance']
+
+  # If disk is critical, suppress disk warning
+  - source_match:
+      alertname: 'DiskSpaceCritical'
+    target_match:
+      alertname: 'DiskSpaceLow'
+    equal: ['instance']