admin/changemaker.lite
1
0
Fork 0
Code Issues Packages Projects Releases 31 Wiki Activity
changemaker.lite/mkdocs/site/v2/deployment/monitoring-stack/index.html

6125 lines
151 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Build Power. Not Rent It. Own your digital infrastructure.">
<meta name="author" content="Bunker Operations">
<link rel="canonical" href="https://bnkserve.org/v2/deployment/monitoring-stack/">
<link rel="prev" href="../tunneling/">
<link rel="next" href="../healthchecks/">
<link rel="icon" href="../../../assets/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.1">
<title>Monitoring Stack - Changemaker Lite</title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Inter:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Inter";--md-code-font:"JetBrains Mono"}</style>
<link rel="stylesheet" href="../../../stylesheets/extra.css">
<link rel="stylesheet" href="../../../stylesheets/home.css">
<link rel="stylesheet" href="../../../assets/css/video-player.css">
<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
<meta property="og:type" content="website" />
<meta property="og:title" content="Monitoring Stack - Changemaker Lite" />
<meta property="og:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
<meta property="og:image" content="https://bnkserve.org/assets/images/social/v2/deployment/monitoring-stack.png" />
<meta property="og:image:type" content="image/png" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="630" />
<meta property="og:url" content="https://bnkserve.org/v2/deployment/monitoring-stack/" />
<meta property="twitter:card" content="summary_large_image" />
<meta property="twitter:title" content="Monitoring Stack - Changemaker Lite" />
<meta property="twitter:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
<meta property="twitter:image" content="https://bnkserve.org/assets/images/social/v2/deployment/monitoring-stack.png" />
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#monitoring-stack-prometheus-grafana" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../../.." title="Changemaker Lite" class="md-header__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
<img src="../../../assets/logo.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
Changemaker Lite
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Monitoring Stack
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m17.75 4.09-2.53 1.94.91 3.06-2.63-1.81-2.63 1.81.91-3.06-2.53-1.94L12.44 4l1.06-3 1.06 3zm3.5 6.91-1.64 1.25.59 1.98-1.7-1.17-1.7 1.17.59-1.98L15.75 11l2.06-.05L18.5 9l.69 1.95zm-2.28 4.95c.83-.08 1.72 1.1 1.19 1.85-.32.45-.66.87-1.08 1.27C15.17 23 8.84 23 4.94 19.07c-3.91-3.9-3.91-10.24 0-14.14.4-.4.82-.76 1.27-1.08.75-.53 1.93.36 1.85 1.19-.27 2.86.69 5.83 2.89 8.02a9.96 9.96 0 0 0 8.02 2.89m-1.64 2.02a12.08 12.08 0 0 1-7.8-3.47c-2.17-2.19-3.33-5-3.49-7.82-2.81 3.14-2.7 7.96.31 10.98 3.02 3.01 7.84 3.12 10.98.31"/></svg>
</label>
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 7a5 5 0 0 1 5 5 5 5 0 0 1-5 5 5 5 0 0 1-5-5 5 5 0 0 1 5-5m0 2a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0-7 2.39 3.42C13.65 5.15 12.84 5 12 5s-1.65.15-2.39.42zM3.34 7l4.16-.35A7.2 7.2 0 0 0 5.94 8.5c-.44.74-.69 1.5-.83 2.29zm.02 10 1.76-3.77a7.131 7.131 0 0 0 2.38 4.14zM20.65 7l-1.77 3.79a7.02 7.02 0 0 0-2.38-4.15zm-.01 10-4.14.36c.59-.51 1.12-1.14 1.54-1.86.42-.73.69-1.5.83-2.29zM12 22l-2.41-3.44c.74.27 1.55.44 2.41.44.82 0 1.63-.17 2.37-.44z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
changemaker.lite
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../../" class="md-tabs__link">
V2 Documentation
</a>
</li>
<li class="md-tabs__item">
<a href="../../../phil/" class="md-tabs__link">
Philosophy
</a>
</li>
<li class="md-tabs__item">
<a href="../../../v1/" class="md-tabs__link">
V1 Documentation (Legacy)
</a>
</li>
<li class="md-tabs__item">
<a href="../../../blog/" class="md-tabs__link">
Blog
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../.." title="Changemaker Lite" class="md-nav__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
<img src="../../../assets/logo.png" alt="logo">
</a>
Changemaker Lite
</label>
<div class="md-nav__source">
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
changemaker.lite
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
<div class="md-nav__link md-nav__container">
<a href="../../" class="md-nav__link ">
<span class="md-ellipsis">
V2 Documentation
</span>
</a>
<label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
V2 Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_2" >
<div class="md-nav__link md-nav__container">
<a href="../../getting-started/" class="md-nav__link ">
<span class="md-ellipsis">
Getting Started
</span>
</a>
<label class="md-nav__link " for="__nav_2_2" id="__nav_2_2_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_2">
<span class="md-nav__icon md-icon"></span>
Getting Started
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../getting-started/quick-start/" class="md-nav__link">
<span class="md-ellipsis">
Quick Start
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_3" >
<div class="md-nav__link md-nav__container">
<a href="../../architecture/" class="md-nav__link ">
<span class="md-ellipsis">
Architecture
</span>
</a>
<label class="md-nav__link " for="__nav_2_3" id="__nav_2_3_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_3">
<span class="md-nav__icon md-icon"></span>
Architecture
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../architecture/dual-api/" class="md-nav__link">
<span class="md-ellipsis">
Dual API System
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/authentication/" class="md-nav__link">
<span class="md-ellipsis">
Authentication & Security
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_4" >
<div class="md-nav__link md-nav__container">
<a href="../../backend/" class="md-nav__link ">
<span class="md-ellipsis">
Backend
</span>
</a>
<label class="md-nav__link " for="__nav_2_4" id="__nav_2_4_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_4">
<span class="md-nav__icon md-icon"></span>
Backend
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/modules/" class="md-nav__link">
<span class="md-ellipsis">
Modules
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/services/" class="md-nav__link">
<span class="md-ellipsis">
Services
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/middleware/" class="md-nav__link">
<span class="md-ellipsis">
Middleware
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/utilities/" class="md-nav__link">
<span class="md-ellipsis">
Utilities
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_5" >
<div class="md-nav__link md-nav__container">
<a href="../../frontend/" class="md-nav__link ">
<span class="md-ellipsis">
Frontend
</span>
</a>
<label class="md-nav__link " for="__nav_2_5" id="__nav_2_5_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_5">
<span class="md-nav__icon md-icon"></span>
Frontend
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/components/" class="md-nav__link">
<span class="md-ellipsis">
Components
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/layouts/" class="md-nav__link">
<span class="md-ellipsis">
Layouts
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/pages/" class="md-nav__link">
<span class="md-ellipsis">
Pages
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_6" >
<div class="md-nav__link md-nav__container">
<a href="../../database/" class="md-nav__link ">
<span class="md-ellipsis">
Database
</span>
</a>
<label class="md-nav__link " for="__nav_2_6" id="__nav_2_6_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_6">
<span class="md-nav__icon md-icon"></span>
Database
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../database/schema/" class="md-nav__link">
<span class="md-ellipsis">
Schema Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/migrations/" class="md-nav__link">
<span class="md-ellipsis">
Migrations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/seeding/" class="md-nav__link">
<span class="md-ellipsis">
Seeding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/indexes/" class="md-nav__link">
<span class="md-ellipsis">
Indexes
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../database/models/" class="md-nav__link">
<span class="md-ellipsis">
Models
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_7" >
<div class="md-nav__link md-nav__container">
<a href="../../features/" class="md-nav__link ">
<span class="md-ellipsis">
Features
</span>
</a>
<label class="md-nav__link " for="__nav_2_7" id="__nav_2_7_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_7">
<span class="md-nav__icon md-icon"></span>
Features
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/influence/" class="md-nav__link">
<span class="md-ellipsis">
Influence
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/map/" class="md-nav__link">
<span class="md-ellipsis">
Map
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/landing-pages/" class="md-nav__link">
<span class="md-ellipsis">
Landing Pages
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/email-templates/" class="md-nav__link">
<span class="md-ellipsis">
Email Templates
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/media/" class="md-nav__link">
<span class="md-ellipsis">
Media
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/newsletter/" class="md-nav__link">
<span class="md-ellipsis">
Newsletter
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/observability/" class="md-nav__link">
<span class="md-ellipsis">
Observability
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../features/tunnel/" class="md-nav__link">
<span class="md-ellipsis">
Tunnel
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_8" checked>
<div class="md-nav__link md-nav__container">
<a href="../" class="md-nav__link ">
<span class="md-ellipsis">
Deployment
</span>
</a>
<label class="md-nav__link " for="__nav_2_8" id="__nav_2_8_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_8_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2_8">
<span class="md-nav__icon md-icon"></span>
Deployment
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../docker-compose/" class="md-nav__link">
<span class="md-ellipsis">
Docker Compose
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../environment-variables/" class="md-nav__link">
<span class="md-ellipsis">
Environment Variables
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../nginx/" class="md-nav__link">
<span class="md-ellipsis">
Nginx Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ssl-tls/" class="md-nav__link">
<span class="md-ellipsis">
SSL/TLS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../tunneling/" class="md-nav__link">
<span class="md-ellipsis">
Tunneling
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Monitoring Stack
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Monitoring Stack
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="On this page">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
On this page
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start" class="md-nav__link">
<span class="md-ellipsis">
Quick Start
</span>
</a>
<nav class="md-nav" aria-label="Quick Start">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#enable-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Enable Monitoring
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#prometheus-configuration" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Configuration
</span>
</a>
<nav class="md-nav" aria-label="Prometheus Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#scrape-targets" class="md-nav__link">
<span class="md-ellipsis">
Scrape Targets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#custom-metrics-cm_" class="md-nav__link">
<span class="md-ellipsis">
Custom Metrics (cm_*)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-rules" class="md-nav__link">
<span class="md-ellipsis">
Alert Rules
</span>
</a>
<nav class="md-nav" aria-label="Alert Rules">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-alerts" class="md-nav__link">
<span class="md-ellipsis">
Application Alerts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#system-alerts" class="md-nav__link">
<span class="md-ellipsis">
System Alerts
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#data-retention" class="md-nav__link">
<span class="md-ellipsis">
Data Retention
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#grafana-configuration" class="md-nav__link">
<span class="md-ellipsis">
Grafana Configuration
</span>
</a>
<nav class="md-nav" aria-label="Grafana Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#datasource" class="md-nav__link">
<span class="md-ellipsis">
Datasource
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dashboards" class="md-nav__link">
<span class="md-ellipsis">
Dashboards
</span>
</a>
<nav class="md-nav" aria-label="Dashboards">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-application-overview" class="md-nav__link">
<span class="md-ellipsis">
1. Application Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2-api-performance" class="md-nav__link">
<span class="md-ellipsis">
2. API Performance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#3-system-health" class="md-nav__link">
<span class="md-ellipsis">
3. System Health
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#first-login" class="md-nav__link">
<span class="md-ellipsis">
First Login
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#alertmanager-configuration" class="md-nav__link">
<span class="md-ellipsis">
Alertmanager Configuration
</span>
</a>
<nav class="md-nav" aria-label="Alertmanager Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#notification-receivers" class="md-nav__link">
<span class="md-ellipsis">
Notification Receivers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#testing-alerts" class="md-nav__link">
<span class="md-ellipsis">
Testing Alerts
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#exporters" class="md-nav__link">
<span class="md-ellipsis">
Exporters
</span>
</a>
<nav class="md-nav" aria-label="Exporters">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cadvisor-container-metrics" class="md-nav__link">
<span class="md-ellipsis">
cAdvisor (Container Metrics)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#node-exporter-host-metrics" class="md-nav__link">
<span class="md-ellipsis">
Node Exporter (Host Metrics)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#redis-exporter" class="md-nav__link">
<span class="md-ellipsis">
Redis Exporter
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#gotify-push-notifications" class="md-nav__link">
<span class="md-ellipsis">
Gotify (Push Notifications)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#accessing-services" class="md-nav__link">
<span class="md-ellipsis">
Accessing Services
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<nav class="md-nav" aria-label="Troubleshooting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prometheus-not-scraping" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Not Scraping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards-not-loading" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards Not Loading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alerts-not-firing" class="md-nav__link">
<span class="md-ellipsis">
Alerts Not Firing
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#production-best-practices" class="md-nav__link">
<span class="md-ellipsis">
Production Best Practices
</span>
</a>
<nav class="md-nav" aria-label="Production Best Practices">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#secure-grafana" class="md-nav__link">
<span class="md-ellipsis">
Secure Grafana
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-tuning" class="md-nav__link">
<span class="md-ellipsis">
Alert Tuning
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#external-storage-long-term" class="md-nav__link">
<span class="md-ellipsis">
External Storage (Long-Term)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#related-documentation" class="md-nav__link">
<span class="md-ellipsis">
Related Documentation
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../healthchecks/" class="md-nav__link">
<span class="md-ellipsis">
Health Checks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../scaling/" class="md-nav__link">
<span class="md-ellipsis">
Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../backup-restore/" class="md-nav__link">
<span class="md-ellipsis">
Backup & Restore
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_9" >
<div class="md-nav__link md-nav__container">
<a href="../../development/" class="md-nav__link ">
<span class="md-ellipsis">
Development
</span>
</a>
<label class="md-nav__link " for="__nav_2_9" id="__nav_2_9_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_9">
<span class="md-nav__icon md-icon"></span>
Development
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../development/local-setup/" class="md-nav__link">
<span class="md-ellipsis">
Local Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/docker-workflow/" class="md-nav__link">
<span class="md-ellipsis">
Docker Workflow
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/git-workflow/" class="md-nav__link">
<span class="md-ellipsis">
Git Workflow
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/npm-commands/" class="md-nav__link">
<span class="md-ellipsis">
NPM Commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/migrations/" class="md-nav__link">
<span class="md-ellipsis">
Migrations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/typescript/" class="md-nav__link">
<span class="md-ellipsis">
TypeScript
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/testing/" class="md-nav__link">
<span class="md-ellipsis">
Testing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/debugging/" class="md-nav__link">
<span class="md-ellipsis">
Debugging
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/code-style/" class="md-nav__link">
<span class="md-ellipsis">
Code Style
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_10" >
<div class="md-nav__link md-nav__container">
<a href="../../api-reference/" class="md-nav__link ">
<span class="md-ellipsis">
API Reference
</span>
</a>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_10">
<span class="md-nav__icon md-icon"></span>
API Reference
</label>
<ul class="md-nav__list" data-md-scrollfix>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
<div class="md-nav__link md-nav__container">
<a href="../../user-guides/" class="md-nav__link ">
<span class="md-ellipsis">
User Guides
</span>
</a>
<label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_11">
<span class="md-nav__icon md-icon"></span>
User Guides
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../user-guides/admin-guide/" class="md-nav__link">
<span class="md-ellipsis">
Admin Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/campaign-manager-guide/" class="md-nav__link">
<span class="md-ellipsis">
Campaign Manager Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/map-organizer-guide/" class="md-nav__link">
<span class="md-ellipsis">
Map Organizer Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/content-editor-guide/" class="md-nav__link">
<span class="md-ellipsis">
Content Editor Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/volunteer-guide/" class="md-nav__link">
<span class="md-ellipsis">
Volunteer Guide
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
<div class="md-nav__link md-nav__container">
<a href="../../troubleshooting/" class="md-nav__link ">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_12">
<span class="md-nav__icon md-icon"></span>
Troubleshooting
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../troubleshooting/faq/" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/common-errors/" class="md-nav__link">
<span class="md-ellipsis">
Common Errors
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/auth-issues/" class="md-nav__link">
<span class="md-ellipsis">
Auth Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/database-issues/" class="md-nav__link">
<span class="md-ellipsis">
Database Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/docker-issues/" class="md-nav__link">
<span class="md-ellipsis">
Docker Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/email-issues/" class="md-nav__link">
<span class="md-ellipsis">
Email Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/geocoding-issues/" class="md-nav__link">
<span class="md-ellipsis">
Geocoding Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/monitoring-issues/" class="md-nav__link">
<span class="md-ellipsis">
Monitoring Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/performance-optimization/" class="md-nav__link">
<span class="md-ellipsis">
Performance Optimization
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
<div class="md-nav__link md-nav__container">
<a href="../../migration/" class="md-nav__link ">
<span class="md-ellipsis">
Migration
</span>
</a>
<label class="md-nav__link " for="__nav_2_13" id="__nav_2_13_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_13">
<span class="md-nav__icon md-icon"></span>
Migration
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../migration/feature-parity/" class="md-nav__link">
<span class="md-ellipsis">
Feature Parity
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/breaking-changes/" class="md-nav__link">
<span class="md-ellipsis">
Breaking Changes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/api-changes/" class="md-nav__link">
<span class="md-ellipsis">
API Changes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/data-migration/" class="md-nav__link">
<span class="md-ellipsis">
Data Migration
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
<div class="md-nav__link md-nav__container">
<a href="../../contributing/" class="md-nav__link ">
<span class="md-ellipsis">
Contributing
</span>
</a>
<label class="md-nav__link " for="__nav_2_14" id="__nav_2_14_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_14">
<span class="md-nav__icon md-icon"></span>
Contributing
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../contributing/development-setup/" class="md-nav__link">
<span class="md-ellipsis">
Development Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/code-of-conduct/" class="md-nav__link">
<span class="md-ellipsis">
Code of Conduct
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/pull-requests/" class="md-nav__link">
<span class="md-ellipsis">
Pull Requests
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/roadmap/" class="md-nav__link">
<span class="md-ellipsis">
Roadmap
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../phil/" class="md-nav__link">
<span class="md-ellipsis">
Philosophy
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../v1/" class="md-nav__link">
<span class="md-ellipsis">
V1 Documentation (Legacy)
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../blog/" class="md-nav__link">
<span class="md-ellipsis">
Blog
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="On this page">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
On this page
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start" class="md-nav__link">
<span class="md-ellipsis">
Quick Start
</span>
</a>
<nav class="md-nav" aria-label="Quick Start">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#enable-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Enable Monitoring
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#prometheus-configuration" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Configuration
</span>
</a>
<nav class="md-nav" aria-label="Prometheus Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#scrape-targets" class="md-nav__link">
<span class="md-ellipsis">
Scrape Targets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#custom-metrics-cm_" class="md-nav__link">
<span class="md-ellipsis">
Custom Metrics (cm_*)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-rules" class="md-nav__link">
<span class="md-ellipsis">
Alert Rules
</span>
</a>
<nav class="md-nav" aria-label="Alert Rules">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-alerts" class="md-nav__link">
<span class="md-ellipsis">
Application Alerts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#system-alerts" class="md-nav__link">
<span class="md-ellipsis">
System Alerts
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#data-retention" class="md-nav__link">
<span class="md-ellipsis">
Data Retention
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#grafana-configuration" class="md-nav__link">
<span class="md-ellipsis">
Grafana Configuration
</span>
</a>
<nav class="md-nav" aria-label="Grafana Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#datasource" class="md-nav__link">
<span class="md-ellipsis">
Datasource
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dashboards" class="md-nav__link">
<span class="md-ellipsis">
Dashboards
</span>
</a>
<nav class="md-nav" aria-label="Dashboards">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-application-overview" class="md-nav__link">
<span class="md-ellipsis">
1. Application Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2-api-performance" class="md-nav__link">
<span class="md-ellipsis">
2. API Performance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#3-system-health" class="md-nav__link">
<span class="md-ellipsis">
3. System Health
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#first-login" class="md-nav__link">
<span class="md-ellipsis">
First Login
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#alertmanager-configuration" class="md-nav__link">
<span class="md-ellipsis">
Alertmanager Configuration
</span>
</a>
<nav class="md-nav" aria-label="Alertmanager Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#notification-receivers" class="md-nav__link">
<span class="md-ellipsis">
Notification Receivers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#testing-alerts" class="md-nav__link">
<span class="md-ellipsis">
Testing Alerts
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#exporters" class="md-nav__link">
<span class="md-ellipsis">
Exporters
</span>
</a>
<nav class="md-nav" aria-label="Exporters">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cadvisor-container-metrics" class="md-nav__link">
<span class="md-ellipsis">
cAdvisor (Container Metrics)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#node-exporter-host-metrics" class="md-nav__link">
<span class="md-ellipsis">
Node Exporter (Host Metrics)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#redis-exporter" class="md-nav__link">
<span class="md-ellipsis">
Redis Exporter
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#gotify-push-notifications" class="md-nav__link">
<span class="md-ellipsis">
Gotify (Push Notifications)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#accessing-services" class="md-nav__link">
<span class="md-ellipsis">
Accessing Services
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<nav class="md-nav" aria-label="Troubleshooting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prometheus-not-scraping" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Not Scraping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards-not-loading" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards Not Loading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alerts-not-firing" class="md-nav__link">
<span class="md-ellipsis">
Alerts Not Firing
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#production-best-practices" class="md-nav__link">
<span class="md-ellipsis">
Production Best Practices
</span>
</a>
<nav class="md-nav" aria-label="Production Best Practices">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#secure-grafana" class="md-nav__link">
<span class="md-ellipsis">
Secure Grafana
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-tuning" class="md-nav__link">
<span class="md-ellipsis">
Alert Tuning
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#external-storage-long-term" class="md-nav__link">
<span class="md-ellipsis">
External Storage (Long-Term)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#related-documentation" class="md-nav__link">
<span class="md-ellipsis">
Related Documentation
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<nav class="md-path" aria-label="Navigation" >
<ol class="md-path__list">
<li class="md-path__item">
<a href="../../.." class="md-path__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-path__item">
<a href="../../" class="md-path__link">
<span class="md-ellipsis">
V2 Documentation
</span>
</a>
</li>
<li class="md-path__item">
<a href="../" class="md-path__link">
<span class="md-ellipsis">
Deployment
</span>
</a>
</li>
</ol>
</nav>
<article class="md-content__inner md-typeset">
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/deployment/monitoring-stack.md" title="Edit this page" class="md-content__button md-icon" rel="edit">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"/></svg>
</a>
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/deployment/monitoring-stack.md" title="View source of this page" class="md-content__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 18c.56 0 1 .44 1 1s-.44 1-1 1-1-.44-1-1 .44-1 1-1m0-3c-2.73 0-5.06 1.66-6 4 .94 2.34 3.27 4 6 4s5.06-1.66 6-4c-.94-2.34-3.27-4-6-4m0 6.5a2.5 2.5 0 0 1-2.5-2.5 2.5 2.5 0 0 1 2.5-2.5 2.5 2.5 0 0 1 2.5 2.5 2.5 2.5 0 0 1-2.5 2.5M9.27 20H6V4h7v5h5v4.07c.7.08 1.36.25 2 .49V8l-6-6H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h4.5a8.2 8.2 0 0 1-1.23-2"/></svg>
</a>
<h1 id="monitoring-stack-prometheus-grafana">Monitoring Stack (Prometheus + Grafana)<a class="headerlink" href="#monitoring-stack-prometheus-grafana" title="Permanent link">&para;</a></h1>
<h2 id="overview">Overview<a class="headerlink" href="#overview" title="Permanent link">&para;</a></h2>
<p>Changemaker Lite V2 includes a complete observability stack for production monitoring:</p>
<ul>
<li><strong>Prometheus</strong>: Metrics collection + alerting rules</li>
<li><strong>Grafana</strong>: Visualization + pre-configured dashboards</li>
<li><strong>Alertmanager</strong>: Alert routing + notifications</li>
<li><strong>cAdvisor</strong>: Docker container metrics</li>
<li><strong>Node Exporter</strong>: Host system metrics</li>
<li><strong>Redis Exporter</strong>: Redis-specific metrics</li>
<li><strong>Gotify</strong>: Push notifications (optional)</li>
</ul>
<p><strong>All monitoring services</strong> behind Docker Compose profile flag (opt-in).</p>
<hr />
<h2 id="architecture">Architecture<a class="headerlink" href="#architecture" title="Permanent link">&para;</a></h2>
<pre class="mermaid"><code>graph LR
subgraph "Application Metrics"
API[API&lt;br/&gt;:4000/api/metrics]
MEDIA[Media API&lt;br/&gt;:4100/metrics]
end
subgraph "Infrastructure Metrics"
CADVISOR[cAdvisor&lt;br/&gt;Container Stats]
NODE[Node Exporter&lt;br/&gt;Host Stats]
REDIS_EXP[Redis Exporter&lt;br/&gt;Redis Stats]
end
subgraph "Monitoring Stack"
PROM[Prometheus&lt;br/&gt;:9090]
GRAFANA[Grafana&lt;br/&gt;:3001]
ALERT[Alertmanager&lt;br/&gt;:9093]
GOTIFY[Gotify&lt;br/&gt;:8889]
end
API --&gt; PROM
MEDIA --&gt; PROM
CADVISOR --&gt; PROM
NODE --&gt; PROM
REDIS_EXP --&gt; PROM
PROM --&gt; GRAFANA
PROM --&gt; ALERT
ALERT --&gt; GOTIFY</code></pre>
<hr />
<h2 id="quick-start">Quick Start<a class="headerlink" href="#quick-start" title="Permanent link">&para;</a></h2>
<h3 id="enable-monitoring">Enable Monitoring<a class="headerlink" href="#enable-monitoring" title="Permanent link">&para;</a></h3>
<div class="language-bash highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1"># Start with monitoring profile</span>
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>up<span class="w"> </span>-d
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>
</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="c1"># Check services</span>
</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a>docker<span class="w"> </span>compose<span class="w"> </span>ps<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>monitoring
</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a>
</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="c1"># Access dashboards</span>
</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a>open<span class="w"> </span>http://localhost:3001<span class="w"> </span><span class="c1"># Grafana (admin/admin)</span>
</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a>open<span class="w"> </span>http://localhost:9090<span class="w"> </span><span class="c1"># Prometheus</span>
</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a>open<span class="w"> </span>http://localhost:9093<span class="w"> </span><span class="c1"># Alertmanager</span>
</span></code></pre></div>
<hr />
<h2 id="prometheus-configuration">Prometheus Configuration<a class="headerlink" href="#prometheus-configuration" title="Permanent link">&para;</a></h2>
<h3 id="scrape-targets">Scrape Targets<a class="headerlink" href="#scrape-targets" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>configs/prometheus/prometheus.yml</code></p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="w"> </span><span class="c1"># V2 Unified API Metrics (10s interval)</span>
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;changemaker-v2-api&#39;</span>
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;changemaker-v2-api:4000&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="w"> </span><span class="nt">metrics_path</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;/api/metrics&#39;</span>
</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10s</span>
</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="w"> </span><span class="nt">scrape_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5s</span>
</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>
</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="w"> </span><span class="c1"># Redis Metrics (15s interval)</span>
</span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;redis&#39;</span>
</span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;redis-exporter:9121&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-14"><a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span>
</span><span id="__span-1-15"><a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a>
</span><span id="__span-1-16"><a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="w"> </span><span class="c1"># cAdvisor - Docker container metrics</span>
</span><span id="__span-1-17"><a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;cadvisor&#39;</span>
</span><span id="__span-1-18"><a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-19"><a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;cadvisor:8080&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-20"><a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span>
</span><span id="__span-1-21"><a id="__codelineno-1-21" name="__codelineno-1-21" href="#__codelineno-1-21"></a>
</span><span id="__span-1-22"><a id="__codelineno-1-22" name="__codelineno-1-22" href="#__codelineno-1-22"></a><span class="w"> </span><span class="c1"># Node Exporter - System metrics</span>
</span><span id="__span-1-23"><a id="__codelineno-1-23" name="__codelineno-1-23" href="#__codelineno-1-23"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;node&#39;</span>
</span><span id="__span-1-24"><a id="__codelineno-1-24" name="__codelineno-1-24" href="#__codelineno-1-24"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-25"><a id="__codelineno-1-25" name="__codelineno-1-25" href="#__codelineno-1-25"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;node-exporter:9100&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-26"><a id="__codelineno-1-26" name="__codelineno-1-26" href="#__codelineno-1-26"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span>
</span><span id="__span-1-27"><a id="__codelineno-1-27" name="__codelineno-1-27" href="#__codelineno-1-27"></a>
</span><span id="__span-1-28"><a id="__codelineno-1-28" name="__codelineno-1-28" href="#__codelineno-1-28"></a><span class="w"> </span><span class="c1"># Prometheus self-monitoring</span>
</span><span id="__span-1-29"><a id="__codelineno-1-29" name="__codelineno-1-29" href="#__codelineno-1-29"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;prometheus&#39;</span>
</span><span id="__span-1-30"><a id="__codelineno-1-30" name="__codelineno-1-30" href="#__codelineno-1-30"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-31"><a id="__codelineno-1-31" name="__codelineno-1-31" href="#__codelineno-1-31"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;localhost:9090&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-32"><a id="__codelineno-1-32" name="__codelineno-1-32" href="#__codelineno-1-32"></a>
</span><span id="__span-1-33"><a id="__codelineno-1-33" name="__codelineno-1-33" href="#__codelineno-1-33"></a><span class="w"> </span><span class="c1"># Alertmanager monitoring</span>
</span><span id="__span-1-34"><a id="__codelineno-1-34" name="__codelineno-1-34" href="#__codelineno-1-34"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;alertmanager&#39;</span>
</span><span id="__span-1-35"><a id="__codelineno-1-35" name="__codelineno-1-35" href="#__codelineno-1-35"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-1-36"><a id="__codelineno-1-36" name="__codelineno-1-36" href="#__codelineno-1-36"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;alertmanager:9093&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-1-37"><a id="__codelineno-1-37" name="__codelineno-1-37" href="#__codelineno-1-37"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
</span></code></pre></div>
<p><strong>Intervals:</strong>
- <strong>10s</strong>: API (real-time application metrics)
- <strong>15s</strong>: Infrastructure (host + containers + Redis)
- <strong>30s</strong>: Monitoring stack itself</p>
<hr />
<h3 id="custom-metrics-cm_">Custom Metrics (cm_*)<a class="headerlink" href="#custom-metrics-cm_" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>api/src/utils/metrics.ts</code></p>
<p><strong>12 custom metrics</strong> for domain-specific monitoring:</p>
<table>
<thead>
<tr>
<th>Metric</th>
<th>Type</th>
<th>Labels</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>cm_emails_sent_total</code></td>
<td>Counter</td>
<td><code>campaign_id</code></td>
<td>Campaign emails sent successfully</td>
</tr>
<tr>
<td><code>cm_emails_failed_total</code></td>
<td>Counter</td>
<td><code>campaign_id</code>, <code>error_type</code></td>
<td>Failed email sends</td>
</tr>
<tr>
<td><code>cm_email_queue_size</code></td>
<td>Gauge</td>
<td>-</td>
<td>Current email queue size</td>
</tr>
<tr>
<td><code>cm_email_send_duration_seconds</code></td>
<td>Histogram</td>
<td>-</td>
<td>Email send latency</td>
</tr>
<tr>
<td><code>cm_login_attempts_total</code></td>
<td>Counter</td>
<td><code>status</code></td>
<td>Login attempts (success/failure)</td>
</tr>
<tr>
<td><code>cm_active_sessions</code></td>
<td>Gauge</td>
<td>-</td>
<td>Active refresh tokens</td>
</tr>
<tr>
<td><code>cm_campaign_emails_total</code></td>
<td>Counter</td>
<td><code>campaign_id</code></td>
<td>Total campaign emails created</td>
</tr>
<tr>
<td><code>cm_response_submissions_total</code></td>
<td>Counter</td>
<td>-</td>
<td>Response wall submissions</td>
</tr>
<tr>
<td><code>cm_canvass_visits_total</code></td>
<td>Counter</td>
<td><code>outcome</code></td>
<td>Canvass visits by outcome</td>
</tr>
<tr>
<td><code>cm_active_canvass_sessions</code></td>
<td>Gauge</td>
<td>-</td>
<td>Active canvass sessions</td>
</tr>
<tr>
<td><code>cm_shift_signups_total</code></td>
<td>Counter</td>
<td>-</td>
<td>Shift signups</td>
</tr>
<tr>
<td><code>cm_external_service_up</code></td>
<td>Gauge</td>
<td><code>service</code></td>
<td>External service health (1=up, 0=down)</td>
</tr>
</tbody>
</table>
<p><strong>HTTP metrics</strong> (standard prom-client):
- <code>http_requests_total</code>
- <code>http_request_duration_seconds</code></p>
<p><strong>Geocoding metrics:</strong>
- <code>cm_geocode_cache_hits_total</code>
- <code>cm_geocode_cache_misses_total</code>
- <code>cm_geocode_requests_total</code>
- <code>cm_geocode_duration_seconds</code></p>
<p><strong>Email template metrics:</strong>
- <code>cm_email_templates_updated_total</code>
- <code>cm_email_test_sent_total</code>
- <code>cm_email_template_rollback_total</code>
- <code>cm_email_template_cache_hit/miss_total</code></p>
<p><strong>Location query metrics:</strong>
- <code>cm_map_location_query_duration_seconds</code>
- <code>cm_map_location_query_count_total</code>
- <code>cm_map_location_result_count</code></p>
<hr />
<h3 id="alert-rules">Alert Rules<a class="headerlink" href="#alert-rules" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>configs/prometheus/alerts.yml</code></p>
<p><strong>12 alert rules</strong> across 4 groups:</p>
<h4 id="application-alerts">Application Alerts<a class="headerlink" href="#application-alerts" title="Permanent link">&para;</a></h4>
<ol>
<li><strong>ApplicationDown</strong>: API unreachable for 2 minutes</li>
<li><strong>HighErrorRate</strong>: &gt;10% 5xx errors for 5 minutes</li>
<li><strong>EmailQueueBacklog</strong>: Queue size &gt;100 for 10 minutes</li>
<li><strong>HighEmailFailureRate</strong>: &gt;20% email failures for 10 minutes</li>
<li><strong>SuspiciousLoginActivity</strong>: &gt;5 failed logins/sec for 2 minutes</li>
<li><strong>HighAPILatency</strong>: P95 latency &gt;2s for 5 minutes</li>
<li><strong>ExternalServiceDown</strong>: External service unreachable for 5 minutes</li>
</ol>
<h4 id="system-alerts">System Alerts<a class="headerlink" href="#system-alerts" title="Permanent link">&para;</a></h4>
<ol>
<li><strong>RedisDown</strong>: Redis unreachable for 1 minute</li>
<li><strong>DiskSpaceLow</strong>: &lt;15% disk space for 5 minutes</li>
<li><strong>DiskSpaceCritical</strong>: &lt;10% disk space for 2 minutes</li>
<li><strong>HighCPUUsage</strong>: &gt;85% CPU for 10 minutes</li>
<li><strong>HighMemoryUsage</strong>: &gt;85% memory for 10 minutes</li>
</ol>
<p><strong>Example Alert</strong>:
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ApplicationDown</span>
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">up{job=&quot;changemaker-v2-api&quot;} == 0</span>
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2m</span>
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w"> </span><span class="nt">labels</span><span class="p">:</span>
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="w"> </span><span class="nt">annotations</span><span class="p">:</span>
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w"> </span><span class="nt">summary</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;V2</span><span class="nv"> </span><span class="s">API</span><span class="nv"> </span><span class="s">is</span><span class="nv"> </span><span class="s">down&quot;</span>
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;The</span><span class="nv"> </span><span class="s">Changemaker</span><span class="nv"> </span><span class="s">V2</span><span class="nv"> </span><span class="s">API</span><span class="nv"> </span><span class="s">has</span><span class="nv"> </span><span class="s">been</span><span class="nv"> </span><span class="s">down</span><span class="nv"> </span><span class="s">for</span><span class="nv"> </span><span class="s">more</span><span class="nv"> </span><span class="s">than</span><span class="nv"> </span><span class="s">2</span><span class="nv"> </span><span class="s">minutes.&quot;</span>
</span></code></pre></div></p>
<hr />
<h3 id="data-retention">Data Retention<a class="headerlink" href="#data-retention" title="Permanent link">&para;</a></h3>
<p><strong>docker-compose.yml</strong>:
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="nt">prometheus</span><span class="p">:</span>
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="w"> </span><span class="nt">command</span><span class="p">:</span>
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&#39;--storage.tsdb.retention.time=30d&#39;</span><span class="w"> </span><span class="c1"># 30 days</span>
</span></code></pre></div></p>
<p><strong>Disk usage</strong>: ~1-5GB for 30 days (depends on scrape frequency + cardinality).</p>
<p><strong>Increase retention</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="c1"># Edit docker-compose.yml</span>
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="c1"># Change to &#39;--storage.tsdb.retention.time=90d&#39;</span>
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a>
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="c1"># Recreate container</span>
</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>up<span class="w"> </span>-d<span class="w"> </span>--force-recreate<span class="w"> </span>prometheus
</span></code></pre></div></p>
<hr />
<h2 id="grafana-configuration">Grafana Configuration<a class="headerlink" href="#grafana-configuration" title="Permanent link">&para;</a></h2>
<h3 id="datasource">Datasource<a class="headerlink" href="#datasource" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>configs/grafana/datasources.yml</code></p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="nt">apiVersion</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
</span><span id="__span-5-2"><a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a>
</span><span id="__span-5-3"><a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a><span class="nt">datasources</span><span class="p">:</span>
</span><span id="__span-5-4"><a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Prometheus</span>
</span><span id="__span-5-5"><a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prometheus</span>
</span><span id="__span-5-6"><a id="__codelineno-5-6" name="__codelineno-5-6" href="#__codelineno-5-6"></a><span class="w"> </span><span class="nt">access</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">proxy</span>
</span><span id="__span-5-7"><a id="__codelineno-5-7" name="__codelineno-5-7" href="#__codelineno-5-7"></a><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://prometheus:9090</span>
</span><span id="__span-5-8"><a id="__codelineno-5-8" name="__codelineno-5-8" href="#__codelineno-5-8"></a><span class="w"> </span><span class="nt">isDefault</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
</span><span id="__span-5-9"><a id="__codelineno-5-9" name="__codelineno-5-9" href="#__codelineno-5-9"></a><span class="w"> </span><span class="nt">editable</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
</span></code></pre></div>
<p><strong>Auto-provisioned</strong> on Grafana startup.</p>
<hr />
<h3 id="dashboards">Dashboards<a class="headerlink" href="#dashboards" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>configs/grafana/dashboards.yml</code></p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="nt">apiVersion</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a>
</span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a><span class="nt">providers</span><span class="p">:</span>
</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;Default&#39;</span>
</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5" href="#__codelineno-6-5"></a><span class="w"> </span><span class="nt">folder</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;Changemaker</span><span class="nv"> </span><span class="s">Lite&#39;</span>
</span><span id="__span-6-6"><a id="__codelineno-6-6" name="__codelineno-6-6" href="#__codelineno-6-6"></a><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">file</span>
</span><span id="__span-6-7"><a id="__codelineno-6-7" name="__codelineno-6-7" href="#__codelineno-6-7"></a><span class="w"> </span><span class="nt">options</span><span class="p">:</span>
</span><span id="__span-6-8"><a id="__codelineno-6-8" name="__codelineno-6-8" href="#__codelineno-6-8"></a><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/etc/grafana/provisioning/dashboards</span>
</span></code></pre></div>
<p><strong>3 pre-configured dashboards</strong>:</p>
<h4 id="1-application-overview">1. Application Overview<a class="headerlink" href="#1-application-overview" title="Permanent link">&para;</a></h4>
<p><strong>File</strong>: <code>configs/grafana/application-overview.json</code></p>
<p><strong>Panels</strong>:
- API uptime (last 24h)
- Request rate (req/sec)
- Error rate (%)
- Email queue size
- Active sessions
- Campaign emails sent</p>
<p><strong>Refresh</strong>: 10s</p>
<hr />
<h4 id="2-api-performance">2. API Performance<a class="headerlink" href="#2-api-performance" title="Permanent link">&para;</a></h4>
<p><strong>File</strong>: <code>configs/grafana/api-performance.json</code></p>
<p><strong>Panels</strong>:
- Request latency (P50, P95, P99)
- Requests by status code
- Top 10 slowest endpoints
- HTTP errors by route
- Geocoding cache hit rate
- Email send duration</p>
<p><strong>Refresh</strong>: 30s</p>
<hr />
<h4 id="3-system-health">3. System Health<a class="headerlink" href="#3-system-health" title="Permanent link">&para;</a></h4>
<p><strong>File</strong>: <code>configs/grafana/system-health.json</code></p>
<p><strong>Panels</strong>:
- CPU usage (%)
- Memory usage (%)
- Disk space (GB free)
- Network I/O (MB/s)
- Container CPU throttling
- Redis memory usage</p>
<p><strong>Refresh</strong>: 1m</p>
<hr />
<h3 id="first-login">First Login<a class="headerlink" href="#first-login" title="Permanent link">&para;</a></h3>
<div class="language-bash highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a><span class="c1"># Access Grafana</span>
</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a>open<span class="w"> </span>http://localhost:3001
</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a>
</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a><span class="c1"># Default credentials</span>
</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5" href="#__codelineno-7-5"></a>Username:<span class="w"> </span>admin
</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6" href="#__codelineno-7-6"></a>Password:<span class="w"> </span>admin
</span><span id="__span-7-7"><a id="__codelineno-7-7" name="__codelineno-7-7" href="#__codelineno-7-7"></a>
</span><span id="__span-7-8"><a id="__codelineno-7-8" name="__codelineno-7-8" href="#__codelineno-7-8"></a><span class="c1"># Change password on first login</span>
</span></code></pre></div>
<p><strong>Navigate</strong>: Dashboards → Changemaker Lite folder → Select dashboard</p>
<hr />
<h2 id="alertmanager-configuration">Alertmanager Configuration<a class="headerlink" href="#alertmanager-configuration" title="Permanent link">&para;</a></h2>
<h3 id="notification-receivers">Notification Receivers<a class="headerlink" href="#notification-receivers" title="Permanent link">&para;</a></h3>
<p><strong>File</strong>: <code>configs/alertmanager/alertmanager.yml</code></p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="nt">global</span><span class="p">:</span>
</span><span id="__span-8-2"><a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a><span class="w"> </span><span class="nt">resolve_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5m</span>
</span><span id="__span-8-3"><a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a>
</span><span id="__span-8-4"><a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a><span class="nt">route</span><span class="p">:</span>
</span><span id="__span-8-5"><a id="__codelineno-8-5" name="__codelineno-8-5" href="#__codelineno-8-5"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;default&#39;</span>
</span><span id="__span-8-6"><a id="__codelineno-8-6" name="__codelineno-8-6" href="#__codelineno-8-6"></a><span class="w"> </span><span class="nt">group_by</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;alertname&#39;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&#39;severity&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-8-7"><a id="__codelineno-8-7" name="__codelineno-8-7" href="#__codelineno-8-7"></a><span class="w"> </span><span class="nt">group_wait</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
</span><span id="__span-8-8"><a id="__codelineno-8-8" name="__codelineno-8-8" href="#__codelineno-8-8"></a><span class="w"> </span><span class="nt">group_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5m</span>
</span><span id="__span-8-9"><a id="__codelineno-8-9" name="__codelineno-8-9" href="#__codelineno-8-9"></a><span class="w"> </span><span class="nt">repeat_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">4h</span>
</span><span id="__span-8-10"><a id="__codelineno-8-10" name="__codelineno-8-10" href="#__codelineno-8-10"></a>
</span><span id="__span-8-11"><a id="__codelineno-8-11" name="__codelineno-8-11" href="#__codelineno-8-11"></a><span class="nt">receivers</span><span class="p">:</span>
</span><span id="__span-8-12"><a id="__codelineno-8-12" name="__codelineno-8-12" href="#__codelineno-8-12"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;default&#39;</span>
</span><span id="__span-8-13"><a id="__codelineno-8-13" name="__codelineno-8-13" href="#__codelineno-8-13"></a><span class="w"> </span><span class="c1"># Email (example)</span>
</span><span id="__span-8-14"><a id="__codelineno-8-14" name="__codelineno-8-14" href="#__codelineno-8-14"></a><span class="w"> </span><span class="nt">email_configs</span><span class="p">:</span>
</span><span id="__span-8-15"><a id="__codelineno-8-15" name="__codelineno-8-15" href="#__codelineno-8-15"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">to</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;admin@cmlite.org&#39;</span>
</span><span id="__span-8-16"><a id="__codelineno-8-16" name="__codelineno-8-16" href="#__codelineno-8-16"></a><span class="w"> </span><span class="nt">from</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;alerts@cmlite.org&#39;</span>
</span><span id="__span-8-17"><a id="__codelineno-8-17" name="__codelineno-8-17" href="#__codelineno-8-17"></a><span class="w"> </span><span class="nt">smarthost</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;smtp.example.com:587&#39;</span>
</span><span id="__span-8-18"><a id="__codelineno-8-18" name="__codelineno-8-18" href="#__codelineno-8-18"></a><span class="w"> </span><span class="nt">auth_username</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;alerts@cmlite.org&#39;</span>
</span><span id="__span-8-19"><a id="__codelineno-8-19" name="__codelineno-8-19" href="#__codelineno-8-19"></a><span class="w"> </span><span class="nt">auth_password</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;your-password&#39;</span>
</span><span id="__span-8-20"><a id="__codelineno-8-20" name="__codelineno-8-20" href="#__codelineno-8-20"></a>
</span><span id="__span-8-21"><a id="__codelineno-8-21" name="__codelineno-8-21" href="#__codelineno-8-21"></a><span class="w"> </span><span class="c1"># Slack (example)</span>
</span><span id="__span-8-22"><a id="__codelineno-8-22" name="__codelineno-8-22" href="#__codelineno-8-22"></a><span class="w"> </span><span class="nt">slack_configs</span><span class="p">:</span>
</span><span id="__span-8-23"><a id="__codelineno-8-23" name="__codelineno-8-23" href="#__codelineno-8-23"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">api_url</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK&#39;</span>
</span><span id="__span-8-24"><a id="__codelineno-8-24" name="__codelineno-8-24" href="#__codelineno-8-24"></a><span class="w"> </span><span class="nt">channel</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;#alerts&#39;</span>
</span><span id="__span-8-25"><a id="__codelineno-8-25" name="__codelineno-8-25" href="#__codelineno-8-25"></a><span class="w"> </span><span class="nt">title</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;{{</span><span class="nv"> </span><span class="s">.GroupLabels.alertname</span><span class="nv"> </span><span class="s">}}&#39;</span>
</span><span id="__span-8-26"><a id="__codelineno-8-26" name="__codelineno-8-26" href="#__codelineno-8-26"></a><span class="w"> </span><span class="nt">text</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;{{</span><span class="nv"> </span><span class="s">range</span><span class="nv"> </span><span class="s">.Alerts</span><span class="nv"> </span><span class="s">}}{{</span><span class="nv"> </span><span class="s">.Annotations.summary</span><span class="nv"> </span><span class="s">}}\n{{</span><span class="nv"> </span><span class="s">end</span><span class="nv"> </span><span class="s">}}&#39;</span>
</span><span id="__span-8-27"><a id="__codelineno-8-27" name="__codelineno-8-27" href="#__codelineno-8-27"></a>
</span><span id="__span-8-28"><a id="__codelineno-8-28" name="__codelineno-8-28" href="#__codelineno-8-28"></a><span class="w"> </span><span class="c1"># Gotify (push notifications)</span>
</span><span id="__span-8-29"><a id="__codelineno-8-29" name="__codelineno-8-29" href="#__codelineno-8-29"></a><span class="w"> </span><span class="nt">webhook_configs</span><span class="p">:</span>
</span><span id="__span-8-30"><a id="__codelineno-8-30" name="__codelineno-8-30" href="#__codelineno-8-30"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;http://gotify:80/message?token=YOUR_GOTIFY_TOKEN&#39;</span>
</span></code></pre></div>
<p><strong>Grouping</strong>: Combines similar alerts (prevents spam).</p>
<p><strong>Repeat</strong>: Re-sends unresolved alerts every 4 hours.</p>
<hr />
<h3 id="testing-alerts">Testing Alerts<a class="headerlink" href="#testing-alerts" title="Permanent link">&para;</a></h3>
<p><strong>Manual test</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="c1"># Trigger test alert</span>
</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:9093/api/v1/alerts<span class="w"> </span><span class="se">\</span>
</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a><span class="w"> </span>-d<span class="w"> </span><span class="s1">&#39;[{</span>
</span><span id="__span-9-4"><a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="s1"> &quot;labels&quot;: {&quot;alertname&quot;:&quot;TestAlert&quot;,&quot;severity&quot;:&quot;warning&quot;},</span>
</span><span id="__span-9-5"><a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a><span class="s1"> &quot;annotations&quot;: {&quot;summary&quot;:&quot;Test alert from curl&quot;}</span>
</span><span id="__span-9-6"><a id="__codelineno-9-6" name="__codelineno-9-6" href="#__codelineno-9-6"></a><span class="s1"> }]&#39;</span>
</span><span id="__span-9-7"><a id="__codelineno-9-7" name="__codelineno-9-7" href="#__codelineno-9-7"></a>
</span><span id="__span-9-8"><a id="__codelineno-9-8" name="__codelineno-9-8" href="#__codelineno-9-8"></a><span class="c1"># Check Alertmanager UI</span>
</span><span id="__span-9-9"><a id="__codelineno-9-9" name="__codelineno-9-9" href="#__codelineno-9-9"></a>open<span class="w"> </span>http://localhost:9093
</span></code></pre></div></p>
<p><strong>Force alert</strong> (stop API):
<div class="language-bash highlight"><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a><span class="c1"># Stop API (triggers ApplicationDown alert after 2m)</span>
</span><span id="__span-10-2"><a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>stop<span class="w"> </span>api
</span><span id="__span-10-3"><a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a>
</span><span id="__span-10-4"><a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="c1"># Check Prometheus alerts</span>
</span><span id="__span-10-5"><a id="__codelineno-10-5" name="__codelineno-10-5" href="#__codelineno-10-5"></a>open<span class="w"> </span>http://localhost:9090/alerts
</span><span id="__span-10-6"><a id="__codelineno-10-6" name="__codelineno-10-6" href="#__codelineno-10-6"></a>
</span><span id="__span-10-7"><a id="__codelineno-10-7" name="__codelineno-10-7" href="#__codelineno-10-7"></a><span class="c1"># Wait 2 minutes → Alert fires → Notification sent</span>
</span></code></pre></div></p>
<hr />
<h2 id="exporters">Exporters<a class="headerlink" href="#exporters" title="Permanent link">&para;</a></h2>
<h3 id="cadvisor-container-metrics">cAdvisor (Container Metrics)<a class="headerlink" href="#cadvisor-container-metrics" title="Permanent link">&para;</a></h3>
<p><strong>Metrics</strong>:
- CPU usage per container
- Memory usage per container
- Network I/O
- Disk I/O</p>
<p><strong>Access</strong>: http://localhost:8080</p>
<p><strong>Configuration</strong> (docker-compose.yml):
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a><span class="nt">cadvisor</span><span class="p">:</span>
</span><span id="__span-11-2"><a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gcr.io/cadvisor/cadvisor:latest</span>
</span><span id="__span-11-3"><a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a><span class="w"> </span><span class="nt">container_name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">cadvisor-changemaker</span>
</span><span id="__span-11-4"><a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a><span class="w"> </span><span class="nt">privileged</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span><span class="w"> </span><span class="c1"># Required for full access</span>
</span><span id="__span-11-5"><a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a><span class="w"> </span><span class="nt">volumes</span><span class="p">:</span>
</span><span id="__span-11-6"><a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/:/rootfs:ro</span>
</span><span id="__span-11-7"><a id="__codelineno-11-7" name="__codelineno-11-7" href="#__codelineno-11-7"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/var/run:/var/run:ro</span>
</span><span id="__span-11-8"><a id="__codelineno-11-8" name="__codelineno-11-8" href="#__codelineno-11-8"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/sys:/sys:ro</span>
</span><span id="__span-11-9"><a id="__codelineno-11-9" name="__codelineno-11-9" href="#__codelineno-11-9"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/var/lib/docker/:/var/lib/docker:ro</span>
</span><span id="__span-11-10"><a id="__codelineno-11-10" name="__codelineno-11-10" href="#__codelineno-11-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/dev/disk/:/dev/disk:ro</span>
</span><span id="__span-11-11"><a id="__codelineno-11-11" name="__codelineno-11-11" href="#__codelineno-11-11"></a><span class="w"> </span><span class="nt">devices</span><span class="p">:</span>
</span><span id="__span-11-12"><a id="__codelineno-11-12" name="__codelineno-11-12" href="#__codelineno-11-12"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/dev/kmsg</span>
</span></code></pre></div></p>
<hr />
<h3 id="node-exporter-host-metrics">Node Exporter (Host Metrics)<a class="headerlink" href="#node-exporter-host-metrics" title="Permanent link">&para;</a></h3>
<p><strong>Metrics</strong>:
- CPU usage (all cores)
- Memory usage (total, free, cached)
- Disk usage (filesystem, mountpoints)
- Network I/O (bytes, packets)</p>
<p><strong>Access</strong>: http://localhost:9100/metrics</p>
<p><strong>Configuration</strong>:
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a><span class="nt">node-exporter</span><span class="p">:</span>
</span><span id="__span-12-2"><a id="__codelineno-12-2" name="__codelineno-12-2" href="#__codelineno-12-2"></a><span class="w"> </span><span class="nt">command</span><span class="p">:</span>
</span><span id="__span-12-3"><a id="__codelineno-12-3" name="__codelineno-12-3" href="#__codelineno-12-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&#39;--path.rootfs=/host&#39;</span>
</span><span id="__span-12-4"><a id="__codelineno-12-4" name="__codelineno-12-4" href="#__codelineno-12-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&#39;--path.procfs=/host/proc&#39;</span>
</span><span id="__span-12-5"><a id="__codelineno-12-5" name="__codelineno-12-5" href="#__codelineno-12-5"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&#39;--path.sysfs=/host/sys&#39;</span>
</span><span id="__span-12-6"><a id="__codelineno-12-6" name="__codelineno-12-6" href="#__codelineno-12-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&#39;--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)&#39;</span>
</span><span id="__span-12-7"><a id="__codelineno-12-7" name="__codelineno-12-7" href="#__codelineno-12-7"></a><span class="w"> </span><span class="nt">volumes</span><span class="p">:</span>
</span><span id="__span-12-8"><a id="__codelineno-12-8" name="__codelineno-12-8" href="#__codelineno-12-8"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/proc:/host/proc:ro</span>
</span><span id="__span-12-9"><a id="__codelineno-12-9" name="__codelineno-12-9" href="#__codelineno-12-9"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/sys:/host/sys:ro</span>
</span><span id="__span-12-10"><a id="__codelineno-12-10" name="__codelineno-12-10" href="#__codelineno-12-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/:/rootfs:ro</span>
</span></code></pre></div></p>
<hr />
<h3 id="redis-exporter">Redis Exporter<a class="headerlink" href="#redis-exporter" title="Permanent link">&para;</a></h3>
<p><strong>Metrics</strong>:
- Memory usage
- Commands per second
- Connected clients
- Keyspace hits/misses
- Evicted keys</p>
<p><strong>Access</strong>: http://localhost:9121/metrics</p>
<p><strong>Configuration</strong>:
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-13-1"><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a><span class="nt">redis-exporter</span><span class="p">:</span>
</span><span id="__span-13-2"><a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a><span class="w"> </span><span class="nt">environment</span><span class="p">:</span>
</span><span id="__span-13-3"><a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">REDIS_ADDR=redis:6379</span>
</span><span id="__span-13-4"><a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">REDIS_PASSWORD=${REDIS_PASSWORD}</span><span class="w"> </span><span class="c1"># Authenticates with Redis</span>
</span></code></pre></div></p>
<hr />
<h2 id="gotify-push-notifications">Gotify (Push Notifications)<a class="headerlink" href="#gotify-push-notifications" title="Permanent link">&para;</a></h2>
<p><strong>Setup</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-14-1"><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a><span class="c1"># Access Gotify UI</span>
</span><span id="__span-14-2"><a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a>open<span class="w"> </span>http://localhost:8889
</span><span id="__span-14-3"><a id="__codelineno-14-3" name="__codelineno-14-3" href="#__codelineno-14-3"></a>
</span><span id="__span-14-4"><a id="__codelineno-14-4" name="__codelineno-14-4" href="#__codelineno-14-4"></a><span class="c1"># Login (default: admin/admin)</span>
</span><span id="__span-14-5"><a id="__codelineno-14-5" name="__codelineno-14-5" href="#__codelineno-14-5"></a>
</span><span id="__span-14-6"><a id="__codelineno-14-6" name="__codelineno-14-6" href="#__codelineno-14-6"></a><span class="c1"># Create app → Copy token</span>
</span><span id="__span-14-7"><a id="__codelineno-14-7" name="__codelineno-14-7" href="#__codelineno-14-7"></a>
</span><span id="__span-14-8"><a id="__codelineno-14-8" name="__codelineno-14-8" href="#__codelineno-14-8"></a><span class="c1"># Add to Alertmanager config:</span>
</span><span id="__span-14-9"><a id="__codelineno-14-9" name="__codelineno-14-9" href="#__codelineno-14-9"></a>webhook_configs:
</span><span id="__span-14-10"><a id="__codelineno-14-10" name="__codelineno-14-10" href="#__codelineno-14-10"></a><span class="w"> </span>-<span class="w"> </span>url:<span class="w"> </span><span class="s1">&#39;http://gotify:80/message?token=YOUR_TOKEN&#39;</span>
</span></code></pre></div></p>
<p><strong>Mobile apps</strong>: Available for iOS/Android (receive push notifications).</p>
<hr />
<h2 id="accessing-services">Accessing Services<a class="headerlink" href="#accessing-services" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>Service</th>
<th>URL</th>
<th>Default Credentials</th>
</tr>
</thead>
<tbody>
<tr>
<td>Prometheus</td>
<td>http://localhost:9090</td>
<td>None</td>
</tr>
<tr>
<td>Grafana</td>
<td>http://localhost:3001</td>
<td>admin / admin</td>
</tr>
<tr>
<td>Alertmanager</td>
<td>http://localhost:9093</td>
<td>None</td>
</tr>
<tr>
<td>cAdvisor</td>
<td>http://localhost:8080</td>
<td>None</td>
</tr>
<tr>
<td>Node Exporter</td>
<td>http://localhost:9100/metrics</td>
<td>None</td>
</tr>
<tr>
<td>Redis Exporter</td>
<td>http://localhost:9121/metrics</td>
<td>None</td>
</tr>
<tr>
<td>Gotify</td>
<td>http://localhost:8889</td>
<td>admin / admin</td>
</tr>
</tbody>
</table>
<hr />
<h2 id="troubleshooting">Troubleshooting<a class="headerlink" href="#troubleshooting" title="Permanent link">&para;</a></h2>
<h3 id="prometheus-not-scraping">Prometheus Not Scraping<a class="headerlink" href="#prometheus-not-scraping" title="Permanent link">&para;</a></h3>
<p><strong>Symptoms</strong>: Missing data in Grafana dashboards</p>
<p><strong>Diagnosis</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-15-1"><a id="__codelineno-15-1" name="__codelineno-15-1" href="#__codelineno-15-1"></a><span class="c1"># Check Prometheus targets</span>
</span><span id="__span-15-2"><a id="__codelineno-15-2" name="__codelineno-15-2" href="#__codelineno-15-2"></a>open<span class="w"> </span>http://localhost:9090/targets
</span><span id="__span-15-3"><a id="__codelineno-15-3" name="__codelineno-15-3" href="#__codelineno-15-3"></a>
</span><span id="__span-15-4"><a id="__codelineno-15-4" name="__codelineno-15-4" href="#__codelineno-15-4"></a><span class="c1"># Look for errors (red) vs success (green)</span>
</span><span id="__span-15-5"><a id="__codelineno-15-5" name="__codelineno-15-5" href="#__codelineno-15-5"></a>
</span><span id="__span-15-6"><a id="__codelineno-15-6" name="__codelineno-15-6" href="#__codelineno-15-6"></a><span class="c1"># Check API metrics endpoint</span>
</span><span id="__span-15-7"><a id="__codelineno-15-7" name="__codelineno-15-7" href="#__codelineno-15-7"></a>curl<span class="w"> </span>http://localhost:4000/api/metrics
</span></code></pre></div></p>
<p><strong>Common causes</strong>:
- API container not running
- Wrong port in <code>prometheus.yml</code>
- Network connectivity issue</p>
<p><strong>Solution</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-16-1"><a id="__codelineno-16-1" name="__codelineno-16-1" href="#__codelineno-16-1"></a><span class="c1"># Restart API</span>
</span><span id="__span-16-2"><a id="__codelineno-16-2" name="__codelineno-16-2" href="#__codelineno-16-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>restart<span class="w"> </span>api
</span><span id="__span-16-3"><a id="__codelineno-16-3" name="__codelineno-16-3" href="#__codelineno-16-3"></a>
</span><span id="__span-16-4"><a id="__codelineno-16-4" name="__codelineno-16-4" href="#__codelineno-16-4"></a><span class="c1"># Reload Prometheus config</span>
</span><span id="__span-16-5"><a id="__codelineno-16-5" name="__codelineno-16-5" href="#__codelineno-16-5"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span><span class="nb">kill</span><span class="w"> </span>-HUP<span class="w"> </span><span class="m">1</span>
</span><span id="__span-16-6"><a id="__codelineno-16-6" name="__codelineno-16-6" href="#__codelineno-16-6"></a>
</span><span id="__span-16-7"><a id="__codelineno-16-7" name="__codelineno-16-7" href="#__codelineno-16-7"></a><span class="c1"># Or restart Prometheus</span>
</span><span id="__span-16-8"><a id="__codelineno-16-8" name="__codelineno-16-8" href="#__codelineno-16-8"></a>docker<span class="w"> </span>compose<span class="w"> </span>restart<span class="w"> </span>prometheus
</span></code></pre></div></p>
<hr />
<h3 id="grafana-dashboards-not-loading">Grafana Dashboards Not Loading<a class="headerlink" href="#grafana-dashboards-not-loading" title="Permanent link">&para;</a></h3>
<p><strong>Symptoms</strong>: Blank dashboards or "No data" errors</p>
<p><strong>Diagnosis</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-17-1"><a id="__codelineno-17-1" name="__codelineno-17-1" href="#__codelineno-17-1"></a><span class="c1"># Check Grafana logs</span>
</span><span id="__span-17-2"><a id="__codelineno-17-2" name="__codelineno-17-2" href="#__codelineno-17-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>grafana<span class="w"> </span><span class="p">|</span><span class="w"> </span>tail<span class="w"> </span>-50
</span><span id="__span-17-3"><a id="__codelineno-17-3" name="__codelineno-17-3" href="#__codelineno-17-3"></a>
</span><span id="__span-17-4"><a id="__codelineno-17-4" name="__codelineno-17-4" href="#__codelineno-17-4"></a><span class="c1"># Check datasource</span>
</span><span id="__span-17-5"><a id="__codelineno-17-5" name="__codelineno-17-5" href="#__codelineno-17-5"></a>open<span class="w"> </span>http://localhost:3001/datasources
</span><span id="__span-17-6"><a id="__codelineno-17-6" name="__codelineno-17-6" href="#__codelineno-17-6"></a>
</span><span id="__span-17-7"><a id="__codelineno-17-7" name="__codelineno-17-7" href="#__codelineno-17-7"></a><span class="c1"># Test Prometheus query</span>
</span><span id="__span-17-8"><a id="__codelineno-17-8" name="__codelineno-17-8" href="#__codelineno-17-8"></a>curl<span class="w"> </span>http://prometheus:9090/api/v1/query?query<span class="o">=</span>up
</span></code></pre></div></p>
<p><strong>Solution</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-18-1"><a id="__codelineno-18-1" name="__codelineno-18-1" href="#__codelineno-18-1"></a><span class="c1"># Verify datasource URL</span>
</span><span id="__span-18-2"><a id="__codelineno-18-2" name="__codelineno-18-2" href="#__codelineno-18-2"></a><span class="c1"># Should be http://prometheus:9090 (container name, not localhost)</span>
</span><span id="__span-18-3"><a id="__codelineno-18-3" name="__codelineno-18-3" href="#__codelineno-18-3"></a>
</span><span id="__span-18-4"><a id="__codelineno-18-4" name="__codelineno-18-4" href="#__codelineno-18-4"></a><span class="c1"># Restart Grafana</span>
</span><span id="__span-18-5"><a id="__codelineno-18-5" name="__codelineno-18-5" href="#__codelineno-18-5"></a>docker<span class="w"> </span>compose<span class="w"> </span>restart<span class="w"> </span>grafana
</span></code></pre></div></p>
<hr />
<h3 id="alerts-not-firing">Alerts Not Firing<a class="headerlink" href="#alerts-not-firing" title="Permanent link">&para;</a></h3>
<p><strong>Symptoms</strong>: No notifications despite issues</p>
<p><strong>Diagnosis</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-19-1"><a id="__codelineno-19-1" name="__codelineno-19-1" href="#__codelineno-19-1"></a><span class="c1"># Check Prometheus alerts</span>
</span><span id="__span-19-2"><a id="__codelineno-19-2" name="__codelineno-19-2" href="#__codelineno-19-2"></a>open<span class="w"> </span>http://localhost:9090/alerts
</span><span id="__span-19-3"><a id="__codelineno-19-3" name="__codelineno-19-3" href="#__codelineno-19-3"></a>
</span><span id="__span-19-4"><a id="__codelineno-19-4" name="__codelineno-19-4" href="#__codelineno-19-4"></a><span class="c1"># Check Alertmanager</span>
</span><span id="__span-19-5"><a id="__codelineno-19-5" name="__codelineno-19-5" href="#__codelineno-19-5"></a>open<span class="w"> </span>http://localhost:9093
</span><span id="__span-19-6"><a id="__codelineno-19-6" name="__codelineno-19-6" href="#__codelineno-19-6"></a>
</span><span id="__span-19-7"><a id="__codelineno-19-7" name="__codelineno-19-7" href="#__codelineno-19-7"></a><span class="c1"># Verify alert rules loaded</span>
</span><span id="__span-19-8"><a id="__codelineno-19-8" name="__codelineno-19-8" href="#__codelineno-19-8"></a>curl<span class="w"> </span>http://localhost:9090/api/v1/rules
</span></code></pre></div></p>
<p><strong>Solution</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-20-1"><a id="__codelineno-20-1" name="__codelineno-20-1" href="#__codelineno-20-1"></a><span class="c1"># Reload Prometheus config</span>
</span><span id="__span-20-2"><a id="__codelineno-20-2" name="__codelineno-20-2" href="#__codelineno-20-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span><span class="nb">kill</span><span class="w"> </span>-HUP<span class="w"> </span><span class="m">1</span>
</span><span id="__span-20-3"><a id="__codelineno-20-3" name="__codelineno-20-3" href="#__codelineno-20-3"></a>
</span><span id="__span-20-4"><a id="__codelineno-20-4" name="__codelineno-20-4" href="#__codelineno-20-4"></a><span class="c1"># Check alerts.yml syntax</span>
</span><span id="__span-20-5"><a id="__codelineno-20-5" name="__codelineno-20-5" href="#__codelineno-20-5"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span>promtool<span class="w"> </span>check<span class="w"> </span>rules<span class="w"> </span>/etc/prometheus/alerts.yml
</span><span id="__span-20-6"><a id="__codelineno-20-6" name="__codelineno-20-6" href="#__codelineno-20-6"></a>
</span><span id="__span-20-7"><a id="__codelineno-20-7" name="__codelineno-20-7" href="#__codelineno-20-7"></a><span class="c1"># Test notification receiver</span>
</span><span id="__span-20-8"><a id="__codelineno-20-8" name="__codelineno-20-8" href="#__codelineno-20-8"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:9093/api/v1/alerts<span class="w"> </span>-d<span class="w"> </span><span class="s1">&#39;[...]&#39;</span>
</span></code></pre></div></p>
<hr />
<h2 id="production-best-practices">Production Best Practices<a class="headerlink" href="#production-best-practices" title="Permanent link">&para;</a></h2>
<h3 id="secure-grafana">Secure Grafana<a class="headerlink" href="#secure-grafana" title="Permanent link">&para;</a></h3>
<p><strong>Change admin password</strong>:
<div class="language-bash highlight"><pre><span></span><code><span id="__span-21-1"><a id="__codelineno-21-1" name="__codelineno-21-1" href="#__codelineno-21-1"></a><span class="c1"># Via UI: Admin → Profile → Change Password</span>
</span><span id="__span-21-2"><a id="__codelineno-21-2" name="__codelineno-21-2" href="#__codelineno-21-2"></a>
</span><span id="__span-21-3"><a id="__codelineno-21-3" name="__codelineno-21-3" href="#__codelineno-21-3"></a><span class="c1"># Via env var (docker-compose.yml):</span>
</span><span id="__span-21-4"><a id="__codelineno-21-4" name="__codelineno-21-4" href="#__codelineno-21-4"></a>environment:
</span><span id="__span-21-5"><a id="__codelineno-21-5" name="__codelineno-21-5" href="#__codelineno-21-5"></a><span class="w"> </span>-<span class="w"> </span><span class="nv">GF_SECURITY_ADMIN_PASSWORD</span><span class="o">=</span>&lt;strong-password&gt;
</span></code></pre></div></p>
<p><strong>Disable signup</strong>:
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-22-1"><a id="__codelineno-22-1" name="__codelineno-22-1" href="#__codelineno-22-1"></a><span class="nt">environment</span><span class="p">:</span>
</span><span id="__span-22-2"><a id="__codelineno-22-2" name="__codelineno-22-2" href="#__codelineno-22-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">GF_USERS_ALLOW_SIGN_UP=false</span><span class="w"> </span><span class="c1"># Already set</span>
</span></code></pre></div></p>
<hr />
<h3 id="alert-tuning">Alert Tuning<a class="headerlink" href="#alert-tuning" title="Permanent link">&para;</a></h3>
<p><strong>Avoid false positives</strong>: Increase <code>for</code> duration in critical alerts.</p>
<p><strong>Example</strong> (before):
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-23-1"><a id="__codelineno-23-1" name="__codelineno-23-1" href="#__codelineno-23-1"></a><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">DiskSpaceLow</span>
</span><span id="__span-23-2"><a id="__codelineno-23-2" name="__codelineno-23-2" href="#__codelineno-23-2"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">disk_free_percent &lt; 15</span>
</span><span id="__span-23-3"><a id="__codelineno-23-3" name="__codelineno-23-3" href="#__codelineno-23-3"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1m</span><span class="w"> </span><span class="c1"># Too aggressive</span>
</span></code></pre></div></p>
<p><strong>Example</strong> (after):
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-24-1"><a id="__codelineno-24-1" name="__codelineno-24-1" href="#__codelineno-24-1"></a><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">DiskSpaceLow</span>
</span><span id="__span-24-2"><a id="__codelineno-24-2" name="__codelineno-24-2" href="#__codelineno-24-2"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">disk_free_percent &lt; 15</span>
</span><span id="__span-24-3"><a id="__codelineno-24-3" name="__codelineno-24-3" href="#__codelineno-24-3"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10m</span><span class="w"> </span><span class="c1"># More reasonable</span>
</span></code></pre></div></p>
<hr />
<h3 id="external-storage-long-term">External Storage (Long-Term)<a class="headerlink" href="#external-storage-long-term" title="Permanent link">&para;</a></h3>
<p><strong>Prometheus</strong> supports remote write to:
- <strong>Thanos</strong>: Long-term storage (S3/GCS)
- <strong>Cortex</strong>: Multi-tenant Prometheus
- <strong>VictoriaMetrics</strong>: High-performance storage</p>
<p><strong>Example</strong> (Thanos):
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-25-1"><a id="__codelineno-25-1" name="__codelineno-25-1" href="#__codelineno-25-1"></a><span class="c1"># prometheus.yml</span>
</span><span id="__span-25-2"><a id="__codelineno-25-2" name="__codelineno-25-2" href="#__codelineno-25-2"></a><span class="nt">remote_write</span><span class="p">:</span>
</span><span id="__span-25-3"><a id="__codelineno-25-3" name="__codelineno-25-3" href="#__codelineno-25-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;http://thanos-receive:19291/api/v1/receive&quot;</span>
</span></code></pre></div></p>
<hr />
<h2 id="related-documentation">Related Documentation<a class="headerlink" href="#related-documentation" title="Permanent link">&para;</a></h2>
<ul>
<li><strong><a href="../docker-compose/">Docker Compose</a></strong> — Monitoring services configuration</li>
<li><strong><a href="../environment-variables/">Environment Variables</a></strong> — Monitoring env vars</li>
<li><strong><a href="../api/metrics.md">API Reference</a></strong> — Custom metrics implementation</li>
</ul>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../tunneling/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Tunneling">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</div>
<div class="md-footer__title">
<span class="md-footer__direction">
Previous
</span>
<div class="md-ellipsis">
Tunneling
</div>
</div>
</a>
<a href="../healthchecks/" class="md-footer__link md-footer__link--next" aria-label="Next: Health Checks">
<div class="md-footer__title">
<span class="md-footer__direction">
Next
</span>
<div class="md-ellipsis">
Health Checks
</div>
</div>
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
</div>
</a>
</nav>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright &copy; 2024 The Bunker Operations – <a href="#__consent">Change cookie settings</a>
</div>
</div>
<div class="md-social">
<a href="https://gitea.bnkops.com/admin" target="_blank" rel="noopener" title="Gitea Repository" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
</a>
<a href="https://listmonk.bnkops.com/subscription/form" target="_blank" rel="noopener" title="Newsletter" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M536.4-26.3c9.8-3.5 20.6-1 28 6.3s9.8 18.2 6.3 28l-178 496.9c-5 13.9-18.1 23.1-32.8 23.1-14.2 0-27-8.6-32.3-21.7l-64.2-158c-4.5-11-2.5-23.6 5.2-32.6l94.5-112.4c5.1-6.1 4.7-15-.9-20.6s-14.6-6-20.6-.9l-112.4 94.3c-9.1 7.6-21.6 9.6-32.6 5.2L38.1 216.8c-13.1-5.3-21.7-18.1-21.7-32.3 0-14.7 9.2-27.8 23.1-32.8z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../../..", "features": ["announce.dismiss", "content.action.edit", "content.action.view", "content.code.annotate", "content.code.copy", "content.tooltips", "navigation.expand", "navigation.footer", "navigation.indexes", "navigation.path", "navigation.prune", "navigation.sections", "navigation.tabs", "navigation.tabs.sticky", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest", "toc.follow"], "search": "../../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../../assets/javascripts/bundle.79ae519e.min.js"></script>
<script src="../../../javascripts/home.js"></script>
<script src="../../../javascripts/github-widget.js"></script>
<script src="../../../javascripts/gitea-widget.js"></script>
<script src="../../../assets/js/env-config.js"></script>
<script src="../../../assets/js/video-player.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink