5471 lines
128 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Build Power. Not Rent It. Own your digital infrastructure.">
<meta name="author" content="Bunker Operations">
<link rel="canonical" href="https://bnkserve.org/v2/features/observability/">
<link rel="prev" href="../newsletter/">
<link rel="next" href="../tunnel/">
<link rel="icon" href="../../../assets/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.1">
<title>Observability & Monitoring - Changemaker Lite</title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Inter:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Inter";--md-code-font:"JetBrains Mono"}</style>
<link rel="stylesheet" href="../../../stylesheets/extra.css">
<link rel="stylesheet" href="../../../stylesheets/home.css">
<link rel="stylesheet" href="../../../assets/css/video-player.css">
<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
<meta property="og:type" content="website" />
<meta property="og:title" content="Observability &amp; Monitoring - Changemaker Lite" />
<meta property="og:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
<meta property="og:image" content="https://bnkserve.org/assets/images/social/v2/features/observability/index.png" />
<meta property="og:image:type" content="image/png" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="630" />
<meta property="og:url" content="https://bnkserve.org/v2/features/observability/" />
<meta property="twitter:card" content="summary_large_image" />
<meta property="twitter:title" content="Observability &amp; Monitoring - Changemaker Lite" />
<meta property="twitter:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
<meta property="twitter:image" content="https://bnkserve.org/assets/images/social/v2/features/observability/index.png" />
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#observability-monitoring" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../../.." title="Changemaker Lite" class="md-header__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
<img src="../../../assets/logo.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
Changemaker Lite
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Observability &amp; Monitoring
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m17.75 4.09-2.53 1.94.91 3.06-2.63-1.81-2.63 1.81.91-3.06-2.53-1.94L12.44 4l1.06-3 1.06 3zm3.5 6.91-1.64 1.25.59 1.98-1.7-1.17-1.7 1.17.59-1.98L15.75 11l2.06-.05L18.5 9l.69 1.95zm-2.28 4.95c.83-.08 1.72 1.1 1.19 1.85-.32.45-.66.87-1.08 1.27C15.17 23 8.84 23 4.94 19.07c-3.91-3.9-3.91-10.24 0-14.14.4-.4.82-.76 1.27-1.08.75-.53 1.93.36 1.85 1.19-.27 2.86.69 5.83 2.89 8.02a9.96 9.96 0 0 0 8.02 2.89m-1.64 2.02a12.08 12.08 0 0 1-7.8-3.47c-2.17-2.19-3.33-5-3.49-7.82-2.81 3.14-2.7 7.96.31 10.98 3.02 3.01 7.84 3.12 10.98.31"/></svg>
</label>
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 7a5 5 0 0 1 5 5 5 5 0 0 1-5 5 5 5 0 0 1-5-5 5 5 0 0 1 5-5m0 2a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0-7 2.39 3.42C13.65 5.15 12.84 5 12 5s-1.65.15-2.39.42zM3.34 7l4.16-.35A7.2 7.2 0 0 0 5.94 8.5c-.44.74-.69 1.5-.83 2.29zm.02 10 1.76-3.77a7.131 7.131 0 0 0 2.38 4.14zM20.65 7l-1.77 3.79a7.02 7.02 0 0 0-2.38-4.15zm-.01 10-4.14.36c.59-.51 1.12-1.14 1.54-1.86.42-.73.69-1.5.83-2.29zM12 22l-2.41-3.44c.74.27 1.55.44 2.41.44.82 0 1.63-.17 2.37-.44z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
changemaker.lite
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../../" class="md-tabs__link">
V2 Documentation
</a>
</li>
<li class="md-tabs__item">
<a href="../../../phil/" class="md-tabs__link">
Philosophy
</a>
</li>
<li class="md-tabs__item">
<a href="../../../v1/" class="md-tabs__link">
V1 Documentation (Legacy)
</a>
</li>
<li class="md-tabs__item">
<a href="../../../blog/" class="md-tabs__link">
Blog
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../.." title="Changemaker Lite" class="md-nav__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
<img src="../../../assets/logo.png" alt="logo">
</a>
Changemaker Lite
</label>
<div class="md-nav__source">
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
changemaker.lite
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
<div class="md-nav__link md-nav__container">
<a href="../../" class="md-nav__link ">
<span class="md-ellipsis">
V2 Documentation
</span>
</a>
<label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
V2 Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_2" >
<div class="md-nav__link md-nav__container">
<a href="../../getting-started/" class="md-nav__link ">
<span class="md-ellipsis">
Getting Started
</span>
</a>
<label class="md-nav__link " for="__nav_2_2" id="__nav_2_2_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_2">
<span class="md-nav__icon md-icon"></span>
Getting Started
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../getting-started/quick-start/" class="md-nav__link">
<span class="md-ellipsis">
Quick Start
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_3" >
<div class="md-nav__link md-nav__container">
<a href="../../architecture/" class="md-nav__link ">
<span class="md-ellipsis">
Architecture
</span>
</a>
<label class="md-nav__link " for="__nav_2_3" id="__nav_2_3_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_3">
<span class="md-nav__icon md-icon"></span>
Architecture
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../architecture/dual-api/" class="md-nav__link">
<span class="md-ellipsis">
Dual API System
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/authentication/" class="md-nav__link">
<span class="md-ellipsis">
Authentication & Security
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_4" >
<div class="md-nav__link md-nav__container">
<a href="../../backend/" class="md-nav__link ">
<span class="md-ellipsis">
Backend
</span>
</a>
<label class="md-nav__link " for="__nav_2_4" id="__nav_2_4_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_4">
<span class="md-nav__icon md-icon"></span>
Backend
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/modules/" class="md-nav__link">
<span class="md-ellipsis">
Modules
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/services/" class="md-nav__link">
<span class="md-ellipsis">
Services
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/middleware/" class="md-nav__link">
<span class="md-ellipsis">
Middleware
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../backend/utilities/" class="md-nav__link">
<span class="md-ellipsis">
Utilities
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_5" >
<div class="md-nav__link md-nav__container">
<a href="../../frontend/" class="md-nav__link ">
<span class="md-ellipsis">
Frontend
</span>
</a>
<label class="md-nav__link " for="__nav_2_5" id="__nav_2_5_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_5">
<span class="md-nav__icon md-icon"></span>
Frontend
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/components/" class="md-nav__link">
<span class="md-ellipsis">
Components
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/layouts/" class="md-nav__link">
<span class="md-ellipsis">
Layouts
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../frontend/pages/" class="md-nav__link">
<span class="md-ellipsis">
Pages
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_6" >
<div class="md-nav__link md-nav__container">
<a href="../../database/" class="md-nav__link ">
<span class="md-ellipsis">
Database
</span>
</a>
<label class="md-nav__link " for="__nav_2_6" id="__nav_2_6_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_6">
<span class="md-nav__icon md-icon"></span>
Database
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../database/schema/" class="md-nav__link">
<span class="md-ellipsis">
Schema Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/migrations/" class="md-nav__link">
<span class="md-ellipsis">
Migrations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/seeding/" class="md-nav__link">
<span class="md-ellipsis">
Seeding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../database/indexes/" class="md-nav__link">
<span class="md-ellipsis">
Indexes
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../database/models/" class="md-nav__link">
<span class="md-ellipsis">
Models
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_7" checked>
<div class="md-nav__link md-nav__container">
<a href="../" class="md-nav__link ">
<span class="md-ellipsis">
Features
</span>
</a>
<label class="md-nav__link " for="__nav_2_7" id="__nav_2_7_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_7_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2_7">
<span class="md-nav__icon md-icon"></span>
Features
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../influence/" class="md-nav__link">
<span class="md-ellipsis">
Influence
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../map/" class="md-nav__link">
<span class="md-ellipsis">
Map
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../landing-pages/" class="md-nav__link">
<span class="md-ellipsis">
Landing Pages
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../email-templates/" class="md-nav__link">
<span class="md-ellipsis">
Email Templates
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../media/" class="md-nav__link">
<span class="md-ellipsis">
Media
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../newsletter/" class="md-nav__link">
<span class="md-ellipsis">
Newsletter
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_7_8" checked>
<div class="md-nav__link md-nav__container">
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Observability
</span>
</a>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_2_7_8_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2_7_8">
<span class="md-nav__icon md-icon"></span>
Observability
</label>
<ul class="md-nav__list" data-md-scrollfix>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../tunnel/" class="md-nav__link">
<span class="md-ellipsis">
Tunnel
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_8" >
<div class="md-nav__link md-nav__container">
<a href="../../deployment/" class="md-nav__link ">
<span class="md-ellipsis">
Deployment
</span>
</a>
<label class="md-nav__link " for="__nav_2_8" id="__nav_2_8_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_8">
<span class="md-nav__icon md-icon"></span>
Deployment
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../deployment/docker-compose/" class="md-nav__link">
<span class="md-ellipsis">
Docker Compose
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/environment-variables/" class="md-nav__link">
<span class="md-ellipsis">
Environment Variables
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/nginx/" class="md-nav__link">
<span class="md-ellipsis">
Nginx Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/ssl-tls/" class="md-nav__link">
<span class="md-ellipsis">
SSL/TLS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/tunneling/" class="md-nav__link">
<span class="md-ellipsis">
Tunneling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/monitoring-stack/" class="md-nav__link">
<span class="md-ellipsis">
Monitoring Stack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/healthchecks/" class="md-nav__link">
<span class="md-ellipsis">
Health Checks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/scaling/" class="md-nav__link">
<span class="md-ellipsis">
Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../deployment/backup-restore/" class="md-nav__link">
<span class="md-ellipsis">
Backup & Restore
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_9" >
<div class="md-nav__link md-nav__container">
<a href="../../development/" class="md-nav__link ">
<span class="md-ellipsis">
Development
</span>
</a>
<label class="md-nav__link " for="__nav_2_9" id="__nav_2_9_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_9">
<span class="md-nav__icon md-icon"></span>
Development
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../development/local-setup/" class="md-nav__link">
<span class="md-ellipsis">
Local Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/docker-workflow/" class="md-nav__link">
<span class="md-ellipsis">
Docker Workflow
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/git-workflow/" class="md-nav__link">
<span class="md-ellipsis">
Git Workflow
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/npm-commands/" class="md-nav__link">
<span class="md-ellipsis">
NPM Commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/migrations/" class="md-nav__link">
<span class="md-ellipsis">
Migrations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/typescript/" class="md-nav__link">
<span class="md-ellipsis">
TypeScript
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/testing/" class="md-nav__link">
<span class="md-ellipsis">
Testing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/debugging/" class="md-nav__link">
<span class="md-ellipsis">
Debugging
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../development/code-style/" class="md-nav__link">
<span class="md-ellipsis">
Code Style
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_10" >
<div class="md-nav__link md-nav__container">
<a href="../../api-reference/" class="md-nav__link ">
<span class="md-ellipsis">
API Reference
</span>
</a>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_10">
<span class="md-nav__icon md-icon"></span>
API Reference
</label>
<ul class="md-nav__list" data-md-scrollfix>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
<div class="md-nav__link md-nav__container">
<a href="../../user-guides/" class="md-nav__link ">
<span class="md-ellipsis">
User Guides
</span>
</a>
<label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_11">
<span class="md-nav__icon md-icon"></span>
User Guides
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../user-guides/admin-guide/" class="md-nav__link">
<span class="md-ellipsis">
Admin Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/campaign-manager-guide/" class="md-nav__link">
<span class="md-ellipsis">
Campaign Manager Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/map-organizer-guide/" class="md-nav__link">
<span class="md-ellipsis">
Map Organizer Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/content-editor-guide/" class="md-nav__link">
<span class="md-ellipsis">
Content Editor Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../user-guides/volunteer-guide/" class="md-nav__link">
<span class="md-ellipsis">
Volunteer Guide
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
<div class="md-nav__link md-nav__container">
<a href="../../troubleshooting/" class="md-nav__link ">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_12">
<span class="md-nav__icon md-icon"></span>
Troubleshooting
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../troubleshooting/faq/" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/common-errors/" class="md-nav__link">
<span class="md-ellipsis">
Common Errors
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/auth-issues/" class="md-nav__link">
<span class="md-ellipsis">
Auth Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/database-issues/" class="md-nav__link">
<span class="md-ellipsis">
Database Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/docker-issues/" class="md-nav__link">
<span class="md-ellipsis">
Docker Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/email-issues/" class="md-nav__link">
<span class="md-ellipsis">
Email Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/geocoding-issues/" class="md-nav__link">
<span class="md-ellipsis">
Geocoding Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/monitoring-issues/" class="md-nav__link">
<span class="md-ellipsis">
Monitoring Issues
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshooting/performance-optimization/" class="md-nav__link">
<span class="md-ellipsis">
Performance Optimization
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
<div class="md-nav__link md-nav__container">
<a href="../../migration/" class="md-nav__link ">
<span class="md-ellipsis">
Migration
</span>
</a>
<label class="md-nav__link " for="__nav_2_13" id="__nav_2_13_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_13">
<span class="md-nav__icon md-icon"></span>
Migration
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../migration/feature-parity/" class="md-nav__link">
<span class="md-ellipsis">
Feature Parity
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/breaking-changes/" class="md-nav__link">
<span class="md-ellipsis">
Breaking Changes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/api-changes/" class="md-nav__link">
<span class="md-ellipsis">
API Changes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../migration/data-migration/" class="md-nav__link">
<span class="md-ellipsis">
Data Migration
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
<div class="md-nav__link md-nav__container">
<a href="../../contributing/" class="md-nav__link ">
<span class="md-ellipsis">
Contributing
</span>
</a>
<label class="md-nav__link " for="__nav_2_14" id="__nav_2_14_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_14">
<span class="md-nav__icon md-icon"></span>
Contributing
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../contributing/development-setup/" class="md-nav__link">
<span class="md-ellipsis">
Development Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/code-of-conduct/" class="md-nav__link">
<span class="md-ellipsis">
Code of Conduct
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/pull-requests/" class="md-nav__link">
<span class="md-ellipsis">
Pull Requests
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../contributing/roadmap/" class="md-nav__link">
<span class="md-ellipsis">
Roadmap
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../phil/" class="md-nav__link">
<span class="md-ellipsis">
Philosophy
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../v1/" class="md-nav__link">
<span class="md-ellipsis">
V1 Documentation (Legacy)
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
<a href="../../../blog/" class="md-nav__link">
<span class="md-ellipsis">
Blog
</span>
<span class="md-nav__icon md-icon"></span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="On this page">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
On this page
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#features" class="md-nav__link">
<span class="md-ellipsis">
Features
</span>
</a>
<nav class="md-nav" aria-label="Features">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#metrics-collection" class="md-nav__link">
<span class="md-ellipsis">
Metrics Collection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-rules" class="md-nav__link">
<span class="md-ellipsis">
Alert Rules
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#admin-interface" class="md-nav__link">
<span class="md-ellipsis">
Admin Interface
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
<nav class="md-nav" aria-label="Architecture">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#backend-components" class="md-nav__link">
<span class="md-ellipsis">
Backend Components
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#frontend-components" class="md-nav__link">
<span class="md-ellipsis">
Frontend Components
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docker-services" class="md-nav__link">
<span class="md-ellipsis">
Docker Services
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#configuration" class="md-nav__link">
<span class="md-ellipsis">
Configuration
</span>
</a>
<nav class="md-nav" aria-label="Configuration">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#environment-variables" class="md-nav__link">
<span class="md-ellipsis">
Environment Variables
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#prometheus-scrape-targets" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Scrape Targets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alert-rules_1" class="md-nav__link">
<span class="md-ellipsis">
Alert Rules
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#metrics-usage" class="md-nav__link">
<span class="md-ellipsis">
Metrics Usage
</span>
</a>
<nav class="md-nav" aria-label="Metrics Usage">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#increment-counter" class="md-nav__link">
<span class="md-ellipsis">
Increment Counter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#set-gauge" class="md-nav__link">
<span class="md-ellipsis">
Set Gauge
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#observe-histogram" class="md-nav__link">
<span class="md-ellipsis">
Observe Histogram
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards_1" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards
</span>
</a>
<nav class="md-nav" aria-label="Grafana Dashboards">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dashboard-setup" class="md-nav__link">
<span class="md-ellipsis">
Dashboard Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#accessing-dashboards" class="md-nav__link">
<span class="md-ellipsis">
Accessing Dashboards
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#alertmanager" class="md-nav__link">
<span class="md-ellipsis">
Alertmanager
</span>
</a>
<nav class="md-nav" aria-label="Alertmanager">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#alert-routing" class="md-nav__link">
<span class="md-ellipsis">
Alert Routing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#notification-channels" class="md-nav__link">
<span class="md-ellipsis">
Notification Channels
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#service-health-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Service Health Monitoring
</span>
</a>
<nav class="md-nav" aria-label="Service Health Monitoring">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#external-service-checks" class="md-nav__link">
<span class="md-ellipsis">
External Service Checks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docker-healthchecks" class="md-nav__link">
<span class="md-ellipsis">
Docker Healthchecks
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#performance-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Performance Monitoring
</span>
</a>
<nav class="md-nav" aria-label="Performance Monitoring">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#http-request-tracking" class="md-nav__link">
<span class="md-ellipsis">
HTTP Request Tracking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#queue-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Queue Monitoring
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#resource-monitoring" class="md-nav__link">
<span class="md-ellipsis">
Resource Monitoring
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#admin-interface_1" class="md-nav__link">
<span class="md-ellipsis">
Admin Interface
</span>
</a>
<nav class="md-nav" aria-label="Admin Interface">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#metrics-tab" class="md-nav__link">
<span class="md-ellipsis">
Metrics Tab
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dashboards-tab" class="md-nav__link">
<span class="md-ellipsis">
Dashboards Tab
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#alerts-tab" class="md-nav__link">
<span class="md-ellipsis">
Alerts Tab
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#starting-monitoring-stack" class="md-nav__link">
<span class="md-ellipsis">
Starting Monitoring Stack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#api-endpoints" class="md-nav__link">
<span class="md-ellipsis">
API Endpoints
</span>
</a>
<nav class="md-nav" aria-label="API Endpoints">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#observability-endpoints" class="md-nav__link">
<span class="md-ellipsis">
Observability Endpoints
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#metrics-endpoint" class="md-nav__link">
<span class="md-ellipsis">
Metrics Endpoint
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#related-documentation" class="md-nav__link">
<span class="md-ellipsis">
Related Documentation
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<nav class="md-path" aria-label="Navigation" >
<ol class="md-path__list">
<li class="md-path__item">
<a href="../../.." class="md-path__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-path__item">
<a href="../../" class="md-path__link">
<span class="md-ellipsis">
V2 Documentation
</span>
</a>
</li>
<li class="md-path__item">
<a href="../" class="md-path__link">
<span class="md-ellipsis">
Features
</span>
</a>
</li>
<li class="md-path__item">
<a href="./" class="md-path__link">
<span class="md-ellipsis">
Observability
</span>
</a>
</li>
</ol>
</nav>
<article class="md-content__inner md-typeset">
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/features/observability/index.md" title="Edit this page" class="md-content__button md-icon" rel="edit">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"/></svg>
</a>
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/features/observability/index.md" title="View source of this page" class="md-content__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 18c.56 0 1 .44 1 1s-.44 1-1 1-1-.44-1-1 .44-1 1-1m0-3c-2.73 0-5.06 1.66-6 4 .94 2.34 3.27 4 6 4s5.06-1.66 6-4c-.94-2.34-3.27-4-6-4m0 6.5a2.5 2.5 0 0 1-2.5-2.5 2.5 2.5 0 0 1 2.5-2.5 2.5 2.5 0 0 1 2.5 2.5 2.5 2.5 0 0 1-2.5 2.5M9.27 20H6V4h7v5h5v4.07c.7.08 1.36.25 2 .49V8l-6-6H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h4.5a8.2 8.2 0 0 1-1.23-2"/></svg>
</a>
<h1 id="observability-monitoring">Observability &amp; Monitoring<a class="headerlink" href="#observability-monitoring" title="Permanent link">&para;</a></h1>
<p>The Observability feature provides comprehensive monitoring, metrics collection, and alerting for the Changemaker Lite platform. Built on the Prometheus ecosystem with Grafana dashboards and Alertmanager integration.</p>
<h2 id="overview">Overview<a class="headerlink" href="#overview" title="Permanent link">&para;</a></h2>
<p>The Observability stack consists of:</p>
<ol>
<li><strong>Prometheus</strong> - Metrics collection and storage</li>
<li><strong>Grafana</strong> - Visualization dashboards</li>
<li><strong>Alertmanager</strong> - Alert routing and notifications</li>
<li><strong>Custom Metrics</strong> - 12 domain-specific <code>cm_*</code> metrics</li>
<li><strong>HTTP Metrics</strong> - Request tracking and performance</li>
<li><strong>Service Health</strong> - External service monitoring</li>
</ol>
<h2 id="features">Features<a class="headerlink" href="#features" title="Permanent link">&para;</a></h2>
<h3 id="metrics-collection">Metrics Collection<a class="headerlink" href="#metrics-collection" title="Permanent link">&para;</a></h3>
<p><strong>Custom Domain Metrics (12 total):</strong></p>
<p><strong>Counters:</strong>
- <code>cm_api_uptime_seconds</code> - API uptime counter
- <code>cm_canvass_visits_total</code> - Total canvass visits
- <code>cm_campaign_emails_sent_total</code> - Total campaign emails sent
- <code>cm_geocode_requests_total</code> - Total geocode requests</p>
<p><strong>Gauges:</strong>
- <code>cm_canvass_sessions_active</code> - Active canvass sessions
- <code>cm_email_queue_size</code> - Email queue depth
- <code>cm_geocode_queue_size</code> - Geocode queue depth
- <code>cm_external_service_health</code> - Service health (0/1)</p>
<p><strong>Histograms:</strong>
- <code>cm_geocode_duration_seconds</code> - Geocoding latency
- <code>http_request_duration_ms</code> - HTTP request duration</p>
<p><strong>HTTP Metrics:</strong>
- Request count by method/route/status
- Request duration percentiles (p50, p95, p99)
- Active requests gauge
- Error rate tracking</p>
<h3 id="grafana-dashboards">Grafana Dashboards<a class="headerlink" href="#grafana-dashboards" title="Permanent link">&para;</a></h3>
<p>Three pre-configured dashboards:</p>
<ol>
<li><strong>Changemaker Lite Overview</strong> - System-wide metrics</li>
<li>API uptime and request rates</li>
<li>Queue sizes and health</li>
<li>Active sessions</li>
<li>
<p>Error rates</p>
</li>
<li>
<p><strong>Canvassing Metrics</strong> - Canvass-specific metrics</p>
</li>
<li>Active sessions over time</li>
<li>Visits by outcome</li>
<li>Session duration</li>
<li>
<p>Volunteer leaderboard</p>
</li>
<li>
<p><strong>External Services</strong> - Integration health</p>
</li>
<li>Redis health</li>
<li>PostgreSQL health</li>
<li>Listmonk status</li>
<li>Geocoding providers</li>
</ol>
<h3 id="alert-rules">Alert Rules<a class="headerlink" href="#alert-rules" title="Permanent link">&para;</a></h3>
<p>12 predefined alert rules:</p>
<p><strong>Critical Alerts:</strong>
- API down (&gt;5 min)
- Database unreachable
- Redis connection lost</p>
<p><strong>Warning Alerts:</strong>
- High error rate (&gt;5%)
- Queue backup (&gt;1000 jobs)
- Slow requests (p95 &gt;2s)
- Service degradation</p>
<p><strong>Info Alerts:</strong>
- New deployment
- Service restart
- Configuration change</p>
<h3 id="admin-interface">Admin Interface<a class="headerlink" href="#admin-interface" title="Permanent link">&para;</a></h3>
<p>Observability page (<code>/app/observability</code>) with:</p>
<ul>
<li><strong>Metrics Tab</strong> - Live metrics display</li>
<li><strong>Dashboards Tab</strong> - Embedded Grafana</li>
<li><strong>Alerts Tab</strong> - Active alerts and rules</li>
</ul>
<h2 id="architecture">Architecture<a class="headerlink" href="#architecture" title="Permanent link">&para;</a></h2>
<h3 id="backend-components">Backend Components<a class="headerlink" href="#backend-components" title="Permanent link">&para;</a></h3>
<p><strong>Metrics Module:</strong>
- <code>api/src/utils/metrics.ts</code> - Prometheus metrics definitions
- <code>api/src/modules/observability/observability.routes.ts</code> - Admin API</p>
<p><strong>Instrumentation:</strong>
- Express middleware for HTTP metrics
- Service-level metric updates
- Queue size tracking
- External service health checks</p>
<p><strong>Configuration:</strong>
- <code>configs/prometheus/prometheus.yml</code> - Scrape config
- <code>configs/prometheus/alerts.yml</code> - Alert rules
- <code>configs/grafana/dashboards/</code> - Dashboard JSON</p>
<h3 id="frontend-components">Frontend Components<a class="headerlink" href="#frontend-components" title="Permanent link">&para;</a></h3>
<p><strong>Admin Page:</strong>
- <code>admin/src/pages/ObservabilityPage.tsx</code> - Monitoring dashboard
- Three tabs: Metrics, Dashboards, Alerts
- Embedded Grafana iframes
- Live metric cards</p>
<p><strong>Observability Components:</strong>
- <code>admin/src/components/observability/MetricsChart.tsx</code> - Chart component
- <code>admin/src/components/observability/ServiceHealthCard.tsx</code> - Health display</p>
<h3 id="docker-services">Docker Services<a class="headerlink" href="#docker-services" title="Permanent link">&para;</a></h3>
<p><strong>Monitoring Profile:</strong></p>
<p>Services run with <code>--profile monitoring</code>:</p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nt">profiles</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">monitoring</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="w"> </span><span class="nt">prometheus</span><span class="p">:</span>
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prom/prometheus:latest</span>
</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;9090:9090&quot;</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a>
</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="w"> </span><span class="nt">grafana</span><span class="p">:</span>
</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">grafana/grafana:latest</span>
</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;3001:3000&quot;</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a>
</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="w"> </span><span class="nt">alertmanager</span><span class="p">:</span>
</span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prom/alertmanager:latest</span>
</span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12" href="#__codelineno-0-12"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;9093:9093&quot;</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13" href="#__codelineno-0-13"></a>
</span><span id="__span-0-14"><a id="__codelineno-0-14" name="__codelineno-0-14" href="#__codelineno-0-14"></a><span class="w"> </span><span class="nt">cadvisor</span><span class="p">:</span>
</span><span id="__span-0-15"><a id="__codelineno-0-15" name="__codelineno-0-15" href="#__codelineno-0-15"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gcr.io/cadvisor/cadvisor:latest</span>
</span><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16" href="#__codelineno-0-16"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;8080:8080&quot;</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-17"><a id="__codelineno-0-17" name="__codelineno-0-17" href="#__codelineno-0-17"></a>
</span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18" href="#__codelineno-0-18"></a><span class="w"> </span><span class="nt">node-exporter</span><span class="p">:</span>
</span><span id="__span-0-19"><a id="__codelineno-0-19" name="__codelineno-0-19" href="#__codelineno-0-19"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prom/node-exporter:latest</span>
</span><span id="__span-0-20"><a id="__codelineno-0-20" name="__codelineno-0-20" href="#__codelineno-0-20"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;9100:9100&quot;</span><span class="p p-Indicator">]</span>
</span><span id="__span-0-21"><a id="__codelineno-0-21" name="__codelineno-0-21" href="#__codelineno-0-21"></a>
</span><span id="__span-0-22"><a id="__codelineno-0-22" name="__codelineno-0-22" href="#__codelineno-0-22"></a><span class="w"> </span><span class="nt">redis-exporter</span><span class="p">:</span>
</span><span id="__span-0-23"><a id="__codelineno-0-23" name="__codelineno-0-23" href="#__codelineno-0-23"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">oliver006/redis_exporter:latest</span>
</span><span id="__span-0-24"><a id="__codelineno-0-24" name="__codelineno-0-24" href="#__codelineno-0-24"></a><span class="w"> </span><span class="nt">ports</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;9121:9121&quot;</span><span class="p p-Indicator">]</span>
</span></code></pre></div>
<h2 id="configuration">Configuration<a class="headerlink" href="#configuration" title="Permanent link">&para;</a></h2>
<h3 id="environment-variables">Environment Variables<a class="headerlink" href="#environment-variables" title="Permanent link">&para;</a></h3>
<div class="language-bash highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="c1"># Enable metrics</span>
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="nv">METRICS_ENABLED</span><span class="o">=</span><span class="nb">true</span>
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="c1"># Prometheus</span>
</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="nv">PROMETHEUS_PORT</span><span class="o">=</span><span class="m">9090</span>
</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a>
</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="c1"># Grafana</span>
</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="nv">GRAFANA_PORT</span><span class="o">=</span><span class="m">3001</span>
</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a><span class="nv">GRAFANA_ADMIN_USER</span><span class="o">=</span>admin
</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="nv">GRAFANA_ADMIN_PASSWORD</span><span class="o">=</span>admin
</span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a>
</span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="c1"># Alertmanager</span>
</span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="nv">ALERTMANAGER_PORT</span><span class="o">=</span><span class="m">9093</span>
</span></code></pre></div>
<h3 id="prometheus-scrape-targets">Prometheus Scrape Targets<a class="headerlink" href="#prometheus-scrape-targets" title="Permanent link">&para;</a></h3>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;changemaker-api&#39;</span>
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;api:4000&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a>
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;media-api&#39;</span>
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;media-api:4100&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a>
</span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;redis&#39;</span>
</span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;redis-exporter:9121&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-2-13"><a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a>
</span><span id="__span-2-14"><a id="__codelineno-2-14" name="__codelineno-2-14" href="#__codelineno-2-14"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;node&#39;</span>
</span><span id="__span-2-15"><a id="__codelineno-2-15" name="__codelineno-2-15" href="#__codelineno-2-15"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-2-16"><a id="__codelineno-2-16" name="__codelineno-2-16" href="#__codelineno-2-16"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;node-exporter:9100&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-2-17"><a id="__codelineno-2-17" name="__codelineno-2-17" href="#__codelineno-2-17"></a>
</span><span id="__span-2-18"><a id="__codelineno-2-18" name="__codelineno-2-18" href="#__codelineno-2-18"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;cadvisor&#39;</span>
</span><span id="__span-2-19"><a id="__codelineno-2-19" name="__codelineno-2-19" href="#__codelineno-2-19"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
</span><span id="__span-2-20"><a id="__codelineno-2-20" name="__codelineno-2-20" href="#__codelineno-2-20"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;cadvisor:8080&#39;</span><span class="p p-Indicator">]</span>
</span></code></pre></div>
<h3 id="alert-rules_1">Alert Rules<a class="headerlink" href="#alert-rules_1" title="Permanent link">&para;</a></h3>
<p>Example alert rule:</p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="nt">groups</span><span class="p">:</span>
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">api_alerts</span>
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="w"> </span><span class="nt">rules</span><span class="p">:</span>
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">APIDown</span>
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">up{job=&quot;changemaker-api&quot;} == 0</span>
</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5m</span>
</span><span id="__span-3-7"><a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a><span class="w"> </span><span class="nt">labels</span><span class="p">:</span>
</span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
</span><span id="__span-3-9"><a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a><span class="w"> </span><span class="nt">annotations</span><span class="p">:</span>
</span><span id="__span-3-10"><a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a><span class="w"> </span><span class="nt">summary</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;API</span><span class="nv"> </span><span class="s">is</span><span class="nv"> </span><span class="s">down&quot;</span>
</span><span id="__span-3-11"><a id="__codelineno-3-11" name="__codelineno-3-11" href="#__codelineno-3-11"></a><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;API</span><span class="nv"> </span><span class="s">has</span><span class="nv"> </span><span class="s">been</span><span class="nv"> </span><span class="s">down</span><span class="nv"> </span><span class="s">for</span><span class="nv"> </span><span class="s">5</span><span class="nv"> </span><span class="s">minutes&quot;</span>
</span><span id="__span-3-12"><a id="__codelineno-3-12" name="__codelineno-3-12" href="#__codelineno-3-12"></a>
</span><span id="__span-3-13"><a id="__codelineno-3-13" name="__codelineno-3-13" href="#__codelineno-3-13"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">HighErrorRate</span>
</span><span id="__span-3-14"><a id="__codelineno-3-14" name="__codelineno-3-14" href="#__codelineno-3-14"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rate(http_request_duration_ms_count{status=~&quot;5..&quot;}[5m]) &gt; 0.05</span>
</span><span id="__span-3-15"><a id="__codelineno-3-15" name="__codelineno-3-15" href="#__codelineno-3-15"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10m</span>
</span><span id="__span-3-16"><a id="__codelineno-3-16" name="__codelineno-3-16" href="#__codelineno-3-16"></a><span class="w"> </span><span class="nt">labels</span><span class="p">:</span>
</span><span id="__span-3-17"><a id="__codelineno-3-17" name="__codelineno-3-17" href="#__codelineno-3-17"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">warning</span>
</span><span id="__span-3-18"><a id="__codelineno-3-18" name="__codelineno-3-18" href="#__codelineno-3-18"></a><span class="w"> </span><span class="nt">annotations</span><span class="p">:</span>
</span><span id="__span-3-19"><a id="__codelineno-3-19" name="__codelineno-3-19" href="#__codelineno-3-19"></a><span class="w"> </span><span class="nt">summary</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;High</span><span class="nv"> </span><span class="s">error</span><span class="nv"> </span><span class="s">rate</span><span class="nv"> </span><span class="s">detected&quot;</span>
</span></code></pre></div>
<h2 id="metrics-usage">Metrics Usage<a class="headerlink" href="#metrics-usage" title="Permanent link">&para;</a></h2>
<h3 id="increment-counter">Increment Counter<a class="headerlink" href="#increment-counter" title="Permanent link">&para;</a></h3>
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="k">import</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nx">metrics</span><span class="w"> </span><span class="p">}</span><span class="w"> </span><span class="kr">from</span><span class="w"> </span><span class="s1">&#39;../utils/metrics&#39;</span><span class="p">;</span>
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a>
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="c1">// Campaign email sent</span>
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="nx">metrics</span><span class="p">.</span><span class="nx">campaignEmailsSent</span><span class="p">.</span><span class="nx">inc</span><span class="p">();</span>
</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a>
</span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a><span class="c1">// Geocode request</span>
</span><span id="__span-4-7"><a id="__codelineno-4-7" name="__codelineno-4-7" href="#__codelineno-4-7"></a><span class="nx">metrics</span><span class="p">.</span><span class="nx">geocodeRequests</span><span class="p">.</span><span class="nx">inc</span><span class="p">({</span><span class="w"> </span><span class="nx">provider</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;nominatim&#39;</span><span class="w"> </span><span class="p">});</span>
</span></code></pre></div>
<h3 id="set-gauge">Set Gauge<a class="headerlink" href="#set-gauge" title="Permanent link">&para;</a></h3>
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="c1">// Update queue size</span>
</span><span id="__span-5-2"><a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a><span class="nx">metrics</span><span class="p">.</span><span class="nx">emailQueueSize</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">queueSize</span><span class="p">);</span>
</span><span id="__span-5-3"><a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a>
</span><span id="__span-5-4"><a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a><span class="c1">// Update active sessions</span>
</span><span id="__span-5-5"><a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a><span class="nx">metrics</span><span class="p">.</span><span class="nx">canvassSessionsActive</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">activeSessions</span><span class="p">);</span>
</span><span id="__span-5-6"><a id="__codelineno-5-6" name="__codelineno-5-6" href="#__codelineno-5-6"></a>
</span><span id="__span-5-7"><a id="__codelineno-5-7" name="__codelineno-5-7" href="#__codelineno-5-7"></a><span class="c1">// Set service health (1 = healthy, 0 = unhealthy)</span>
</span><span id="__span-5-8"><a id="__codelineno-5-8" name="__codelineno-5-8" href="#__codelineno-5-8"></a><span class="nx">metrics</span><span class="p">.</span><span class="nx">externalServiceHealth</span><span class="p">.</span><span class="nx">set</span><span class="p">({</span><span class="w"> </span><span class="nx">service</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;redis&#39;</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="mf">1</span><span class="p">);</span>
</span></code></pre></div>
<h3 id="observe-histogram">Observe Histogram<a class="headerlink" href="#observe-histogram" title="Permanent link">&para;</a></h3>
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="c1">// Time geocoding request</span>
</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">end</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nx">metrics</span><span class="p">.</span><span class="nx">geocodeDuration</span><span class="p">.</span><span class="nx">startTimer</span><span class="p">();</span>
</span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a><span class="k">try</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">geocode</span><span class="p">(</span><span class="nx">address</span><span class="p">);</span>
</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5" href="#__codelineno-6-5"></a><span class="w"> </span><span class="nx">end</span><span class="p">({</span><span class="w"> </span><span class="nx">success</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;true&#39;</span><span class="w"> </span><span class="p">});</span>
</span><span id="__span-6-6"><a id="__codelineno-6-6" name="__codelineno-6-6" href="#__codelineno-6-6"></a><span class="p">}</span><span class="w"> </span><span class="k">catch</span><span class="w"> </span><span class="p">(</span><span class="nx">error</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-6-7"><a id="__codelineno-6-7" name="__codelineno-6-7" href="#__codelineno-6-7"></a><span class="w"> </span><span class="nx">end</span><span class="p">({</span><span class="w"> </span><span class="nx">success</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;false&#39;</span><span class="w"> </span><span class="p">});</span>
</span><span id="__span-6-8"><a id="__codelineno-6-8" name="__codelineno-6-8" href="#__codelineno-6-8"></a><span class="p">}</span>
</span></code></pre></div>
<h2 id="grafana-dashboards_1">Grafana Dashboards<a class="headerlink" href="#grafana-dashboards_1" title="Permanent link">&para;</a></h2>
<h3 id="dashboard-setup">Dashboard Setup<a class="headerlink" href="#dashboard-setup" title="Permanent link">&para;</a></h3>
<p>Dashboards auto-provisioned from <code>configs/grafana/dashboards/</code>:</p>
<div class="language-json highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a><span class="p">{</span>
</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a><span class="w"> </span><span class="nt">&quot;dashboard&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a><span class="w"> </span><span class="nt">&quot;title&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Changemaker Lite Overview&quot;</span><span class="p">,</span>
</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a><span class="w"> </span><span class="nt">&quot;panels&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5" href="#__codelineno-7-5"></a><span class="w"> </span><span class="p">{</span>
</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6" href="#__codelineno-7-6"></a><span class="w"> </span><span class="nt">&quot;title&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;API Request Rate&quot;</span><span class="p">,</span>
</span><span id="__span-7-7"><a id="__codelineno-7-7" name="__codelineno-7-7" href="#__codelineno-7-7"></a><span class="w"> </span><span class="nt">&quot;targets&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
</span><span id="__span-7-8"><a id="__codelineno-7-8" name="__codelineno-7-8" href="#__codelineno-7-8"></a><span class="w"> </span><span class="p">{</span>
</span><span id="__span-7-9"><a id="__codelineno-7-9" name="__codelineno-7-9" href="#__codelineno-7-9"></a><span class="w"> </span><span class="nt">&quot;expr&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;rate(http_request_duration_ms_count[5m])&quot;</span>
</span><span id="__span-7-10"><a id="__codelineno-7-10" name="__codelineno-7-10" href="#__codelineno-7-10"></a><span class="w"> </span><span class="p">}</span>
</span><span id="__span-7-11"><a id="__codelineno-7-11" name="__codelineno-7-11" href="#__codelineno-7-11"></a><span class="w"> </span><span class="p">]</span>
</span><span id="__span-7-12"><a id="__codelineno-7-12" name="__codelineno-7-12" href="#__codelineno-7-12"></a><span class="w"> </span><span class="p">}</span>
</span><span id="__span-7-13"><a id="__codelineno-7-13" name="__codelineno-7-13" href="#__codelineno-7-13"></a><span class="w"> </span><span class="p">]</span>
</span><span id="__span-7-14"><a id="__codelineno-7-14" name="__codelineno-7-14" href="#__codelineno-7-14"></a><span class="w"> </span><span class="p">}</span>
</span><span id="__span-7-15"><a id="__codelineno-7-15" name="__codelineno-7-15" href="#__codelineno-7-15"></a><span class="p">}</span>
</span></code></pre></div>
<h3 id="accessing-dashboards">Accessing Dashboards<a class="headerlink" href="#accessing-dashboards" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Direct:</strong> http://localhost:3001 (admin/admin)</li>
<li><strong>Embedded:</strong> <code>/app/observability</code> → Dashboards tab</li>
<li><strong>Subdomain:</strong> http://grafana.cmlite.org (production)</li>
</ul>
<h2 id="alertmanager">Alertmanager<a class="headerlink" href="#alertmanager" title="Permanent link">&para;</a></h2>
<h3 id="alert-routing">Alert Routing<a class="headerlink" href="#alert-routing" title="Permanent link">&para;</a></h3>
<p>Configure in <code>configs/alertmanager/alertmanager.yml</code>:</p>
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="nt">route</span><span class="p">:</span>
</span><span id="__span-8-2"><a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;default&#39;</span>
</span><span id="__span-8-3"><a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a><span class="w"> </span><span class="nt">group_by</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&#39;alertname&#39;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&#39;severity&#39;</span><span class="p p-Indicator">]</span>
</span><span id="__span-8-4"><a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a><span class="w"> </span><span class="nt">routes</span><span class="p">:</span>
</span><span id="__span-8-5"><a id="__codelineno-8-5" name="__codelineno-8-5" href="#__codelineno-8-5"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">match</span><span class="p">:</span>
</span><span id="__span-8-6"><a id="__codelineno-8-6" name="__codelineno-8-6" href="#__codelineno-8-6"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
</span><span id="__span-8-7"><a id="__codelineno-8-7" name="__codelineno-8-7" href="#__codelineno-8-7"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;critical-alerts&#39;</span>
</span><span id="__span-8-8"><a id="__codelineno-8-8" name="__codelineno-8-8" href="#__codelineno-8-8"></a>
</span><span id="__span-8-9"><a id="__codelineno-8-9" name="__codelineno-8-9" href="#__codelineno-8-9"></a><span class="nt">receivers</span><span class="p">:</span>
</span><span id="__span-8-10"><a id="__codelineno-8-10" name="__codelineno-8-10" href="#__codelineno-8-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;default&#39;</span>
</span><span id="__span-8-11"><a id="__codelineno-8-11" name="__codelineno-8-11" href="#__codelineno-8-11"></a><span class="w"> </span><span class="nt">webhook_configs</span><span class="p">:</span>
</span><span id="__span-8-12"><a id="__codelineno-8-12" name="__codelineno-8-12" href="#__codelineno-8-12"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;http://gotify:8889/message&#39;</span>
</span><span id="__span-8-13"><a id="__codelineno-8-13" name="__codelineno-8-13" href="#__codelineno-8-13"></a>
</span><span id="__span-8-14"><a id="__codelineno-8-14" name="__codelineno-8-14" href="#__codelineno-8-14"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;critical-alerts&#39;</span>
</span><span id="__span-8-15"><a id="__codelineno-8-15" name="__codelineno-8-15" href="#__codelineno-8-15"></a><span class="w"> </span><span class="nt">email_configs</span><span class="p">:</span>
</span><span id="__span-8-16"><a id="__codelineno-8-16" name="__codelineno-8-16" href="#__codelineno-8-16"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">to</span><span class="p">:</span><span class="w"> </span><span class="s">&#39;admin@example.com&#39;</span>
</span></code></pre></div>
<h3 id="notification-channels">Notification Channels<a class="headerlink" href="#notification-channels" title="Permanent link">&para;</a></h3>
<p>Supported receivers:</p>
<ul>
<li><strong>Webhook</strong> - Gotify, Slack, Discord</li>
<li><strong>Email</strong> - SMTP notifications</li>
<li><strong>PagerDuty</strong> - Incident management</li>
<li><strong>Opsgenie</strong> - Alert management</li>
</ul>
<h2 id="service-health-monitoring">Service Health Monitoring<a class="headerlink" href="#service-health-monitoring" title="Permanent link">&para;</a></h2>
<h3 id="external-service-checks">External Service Checks<a class="headerlink" href="#external-service-checks" title="Permanent link">&para;</a></h3>
<p>Monitor services via health gauges:</p>
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="c1">// Check Redis</span>
</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a><span class="k">try</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">redisClient</span><span class="p">.</span><span class="nx">ping</span><span class="p">();</span>
</span><span id="__span-9-4"><a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="w"> </span><span class="nx">metrics</span><span class="p">.</span><span class="nx">externalServiceHealth</span><span class="p">.</span><span class="nx">set</span><span class="p">({</span><span class="w"> </span><span class="nx">service</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;redis&#39;</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="mf">1</span><span class="p">);</span>
</span><span id="__span-9-5"><a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a><span class="p">}</span><span class="w"> </span><span class="k">catch</span><span class="w"> </span><span class="p">(</span><span class="nx">error</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-9-6"><a id="__codelineno-9-6" name="__codelineno-9-6" href="#__codelineno-9-6"></a><span class="w"> </span><span class="nx">metrics</span><span class="p">.</span><span class="nx">externalServiceHealth</span><span class="p">.</span><span class="nx">set</span><span class="p">({</span><span class="w"> </span><span class="nx">service</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;redis&#39;</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="mf">0</span><span class="p">);</span>
</span><span id="__span-9-7"><a id="__codelineno-9-7" name="__codelineno-9-7" href="#__codelineno-9-7"></a><span class="p">}</span>
</span><span id="__span-9-8"><a id="__codelineno-9-8" name="__codelineno-9-8" href="#__codelineno-9-8"></a>
</span><span id="__span-9-9"><a id="__codelineno-9-9" name="__codelineno-9-9" href="#__codelineno-9-9"></a><span class="c1">// Check PostgreSQL</span>
</span><span id="__span-9-10"><a id="__codelineno-9-10" name="__codelineno-9-10" href="#__codelineno-9-10"></a><span class="k">try</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-9-11"><a id="__codelineno-9-11" name="__codelineno-9-11" href="#__codelineno-9-11"></a><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">prisma</span><span class="p">.</span><span class="nx">$queryRaw</span><span class="sb">`SELECT 1`</span><span class="p">;</span>
</span><span id="__span-9-12"><a id="__codelineno-9-12" name="__codelineno-9-12" href="#__codelineno-9-12"></a><span class="w"> </span><span class="nx">metrics</span><span class="p">.</span><span class="nx">externalServiceHealth</span><span class="p">.</span><span class="nx">set</span><span class="p">({</span><span class="w"> </span><span class="nx">service</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;postgres&#39;</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="mf">1</span><span class="p">);</span>
</span><span id="__span-9-13"><a id="__codelineno-9-13" name="__codelineno-9-13" href="#__codelineno-9-13"></a><span class="p">}</span><span class="w"> </span><span class="k">catch</span><span class="w"> </span><span class="p">(</span><span class="nx">error</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-9-14"><a id="__codelineno-9-14" name="__codelineno-9-14" href="#__codelineno-9-14"></a><span class="w"> </span><span class="nx">metrics</span><span class="p">.</span><span class="nx">externalServiceHealth</span><span class="p">.</span><span class="nx">set</span><span class="p">({</span><span class="w"> </span><span class="nx">service</span><span class="o">:</span><span class="w"> </span><span class="s1">&#39;postgres&#39;</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="mf">0</span><span class="p">);</span>
</span><span id="__span-9-15"><a id="__codelineno-9-15" name="__codelineno-9-15" href="#__codelineno-9-15"></a><span class="p">}</span>
</span></code></pre></div>
<h3 id="docker-healthchecks">Docker Healthchecks<a class="headerlink" href="#docker-healthchecks" title="Permanent link">&para;</a></h3>
<p>Services with healthchecks:</p>
<ul>
<li><strong>API</strong> - <code>wget --spider http://localhost:4000/health</code></li>
<li><strong>Media API</strong> - <code>wget --spider http://localhost:4100/health</code></li>
<li><strong>PostgreSQL</strong> - <code>pg_isready</code></li>
<li><strong>Redis</strong> - <code>redis-cli ping</code></li>
<li><strong>Listmonk</strong> - <code>wget --spider http://localhost:9000/health</code></li>
</ul>
<h2 id="performance-monitoring">Performance Monitoring<a class="headerlink" href="#performance-monitoring" title="Permanent link">&para;</a></h2>
<h3 id="http-request-tracking">HTTP Request Tracking<a class="headerlink" href="#http-request-tracking" title="Permanent link">&para;</a></h3>
<p>Automatic tracking of:</p>
<ul>
<li>Request count by route</li>
<li>Request duration percentiles</li>
<li>Status code distribution</li>
<li>Error rates</li>
</ul>
<h3 id="queue-monitoring">Queue Monitoring<a class="headerlink" href="#queue-monitoring" title="Permanent link">&para;</a></h3>
<p>Track queue depths:</p>
<ul>
<li>Email queue size</li>
<li>Geocode queue size</li>
<li>Failed job count</li>
<li>Processing rate</li>
</ul>
<h3 id="resource-monitoring">Resource Monitoring<a class="headerlink" href="#resource-monitoring" title="Permanent link">&para;</a></h3>
<p>Via cAdvisor and Node Exporter:</p>
<ul>
<li>CPU usage</li>
<li>Memory usage</li>
<li>Disk I/O</li>
<li>Network traffic</li>
</ul>
<h2 id="admin-interface_1">Admin Interface<a class="headerlink" href="#admin-interface_1" title="Permanent link">&para;</a></h2>
<h3 id="metrics-tab">Metrics Tab<a class="headerlink" href="#metrics-tab" title="Permanent link">&para;</a></h3>
<p>Display cards:</p>
<ul>
<li>API uptime</li>
<li>Request rate (req/sec)</li>
<li>Error rate (%)</li>
<li>Queue sizes</li>
<li>Active sessions</li>
<li>Service health</li>
</ul>
<h3 id="dashboards-tab">Dashboards Tab<a class="headerlink" href="#dashboards-tab" title="Permanent link">&para;</a></h3>
<p>Embedded Grafana:</p>
<ul>
<li>Overview dashboard</li>
<li>Canvassing metrics</li>
<li>External services</li>
<li>Custom queries</li>
</ul>
<h3 id="alerts-tab">Alerts Tab<a class="headerlink" href="#alerts-tab" title="Permanent link">&para;</a></h3>
<p>Active alerts list:</p>
<ul>
<li>Alert name</li>
<li>Severity</li>
<li>Status (firing/pending/resolved)</li>
<li>Duration</li>
<li>Quick actions (silence, resolve)</li>
</ul>
<h2 id="starting-monitoring-stack">Starting Monitoring Stack<a class="headerlink" href="#starting-monitoring-stack" title="Permanent link">&para;</a></h2>
<div class="language-bash highlight"><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a><span class="c1"># Start with monitoring profile</span>
</span><span id="__span-10-2"><a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>up<span class="w"> </span>-d
</span><span id="__span-10-3"><a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a>
</span><span id="__span-10-4"><a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="c1"># Access services</span>
</span><span id="__span-10-5"><a id="__codelineno-10-5" name="__codelineno-10-5" href="#__codelineno-10-5"></a><span class="c1"># Prometheus: http://localhost:9090</span>
</span><span id="__span-10-6"><a id="__codelineno-10-6" name="__codelineno-10-6" href="#__codelineno-10-6"></a><span class="c1"># Grafana: http://localhost:3001 (admin/admin)</span>
</span><span id="__span-10-7"><a id="__codelineno-10-7" name="__codelineno-10-7" href="#__codelineno-10-7"></a><span class="c1"># Alertmanager: http://localhost:9093</span>
</span></code></pre></div>
<h2 id="api-endpoints">API Endpoints<a class="headerlink" href="#api-endpoints" title="Permanent link">&para;</a></h2>
<h3 id="observability-endpoints">Observability Endpoints<a class="headerlink" href="#observability-endpoints" title="Permanent link">&para;</a></h3>
<div class="language-text highlight"><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a>GET /api/observability/prometheus # Prometheus status
</span><span id="__span-11-2"><a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a>GET /api/observability/grafana # Grafana status
</span><span id="__span-11-3"><a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a>GET /api/observability/alertmanager # Alertmanager status
</span><span id="__span-11-4"><a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a>GET /api/observability/metrics # Current metrics values
</span></code></pre></div>
<h3 id="metrics-endpoint">Metrics Endpoint<a class="headerlink" href="#metrics-endpoint" title="Permanent link">&para;</a></h3>
<div class="language-text highlight"><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a>GET /metrics # Prometheus scrape endpoint
</span></code></pre></div>
<h2 id="related-documentation">Related Documentation<a class="headerlink" href="#related-documentation" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="../../frontend/pages/admin/observability-page/">Observability Page</a></li>
<li><a href="../../backend/utilities/">Metrics Utilities</a></li>
<li><a href="../../deployment/docker-compose/">Docker Compose</a></li>
<li><a href="../../deployment/monitoring-stack/">Monitoring Stack</a></li>
<li><a href="../../deployment/healthchecks/">Healthchecks</a></li>
<li><a href="../../troubleshooting/performance-optimization/">Performance Optimization</a></li>
</ul>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../newsletter/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Newsletter Integration (Listmonk)">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</div>
<div class="md-footer__title">
<span class="md-footer__direction">
Previous
</span>
<div class="md-ellipsis">
Newsletter Integration (Listmonk)
</div>
</div>
</a>
<a href="../tunnel/" class="md-footer__link md-footer__link--next" aria-label="Next: Tunnel Management (Pangolin)">
<div class="md-footer__title">
<span class="md-footer__direction">
Next
</span>
<div class="md-ellipsis">
Tunnel Management (Pangolin)
</div>
</div>
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
</div>
</a>
</nav>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright &copy; 2024 The Bunker Operations <a href="#__consent">Change cookie settings</a>
</div>
</div>
<div class="md-social">
<a href="https://gitea.bnkops.com/admin" target="_blank" rel="noopener" title="Gitea Repository" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
</a>
<a href="https://listmonk.bnkops.com/subscription/form" target="_blank" rel="noopener" title="Newsletter" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M536.4-26.3c9.8-3.5 20.6-1 28 6.3s9.8 18.2 6.3 28l-178 496.9c-5 13.9-18.1 23.1-32.8 23.1-14.2 0-27-8.6-32.3-21.7l-64.2-158c-4.5-11-2.5-23.6 5.2-32.6l94.5-112.4c5.1-6.1 4.7-15-.9-20.6s-14.6-6-20.6-.9l-112.4 94.3c-9.1 7.6-21.6 9.6-32.6 5.2L38.1 216.8c-13.1-5.3-21.7-18.1-21.7-32.3 0-14.7 9.2-27.8 23.1-32.8z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../../..", "features": ["announce.dismiss", "content.action.edit", "content.action.view", "content.code.annotate", "content.code.copy", "content.tooltips", "navigation.expand", "navigation.footer", "navigation.indexes", "navigation.path", "navigation.prune", "navigation.sections", "navigation.tabs", "navigation.tabs.sticky", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest", "toc.follow"], "search": "../../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../../assets/javascripts/bundle.79ae519e.min.js"></script>
<script src="../../../javascripts/home.js"></script>
<script src="../../../javascripts/github-widget.js"></script>
<script src="../../../javascripts/gitea-widget.js"></script>
<script src="../../../assets/js/env-config.js"></script>
<script src="../../../assets/js/video-player.js"></script>
</body>
</html>