7333 lines
225 KiB
HTML
7333 lines
225 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="en" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
<meta name="description" content="Build Power. Not Rent It. Own your digital infrastructure.">
|
||
|
||
|
||
<meta name="author" content="Bunker Operations">
|
||
|
||
|
||
<link rel="canonical" href="https://bnkserve.org/v2/troubleshooting/monitoring-issues/">
|
||
|
||
|
||
<link rel="prev" href="../geocoding-issues/">
|
||
|
||
|
||
<link rel="next" href="../performance-optimization/">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="icon" href="../../../assets/favicon.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.1">
|
||
|
||
|
||
|
||
<title>Monitoring Issues - Changemaker Lite</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../../assets/stylesheets/main.484c7ddc.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="../../../assets/stylesheets/palette.ab4e12ef.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Inter:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Inter";--md-code-font:"JetBrains Mono"}</style>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../../stylesheets/extra.css">
|
||
|
||
<link rel="stylesheet" href="../../../stylesheets/home.css">
|
||
|
||
<link rel="stylesheet" href="../../../assets/css/video-player.css">
|
||
|
||
<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta property="og:type" content="website" />
|
||
<meta property="og:title" content="Monitoring Issues - Changemaker Lite" />
|
||
<meta property="og:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
|
||
<meta property="og:image" content="https://bnkserve.org/assets/images/social/v2/troubleshooting/monitoring-issues.png" />
|
||
<meta property="og:image:type" content="image/png" />
|
||
<meta property="og:image:width" content="1200" />
|
||
<meta property="og:image:height" content="630" />
|
||
<meta property="og:url" content="https://bnkserve.org/v2/troubleshooting/monitoring-issues/" />
|
||
<meta property="twitter:card" content="summary_large_image" />
|
||
<meta property="twitter:title" content="Monitoring Issues - Changemaker Lite" />
|
||
<meta property="twitter:description" content="Build Power. Not Rent It. Own your digital infrastructure." />
|
||
<meta property="twitter:image" content="https://bnkserve.org/assets/images/social/v2/troubleshooting/monitoring-issues.png" />
|
||
</head>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#monitoring-and-observability-issues" class="md-skip">
|
||
Skip to content
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
|
||
<nav class="md-header__inner md-grid" aria-label="Header">
|
||
<a href="../../.." title="Changemaker Lite" class="md-header__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
|
||
|
||
<img src="../../../assets/logo.png" alt="logo">
|
||
|
||
</a>
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
Changemaker Lite
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
Monitoring Issues
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<form class="md-header__option" data-md-component="palette">
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_0">
|
||
|
||
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m17.75 4.09-2.53 1.94.91 3.06-2.63-1.81-2.63 1.81.91-3.06-2.53-1.94L12.44 4l1.06-3 1.06 3zm3.5 6.91-1.64 1.25.59 1.98-1.7-1.17-1.7 1.17.59-1.98L15.75 11l2.06-.05L18.5 9l.69 1.95zm-2.28 4.95c.83-.08 1.72 1.1 1.19 1.85-.32.45-.66.87-1.08 1.27C15.17 23 8.84 23 4.94 19.07c-3.91-3.9-3.91-10.24 0-14.14.4-.4.82-.76 1.27-1.08.75-.53 1.93.36 1.85 1.19-.27 2.86.69 5.83 2.89 8.02a9.96 9.96 0 0 0 8.02 2.89m-1.64 2.02a12.08 12.08 0 0 1-7.8-3.47c-2.17-2.19-3.33-5-3.49-7.82-2.81 3.14-2.7 7.96.31 10.98 3.02 3.01 7.84 3.12 10.98.31"/></svg>
|
||
</label>
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="deep-purple" data-md-color-accent="amber" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||
|
||
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_0" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 7a5 5 0 0 1 5 5 5 5 0 0 1-5 5 5 5 0 0 1-5-5 5 5 0 0 1 5-5m0 2a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0-7 2.39 3.42C13.65 5.15 12.84 5 12 5s-1.65.15-2.39.42zM3.34 7l4.16-.35A7.2 7.2 0 0 0 5.94 8.5c-.44.74-.69 1.5-.83 2.29zm.02 10 1.76-3.77a7.131 7.131 0 0 0 2.38 4.14zM20.65 7l-1.77 3.79a7.02 7.02 0 0 0-2.38-4.15zm-.01 10-4.14.36c.59-.51 1.12-1.14 1.54-1.86.42-.73.69-1.5.83-2.29zM12 22l-2.41-3.44c.74.27 1.55.44 2.41.44.82 0 1.63-.17 2.37-.44z"/></svg>
|
||
</label>
|
||
|
||
|
||
</form>
|
||
|
||
|
||
|
||
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-header__button md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
</label>
|
||
<div class="md-search" data-md-component="search" role="dialog">
|
||
<label class="md-search__overlay" for="__search"></label>
|
||
<div class="md-search__inner" role="search">
|
||
<form class="md-search__form" name="search">
|
||
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||
<label class="md-search__icon md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
|
||
</label>
|
||
<nav class="md-search__options" aria-label="Search">
|
||
|
||
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
|
||
</a>
|
||
|
||
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
|
||
</button>
|
||
</nav>
|
||
|
||
<div class="md-search__suggest" data-md-component="search-suggest"></div>
|
||
|
||
</form>
|
||
<div class="md-search__output">
|
||
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
|
||
<div class="md-search-result" data-md-component="search-result">
|
||
<div class="md-search-result__meta">
|
||
Initializing search
|
||
</div>
|
||
<ol class="md-search-result__list" role="presentation"></ol>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-header__source">
|
||
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
changemaker.lite
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
</nav>
|
||
|
||
|
||
|
||
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
|
||
<div class="md-grid">
|
||
<ul class="md-tabs__list">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../.." class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Home
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item md-tabs__item--active">
|
||
<a href="../../" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
V2 Documentation
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../../phil/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Philosophy
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../../v1/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
V1 Documentation (Legacy)
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../../blog/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Blog
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
|
||
<label class="md-nav__title" for="__drawer">
|
||
<a href="../../.." title="Changemaker Lite" class="md-nav__button md-logo" aria-label="Changemaker Lite" data-md-component="logo">
|
||
|
||
<img src="../../../assets/logo.png" alt="logo">
|
||
|
||
</a>
|
||
Changemaker Lite
|
||
</label>
|
||
|
||
<div class="md-nav__source">
|
||
<a href="https://gitea.bnkops.com/admin/changemaker.lite" title="Go to repository" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
changemaker.lite
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../.." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Home
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
V2 Documentation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
V2 Documentation
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../getting-started/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Getting Started
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_2" id="__nav_2_2_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Getting Started
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../getting-started/quick-start/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Quick Start
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../architecture/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_3" id="__nav_2_3_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/dual-api/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Dual API System
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/authentication/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Authentication & Security
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_4" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../backend/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Backend
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_4" id="__nav_2_4_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Backend
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../backend/modules/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Modules
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../backend/services/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Services
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../backend/middleware/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Middleware
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../backend/utilities/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Utilities
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_5" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../frontend/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Frontend
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_5" id="__nav_2_5_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Frontend
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../frontend/components/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Components
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../frontend/layouts/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Layouts
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../frontend/pages/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Pages
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_6" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../database/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Database
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_6" id="__nav_2_6_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Database
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../database/schema/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Schema Overview
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../database/migrations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Migrations
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../database/seeding/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Seeding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../database/indexes/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Indexes
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../database/models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Models
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_7" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../features/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Features
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_7" id="__nav_2_7_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_7_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_7">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Features
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/influence/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Influence
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/map/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Map
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/landing-pages/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Landing Pages
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/email-templates/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Email Templates
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/media/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Media
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/newsletter/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Newsletter
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/observability/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Observability
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../features/tunnel/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Tunnel
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_8" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../deployment/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Deployment
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_8" id="__nav_2_8_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_8_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_8">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Deployment
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/docker-compose/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Docker Compose
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/environment-variables/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Environment Variables
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/nginx/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Nginx Configuration
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/ssl-tls/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
SSL/TLS
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/tunneling/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Tunneling
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/monitoring-stack/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Monitoring Stack
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/healthchecks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Health Checks
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/scaling/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Scaling
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../deployment/backup-restore/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Backup & Restore
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_9" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../development/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Development
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_9" id="__nav_2_9_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_9_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_9">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Development
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/local-setup/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Local Setup
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/docker-workflow/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Docker Workflow
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/git-workflow/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Git Workflow
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/npm-commands/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
NPM Commands
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/migrations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Migrations
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/typescript/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
TypeScript
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/testing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Testing
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/debugging/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Debugging
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../development/code-style/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Code Style
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_10" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../api-reference/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
API Reference
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_10_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_10">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
API Reference
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../user-guides/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
User Guides
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_11">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
User Guides
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../user-guides/admin-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Admin Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../user-guides/campaign-manager-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Campaign Manager Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../user-guides/map-organizer-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Map Organizer Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../user-guides/content-editor-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Content Editor Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../user-guides/volunteer-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Volunteer Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_12" checked>
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Troubleshooting
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_2_12">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Troubleshooting
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../faq/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
FAQ
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../common-errors/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Common Errors
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../auth-issues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Auth Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../database-issues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Database Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../docker-issues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Docker Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../email-issues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Email Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../geocoding-issues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Geocoding Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active">
|
||
|
||
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Monitoring Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Monitoring Issues
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="On this page">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
On this page
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#overview" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Overview
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Overview">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#monitoring-stack" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Monitoring Stack
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#custom-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Custom Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prometheus-not-scraping" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prometheus Not Scraping
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Prometheus Not Scraping">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#target-down" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Target Down
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Target Down">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#common-causes" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Common Causes
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#scrape-timeout" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Scrape Timeout
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Scrape Timeout">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#authentication-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Authentication Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Authentication Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#grafana-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Grafana Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Grafana Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dashboards-not-loading" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Dashboards Not Loading
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Dashboards Not Loading">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#datasource-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Datasource Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Datasource Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#query-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Query Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Query Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alertmanager-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alertmanager Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Alertmanager Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alerts-not-firing" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alerts Not Firing
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Alerts Not Firing">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#notifications-not-sent" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Notifications Not Sent
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Notifications Not Sent">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#routing-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Routing Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Routing Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#metrics-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Metrics Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Metrics Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#missing-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Missing Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Missing Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#incorrect-values" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Incorrect Values
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Incorrect Values">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#stale-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Stale Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Stale Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#performance-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Performance Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Performance Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#high-memory-usage" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
High Memory Usage
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="High Memory Usage">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#slow-queries" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Slow Queries
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Slow Queries">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#useful-commands" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Useful Commands
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Useful Commands">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prometheus-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prometheus Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#grafana-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Grafana Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alertmanager-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alertmanager Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#related-documentation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Related Documentation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Related Documentation">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#monitoring-documentation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Monitoring Documentation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#other-troubleshooting" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Other Troubleshooting
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#external-resources" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
External Resources
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../performance-optimization/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Performance Optimization
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../migration/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Migration
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_13" id="__nav_2_13_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_13">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Migration
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../migration/feature-parity/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Feature Parity
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../migration/breaking-changes/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Breaking Changes
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../migration/api-changes/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
API Changes
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../migration/data-migration/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Data Migration
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../../contributing/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Contributing
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_14" id="__nav_2_14_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_14">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Contributing
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../contributing/development-setup/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Development Setup
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../contributing/code-of-conduct/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Code of Conduct
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../contributing/pull-requests/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Pull Requests
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../contributing/roadmap/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Roadmap
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../../phil/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Philosophy
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../../v1/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
V1 Documentation (Legacy)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--pruned md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="../../../blog/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Blog
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="On this page">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
On this page
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#overview" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Overview
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Overview">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#monitoring-stack" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Monitoring Stack
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#custom-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Custom Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prometheus-not-scraping" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prometheus Not Scraping
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Prometheus Not Scraping">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#target-down" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Target Down
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Target Down">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#common-causes" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Common Causes
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#scrape-timeout" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Scrape Timeout
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Scrape Timeout">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_1" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#authentication-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Authentication Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Authentication Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#grafana-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Grafana Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Grafana Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dashboards-not-loading" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Dashboards Not Loading
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Dashboards Not Loading">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#datasource-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Datasource Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Datasource Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#query-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Query Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Query Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alertmanager-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alertmanager Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Alertmanager Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alerts-not-firing" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alerts Not Firing
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Alerts Not Firing">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#notifications-not-sent" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Notifications Not Sent
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Notifications Not Sent">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#routing-errors" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Routing Errors
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Routing Errors">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#metrics-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Metrics Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Metrics Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#missing-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Missing Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Missing Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#incorrect-values" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Incorrect Values
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Incorrect Values">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_10" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#stale-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Stale Metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Stale Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_11" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#performance-issues" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Performance Issues
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Performance Issues">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#high-memory-usage" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
High Memory Usage
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="High Memory Usage">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_12" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#slow-queries" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Slow Queries
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Slow Queries">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#symptoms_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Symptoms
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solutions_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solutions
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prevention_13" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prevention
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#useful-commands" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Useful Commands
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Useful Commands">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prometheus-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prometheus Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#grafana-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Grafana Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alertmanager-operations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Alertmanager Operations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#related-documentation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Related Documentation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Related Documentation">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#monitoring-documentation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Monitoring Documentation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#other-troubleshooting" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Other Troubleshooting
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#external-resources" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
External Resources
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-path" aria-label="Navigation" >
|
||
<ol class="md-path__list">
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href="../../.." class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
Home
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href="../../" class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
V2 Documentation
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href="../" class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
Troubleshooting
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ol>
|
||
</nav>
|
||
|
||
|
||
<article class="md-content__inner md-typeset">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/troubleshooting/monitoring-issues.md" title="Edit this page" class="md-content__button md-icon" rel="edit">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"/></svg>
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://gitea.bnkops.com/admin/changemaker.lite/src/branch/main/mkdocs/docs/v2/troubleshooting/monitoring-issues.md" title="View source of this page" class="md-content__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 18c.56 0 1 .44 1 1s-.44 1-1 1-1-.44-1-1 .44-1 1-1m0-3c-2.73 0-5.06 1.66-6 4 .94 2.34 3.27 4 6 4s5.06-1.66 6-4c-.94-2.34-3.27-4-6-4m0 6.5a2.5 2.5 0 0 1-2.5-2.5 2.5 2.5 0 0 1 2.5-2.5 2.5 2.5 0 0 1 2.5 2.5 2.5 2.5 0 0 1-2.5 2.5M9.27 20H6V4h7v5h5v4.07c.7.08 1.36.25 2 .49V8l-6-6H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h4.5a8.2 8.2 0 0 1-1.23-2"/></svg>
|
||
</a>
|
||
|
||
|
||
|
||
<h1 id="monitoring-and-observability-issues">Monitoring and Observability Issues<a class="headerlink" href="#monitoring-and-observability-issues" title="Permanent link">¶</a></h1>
|
||
<p>This guide covers Prometheus, Grafana, and observability stack problems in Changemaker Lite V2.</p>
|
||
<h2 id="overview">Overview<a class="headerlink" href="#overview" title="Permanent link">¶</a></h2>
|
||
<h3 id="monitoring-stack">Monitoring Stack<a class="headerlink" href="#monitoring-stack" title="Permanent link">¶</a></h3>
|
||
<p>Changemaker Lite V2 uses <strong>profile-based monitoring</strong> (optional):</p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1"># Start with monitoring</span>
|
||
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>up<span class="w"> </span>-d
|
||
</span></code></pre></div>
|
||
<p><strong>Components:</strong></p>
|
||
<ul>
|
||
<li><strong>Prometheus</strong> - Metrics collection and storage (port 9090)</li>
|
||
<li><strong>Grafana</strong> - Metrics visualization (port 3001)</li>
|
||
<li><strong>Alertmanager</strong> - Alert routing and notification (port 9093)</li>
|
||
<li><strong>cAdvisor</strong> - Container metrics (port 8080)</li>
|
||
<li><strong>Node Exporter</strong> - Host metrics (port 9100)</li>
|
||
<li><strong>Redis Exporter</strong> - Redis metrics (port 9121)</li>
|
||
</ul>
|
||
<h3 id="custom-metrics">Custom Metrics<a class="headerlink" href="#custom-metrics" title="Permanent link">¶</a></h3>
|
||
<p>12 custom <code>cm_*</code> Prometheus metrics:</p>
|
||
<ol>
|
||
<li><code>cm_api_uptime_seconds</code> - API uptime</li>
|
||
<li><code>cm_database_uptime_seconds</code> - Database uptime</li>
|
||
<li><code>cm_email_queue_size</code> - Email queue depth</li>
|
||
<li><code>cm_geocoding_queue_size</code> - Geocoding queue depth</li>
|
||
<li><code>cm_users_total</code> - Total users</li>
|
||
<li><code>cm_campaigns_total</code> - Total campaigns</li>
|
||
<li><code>cm_locations_total</code> - Total locations</li>
|
||
<li><code>cm_geocoded_locations_total</code> - Geocoded locations</li>
|
||
<li><code>cm_active_canvass_sessions</code> - Active sessions</li>
|
||
<li><code>cm_external_service_up</code> - Service health (0/1)</li>
|
||
<li><code>cm_listmonk_subscribers_total</code> - Listmonk subscribers</li>
|
||
<li><code>cm_media_videos_total</code> - Total videos</li>
|
||
</ol>
|
||
<p>Plus standard HTTP metrics:
|
||
- <code>http_request_duration_seconds</code>
|
||
- <code>http_requests_total</code></p>
|
||
<hr />
|
||
<h2 id="prometheus-not-scraping">Prometheus Not Scraping<a class="headerlink" href="#prometheus-not-scraping" title="Permanent link">¶</a></h2>
|
||
<h3 id="target-down">Target Down<a class="headerlink" href="#target-down" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🔴 Critical</p>
|
||
<h4 id="symptoms">Symptoms<a class="headerlink" href="#symptoms" title="Permanent link">¶</a></h4>
|
||
<p>Prometheus UI (localhost:9090) shows targets as "DOWN":</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>Target: api (localhost:4000/metrics)
|
||
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>State: DOWN
|
||
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>Error: Get "http://api:4000/metrics": connection refused
|
||
</span></code></pre></div>
|
||
<p>No data in Grafana dashboards.</p>
|
||
<h4 id="common-causes">Common Causes<a class="headerlink" href="#common-causes" title="Permanent link">¶</a></h4>
|
||
<ol>
|
||
<li><strong>Service not running</strong> - API container stopped</li>
|
||
<li><strong>Metrics endpoint missing</strong> - /metrics endpoint not registered</li>
|
||
<li><strong>Network issue</strong> - Prometheus can't reach service</li>
|
||
<li><strong>Authentication required</strong> - Metrics endpoint requires auth</li>
|
||
</ol>
|
||
<h4 id="solutions">Solutions<a class="headerlink" href="#solutions" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check service is running</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="c1"># Is API running?</span>
|
||
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>ps<span class="w"> </span>api
|
||
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a>
|
||
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="c1"># Should show "Up"</span>
|
||
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="c1"># If not:</span>
|
||
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a>docker<span class="w"> </span>compose<span class="w"> </span>up<span class="w"> </span>-d<span class="w"> </span>api
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Test metrics endpoint</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="c1"># From host</span>
|
||
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>curl<span class="w"> </span>http://localhost:4000/metrics
|
||
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a>
|
||
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="c1"># Should return Prometheus metrics:</span>
|
||
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a><span class="c1"># # HELP cm_api_uptime_seconds API uptime in seconds</span>
|
||
</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a><span class="c1"># # TYPE cm_api_uptime_seconds gauge</span>
|
||
</span><span id="__span-3-7"><a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a><span class="c1"># cm_api_uptime_seconds 123.45</span>
|
||
</span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a>
|
||
</span><span id="__span-3-9"><a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a><span class="c1"># From Prometheus container</span>
|
||
</span><span id="__span-3-10"><a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span>wget<span class="w"> </span>-O-<span class="w"> </span>http://api:4000/metrics
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check Prometheus config</strong></p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
|
||
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">'api'</span>
|
||
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
|
||
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'api:4000'</span><span class="p p-Indicator">]</span><span class="w"> </span><span class="c1"># Use service name, not localhost</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Verify network</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="c1"># Both on same network?</span>
|
||
</span><span id="__span-5-2"><a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a>docker<span class="w"> </span>inspect<span class="w"> </span>changemaker-lite-prometheus-1<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>NetworkMode
|
||
</span><span id="__span-5-3"><a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a>docker<span class="w"> </span>inspect<span class="w"> </span>changemaker-lite-api-1<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>NetworkMode
|
||
</span><span id="__span-5-4"><a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a>
|
||
</span><span id="__span-5-5"><a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a><span class="c1"># Should both show "changemaker-lite"</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 5: Check metrics are registered</strong></p>
|
||
<p>In API logs:</p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>api<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-i<span class="w"> </span><span class="s2">"metrics\|prometheus"</span>
|
||
</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a>
|
||
</span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a><span class="c1"># Should show:</span>
|
||
</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a><span class="c1"># Metrics endpoint registered at /metrics</span>
|
||
</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5" href="#__codelineno-6-5"></a><span class="c1"># Prometheus metrics initialized</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention">Prevention<a class="headerlink" href="#prevention" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Health checks</strong> - Monitor Prometheus target health</li>
|
||
<li><strong>Service dependencies</strong> - Ensure services start in order</li>
|
||
<li><strong>Network config</strong> - Use Docker service names</li>
|
||
<li><strong>Testing</strong> - Test /metrics endpoint on deploy</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="scrape-timeout">Scrape Timeout<a class="headerlink" href="#scrape-timeout" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_1">Symptoms<a class="headerlink" href="#symptoms_1" title="Permanent link">¶</a></h4>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>Target: api
|
||
</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a>State: UP
|
||
</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a>Last Scrape: 5.2s (slow)
|
||
</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a>Last Error: context deadline exceeded
|
||
</span></code></pre></div>
|
||
<p>Scrapes taking too long or timing out.</p>
|
||
<h4 id="solutions_1">Solutions<a class="headerlink" href="#solutions_1" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Increase scrape timeout</strong></p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="nt">global</span><span class="p">:</span>
|
||
</span><span id="__span-8-2"><a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span>
|
||
</span><span id="__span-8-3"><a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a><span class="w"> </span><span class="nt">scrape_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10s</span><span class="w"> </span><span class="c1"># Increase from 10s to 30s</span>
|
||
</span><span id="__span-8-4"><a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a>
|
||
</span><span id="__span-8-5"><a id="__codelineno-8-5" name="__codelineno-8-5" href="#__codelineno-8-5"></a><span class="nt">scrape_configs</span><span class="p">:</span>
|
||
</span><span id="__span-8-6"><a id="__codelineno-8-6" name="__codelineno-8-6" href="#__codelineno-8-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">'api'</span>
|
||
</span><span id="__span-8-7"><a id="__codelineno-8-7" name="__codelineno-8-7" href="#__codelineno-8-7"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span><span class="w"> </span><span class="c1"># Scrape less frequently</span>
|
||
</span><span id="__span-8-8"><a id="__codelineno-8-8" name="__codelineno-8-8" href="#__codelineno-8-8"></a><span class="w"> </span><span class="nt">scrape_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">20s</span>
|
||
</span><span id="__span-8-9"><a id="__codelineno-8-9" name="__codelineno-8-9" href="#__codelineno-8-9"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
|
||
</span><span id="__span-8-10"><a id="__codelineno-8-10" name="__codelineno-8-10" href="#__codelineno-8-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'api:4000'</span><span class="p p-Indicator">]</span>
|
||
</span></code></pre></div>
|
||
<p>Reload config:</p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="c1"># Reload Prometheus config</span>
|
||
</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span><span class="nb">kill</span><span class="w"> </span>-HUP<span class="w"> </span><span class="m">1</span>
|
||
</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a>
|
||
</span><span id="__span-9-4"><a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="c1"># Or restart</span>
|
||
</span><span id="__span-9-5"><a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a>docker<span class="w"> </span>compose<span class="w"> </span>restart<span class="w"> </span>prometheus
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Optimize metrics generation</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a><span class="c1">// In api/src/utils/metrics.ts</span>
|
||
</span><span id="__span-10-2"><a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a><span class="c1">// Cache expensive metrics</span>
|
||
</span><span id="__span-10-3"><a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a><span class="kd">let</span><span class="w"> </span><span class="nx">cachedUserCount</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mf">0</span><span class="p">;</span>
|
||
</span><span id="__span-10-4"><a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="kd">let</span><span class="w"> </span><span class="nx">lastUserCountUpdate</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mf">0</span><span class="p">;</span>
|
||
</span><span id="__span-10-5"><a id="__codelineno-10-5" name="__codelineno-10-5" href="#__codelineno-10-5"></a>
|
||
</span><span id="__span-10-6"><a id="__codelineno-10-6" name="__codelineno-10-6" href="#__codelineno-10-6"></a><span class="nx">register</span><span class="p">.</span><span class="nx">registerMetric</span><span class="p">(</span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span>
|
||
</span><span id="__span-10-7"><a id="__codelineno-10-7" name="__codelineno-10-7" href="#__codelineno-10-7"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_users_total'</span><span class="p">,</span>
|
||
</span><span id="__span-10-8"><a id="__codelineno-10-8" name="__codelineno-10-8" href="#__codelineno-10-8"></a><span class="w"> </span><span class="nx">help</span><span class="o">:</span><span class="w"> </span><span class="s1">'Total number of users'</span><span class="p">,</span>
|
||
</span><span id="__span-10-9"><a id="__codelineno-10-9" name="__codelineno-10-9" href="#__codelineno-10-9"></a><span class="w"> </span><span class="k">async</span><span class="w"> </span><span class="nx">collect</span><span class="p">()</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-10-10"><a id="__codelineno-10-10" name="__codelineno-10-10" href="#__codelineno-10-10"></a><span class="w"> </span><span class="kd">const</span><span class="w"> </span><span class="nx">now</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">Date</span><span class="p">.</span><span class="nx">now</span><span class="p">();</span>
|
||
</span><span id="__span-10-11"><a id="__codelineno-10-11" name="__codelineno-10-11" href="#__codelineno-10-11"></a><span class="w"> </span><span class="c1">// Only query database every 60 seconds</span>
|
||
</span><span id="__span-10-12"><a id="__codelineno-10-12" name="__codelineno-10-12" href="#__codelineno-10-12"></a><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="nx">now</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="nx">lastUserCountUpdate</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mf">60000</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-10-13"><a id="__codelineno-10-13" name="__codelineno-10-13" href="#__codelineno-10-13"></a><span class="w"> </span><span class="nx">cachedUserCount</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">prisma</span><span class="p">.</span><span class="nx">user</span><span class="p">.</span><span class="nx">count</span><span class="p">();</span>
|
||
</span><span id="__span-10-14"><a id="__codelineno-10-14" name="__codelineno-10-14" href="#__codelineno-10-14"></a><span class="w"> </span><span class="nx">lastUserCountUpdate</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nx">now</span><span class="p">;</span>
|
||
</span><span id="__span-10-15"><a id="__codelineno-10-15" name="__codelineno-10-15" href="#__codelineno-10-15"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-10-16"><a id="__codelineno-10-16" name="__codelineno-10-16" href="#__codelineno-10-16"></a><span class="w"> </span><span class="k">this</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">cachedUserCount</span><span class="p">);</span>
|
||
</span><span id="__span-10-17"><a id="__codelineno-10-17" name="__codelineno-10-17" href="#__codelineno-10-17"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-10-18"><a id="__codelineno-10-18" name="__codelineno-10-18" href="#__codelineno-10-18"></a><span class="p">}));</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Reduce metric cardinality</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a><span class="c1">// Bad - high cardinality (creates metric per user)</span>
|
||
</span><span id="__span-11-2"><a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-11-3"><a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'requests_by_user'</span><span class="p">,</span>
|
||
</span><span id="__span-11-4"><a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a><span class="w"> </span><span class="nx">labelNames</span><span class="o">:</span><span class="w"> </span><span class="p">[</span><span class="s1">'userId'</span><span class="p">]</span><span class="w"> </span><span class="c1">// Don't do this!</span>
|
||
</span><span id="__span-11-5"><a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a><span class="p">});</span>
|
||
</span><span id="__span-11-6"><a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a>
|
||
</span><span id="__span-11-7"><a id="__codelineno-11-7" name="__codelineno-11-7" href="#__codelineno-11-7"></a><span class="c1">// Good - low cardinality</span>
|
||
</span><span id="__span-11-8"><a id="__codelineno-11-8" name="__codelineno-11-8" href="#__codelineno-11-8"></a><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-11-9"><a id="__codelineno-11-9" name="__codelineno-11-9" href="#__codelineno-11-9"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'requests_by_role'</span><span class="p">,</span>
|
||
</span><span id="__span-11-10"><a id="__codelineno-11-10" name="__codelineno-11-10" href="#__codelineno-11-10"></a><span class="w"> </span><span class="nx">labelNames</span><span class="o">:</span><span class="w"> </span><span class="p">[</span><span class="s1">'role'</span><span class="p">]</span><span class="w"> </span><span class="c1">// Only 5 roles</span>
|
||
</span><span id="__span-11-11"><a id="__codelineno-11-11" name="__codelineno-11-11" href="#__codelineno-11-11"></a><span class="p">});</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_1">Prevention<a class="headerlink" href="#prevention_1" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Cache expensive metrics</strong> - Don't query DB on every scrape</li>
|
||
<li><strong>Reasonable timeouts</strong> - 10-30s timeouts</li>
|
||
<li><strong>Low cardinality</strong> - Avoid high-cardinality labels</li>
|
||
<li><strong>Optimize queries</strong> - Fast metric queries</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="authentication-errors">Authentication Errors<a class="headerlink" href="#authentication-errors" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_2">Symptoms<a class="headerlink" href="#symptoms_2" title="Permanent link">¶</a></h4>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a>Error: 401 Unauthorized when scraping /metrics
|
||
</span></code></pre></div>
|
||
<h4 id="solutions_2">Solutions<a class="headerlink" href="#solutions_2" title="Permanent link">¶</a></h4>
|
||
<p>Changemaker Lite V2 metrics endpoint is <strong>public</strong> (no auth required).</p>
|
||
<p>If you see auth errors:</p>
|
||
<p><strong>Solution 1: Remove auth middleware from /metrics</strong></p>
|
||
<p>In <code>api/src/server.ts</code>:</p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-13-1"><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a><span class="c1">// Metrics endpoint should be BEFORE authenticate middleware</span>
|
||
</span><span id="__span-13-2"><a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a><span class="nx">app</span><span class="p">.</span><span class="nx">get</span><span class="p">(</span><span class="s1">'/metrics'</span><span class="p">,</span><span class="w"> </span><span class="k">async</span><span class="w"> </span><span class="p">(</span><span class="nx">req</span><span class="p">,</span><span class="w"> </span><span class="nx">res</span><span class="p">)</span><span class="w"> </span><span class="p">=></span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-13-3"><a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a><span class="w"> </span><span class="nx">res</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="s1">'Content-Type'</span><span class="p">,</span><span class="w"> </span><span class="nx">register</span><span class="p">.</span><span class="nx">contentType</span><span class="p">);</span>
|
||
</span><span id="__span-13-4"><a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a><span class="w"> </span><span class="nx">res</span><span class="p">.</span><span class="nx">end</span><span class="p">(</span><span class="k">await</span><span class="w"> </span><span class="nx">register</span><span class="p">.</span><span class="nx">metrics</span><span class="p">());</span>
|
||
</span><span id="__span-13-5"><a id="__codelineno-13-5" name="__codelineno-13-5" href="#__codelineno-13-5"></a><span class="p">});</span>
|
||
</span><span id="__span-13-6"><a id="__codelineno-13-6" name="__codelineno-13-6" href="#__codelineno-13-6"></a>
|
||
</span><span id="__span-13-7"><a id="__codelineno-13-7" name="__codelineno-13-7" href="#__codelineno-13-7"></a><span class="c1">// Auth middleware comes after</span>
|
||
</span><span id="__span-13-8"><a id="__codelineno-13-8" name="__codelineno-13-8" href="#__codelineno-13-8"></a><span class="nx">app</span><span class="p">.</span><span class="nx">use</span><span class="p">(</span><span class="nx">authenticate</span><span class="p">);</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Configure basic auth in Prometheus</strong></p>
|
||
<p>If you DO want to protect /metrics:</p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-14-1"><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
|
||
</span><span id="__span-14-2"><a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">'api'</span>
|
||
</span><span id="__span-14-3"><a id="__codelineno-14-3" name="__codelineno-14-3" href="#__codelineno-14-3"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
|
||
</span><span id="__span-14-4"><a id="__codelineno-14-4" name="__codelineno-14-4" href="#__codelineno-14-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'api:4000'</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-14-5"><a id="__codelineno-14-5" name="__codelineno-14-5" href="#__codelineno-14-5"></a><span class="w"> </span><span class="nt">basic_auth</span><span class="p">:</span>
|
||
</span><span id="__span-14-6"><a id="__codelineno-14-6" name="__codelineno-14-6" href="#__codelineno-14-6"></a><span class="w"> </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">'prometheus'</span>
|
||
</span><span id="__span-14-7"><a id="__codelineno-14-7" name="__codelineno-14-7" href="#__codelineno-14-7"></a><span class="w"> </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">'your-password'</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_2">Prevention<a class="headerlink" href="#prevention_2" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Public metrics</strong> - Keep /metrics public for simplicity</li>
|
||
<li><strong>Network isolation</strong> - Use Docker networks for security</li>
|
||
<li><strong>IP whitelist</strong> - Only allow Prometheus IP</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="grafana-issues">Grafana Issues<a class="headerlink" href="#grafana-issues" title="Permanent link">¶</a></h2>
|
||
<h3 id="dashboards-not-loading">Dashboards Not Loading<a class="headerlink" href="#dashboards-not-loading" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟠 High</p>
|
||
<h4 id="symptoms_3">Symptoms<a class="headerlink" href="#symptoms_3" title="Permanent link">¶</a></h4>
|
||
<p>Grafana shows blank dashboards or "No data" panels.</p>
|
||
<h4 id="solutions_3">Solutions<a class="headerlink" href="#solutions_3" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check Grafana is running</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-15-1"><a id="__codelineno-15-1" name="__codelineno-15-1" href="#__codelineno-15-1"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>ps<span class="w"> </span>grafana
|
||
</span><span id="__span-15-2"><a id="__codelineno-15-2" name="__codelineno-15-2" href="#__codelineno-15-2"></a>
|
||
</span><span id="__span-15-3"><a id="__codelineno-15-3" name="__codelineno-15-3" href="#__codelineno-15-3"></a><span class="c1"># Should show "Up"</span>
|
||
</span><span id="__span-15-4"><a id="__codelineno-15-4" name="__codelineno-15-4" href="#__codelineno-15-4"></a><span class="c1"># If not:</span>
|
||
</span><span id="__span-15-5"><a id="__codelineno-15-5" name="__codelineno-15-5" href="#__codelineno-15-5"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>up<span class="w"> </span>-d<span class="w"> </span>grafana
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Verify Prometheus datasource</strong></p>
|
||
<ol>
|
||
<li>Open Grafana: http://localhost:3001</li>
|
||
<li>Login (admin/admin)</li>
|
||
<li>Settings → Data Sources</li>
|
||
<li>Click Prometheus</li>
|
||
<li>URL should be: <code>http://prometheus:9090</code></li>
|
||
<li>Click "Save & Test"</li>
|
||
<li>Should show "Data source is working"</li>
|
||
</ol>
|
||
<p><strong>Solution 3: Check dashboard provisioning</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-16-1"><a id="__codelineno-16-1" name="__codelineno-16-1" href="#__codelineno-16-1"></a><span class="c1"># List provisioned dashboards</span>
|
||
</span><span id="__span-16-2"><a id="__codelineno-16-2" name="__codelineno-16-2" href="#__codelineno-16-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>grafana<span class="w"> </span>ls<span class="w"> </span>-la<span class="w"> </span>/etc/grafana/provisioning/dashboards/
|
||
</span><span id="__span-16-3"><a id="__codelineno-16-3" name="__codelineno-16-3" href="#__codelineno-16-3"></a>
|
||
</span><span id="__span-16-4"><a id="__codelineno-16-4" name="__codelineno-16-4" href="#__codelineno-16-4"></a><span class="c1"># Should show:</span>
|
||
</span><span id="__span-16-5"><a id="__codelineno-16-5" name="__codelineno-16-5" href="#__codelineno-16-5"></a><span class="c1"># dashboard-provider.yml</span>
|
||
</span><span id="__span-16-6"><a id="__codelineno-16-6" name="__codelineno-16-6" href="#__codelineno-16-6"></a><span class="c1"># changemaker-api.json</span>
|
||
</span><span id="__span-16-7"><a id="__codelineno-16-7" name="__codelineno-16-7" href="#__codelineno-16-7"></a><span class="c1"># changemaker-queue.json</span>
|
||
</span><span id="__span-16-8"><a id="__codelineno-16-8" name="__codelineno-16-8" href="#__codelineno-16-8"></a><span class="c1"># changemaker-external-services.json</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Import dashboard manually</strong></p>
|
||
<p>If auto-provisioning fails:</p>
|
||
<ol>
|
||
<li>Grafana → Dashboards → Import</li>
|
||
<li>Upload JSON from <code>configs/grafana/dashboards/</code></li>
|
||
<li>Select Prometheus datasource</li>
|
||
<li>Click Import</li>
|
||
</ol>
|
||
<p><strong>Solution 5: Check for data</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-17-1"><a id="__codelineno-17-1" name="__codelineno-17-1" href="#__codelineno-17-1"></a><span class="c1"># Test query in Grafana Explore</span>
|
||
</span><span id="__span-17-2"><a id="__codelineno-17-2" name="__codelineno-17-2" href="#__codelineno-17-2"></a><span class="c1"># Query: cm_api_uptime_seconds</span>
|
||
</span><span id="__span-17-3"><a id="__codelineno-17-3" name="__codelineno-17-3" href="#__codelineno-17-3"></a>
|
||
</span><span id="__span-17-4"><a id="__codelineno-17-4" name="__codelineno-17-4" href="#__codelineno-17-4"></a><span class="c1"># Or test in Prometheus:</span>
|
||
</span><span id="__span-17-5"><a id="__codelineno-17-5" name="__codelineno-17-5" href="#__codelineno-17-5"></a>curl<span class="w"> </span><span class="s1">'http://localhost:9090/api/v1/query?query=cm_api_uptime_seconds'</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_3">Prevention<a class="headerlink" href="#prevention_3" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Dashboard versioning</strong> - Keep dashboards in git</li>
|
||
<li><strong>Auto-provisioning</strong> - Use provisioning instead of manual import</li>
|
||
<li><strong>Testing</strong> - Test dashboards after changes</li>
|
||
<li><strong>Documentation</strong> - Document dashboard variables</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="datasource-errors">Datasource Errors<a class="headerlink" href="#datasource-errors" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟠 High</p>
|
||
<h4 id="symptoms_4">Symptoms<a class="headerlink" href="#symptoms_4" title="Permanent link">¶</a></h4>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-18-1"><a id="__codelineno-18-1" name="__codelineno-18-1" href="#__codelineno-18-1"></a>Error: Failed to query Prometheus
|
||
</span><span id="__span-18-2"><a id="__codelineno-18-2" name="__codelineno-18-2" href="#__codelineno-18-2"></a>Error: connection refused
|
||
</span></code></pre></div>
|
||
<p>Red error bars on Grafana panels.</p>
|
||
<h4 id="solutions_4">Solutions<a class="headerlink" href="#solutions_4" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Test Prometheus connection</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-19-1"><a id="__codelineno-19-1" name="__codelineno-19-1" href="#__codelineno-19-1"></a><span class="c1"># From Grafana container</span>
|
||
</span><span id="__span-19-2"><a id="__codelineno-19-2" name="__codelineno-19-2" href="#__codelineno-19-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>grafana<span class="w"> </span>wget<span class="w"> </span>-O-<span class="w"> </span>http://prometheus:9090/api/v1/query?query<span class="o">=</span>up
|
||
</span><span id="__span-19-3"><a id="__codelineno-19-3" name="__codelineno-19-3" href="#__codelineno-19-3"></a>
|
||
</span><span id="__span-19-4"><a id="__codelineno-19-4" name="__codelineno-19-4" href="#__codelineno-19-4"></a><span class="c1"># Should return JSON:</span>
|
||
</span><span id="__span-19-5"><a id="__codelineno-19-5" name="__codelineno-19-5" href="#__codelineno-19-5"></a><span class="c1"># {"status":"success","data":{"resultType":"vector","result":[...]}}</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Check Prometheus is running</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-20-1"><a id="__codelineno-20-1" name="__codelineno-20-1" href="#__codelineno-20-1"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>ps<span class="w"> </span>prometheus
|
||
</span><span id="__span-20-2"><a id="__codelineno-20-2" name="__codelineno-20-2" href="#__codelineno-20-2"></a>
|
||
</span><span id="__span-20-3"><a id="__codelineno-20-3" name="__codelineno-20-3" href="#__codelineno-20-3"></a><span class="c1"># Should show "Up"</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Verify datasource URL</strong></p>
|
||
<p>In Grafana datasource settings:
|
||
- URL: <code>http://prometheus:9090</code> (NOT <code>http://localhost:9090</code>)
|
||
- Access: Server (NOT Browser)</p>
|
||
<p><strong>Solution 4: Check Docker network</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-21-1"><a id="__codelineno-21-1" name="__codelineno-21-1" href="#__codelineno-21-1"></a><span class="c1"># Same network?</span>
|
||
</span><span id="__span-21-2"><a id="__codelineno-21-2" name="__codelineno-21-2" href="#__codelineno-21-2"></a>docker<span class="w"> </span>inspect<span class="w"> </span>changemaker-lite-grafana-1<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>NetworkMode
|
||
</span><span id="__span-21-3"><a id="__codelineno-21-3" name="__codelineno-21-3" href="#__codelineno-21-3"></a>docker<span class="w"> </span>inspect<span class="w"> </span>changemaker-lite-prometheus-1<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>NetworkMode
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_4">Prevention<a class="headerlink" href="#prevention_4" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Health checks</strong> - Monitor datasource health</li>
|
||
<li><strong>Service dependencies</strong> - Start Prometheus before Grafana</li>
|
||
<li><strong>Error handling</strong> - Graceful error messages</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="query-errors">Query Errors<a class="headerlink" href="#query-errors" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_5">Symptoms<a class="headerlink" href="#symptoms_5" title="Permanent link">¶</a></h4>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-22-1"><a id="__codelineno-22-1" name="__codelineno-22-1" href="#__codelineno-22-1"></a>Error executing query: parse error at char X: unexpected identifier
|
||
</span></code></pre></div>
|
||
<p>Panel shows "Error loading data".</p>
|
||
<h4 id="solutions_5">Solutions<a class="headerlink" href="#solutions_5" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Validate PromQL syntax</strong></p>
|
||
<p>Common errors:</p>
|
||
<div class="language-promql highlight"><pre><span></span><code><span id="__span-23-1"><a id="__codelineno-23-1" name="__codelineno-23-1" href="#__codelineno-23-1"></a><span class="c1"># Bad - missing {}</span>
|
||
</span><span id="__span-23-2"><a id="__codelineno-23-2" name="__codelineno-23-2" href="#__codelineno-23-2"></a><span class="nv">cm_api_uptime_seconds</span><span class="p">{</span><span class="err">job=api</span><span class="p">}</span>
|
||
</span><span id="__span-23-3"><a id="__codelineno-23-3" name="__codelineno-23-3" href="#__codelineno-23-3"></a>
|
||
</span><span id="__span-23-4"><a id="__codelineno-23-4" name="__codelineno-23-4" href="#__codelineno-23-4"></a><span class="c1"># Good</span>
|
||
</span><span id="__span-23-5"><a id="__codelineno-23-5" name="__codelineno-23-5" href="#__codelineno-23-5"></a><span class="nv">cm_api_uptime_seconds</span><span class="p">{</span><span class="nl">job</span><span class="o">=</span><span class="p">"</span><span class="s">api</span><span class="p">"}</span>
|
||
</span><span id="__span-23-6"><a id="__codelineno-23-6" name="__codelineno-23-6" href="#__codelineno-23-6"></a>
|
||
</span><span id="__span-23-7"><a id="__codelineno-23-7" name="__codelineno-23-7" href="#__codelineno-23-7"></a><span class="c1"># Bad - wrong function</span>
|
||
</span><span id="__span-23-8"><a id="__codelineno-23-8" name="__codelineno-23-8" href="#__codelineno-23-8"></a><span class="nv">average</span><span class="o">(</span><span class="nv">cm_api_uptime_seconds</span><span class="o">)</span>
|
||
</span><span id="__span-23-9"><a id="__codelineno-23-9" name="__codelineno-23-9" href="#__codelineno-23-9"></a>
|
||
</span><span id="__span-23-10"><a id="__codelineno-23-10" name="__codelineno-23-10" href="#__codelineno-23-10"></a><span class="c1"># Good</span>
|
||
</span><span id="__span-23-11"><a id="__codelineno-23-11" name="__codelineno-23-11" href="#__codelineno-23-11"></a><span class="k">avg</span><span class="o">(</span><span class="nv">cm_api_uptime_seconds</span><span class="o">)</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Test query in Explore</strong></p>
|
||
<ol>
|
||
<li>Grafana → Explore</li>
|
||
<li>Enter query</li>
|
||
<li>Run</li>
|
||
<li>Fix errors before adding to dashboard</li>
|
||
</ol>
|
||
<p><strong>Solution 3: Check metric exists</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-24-1"><a id="__codelineno-24-1" name="__codelineno-24-1" href="#__codelineno-24-1"></a><span class="c1"># List all metrics</span>
|
||
</span><span id="__span-24-2"><a id="__codelineno-24-2" name="__codelineno-24-2" href="#__codelineno-24-2"></a>curl<span class="w"> </span>http://localhost:9090/api/v1/label/__name__/values<span class="w"> </span><span class="p">|</span><span class="w"> </span>jq
|
||
</span><span id="__span-24-3"><a id="__codelineno-24-3" name="__codelineno-24-3" href="#__codelineno-24-3"></a>
|
||
</span><span id="__span-24-4"><a id="__codelineno-24-4" name="__codelineno-24-4" href="#__codelineno-24-4"></a><span class="c1"># Search for metric</span>
|
||
</span><span id="__span-24-5"><a id="__codelineno-24-5" name="__codelineno-24-5" href="#__codelineno-24-5"></a>curl<span class="w"> </span>http://localhost:9090/api/v1/label/__name__/values<span class="w"> </span><span class="p">|</span><span class="w"> </span>jq<span class="w"> </span><span class="s1">'.data[]'</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>cm_
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Use metric browser</strong></p>
|
||
<p>In Grafana query editor:
|
||
1. Click "Metrics" button
|
||
2. Browse available metrics
|
||
3. Select metric (auto-fills query)</p>
|
||
<h4 id="prevention_5">Prevention<a class="headerlink" href="#prevention_5" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Query validation</strong> - Validate before saving</li>
|
||
<li><strong>Testing</strong> - Test queries in Explore</li>
|
||
<li><strong>Documentation</strong> - Document available metrics</li>
|
||
<li><strong>Examples</strong> - Provide query examples</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="alertmanager-issues">Alertmanager Issues<a class="headerlink" href="#alertmanager-issues" title="Permanent link">¶</a></h2>
|
||
<h3 id="alerts-not-firing">Alerts Not Firing<a class="headerlink" href="#alerts-not-firing" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟠 High</p>
|
||
<h4 id="symptoms_6">Symptoms<a class="headerlink" href="#symptoms_6" title="Permanent link">¶</a></h4>
|
||
<p>Conditions met but alert not triggering.</p>
|
||
<h4 id="solutions_6">Solutions<a class="headerlink" href="#solutions_6" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check alert rules</strong></p>
|
||
<p>In Prometheus UI (localhost:9090):</p>
|
||
<ol>
|
||
<li>Click "Alerts"</li>
|
||
<li>Find your alert</li>
|
||
<li>Check state:</li>
|
||
<li>Inactive: Condition not met</li>
|
||
<li>Pending: Met but waiting for <code>for:</code> duration</li>
|
||
<li>Firing: Alert active</li>
|
||
</ol>
|
||
<p><strong>Solution 2: Verify alert rule syntax</strong></p>
|
||
<p>In <code>configs/prometheus/alerts.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-25-1"><a id="__codelineno-25-1" name="__codelineno-25-1" href="#__codelineno-25-1"></a><span class="nt">groups</span><span class="p">:</span>
|
||
</span><span id="__span-25-2"><a id="__codelineno-25-2" name="__codelineno-25-2" href="#__codelineno-25-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">changemaker_alerts</span>
|
||
</span><span id="__span-25-3"><a id="__codelineno-25-3" name="__codelineno-25-3" href="#__codelineno-25-3"></a><span class="w"> </span><span class="nt">interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||
</span><span id="__span-25-4"><a id="__codelineno-25-4" name="__codelineno-25-4" href="#__codelineno-25-4"></a><span class="w"> </span><span class="nt">rules</span><span class="p">:</span>
|
||
</span><span id="__span-25-5"><a id="__codelineno-25-5" name="__codelineno-25-5" href="#__codelineno-25-5"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">alert</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">APIDown</span>
|
||
</span><span id="__span-25-6"><a id="__codelineno-25-6" name="__codelineno-25-6" href="#__codelineno-25-6"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">up{job="api"} == 0</span>
|
||
</span><span id="__span-25-7"><a id="__codelineno-25-7" name="__codelineno-25-7" href="#__codelineno-25-7"></a><span class="w"> </span><span class="nt">for</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1m</span><span class="w"> </span><span class="c1"># Must be down for 1 minute before firing</span>
|
||
</span><span id="__span-25-8"><a id="__codelineno-25-8" name="__codelineno-25-8" href="#__codelineno-25-8"></a><span class="w"> </span><span class="nt">labels</span><span class="p">:</span>
|
||
</span><span id="__span-25-9"><a id="__codelineno-25-9" name="__codelineno-25-9" href="#__codelineno-25-9"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
|
||
</span><span id="__span-25-10"><a id="__codelineno-25-10" name="__codelineno-25-10" href="#__codelineno-25-10"></a><span class="w"> </span><span class="nt">annotations</span><span class="p">:</span>
|
||
</span><span id="__span-25-11"><a id="__codelineno-25-11" name="__codelineno-25-11" href="#__codelineno-25-11"></a><span class="w"> </span><span class="nt">summary</span><span class="p">:</span><span class="w"> </span><span class="s">"API</span><span class="nv"> </span><span class="s">is</span><span class="nv"> </span><span class="s">down"</span>
|
||
</span><span id="__span-25-12"><a id="__codelineno-25-12" name="__codelineno-25-12" href="#__codelineno-25-12"></a><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="s">"API</span><span class="nv"> </span><span class="s">has</span><span class="nv"> </span><span class="s">been</span><span class="nv"> </span><span class="s">down</span><span class="nv"> </span><span class="s">for</span><span class="nv"> </span><span class="s">1</span><span class="nv"> </span><span class="s">minute"</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check Alertmanager config</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-26-1"><a id="__codelineno-26-1" name="__codelineno-26-1" href="#__codelineno-26-1"></a><span class="c1"># Test Alertmanager</span>
|
||
</span><span id="__span-26-2"><a id="__codelineno-26-2" name="__codelineno-26-2" href="#__codelineno-26-2"></a>curl<span class="w"> </span>http://localhost:9093/api/v1/alerts
|
||
</span><span id="__span-26-3"><a id="__codelineno-26-3" name="__codelineno-26-3" href="#__codelineno-26-3"></a>
|
||
</span><span id="__span-26-4"><a id="__codelineno-26-4" name="__codelineno-26-4" href="#__codelineno-26-4"></a><span class="c1"># Should return alert list</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: View Prometheus logs</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-27-1"><a id="__codelineno-27-1" name="__codelineno-27-1" href="#__codelineno-27-1"></a>docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>prometheus<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-i<span class="w"> </span>alert
|
||
</span><span id="__span-27-2"><a id="__codelineno-27-2" name="__codelineno-27-2" href="#__codelineno-27-2"></a>
|
||
</span><span id="__span-27-3"><a id="__codelineno-27-3" name="__codelineno-27-3" href="#__codelineno-27-3"></a><span class="c1"># Shows:</span>
|
||
</span><span id="__span-27-4"><a id="__codelineno-27-4" name="__codelineno-27-4" href="#__codelineno-27-4"></a><span class="c1"># Loaded alert rules</span>
|
||
</span><span id="__span-27-5"><a id="__codelineno-27-5" name="__codelineno-27-5" href="#__codelineno-27-5"></a><span class="c1"># Alert X is firing</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 5: Reload alert rules</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-28-1"><a id="__codelineno-28-1" name="__codelineno-28-1" href="#__codelineno-28-1"></a><span class="c1"># Reload Prometheus config</span>
|
||
</span><span id="__span-28-2"><a id="__codelineno-28-2" name="__codelineno-28-2" href="#__codelineno-28-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span><span class="nb">kill</span><span class="w"> </span>-HUP<span class="w"> </span><span class="m">1</span>
|
||
</span><span id="__span-28-3"><a id="__codelineno-28-3" name="__codelineno-28-3" href="#__codelineno-28-3"></a>
|
||
</span><span id="__span-28-4"><a id="__codelineno-28-4" name="__codelineno-28-4" href="#__codelineno-28-4"></a><span class="c1"># Check rules loaded</span>
|
||
</span><span id="__span-28-5"><a id="__codelineno-28-5" name="__codelineno-28-5" href="#__codelineno-28-5"></a>curl<span class="w"> </span>http://localhost:9090/api/v1/rules
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_6">Prevention<a class="headerlink" href="#prevention_6" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Test alert conditions</strong> - Trigger manually to test</li>
|
||
<li><strong>Reasonable thresholds</strong> - Not too sensitive or too lenient</li>
|
||
<li><strong>Documentation</strong> - Document alert thresholds</li>
|
||
<li><strong>Regular review</strong> - Review alert effectiveness</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="notifications-not-sent">Notifications Not Sent<a class="headerlink" href="#notifications-not-sent" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_7">Symptoms<a class="headerlink" href="#symptoms_7" title="Permanent link">¶</a></h4>
|
||
<p>Alert firing in Prometheus but no notification received.</p>
|
||
<h4 id="solutions_7">Solutions<a class="headerlink" href="#solutions_7" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check Alertmanager config</strong></p>
|
||
<p>In <code>configs/alertmanager/alertmanager.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-29-1"><a id="__codelineno-29-1" name="__codelineno-29-1" href="#__codelineno-29-1"></a><span class="nt">route</span><span class="p">:</span>
|
||
</span><span id="__span-29-2"><a id="__codelineno-29-2" name="__codelineno-29-2" href="#__codelineno-29-2"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">'email'</span>
|
||
</span><span id="__span-29-3"><a id="__codelineno-29-3" name="__codelineno-29-3" href="#__codelineno-29-3"></a><span class="w"> </span><span class="nt">group_wait</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||
</span><span id="__span-29-4"><a id="__codelineno-29-4" name="__codelineno-29-4" href="#__codelineno-29-4"></a><span class="w"> </span><span class="nt">group_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5m</span>
|
||
</span><span id="__span-29-5"><a id="__codelineno-29-5" name="__codelineno-29-5" href="#__codelineno-29-5"></a><span class="w"> </span><span class="nt">repeat_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12h</span>
|
||
</span><span id="__span-29-6"><a id="__codelineno-29-6" name="__codelineno-29-6" href="#__codelineno-29-6"></a>
|
||
</span><span id="__span-29-7"><a id="__codelineno-29-7" name="__codelineno-29-7" href="#__codelineno-29-7"></a><span class="nt">receivers</span><span class="p">:</span>
|
||
</span><span id="__span-29-8"><a id="__codelineno-29-8" name="__codelineno-29-8" href="#__codelineno-29-8"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">'email'</span>
|
||
</span><span id="__span-29-9"><a id="__codelineno-29-9" name="__codelineno-29-9" href="#__codelineno-29-9"></a><span class="w"> </span><span class="nt">email_configs</span><span class="p">:</span>
|
||
</span><span id="__span-29-10"><a id="__codelineno-29-10" name="__codelineno-29-10" href="#__codelineno-29-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">to</span><span class="p">:</span><span class="w"> </span><span class="s">'alerts@example.com'</span>
|
||
</span><span id="__span-29-11"><a id="__codelineno-29-11" name="__codelineno-29-11" href="#__codelineno-29-11"></a><span class="w"> </span><span class="nt">from</span><span class="p">:</span><span class="w"> </span><span class="s">'alertmanager@example.com'</span>
|
||
</span><span id="__span-29-12"><a id="__codelineno-29-12" name="__codelineno-29-12" href="#__codelineno-29-12"></a><span class="w"> </span><span class="nt">smarthost</span><span class="p">:</span><span class="w"> </span><span class="s">'smtp.gmail.com:587'</span>
|
||
</span><span id="__span-29-13"><a id="__codelineno-29-13" name="__codelineno-29-13" href="#__codelineno-29-13"></a><span class="w"> </span><span class="nt">auth_username</span><span class="p">:</span><span class="w"> </span><span class="s">'your-email@gmail.com'</span>
|
||
</span><span id="__span-29-14"><a id="__codelineno-29-14" name="__codelineno-29-14" href="#__codelineno-29-14"></a><span class="w"> </span><span class="nt">auth_password</span><span class="p">:</span><span class="w"> </span><span class="s">'your-app-password'</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Test Alertmanager notification</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-30-1"><a id="__codelineno-30-1" name="__codelineno-30-1" href="#__codelineno-30-1"></a><span class="c1"># Send test alert</span>
|
||
</span><span id="__span-30-2"><a id="__codelineno-30-2" name="__codelineno-30-2" href="#__codelineno-30-2"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:9093/api/v1/alerts<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-30-3"><a id="__codelineno-30-3" name="__codelineno-30-3" href="#__codelineno-30-3"></a><span class="w"> </span>-H<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-30-4"><a id="__codelineno-30-4" name="__codelineno-30-4" href="#__codelineno-30-4"></a><span class="w"> </span>-d<span class="w"> </span><span class="s1">'[{</span>
|
||
</span><span id="__span-30-5"><a id="__codelineno-30-5" name="__codelineno-30-5" href="#__codelineno-30-5"></a><span class="s1"> "labels": {</span>
|
||
</span><span id="__span-30-6"><a id="__codelineno-30-6" name="__codelineno-30-6" href="#__codelineno-30-6"></a><span class="s1"> "alertname": "Test",</span>
|
||
</span><span id="__span-30-7"><a id="__codelineno-30-7" name="__codelineno-30-7" href="#__codelineno-30-7"></a><span class="s1"> "severity": "critical"</span>
|
||
</span><span id="__span-30-8"><a id="__codelineno-30-8" name="__codelineno-30-8" href="#__codelineno-30-8"></a><span class="s1"> },</span>
|
||
</span><span id="__span-30-9"><a id="__codelineno-30-9" name="__codelineno-30-9" href="#__codelineno-30-9"></a><span class="s1"> "annotations": {</span>
|
||
</span><span id="__span-30-10"><a id="__codelineno-30-10" name="__codelineno-30-10" href="#__codelineno-30-10"></a><span class="s1"> "summary": "Test alert"</span>
|
||
</span><span id="__span-30-11"><a id="__codelineno-30-11" name="__codelineno-30-11" href="#__codelineno-30-11"></a><span class="s1"> }</span>
|
||
</span><span id="__span-30-12"><a id="__codelineno-30-12" name="__codelineno-30-12" href="#__codelineno-30-12"></a><span class="s1"> }]'</span>
|
||
</span><span id="__span-30-13"><a id="__codelineno-30-13" name="__codelineno-30-13" href="#__codelineno-30-13"></a>
|
||
</span><span id="__span-30-14"><a id="__codelineno-30-14" name="__codelineno-30-14" href="#__codelineno-30-14"></a><span class="c1"># Check if notification sent</span>
|
||
</span><span id="__span-30-15"><a id="__codelineno-30-15" name="__codelineno-30-15" href="#__codelineno-30-15"></a>docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>alertmanager<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-i<span class="w"> </span><span class="s2">"notification\|email"</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check SMTP config</strong></p>
|
||
<p>See <a href="../email-issues/#smtp-configuration">Email Issues</a> for SMTP troubleshooting.</p>
|
||
<p><strong>Solution 4: Use alternative notification channels</strong></p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-31-1"><a id="__codelineno-31-1" name="__codelineno-31-1" href="#__codelineno-31-1"></a><span class="nt">receivers</span><span class="p">:</span>
|
||
</span><span id="__span-31-2"><a id="__codelineno-31-2" name="__codelineno-31-2" href="#__codelineno-31-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">'slack'</span>
|
||
</span><span id="__span-31-3"><a id="__codelineno-31-3" name="__codelineno-31-3" href="#__codelineno-31-3"></a><span class="w"> </span><span class="nt">slack_configs</span><span class="p">:</span>
|
||
</span><span id="__span-31-4"><a id="__codelineno-31-4" name="__codelineno-31-4" href="#__codelineno-31-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">api_url</span><span class="p">:</span><span class="w"> </span><span class="s">'https://hooks.slack.com/services/...'</span>
|
||
</span><span id="__span-31-5"><a id="__codelineno-31-5" name="__codelineno-31-5" href="#__codelineno-31-5"></a><span class="w"> </span><span class="nt">channel</span><span class="p">:</span><span class="w"> </span><span class="s">'#alerts'</span>
|
||
</span><span id="__span-31-6"><a id="__codelineno-31-6" name="__codelineno-31-6" href="#__codelineno-31-6"></a>
|
||
</span><span id="__span-31-7"><a id="__codelineno-31-7" name="__codelineno-31-7" href="#__codelineno-31-7"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">'webhook'</span>
|
||
</span><span id="__span-31-8"><a id="__codelineno-31-8" name="__codelineno-31-8" href="#__codelineno-31-8"></a><span class="w"> </span><span class="nt">webhook_configs</span><span class="p">:</span>
|
||
</span><span id="__span-31-9"><a id="__codelineno-31-9" name="__codelineno-31-9" href="#__codelineno-31-9"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">'http://your-webhook-url.com/alerts'</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_7">Prevention<a class="headerlink" href="#prevention_7" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Test notifications</strong> - Regular notification tests</li>
|
||
<li><strong>Multiple channels</strong> - Email + Slack + webhook</li>
|
||
<li><strong>Fallback receivers</strong> - Backup notification method</li>
|
||
<li><strong>Documentation</strong> - Document notification setup</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="routing-errors">Routing Errors<a class="headerlink" href="#routing-errors" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_8">Symptoms<a class="headerlink" href="#symptoms_8" title="Permanent link">¶</a></h4>
|
||
<p>Alerts going to wrong receiver or being silenced incorrectly.</p>
|
||
<h4 id="solutions_8">Solutions<a class="headerlink" href="#solutions_8" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check routing rules</strong></p>
|
||
<p>In <code>configs/alertmanager/alertmanager.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-32-1"><a id="__codelineno-32-1" name="__codelineno-32-1" href="#__codelineno-32-1"></a><span class="nt">route</span><span class="p">:</span>
|
||
</span><span id="__span-32-2"><a id="__codelineno-32-2" name="__codelineno-32-2" href="#__codelineno-32-2"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">'default'</span>
|
||
</span><span id="__span-32-3"><a id="__codelineno-32-3" name="__codelineno-32-3" href="#__codelineno-32-3"></a><span class="w"> </span><span class="nt">routes</span><span class="p">:</span>
|
||
</span><span id="__span-32-4"><a id="__codelineno-32-4" name="__codelineno-32-4" href="#__codelineno-32-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">match</span><span class="p">:</span>
|
||
</span><span id="__span-32-5"><a id="__codelineno-32-5" name="__codelineno-32-5" href="#__codelineno-32-5"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
|
||
</span><span id="__span-32-6"><a id="__codelineno-32-6" name="__codelineno-32-6" href="#__codelineno-32-6"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">'pager'</span>
|
||
</span><span id="__span-32-7"><a id="__codelineno-32-7" name="__codelineno-32-7" href="#__codelineno-32-7"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">match</span><span class="p">:</span>
|
||
</span><span id="__span-32-8"><a id="__codelineno-32-8" name="__codelineno-32-8" href="#__codelineno-32-8"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">warning</span>
|
||
</span><span id="__span-32-9"><a id="__codelineno-32-9" name="__codelineno-32-9" href="#__codelineno-32-9"></a><span class="w"> </span><span class="nt">receiver</span><span class="p">:</span><span class="w"> </span><span class="s">'email'</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Test routing</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-33-1"><a id="__codelineno-33-1" name="__codelineno-33-1" href="#__codelineno-33-1"></a><span class="c1"># Use amtool to test routing</span>
|
||
</span><span id="__span-33-2"><a id="__codelineno-33-2" name="__codelineno-33-2" href="#__codelineno-33-2"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>alertmanager<span class="w"> </span>amtool<span class="w"> </span>config<span class="w"> </span>routes<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-33-3"><a id="__codelineno-33-3" name="__codelineno-33-3" href="#__codelineno-33-3"></a><span class="w"> </span>--config.file<span class="o">=</span>/etc/alertmanager/alertmanager.yml<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-33-4"><a id="__codelineno-33-4" name="__codelineno-33-4" href="#__codelineno-33-4"></a><span class="w"> </span><span class="nv">alertname</span><span class="o">=</span>TestAlert<span class="w"> </span><span class="nv">severity</span><span class="o">=</span>critical
|
||
</span><span id="__span-33-5"><a id="__codelineno-33-5" name="__codelineno-33-5" href="#__codelineno-33-5"></a>
|
||
</span><span id="__span-33-6"><a id="__codelineno-33-6" name="__codelineno-33-6" href="#__codelineno-33-6"></a><span class="c1"># Shows which receiver will be used</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: View active silences</strong></p>
|
||
<p>In Alertmanager UI (localhost:9093):</p>
|
||
<ol>
|
||
<li>Click "Silences"</li>
|
||
<li>Check if alert is silenced</li>
|
||
<li>Expire or delete silence if wrong</li>
|
||
</ol>
|
||
<p><strong>Solution 4: Check inhibition rules</strong></p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-34-1"><a id="__codelineno-34-1" name="__codelineno-34-1" href="#__codelineno-34-1"></a><span class="nt">inhibit_rules</span><span class="p">:</span>
|
||
</span><span id="__span-34-2"><a id="__codelineno-34-2" name="__codelineno-34-2" href="#__codelineno-34-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">source_match</span><span class="p">:</span>
|
||
</span><span id="__span-34-3"><a id="__codelineno-34-3" name="__codelineno-34-3" href="#__codelineno-34-3"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">critical</span>
|
||
</span><span id="__span-34-4"><a id="__codelineno-34-4" name="__codelineno-34-4" href="#__codelineno-34-4"></a><span class="w"> </span><span class="nt">target_match</span><span class="p">:</span>
|
||
</span><span id="__span-34-5"><a id="__codelineno-34-5" name="__codelineno-34-5" href="#__codelineno-34-5"></a><span class="w"> </span><span class="nt">severity</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">warning</span>
|
||
</span><span id="__span-34-6"><a id="__codelineno-34-6" name="__codelineno-34-6" href="#__codelineno-34-6"></a><span class="w"> </span><span class="nt">equal</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'alertname'</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">'instance'</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-34-7"><a id="__codelineno-34-7" name="__codelineno-34-7" href="#__codelineno-34-7"></a><span class="c1"># Critical alerts inhibit warnings for same instance</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_8">Prevention<a class="headerlink" href="#prevention_8" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Clear routing logic</strong> - Simple, understandable rules</li>
|
||
<li><strong>Test routing</strong> - Test before deploying</li>
|
||
<li><strong>Documentation</strong> - Document routing rules</li>
|
||
<li><strong>Regular review</strong> - Review silences and inhibitions</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="metrics-issues">Metrics Issues<a class="headerlink" href="#metrics-issues" title="Permanent link">¶</a></h2>
|
||
<h3 id="missing-metrics">Missing Metrics<a class="headerlink" href="#missing-metrics" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_9">Symptoms<a class="headerlink" href="#symptoms_9" title="Permanent link">¶</a></h4>
|
||
<p>Expected metric not appearing in Prometheus or Grafana.</p>
|
||
<h4 id="solutions_9">Solutions<a class="headerlink" href="#solutions_9" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check metric is registered</strong></p>
|
||
<p>In API code (<code>api/src/utils/metrics.ts</code>):</p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-35-1"><a id="__codelineno-35-1" name="__codelineno-35-1" href="#__codelineno-35-1"></a><span class="k">import</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nx">Counter</span><span class="w"> </span><span class="p">}</span><span class="w"> </span><span class="kr">from</span><span class="w"> </span><span class="s1">'prom-client'</span><span class="p">;</span>
|
||
</span><span id="__span-35-2"><a id="__codelineno-35-2" name="__codelineno-35-2" href="#__codelineno-35-2"></a>
|
||
</span><span id="__span-35-3"><a id="__codelineno-35-3" name="__codelineno-35-3" href="#__codelineno-35-3"></a><span class="kd">const</span><span class="w"> </span><span class="nx">requestCounter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-35-4"><a id="__codelineno-35-4" name="__codelineno-35-4" href="#__codelineno-35-4"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_my_metric_total'</span><span class="p">,</span>
|
||
</span><span id="__span-35-5"><a id="__codelineno-35-5" name="__codelineno-35-5" href="#__codelineno-35-5"></a><span class="w"> </span><span class="nx">help</span><span class="o">:</span><span class="w"> </span><span class="s1">'Description of metric'</span>
|
||
</span><span id="__span-35-6"><a id="__codelineno-35-6" name="__codelineno-35-6" href="#__codelineno-35-6"></a><span class="p">});</span>
|
||
</span><span id="__span-35-7"><a id="__codelineno-35-7" name="__codelineno-35-7" href="#__codelineno-35-7"></a>
|
||
</span><span id="__span-35-8"><a id="__codelineno-35-8" name="__codelineno-35-8" href="#__codelineno-35-8"></a><span class="nx">register</span><span class="p">.</span><span class="nx">registerMetric</span><span class="p">(</span><span class="nx">requestCounter</span><span class="p">);</span><span class="w"> </span><span class="c1">// Must register!</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Check metric is collected</strong></p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-36-1"><a id="__codelineno-36-1" name="__codelineno-36-1" href="#__codelineno-36-1"></a><span class="c1"># Test /metrics endpoint</span>
|
||
</span><span id="__span-36-2"><a id="__codelineno-36-2" name="__codelineno-36-2" href="#__codelineno-36-2"></a>curl<span class="w"> </span>http://localhost:4000/metrics<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>cm_my_metric
|
||
</span><span id="__span-36-3"><a id="__codelineno-36-3" name="__codelineno-36-3" href="#__codelineno-36-3"></a>
|
||
</span><span id="__span-36-4"><a id="__codelineno-36-4" name="__codelineno-36-4" href="#__codelineno-36-4"></a><span class="c1"># Should show:</span>
|
||
</span><span id="__span-36-5"><a id="__codelineno-36-5" name="__codelineno-36-5" href="#__codelineno-36-5"></a><span class="c1"># # HELP cm_my_metric_total Description of metric</span>
|
||
</span><span id="__span-36-6"><a id="__codelineno-36-6" name="__codelineno-36-6" href="#__codelineno-36-6"></a><span class="c1"># # TYPE cm_my_metric_total counter</span>
|
||
</span><span id="__span-36-7"><a id="__codelineno-36-7" name="__codelineno-36-7" href="#__codelineno-36-7"></a><span class="c1"># cm_my_metric_total 42</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check scrape config</strong></p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-37-1"><a id="__codelineno-37-1" name="__codelineno-37-1" href="#__codelineno-37-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
|
||
</span><span id="__span-37-2"><a id="__codelineno-37-2" name="__codelineno-37-2" href="#__codelineno-37-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">'api'</span>
|
||
</span><span id="__span-37-3"><a id="__codelineno-37-3" name="__codelineno-37-3" href="#__codelineno-37-3"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
|
||
</span><span id="__span-37-4"><a id="__codelineno-37-4" name="__codelineno-37-4" href="#__codelineno-37-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'api:4000'</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-37-5"><a id="__codelineno-37-5" name="__codelineno-37-5" href="#__codelineno-37-5"></a><span class="w"> </span><span class="nt">metric_relabel_configs</span><span class="p">:</span><span class="w"> </span><span class="c1"># Don't accidentally drop metric</span>
|
||
</span><span id="__span-37-6"><a id="__codelineno-37-6" name="__codelineno-37-6" href="#__codelineno-37-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">source_labels</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">__name__</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-37-7"><a id="__codelineno-37-7" name="__codelineno-37-7" href="#__codelineno-37-7"></a><span class="w"> </span><span class="nt">regex</span><span class="p">:</span><span class="w"> </span><span class="s">'cm_.*'</span><span class="w"> </span><span class="c1"># Keep cm_* metrics</span>
|
||
</span><span id="__span-37-8"><a id="__codelineno-37-8" name="__codelineno-37-8" href="#__codelineno-37-8"></a><span class="w"> </span><span class="nt">action</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">keep</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Verify metric type</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-38-1"><a id="__codelineno-38-1" name="__codelineno-38-1" href="#__codelineno-38-1"></a><span class="c1">// Counter - only increases (counts)</span>
|
||
</span><span id="__span-38-2"><a id="__codelineno-38-2" name="__codelineno-38-2" href="#__codelineno-38-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">counter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_requests_total'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-38-3"><a id="__codelineno-38-3" name="__codelineno-38-3" href="#__codelineno-38-3"></a><span class="nx">counter</span><span class="p">.</span><span class="nx">inc</span><span class="p">();</span><span class="w"> </span><span class="c1">// Increment</span>
|
||
</span><span id="__span-38-4"><a id="__codelineno-38-4" name="__codelineno-38-4" href="#__codelineno-38-4"></a>
|
||
</span><span id="__span-38-5"><a id="__codelineno-38-5" name="__codelineno-38-5" href="#__codelineno-38-5"></a><span class="c1">// Gauge - can go up or down (current value)</span>
|
||
</span><span id="__span-38-6"><a id="__codelineno-38-6" name="__codelineno-38-6" href="#__codelineno-38-6"></a><span class="kd">const</span><span class="w"> </span><span class="nx">gauge</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_queue_size'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-38-7"><a id="__codelineno-38-7" name="__codelineno-38-7" href="#__codelineno-38-7"></a><span class="nx">gauge</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="mf">42</span><span class="p">);</span><span class="w"> </span><span class="c1">// Set value</span>
|
||
</span><span id="__span-38-8"><a id="__codelineno-38-8" name="__codelineno-38-8" href="#__codelineno-38-8"></a>
|
||
</span><span id="__span-38-9"><a id="__codelineno-38-9" name="__codelineno-38-9" href="#__codelineno-38-9"></a><span class="c1">// Histogram - distribution of values</span>
|
||
</span><span id="__span-38-10"><a id="__codelineno-38-10" name="__codelineno-38-10" href="#__codelineno-38-10"></a><span class="kd">const</span><span class="w"> </span><span class="nx">histogram</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Histogram</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_request_duration_seconds'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-38-11"><a id="__codelineno-38-11" name="__codelineno-38-11" href="#__codelineno-38-11"></a><span class="nx">histogram</span><span class="p">.</span><span class="nx">observe</span><span class="p">(</span><span class="mf">0.5</span><span class="p">);</span><span class="w"> </span><span class="c1">// Record duration</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_9">Prevention<a class="headerlink" href="#prevention_9" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Register all metrics</strong> - Don't forget register.registerMetric()</li>
|
||
<li><strong>Test endpoint</strong> - Check /metrics shows metric</li>
|
||
<li><strong>Naming convention</strong> - Use cm_* prefix for custom metrics</li>
|
||
<li><strong>Documentation</strong> - Document all custom metrics</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="incorrect-values">Incorrect Values<a class="headerlink" href="#incorrect-values" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_10">Symptoms<a class="headerlink" href="#symptoms_10" title="Permanent link">¶</a></h4>
|
||
<p>Metric showing wrong or unexpected values.</p>
|
||
<h4 id="solutions_10">Solutions<a class="headerlink" href="#solutions_10" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check metric logic</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-39-1"><a id="__codelineno-39-1" name="__codelineno-39-1" href="#__codelineno-39-1"></a><span class="c1">// Wrong - gauge not updated</span>
|
||
</span><span id="__span-39-2"><a id="__codelineno-39-2" name="__codelineno-39-2" href="#__codelineno-39-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">gauge</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_users_total'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-39-3"><a id="__codelineno-39-3" name="__codelineno-39-3" href="#__codelineno-39-3"></a><span class="c1">// Never set, always 0</span>
|
||
</span><span id="__span-39-4"><a id="__codelineno-39-4" name="__codelineno-39-4" href="#__codelineno-39-4"></a>
|
||
</span><span id="__span-39-5"><a id="__codelineno-39-5" name="__codelineno-39-5" href="#__codelineno-39-5"></a><span class="c1">// Right - gauge updated</span>
|
||
</span><span id="__span-39-6"><a id="__codelineno-39-6" name="__codelineno-39-6" href="#__codelineno-39-6"></a><span class="kd">const</span><span class="w"> </span><span class="nx">gauge</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span>
|
||
</span><span id="__span-39-7"><a id="__codelineno-39-7" name="__codelineno-39-7" href="#__codelineno-39-7"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_users_total'</span><span class="p">,</span>
|
||
</span><span id="__span-39-8"><a id="__codelineno-39-8" name="__codelineno-39-8" href="#__codelineno-39-8"></a><span class="w"> </span><span class="k">async</span><span class="w"> </span><span class="nx">collect</span><span class="p">()</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-39-9"><a id="__codelineno-39-9" name="__codelineno-39-9" href="#__codelineno-39-9"></a><span class="w"> </span><span class="kd">const</span><span class="w"> </span><span class="nx">count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">prisma</span><span class="p">.</span><span class="nx">user</span><span class="p">.</span><span class="nx">count</span><span class="p">();</span>
|
||
</span><span id="__span-39-10"><a id="__codelineno-39-10" name="__codelineno-39-10" href="#__codelineno-39-10"></a><span class="w"> </span><span class="k">this</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">count</span><span class="p">);</span>
|
||
</span><span id="__span-39-11"><a id="__codelineno-39-11" name="__codelineno-39-11" href="#__codelineno-39-11"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-39-12"><a id="__codelineno-39-12" name="__codelineno-39-12" href="#__codelineno-39-12"></a><span class="p">});</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Check metric type</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-40-1"><a id="__codelineno-40-1" name="__codelineno-40-1" href="#__codelineno-40-1"></a><span class="c1">// Wrong - using Counter for value that can decrease</span>
|
||
</span><span id="__span-40-2"><a id="__codelineno-40-2" name="__codelineno-40-2" href="#__codelineno-40-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">queueSize</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_queue_size'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-40-3"><a id="__codelineno-40-3" name="__codelineno-40-3" href="#__codelineno-40-3"></a><span class="nx">queueSize</span><span class="p">.</span><span class="nx">inc</span><span class="p">(</span><span class="mf">50</span><span class="p">);</span><span class="w"> </span><span class="c1">// Add 50</span>
|
||
</span><span id="__span-40-4"><a id="__codelineno-40-4" name="__codelineno-40-4" href="#__codelineno-40-4"></a><span class="nx">queueSize</span><span class="p">.</span><span class="nx">inc</span><span class="p">(</span><span class="o">-</span><span class="mf">20</span><span class="p">);</span><span class="w"> </span><span class="c1">// Try to subtract 20 - ERROR!</span>
|
||
</span><span id="__span-40-5"><a id="__codelineno-40-5" name="__codelineno-40-5" href="#__codelineno-40-5"></a>
|
||
</span><span id="__span-40-6"><a id="__codelineno-40-6" name="__codelineno-40-6" href="#__codelineno-40-6"></a><span class="c1">// Right - use Gauge</span>
|
||
</span><span id="__span-40-7"><a id="__codelineno-40-7" name="__codelineno-40-7" href="#__codelineno-40-7"></a><span class="kd">const</span><span class="w"> </span><span class="nx">queueSize</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_queue_size'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-40-8"><a id="__codelineno-40-8" name="__codelineno-40-8" href="#__codelineno-40-8"></a><span class="nx">queueSize</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="mf">50</span><span class="p">);</span><span class="w"> </span><span class="c1">// Set to 50</span>
|
||
</span><span id="__span-40-9"><a id="__codelineno-40-9" name="__codelineno-40-9" href="#__codelineno-40-9"></a><span class="nx">queueSize</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="mf">30</span><span class="p">);</span><span class="w"> </span><span class="c1">// Set to 30 (can decrease)</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check label values</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-41-1"><a id="__codelineno-41-1" name="__codelineno-41-1" href="#__codelineno-41-1"></a><span class="c1">// Labels must match exactly</span>
|
||
</span><span id="__span-41-2"><a id="__codelineno-41-2" name="__codelineno-41-2" href="#__codelineno-41-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">counter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-41-3"><a id="__codelineno-41-3" name="__codelineno-41-3" href="#__codelineno-41-3"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'requests_total'</span><span class="p">,</span>
|
||
</span><span id="__span-41-4"><a id="__codelineno-41-4" name="__codelineno-41-4" href="#__codelineno-41-4"></a><span class="w"> </span><span class="nx">labelNames</span><span class="o">:</span><span class="w"> </span><span class="p">[</span><span class="s1">'method'</span><span class="p">,</span><span class="w"> </span><span class="s1">'status'</span><span class="p">]</span>
|
||
</span><span id="__span-41-5"><a id="__codelineno-41-5" name="__codelineno-41-5" href="#__codelineno-41-5"></a><span class="p">});</span>
|
||
</span><span id="__span-41-6"><a id="__codelineno-41-6" name="__codelineno-41-6" href="#__codelineno-41-6"></a>
|
||
</span><span id="__span-41-7"><a id="__codelineno-41-7" name="__codelineno-41-7" href="#__codelineno-41-7"></a><span class="nx">counter</span><span class="p">.</span><span class="nx">inc</span><span class="p">({</span><span class="w"> </span><span class="nx">method</span><span class="o">:</span><span class="w"> </span><span class="s1">'GET'</span><span class="p">,</span><span class="w"> </span><span class="nx">status</span><span class="o">:</span><span class="w"> </span><span class="s1">'200'</span><span class="w"> </span><span class="p">});</span>
|
||
</span><span id="__span-41-8"><a id="__codelineno-41-8" name="__codelineno-41-8" href="#__codelineno-41-8"></a><span class="c1">// Creates: requests_total{method="GET",status="200"} 1</span>
|
||
</span><span id="__span-41-9"><a id="__codelineno-41-9" name="__codelineno-41-9" href="#__codelineno-41-9"></a>
|
||
</span><span id="__span-41-10"><a id="__codelineno-41-10" name="__codelineno-41-10" href="#__codelineno-41-10"></a><span class="nx">counter</span><span class="p">.</span><span class="nx">inc</span><span class="p">({</span><span class="w"> </span><span class="nx">method</span><span class="o">:</span><span class="w"> </span><span class="s1">'GET'</span><span class="p">,</span><span class="w"> </span><span class="nx">status</span><span class="o">:</span><span class="w"> </span><span class="kt">200</span><span class="w"> </span><span class="p">});</span><span class="w"> </span><span class="c1">// Wrong - number not string</span>
|
||
</span><span id="__span-41-11"><a id="__codelineno-41-11" name="__codelineno-41-11" href="#__codelineno-41-11"></a><span class="c1">// Creates separate metric: requests_total{method="GET",status=200} 1</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Check query aggregation</strong></p>
|
||
<div class="language-promql highlight"><pre><span></span><code><span id="__span-42-1"><a id="__codelineno-42-1" name="__codelineno-42-1" href="#__codelineno-42-1"></a><span class="c1"># Wrong - sums across all labels</span>
|
||
</span><span id="__span-42-2"><a id="__codelineno-42-2" name="__codelineno-42-2" href="#__codelineno-42-2"></a><span class="k">sum</span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="o">)</span>
|
||
</span><span id="__span-42-3"><a id="__codelineno-42-3" name="__codelineno-42-3" href="#__codelineno-42-3"></a>
|
||
</span><span id="__span-42-4"><a id="__codelineno-42-4" name="__codelineno-42-4" href="#__codelineno-42-4"></a><span class="c1"># Right - sum by specific label</span>
|
||
</span><span id="__span-42-5"><a id="__codelineno-42-5" name="__codelineno-42-5" href="#__codelineno-42-5"></a><span class="k">sum</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">status</span><span class="o">)</span><span class="w"> </span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="o">)</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_10">Prevention<a class="headerlink" href="#prevention_10" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Correct metric type</strong> - Counter vs Gauge vs Histogram</li>
|
||
<li><strong>Type consistency</strong> - Label values always same type</li>
|
||
<li><strong>Testing</strong> - Test metric values with sample data</li>
|
||
<li><strong>Validation</strong> - Validate metric values are reasonable</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="stale-metrics">Stale Metrics<a class="headerlink" href="#stale-metrics" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟢 Low</p>
|
||
<h4 id="symptoms_11">Symptoms<a class="headerlink" href="#symptoms_11" title="Permanent link">¶</a></h4>
|
||
<p>Metric values not updating, showing old data.</p>
|
||
<h4 id="solutions_11">Solutions<a class="headerlink" href="#solutions_11" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Check collection frequency</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-43-1"><a id="__codelineno-43-1" name="__codelineno-43-1" href="#__codelineno-43-1"></a><span class="c1">// Metrics only updated when scraped</span>
|
||
</span><span id="__span-43-2"><a id="__codelineno-43-2" name="__codelineno-43-2" href="#__codelineno-43-2"></a><span class="kd">const</span><span class="w"> </span><span class="nx">gauge</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ow">new</span><span class="w"> </span><span class="nx">Gauge</span><span class="p">({</span>
|
||
</span><span id="__span-43-3"><a id="__codelineno-43-3" name="__codelineno-43-3" href="#__codelineno-43-3"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'cm_queue_size'</span><span class="p">,</span>
|
||
</span><span id="__span-43-4"><a id="__codelineno-43-4" name="__codelineno-43-4" href="#__codelineno-43-4"></a><span class="w"> </span><span class="k">async</span><span class="w"> </span><span class="nx">collect</span><span class="p">()</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-43-5"><a id="__codelineno-43-5" name="__codelineno-43-5" href="#__codelineno-43-5"></a><span class="w"> </span><span class="c1">// This runs on every Prometheus scrape (every 15s)</span>
|
||
</span><span id="__span-43-6"><a id="__codelineno-43-6" name="__codelineno-43-6" href="#__codelineno-43-6"></a><span class="w"> </span><span class="kd">const</span><span class="w"> </span><span class="nx">size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">await</span><span class="w"> </span><span class="nx">getQueueSize</span><span class="p">();</span>
|
||
</span><span id="__span-43-7"><a id="__codelineno-43-7" name="__codelineno-43-7" href="#__codelineno-43-7"></a><span class="w"> </span><span class="k">this</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">size</span><span class="p">);</span>
|
||
</span><span id="__span-43-8"><a id="__codelineno-43-8" name="__codelineno-43-8" href="#__codelineno-43-8"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-43-9"><a id="__codelineno-43-9" name="__codelineno-43-9" href="#__codelineno-43-9"></a><span class="p">});</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Force metric update</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-44-1"><a id="__codelineno-44-1" name="__codelineno-44-1" href="#__codelineno-44-1"></a><span class="c1">// Update metric on event, not just scrape</span>
|
||
</span><span id="__span-44-2"><a id="__codelineno-44-2" name="__codelineno-44-2" href="#__codelineno-44-2"></a><span class="nx">eventEmitter</span><span class="p">.</span><span class="nx">on</span><span class="p">(</span><span class="s1">'queueSizeChanged'</span><span class="p">,</span><span class="w"> </span><span class="p">(</span><span class="nx">size</span><span class="p">)</span><span class="w"> </span><span class="p">=></span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-44-3"><a id="__codelineno-44-3" name="__codelineno-44-3" href="#__codelineno-44-3"></a><span class="w"> </span><span class="nx">queueSizeGauge</span><span class="p">.</span><span class="nx">set</span><span class="p">(</span><span class="nx">size</span><span class="p">);</span>
|
||
</span><span id="__span-44-4"><a id="__codelineno-44-4" name="__codelineno-44-4" href="#__codelineno-44-4"></a><span class="p">});</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Check scrape interval</strong></p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-45-1"><a id="__codelineno-45-1" name="__codelineno-45-1" href="#__codelineno-45-1"></a><span class="nt">global</span><span class="p">:</span>
|
||
</span><span id="__span-45-2"><a id="__codelineno-45-2" name="__codelineno-45-2" href="#__codelineno-45-2"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span><span class="w"> </span><span class="c1"># Scrape every 15 seconds</span>
|
||
</span><span id="__span-45-3"><a id="__codelineno-45-3" name="__codelineno-45-3" href="#__codelineno-45-3"></a>
|
||
</span><span id="__span-45-4"><a id="__codelineno-45-4" name="__codelineno-45-4" href="#__codelineno-45-4"></a><span class="c1"># Increase for more frequent updates</span>
|
||
</span><span id="__span-45-5"><a id="__codelineno-45-5" name="__codelineno-45-5" href="#__codelineno-45-5"></a><span class="nt">global</span><span class="p">:</span>
|
||
</span><span id="__span-45-6"><a id="__codelineno-45-6" name="__codelineno-45-6" href="#__codelineno-45-6"></a><span class="w"> </span><span class="nt">scrape_interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5s</span><span class="w"> </span><span class="c1"># Scrape every 5 seconds</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_11">Prevention<a class="headerlink" href="#prevention_11" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Appropriate intervals</strong> - Balance freshness vs overhead</li>
|
||
<li><strong>Event-driven updates</strong> - Update on change, not just scrape</li>
|
||
<li><strong>Cache expensive metrics</strong> - Don't query DB every scrape</li>
|
||
<li><strong>Staleness markers</strong> - Set metrics to NaN when stale</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="performance-issues">Performance Issues<a class="headerlink" href="#performance-issues" title="Permanent link">¶</a></h2>
|
||
<h3 id="high-memory-usage">High Memory Usage<a class="headerlink" href="#high-memory-usage" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟠 High</p>
|
||
<h4 id="symptoms_12">Symptoms<a class="headerlink" href="#symptoms_12" title="Permanent link">¶</a></h4>
|
||
<p>Prometheus container using excessive memory (multiple GB).</p>
|
||
<h4 id="solutions_12">Solutions<a class="headerlink" href="#solutions_12" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Reduce retention period</strong></p>
|
||
<p>In <code>docker-compose.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-46-1"><a id="__codelineno-46-1" name="__codelineno-46-1" href="#__codelineno-46-1"></a><span class="nt">prometheus</span><span class="p">:</span>
|
||
</span><span id="__span-46-2"><a id="__codelineno-46-2" name="__codelineno-46-2" href="#__codelineno-46-2"></a><span class="w"> </span><span class="nt">command</span><span class="p">:</span>
|
||
</span><span id="__span-46-3"><a id="__codelineno-46-3" name="__codelineno-46-3" href="#__codelineno-46-3"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">'--config.file=/etc/prometheus/prometheus.yml'</span>
|
||
</span><span id="__span-46-4"><a id="__codelineno-46-4" name="__codelineno-46-4" href="#__codelineno-46-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">'--storage.tsdb.retention.time=7d'</span><span class="w"> </span><span class="c1"># Reduce from 15d to 7d</span>
|
||
</span><span id="__span-46-5"><a id="__codelineno-46-5" name="__codelineno-46-5" href="#__codelineno-46-5"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">'--storage.tsdb.retention.size=10GB'</span><span class="w"> </span><span class="c1"># Add size limit</span>
|
||
</span></code></pre></div>
|
||
<p>Restart:</p>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-47-1"><a id="__codelineno-47-1" name="__codelineno-47-1" href="#__codelineno-47-1"></a>docker<span class="w"> </span>compose<span class="w"> </span>--profile<span class="w"> </span>monitoring<span class="w"> </span>restart<span class="w"> </span>prometheus
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Reduce metric cardinality</strong></p>
|
||
<div class="language-typescript highlight"><pre><span></span><code><span id="__span-48-1"><a id="__codelineno-48-1" name="__codelineno-48-1" href="#__codelineno-48-1"></a><span class="c1">// Bad - creates metric per user (thousands)</span>
|
||
</span><span id="__span-48-2"><a id="__codelineno-48-2" name="__codelineno-48-2" href="#__codelineno-48-2"></a><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-48-3"><a id="__codelineno-48-3" name="__codelineno-48-3" href="#__codelineno-48-3"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'requests_by_user'</span><span class="p">,</span>
|
||
</span><span id="__span-48-4"><a id="__codelineno-48-4" name="__codelineno-48-4" href="#__codelineno-48-4"></a><span class="w"> </span><span class="nx">labelNames</span><span class="o">:</span><span class="w"> </span><span class="p">[</span><span class="s1">'userId'</span><span class="p">]</span>
|
||
</span><span id="__span-48-5"><a id="__codelineno-48-5" name="__codelineno-48-5" href="#__codelineno-48-5"></a><span class="p">});</span>
|
||
</span><span id="__span-48-6"><a id="__codelineno-48-6" name="__codelineno-48-6" href="#__codelineno-48-6"></a>
|
||
</span><span id="__span-48-7"><a id="__codelineno-48-7" name="__codelineno-48-7" href="#__codelineno-48-7"></a><span class="c1">// Good - creates metric per role (5)</span>
|
||
</span><span id="__span-48-8"><a id="__codelineno-48-8" name="__codelineno-48-8" href="#__codelineno-48-8"></a><span class="ow">new</span><span class="w"> </span><span class="nx">Counter</span><span class="p">({</span>
|
||
</span><span id="__span-48-9"><a id="__codelineno-48-9" name="__codelineno-48-9" href="#__codelineno-48-9"></a><span class="w"> </span><span class="nx">name</span><span class="o">:</span><span class="w"> </span><span class="s1">'requests_by_role'</span><span class="p">,</span>
|
||
</span><span id="__span-48-10"><a id="__codelineno-48-10" name="__codelineno-48-10" href="#__codelineno-48-10"></a><span class="w"> </span><span class="nx">labelNames</span><span class="o">:</span><span class="w"> </span><span class="p">[</span><span class="s1">'role'</span><span class="p">]</span>
|
||
</span><span id="__span-48-11"><a id="__codelineno-48-11" name="__codelineno-48-11" href="#__codelineno-48-11"></a><span class="p">});</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Drop unnecessary metrics</strong></p>
|
||
<p>In <code>configs/prometheus/prometheus.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-49-1"><a id="__codelineno-49-1" name="__codelineno-49-1" href="#__codelineno-49-1"></a><span class="nt">scrape_configs</span><span class="p">:</span>
|
||
</span><span id="__span-49-2"><a id="__codelineno-49-2" name="__codelineno-49-2" href="#__codelineno-49-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="s">'api'</span>
|
||
</span><span id="__span-49-3"><a id="__codelineno-49-3" name="__codelineno-49-3" href="#__codelineno-49-3"></a><span class="w"> </span><span class="nt">static_configs</span><span class="p">:</span>
|
||
</span><span id="__span-49-4"><a id="__codelineno-49-4" name="__codelineno-49-4" href="#__codelineno-49-4"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">targets</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">'api:4000'</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-49-5"><a id="__codelineno-49-5" name="__codelineno-49-5" href="#__codelineno-49-5"></a><span class="w"> </span><span class="nt">metric_relabel_configs</span><span class="p">:</span>
|
||
</span><span id="__span-49-6"><a id="__codelineno-49-6" name="__codelineno-49-6" href="#__codelineno-49-6"></a><span class="w"> </span><span class="c1"># Drop metrics we don't use</span>
|
||
</span><span id="__span-49-7"><a id="__codelineno-49-7" name="__codelineno-49-7" href="#__codelineno-49-7"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">source_labels</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">__name__</span><span class="p p-Indicator">]</span>
|
||
</span><span id="__span-49-8"><a id="__codelineno-49-8" name="__codelineno-49-8" href="#__codelineno-49-8"></a><span class="w"> </span><span class="nt">regex</span><span class="p">:</span><span class="w"> </span><span class="s">'go_.*|process_.*'</span><span class="w"> </span><span class="c1"># Drop Go/process metrics</span>
|
||
</span><span id="__span-49-9"><a id="__codelineno-49-9" name="__codelineno-49-9" href="#__codelineno-49-9"></a><span class="w"> </span><span class="nt">action</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">drop</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 4: Increase memory limit</strong></p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-50-1"><a id="__codelineno-50-1" name="__codelineno-50-1" href="#__codelineno-50-1"></a><span class="nt">prometheus</span><span class="p">:</span>
|
||
</span><span id="__span-50-2"><a id="__codelineno-50-2" name="__codelineno-50-2" href="#__codelineno-50-2"></a><span class="w"> </span><span class="nt">deploy</span><span class="p">:</span>
|
||
</span><span id="__span-50-3"><a id="__codelineno-50-3" name="__codelineno-50-3" href="#__codelineno-50-3"></a><span class="w"> </span><span class="nt">resources</span><span class="p">:</span>
|
||
</span><span id="__span-50-4"><a id="__codelineno-50-4" name="__codelineno-50-4" href="#__codelineno-50-4"></a><span class="w"> </span><span class="nt">limits</span><span class="p">:</span>
|
||
</span><span id="__span-50-5"><a id="__codelineno-50-5" name="__codelineno-50-5" href="#__codelineno-50-5"></a><span class="w"> </span><span class="nt">memory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">4G</span><span class="w"> </span><span class="c1"># Increase from 2G</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_12">Prevention<a class="headerlink" href="#prevention_12" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Low cardinality</strong> - Avoid high-cardinality labels</li>
|
||
<li><strong>Appropriate retention</strong> - 7-30 days is usually enough</li>
|
||
<li><strong>Regular cleanup</strong> - Drop unused metrics</li>
|
||
<li><strong>Monitor memory</strong> - Alert on high usage</li>
|
||
</ul>
|
||
<hr />
|
||
<h3 id="slow-queries">Slow Queries<a class="headerlink" href="#slow-queries" title="Permanent link">¶</a></h3>
|
||
<p><strong>Severity:</strong> 🟡 Medium</p>
|
||
<h4 id="symptoms_13">Symptoms<a class="headerlink" href="#symptoms_13" title="Permanent link">¶</a></h4>
|
||
<p>Grafana dashboards slow to load. Queries taking 10+ seconds.</p>
|
||
<h4 id="solutions_13">Solutions<a class="headerlink" href="#solutions_13" title="Permanent link">¶</a></h4>
|
||
<p><strong>Solution 1: Optimize query</strong></p>
|
||
<div class="language-promql highlight"><pre><span></span><code><span id="__span-51-1"><a id="__codelineno-51-1" name="__codelineno-51-1" href="#__codelineno-51-1"></a><span class="c1"># Slow - calculates rate for all time</span>
|
||
</span><span id="__span-51-2"><a id="__codelineno-51-2" name="__codelineno-51-2" href="#__codelineno-51-2"></a><span class="kr">rate</span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="p">[</span><span class="s">1y</span><span class="p">]</span><span class="o">)</span>
|
||
</span><span id="__span-51-3"><a id="__codelineno-51-3" name="__codelineno-51-3" href="#__codelineno-51-3"></a>
|
||
</span><span id="__span-51-4"><a id="__codelineno-51-4" name="__codelineno-51-4" href="#__codelineno-51-4"></a><span class="c1"># Fast - only last 5 minutes</span>
|
||
</span><span id="__span-51-5"><a id="__codelineno-51-5" name="__codelineno-51-5" href="#__codelineno-51-5"></a><span class="kr">rate</span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="p">[</span><span class="s">5m</span><span class="p">]</span><span class="o">)</span>
|
||
</span><span id="__span-51-6"><a id="__codelineno-51-6" name="__codelineno-51-6" href="#__codelineno-51-6"></a>
|
||
</span><span id="__span-51-7"><a id="__codelineno-51-7" name="__codelineno-51-7" href="#__codelineno-51-7"></a><span class="c1"># Slow - many time series</span>
|
||
</span><span id="__span-51-8"><a id="__codelineno-51-8" name="__codelineno-51-8" href="#__codelineno-51-8"></a><span class="k">sum</span><span class="o">(</span><span class="kr">rate</span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="p">[</span><span class="s">5m</span><span class="p">]</span><span class="o">))</span>
|
||
</span><span id="__span-51-9"><a id="__codelineno-51-9" name="__codelineno-51-9" href="#__codelineno-51-9"></a>
|
||
</span><span id="__span-51-10"><a id="__codelineno-51-10" name="__codelineno-51-10" href="#__codelineno-51-10"></a><span class="c1"># Faster - aggregate before rate</span>
|
||
</span><span id="__span-51-11"><a id="__codelineno-51-11" name="__codelineno-51-11" href="#__codelineno-51-11"></a><span class="k">sum</span><span class="o">(</span><span class="kr">increase</span><span class="o">(</span><span class="nv">cm_requests_total</span><span class="p">[</span><span class="s">5m</span><span class="p">]</span><span class="o">))</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="mi">300</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 2: Use recording rules</strong></p>
|
||
<p>In <code>configs/prometheus/alerts.yml</code>:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-52-1"><a id="__codelineno-52-1" name="__codelineno-52-1" href="#__codelineno-52-1"></a><span class="nt">groups</span><span class="p">:</span>
|
||
</span><span id="__span-52-2"><a id="__codelineno-52-2" name="__codelineno-52-2" href="#__codelineno-52-2"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">recording_rules</span>
|
||
</span><span id="__span-52-3"><a id="__codelineno-52-3" name="__codelineno-52-3" href="#__codelineno-52-3"></a><span class="w"> </span><span class="nt">interval</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||
</span><span id="__span-52-4"><a id="__codelineno-52-4" name="__codelineno-52-4" href="#__codelineno-52-4"></a><span class="w"> </span><span class="nt">rules</span><span class="p">:</span>
|
||
</span><span id="__span-52-5"><a id="__codelineno-52-5" name="__codelineno-52-5" href="#__codelineno-52-5"></a><span class="w"> </span><span class="c1"># Pre-calculate expensive query every 30s</span>
|
||
</span><span id="__span-52-6"><a id="__codelineno-52-6" name="__codelineno-52-6" href="#__codelineno-52-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">record</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">job:cm_request_rate:sum</span>
|
||
</span><span id="__span-52-7"><a id="__codelineno-52-7" name="__codelineno-52-7" href="#__codelineno-52-7"></a><span class="w"> </span><span class="nt">expr</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">sum(rate(cm_requests_total[5m])) by (job)</span>
|
||
</span><span id="__span-52-8"><a id="__codelineno-52-8" name="__codelineno-52-8" href="#__codelineno-52-8"></a>
|
||
</span><span id="__span-52-9"><a id="__codelineno-52-9" name="__codelineno-52-9" href="#__codelineno-52-9"></a><span class="c1"># Then use in dashboard:</span>
|
||
</span><span id="__span-52-10"><a id="__codelineno-52-10" name="__codelineno-52-10" href="#__codelineno-52-10"></a><span class="c1"># job:cm_request_rate:sum # Fast!</span>
|
||
</span></code></pre></div>
|
||
<p><strong>Solution 3: Reduce time range</strong></p>
|
||
<p>In Grafana:
|
||
- Change dashboard time range from "Last 30 days" to "Last 24 hours"
|
||
- Queries are faster with less data</p>
|
||
<p><strong>Solution 4: Increase Prometheus resources</strong></p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-53-1"><a id="__codelineno-53-1" name="__codelineno-53-1" href="#__codelineno-53-1"></a><span class="nt">prometheus</span><span class="p">:</span>
|
||
</span><span id="__span-53-2"><a id="__codelineno-53-2" name="__codelineno-53-2" href="#__codelineno-53-2"></a><span class="w"> </span><span class="nt">deploy</span><span class="p">:</span>
|
||
</span><span id="__span-53-3"><a id="__codelineno-53-3" name="__codelineno-53-3" href="#__codelineno-53-3"></a><span class="w"> </span><span class="nt">resources</span><span class="p">:</span>
|
||
</span><span id="__span-53-4"><a id="__codelineno-53-4" name="__codelineno-53-4" href="#__codelineno-53-4"></a><span class="w"> </span><span class="nt">limits</span><span class="p">:</span>
|
||
</span><span id="__span-53-5"><a id="__codelineno-53-5" name="__codelineno-53-5" href="#__codelineno-53-5"></a><span class="w"> </span><span class="nt">cpus</span><span class="p">:</span><span class="w"> </span><span class="s">'2.0'</span><span class="w"> </span><span class="c1"># More CPU for queries</span>
|
||
</span><span id="__span-53-6"><a id="__codelineno-53-6" name="__codelineno-53-6" href="#__codelineno-53-6"></a><span class="w"> </span><span class="nt">memory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">4G</span>
|
||
</span></code></pre></div>
|
||
<h4 id="prevention_13">Prevention<a class="headerlink" href="#prevention_13" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Efficient queries</strong> - Keep queries simple</li>
|
||
<li><strong>Recording rules</strong> - Pre-calculate expensive queries</li>
|
||
<li><strong>Appropriate time ranges</strong> - Don't query months of data</li>
|
||
<li><strong>Indexing</strong> - Prometheus auto-indexes, but cardinality affects performance</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="useful-commands">Useful Commands<a class="headerlink" href="#useful-commands" title="Permanent link">¶</a></h2>
|
||
<h3 id="prometheus-operations">Prometheus Operations<a class="headerlink" href="#prometheus-operations" title="Permanent link">¶</a></h3>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-54-1"><a id="__codelineno-54-1" name="__codelineno-54-1" href="#__codelineno-54-1"></a><span class="c1"># Check targets</span>
|
||
</span><span id="__span-54-2"><a id="__codelineno-54-2" name="__codelineno-54-2" href="#__codelineno-54-2"></a>curl<span class="w"> </span>http://localhost:9090/api/v1/targets
|
||
</span><span id="__span-54-3"><a id="__codelineno-54-3" name="__codelineno-54-3" href="#__codelineno-54-3"></a>
|
||
</span><span id="__span-54-4"><a id="__codelineno-54-4" name="__codelineno-54-4" href="#__codelineno-54-4"></a><span class="c1"># Query metric</span>
|
||
</span><span id="__span-54-5"><a id="__codelineno-54-5" name="__codelineno-54-5" href="#__codelineno-54-5"></a>curl<span class="w"> </span><span class="s1">'http://localhost:9090/api/v1/query?query=cm_api_uptime_seconds'</span>
|
||
</span><span id="__span-54-6"><a id="__codelineno-54-6" name="__codelineno-54-6" href="#__codelineno-54-6"></a>
|
||
</span><span id="__span-54-7"><a id="__codelineno-54-7" name="__codelineno-54-7" href="#__codelineno-54-7"></a><span class="c1"># Query range</span>
|
||
</span><span id="__span-54-8"><a id="__codelineno-54-8" name="__codelineno-54-8" href="#__codelineno-54-8"></a>curl<span class="w"> </span><span class="s1">'http://localhost:9090/api/v1/query_range?query=cm_api_uptime_seconds&start=2026-02-13T00:00:00Z&end=2026-02-13T23:59:59Z&step=15s'</span>
|
||
</span><span id="__span-54-9"><a id="__codelineno-54-9" name="__codelineno-54-9" href="#__codelineno-54-9"></a>
|
||
</span><span id="__span-54-10"><a id="__codelineno-54-10" name="__codelineno-54-10" href="#__codelineno-54-10"></a><span class="c1"># Reload config</span>
|
||
</span><span id="__span-54-11"><a id="__codelineno-54-11" name="__codelineno-54-11" href="#__codelineno-54-11"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span><span class="nb">kill</span><span class="w"> </span>-HUP<span class="w"> </span><span class="m">1</span>
|
||
</span><span id="__span-54-12"><a id="__codelineno-54-12" name="__codelineno-54-12" href="#__codelineno-54-12"></a>
|
||
</span><span id="__span-54-13"><a id="__codelineno-54-13" name="__codelineno-54-13" href="#__codelineno-54-13"></a><span class="c1"># Check config</span>
|
||
</span><span id="__span-54-14"><a id="__codelineno-54-14" name="__codelineno-54-14" href="#__codelineno-54-14"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span>promtool<span class="w"> </span>check<span class="w"> </span>config<span class="w"> </span>/etc/prometheus/prometheus.yml
|
||
</span><span id="__span-54-15"><a id="__codelineno-54-15" name="__codelineno-54-15" href="#__codelineno-54-15"></a>
|
||
</span><span id="__span-54-16"><a id="__codelineno-54-16" name="__codelineno-54-16" href="#__codelineno-54-16"></a><span class="c1"># Check rules</span>
|
||
</span><span id="__span-54-17"><a id="__codelineno-54-17" name="__codelineno-54-17" href="#__codelineno-54-17"></a>docker<span class="w"> </span>compose<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>prometheus<span class="w"> </span>promtool<span class="w"> </span>check<span class="w"> </span>rules<span class="w"> </span>/etc/prometheus/alerts.yml
|
||
</span></code></pre></div>
|
||
<h3 id="grafana-operations">Grafana Operations<a class="headerlink" href="#grafana-operations" title="Permanent link">¶</a></h3>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-55-1"><a id="__codelineno-55-1" name="__codelineno-55-1" href="#__codelineno-55-1"></a><span class="c1"># Test datasource</span>
|
||
</span><span id="__span-55-2"><a id="__codelineno-55-2" name="__codelineno-55-2" href="#__codelineno-55-2"></a>curl<span class="w"> </span>http://admin:admin@localhost:3001/api/datasources/1/health
|
||
</span><span id="__span-55-3"><a id="__codelineno-55-3" name="__codelineno-55-3" href="#__codelineno-55-3"></a>
|
||
</span><span id="__span-55-4"><a id="__codelineno-55-4" name="__codelineno-55-4" href="#__codelineno-55-4"></a><span class="c1"># List dashboards</span>
|
||
</span><span id="__span-55-5"><a id="__codelineno-55-5" name="__codelineno-55-5" href="#__codelineno-55-5"></a>curl<span class="w"> </span>http://admin:admin@localhost:3001/api/search?type<span class="o">=</span>dash-db
|
||
</span><span id="__span-55-6"><a id="__codelineno-55-6" name="__codelineno-55-6" href="#__codelineno-55-6"></a>
|
||
</span><span id="__span-55-7"><a id="__codelineno-55-7" name="__codelineno-55-7" href="#__codelineno-55-7"></a><span class="c1"># Export dashboard</span>
|
||
</span><span id="__span-55-8"><a id="__codelineno-55-8" name="__codelineno-55-8" href="#__codelineno-55-8"></a>curl<span class="w"> </span>http://admin:admin@localhost:3001/api/dashboards/uid/YOUR_UID<span class="w"> </span><span class="p">|</span><span class="w"> </span>jq<span class="w"> </span>.dashboard<span class="w"> </span>><span class="w"> </span>dashboard.json
|
||
</span><span id="__span-55-9"><a id="__codelineno-55-9" name="__codelineno-55-9" href="#__codelineno-55-9"></a>
|
||
</span><span id="__span-55-10"><a id="__codelineno-55-10" name="__codelineno-55-10" href="#__codelineno-55-10"></a><span class="c1"># Import dashboard</span>
|
||
</span><span id="__span-55-11"><a id="__codelineno-55-11" name="__codelineno-55-11" href="#__codelineno-55-11"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://admin:admin@localhost:3001/api/dashboards/db<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-55-12"><a id="__codelineno-55-12" name="__codelineno-55-12" href="#__codelineno-55-12"></a><span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-55-13"><a id="__codelineno-55-13" name="__codelineno-55-13" href="#__codelineno-55-13"></a><span class="w"> </span>-d<span class="w"> </span>@dashboard.json
|
||
</span></code></pre></div>
|
||
<h3 id="alertmanager-operations">Alertmanager Operations<a class="headerlink" href="#alertmanager-operations" title="Permanent link">¶</a></h3>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-56-1"><a id="__codelineno-56-1" name="__codelineno-56-1" href="#__codelineno-56-1"></a><span class="c1"># Check alerts</span>
|
||
</span><span id="__span-56-2"><a id="__codelineno-56-2" name="__codelineno-56-2" href="#__codelineno-56-2"></a>curl<span class="w"> </span>http://localhost:9093/api/v1/alerts
|
||
</span><span id="__span-56-3"><a id="__codelineno-56-3" name="__codelineno-56-3" href="#__codelineno-56-3"></a>
|
||
</span><span id="__span-56-4"><a id="__codelineno-56-4" name="__codelineno-56-4" href="#__codelineno-56-4"></a><span class="c1"># Send test alert</span>
|
||
</span><span id="__span-56-5"><a id="__codelineno-56-5" name="__codelineno-56-5" href="#__codelineno-56-5"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:9093/api/v1/alerts<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-56-6"><a id="__codelineno-56-6" name="__codelineno-56-6" href="#__codelineno-56-6"></a><span class="w"> </span>-H<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-56-7"><a id="__codelineno-56-7" name="__codelineno-56-7" href="#__codelineno-56-7"></a><span class="w"> </span>-d<span class="w"> </span><span class="s1">'[{"labels":{"alertname":"Test","severity":"critical"},"annotations":{"summary":"Test"}}]'</span>
|
||
</span><span id="__span-56-8"><a id="__codelineno-56-8" name="__codelineno-56-8" href="#__codelineno-56-8"></a>
|
||
</span><span id="__span-56-9"><a id="__codelineno-56-9" name="__codelineno-56-9" href="#__codelineno-56-9"></a><span class="c1"># List silences</span>
|
||
</span><span id="__span-56-10"><a id="__codelineno-56-10" name="__codelineno-56-10" href="#__codelineno-56-10"></a>curl<span class="w"> </span>http://localhost:9093/api/v1/silences
|
||
</span><span id="__span-56-11"><a id="__codelineno-56-11" name="__codelineno-56-11" href="#__codelineno-56-11"></a>
|
||
</span><span id="__span-56-12"><a id="__codelineno-56-12" name="__codelineno-56-12" href="#__codelineno-56-12"></a><span class="c1"># Create silence</span>
|
||
</span><span id="__span-56-13"><a id="__codelineno-56-13" name="__codelineno-56-13" href="#__codelineno-56-13"></a>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:9093/api/v1/silences<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-56-14"><a id="__codelineno-56-14" name="__codelineno-56-14" href="#__codelineno-56-14"></a><span class="w"> </span>-H<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-56-15"><a id="__codelineno-56-15" name="__codelineno-56-15" href="#__codelineno-56-15"></a><span class="w"> </span>-d<span class="w"> </span><span class="s1">'{"matchers":[{"name":"alertname","value":"Test"}],"startsAt":"2026-02-13T00:00:00Z","endsAt":"2026-02-14T00:00:00Z","createdBy":"admin","comment":"Test silence"}'</span>
|
||
</span></code></pre></div>
|
||
<hr />
|
||
<h2 id="related-documentation">Related Documentation<a class="headerlink" href="#related-documentation" title="Permanent link">¶</a></h2>
|
||
<h3 id="monitoring-documentation">Monitoring Documentation<a class="headerlink" href="#monitoring-documentation" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li><a href="./">Monitoring Issues</a> - This guide</li>
|
||
<li><a href="../user-guides/observability-dashboard.md">Observability Dashboard</a> - Using dashboard</li>
|
||
<li><a href="../deployment/monitoring.md">Monitoring Guide</a> - Setup and configuration</li>
|
||
</ul>
|
||
<h3 id="other-troubleshooting">Other Troubleshooting<a class="headerlink" href="#other-troubleshooting" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li><a href="../common-errors/">Common Errors</a> - General errors</li>
|
||
<li><a href="../performance-optimization/">Performance Optimization</a> - Performance tuning</li>
|
||
</ul>
|
||
<h3 id="external-resources">External Resources<a class="headerlink" href="#external-resources" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li><a href="https://prometheus.io/docs/">Prometheus Documentation</a></li>
|
||
<li><a href="https://grafana.com/docs/">Grafana Documentation</a></li>
|
||
<li><a href="https://prometheus.io/docs/alerting/latest/alertmanager/">Alertmanager Documentation</a></li>
|
||
<li><a href="https://prometheus.io/docs/prometheus/latest/querying/basics/">PromQL Tutorial</a></li>
|
||
</ul>
|
||
<hr />
|
||
<p><strong>Last Updated:</strong> February 2026
|
||
<strong>Version:</strong> V2.0
|
||
<strong>Status:</strong> Complete</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
|
||
Back to top
|
||
</button>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
|
||
|
||
<nav class="md-footer__inner md-grid" aria-label="Footer" >
|
||
|
||
|
||
<a href="../geocoding-issues/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Geocoding Issues">
|
||
<div class="md-footer__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
|
||
</div>
|
||
<div class="md-footer__title">
|
||
<span class="md-footer__direction">
|
||
Previous
|
||
</span>
|
||
<div class="md-ellipsis">
|
||
Geocoding Issues
|
||
</div>
|
||
</div>
|
||
</a>
|
||
|
||
|
||
|
||
<a href="../performance-optimization/" class="md-footer__link md-footer__link--next" aria-label="Next: Performance Optimization">
|
||
<div class="md-footer__title">
|
||
<span class="md-footer__direction">
|
||
Next
|
||
</span>
|
||
<div class="md-ellipsis">
|
||
Performance Optimization
|
||
</div>
|
||
</div>
|
||
<div class="md-footer__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
|
||
</div>
|
||
</a>
|
||
|
||
</nav>
|
||
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
<div class="md-copyright__highlight">
|
||
Copyright © 2024 The Bunker Operations – <a href="#__consent">Change cookie settings</a>
|
||
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
<div class="md-social">
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://gitea.bnkops.com/admin" target="_blank" rel="noopener" title="Gitea Repository" class="md-social__link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://listmonk.bnkops.com/subscription/form" target="_blank" rel="noopener" title="Newsletter" class="md-social__link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M536.4-26.3c9.8-3.5 20.6-1 28 6.3s9.8 18.2 6.3 28l-178 496.9c-5 13.9-18.1 23.1-32.8 23.1-14.2 0-27-8.6-32.3-21.7l-64.2-158c-4.5-11-2.5-23.6 5.2-32.6l94.5-112.4c5.1-6.1 4.7-15-.9-20.6s-14.6-6-20.6-.9l-112.4 94.3c-9.1 7.6-21.6 9.6-32.6 5.2L38.1 216.8c-13.1-5.3-21.7-18.1-21.7-32.3 0-14.7 9.2-27.8 23.1-32.8z"/></svg>
|
||
</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"annotate": null, "base": "../../..", "features": ["announce.dismiss", "content.action.edit", "content.action.view", "content.code.annotate", "content.code.copy", "content.tooltips", "navigation.expand", "navigation.footer", "navigation.indexes", "navigation.path", "navigation.prune", "navigation.sections", "navigation.tabs", "navigation.tabs.sticky", "navigation.top", "navigation.tracking", "search.highlight", "search.share", "search.suggest", "toc.follow"], "search": "../../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
||
|
||
<script src="../../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
||
|
||
<script src="../../../javascripts/home.js"></script>
|
||
|
||
<script src="../../../javascripts/github-widget.js"></script>
|
||
|
||
<script src="../../../javascripts/gitea-widget.js"></script>
|
||
|
||
<script src="../../../assets/js/env-config.js"></script>
|
||
|
||
<script src="../../../assets/js/video-player.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |