Harden install pipeline: health checks, log rotation, backup timer

- install.sh: Add Docker daemon check, 10GB disk space pre-flight, error handling on pull/up, post-startup health polling with crash detection, cleanup trap on failure - docker-compose: Fix nginx/listmonk depends_on to service_healthy, add x-logging anchor (10m/3 files) to all ~39 services - config.sh: Preserve existing secrets on re-run (reconfigure mode), add automated daily backup timer (systemd, 02:00, 30-day retention) - mirror-images.sh: Fix gotify source tag (2.9.0 not v2.9.0) - build-release.sh: Ensure mkdocs/docs and mkdocs/overrides dirs exist - .env.example: Add COMPOSE_PROFILES variable Bunker Admin
2026-03-25 19:33:11 -06:00
parent 3262d92065
commit 7287328148
9 changed files with 556 additions and 67 deletions
--- a/scripts/build-release.sh
+++ b/scripts/build-release.sh
@@ -144,8 +144,8 @@ info "Nginx templates (for reference)"
 if [[ -d "$PROJECT_DIR/mkdocs" ]]; then
  mkdir -p "$STAGE_DIR/mkdocs"
  cp "$PROJECT_DIR/mkdocs/mkdocs.yml" "$STAGE_DIR/mkdocs/" 2>/dev/null || true
-  cp -r "$PROJECT_DIR/mkdocs/docs" "$STAGE_DIR/mkdocs/" 2>/dev/null || true
-  cp -r "$PROJECT_DIR/mkdocs/overrides" "$STAGE_DIR/mkdocs/" 2>/dev/null || true
+  cp -r "$PROJECT_DIR/mkdocs/docs" "$STAGE_DIR/mkdocs/" 2>/dev/null || mkdir -p "$STAGE_DIR/mkdocs/docs"
+  cp -r "$PROJECT_DIR/mkdocs/overrides" "$STAGE_DIR/mkdocs/" 2>/dev/null || mkdir -p "$STAGE_DIR/mkdocs/overrides"
  mkdir -p "$STAGE_DIR/mkdocs/.cache"
  mkdir -p "$STAGE_DIR/mkdocs/site"
  info "MkDocs (starter documentation)"
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -21,6 +21,14 @@ REPO="admin/changemaker.lite"
 INSTALL_DIR="${HOME}/changemaker.lite"
 VERSION=""
 LOCAL_TARBALL=""
+MIN_DISK_MB=10000
+HEALTH_TIMEOUT=180
+HEALTH_INTERVAL=5
+
+# --- State flags for cleanup ---
+EXTRACT_DIR=""
+TARBALL_PATH=""
+CONFIG_COMPLETE=false

 # --- Colors ---
 if [[ -t 1 ]] && [[ -z "${NO_COLOR:-}" ]]; then
@@ -35,6 +43,28 @@ success() { echo -e "${GREEN}[OK]${NC}    $*"; }
 warn()    { echo -e "${YELLOW}[WARN]${NC}  $*"; }
 error()   { echo -e "${RED}[ERROR]${NC} $*" >&2; }

+# --- Cleanup on failure ---
+cleanup() {
+  local exit_code=$?
+  if [[ $exit_code -ne 0 ]]; then
+    # Clean up temp extraction directory
+    if [[ -n "$EXTRACT_DIR" ]] && [[ -d "$EXTRACT_DIR" ]]; then
+      rm -rf "$EXTRACT_DIR"
+    fi
+    # Clean up downloaded tarball (but not user-provided ones)
+    if [[ -z "$LOCAL_TARBALL" ]] && [[ -n "$TARBALL_PATH" ]] && [[ -f "$TARBALL_PATH" ]]; then
+      rm -f "$TARBALL_PATH"
+    fi
+    # Remove install dir only if config wizard never ran (no user data to lose)
+    if [[ "$CONFIG_COMPLETE" == "false" ]] && [[ -d "$INSTALL_DIR" ]] && [[ ! -f "$INSTALL_DIR/.env" ]]; then
+      rm -rf "$INSTALL_DIR"
+    fi
+    echo ""
+    error "Installation failed. See errors above."
+  fi
+}
+trap cleanup EXIT
+
 # --- Arg parser ---
 while [[ $# -gt 0 ]]; do
  case "$1" in
@@ -51,7 +81,10 @@ done
 echo -e "${BOLD}Changemaker Lite — Installer${NC}"
 echo ""

-# --- Step 1: Check prerequisites ---
+# =============================================================================
+# Step 1: Check prerequisites
+# =============================================================================
+
 info "Checking prerequisites..."
 MISSING=()
 command -v docker    >/dev/null 2>&1 || MISSING+=("docker")
@@ -68,7 +101,30 @@ if [[ ${#MISSING[@]} -gt 0 ]]; then
 fi
 success "Prerequisites OK (Docker $(docker --version | grep -oP '\d+\.\d+\.\d+'), OpenSSL available)"

-# --- Step 2: Check install directory ---
+# Docker daemon must be running (not just installed)
+if ! docker info >/dev/null 2>&1; then
+  error "Docker daemon is not running."
+  echo ""
+  echo "  Start it with:  sudo systemctl start docker"
+  echo "  Or:             sudo service docker start"
+  exit 1
+fi
+success "Docker daemon is running"
+
+# Disk space check
+AVAILABLE_MB=$(df -m "$(dirname "$INSTALL_DIR")" | awk 'NR==2 {print $4}')
+if [[ "$AVAILABLE_MB" -lt "$MIN_DISK_MB" ]]; then
+  error "Insufficient disk space: ${AVAILABLE_MB}MB available, ${MIN_DISK_MB}MB required."
+  echo ""
+  echo "  The full stack (images + volumes) needs ~10GB of free space."
+  exit 1
+fi
+success "Disk space: ${AVAILABLE_MB}MB available (${MIN_DISK_MB}MB required)"
+
+# =============================================================================
+# Step 2: Check install directory
+# =============================================================================
+
 if [[ -d "$INSTALL_DIR" ]]; then
  if [[ -f "$INSTALL_DIR/docker-compose.yml" ]]; then
    error "Changemaker Lite is already installed at $INSTALL_DIR"
@@ -78,8 +134,10 @@ if [[ -d "$INSTALL_DIR" ]]; then
  fi
 fi

-# --- Step 3: Get tarball ---
-TARBALL_PATH=""
+# =============================================================================
+# Step 3: Get tarball
+# =============================================================================
+
 if [[ -n "$LOCAL_TARBALL" ]]; then
  if [[ ! -f "$LOCAL_TARBALL" ]]; then
    error "Tarball not found: $LOCAL_TARBALL"
@@ -101,7 +159,7 @@ else
  if [[ -z "$RELEASE_JSON" ]]; then
    error "Could not fetch release info from ${GITEA_URL}"
    echo ""
-    echo "If the registry requires authentication:"
+    echo "If the server is unreachable:"
    echo "  1. Download the tarball manually from ${GITEA_URL}/${REPO}/releases"
    echo "  2. Run: bash install.sh --tarball /path/to/changemaker-lite-*.tar.gz"
    exit 1
@@ -129,7 +187,10 @@ for a in assets:
  success "Downloaded $(du -h "$TARBALL_PATH" | cut -f1)"
 fi

-# --- Step 4: Extract ---
+# =============================================================================
+# Step 4: Extract
+# =============================================================================
+
 info "Extracting to ${INSTALL_DIR}..."
 mkdir -p "$(dirname "$INSTALL_DIR")"

@@ -141,12 +202,12 @@ tar xzf "$TARBALL_PATH" -C "$EXTRACT_DIR"
 EXTRACTED=$(find "$EXTRACT_DIR" -maxdepth 1 -mindepth 1 -type d | head -1)
 if [[ -z "$EXTRACTED" ]]; then
  error "Tarball extraction failed — no directory found"
-  rm -rf "$EXTRACT_DIR"
  exit 1
 fi

 mv "$EXTRACTED" "$INSTALL_DIR"
 rm -rf "$EXTRACT_DIR"
+EXTRACT_DIR=""  # Clear so cleanup doesn't try to remove it

 # Clean up downloaded tarball
 if [[ -z "$LOCAL_TARBALL" ]] && [[ -f "$TARBALL_PATH" ]]; then
@@ -155,23 +216,148 @@ fi

 success "Extracted to ${INSTALL_DIR}"

-# --- Step 5: Run config wizard ---
+# =============================================================================
+# Step 5: Run config wizard
+# =============================================================================
+
 echo ""
 echo -e "${BOLD}Starting configuration wizard...${NC}"
 echo ""
 cd "$INSTALL_DIR"
 bash config.sh
+CONFIG_COMPLETE=true
+
+# =============================================================================
+# Step 6: Start services
+# =============================================================================
+
+echo ""
+echo -e "${BOLD}Configuration complete!${NC}"
+echo ""
+
+START_SERVICES="y"
+if [[ -t 0 ]]; then
+  read -rp "Start all services now? [Y/n]: " START_SERVICES
+  START_SERVICES=${START_SERVICES:-y}
+fi
+
+if [[ "$START_SERVICES" =~ ^[Yy]$ ]]; then
+  echo ""
+  info "Pulling images from registry (this may take a few minutes on first run)..."
+  echo ""
+  cd "$INSTALL_DIR"
+
+  if ! docker compose pull 2>&1; then
+    echo ""
+    error "Failed to pull images from the registry."
+    echo ""
+    echo "  Check that the registry is reachable:"
+    echo "    curl -sf https://gitea.bnkops.com/api/v1/repos/admin/changemaker.lite/releases/latest"
+    echo ""
+    echo "  If pulling from a private registry, log in first:"
+    echo "    docker login gitea.bnkops.com"
+    echo ""
+    echo "  Then retry:"
+    echo "    cd ${INSTALL_DIR} && docker compose pull && docker compose up -d"
+    exit 1
+  fi
+  success "All images pulled"
+
+  echo ""
+  info "Starting services..."
+  if ! docker compose up -d 2>&1; then
+    echo ""
+    error "Failed to start services."
+    echo "  Check logs: docker compose logs --tail 30"
+    exit 1
+  fi
+
+  # --- Post-startup health verification ---
+  echo ""
+  info "Waiting for services to become healthy (up to ${HEALTH_TIMEOUT}s)..."
+  info "  Database migrations and seeding run automatically on first boot."
+  echo ""
+
+  CORE_SERVICES=("v2-postgres" "redis" "api" "admin")
+  ELAPSED=0
+  ALL_HEALTHY=false
+
+  while [[ $ELAPSED -lt $HEALTH_TIMEOUT ]]; do
+    ALL_HEALTHY=true
+
+    for svc in "${CORE_SERVICES[@]}"; do
+      # Detect crashed containers early
+      state=$(docker compose ps "$svc" --format '{{.State}}' 2>/dev/null || echo "missing")
+      if [[ "$state" == "exited" || "$state" == "dead" ]]; then
+        echo ""
+        error "Service '${svc}' exited unexpectedly. Last logs:"
+        docker compose logs "$svc" --tail 20 2>/dev/null || true
+        echo ""
+        error "Fix the issue and retry: cd ${INSTALL_DIR} && docker compose up -d"
+        exit 1
+      fi
+
+      # Check health status
+      health=$(docker compose ps "$svc" --format '{{.Health}}' 2>/dev/null || echo "")
+      if [[ "$health" != "healthy" ]]; then
+        ALL_HEALTHY=false
+      fi
+    done
+
+    if [[ "$ALL_HEALTHY" == "true" ]]; then
+      break
+    fi
+
+    sleep "$HEALTH_INTERVAL"
+    ELAPSED=$((ELAPSED + HEALTH_INTERVAL))
+  done
+
+  # Print status table
+  echo ""
+  echo -e "${BOLD}  Service Status:${NC}"
+  for svc in "${CORE_SERVICES[@]}"; do
+    health=$(docker compose ps "$svc" --format '{{.Health}}' 2>/dev/null || echo "unknown")
+    state=$(docker compose ps "$svc" --format '{{.State}}' 2>/dev/null || echo "unknown")
+    if [[ "$health" == "healthy" ]]; then
+      echo -e "    ${GREEN}[healthy]${NC}  $svc"
+    elif [[ "$state" == "running" ]]; then
+      echo -e "    ${YELLOW}[starting]${NC} $svc"
+    else
+      echo -e "    ${RED}[${state}]${NC}    $svc"
+    fi
+  done
+  echo ""
+
+  if [[ "$ALL_HEALTHY" == "true" ]]; then
+    success "All core services are healthy! (${ELAPSED}s)"
+    echo ""
+    echo "  Admin GUI:  http://localhost:3000"
+    echo "  API:        http://localhost:4000"
+    echo ""
+    echo "  Check full stack:  docker compose ps"
+    echo "  View API logs:     docker compose logs -f api --tail 20"
+  else
+    warn "Some services are still starting after ${HEALTH_TIMEOUT}s."
+    echo ""
+    echo "  This may be normal on first boot (migrations + seeding can be slow)."
+    echo "  Monitor progress with:"
+    echo "    docker compose logs -f api --tail 30"
+    echo ""
+    echo "  Check status with:"
+    echo "    docker compose ps"
+  fi
+else
+  echo ""
+  info "Skipped. Start services manually when ready:"
+  echo ""
+  echo "    cd ${INSTALL_DIR} && docker compose up -d"
+  echo ""
+  echo "  Pre-built images will be pulled from the registry (~2 min first time)."
+  echo "  Database migrations and seeding run automatically on startup."
+fi

-# --- Done ---
 echo ""
 echo -e "${BOLD}${GREEN}Installation complete!${NC}"
 echo ""
-echo "  Start all services:"
-echo "    cd ${INSTALL_DIR} && docker compose up -d"
-echo ""
-echo "  Check status:"
-echo "    docker compose ps"
-echo ""
-echo "  View API logs:"
-echo "    docker compose logs -f api --tail 20"
+echo -e "  \033[0;33mIMPORTANT: Change your admin password after first login!\033[0m"
 echo ""
--- a/scripts/mirror-images.sh
+++ b/scripts/mirror-images.sh
@@ -144,7 +144,7 @@ declare -A MONITORING_IMAGES=(
  ["oliver006/redis_exporter:v1.81.0"]="redis_exporter:v1.81.0"
  ["gcr.io/cadvisor/cadvisor:v0.55.1"]="cadvisor:v0.55.1"
  ["prom/node-exporter:v1.10.2"]="node-exporter:v1.10.2"
-  ["gotify/server:v2.9.0"]="gotify:v2.9.0"
+  ["gotify/server:2.9.0"]="gotify:v2.9.0"
 )

 # =============================================================================
--- a/scripts/systemd/changemaker-backup.service
+++ b/scripts/systemd/changemaker-backup.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Changemaker Lite database and uploads backup
+Documentation=https://docs.cmlite.org/docs/admin/backups/
+
+[Service]
+Type=oneshot
+User=__USER__
+Group=__USER__
+WorkingDirectory=__PROJECT_DIR__
+ExecStart=__PROJECT_DIR__/scripts/backup.sh --retention 30
+TimeoutStartSec=600
+StandardOutput=journal
+StandardError=journal
--- a/scripts/systemd/changemaker-backup.timer
+++ b/scripts/systemd/changemaker-backup.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily Changemaker Lite backup
+
+[Timer]
+OnCalendar=*-*-* 02:00:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target