From 7287328148cef60c8e75b6fb84bb467620e81204 Mon Sep 17 00:00:00 2001 From: bunker-admin Date: Wed, 25 Mar 2026 19:33:11 -0600 Subject: [PATCH] Harden install pipeline: health checks, log rotation, backup timer - install.sh: Add Docker daemon check, 10GB disk space pre-flight, error handling on pull/up, post-startup health polling with crash detection, cleanup trap on failure - docker-compose: Fix nginx/listmonk depends_on to service_healthy, add x-logging anchor (10m/3 files) to all ~39 services - config.sh: Preserve existing secrets on re-run (reconfigure mode), add automated daily backup timer (systemd, 02:00, 30-day retention) - mirror-images.sh: Fix gotify source tag (2.9.0 not v2.9.0) - build-release.sh: Ensure mkdocs/docs and mkdocs/overrides dirs exist - .env.example: Add COMPOSE_PROFILES variable Bunker Admin --- .env.example | 4 + config.sh | 262 +++++++++++++++++---- docker-compose.prod.yml | 54 ++++- docker-compose.yml | 54 ++++- scripts/build-release.sh | 4 +- scripts/install.sh | 220 +++++++++++++++-- scripts/mirror-images.sh | 2 +- scripts/systemd/changemaker-backup.service | 13 + scripts/systemd/changemaker-backup.timer | 10 + 9 files changed, 556 insertions(+), 67 deletions(-) create mode 100644 scripts/systemd/changemaker-backup.service create mode 100644 scripts/systemd/changemaker-backup.timer diff --git a/.env.example b/.env.example index 161eb6d..3f58567 100644 --- a/.env.example +++ b/.env.example @@ -179,6 +179,10 @@ VIDEO_PREVIEW_LINK_EXPIRY_HOURS=24 # Leave IMAGE_TAG blank/unset (defaults to 'local') to build locally from source. GITEA_REGISTRY=gitea.bnkops.com/admin IMAGE_TAG= + +# Docker Compose profiles — set to 'monitoring' to include Prometheus/Grafana/Alertmanager +# in every 'docker compose up -d'. Leave blank to start monitoring separately. +COMPOSE_PROFILES= # Credentials used by the registry status API endpoint (GET /api/registry/status) # For docker push/pull, run: docker login gitea.bnkops.com GITEA_REGISTRY_USER=admin diff --git a/config.sh b/config.sh index 7b2d0a8..4c00cff 100755 --- a/config.sh +++ b/config.sh @@ -180,6 +180,8 @@ initialize_env() { exit 1 fi + RECONFIGURE_MODE=false + if [[ -f "$ENV_FILE" ]]; then warn "Existing .env file found at $ENV_FILE" if prompt_yes_no "Back up existing .env and create a fresh one?"; then @@ -187,7 +189,8 @@ initialize_env() { cp "$ENV_EXAMPLE" "$ENV_FILE" success "Created fresh .env from .env.example" else - info "Keeping existing .env. Will update values in place." + info "Keeping existing .env. Existing secrets will be preserved." + RECONFIGURE_MODE=true fi else cp "$ENV_EXAMPLE" "$ENV_FILE" @@ -283,12 +286,37 @@ configure_admin() { success "Admin credentials configured" } +# Check if a key already has a real value in .env (non-empty, not a placeholder) +env_var_is_set() { + local key=$1 + local val + val=$(grep "^${key}=" "$ENV_FILE" 2>/dev/null | head -1 | cut -d= -f2-) + [[ -n "$val" && "$val" != "changeme" && "$val" != *"example"* && "$val" != *"CHANGEME"* ]] +} + +# Update an env var only if not already set (for reconfigure mode) +update_env_var_if_empty() { + local key=$1 + local value=$2 + if [[ "$RECONFIGURE_MODE" == "true" ]] && env_var_is_set "$key"; then + return 1 # signal: kept existing + fi + update_env_var "$key" "$value" + return 0 +} + generate_all_secrets() { header "Generating Secrets" - info "Auto-generating 22 unique secrets and passwords..." + if [[ "$RECONFIGURE_MODE" == "true" ]]; then + info "Reconfigure mode: existing secrets will be preserved." + else + info "Auto-generating 22 unique secrets and passwords..." + fi echo "" + local generated=0 kept=0 + # JWT & Encryption (64-char hex) local jwt_access jwt_refresh jwt_invite enc_key jwt_access=$(generate_secret) @@ -296,22 +324,41 @@ generate_all_secrets() { jwt_invite=$(generate_secret) enc_key=$(generate_secret) - update_env_var "JWT_ACCESS_SECRET" "$jwt_access" - update_env_var "JWT_REFRESH_SECRET" "$jwt_refresh" - update_env_var "JWT_INVITE_SECRET" "$jwt_invite" - update_env_var "ENCRYPTION_KEY" "$enc_key" - success "JWT secrets + encryption key" + local jwt_changed=false + update_env_var_if_empty "JWT_ACCESS_SECRET" "$jwt_access" && jwt_changed=true + update_env_var_if_empty "JWT_REFRESH_SECRET" "$jwt_refresh" && jwt_changed=true + update_env_var_if_empty "JWT_INVITE_SECRET" "$jwt_invite" && jwt_changed=true + update_env_var_if_empty "ENCRYPTION_KEY" "$enc_key" && jwt_changed=true + if [[ "$jwt_changed" == "true" ]]; then + success "JWT secrets + encryption key" + ((generated+=4)) + else + info "JWT secrets + encryption key (kept existing)" + ((kept+=4)) + fi # Database passwords (24-char alphanum) local pg_pass redis_pass pg_pass=$(generate_password 24) redis_pass=$(generate_password 24) - update_env_var "V2_POSTGRES_PASSWORD" "$pg_pass" - update_env_var "DATABASE_URL" "postgresql://changemaker:${pg_pass}@localhost:5433/changemaker_v2" - update_env_var "REDIS_PASSWORD" "$redis_pass" - update_env_var "REDIS_URL" "redis://:${redis_pass}@redis-changemaker:6379" - success "PostgreSQL + Redis passwords" + local db_changed=false + if update_env_var_if_empty "V2_POSTGRES_PASSWORD" "$pg_pass"; then + update_env_var "DATABASE_URL" "postgresql://changemaker:${pg_pass}@localhost:5433/changemaker_v2" + db_changed=true + fi + update_env_var_if_empty "REDIS_PASSWORD" "$redis_pass" && db_changed=true + if [[ "$db_changed" == "true" ]]; then + # Rebuild REDIS_URL if password changed + local current_redis_pass + current_redis_pass=$(grep "^REDIS_PASSWORD=" "$ENV_FILE" | cut -d= -f2-) + update_env_var "REDIS_URL" "redis://:${current_redis_pass}@redis-changemaker:6379" + success "PostgreSQL + Redis passwords" + ((generated+=2)) + else + info "PostgreSQL + Redis passwords (kept existing)" + ((kept+=2)) + fi # Listmonk local lm_db_pass lm_web_pass lm_api_token @@ -319,72 +366,133 @@ generate_all_secrets() { lm_web_pass=$(generate_password 20) lm_api_token=$(openssl rand -hex 16) - update_env_var "LISTMONK_DB_PASSWORD" "$lm_db_pass" - update_env_var "LISTMONK_WEB_ADMIN_PASSWORD" "$lm_web_pass" - update_env_var "LISTMONK_API_TOKEN" "$lm_api_token" - update_env_var "LISTMONK_ADMIN_PASSWORD" "$lm_api_token" - success "Listmonk passwords + API token" + local lm_changed=false + update_env_var_if_empty "LISTMONK_DB_PASSWORD" "$lm_db_pass" && lm_changed=true + update_env_var_if_empty "LISTMONK_WEB_ADMIN_PASSWORD" "$lm_web_pass" && lm_changed=true + if update_env_var_if_empty "LISTMONK_API_TOKEN" "$lm_api_token"; then + update_env_var "LISTMONK_ADMIN_PASSWORD" "$lm_api_token" + lm_changed=true + fi + if [[ "$lm_changed" == "true" ]]; then + success "Listmonk passwords + API token" + ((generated+=3)) + else + info "Listmonk passwords + API token (kept existing)" + ((kept+=3)) + fi # NocoDB local nc_pass nc_pass=$(generate_password 20) - update_env_var "NC_ADMIN_PASSWORD" "$nc_pass" - success "NocoDB admin password" + if update_env_var_if_empty "NC_ADMIN_PASSWORD" "$nc_pass"; then + success "NocoDB admin password" + ((generated++)) + else + info "NocoDB admin password (kept existing)" + ((kept++)) + fi # Gitea local gitea_db gitea_root gitea_db=$(generate_password 20) gitea_root=$(generate_password 20) - update_env_var "GITEA_DB_PASSWD" "$gitea_db" - update_env_var "GITEA_DB_ROOT_PASSWORD" "$gitea_root" - success "Gitea database passwords" + local gitea_changed=false + update_env_var_if_empty "GITEA_DB_PASSWD" "$gitea_db" && gitea_changed=true + update_env_var_if_empty "GITEA_DB_ROOT_PASSWORD" "$gitea_root" && gitea_changed=true + if [[ "$gitea_changed" == "true" ]]; then + success "Gitea database passwords" + ((generated+=2)) + else + info "Gitea database passwords (kept existing)" + ((kept+=2)) + fi # n8n local n8n_enc n8n_pass n8n_enc=$(generate_password 32) n8n_pass=$(generate_password 20) - update_env_var "N8N_ENCRYPTION_KEY" "$n8n_enc" - update_env_var "N8N_USER_PASSWORD" "$n8n_pass" - success "n8n encryption key + admin password" + local n8n_changed=false + update_env_var_if_empty "N8N_ENCRYPTION_KEY" "$n8n_enc" && n8n_changed=true + update_env_var_if_empty "N8N_USER_PASSWORD" "$n8n_pass" && n8n_changed=true + if [[ "$n8n_changed" == "true" ]]; then + success "n8n encryption key + admin password" + ((generated+=2)) + else + info "n8n encryption key + admin password (kept existing)" + ((kept+=2)) + fi # Monitoring local grafana_pass gotify_pass grafana_pass=$(generate_password 20) gotify_pass=$(generate_password 20) - update_env_var "GRAFANA_ADMIN_PASSWORD" "$grafana_pass" - update_env_var "GOTIFY_ADMIN_PASSWORD" "$gotify_pass" - success "Grafana + Gotify admin passwords" + local mon_changed=false + update_env_var_if_empty "GRAFANA_ADMIN_PASSWORD" "$grafana_pass" && mon_changed=true + update_env_var_if_empty "GOTIFY_ADMIN_PASSWORD" "$gotify_pass" && mon_changed=true + if [[ "$mon_changed" == "true" ]]; then + success "Grafana + Gotify admin passwords" + ((generated+=2)) + else + info "Grafana + Gotify admin passwords (kept existing)" + ((kept+=2)) + fi # Vaultwarden local vw_admin_token vw_admin_token=$(generate_secret) - update_env_var "VAULTWARDEN_ADMIN_TOKEN" "$vw_admin_token" - success "Vaultwarden admin token" + if update_env_var_if_empty "VAULTWARDEN_ADMIN_TOKEN" "$vw_admin_token"; then + success "Vaultwarden admin token" + ((generated++)) + else + info "Vaultwarden admin token (kept existing)" + ((kept++)) + fi # Rocket.Chat local rc_pass rc_pass=$(generate_password 20) - update_env_var "ROCKETCHAT_ADMIN_PASSWORD" "$rc_pass" - success "Rocket.Chat admin password" + if update_env_var_if_empty "ROCKETCHAT_ADMIN_PASSWORD" "$rc_pass"; then + success "Rocket.Chat admin password" + ((generated++)) + else + info "Rocket.Chat admin password (kept existing)" + ((kept++)) + fi # Gancio local gancio_pass gancio_pass=$(generate_password 20) - update_env_var "GANCIO_ADMIN_PASSWORD" "$gancio_pass" - success "Gancio admin password" + if update_env_var_if_empty "GANCIO_ADMIN_PASSWORD" "$gancio_pass"; then + success "Gancio admin password" + ((generated++)) + else + info "Gancio admin password (kept existing)" + ((kept++)) + fi # Jitsi Meet local jitsi_secret jitsi_jicofo jitsi_jvb jitsi_secret=$(generate_secret) jitsi_jicofo=$(openssl rand -hex 16) jitsi_jvb=$(openssl rand -hex 16) - update_env_var "JITSI_APP_SECRET" "$jitsi_secret" - update_env_var "JITSI_JICOFO_AUTH_PASSWORD" "$jitsi_jicofo" - update_env_var "JITSI_JVB_AUTH_PASSWORD" "$jitsi_jvb" - success "Jitsi Meet secrets (JWT + XMPP)" + local jitsi_changed=false + update_env_var_if_empty "JITSI_APP_SECRET" "$jitsi_secret" && jitsi_changed=true + update_env_var_if_empty "JITSI_JICOFO_AUTH_PASSWORD" "$jitsi_jicofo" && jitsi_changed=true + update_env_var_if_empty "JITSI_JVB_AUTH_PASSWORD" "$jitsi_jvb" && jitsi_changed=true + if [[ "$jitsi_changed" == "true" ]]; then + success "Jitsi Meet secrets (JWT + XMPP)" + ((generated+=3)) + else + info "Jitsi Meet secrets (kept existing)" + ((kept+=3)) + fi echo "" - success "All 21 secrets generated. No placeholder passwords remain." + if [[ $kept -gt 0 ]]; then + success "Secrets: ${generated} generated, ${kept} preserved from existing .env" + else + success "All 22 secrets generated. No placeholder passwords remain." + fi } configure_smtp() { @@ -519,6 +627,14 @@ configure_features() { DOCS_COMMENTS_ENABLED="no" fi + if prompt_yes_no "Enable Monitoring stack (Prometheus, Grafana, Alertmanager, cAdvisor)?" "y"; then + update_env_var "COMPOSE_PROFILES" "monitoring" + success "Monitoring enabled (COMPOSE_PROFILES=monitoring)" + MONITORING_ENABLED="yes" + else + MONITORING_ENABLED="no" + fi + if prompt_yes_no "Enable Bunker Ops (fleet metrics push to central server)?"; then update_env_var "BUNKER_OPS_ENABLED" "true" success "Bunker Ops enabled" @@ -994,16 +1110,21 @@ fix_container_permissions() { local -a dirs=( "configs/code-server/.config:Code Server config" "configs/code-server/.local:Code Server local data" + "mkdocs:MkDocs root" + "mkdocs/docs:MkDocs source docs" + "mkdocs/overrides:MkDocs template overrides" "mkdocs/.cache:MkDocs cache" "mkdocs/site:MkDocs built site" "assets/uploads:Listmonk uploads" "assets/images:Shared images" "assets/icons:Homepage icons" "media/local/inbox:Media upload inbox" + "media/local/photos:Media photos" "media/local/thumbnails:Video thumbnails" "media/public:Public media files" "local-files:n8n local files" "data:NAR import data" + "data/upgrade:Upgrade trigger directory" ) local errors=0 @@ -1085,6 +1206,59 @@ install_upgrade_watcher() { fi } +# ============================================================================= +# Automated Backups (systemd) +# ============================================================================= + +install_backup_timer() { + header "Automated Backups" + + info "Daily automated backups protect against data loss." + info "Backs up PostgreSQL databases + uploads to ./backups/ with 30-day retention." + echo "" + + local unit_src="$SCRIPT_DIR/scripts/systemd" + if [[ ! -f "$unit_src/changemaker-backup.timer" ]] || [[ ! -f "$unit_src/changemaker-backup.service" ]]; then + warn "Systemd backup unit templates not found in scripts/systemd/ — skipping" + BACKUP_TIMER="skipped" + return + fi + + if ! command -v systemctl &>/dev/null; then + warn "systemctl not found — skipping (not a systemd host?)" + BACKUP_TIMER="skipped" + return + fi + + if prompt_yes_no "Install daily automated backups (requires sudo)?" "y"; then + local tmp_timer tmp_service + tmp_timer=$(mktemp) + tmp_service=$(mktemp) + + sed -e "s|__PROJECT_DIR__|$SCRIPT_DIR|g" "$unit_src/changemaker-backup.timer" > "$tmp_timer" + sed -e "s|__PROJECT_DIR__|$SCRIPT_DIR|g" -e "s|__USER__|$(whoami)|g" "$unit_src/changemaker-backup.service" > "$tmp_service" + + if sudo cp "$tmp_timer" /etc/systemd/system/changemaker-backup.timer \ + && sudo cp "$tmp_service" /etc/systemd/system/changemaker-backup.service \ + && sudo systemctl daemon-reload \ + && sudo systemctl enable --now changemaker-backup.timer; then + success "Daily backup timer installed and enabled (runs at 02:00)" + BACKUP_TIMER="yes" + else + warn "Failed to install systemd units (sudo may have failed)" + warn "Install manually later:" + echo -e " ${CYAN}sudo cp scripts/systemd/changemaker-backup.* /etc/systemd/system/${NC}" + echo -e " ${CYAN}sudo systemctl daemon-reload && sudo systemctl enable --now changemaker-backup.timer${NC}" + BACKUP_TIMER="manual" + fi + + rm -f "$tmp_timer" "$tmp_service" + else + info "Skipped. Run backups manually: ./scripts/backup.sh" + BACKUP_TIMER="skipped" + fi +} + # ============================================================================= # Summary & Next Steps # ============================================================================= @@ -1103,10 +1277,12 @@ print_summary() { echo -e " ${BOLD}Gancio sync:${NC} ${GANCIO_SYNC:-no}" echo -e " ${BOLD}Jitsi Meet:${NC} ${MEET_ENABLED:-no}" echo -e " ${BOLD}SMS Campaigns:${NC} ${SMS_ENABLED:-no}" + echo -e " ${BOLD}Monitoring:${NC} ${MONITORING_ENABLED:-no}" echo -e " ${BOLD}Docs Comments:${NC} ${DOCS_COMMENTS_ENABLED:-no}" echo -e " ${BOLD}Bunker Ops:${NC} ${BUNKER_OPS_ENABLED:-no}" echo -e " ${BOLD}Pangolin:${NC} ${PANGOLIN_CONFIGURED:-no}" echo -e " ${BOLD}Upgrade watcher:${NC} ${UPGRADE_WATCHER:-skipped}" + echo -e " ${BOLD}Backup timer:${NC} ${BACKUP_TIMER:-skipped}" echo -e " ${BOLD}Secrets:${NC} 22 auto-generated" echo "" echo -e " ${DIM}Config file: $ENV_FILE${NC}" @@ -1155,9 +1331,8 @@ print_next_steps() { echo -e " ${CYAN}docker compose up -d rocketchat${NC} # Team chat" echo -e " ${CYAN}docker compose up -d jitsi-web jitsi-prosody jitsi-jicofo jitsi-jvb${NC} # Video calls" echo -e " ${CYAN}docker compose up -d homepage${NC} # Service dashboard" - echo -e " ${CYAN}docker compose --profile monitoring up -d${NC} # Monitoring" echo "" - echo -e " ${BOLD}5.${NC} Or start everything at once:" + echo -e " ${BOLD}5.${NC} Or start everything at once (monitoring included if enabled above):" echo -e " ${CYAN}docker compose up -d${NC}" echo "" fi @@ -1187,12 +1362,17 @@ main() { generate_services_yaml fix_container_permissions install_upgrade_watcher + install_backup_timer # Release mode: auto-set production defaults if [[ "$INSTALL_MODE" == "release" ]]; then header "Release Mode Settings" update_env_var "IMAGE_TAG" "latest" update_env_var "NODE_ENV" "production" + # Ensure monitoring is included if user opted in + if [[ "${MONITORING_ENABLED:-no}" == "yes" ]]; then + update_env_var "COMPOSE_PROFILES" "monitoring" + fi success "Set IMAGE_TAG=latest, NODE_ENV=production (pre-built images)" fi diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 0501673..6a08a26 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -6,6 +6,12 @@ ############################################################################### ############################################################################### +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + services: # ========================================================================= # V2 CORE SERVICES @@ -129,6 +135,7 @@ services: condition: service_healthy redis: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -176,6 +183,7 @@ services: depends_on: v2-postgres: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -202,6 +210,7 @@ services: - VITE_MKDOCS_SITE_PORT=${MKDOCS_SITE_SERVER_PORT:-4004} depends_on: - api + logging: *default-logging networks: - changemaker-lite @@ -225,6 +234,7 @@ services: interval: 10s timeout: 5s retries: 5 + logging: *default-logging networks: - changemaker-lite @@ -277,8 +287,11 @@ services: - ./public-web:/usr/share/nginx/public-web:ro - ./configs/pangolin:/etc/pangolin:ro depends_on: - - api - - admin + api: + condition: service_healthy + admin: + condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -305,6 +318,7 @@ services: depends_on: v2-postgres: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -330,6 +344,7 @@ services: volumes: - ./scripts/nocodb-init.sh:/init.sh:ro entrypoint: ["/bin/sh", "/init.sh"] + logging: *default-logging networks: - changemaker-lite @@ -382,7 +397,8 @@ services: retries: 3 start_period: 30s depends_on: - - listmonk-db + listmonk-db: + condition: service_healthy command: [sh, -c, "./listmonk --install --idempotent --yes --config '' && ./listmonk --upgrade --yes --config '' && ./listmonk --config ''"] environment: LISTMONK_app__address: 0.0.0.0:9000 @@ -397,6 +413,7 @@ services: LISTMONK_ADMIN_PASSWORD: ${LISTMONK_WEB_ADMIN_PASSWORD:-} volumes: - ./assets/uploads:/listmonk/uploads:rw + logging: *default-logging networks: - changemaker-lite @@ -417,6 +434,7 @@ services: retries: 6 volumes: - listmonk-data:/var/lib/postgresql/data + logging: *default-logging networks: - changemaker-lite @@ -485,6 +503,7 @@ services: fi echo "[listmonk-init] Done" + logging: *default-logging networks: - changemaker-lite @@ -512,6 +531,7 @@ services: ports: - "127.0.0.1:${CODE_SERVER_PORT:-8888}:8080" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -540,6 +560,7 @@ services: - GANCIO_PORT=${GANCIO_PORT:-8092} entrypoint: ["/bin/sh", "/scripts/mkdocs-entrypoint.sh"] restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -557,6 +578,7 @@ services: ports: - "127.0.0.1:${MKDOCS_SITE_SERVER_PORT:-4004}:80" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -587,6 +609,7 @@ services: volumes: - n8n-data:/home/node/.n8n - ./local-files:/files + logging: *default-logging networks: - changemaker-lite @@ -608,6 +631,7 @@ services: - HOMEPAGE_ALLOWED_HOSTS=* - HOMEPAGE_VAR_BASE_URL=${HOMEPAGE_VAR_BASE_URL:-http://localhost} restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -649,6 +673,7 @@ services: - "127.0.0.1:${GITEA_SSH_PORT:-2222}:22" depends_on: - gitea-db + logging: *default-logging networks: - changemaker-lite @@ -668,6 +693,7 @@ services: interval: 10s timeout: 5s retries: 5 + logging: *default-logging networks: - changemaker-lite @@ -678,6 +704,7 @@ services: ports: - "127.0.0.1:${MINI_QR_PORT:-8089}:8080" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -696,6 +723,7 @@ services: start_period: 20s environment: - VITE_APP_COLLAB_SERVER_URL=${EXCALIDRAW_WS_URL:-wss://draw.cmlite.org} + logging: *default-logging networks: - changemaker-lite @@ -728,6 +756,7 @@ services: - SMTP_PASSWORD=${SMTP_PASS:-} volumes: - vaultwarden-data:/data + logging: *default-logging networks: - changemaker-lite @@ -794,6 +823,7 @@ services: rm -f "$$SESSION_COOKIE" echo "[vaultwarden-init] Done" + logging: *default-logging networks: - changemaker-lite @@ -834,6 +864,7 @@ services: - OVERWRITE_SETTING_VideoConf_Default_Provider=jitsi volumes: - rocketchat-uploads:/app/uploads + logging: *default-logging networks: - changemaker-lite healthcheck: @@ -849,6 +880,7 @@ services: container_name: nats-rocketchat restart: unless-stopped command: --http_port 8222 + logging: *default-logging networks: - changemaker-lite @@ -860,6 +892,7 @@ services: command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] volumes: - mongodb-rocketchat-data:/data/db + logging: *default-logging networks: - changemaker-lite healthcheck: @@ -897,6 +930,7 @@ services: - server__baseurl=${GANCIO_BASE_URL:-https://events.cmlite.org} volumes: - gancio-data:/home/node/data + logging: *default-logging networks: - changemaker-lite @@ -938,6 +972,7 @@ services: ON CONFLICT (key) DO NOTHING;" echo "Gancio theme settings seeded." restart: "no" + logging: *default-logging networks: - changemaker-lite @@ -976,6 +1011,7 @@ services: timeout: 5s retries: 5 start_period: 30s + logging: *default-logging networks: - changemaker-lite @@ -1009,6 +1045,7 @@ services: timeout: 5s retries: 5 start_period: 30s + logging: *default-logging networks: - changemaker-lite @@ -1032,6 +1069,7 @@ services: - TZ=UTC volumes: - jitsi-jicofo-config:/config + logging: *default-logging networks: - changemaker-lite @@ -1057,6 +1095,7 @@ services: - TZ=UTC volumes: - jitsi-jvb-config:/config + logging: *default-logging networks: - changemaker-lite @@ -1091,6 +1130,7 @@ services: - NEWT_SECRET=${PANGOLIN_NEWT_SECRET} depends_on: - nginx + logging: *default-logging networks: - changemaker-lite @@ -1109,6 +1149,7 @@ services: - SWARM=0 volumes: - /var/run/docker.sock:/var/run/docker.sock:ro + logging: *default-logging networks: - changemaker-lite @@ -1129,6 +1170,7 @@ services: - ./configs/prometheus:/etc/prometheus - prometheus-data:/prometheus restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1152,6 +1194,7 @@ services: restart: always depends_on: - prometheus + logging: *default-logging networks: - changemaker-lite profiles: @@ -1173,6 +1216,7 @@ services: devices: - /dev/kmsg restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1193,6 +1237,7 @@ services: - /sys:/host/sys:ro - /:/rootfs:ro restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1209,6 +1254,7 @@ services: restart: always depends_on: - redis + logging: *default-logging networks: - changemaker-lite profiles: @@ -1226,6 +1272,7 @@ services: - '--config.file=/etc/alertmanager/alertmanager.yml' - '--storage.path=/alertmanager' restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1243,6 +1290,7 @@ services: volumes: - gotify-data:/app/data restart: always + logging: *default-logging networks: - changemaker-lite profiles: diff --git a/docker-compose.yml b/docker-compose.yml index db8d188..f3f79c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,12 @@ # Changemaker Lite v2 — Docker Compose ############################################################################### +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + services: # ========================================================================= # V2 CORE SERVICES @@ -130,6 +136,7 @@ services: condition: service_healthy redis: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -183,6 +190,7 @@ services: depends_on: v2-postgres: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -215,6 +223,7 @@ services: - /app/node_modules depends_on: - api + logging: *default-logging networks: - changemaker-lite @@ -238,6 +247,7 @@ services: interval: 10s timeout: 5s retries: 5 + logging: *default-logging networks: - changemaker-lite @@ -292,8 +302,11 @@ services: - ./public-web:/usr/share/nginx/public-web:ro - ./configs/pangolin:/etc/pangolin:ro depends_on: - - api - - admin + api: + condition: service_healthy + admin: + condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -320,6 +333,7 @@ services: depends_on: v2-postgres: condition: service_healthy + logging: *default-logging networks: - changemaker-lite @@ -345,6 +359,7 @@ services: volumes: - ./scripts/nocodb-init.sh:/init.sh:ro entrypoint: ["/bin/sh", "/init.sh"] + logging: *default-logging networks: - changemaker-lite @@ -397,7 +412,8 @@ services: retries: 3 start_period: 30s depends_on: - - listmonk-db + listmonk-db: + condition: service_healthy command: [sh, -c, "./listmonk --install --idempotent --yes --config '' && ./listmonk --upgrade --yes --config '' && ./listmonk --config ''"] environment: LISTMONK_app__address: 0.0.0.0:9000 @@ -412,6 +428,7 @@ services: LISTMONK_ADMIN_PASSWORD: ${LISTMONK_WEB_ADMIN_PASSWORD:-} volumes: - ./assets/uploads:/listmonk/uploads:rw + logging: *default-logging networks: - changemaker-lite @@ -432,6 +449,7 @@ services: retries: 6 volumes: - listmonk-data:/var/lib/postgresql/data + logging: *default-logging networks: - changemaker-lite @@ -500,6 +518,7 @@ services: fi echo "[listmonk-init] Done" + logging: *default-logging networks: - changemaker-lite @@ -532,6 +551,7 @@ services: ports: - "127.0.0.1:${CODE_SERVER_PORT:-8888}:8080" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -560,6 +580,7 @@ services: - GANCIO_PORT=${GANCIO_PORT:-8092} entrypoint: ["/bin/sh", "/scripts/mkdocs-entrypoint.sh"] restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -577,6 +598,7 @@ services: ports: - "127.0.0.1:${MKDOCS_SITE_SERVER_PORT:-4004}:80" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -607,6 +629,7 @@ services: volumes: - n8n-data:/home/node/.n8n - ./local-files:/files + logging: *default-logging networks: - changemaker-lite @@ -628,6 +651,7 @@ services: - HOMEPAGE_ALLOWED_HOSTS=* - HOMEPAGE_VAR_BASE_URL=${HOMEPAGE_VAR_BASE_URL:-http://localhost} restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -669,6 +693,7 @@ services: - "127.0.0.1:${GITEA_SSH_PORT:-2222}:22" depends_on: - gitea-db + logging: *default-logging networks: - changemaker-lite @@ -688,6 +713,7 @@ services: interval: 10s timeout: 5s retries: 5 + logging: *default-logging networks: - changemaker-lite @@ -698,6 +724,7 @@ services: ports: - "127.0.0.1:${MINI_QR_PORT:-8089}:8080" restart: unless-stopped + logging: *default-logging networks: - changemaker-lite @@ -716,6 +743,7 @@ services: start_period: 20s environment: - VITE_APP_COLLAB_SERVER_URL=${EXCALIDRAW_WS_URL:-wss://draw.cmlite.org} + logging: *default-logging networks: - changemaker-lite @@ -748,6 +776,7 @@ services: - SMTP_PASSWORD=${SMTP_PASS:-} volumes: - vaultwarden-data:/data + logging: *default-logging networks: - changemaker-lite @@ -814,6 +843,7 @@ services: rm -f "$$SESSION_COOKIE" echo "[vaultwarden-init] Done" + logging: *default-logging networks: - changemaker-lite @@ -854,6 +884,7 @@ services: - OVERWRITE_SETTING_VideoConf_Default_Provider=jitsi volumes: - rocketchat-uploads:/app/uploads + logging: *default-logging networks: - changemaker-lite healthcheck: @@ -869,6 +900,7 @@ services: container_name: nats-rocketchat restart: unless-stopped command: --http_port 8222 + logging: *default-logging networks: - changemaker-lite @@ -880,6 +912,7 @@ services: command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] volumes: - mongodb-rocketchat-data:/data/db + logging: *default-logging networks: - changemaker-lite healthcheck: @@ -917,6 +950,7 @@ services: - server__baseurl=${GANCIO_BASE_URL:-https://events.cmlite.org} volumes: - gancio-data:/home/node/data + logging: *default-logging networks: - changemaker-lite @@ -958,6 +992,7 @@ services: ON CONFLICT (key) DO NOTHING;" echo "Gancio theme settings seeded." restart: "no" + logging: *default-logging networks: - changemaker-lite @@ -996,6 +1031,7 @@ services: timeout: 5s retries: 5 start_period: 30s + logging: *default-logging networks: - changemaker-lite @@ -1029,6 +1065,7 @@ services: timeout: 5s retries: 5 start_period: 30s + logging: *default-logging networks: - changemaker-lite @@ -1052,6 +1089,7 @@ services: - TZ=UTC volumes: - jitsi-jicofo-config:/config + logging: *default-logging networks: - changemaker-lite @@ -1077,6 +1115,7 @@ services: - TZ=UTC volumes: - jitsi-jvb-config:/config + logging: *default-logging networks: - changemaker-lite @@ -1111,6 +1150,7 @@ services: - NEWT_SECRET=${PANGOLIN_NEWT_SECRET} depends_on: - nginx + logging: *default-logging networks: - changemaker-lite @@ -1129,6 +1169,7 @@ services: - SWARM=0 volumes: - /var/run/docker.sock:/var/run/docker.sock:ro + logging: *default-logging networks: - changemaker-lite @@ -1149,6 +1190,7 @@ services: - ./configs/prometheus:/etc/prometheus - prometheus-data:/prometheus restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1172,6 +1214,7 @@ services: restart: always depends_on: - prometheus + logging: *default-logging networks: - changemaker-lite profiles: @@ -1193,6 +1236,7 @@ services: devices: - /dev/kmsg restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1213,6 +1257,7 @@ services: - /sys:/host/sys:ro - /:/rootfs:ro restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1229,6 +1274,7 @@ services: restart: always depends_on: - redis + logging: *default-logging networks: - changemaker-lite profiles: @@ -1246,6 +1292,7 @@ services: - '--config.file=/etc/alertmanager/alertmanager.yml' - '--storage.path=/alertmanager' restart: always + logging: *default-logging networks: - changemaker-lite profiles: @@ -1263,6 +1310,7 @@ services: volumes: - gotify-data:/app/data restart: always + logging: *default-logging networks: - changemaker-lite profiles: diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 9d96541..ab3773a 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -144,8 +144,8 @@ info "Nginx templates (for reference)" if [[ -d "$PROJECT_DIR/mkdocs" ]]; then mkdir -p "$STAGE_DIR/mkdocs" cp "$PROJECT_DIR/mkdocs/mkdocs.yml" "$STAGE_DIR/mkdocs/" 2>/dev/null || true - cp -r "$PROJECT_DIR/mkdocs/docs" "$STAGE_DIR/mkdocs/" 2>/dev/null || true - cp -r "$PROJECT_DIR/mkdocs/overrides" "$STAGE_DIR/mkdocs/" 2>/dev/null || true + cp -r "$PROJECT_DIR/mkdocs/docs" "$STAGE_DIR/mkdocs/" 2>/dev/null || mkdir -p "$STAGE_DIR/mkdocs/docs" + cp -r "$PROJECT_DIR/mkdocs/overrides" "$STAGE_DIR/mkdocs/" 2>/dev/null || mkdir -p "$STAGE_DIR/mkdocs/overrides" mkdir -p "$STAGE_DIR/mkdocs/.cache" mkdir -p "$STAGE_DIR/mkdocs/site" info "MkDocs (starter documentation)" diff --git a/scripts/install.sh b/scripts/install.sh index 0f6a19d..9187ec8 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -21,6 +21,14 @@ REPO="admin/changemaker.lite" INSTALL_DIR="${HOME}/changemaker.lite" VERSION="" LOCAL_TARBALL="" +MIN_DISK_MB=10000 +HEALTH_TIMEOUT=180 +HEALTH_INTERVAL=5 + +# --- State flags for cleanup --- +EXTRACT_DIR="" +TARBALL_PATH="" +CONFIG_COMPLETE=false # --- Colors --- if [[ -t 1 ]] && [[ -z "${NO_COLOR:-}" ]]; then @@ -35,6 +43,28 @@ success() { echo -e "${GREEN}[OK]${NC} $*"; } warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } +# --- Cleanup on failure --- +cleanup() { + local exit_code=$? + if [[ $exit_code -ne 0 ]]; then + # Clean up temp extraction directory + if [[ -n "$EXTRACT_DIR" ]] && [[ -d "$EXTRACT_DIR" ]]; then + rm -rf "$EXTRACT_DIR" + fi + # Clean up downloaded tarball (but not user-provided ones) + if [[ -z "$LOCAL_TARBALL" ]] && [[ -n "$TARBALL_PATH" ]] && [[ -f "$TARBALL_PATH" ]]; then + rm -f "$TARBALL_PATH" + fi + # Remove install dir only if config wizard never ran (no user data to lose) + if [[ "$CONFIG_COMPLETE" == "false" ]] && [[ -d "$INSTALL_DIR" ]] && [[ ! -f "$INSTALL_DIR/.env" ]]; then + rm -rf "$INSTALL_DIR" + fi + echo "" + error "Installation failed. See errors above." + fi +} +trap cleanup EXIT + # --- Arg parser --- while [[ $# -gt 0 ]]; do case "$1" in @@ -51,7 +81,10 @@ done echo -e "${BOLD}Changemaker Lite — Installer${NC}" echo "" -# --- Step 1: Check prerequisites --- +# ============================================================================= +# Step 1: Check prerequisites +# ============================================================================= + info "Checking prerequisites..." MISSING=() command -v docker >/dev/null 2>&1 || MISSING+=("docker") @@ -68,7 +101,30 @@ if [[ ${#MISSING[@]} -gt 0 ]]; then fi success "Prerequisites OK (Docker $(docker --version | grep -oP '\d+\.\d+\.\d+'), OpenSSL available)" -# --- Step 2: Check install directory --- +# Docker daemon must be running (not just installed) +if ! docker info >/dev/null 2>&1; then + error "Docker daemon is not running." + echo "" + echo " Start it with: sudo systemctl start docker" + echo " Or: sudo service docker start" + exit 1 +fi +success "Docker daemon is running" + +# Disk space check +AVAILABLE_MB=$(df -m "$(dirname "$INSTALL_DIR")" | awk 'NR==2 {print $4}') +if [[ "$AVAILABLE_MB" -lt "$MIN_DISK_MB" ]]; then + error "Insufficient disk space: ${AVAILABLE_MB}MB available, ${MIN_DISK_MB}MB required." + echo "" + echo " The full stack (images + volumes) needs ~10GB of free space." + exit 1 +fi +success "Disk space: ${AVAILABLE_MB}MB available (${MIN_DISK_MB}MB required)" + +# ============================================================================= +# Step 2: Check install directory +# ============================================================================= + if [[ -d "$INSTALL_DIR" ]]; then if [[ -f "$INSTALL_DIR/docker-compose.yml" ]]; then error "Changemaker Lite is already installed at $INSTALL_DIR" @@ -78,8 +134,10 @@ if [[ -d "$INSTALL_DIR" ]]; then fi fi -# --- Step 3: Get tarball --- -TARBALL_PATH="" +# ============================================================================= +# Step 3: Get tarball +# ============================================================================= + if [[ -n "$LOCAL_TARBALL" ]]; then if [[ ! -f "$LOCAL_TARBALL" ]]; then error "Tarball not found: $LOCAL_TARBALL" @@ -101,7 +159,7 @@ else if [[ -z "$RELEASE_JSON" ]]; then error "Could not fetch release info from ${GITEA_URL}" echo "" - echo "If the registry requires authentication:" + echo "If the server is unreachable:" echo " 1. Download the tarball manually from ${GITEA_URL}/${REPO}/releases" echo " 2. Run: bash install.sh --tarball /path/to/changemaker-lite-*.tar.gz" exit 1 @@ -129,7 +187,10 @@ for a in assets: success "Downloaded $(du -h "$TARBALL_PATH" | cut -f1)" fi -# --- Step 4: Extract --- +# ============================================================================= +# Step 4: Extract +# ============================================================================= + info "Extracting to ${INSTALL_DIR}..." mkdir -p "$(dirname "$INSTALL_DIR")" @@ -141,12 +202,12 @@ tar xzf "$TARBALL_PATH" -C "$EXTRACT_DIR" EXTRACTED=$(find "$EXTRACT_DIR" -maxdepth 1 -mindepth 1 -type d | head -1) if [[ -z "$EXTRACTED" ]]; then error "Tarball extraction failed — no directory found" - rm -rf "$EXTRACT_DIR" exit 1 fi mv "$EXTRACTED" "$INSTALL_DIR" rm -rf "$EXTRACT_DIR" +EXTRACT_DIR="" # Clear so cleanup doesn't try to remove it # Clean up downloaded tarball if [[ -z "$LOCAL_TARBALL" ]] && [[ -f "$TARBALL_PATH" ]]; then @@ -155,23 +216,148 @@ fi success "Extracted to ${INSTALL_DIR}" -# --- Step 5: Run config wizard --- +# ============================================================================= +# Step 5: Run config wizard +# ============================================================================= + echo "" echo -e "${BOLD}Starting configuration wizard...${NC}" echo "" cd "$INSTALL_DIR" bash config.sh +CONFIG_COMPLETE=true + +# ============================================================================= +# Step 6: Start services +# ============================================================================= + +echo "" +echo -e "${BOLD}Configuration complete!${NC}" +echo "" + +START_SERVICES="y" +if [[ -t 0 ]]; then + read -rp "Start all services now? [Y/n]: " START_SERVICES + START_SERVICES=${START_SERVICES:-y} +fi + +if [[ "$START_SERVICES" =~ ^[Yy]$ ]]; then + echo "" + info "Pulling images from registry (this may take a few minutes on first run)..." + echo "" + cd "$INSTALL_DIR" + + if ! docker compose pull 2>&1; then + echo "" + error "Failed to pull images from the registry." + echo "" + echo " Check that the registry is reachable:" + echo " curl -sf https://gitea.bnkops.com/api/v1/repos/admin/changemaker.lite/releases/latest" + echo "" + echo " If pulling from a private registry, log in first:" + echo " docker login gitea.bnkops.com" + echo "" + echo " Then retry:" + echo " cd ${INSTALL_DIR} && docker compose pull && docker compose up -d" + exit 1 + fi + success "All images pulled" + + echo "" + info "Starting services..." + if ! docker compose up -d 2>&1; then + echo "" + error "Failed to start services." + echo " Check logs: docker compose logs --tail 30" + exit 1 + fi + + # --- Post-startup health verification --- + echo "" + info "Waiting for services to become healthy (up to ${HEALTH_TIMEOUT}s)..." + info " Database migrations and seeding run automatically on first boot." + echo "" + + CORE_SERVICES=("v2-postgres" "redis" "api" "admin") + ELAPSED=0 + ALL_HEALTHY=false + + while [[ $ELAPSED -lt $HEALTH_TIMEOUT ]]; do + ALL_HEALTHY=true + + for svc in "${CORE_SERVICES[@]}"; do + # Detect crashed containers early + state=$(docker compose ps "$svc" --format '{{.State}}' 2>/dev/null || echo "missing") + if [[ "$state" == "exited" || "$state" == "dead" ]]; then + echo "" + error "Service '${svc}' exited unexpectedly. Last logs:" + docker compose logs "$svc" --tail 20 2>/dev/null || true + echo "" + error "Fix the issue and retry: cd ${INSTALL_DIR} && docker compose up -d" + exit 1 + fi + + # Check health status + health=$(docker compose ps "$svc" --format '{{.Health}}' 2>/dev/null || echo "") + if [[ "$health" != "healthy" ]]; then + ALL_HEALTHY=false + fi + done + + if [[ "$ALL_HEALTHY" == "true" ]]; then + break + fi + + sleep "$HEALTH_INTERVAL" + ELAPSED=$((ELAPSED + HEALTH_INTERVAL)) + done + + # Print status table + echo "" + echo -e "${BOLD} Service Status:${NC}" + for svc in "${CORE_SERVICES[@]}"; do + health=$(docker compose ps "$svc" --format '{{.Health}}' 2>/dev/null || echo "unknown") + state=$(docker compose ps "$svc" --format '{{.State}}' 2>/dev/null || echo "unknown") + if [[ "$health" == "healthy" ]]; then + echo -e " ${GREEN}[healthy]${NC} $svc" + elif [[ "$state" == "running" ]]; then + echo -e " ${YELLOW}[starting]${NC} $svc" + else + echo -e " ${RED}[${state}]${NC} $svc" + fi + done + echo "" + + if [[ "$ALL_HEALTHY" == "true" ]]; then + success "All core services are healthy! (${ELAPSED}s)" + echo "" + echo " Admin GUI: http://localhost:3000" + echo " API: http://localhost:4000" + echo "" + echo " Check full stack: docker compose ps" + echo " View API logs: docker compose logs -f api --tail 20" + else + warn "Some services are still starting after ${HEALTH_TIMEOUT}s." + echo "" + echo " This may be normal on first boot (migrations + seeding can be slow)." + echo " Monitor progress with:" + echo " docker compose logs -f api --tail 30" + echo "" + echo " Check status with:" + echo " docker compose ps" + fi +else + echo "" + info "Skipped. Start services manually when ready:" + echo "" + echo " cd ${INSTALL_DIR} && docker compose up -d" + echo "" + echo " Pre-built images will be pulled from the registry (~2 min first time)." + echo " Database migrations and seeding run automatically on startup." +fi -# --- Done --- echo "" echo -e "${BOLD}${GREEN}Installation complete!${NC}" echo "" -echo " Start all services:" -echo " cd ${INSTALL_DIR} && docker compose up -d" -echo "" -echo " Check status:" -echo " docker compose ps" -echo "" -echo " View API logs:" -echo " docker compose logs -f api --tail 20" +echo -e " \033[0;33mIMPORTANT: Change your admin password after first login!\033[0m" echo "" diff --git a/scripts/mirror-images.sh b/scripts/mirror-images.sh index ae69a90..30d5255 100755 --- a/scripts/mirror-images.sh +++ b/scripts/mirror-images.sh @@ -144,7 +144,7 @@ declare -A MONITORING_IMAGES=( ["oliver006/redis_exporter:v1.81.0"]="redis_exporter:v1.81.0" ["gcr.io/cadvisor/cadvisor:v0.55.1"]="cadvisor:v0.55.1" ["prom/node-exporter:v1.10.2"]="node-exporter:v1.10.2" - ["gotify/server:v2.9.0"]="gotify:v2.9.0" + ["gotify/server:2.9.0"]="gotify:v2.9.0" ) # ============================================================================= diff --git a/scripts/systemd/changemaker-backup.service b/scripts/systemd/changemaker-backup.service new file mode 100644 index 0000000..68e7921 --- /dev/null +++ b/scripts/systemd/changemaker-backup.service @@ -0,0 +1,13 @@ +[Unit] +Description=Changemaker Lite database and uploads backup +Documentation=https://docs.cmlite.org/docs/admin/backups/ + +[Service] +Type=oneshot +User=__USER__ +Group=__USER__ +WorkingDirectory=__PROJECT_DIR__ +ExecStart=__PROJECT_DIR__/scripts/backup.sh --retention 30 +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal diff --git a/scripts/systemd/changemaker-backup.timer b/scripts/systemd/changemaker-backup.timer new file mode 100644 index 0000000..121f2e1 --- /dev/null +++ b/scripts/systemd/changemaker-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily Changemaker Lite backup + +[Timer] +OnCalendar=*-*-* 02:00:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target