From f8c8a939d70bd8fc71ebae03e8e86ecc10281195 Mon Sep 17 00:00:00 2001 From: bunker-admin Date: Thu, 9 Apr 2026 12:08:05 -0600 Subject: [PATCH] Add full non-interactive mode to config.sh New CLI flags for scripted deployments: --smtp-host/port/user/pass Production SMTP configuration --pangolin-api-url/key/org-id/endpoint/site Full Pangolin tunnel setup --mapbox-key Mapbox API key --maxmind-account-id/license-key MaxMind GeoIP credentials With --pangolin-site=new, config.sh creates a Pangolin site, fetches Newt credentials, and creates all resources+targets automatically. With --pangolin-site=existing, it connects to the first available site. Bunker Admin --- DEPLOYMENT_TEST_REPORT_2026-04-09.md | 125 ++++++++++++++++++++ config.sh | 171 +++++++++++++++++++++++++-- 2 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 DEPLOYMENT_TEST_REPORT_2026-04-09.md diff --git a/DEPLOYMENT_TEST_REPORT_2026-04-09.md b/DEPLOYMENT_TEST_REPORT_2026-04-09.md new file mode 100644 index 00000000..96c44be1 --- /dev/null +++ b/DEPLOYMENT_TEST_REPORT_2026-04-09.md @@ -0,0 +1,125 @@ +# Deployment Test Report — 2026-04-09 + +**Target:** Fresh curl-install deployment to `cursedknowledge.org` +**Server:** 100.90.78.47 (bunker-admin, Tailscale) +**Release:** v2.8.1 (commit 82546131) +**Pangolin Org:** cursed-knowledge @ bnkserve.org + +--- + +## Summary + +Full end-to-end deployment test: wipe server, build release, curl install, configure, verify all 37 containers and 18 external subdomains. **All services operational.** + +--- + +## Bugs Found & Fixed + +### 1. config.sh — Pangolin API URL default was correct, but endpoint derivation was wrong + +**Problem:** `PANGOLIN_ENDPOINT` was derived from `PANGOLIN_API_URL` by stripping `/v1`. Since the API lives at `api.bnkserve.org` but the Newt endpoint is `pangolin.bnkserve.org` (different hostname), this produced the wrong value. + +**Fix:** Ask for `PANGOLIN_ENDPOINT` as a separate prompt instead of deriving it. + +**Commit:** `599498fc` + +### 2. config.sh — No resources or targets created during Pangolin setup + +**Problem:** config.sh created the Pangolin site and wrote Newt credentials to `.env`, but never created the HTTP resources (subdomain → target mappings) that tell Pangolin how to route traffic. The message said "Resources will be created automatically via the admin GUI or sync endpoint" — but the admin GUI isn't accessible until the tunnel works, creating a chicken-and-egg problem. + +**Fix:** Added `pangolin_create_resources()` function that: +- Looks up the domain ID from registered domains +- Creates an HTTP resource for each of 18 subdomains +- Creates a target for each resource pointing to `nginx:80` +- Sets each resource as public (no SSO, no blockAccess) + +**Commit:** `599498fc` + +### 3. config.sh — Pangolin site creation failed with "Invalid address format" + +**Problem:** `pickSiteDefaults` returns `clientAddress` without CIDR notation (e.g., `100.90.128.0` instead of `100.90.128.0/24`). Pangolin's site creation API rejects this. + +**Fix:** Omit the `address` field from the site creation payload — Pangolin auto-assigns a valid address. + +**Commit:** `a85e153b` + +### 4. MongoDB — User not created on fresh volumes + +**Problem:** The custom entrypoint (`exec mongod --replSet rs0 --bind_ip_all --auth --keyFile ...`) bypassed Docker's standard `docker-entrypoint.sh`, which is responsible for reading `MONGO_INITDB_ROOT_USERNAME`/`PASSWORD` and creating the root user. On fresh volumes, MongoDB started with auth enabled but no users existed, causing the healthcheck and Rocket.Chat to fail. + +**Fix:** Changed entrypoint to generate the keyfile then delegate to `docker-entrypoint.sh`: +``` +exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /data/replica.key +``` +Docker's entrypoint handles user creation, then starts mongod with our flags. + +**Commit:** `599498fc` + +### 5. Missing `media` subdomain in Pangolin resources (non-issue) + +**Investigation:** The media API is accessed via path routing (`api.domain/media/`), not a separate subdomain. No nginx `server_name` block for `media.*` exists. The CLAUDE.md routing table listing was inaccurate. No fix needed. + +--- + +## Test Results + +### Container Status (37/37 running) + +All containers up and healthy where healthchecks are configured: +- Core: postgres, redis, api, admin, nginx, media-api — all healthy +- MongoDB: **healthy on first boot** (entrypoint fix confirmed) +- Rocket.Chat: healthy (after MongoDB) +- All other services: running/healthy + +### External Access (18/18 subdomains) + +| Subdomain | Service | Status | Notes | +|-----------|---------|--------|-------| +| cursedknowledge.org | MkDocs | 200 | Root domain | +| app.cursedknowledge.org | Admin GUI | 200 | | +| api.cursedknowledge.org | API | 200 | /api/health returns healthy | +| docs.cursedknowledge.org | MkDocs Live | 200 | | +| git.cursedknowledge.org | Gitea | 200 | Needs first-time setup | +| home.cursedknowledge.org | Homepage | 200 | | +| db.cursedknowledge.org | NocoDB | 302 | Redirects to dashboard | +| n8n.cursedknowledge.org | n8n | 200 | | +| grafana.cursedknowledge.org | Grafana | 302 | Redirects to login | +| draw.cursedknowledge.org | Excalidraw | 200 | | +| vault.cursedknowledge.org | Vaultwarden | 200 | | +| qr.cursedknowledge.org | Mini QR | 200 | | +| code.cursedknowledge.org | Code Server | 302 | Redirects to login | +| listmonk.cursedknowledge.org | Listmonk | 403 | Expected (auth proxy) | +| mail.cursedknowledge.org | MailHog | 200 | | +| chat.cursedknowledge.org | Rocket.Chat | 200 | | +| events.cursedknowledge.org | Gancio | 302 | Redirects to home | +| meet.cursedknowledge.org | Jitsi | 200 | | + +### Post-Deploy Manual Steps + +1. **Gitea:** Complete first-time setup at https://git.cursedknowledge.org +2. **Admin password:** Change default admin password at https://app.cursedknowledge.org + +--- + +## Deployment Timeline + +| Step | Duration | Notes | +|------|----------|-------| +| Build images (build-and-push.sh) | ~3 min | 4 services: api, admin, media-api, nginx | +| Build tarball (build-release.sh) | ~5 sec | 15MB tarball, 292 files | +| Upload to Gitea Releases | ~2 sec | v2.8.1 | +| Download tarball on remote | ~1 sec | Via curl | +| config.sh (non-interactive) | ~3 sec | + manual .env patching for SMTP/Pangolin | +| docker compose up -d | ~2 min | Image pulls from Gitea registry | +| All services healthy | ~3 min | Including MongoDB init + seed | +| External access verified | immediate | All 18 subdomains | + +**Total time from wipe to fully operational: ~10 minutes** + +--- + +## Outstanding Items + +- [ ] `config.sh` non-interactive mode should support all variables (SMTP, Pangolin, Mapbox, MaxMind) +- [ ] Newt WireGuard "invalid IP address" warning (cosmetic — TCP proxies work fine, clients feature disabled) +- [ ] Gitea first-time setup should be automated or documented in config.sh next steps diff --git a/config.sh b/config.sh index b14f6e09..76da9671 100755 --- a/config.sh +++ b/config.sh @@ -20,6 +20,24 @@ NI_ADMIN_PASSWORD="" NI_PRODUCTION=true NI_ENABLE_ALL=false +# SMTP flags +NI_SMTP_HOST="" +NI_SMTP_PORT="" +NI_SMTP_USER="" +NI_SMTP_PASS="" + +# Pangolin flags +NI_PANGOLIN_API_URL="" +NI_PANGOLIN_API_KEY="" +NI_PANGOLIN_ORG_ID="" +NI_PANGOLIN_ENDPOINT="" +NI_PANGOLIN_SITE="" # "new", "existing", or "" (skip) + +# Service credential flags +NI_MAPBOX_KEY="" +NI_MAXMIND_ACCOUNT_ID="" +NI_MAXMIND_LICENSE_KEY="" + # --- Arg parser --- while [[ $# -gt 0 ]]; do case "$1" in @@ -29,6 +47,21 @@ while [[ $# -gt 0 ]]; do --admin-password) NI_ADMIN_PASSWORD="$2"; shift 2 ;; --development) NI_PRODUCTION=false; shift ;; --enable-all) NI_ENABLE_ALL=true; shift ;; + # SMTP + --smtp-host) NI_SMTP_HOST="$2"; shift 2 ;; + --smtp-port) NI_SMTP_PORT="$2"; shift 2 ;; + --smtp-user) NI_SMTP_USER="$2"; shift 2 ;; + --smtp-pass) NI_SMTP_PASS="$2"; shift 2 ;; + # Pangolin + --pangolin-api-url) NI_PANGOLIN_API_URL="$2"; shift 2 ;; + --pangolin-api-key) NI_PANGOLIN_API_KEY="$2"; shift 2 ;; + --pangolin-org-id) NI_PANGOLIN_ORG_ID="$2"; shift 2 ;; + --pangolin-endpoint) NI_PANGOLIN_ENDPOINT="$2"; shift 2 ;; + --pangolin-site) NI_PANGOLIN_SITE="$2"; shift 2 ;; + # Services + --mapbox-key) NI_MAPBOX_KEY="$2"; shift 2 ;; + --maxmind-account-id) NI_MAXMIND_ACCOUNT_ID="$2"; shift 2 ;; + --maxmind-license-key) NI_MAXMIND_LICENSE_KEY="$2"; shift 2 ;; --help|-h) echo "Usage: bash config.sh [OPTIONS]" echo "" @@ -39,10 +72,32 @@ while [[ $# -gt 0 ]]; do echo " --admin-password PASS Set admin password (must meet policy: 12+ chars, upper+lower+digit)" echo " --development Set NODE_ENV=development (default: production)" echo " --enable-all Enable all optional features" - echo " --help, -h Show this help" + echo "" + echo "SMTP:" + echo " --smtp-host HOST SMTP server hostname" + echo " --smtp-port PORT SMTP port (default: 587)" + echo " --smtp-user USER SMTP username" + echo " --smtp-pass PASS SMTP password" + echo "" + echo "Pangolin Tunnel:" + echo " --pangolin-api-url URL Pangolin REST API URL" + echo " --pangolin-api-key KEY Pangolin API key" + echo " --pangolin-org-id ID Pangolin organization ID" + echo " --pangolin-endpoint URL Pangolin dashboard/Newt WebSocket URL" + echo " --pangolin-site MODE Site setup: 'new' (create) or 'existing' (connect first)" + echo "" + echo "Services:" + echo " --mapbox-key KEY Mapbox API key for map features" + echo " --maxmind-account-id ID MaxMind GeoIP account ID" + echo " --maxmind-license-key K MaxMind GeoIP license key" echo "" echo "Example:" echo " bash config.sh --non-interactive --domain example.org --admin-password MyStr0ngPass123" + echo " bash config.sh -y --domain example.org --admin-password MyStr0ngPass123 \\" + echo " --smtp-host smtp.example.com --smtp-port 587 --smtp-user me@example.com --smtp-pass secret \\" + echo " --pangolin-api-url https://api.pangolin.example/v1 --pangolin-api-key KEY \\" + echo " --pangolin-org-id myorg --pangolin-endpoint https://pangolin.example --pangolin-site new \\" + echo " --enable-all --mapbox-key pk.xxx --maxmind-account-id 12345 --maxmind-license-key abc" exit 0 ;; *) shift ;; esac @@ -623,10 +678,26 @@ configure_smtp() { header "Email Configuration" if [[ "$NON_INTERACTIVE" == "true" ]]; then - # Non-interactive: use MailHog defaults (production SMTP can be configured later) - update_env_var "VAULTWARDEN_SMTP_SECURITY" "off" - info "Using MailHog for email (configure SMTP later via .env)" - SMTP_MODE="mailhog" + if [[ -n "$NI_SMTP_HOST" ]]; then + update_env_var "SMTP_HOST" "$NI_SMTP_HOST" + update_env_var "SMTP_PORT" "${NI_SMTP_PORT:-587}" + update_env_var "SMTP_USER" "$NI_SMTP_USER" + update_env_var "SMTP_PASS" "$NI_SMTP_PASS" + update_env_var "EMAIL_TEST_MODE" "false" + update_env_var "VAULTWARDEN_SMTP_SECURITY" "starttls" + # Also configure Listmonk SMTP + update_env_var "LISTMONK_SMTP_HOST" "$NI_SMTP_HOST" + update_env_var "LISTMONK_SMTP_PORT" "${NI_SMTP_PORT:-587}" + update_env_var "LISTMONK_SMTP_USER" "$NI_SMTP_USER" + update_env_var "LISTMONK_SMTP_PASSWORD" "$NI_SMTP_PASS" + update_env_var "LISTMONK_SMTP_TLS_TYPE" "STARTTLS" + success "Production SMTP configured ($NI_SMTP_HOST)" + SMTP_MODE="production" + else + update_env_var "VAULTWARDEN_SMTP_SECURITY" "off" + info "Using MailHog for email (configure SMTP later via .env)" + SMTP_MODE="mailhog" + fi return fi @@ -833,7 +904,13 @@ configure_features() { update_env_var "ENABLE_ANALYTICS" "true" success "Analytics enabled" - if [[ "$NON_INTERACTIVE" == "false" ]]; then + if [[ "$NON_INTERACTIVE" == "true" ]]; then + if [[ -n "$NI_MAXMIND_ACCOUNT_ID" ]]; then + update_env_var "MAXMIND_ACCOUNT_ID" "$NI_MAXMIND_ACCOUNT_ID" + update_env_var "MAXMIND_LICENSE_KEY" "$NI_MAXMIND_LICENSE_KEY" + success "MaxMind GeoIP credentials configured" + fi + else echo "" info "GeoIP tracking requires a free MaxMind account." info "Sign up at: https://www.maxmind.com/en/geolite2/signup" @@ -852,14 +929,53 @@ configure_features() { else info "Analytics disabled (can enable later in admin Settings)" fi + + # Mapbox API key (used for map features) + if [[ "$NON_INTERACTIVE" == "true" ]]; then + if [[ -n "$NI_MAPBOX_KEY" ]]; then + update_env_var "MAPBOX_API_KEY" "$NI_MAPBOX_KEY" + success "Mapbox API key configured" + fi + else + echo "" + read -rp " Mapbox API key [leave blank to set later]: " mapbox_key + if [[ -n "$mapbox_key" ]]; then + update_env_var "MAPBOX_API_KEY" "$mapbox_key" + success "Mapbox API key configured" + fi + fi } configure_pangolin() { header "Tunnel Configuration (Pangolin)" if [[ "$NON_INTERACTIVE" == "true" ]]; then - info "Skipping Pangolin setup (configure later via admin GUI or .env)" - PANGOLIN_CONFIGURED="no" + if [[ -n "$NI_PANGOLIN_API_KEY" ]]; then + local pang_url="${NI_PANGOLIN_API_URL:-https://api.bnkserve.org/v1}" + local pang_key="$NI_PANGOLIN_API_KEY" + local pang_org="$NI_PANGOLIN_ORG_ID" + local pang_endpoint="${NI_PANGOLIN_ENDPOINT:-https://pangolin.bnkserve.org}" + + update_env_var "PANGOLIN_API_URL" "$pang_url" + update_env_var "PANGOLIN_API_KEY" "$pang_key" + update_env_var "PANGOLIN_ORG_ID" "$pang_org" + update_env_var "PANGOLIN_ENDPOINT" "$pang_endpoint" + success "Pangolin API credentials saved" + + if command -v curl &>/dev/null && command -v jq &>/dev/null; then + case "${NI_PANGOLIN_SITE:-}" in + new) + pangolin_create_site "$pang_url" "$pang_key" "$pang_org" "$pang_endpoint" ;; + existing) + # Connect to the first available site + pangolin_connect_first_site "$pang_url" "$pang_key" "$pang_org" "$pang_endpoint" ;; + esac + fi + PANGOLIN_CONFIGURED="yes" + else + info "Skipping Pangolin setup (no --pangolin-api-key provided)" + PANGOLIN_CONFIGURED="no" + fi return fi @@ -1229,6 +1345,45 @@ pangolin_connect_site() { pangolin_create_resources "$api_url" "$api_key" "$org_id" "$sel_id" "$domain" } +# Non-interactive helper: connect to the first available site automatically +pangolin_connect_first_site() { + local api_url=$1 api_key=$2 org_id=$3 endpoint=$4 + + local sites_resp + sites_resp=$(pangolin_api GET "$api_url/org/$org_id/sites" "$api_key") + local first_site + first_site=$(echo "$sites_resp" | jq -c '.data.sites[0] // empty' 2>/dev/null) + + if [[ -z "$first_site" || "$first_site" == "null" ]]; then + warn "No existing sites found — cannot auto-connect." + return + fi + + local sel_id sel_name + sel_id=$(echo "$first_site" | jq -r '.siteId') + sel_name=$(echo "$first_site" | jq -r '.name') + + update_env_var "PANGOLIN_SITE_ID" "$sel_id" + success "Connected to site: $sel_name (ID: $sel_id)" + + # Fetch Newt credentials + local defaults_resp + defaults_resp=$(pangolin_api GET "$api_url/org/$org_id/pick-site-defaults" "$api_key") + local newt_id newt_secret + newt_id=$(echo "$defaults_resp" | jq -r '.data.newtId // empty' 2>/dev/null) + newt_secret=$(echo "$defaults_resp" | jq -r '.data.newtSecret // empty' 2>/dev/null) + + if [[ -n "$newt_id" && -n "$newt_secret" ]]; then + update_env_var "PANGOLIN_NEWT_ID" "$newt_id" + update_env_var "PANGOLIN_NEWT_SECRET" "$newt_secret" + success "Newt credentials saved (newtId: $newt_id)" + fi + + # Create resources + local domain="${CONFIGURED_DOMAIN:-cmlite.org}" + pangolin_create_resources "$api_url" "$api_key" "$org_id" "$sel_id" "$domain" +} + configure_control_panel() { header "Control Panel Registration"