admin
62f906d6f0
Two issues occurred during upgrades: 1. Gancio config.json lost when Docker volume name prefix changes (e.g., changemakerlite_ vs changemaker-lite_). Gancio finds existing DB but no config and enters restart loop. Fix: verify_gancio_config() checks the volume and regenerates config.json from .env if missing. 2. mkdocs-site-server (LSIO nginx) returns 403 after upgrade because the anonymous /config volume shadows the ./mkdocs/site bind mount. Fix: docker compose rm -sf the LSIO container before up -d so the anonymous volume is recreated fresh. Also adds Gancio and MkDocs site health checks to Phase 6 verification. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
863 lines
30 KiB
Bash
Executable File
863 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# Changemaker Lite V2 — Upgrade Script
|
|
# Safely pulls updates, rebuilds containers, and restarts services.
|
|
# Usage: ./scripts/upgrade.sh [OPTIONS]
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
# --- Configuration ---
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
|
LOG_DIR="${PROJECT_DIR}/logs"
|
|
LOG_FILE="${LOG_DIR}/upgrade-${TIMESTAMP}.log"
|
|
LOCK_FILE="${PROJECT_DIR}/.upgrade.lock"
|
|
BACKUP_DIR="${BACKUP_DIR:-$PROJECT_DIR/backups}"
|
|
HEALTH_TIMEOUT=120
|
|
HEALTH_INTERVAL=5
|
|
MIN_DISK_MB=2048
|
|
|
|
# Source-built containers (always rebuilt)
|
|
SOURCE_CONTAINERS="api admin media-api"
|
|
# Conditionally rebuilt if Dockerfile changed
|
|
CONDITIONAL_CONTAINERS="nginx code-server"
|
|
# App containers stopped during upgrade
|
|
APP_CONTAINERS="api admin media-api nginx"
|
|
# Infrastructure containers (must stay up)
|
|
INFRA_CONTAINERS="v2-postgres redis"
|
|
# LSIO containers with anonymous /config volumes (must be force-recreated on upgrade
|
|
# to prevent stale anonymous volumes from shadowing bind mounts underneath /config)
|
|
LSIO_VOLUME_CONTAINERS="mkdocs-site-server"
|
|
|
|
# User-modifiable paths (auto-resolve keeps user version on conflict)
|
|
USER_PATHS=(
|
|
"mkdocs/docs/"
|
|
"mkdocs/mkdocs.yml"
|
|
"mkdocs/site/"
|
|
"configs/"
|
|
"nginx/conf.d/services.conf"
|
|
)
|
|
|
|
# --- Defaults ---
|
|
SKIP_BACKUP=false
|
|
PULL_SERVICES=false
|
|
DRY_RUN=false
|
|
FORCE=false
|
|
BRANCH=""
|
|
ROLLBACK=false
|
|
|
|
# --- Colors (respects NO_COLOR convention) ---
|
|
if [[ -t 1 ]] && [[ -z "${NO_COLOR:-}" ]]; then
|
|
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m'
|
|
DIM='\033[2m' NC='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' NC=''
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Utility Functions
|
|
# =============================================================================
|
|
|
|
info() { echo -e "${CYAN}[INFO]${NC} $*"; }
|
|
success() { echo -e "${GREEN}[ OK ]${NC} $*"; }
|
|
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
|
error() { echo -e "${RED}[ERR ]${NC} $*" >&2; }
|
|
|
|
phase() {
|
|
echo ""
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════${NC}"
|
|
echo -e "${BOLD}${BLUE} Phase $1: $2${NC}"
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
}
|
|
|
|
elapsed() {
|
|
local secs=$((SECONDS - START_TIME))
|
|
printf '%dm %ds' $((secs / 60)) $((secs % 60))
|
|
}
|
|
|
|
# --- Save/restore user-modifiable paths across git pull ---
|
|
save_user_paths() {
|
|
USER_SAVE_DIR="$(mktemp -d)"
|
|
for p in "${USER_PATHS[@]}"; do
|
|
if [[ -e "$PROJECT_DIR/$p" ]]; then
|
|
mkdir -p "$USER_SAVE_DIR/$(dirname "$p")"
|
|
cp -a "$PROJECT_DIR/$p" "$USER_SAVE_DIR/$p"
|
|
fi
|
|
done
|
|
}
|
|
|
|
restore_user_paths() {
|
|
if [[ -z "${USER_SAVE_DIR:-}" ]] || [[ ! -d "${USER_SAVE_DIR:-}" ]]; then
|
|
return
|
|
fi
|
|
local restored=0
|
|
for p in "${USER_PATHS[@]}"; do
|
|
if [[ -e "$USER_SAVE_DIR/$p" ]]; then
|
|
# Ensure parent directory exists (in case pull deleted it)
|
|
mkdir -p "$PROJECT_DIR/$(dirname "$p")"
|
|
rm -rf "$PROJECT_DIR/$p"
|
|
cp -a "$USER_SAVE_DIR/$p" "$PROJECT_DIR/$p"
|
|
restored=$((restored + 1))
|
|
fi
|
|
done
|
|
rm -rf "$USER_SAVE_DIR"
|
|
if [[ $restored -gt 0 ]]; then
|
|
success "Restored $restored user-modifiable path(s)"
|
|
fi
|
|
}
|
|
|
|
# --- Verify Gancio config.json in its data volume ---
|
|
# Gancio uses a named Docker volume for /home/node/data. If the volume loses
|
|
# config.json (e.g., volume name prefix mismatch after compose project rename),
|
|
# Gancio detects an existing DB but no config and refuses to start with:
|
|
# "Non empty db! Please move your current db elsewhere than retry."
|
|
# This regenerates config.json from .env vars when missing.
|
|
verify_gancio_config() {
|
|
local gancio_volume
|
|
gancio_volume="$(docker volume ls --format '{{.Name}}' | grep 'gancio-data' | head -1 || true)"
|
|
if [[ -z "$gancio_volume" ]]; then
|
|
return # No gancio volume exists yet; first run will handle it
|
|
fi
|
|
|
|
# Check if config.json exists and is non-empty
|
|
if docker run --rm -v "${gancio_volume}:/data" alpine test -s /data/config.json 2>/dev/null; then
|
|
success "Gancio config.json present in $gancio_volume"
|
|
return
|
|
fi
|
|
|
|
warn "Gancio config.json missing in volume $gancio_volume — regenerating from .env"
|
|
local base_url="${GANCIO_BASE_URL:-https://events.cmlite.org}"
|
|
local pg_user="${V2_POSTGRES_USER:-changemaker}"
|
|
local pg_pass="${V2_POSTGRES_PASSWORD:-changemaker}"
|
|
local config_json="{\"baseurl\":\"${base_url}\",\"server\":{\"host\":\"0.0.0.0\",\"port\":13120},\"db\":{\"dialect\":\"postgres\",\"host\":\"changemaker-v2-postgres\",\"port\":5432,\"database\":\"gancio\",\"username\":\"${pg_user}\",\"password\":\"${pg_pass}\"}}"
|
|
|
|
docker run --rm -v "${gancio_volume}:/data" alpine sh -c \
|
|
"echo '${config_json}' > /data/config.json && chown 1000:1000 /data/config.json"
|
|
success "Gancio config.json regenerated"
|
|
}
|
|
|
|
# --- Lockfile ---
|
|
acquire_lock() {
|
|
if [[ -f "$LOCK_FILE" ]]; then
|
|
local old_pid
|
|
old_pid=$(cat "$LOCK_FILE" 2>/dev/null || echo "")
|
|
if [[ -n "$old_pid" ]] && kill -0 "$old_pid" 2>/dev/null; then
|
|
error "Another upgrade is running (PID $old_pid). If stale, remove $LOCK_FILE"
|
|
exit 1
|
|
fi
|
|
warn "Removing stale lock file (PID $old_pid no longer running)"
|
|
rm -f "$LOCK_FILE"
|
|
fi
|
|
echo $$ > "$LOCK_FILE"
|
|
}
|
|
|
|
release_lock() {
|
|
rm -f "$LOCK_FILE"
|
|
}
|
|
|
|
# --- .env loading (from backup.sh — handles special chars) ---
|
|
load_env() {
|
|
if [[ -f "$PROJECT_DIR/.env" ]]; then
|
|
while IFS='=' read -r key value; do
|
|
[[ -z "$key" || "$key" =~ ^[[:space:]]*# ]] && continue
|
|
key="$(echo "$key" | xargs)"
|
|
value="${value%\"}"
|
|
value="${value#\"}"
|
|
value="${value%\'}"
|
|
value="${value#\'}"
|
|
if [[ "$key" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
|
|
export "$key=$value"
|
|
fi
|
|
done < "$PROJECT_DIR/.env"
|
|
fi
|
|
}
|
|
|
|
# --- Print rollback instructions ---
|
|
print_rollback_help() {
|
|
local commit="${PRE_UPGRADE_COMMIT:-unknown}"
|
|
local backup_path="${LATEST_BACKUP:-$BACKUP_DIR}"
|
|
echo ""
|
|
echo -e "${BOLD}${RED}═══════════════════════════════════════════════${NC}"
|
|
echo -e "${BOLD}${RED} Upgrade Failed — Rollback Instructions${NC}"
|
|
echo -e "${BOLD}${RED}═══════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e " ${BOLD}1.${NC} Restore code to pre-upgrade commit:"
|
|
echo -e " ${CYAN}cd $PROJECT_DIR${NC}"
|
|
echo -e " ${CYAN}git checkout $commit${NC}"
|
|
echo ""
|
|
echo -e " ${BOLD}2.${NC} Rebuild and restart:"
|
|
echo -e " ${CYAN}docker compose build api admin media-api${NC}"
|
|
echo -e " ${CYAN}docker compose up -d${NC}"
|
|
echo ""
|
|
echo -e " ${BOLD}3.${NC} If database rollback is needed (destructive!):"
|
|
echo -e " ${CYAN}# Find backup archive:${NC}"
|
|
echo -e " ${CYAN}ls -lt $BACKUP_DIR/changemaker-v2-backup-*.tar.gz | head -5${NC}"
|
|
echo -e " ${CYAN}# Extract and restore:${NC}"
|
|
echo -e " ${CYAN}tar xzf <backup>.tar.gz -C /tmp${NC}"
|
|
echo -e " ${CYAN}gunzip -c /tmp/<backup>/v2-postgres.sql.gz | docker exec -i changemaker-v2-postgres psql -U changemaker -d changemaker_v2${NC}"
|
|
echo ""
|
|
echo -e " Or use: ${CYAN}./scripts/upgrade.sh --rollback${NC}"
|
|
echo ""
|
|
}
|
|
|
|
# --- Failure trap ---
|
|
on_failure() {
|
|
local exit_code=$?
|
|
# Clean up user path save directory if it exists
|
|
if [[ -n "${USER_SAVE_DIR:-}" ]] && [[ -d "${USER_SAVE_DIR:-}" ]]; then
|
|
rm -rf "$USER_SAVE_DIR"
|
|
fi
|
|
release_lock
|
|
if [[ $exit_code -ne 0 ]] && [[ "$DRY_RUN" != "true" ]]; then
|
|
error "Upgrade failed at line ${BASH_LINENO[0]} (exit code $exit_code)"
|
|
print_rollback_help
|
|
info "Log file: $LOG_FILE"
|
|
fi
|
|
}
|
|
|
|
# =============================================================================
|
|
# Parse Arguments
|
|
# =============================================================================
|
|
|
|
show_help() {
|
|
cat << 'EOF'
|
|
Changemaker Lite V2 — Upgrade Script
|
|
|
|
Usage: ./scripts/upgrade.sh [OPTIONS]
|
|
|
|
Options:
|
|
--skip-backup Skip backup phase (requires --force)
|
|
--pull-services Also pull new third-party Docker images
|
|
--dry-run Show what would happen without executing
|
|
--force Continue past non-critical warnings
|
|
--branch BRANCH Git branch to pull (default: current branch)
|
|
--rollback Rollback to pre-upgrade commit
|
|
--help Show this help message
|
|
|
|
Examples:
|
|
./scripts/upgrade.sh # Standard upgrade
|
|
./scripts/upgrade.sh --dry-run # Preview changes
|
|
./scripts/upgrade.sh --pull-services # Also update PostgreSQL, Redis, etc.
|
|
./scripts/upgrade.sh --rollback # Revert last upgrade
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--skip-backup) SKIP_BACKUP=true; shift ;;
|
|
--pull-services) PULL_SERVICES=true; shift ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
--force) FORCE=true; shift ;;
|
|
--branch) BRANCH="$2"; shift 2 ;;
|
|
--rollback) ROLLBACK=true; shift ;;
|
|
--help|-h) show_help ;;
|
|
*) error "Unknown option: $1"; echo "Run with --help for usage."; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# Validate flag combinations
|
|
if [[ "$SKIP_BACKUP" == "true" ]] && [[ "$FORCE" != "true" ]]; then
|
|
error "--skip-backup requires --force (backup protects your data)"
|
|
exit 1
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Main
|
|
# =============================================================================
|
|
|
|
START_TIME=$SECONDS
|
|
cd "$PROJECT_DIR"
|
|
|
|
# Setup logging
|
|
mkdir -p "$LOG_DIR"
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
|
|
echo ""
|
|
echo -e "${BOLD}${BLUE}══════════════════════════════════════════════════${NC}"
|
|
echo -e "${BOLD}${BLUE} Changemaker Lite V2 — Upgrade${NC}"
|
|
echo -e "${BOLD}${BLUE} ${TIMESTAMP}${NC}"
|
|
echo -e "${BOLD}${BLUE}══════════════════════════════════════════════════${NC}"
|
|
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
echo ""
|
|
echo -e " ${YELLOW}DRY RUN — no changes will be made${NC}"
|
|
fi
|
|
|
|
trap on_failure EXIT
|
|
acquire_lock
|
|
load_env
|
|
|
|
# Determine branch
|
|
if [[ -z "$BRANCH" ]]; then
|
|
BRANCH="$(git rev-parse --abbrev-ref HEAD)"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Rollback Mode
|
|
# =============================================================================
|
|
|
|
if [[ "$ROLLBACK" == "true" ]]; then
|
|
phase "R" "Rollback"
|
|
|
|
# Find latest backup with git commit reference
|
|
LATEST_ARCHIVE="$(ls -t "$BACKUP_DIR"/changemaker-v2-backup-*.tar.gz 2>/dev/null | head -1 || true)"
|
|
if [[ -z "$LATEST_ARCHIVE" ]]; then
|
|
error "No backup archives found in $BACKUP_DIR"
|
|
error "Cannot determine pre-upgrade commit. Manual rollback needed."
|
|
release_lock
|
|
exit 1
|
|
fi
|
|
|
|
info "Latest backup: $(basename "$LATEST_ARCHIVE")"
|
|
|
|
# Extract just the git-commit.txt from the archive
|
|
ARCHIVE_DIR="$(basename "$LATEST_ARCHIVE" .tar.gz)"
|
|
ROLLBACK_COMMIT="$(tar xzf "$LATEST_ARCHIVE" -O "${ARCHIVE_DIR}/git-commit.txt" 2>/dev/null || true)"
|
|
|
|
if [[ -z "$ROLLBACK_COMMIT" ]]; then
|
|
error "No git-commit.txt found in backup archive."
|
|
error "Manually specify: git checkout <commit-hash>"
|
|
release_lock
|
|
exit 1
|
|
fi
|
|
|
|
info "Rolling back to commit: $ROLLBACK_COMMIT"
|
|
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
info "[DRY RUN] Would run: git checkout $ROLLBACK_COMMIT"
|
|
info "[DRY RUN] Would rebuild: docker compose build $SOURCE_CONTAINERS"
|
|
info "[DRY RUN] Would restart: docker compose up -d"
|
|
release_lock
|
|
exit 0
|
|
fi
|
|
|
|
git checkout -B "$BRANCH" "$ROLLBACK_COMMIT"
|
|
docker compose build $SOURCE_CONTAINERS
|
|
docker compose up -d
|
|
success "Rolled back to $ROLLBACK_COMMIT"
|
|
info "Database was NOT rolled back. To restore DB, see backup at:"
|
|
info " $LATEST_ARCHIVE"
|
|
release_lock
|
|
exit 0
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 1: Pre-flight Checks
|
|
# =============================================================================
|
|
|
|
phase "1" "Pre-flight Checks"
|
|
|
|
# Docker
|
|
if command -v docker &>/dev/null; then
|
|
success "Docker: $(docker --version | head -1)"
|
|
else
|
|
error "Docker is not installed."
|
|
exit 1
|
|
fi
|
|
|
|
if docker compose version &>/dev/null; then
|
|
success "Docker Compose: $(docker compose version --short)"
|
|
else
|
|
error "Docker Compose v2 plugin not found."
|
|
exit 1
|
|
fi
|
|
|
|
# Docker daemon running
|
|
if docker info &>/dev/null 2>&1; then
|
|
success "Docker daemon running"
|
|
else
|
|
error "Docker daemon not running."
|
|
exit 1
|
|
fi
|
|
|
|
# Git
|
|
if command -v git &>/dev/null; then
|
|
success "Git: $(git --version)"
|
|
else
|
|
error "Git is not installed."
|
|
exit 1
|
|
fi
|
|
|
|
# Remote reachable
|
|
info "Checking git remote..."
|
|
if timeout 10 git ls-remote origin HEAD &>/dev/null 2>&1; then
|
|
success "Git remote reachable"
|
|
else
|
|
error "Cannot reach git remote. Check your network or remote configuration."
|
|
exit 1
|
|
fi
|
|
|
|
# Working directory checks
|
|
if [[ ! -f "$PROJECT_DIR/docker-compose.yml" ]]; then
|
|
error "docker-compose.yml not found. Are you in the project root?"
|
|
exit 1
|
|
fi
|
|
if [[ ! -f "$PROJECT_DIR/.env" ]]; then
|
|
error ".env not found. Run ./config.sh first."
|
|
exit 1
|
|
fi
|
|
success "Project files verified"
|
|
|
|
# Disk space
|
|
AVAILABLE_MB=$(df -m "$PROJECT_DIR" | awk 'NR==2 {print $4}')
|
|
if [[ "$AVAILABLE_MB" -lt "$MIN_DISK_MB" ]]; then
|
|
error "Insufficient disk space: ${AVAILABLE_MB}MB available, ${MIN_DISK_MB}MB required."
|
|
exit 1
|
|
fi
|
|
success "Disk space: ${AVAILABLE_MB}MB available"
|
|
|
|
# Record pre-upgrade state
|
|
PRE_UPGRADE_COMMIT="$(git rev-parse HEAD)"
|
|
PRE_UPGRADE_SHORT="$(git rev-parse --short HEAD)"
|
|
info "Current commit: $PRE_UPGRADE_SHORT ($(git log -1 --format='%s' HEAD))"
|
|
info "Target branch: $BRANCH"
|
|
|
|
# Record running containers (for restoring monitoring profile later)
|
|
MONITORING_WAS_RUNNING=false
|
|
if docker ps --format '{{.Names}}' | grep -q 'prometheus-changemaker'; then
|
|
MONITORING_WAS_RUNNING=true
|
|
info "Monitoring stack detected (will restart after upgrade)"
|
|
fi
|
|
|
|
# Warn about uncommitted changes in project-owned paths
|
|
PROJECT_OWNED_PATHS="api/ admin/ docker-compose.yml"
|
|
DIRTY_PROJECT_FILES="$(git diff --name-only HEAD -- $PROJECT_OWNED_PATHS 2>/dev/null || true)"
|
|
if [[ -n "$DIRTY_PROJECT_FILES" ]]; then
|
|
warn "Uncommitted changes in project-owned files:"
|
|
echo "$DIRTY_PROJECT_FILES" | while read -r f; do echo " $f"; done
|
|
if [[ "$FORCE" != "true" ]]; then
|
|
error "Commit or stash these changes first, or use --force to continue."
|
|
exit 1
|
|
fi
|
|
warn "Continuing with --force (changes will be stashed)"
|
|
fi
|
|
|
|
# Check for available updates
|
|
LOCAL_HEAD="$(git rev-parse HEAD)"
|
|
REMOTE_HEAD="$(git ls-remote origin "$BRANCH" | cut -f1)"
|
|
if [[ "$LOCAL_HEAD" == "$REMOTE_HEAD" ]]; then
|
|
info "Already up to date ($PRE_UPGRADE_SHORT). No upstream changes."
|
|
if [[ "$FORCE" != "true" ]]; then
|
|
success "Nothing to upgrade."
|
|
release_lock
|
|
exit 0
|
|
fi
|
|
warn "Continuing with --force despite no upstream changes."
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 2: Backup
|
|
# =============================================================================
|
|
|
|
phase "2" "Backup"
|
|
|
|
if [[ "$SKIP_BACKUP" == "true" ]]; then
|
|
warn "Backup skipped (--skip-backup --force)"
|
|
else
|
|
# Run existing backup script
|
|
if [[ -x "$SCRIPT_DIR/backup.sh" ]]; then
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
info "[DRY RUN] Would run: scripts/backup.sh"
|
|
else
|
|
info "Running database backup..."
|
|
"$SCRIPT_DIR/backup.sh"
|
|
success "Database backup complete"
|
|
fi
|
|
else
|
|
warn "scripts/backup.sh not found or not executable, skipping database backup"
|
|
fi
|
|
|
|
# Archive user-modifiable content
|
|
USER_BACKUP="${BACKUP_DIR}/upgrade-user-content-${TIMESTAMP}.tar.gz"
|
|
USER_BACKUP_FILES=()
|
|
for p in "${USER_PATHS[@]}"; do
|
|
if [[ -e "$PROJECT_DIR/$p" ]]; then
|
|
USER_BACKUP_FILES+=("$p")
|
|
fi
|
|
done
|
|
|
|
if [[ ${#USER_BACKUP_FILES[@]} -gt 0 ]]; then
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
info "[DRY RUN] Would archive user content: ${USER_BACKUP_FILES[*]}"
|
|
else
|
|
mkdir -p "$BACKUP_DIR"
|
|
tar -czf "$USER_BACKUP" -C "$PROJECT_DIR" "${USER_BACKUP_FILES[@]}" 2>/dev/null || true
|
|
success "User content archived: $(du -h "$USER_BACKUP" | cut -f1)"
|
|
fi
|
|
fi
|
|
|
|
# Save pre-upgrade commit hash for rollback reference
|
|
LATEST_BACKUP="$(ls -t "$BACKUP_DIR"/changemaker-v2-backup-*.tar.gz 2>/dev/null | head -1 || true)"
|
|
if [[ -n "$LATEST_BACKUP" ]] && [[ "$DRY_RUN" != "true" ]]; then
|
|
# Append git-commit.txt into the latest backup archive
|
|
COMMIT_TMPDIR="$(mktemp -d)"
|
|
ARCHIVE_BASENAME="$(basename "$LATEST_BACKUP" .tar.gz)"
|
|
mkdir -p "$COMMIT_TMPDIR/$ARCHIVE_BASENAME"
|
|
echo "$PRE_UPGRADE_COMMIT" > "$COMMIT_TMPDIR/$ARCHIVE_BASENAME/git-commit.txt"
|
|
|
|
# Re-pack: extract, add file, recompress
|
|
tar xzf "$LATEST_BACKUP" -C "$COMMIT_TMPDIR" 2>/dev/null || true
|
|
tar czf "$LATEST_BACKUP" -C "$COMMIT_TMPDIR" "$ARCHIVE_BASENAME"
|
|
rm -rf "$COMMIT_TMPDIR"
|
|
success "Saved commit reference ($PRE_UPGRADE_SHORT) in backup archive"
|
|
fi
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 3: Code Update
|
|
# =============================================================================
|
|
|
|
phase "3" "Code Update"
|
|
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
info "[DRY RUN] Would fetch and show incoming changes:"
|
|
git fetch origin "$BRANCH" 2>/dev/null || true
|
|
INCOMING="$(git log --oneline HEAD..origin/"$BRANCH" 2>/dev/null || echo "(unable to preview)")"
|
|
if [[ -n "$INCOMING" ]]; then
|
|
echo "$INCOMING"
|
|
else
|
|
info "No new commits to pull."
|
|
fi
|
|
info "[DRY RUN] Would preserve user-modifiable paths: ${USER_PATHS[*]}"
|
|
info "[DRY RUN] Would stash local changes, pull, and pop stash"
|
|
release_lock
|
|
exit 0
|
|
fi
|
|
|
|
# Step 0: Save user-modifiable paths before any git operations
|
|
save_user_paths
|
|
|
|
# Step 1: Stash user changes if any exist
|
|
HAS_CHANGES=false
|
|
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
|
|
HAS_CHANGES=true
|
|
STASH_NAME="upgrade-${TIMESTAMP}"
|
|
info "Stashing local changes as '$STASH_NAME'..."
|
|
git stash push -m "$STASH_NAME"
|
|
success "Local changes stashed"
|
|
fi
|
|
|
|
# Step 3: Pull updates
|
|
info "Pulling updates from origin/$BRANCH..."
|
|
if ! git pull origin "$BRANCH" --no-edit 2>&1; then
|
|
error "git pull failed. This may indicate upstream force-push or branch issues."
|
|
if [[ "$HAS_CHANGES" == "true" ]]; then
|
|
warn "Your stashed changes can be recovered with: git stash pop"
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
POST_PULL_COMMIT="$(git rev-parse --short HEAD)"
|
|
success "Updated to $POST_PULL_COMMIT"
|
|
|
|
# Step 4: Pop stash and handle conflicts
|
|
if [[ "$HAS_CHANGES" == "true" ]]; then
|
|
info "Restoring local changes..."
|
|
if git stash pop 2>&1; then
|
|
success "Local changes restored cleanly"
|
|
else
|
|
warn "Merge conflicts detected during stash pop"
|
|
|
|
# Auto-resolve user-modifiable paths by keeping user's version
|
|
RESOLVED_COUNT=0
|
|
for user_path in "${USER_PATHS[@]}"; do
|
|
CONFLICTED="$(git diff --name-only --diff-filter=U -- "$user_path" 2>/dev/null || true)"
|
|
if [[ -n "$CONFLICTED" ]]; then
|
|
while IFS= read -r cf; do
|
|
info " Auto-resolving (keeping yours): $cf"
|
|
git checkout --theirs "$cf" 2>/dev/null || true
|
|
git add "$cf"
|
|
RESOLVED_COUNT=$((RESOLVED_COUNT + 1))
|
|
done < <(echo "$CONFLICTED")
|
|
fi
|
|
done
|
|
|
|
# Check if any conflicts remain in project-owned files
|
|
REMAINING_CONFLICTS="$(git diff --name-only --diff-filter=U 2>/dev/null || true)"
|
|
if [[ -n "$REMAINING_CONFLICTS" ]]; then
|
|
error "Unresolved conflicts in project-owned files:"
|
|
echo "$REMAINING_CONFLICTS" | while read -r f; do echo " $f"; done
|
|
echo ""
|
|
error "These files have upstream changes that conflict with your edits."
|
|
error "Resolve manually, then run the upgrade again."
|
|
info "Your pre-upgrade commit: $PRE_UPGRADE_COMMIT"
|
|
info "To abort: git merge --abort OR git checkout $PRE_UPGRADE_COMMIT"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $RESOLVED_COUNT -gt 0 ]]; then
|
|
success "Auto-resolved $RESOLVED_COUNT user-modifiable path(s) (kept your versions)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Step 4b: Restore user-modifiable paths (unconditionally overwrites with saved copies)
|
|
restore_user_paths
|
|
|
|
# Step 5: Detect new env vars
|
|
info "Checking for new environment variables..."
|
|
if [[ -f "$PROJECT_DIR/.env.example" ]] && [[ -f "$PROJECT_DIR/.env" ]]; then
|
|
NEW_VARS=()
|
|
while IFS='=' read -r key value; do
|
|
[[ -z "$key" || "$key" =~ ^[[:space:]]*# ]] && continue
|
|
key="$(echo "$key" | xargs)"
|
|
[[ ! "$key" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] && continue
|
|
if ! grep -q "^${key}=" "$PROJECT_DIR/.env" 2>/dev/null; then
|
|
# Strip inline comments and trim whitespace before appending
|
|
value="${value%%#*}"
|
|
value="$(echo "$value" | xargs)"
|
|
echo "${key}=${value}" >> "$PROJECT_DIR/.env"
|
|
NEW_VARS+=("$key")
|
|
fi
|
|
done < "$PROJECT_DIR/.env.example"
|
|
|
|
if [[ ${#NEW_VARS[@]} -gt 0 ]]; then
|
|
warn "New env vars added to .env (review defaults):"
|
|
for v in "${NEW_VARS[@]}"; do
|
|
echo -e " ${CYAN}$v${NC}"
|
|
done
|
|
else
|
|
success "No new environment variables"
|
|
fi
|
|
fi
|
|
|
|
# Step 6: Print update summary
|
|
COMMIT_RANGE="${PRE_UPGRADE_SHORT}..${POST_PULL_COMMIT}"
|
|
COMMIT_COUNT="$(git log --oneline "$PRE_UPGRADE_COMMIT..HEAD" 2>/dev/null | wc -l | xargs)"
|
|
echo ""
|
|
info "Update summary: $COMMIT_COUNT commit(s) ($COMMIT_RANGE)"
|
|
git log --oneline "$PRE_UPGRADE_COMMIT..HEAD" 2>/dev/null | head -20
|
|
if [[ "$COMMIT_COUNT" -gt 20 ]]; then
|
|
info " ... and $((COMMIT_COUNT - 20)) more"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 4: Container Rebuild
|
|
# =============================================================================
|
|
|
|
phase "4" "Container Rebuild"
|
|
|
|
# Always rebuild source-built containers
|
|
info "Rebuilding source containers: $SOURCE_CONTAINERS"
|
|
docker compose build $SOURCE_CONTAINERS
|
|
success "Source containers rebuilt"
|
|
|
|
# Conditionally rebuild containers whose Dockerfiles changed
|
|
CHANGED_FILES="$(git diff --name-only "$PRE_UPGRADE_COMMIT..HEAD" 2>/dev/null || true)"
|
|
for svc in $CONDITIONAL_CONTAINERS; do
|
|
case "$svc" in
|
|
nginx)
|
|
if echo "$CHANGED_FILES" | grep -q "^nginx/"; then
|
|
info "Rebuilding nginx (config changed)..."
|
|
docker compose build nginx
|
|
success "nginx rebuilt"
|
|
else
|
|
info "nginx unchanged, skipping rebuild"
|
|
fi
|
|
;;
|
|
code-server)
|
|
if echo "$CHANGED_FILES" | grep -q "^Dockerfile.code-server"; then
|
|
info "Rebuilding code-server (Dockerfile changed)..."
|
|
docker compose build code-server
|
|
success "code-server rebuilt"
|
|
else
|
|
info "code-server unchanged, skipping rebuild"
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Optionally pull third-party images
|
|
if [[ "$PULL_SERVICES" == "true" ]]; then
|
|
info "Pulling latest third-party images..."
|
|
docker compose pull v2-postgres redis listmonk-app listmonk-db gitea-app nocodb-v2 mailhog 2>/dev/null || true
|
|
success "Third-party images updated"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 5: Service Restart
|
|
# =============================================================================
|
|
|
|
phase "5" "Service Restart"
|
|
|
|
# Stop application containers
|
|
info "Stopping application containers..."
|
|
docker compose stop $APP_CONTAINERS 2>/dev/null || true
|
|
success "Application containers stopped"
|
|
|
|
# Force-recreate LSIO containers to prevent anonymous volume shadowing bind mounts.
|
|
# LSIO images define a VOLUME at /config in their Dockerfile. When a container is
|
|
# merely restarted, Docker reuses the old anonymous volume whose /config/www is empty,
|
|
# which shadows the bind mount (e.g., ./mkdocs/site:/config/www → 403 Forbidden).
|
|
# Removing the container first ensures a fresh anonymous volume that respects bind mounts.
|
|
info "Removing LSIO containers (clearing anonymous volumes)..."
|
|
docker compose rm -sf $LSIO_VOLUME_CONTAINERS 2>/dev/null || true
|
|
success "LSIO containers cleared for fresh recreation"
|
|
|
|
# Verify Gancio config.json exists before starting services
|
|
verify_gancio_config
|
|
|
|
# Ensure infrastructure is running and healthy
|
|
info "Ensuring infrastructure is up..."
|
|
docker compose up -d $INFRA_CONTAINERS
|
|
|
|
# Wait for PostgreSQL to be ready
|
|
info "Waiting for PostgreSQL..."
|
|
PG_WAIT=0
|
|
PG_TIMEOUT=60
|
|
while ! docker compose exec -T v2-postgres pg_isready -U "${V2_POSTGRES_USER:-changemaker}" &>/dev/null 2>&1; do
|
|
sleep 2
|
|
PG_WAIT=$((PG_WAIT + 2))
|
|
if [[ $PG_WAIT -ge $PG_TIMEOUT ]]; then
|
|
error "PostgreSQL did not become ready within ${PG_TIMEOUT}s"
|
|
exit 1
|
|
fi
|
|
done
|
|
success "PostgreSQL ready (${PG_WAIT}s)"
|
|
|
|
# Start API first (entrypoint runs prisma db push + seed)
|
|
info "Starting API (migrations will auto-apply)..."
|
|
docker compose up -d api
|
|
|
|
# Poll API health check
|
|
info "Waiting for API health check..."
|
|
API_WAIT=0
|
|
while true; do
|
|
if docker compose exec -T api wget -q --spider http://localhost:4000/api/health 2>/dev/null; then
|
|
break
|
|
fi
|
|
# Detect container crash early (don't wait full timeout)
|
|
if ! docker compose ps api --format '{{.State}}' 2>/dev/null | grep -q "running"; then
|
|
error "API container exited unexpectedly"
|
|
docker compose logs api --tail 20
|
|
exit 1
|
|
fi
|
|
sleep $HEALTH_INTERVAL
|
|
API_WAIT=$((API_WAIT + HEALTH_INTERVAL))
|
|
if [[ $API_WAIT -ge $HEALTH_TIMEOUT ]]; then
|
|
error "API did not become healthy within ${HEALTH_TIMEOUT}s"
|
|
error "Check logs: docker compose logs api --tail 50"
|
|
exit 1
|
|
fi
|
|
done
|
|
success "API healthy (${API_WAIT}s)"
|
|
|
|
# Start everything else (exclude one-shot init containers)
|
|
info "Starting remaining services..."
|
|
docker compose up -d \
|
|
--scale listmonk-init=0 \
|
|
--scale gancio-init=0 \
|
|
--scale vaultwarden-init=0
|
|
success "All services started"
|
|
|
|
# Restart Pangolin tunnel connector if running (may hold stale state after nginx rebuild)
|
|
if docker ps --format '{{.Names}}' | grep -q 'newt'; then
|
|
info "Restarting Pangolin tunnel connector..."
|
|
docker compose restart newt 2>/dev/null || true
|
|
success "Newt tunnel restarted"
|
|
fi
|
|
|
|
# Restart monitoring if it was running before
|
|
if [[ "$MONITORING_WAS_RUNNING" == "true" ]]; then
|
|
info "Restarting monitoring stack..."
|
|
docker compose --profile monitoring up -d
|
|
success "Monitoring stack restarted"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Phase 6: Post-Upgrade Verification
|
|
# =============================================================================
|
|
|
|
phase "6" "Post-Upgrade Verification"
|
|
|
|
VERIFY_FAILED=false
|
|
|
|
# API health
|
|
if docker compose exec -T api wget -q --spider http://localhost:4000/api/health 2>/dev/null; then
|
|
success "API (port 4000): healthy"
|
|
else
|
|
warn "API (port 4000): not responding"
|
|
VERIFY_FAILED=true
|
|
fi
|
|
|
|
# Admin health
|
|
if docker compose exec -T admin wget -q --spider http://localhost:3000/ 2>/dev/null; then
|
|
success "Admin (port 3000): healthy"
|
|
else
|
|
warn "Admin (port 3000): not responding"
|
|
VERIFY_FAILED=true
|
|
fi
|
|
|
|
# Media API health (optional — may not be enabled)
|
|
if docker ps --format '{{.Names}}' | grep -q 'changemaker-media-api'; then
|
|
if docker compose exec -T media-api wget -q --spider http://127.0.0.1:4100/health 2>/dev/null; then
|
|
success "Media API (port 4100): healthy"
|
|
else
|
|
warn "Media API (port 4100): not responding"
|
|
VERIFY_FAILED=true
|
|
fi
|
|
fi
|
|
|
|
# Gancio health (optional)
|
|
if docker ps --format '{{.Names}}' | grep -q 'gancio-changemaker'; then
|
|
if docker compose ps gancio --format '{{.Status}}' 2>/dev/null | grep -q "healthy"; then
|
|
success "Gancio: healthy"
|
|
elif docker compose ps gancio --format '{{.Status}}' 2>/dev/null | grep -qi "restarting"; then
|
|
warn "Gancio: restart loop detected (check config.json in gancio-data volume)"
|
|
VERIFY_FAILED=true
|
|
else
|
|
info "Gancio: starting (may take up to 60s)"
|
|
fi
|
|
fi
|
|
|
|
# MkDocs static site health
|
|
if docker ps --format '{{.Names}}' | grep -q 'mkdocs-site-server'; then
|
|
if curl -sf http://localhost:${MKDOCS_SITE_SERVER_PORT:-4004}/ -o /dev/null 2>/dev/null; then
|
|
success "MkDocs site (port ${MKDOCS_SITE_SERVER_PORT:-4004}): healthy"
|
|
else
|
|
warn "MkDocs site (port ${MKDOCS_SITE_SERVER_PORT:-4004}): not responding"
|
|
VERIFY_FAILED=true
|
|
fi
|
|
fi
|
|
|
|
# Check for containers in restart loop
|
|
RESTARTING="$(docker compose ps 2>/dev/null | grep -i "restarting" || true)"
|
|
if [[ -n "$RESTARTING" ]]; then
|
|
warn "Containers in restart loop:"
|
|
echo "$RESTARTING"
|
|
VERIFY_FAILED=true
|
|
fi
|
|
|
|
if [[ "$VERIFY_FAILED" == "true" ]]; then
|
|
warn "Some health checks failed. Services may still be starting."
|
|
info "Check logs: docker compose logs --tail 50"
|
|
else
|
|
success "All health checks passed"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Summary
|
|
# =============================================================================
|
|
|
|
ELAPSED="$(elapsed)"
|
|
FINAL_COMMIT="$(git rev-parse --short HEAD)"
|
|
|
|
echo ""
|
|
echo -e "${BOLD}${GREEN}══════════════════════════════════════════════════${NC}"
|
|
echo -e "${BOLD}${GREEN} Upgrade Complete${NC}"
|
|
echo -e "${BOLD}${GREEN}══════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e " ${BOLD}Previous:${NC} $PRE_UPGRADE_SHORT"
|
|
echo -e " ${BOLD}Current:${NC} $FINAL_COMMIT ($(git log -1 --format='%s' HEAD))"
|
|
echo -e " ${BOLD}Commits:${NC} $COMMIT_COUNT"
|
|
echo -e " ${BOLD}Duration:${NC} $ELAPSED"
|
|
echo -e " ${BOLD}Log:${NC} $LOG_FILE"
|
|
echo ""
|
|
|
|
release_lock
|
|
trap - EXIT
|