Fresh-install + upgrade-path hardening bundle

Six independent fixes surfaced during the v2.9.1 → v2.9.2 admin-UI upgrade validation today. Together they make a clean install on a new box work end-to-end without in-session patching. - Fix 1: scripts/validate-compose-parity.sh + build-release.sh hook — fail release builds when api/admin/media-api/nginx healthcheck blocks drift between docker-compose.yml and docker-compose.prod.yml. Previous boot-race fix had to be applied to both files manually. - Fix 2: scripts/systemd/install.sh chowns logs/ to the install user (the API container creates subdirs there as root, locking the host-side watcher out), pre-creates logs/upgrade-watcher.log, and changemaker-upgrade.service adds StartLimitIntervalSec=0 so a single transient failure can't wedge the .path unit permanently. - Fix 3: /api/upgrade/status now returns a `watcher` sub-object that flags the host systemd watcher as stalled when trigger.json has been pending >30s. Admin SettingsPage SystemUpgradeTab renders a warning Alert with the systemctl recovery command when unhealthy. - Fix 4: scripts/upgrade.sh write_result() — prefer head -1 VERSION over `git rev-parse HEAD` so release-mode upgrades report the new tag in result.json instead of "unknown". - Fix 5: admin container healthcheck start_period 20s → 60s in both compose files, same class as the earlier api fix. Matches Gancio convention. - Fix 7: /api/pangolin/sync now detects resources bound to a stale siteId (common after --pangolin-site new rotations), deletes and recreates them against the current site, and reports them under a new `reassigned` response field. Bunker Admin
2026-04-15 11:57:50 -06:00
parent 5115c65691
commit 23df6a8b52
12 changed files with 223 additions and 39 deletions
--- a/api/src/modules/pangolin/pangolin.routes.ts
+++ b/api/src/modules/pangolin/pangolin.routes.ts
@@ -867,11 +867,43 @@ router.post('/sync', pangolinSetupLimiter, async (_req: Request, res: Response)
    const existingByDomain = new Map(existing.map(r => [r.fullDomain || '', r]));

    const created: string[] = [];
+    const reassigned: string[] = [];
    const targetFixed: string[] = [];
    const skipped: string[] = [];
    const warnings: string[] = [];
    const errors: string[] = [];

+    // Create resource + public access + target. Shared by "new" and "reassign"
+    // flows so `--pangolin-site new` installs can rebuild after a site rotation.
+    const createResourceForDef = async (def: ResourceDefinition, fullDomain: string) => {
+      const resource = await pangolinClient.createResource({
+        name: def.name,
+        domainId: matchingDomain.domainId,
+        ...(def.subdomain ? { subdomain: def.subdomain } : {}),
+        http: true,
+        protocol: 'tcp',
+      });
+
+      try {
+        await pangolinClient.updateResource(resource.resourceId, { sso: false, blockAccess: false });
+      } catch {
+        logger.warn(`Created ${fullDomain} but failed to set public access`);
+      }
+
+      try {
+        await pangolinClient.createTarget(resource.resourceId, {
+          siteId,
+          ip: def.target_ip,
+          port: def.target_port,
+          method: 'http',
+          enabled: true,
+        });
+      } catch (targetErr) {
+        const msg = targetErr instanceof Error ? targetErr.message : 'Unknown error';
+        errors.push(`${fullDomain} (target): ${msg}`);
+      }
+    };
+
    for (const def of resourceDefs) {
      const fullDomain = def.subdomain ? `${def.subdomain}.${domain}` : domain;

@@ -890,10 +922,30 @@ router.post('/sync', pangolinSetupLimiter, async (_req: Request, res: Response)
      const existingResource = existingByDomain.get(fullDomain);

      if (existingResource) {
-        // Resource exists — verify it has a target
+        // Resource exists — verify target points at the CURRENT site.
        try {
          const targets = await pangolinClient.listTargets(existingResource.resourceId);
-          if (targets.length === 0) {
+          const currentTargetSiteId = targets[0]?.siteId;
+          const siteMismatch =
+            targets.length > 0 && Number(currentTargetSiteId) !== Number(siteId);
+
+          if (siteMismatch) {
+            // Stale siteId from a previous `--pangolin-site new` install.
+            // Delete and recreate against the current site.
+            logger.warn(
+              `Resource ${fullDomain} bound to stale siteId ${currentTargetSiteId}, reassigning to ${siteId}`,
+            );
+            try {
+              await pangolinClient.deleteResource(existingResource.resourceId);
+              await createResourceForDef(def, fullDomain);
+              reassigned.push(fullDomain);
+              logger.info(`Reassigned ${fullDomain} to siteId ${siteId}`);
+            } catch (err) {
+              const msg = err instanceof Error ? err.message : 'Unknown error';
+              errors.push(`${fullDomain} (reassign): ${msg}`);
+              logger.error(`Failed to reassign resource ${fullDomain}:`, err);
+            }
+          } else if (targets.length === 0) {
            // Missing target — create one
            logger.info(`Resource ${fullDomain} has no target, creating one...`);
            await pangolinClient.createTarget(existingResource.resourceId, {
@@ -927,36 +979,7 @@ router.post('/sync', pangolinSetupLimiter, async (_req: Request, res: Response)
      } else {
        // Create new resource + target
        try {
-          // Root domain: omit subdomain field entirely (Pangolin rejects empty string)
-          const resource = await pangolinClient.createResource({
-            name: def.name,
-            domainId: matchingDomain.domainId,
-            ...(def.subdomain ? { subdomain: def.subdomain } : {}),
-            http: true,
-            protocol: 'tcp',
-          });
-
-          // Make publicly accessible (disable SSO auth + blockAccess)
-          try {
-            await pangolinClient.updateResource(resource.resourceId, { sso: false, blockAccess: false });
-          } catch {
-            logger.warn(`Created ${fullDomain} but failed to set public access`);
-          }
-
-          // Create target
-          try {
-            await pangolinClient.createTarget(resource.resourceId, {
-              siteId,
-              ip: def.target_ip,
-              port: def.target_port,
-              method: 'http',
-              enabled: true,
-            });
-          } catch (targetErr) {
-            const msg = targetErr instanceof Error ? targetErr.message : 'Unknown error';
-            errors.push(`${fullDomain} (target): ${msg}`);
-          }
-
+          await createResourceForDef(def, fullDomain);
          created.push(fullDomain);
          logger.info(`Created resource + target: ${fullDomain}`);
        } catch (err) {
@@ -970,11 +993,12 @@ router.post('/sync', pangolinSetupLimiter, async (_req: Request, res: Response)
    res.json({
      success: true,
      created: created.length,
+      reassigned: reassigned.length,
      targetFixed: targetFixed.length,
      skipped: skipped.length,
      warnings: warnings.length,
      errors: errors.length,
-      details: { created, targetFixed, skipped, warnings, errors },
+      details: { created, reassigned, targetFixed, skipped, warnings, errors },
    });
  } catch (err) {
    const msg = err instanceof Error ? err.message : 'Unknown error';
--- a/api/src/modules/upgrade/upgrade.routes.ts
+++ b/api/src/modules/upgrade/upgrade.routes.ts
@@ -17,8 +17,9 @@ router.get('/status', (_req, res) => {
  const progress = upgradeService.getProgress();
  const result = upgradeService.getResult();
  const running = upgradeService.isRunning();
+  const watcher = upgradeService.getWatcherHealth();

-  res.json({ status, progress: running ? progress : null, result, running });
+  res.json({ status, progress: running ? progress : null, result, running, watcher });
 });

 /**
--- a/api/src/modules/upgrade/upgrade.service.ts
+++ b/api/src/modules/upgrade/upgrade.service.ts
@@ -60,6 +60,12 @@ export interface UpgradeResult {
  triggeredBy?: string;
 }

+export interface WatcherHealth {
+  healthy: boolean;
+  reason?: string;
+  pendingSince?: string;
+}
+
 interface TriggerPayload {
  action: 'check' | 'upgrade';
  branch?: string;
@@ -96,6 +102,31 @@ function getStatus(): UpgradeStatus | null {
  return readJsonFile<UpgradeStatus>(STATUS_FILE);
 }

+/**
+ * Watcher liveness heuristic. The host-side systemd watcher consumes and
+ * DELETES trigger.json within ~1s of it appearing. If trigger.json exists
+ * and is older than the threshold, the `.path` unit is almost certainly
+ * wedged (e.g. StartLimitBurst latch) and the admin UI should surface it.
+ */
+const WATCHER_STALL_MS = 30 * 1000;
+
+function getWatcherHealth(): WatcherHealth {
+  try {
+    if (!fs.existsSync(TRIGGER_FILE)) return { healthy: true };
+    const mtimeMs = fs.statSync(TRIGGER_FILE).mtimeMs;
+    const age = Date.now() - mtimeMs;
+    if (age <= WATCHER_STALL_MS) return { healthy: true };
+    return {
+      healthy: false,
+      reason: `Trigger file has been pending for ${Math.round(age / 1000)}s — host upgrade watcher may be stopped or failed`,
+      pendingSince: new Date(mtimeMs).toISOString(),
+    };
+  } catch (err) {
+    logger.warn('getWatcherHealth failed:', err);
+    return { healthy: true };
+  }
+}
+
 function getProgress(): UpgradeProgress | null {
  return readJsonFile<UpgradeProgress>(PROGRESS_FILE);
 }
@@ -221,6 +252,7 @@ export const upgradeService = {
  getStatus,
  getProgress,
  getResult,
+  getWatcherHealth,
  isRunning,
  triggerCheck,
  triggerUpgrade,