Make embed proxy ports configurable via env vars for multi-instance deployments

All 13 nginx embed proxy ports (8881-8895) are now driven by environment
variables instead of being hardcoded. This prevents port conflicts when
running multiple Changemaker instances on the same host.

Chain: .env → docker-compose port mappings → nginx container env →
entrypoint.sh envsubst → services.conf.template listen directives →
API /services/config endpoint → frontend buildServiceUrl().

Existing deployments are unaffected (all vars default to current values).

Bunker Admin

changemaker-control-panel/api/prisma/migrations/20260324043047_add_upgrades_and_events/migration.sql

@@ -0,0 +1,67 @@
+-- CreateEnum
+CREATE TYPE "UpgradeStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED', 'ROLLED_BACK');
+
+-- CreateEnum
+CREATE TYPE "EventSeverity" AS ENUM ('ERROR', 'WARNING', 'INFO');
+
+-- CreateTable
+CREATE TABLE "instance_upgrades" (
+    "id" TEXT NOT NULL,
+    "instance_id" TEXT NOT NULL,
+    "status" "UpgradeStatus" NOT NULL DEFAULT 'PENDING',
+    "previous_commit" TEXT,
+    "new_commit" TEXT,
+    "commit_count" INTEGER NOT NULL DEFAULT 0,
+    "branch" TEXT NOT NULL DEFAULT 'v2',
+    "current_phase" INTEGER NOT NULL DEFAULT 0,
+    "phase_name" TEXT,
+    "percentage" INTEGER NOT NULL DEFAULT 0,
+    "progress_message" TEXT,
+    "duration_seconds" INTEGER,
+    "error_message" TEXT,
+    "warnings" JSONB,
+    "log" TEXT,
+    "triggered_by_id" TEXT,
+    "started_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "completed_at" TIMESTAMP(3),
+
+    CONSTRAINT "instance_upgrades_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "instance_events" (
+    "id" TEXT NOT NULL,
+    "instance_id" TEXT NOT NULL,
+    "severity" "EventSeverity" NOT NULL,
+    "source" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "message" TEXT NOT NULL,
+    "metadata" JSONB,
+    "acknowledged" BOOLEAN NOT NULL DEFAULT false,
+    "acknowledged_at" TIMESTAMP(3),
+    "acknowledged_by_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "instance_events_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "instance_upgrades_instance_id_started_at_idx" ON "instance_upgrades"("instance_id", "started_at");
+
+-- CreateIndex
+CREATE INDEX "instance_events_instance_id_created_at_idx" ON "instance_events"("instance_id", "created_at");
+
+-- CreateIndex
+CREATE INDEX "instance_events_severity_acknowledged_idx" ON "instance_events"("severity", "acknowledged");
+
+-- AddForeignKey
+ALTER TABLE "instance_upgrades" ADD CONSTRAINT "instance_upgrades_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instances"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "instance_upgrades" ADD CONSTRAINT "instance_upgrades_triggered_by_id_fkey" FOREIGN KEY ("triggered_by_id") REFERENCES "ccp_users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "instance_events" ADD CONSTRAINT "instance_events_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instances"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "instance_events" ADD CONSTRAINT "instance_events_acknowledged_by_id_fkey" FOREIGN KEY ("acknowledged_by_id") REFERENCES "ccp_users"("id") ON DELETE SET NULL ON UPDATE CASCADE;

changemaker-control-panel/api/prisma/schema.prisma

@@ -24,8 +24,10 @@ model CcpUser {
   createdAt     DateTime @default(now()) @map("created_at")
   updatedAt     DateTime @updatedAt @map("updated_at")
 
-  refreshTokens CcpRefreshToken[]
-  auditLogs     AuditLog[]
+  refreshTokens   CcpRefreshToken[]
+  auditLogs       AuditLog[]
+  triggeredUpgrades InstanceUpgrade[]
+  acknowledgedEvents InstanceEvent[]
 
   @@map("ccp_users")
 }
@@ -115,6 +117,8 @@ model Instance {
   healthChecks    HealthCheck[]
   backups         Backup[]
   auditLogs       AuditLog[]
+  upgrades        InstanceUpgrade[]
+  events          InstanceEvent[]
 
   @@map("instances")
 }
@@ -228,6 +232,77 @@ model AuditLog {
   @@map("audit_logs")
 }
 
+// ─── Instance Upgrades ────────────────────────────────────
+
+enum UpgradeStatus {
+  PENDING
+  IN_PROGRESS
+  COMPLETED
+  FAILED
+  ROLLED_BACK
+}
+
+model InstanceUpgrade {
+  id              String        @id @default(uuid())
+  instanceId      String        @map("instance_id")
+  status          UpgradeStatus @default(PENDING)
+  previousCommit  String?       @map("previous_commit")
+  newCommit       String?       @map("new_commit")
+  commitCount     Int           @default(0) @map("commit_count")
+  branch          String        @default("v2")
+
+  // Progress tracking (updated from progress.json polling)
+  currentPhase    Int           @default(0) @map("current_phase")
+  phaseName       String?       @map("phase_name")
+  percentage      Int           @default(0)
+  progressMessage String?       @map("progress_message")
+
+  // Result
+  durationSeconds Int?          @map("duration_seconds")
+  errorMessage    String?       @map("error_message")
+  warnings        Json?
+  log             String?
+
+  triggeredById   String?       @map("triggered_by_id")
+  startedAt       DateTime      @default(now()) @map("started_at")
+  completedAt     DateTime?     @map("completed_at")
+
+  instance    Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
+  triggeredBy CcpUser?  @relation(fields: [triggeredById], references: [id], onDelete: SetNull)
+
+  @@index([instanceId, startedAt])
+  @@map("instance_upgrades")
+}
+
+// ─── Instance Events (Errors/Warnings) ───────────────────
+
+enum EventSeverity {
+  ERROR
+  WARNING
+  INFO
+}
+
+model InstanceEvent {
+  id               String        @id @default(uuid())
+  instanceId       String        @map("instance_id")
+  severity         EventSeverity
+  source           String        // 'health_check', 'upgrade', 'container', 'provisioning'
+  title            String
+  message          String
+  metadata         Json?
+  acknowledged     Boolean       @default(false)
+  acknowledgedAt   DateTime?     @map("acknowledged_at")
+  acknowledgedById String?       @map("acknowledged_by_id")
+  createdAt        DateTime      @default(now()) @map("created_at")
+
+  instance       Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
+  acknowledgedBy CcpUser?  @relation(fields: [acknowledgedById], references: [id], onDelete: SetNull)
+
+  @@index([instanceId, createdAt])
+  @@index([severity, acknowledged])
+  @@map("instance_events")
+}
+
 // ─── CCP Settings ──────────────────────────────────────────
 
 model CcpSetting {

changemaker-control-panel/api/src/modules/events/events.routes.ts

@@ -0,0 +1,80 @@
+import { Router, Request, Response } from 'express';
+import { EventSeverity } from '@prisma/client';
+import { authenticate, requireRole } from '../../middleware/auth';
+import * as eventService from '../../services/event.service';
+
+const router = Router();
+
+router.use(authenticate);
+
+// ─── Cross-Instance Event Queries ────────────────────────────────
+
+// GET /api/events — list events with filters
+router.get(
+  '/',
+  async (req: Request, res: Response) => {
+    const { instanceId, severity, acknowledged, source, from, to, page, limit } = req.query;
+
+    const filters: eventService.EventFilters = {
+      instanceId: instanceId as string | undefined,
+      severity: severity ? (severity as EventSeverity) : undefined,
+      acknowledged: acknowledged !== undefined ? acknowledged === 'true' : undefined,
+      source: source as string | undefined,
+      from: from ? new Date(from as string) : undefined,
+      to: to ? new Date(to as string) : undefined,
+      page: page ? Math.max(1, parseInt(page as string, 10)) : 1,
+      limit: limit ? Math.min(100, Math.max(1, parseInt(limit as string, 10))) : 50,
+    };
+
+    const result = await eventService.listEvents(filters);
+    res.json(result);
+  }
+);
+
+// GET /api/events/summary — unacknowledged counts for dashboard
+router.get(
+  '/summary',
+  async (_req: Request, res: Response) => {
+    const summary = await eventService.getUnacknowledgedSummary();
+    res.json({ data: summary });
+  }
+);
+
+// PUT /api/events/:id/acknowledge — acknowledge a single event
+router.put(
+  '/:id/acknowledge',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const event = await eventService.acknowledgeEvent(req.params.id as string, req.user!.id);
+    res.json({ data: event });
+  }
+);
+
+// ─── Instance-Scoped Event Routes ────────────────────────────────
+// These are mounted at /api/instances/:id/events via a sub-export
+
+export const instanceEventsRouter = Router({ mergeParams: true });
+instanceEventsRouter.use(authenticate);
+
+// GET /api/instances/:id/events
+instanceEventsRouter.get(
+  '/',
+  async (req: Request, res: Response) => {
+    const page = Math.max(1, parseInt(req.query.page as string, 10) || 1);
+    const limit = Math.min(100, Math.max(1, parseInt(req.query.limit as string, 10) || 50));
+    const result = await eventService.listInstanceEvents(req.params.id as string, page, limit);
+    res.json(result);
+  }
+);
+
+// PUT /api/instances/:id/events/acknowledge-all
+instanceEventsRouter.put(
+  '/acknowledge-all',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const result = await eventService.acknowledgeAllForInstance(req.params.id as string, req.user!.id);
+    res.json({ data: result });
+  }
+);
+
+export default router;

changemaker-control-panel/api/src/modules/instances/instances.routes.ts

@@ -8,6 +8,7 @@ import { createInstanceSchema, updateInstanceSchema, registerInstanceSchema, rec
 import * as instancesService from './instances.service';
 import * as healthService from '../../services/health.service';
 import * as backupService from '../../services/backup.service';
+import * as upgradeService from '../../services/upgrade.service';
 import { discoverInstances } from '../../services/discovery.service';
 
 const secretsLimiter = rateLimit({
@@ -265,6 +266,52 @@ router.get(
   }
 );
 
+// ─── Upgrades ──────────────────────────────────────────────────────
+
+router.post(
+  '/:id/check-update',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const status = await upgradeService.checkForUpdates(req.params.id as string);
+    res.json({ data: status });
+  }
+);
+
+router.post(
+  '/:id/upgrade',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const { skipBackup, useRegistry, branch } = req.body || {};
+    const upgrade = await upgradeService.startUpgrade(
+      req.params.id as string,
+      req.user!.id,
+      req.ip,
+      { skipBackup, useRegistry, branch }
+    );
+    res.status(201).json({ data: upgrade });
+  }
+);
+
+router.get(
+  '/:id/upgrade-status',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const status = await upgradeService.getUpgradeStatus(req.params.id as string);
+    res.json({ data: status });
+  }
+);
+
+router.get(
+  '/:id/upgrade-history',
+  requireRole('SUPER_ADMIN', 'OPERATOR'),
+  async (req: Request, res: Response) => {
+    const page = Math.max(1, parseInt(req.query.page as string, 10) || 1);
+    const limit = Math.min(100, Math.max(1, parseInt(req.query.limit as string, 10) || 20));
+    const result = await upgradeService.getUpgradeHistory(req.params.id as string, page, limit);
+    res.json(result);
+  }
+);
+
 // ─── Health Checks ──────────────────────────────────────────────────
 
 router.post(

changemaker-control-panel/api/src/modules/instances/provisioner.ts

@@ -10,6 +10,7 @@ import { decryptJson } from '../../utils/encryption';
 import { renderAllTemplates, buildTemplateContext } from '../../services/template-engine';
 import * as docker from '../../services/docker.service';
 import { logger } from '../../utils/logger';
+import { createEvent } from '../../services/event.service';
 const execFile = promisify(execFileCb);
 
 /**
@@ -142,30 +143,28 @@ export async function provision(instanceId: string): Promise<void> {
       docker.waitForHealthy(redisContainer, 60_000),
     ]);
 
-    // ── Step 10: Run database schema sync ─────────────────────────
-    await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Setting up database schema'));
-    logger.info(`[provisioner] ${instance.slug}: Pushing Prisma schema to database`);
-    // Use composeRun with --entrypoint "" to skip the API's startup entrypoint
-    // (which would try migrate deploy + seed and fail on schema drift)
-    await docker.composeRun(basePath, composeProject, 'api', 'npx prisma db push --accept-data-loss', 180_000);
-
-    // ── Step 11: Seed database ─────────────────────────────────────
-    await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Seeding database'));
-    logger.info(`[provisioner] ${instance.slug}: Seeding database`);
-    await docker.composeRun(basePath, composeProject, 'api', 'npx prisma db seed', 120_000);
-
-    // ── Step 12: Start all services ────────────────────────────────
+    // ── Step 10: Start all services ────────────────────────────────
+    // The API entrypoint (docker-entrypoint.sh) handles:
+    //   1. Wait for Postgres
+    //   2. prisma migrate deploy
+    //   3. prisma db seed (needs tsx — installed in production image)
+    //   4. Start the server
     await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Starting all services'));
-    logger.info(`[provisioner] ${instance.slug}: Starting all services`);
+    logger.info(`[provisioner] ${instance.slug}: Starting all services (entrypoint handles migrate + seed)`);
     await docker.composeUp(basePath, composeProject);
 
-    // ── Step 13: Health check ──────────────────────────────────────
-    await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Verifying instance health'));
+    // ── Step 11: Wait for API healthy ───────────────────────────────
+    await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Waiting for API to be ready'));
     logger.info(`[provisioner] ${instance.slug}: Waiting for API health check`);
     const ports = instance.portConfig as Record<string, number>;
-    // Use host.docker.internal to reach ports exposed on the Docker host
-    // (localhost inside the CCP container refers to the CCP container itself)
-    await docker.waitForHttp(`http://host.docker.internal:${ports.api}/api/health`, 120_000);
+    await docker.waitForHttp(`http://host.docker.internal:${ports.api}/api/health`, 180_000);
+
+    // ── Step 12: Verify instance health ─────────────────────────────
+    await updateStatus(instanceId, InstanceStatus.PROVISIONING, stepMsg('Verifying instance health'));
+    logger.info(`[provisioner] ${instance.slug}: Final health verification`);
+    await docker.waitForHttp(`http://host.docker.internal:${ports.api}/api/health`, 30_000);
+
+    // ── Step 13: done (below) ───────────────────────────────────────
 
     // ── Done! ──────────────────────────────────────────────────────
     await updateStatus(instanceId, InstanceStatus.RUNNING, 'Provisioning complete');
@@ -196,5 +195,15 @@ export async function provision(instanceId: string): Promise<void> {
         details: { event: 'provisioning_failed', error: errorMsg, step },
       },
     }).catch(() => {});
+
+    // Create error event
+    await createEvent(
+      instanceId,
+      'ERROR',
+      'provisioning',
+      'Provisioning failed',
+      `Failed at step ${step}/${totalSteps}: ${errorMsg.slice(0, 500)}`,
+      { step, totalSteps }
+    ).catch(() => {});
   }
 }

changemaker-control-panel/api/src/server.ts

@@ -15,6 +15,7 @@ import settingsRoutes from './modules/settings/settings.routes';
 import healthRoutes from './modules/health/health.routes';
 import auditRoutes from './modules/audit/audit.routes';
 import backupRoutes from './modules/backups/backup.routes';
+import eventsRoutes, { instanceEventsRouter } from './modules/events/events.routes';
 import { startHealthScheduler } from './services/health.service';
 import { autoDiscoverOnStartup } from './services/discovery.service';
 
@@ -57,6 +58,8 @@ app.use('/api/settings', settingsRoutes);
 app.use('/api/health', healthRoutes);
 app.use('/api/audit', auditRoutes);
 app.use('/api/backups', backupRoutes);
+app.use('/api/events', eventsRoutes);
+app.use('/api/instances/:id/events', instanceEventsRouter);
 
 // Error handler (must be last)
 app.use(errorHandler);

changemaker-control-panel/api/src/services/event.service.ts

@@ -0,0 +1,174 @@
+import { EventSeverity, Prisma } from '@prisma/client';
+import { prisma } from '../lib/prisma';
+import { logger } from '../utils/logger';
+
+// ─── Event Creation ───────────────────────────────────────────────
+
+/**
+ * Create an instance event (error, warning, or info).
+ * Deduplicates: won't create a duplicate event if one with the same
+ * source + title exists for this instance within the last 5 minutes.
+ */
+export async function createEvent(
+  instanceId: string,
+  severity: 'ERROR' | 'WARNING' | 'INFO',
+  source: string,
+  title: string,
+  message: string,
+  metadata?: Record<string, unknown>
+): Promise<void> {
+  // Deduplicate: avoid spamming the same event within 5 minutes
+  const fiveMinAgo = new Date(Date.now() - 5 * 60 * 1000);
+  const existing = await prisma.instanceEvent.findFirst({
+    where: {
+      instanceId,
+      source,
+      title,
+      createdAt: { gte: fiveMinAgo },
+    },
+  });
+
+  if (existing) {
+    logger.debug(`[events] Skipping duplicate event: ${title} for instance ${instanceId}`);
+    return;
+  }
+
+  await prisma.instanceEvent.create({
+    data: {
+      instanceId,
+      severity: severity as EventSeverity,
+      source,
+      title,
+      message,
+      metadata: metadata as Prisma.InputJsonValue ?? undefined,
+    },
+  });
+
+  logger.info(`[events] ${severity} event for instance ${instanceId}: ${title}`);
+}
+
+// ─── Event Queries ────────────────────────────────────────────────
+
+export interface EventFilters {
+  instanceId?: string;
+  severity?: EventSeverity;
+  acknowledged?: boolean;
+  source?: string;
+  from?: Date;
+  to?: Date;
+  page?: number;
+  limit?: number;
+}
+
+/**
+ * List events across all instances with filtering and pagination.
+ */
+export async function listEvents(filters: EventFilters) {
+  const page = filters.page || 1;
+  const limit = Math.min(filters.limit || 50, 100);
+
+  const where: Prisma.InstanceEventWhereInput = {};
+  if (filters.instanceId) where.instanceId = filters.instanceId;
+  if (filters.severity) where.severity = filters.severity;
+  if (filters.acknowledged !== undefined) where.acknowledged = filters.acknowledged;
+  if (filters.source) where.source = filters.source;
+  if (filters.from || filters.to) {
+    where.createdAt = {};
+    if (filters.from) where.createdAt.gte = filters.from;
+    if (filters.to) where.createdAt.lte = filters.to;
+  }
+
+  const [data, total] = await Promise.all([
+    prisma.instanceEvent.findMany({
+      where,
+      orderBy: { createdAt: 'desc' },
+      skip: (page - 1) * limit,
+      take: limit,
+      include: {
+        instance: { select: { id: true, name: true, slug: true } },
+        acknowledgedBy: { select: { id: true, name: true, email: true } },
+      },
+    }),
+    prisma.instanceEvent.count({ where }),
+  ]);
+
+  return { data, total, page, limit };
+}
+
+/**
+ * List events for a single instance.
+ */
+export async function listInstanceEvents(instanceId: string, page = 1, limit = 50) {
+  return listEvents({ instanceId, page, limit });
+}
+
+/**
+ * Acknowledge a single event.
+ */
+export async function acknowledgeEvent(eventId: string, userId: string) {
+  const event = await prisma.instanceEvent.findUnique({ where: { id: eventId } });
+  if (!event) throw new Error('Event not found');
+
+  return prisma.instanceEvent.update({
+    where: { id: eventId },
+    data: {
+      acknowledged: true,
+      acknowledgedAt: new Date(),
+      acknowledgedById: userId,
+    },
+  });
+}
+
+/**
+ * Acknowledge all unacknowledged events for an instance.
+ */
+export async function acknowledgeAllForInstance(instanceId: string, userId: string) {
+  const result = await prisma.instanceEvent.updateMany({
+    where: {
+      instanceId,
+      acknowledged: false,
+    },
+    data: {
+      acknowledged: true,
+      acknowledgedAt: new Date(),
+      acknowledgedById: userId,
+    },
+  });
+
+  return { acknowledged: result.count };
+}
+
+/**
+ * Get summary of unacknowledged events by severity (for dashboard).
+ */
+export async function getUnacknowledgedSummary() {
+  const [errors, warnings, infos] = await Promise.all([
+    prisma.instanceEvent.count({
+      where: { acknowledged: false, severity: EventSeverity.ERROR },
+    }),
+    prisma.instanceEvent.count({
+      where: { acknowledged: false, severity: EventSeverity.WARNING },
+    }),
+    prisma.instanceEvent.count({
+      where: { acknowledged: false, severity: EventSeverity.INFO },
+    }),
+  ]);
+
+  // Also get the 5 most recent unacknowledged errors for dashboard display
+  const recentErrors = await prisma.instanceEvent.findMany({
+    where: { acknowledged: false, severity: { in: [EventSeverity.ERROR, EventSeverity.WARNING] } },
+    orderBy: { createdAt: 'desc' },
+    take: 5,
+    include: {
+      instance: { select: { id: true, name: true, slug: true } },
+    },
+  });
+
+  return {
+    errors,
+    warnings,
+    infos,
+    total: errors + warnings + infos,
+    recentErrors,
+  };
+}

changemaker-control-panel/api/src/services/health.service.ts

@@ -2,6 +2,7 @@ import { InstanceStatus, HealthStatus } from '@prisma/client';
 import { prisma } from '../lib/prisma';
 import * as docker from './docker.service';
 import { logger } from '../utils/logger';
+import { createEvent } from './event.service';
 import type { ContainerInfo } from './docker.service';
 
 /**
@@ -94,6 +95,13 @@ export async function checkInstanceHealth(instanceId: string) {
   const responseTimeMs = Date.now() - startTime;
   const { status, serviceStatus, totalServices, healthyServices } = determineHealth(containers);
 
+  // Get the previous health check to detect transitions
+  const previousCheck = await prisma.healthCheck.findFirst({
+    where: { instanceId },
+    orderBy: { checkedAt: 'desc' },
+    select: { status: true },
+  });
+
   const healthCheck = await prisma.healthCheck.create({
     data: {
       instanceId,
@@ -110,6 +118,44 @@ export async function checkInstanceHealth(instanceId: string) {
     data: { lastHealthCheck: new Date() },
   });
 
+  // Create events on health transitions
+  const previousStatus = previousCheck?.status;
+  if (status !== previousStatus) {
+    if (status === HealthStatus.UNHEALTHY) {
+      createEvent(
+        instanceId,
+        'ERROR',
+        'health_check',
+        'Instance unhealthy',
+        `${instance.slug}: ${healthyServices}/${totalServices} services healthy`,
+        { serviceStatus, responseTimeMs }
+      ).catch((e) => logger.warn(`[health] Failed to create event: ${(e as Error).message}`));
+    } else if (status === HealthStatus.DEGRADED && previousStatus !== HealthStatus.UNHEALTHY) {
+      createEvent(
+        instanceId,
+        'WARNING',
+        'health_check',
+        'Instance degraded',
+        `${instance.slug}: ${healthyServices}/${totalServices} services healthy`,
+        { serviceStatus, responseTimeMs }
+      ).catch((e) => logger.warn(`[health] Failed to create event: ${(e as Error).message}`));
+    }
+  }
+
+  // Detect crashed containers (exited with non-zero exit code)
+  for (const c of containers) {
+    if (c.state === 'exited' && c.exitCode !== 0) {
+      createEvent(
+        instanceId,
+        'ERROR',
+        'container',
+        `Container crashed: ${c.service || c.name}`,
+        `${c.service || c.name} exited with code ${c.exitCode}`,
+        { service: c.service, container: c.name, exitCode: c.exitCode, status: c.status }
+      ).catch((e) => logger.warn(`[health] Failed to create container event: ${(e as Error).message}`));
+    }
+  }
+
   return healthCheck;
 }
 

changemaker-control-panel/api/src/services/secret-generator.ts

@@ -43,6 +43,7 @@ export interface InstanceSecrets {
   redisPassword: string;
   jwtAccessSecret: string;
   jwtRefreshSecret: string;
+  jwtInviteSecret: string;
   encryptionKey: string;
   initialAdminPassword: string;
   nocodbAdminPassword: string;
@@ -66,6 +67,7 @@ export function generateSecrets(adminEmail: string): InstanceSecrets & { adminEm
     redisPassword: randomHex(16),
     jwtAccessSecret: randomHex(32),
     jwtRefreshSecret: randomHex(32),
+    jwtInviteSecret: randomHex(32),
     encryptionKey: randomHex(32),
     initialAdminPassword: randomPassword(16),
     nocodbAdminPassword: randomPassword(16),

changemaker-control-panel/api/src/services/template-engine.ts

@@ -49,6 +49,7 @@ export interface TemplateContext {
     redisPassword: string;
     jwtAccessSecret: string;
     jwtRefreshSecret: string;
+    jwtInviteSecret: string;
     encryptionKey: string;
     initialAdminPassword: string;
     adminEmail: string;
@@ -161,6 +162,7 @@ export function buildTemplateContext(
       redisPassword: secrets.redisPassword,
       jwtAccessSecret: secrets.jwtAccessSecret,
       jwtRefreshSecret: secrets.jwtRefreshSecret,
+      jwtInviteSecret: secrets.jwtInviteSecret || secrets.jwtAccessSecret,
       encryptionKey: secrets.encryptionKey,
       initialAdminPassword: secrets.initialAdminPassword,
       adminEmail: secrets.adminEmail,

changemaker-control-panel/api/src/services/upgrade.service.ts

@@ -0,0 +1,410 @@
+import { exec as execCb } from 'child_process';
+import { promisify } from 'util';
+import fs from 'fs/promises';
+import path from 'path';
+import { UpgradeStatus, AuditAction, InstanceStatus, Prisma } from '@prisma/client';
+import { prisma } from '../lib/prisma';
+import { logger } from '../utils/logger';
+import { createEvent } from './event.service';
+
+const exec = promisify(execCb);
+
+const UPGRADE_TIMEOUT = 600_000; // 10 minutes
+const PROGRESS_POLL_INTERVAL = 2_000; // 2 seconds
+
+// ─── Update Check ─────────────────────────────────────────────────
+
+export interface UpdateStatus {
+  branch: string;
+  currentCommit: string;
+  currentMessage?: string;
+  remoteCommit: string | null;
+  commitsBehind: number;
+  changelog: Array<{ hash: string; message: string; date: string; author: string }>;
+  checkedAt: string;
+  error: string | null;
+}
+
+/**
+ * Check for available updates by running upgrade-check.sh in the instance's basePath.
+ * Falls back to reading an existing status.json if the script isn't available.
+ */
+export async function checkForUpdates(instanceId: string): Promise<UpdateStatus> {
+  const instance = await prisma.instance.findUnique({ where: { id: instanceId } });
+  if (!instance) throw new Error('Instance not found');
+
+  const basePath = instance.basePath;
+  const statusFile = path.join(basePath, 'data', 'upgrade', 'status.json');
+  const scriptPath = path.join(basePath, 'scripts', 'upgrade-check.sh');
+
+  // Try to run upgrade-check.sh
+  try {
+    await fs.access(scriptPath);
+    await exec(`bash "${scriptPath}"`, {
+      cwd: basePath,
+      timeout: 30_000,
+      env: { ...process.env, COMPOSE_ANSI: 'never' },
+    });
+  } catch (err) {
+    logger.warn(`[upgrade] upgrade-check.sh failed for ${instance.slug}: ${(err as Error).message}`);
+    // Script may have still written status.json before failing — try reading it
+  }
+
+  // Read status.json
+  try {
+    const raw = await fs.readFile(statusFile, 'utf-8');
+    const status = JSON.parse(raw) as UpdateStatus;
+    return status;
+  } catch {
+    // If no status.json exists, try to gather basic git info
+    try {
+      const { stdout: branch } = await exec('git rev-parse --abbrev-ref HEAD', { cwd: basePath, timeout: 5_000 });
+      const { stdout: commit } = await exec('git rev-parse --short HEAD', { cwd: basePath, timeout: 5_000 });
+      return {
+        branch: branch.trim(),
+        currentCommit: commit.trim(),
+        remoteCommit: null,
+        commitsBehind: 0,
+        changelog: [],
+        checkedAt: new Date().toISOString(),
+        error: 'Could not check for remote updates',
+      };
+    } catch {
+      return {
+        branch: instance.gitBranch,
+        currentCommit: instance.gitCommit || 'unknown',
+        remoteCommit: null,
+        commitsBehind: 0,
+        changelog: [],
+        checkedAt: new Date().toISOString(),
+        error: 'Could not determine version info (no .git directory?)',
+      };
+    }
+  }
+}
+
+// ─── Upgrade Orchestration ────────────────────────────────────────
+
+export interface StartUpgradeOptions {
+  skipBackup?: boolean;
+  useRegistry?: boolean;
+  branch?: string;
+}
+
+/**
+ * Start an upgrade for an instance. Returns the created InstanceUpgrade record.
+ * The actual upgrade runs asynchronously (fire-and-forget).
+ */
+export async function startUpgrade(
+  instanceId: string,
+  userId: string,
+  ipAddress?: string,
+  options?: StartUpgradeOptions
+) {
+  const instance = await prisma.instance.findUnique({ where: { id: instanceId } });
+  if (!instance) throw new Error('Instance not found');
+
+  if (instance.status !== InstanceStatus.RUNNING && instance.status !== InstanceStatus.STOPPED) {
+    throw new Error(`Cannot upgrade instance in ${instance.status} state`);
+  }
+
+  // Check for in-progress upgrades
+  const active = await prisma.instanceUpgrade.findFirst({
+    where: {
+      instanceId,
+      status: { in: [UpgradeStatus.PENDING, UpgradeStatus.IN_PROGRESS] },
+    },
+  });
+  if (active) {
+    throw new Error('An upgrade is already in progress for this instance');
+  }
+
+  // Get current commit for tracking
+  let currentCommit: string | null = null;
+  try {
+    const { stdout } = await exec('git rev-parse --short HEAD', {
+      cwd: instance.basePath,
+      timeout: 5_000,
+    });
+    currentCommit = stdout.trim();
+  } catch {
+    // Non-critical — may be a release install without .git
+  }
+
+  const branch = options?.branch || instance.gitBranch;
+
+  // Create upgrade record
+  const upgrade = await prisma.instanceUpgrade.create({
+    data: {
+      instanceId,
+      status: UpgradeStatus.PENDING,
+      previousCommit: currentCommit,
+      branch,
+      triggeredById: userId,
+    },
+  });
+
+  // Audit log
+  await prisma.auditLog.create({
+    data: {
+      userId,
+      instanceId,
+      action: AuditAction.INSTANCE_UPGRADE,
+      details: {
+        upgradeId: upgrade.id,
+        previousCommit: currentCommit,
+        branch,
+        options: options || {},
+      } as unknown as Prisma.InputJsonValue,
+      ipAddress,
+    },
+  });
+
+  // Fire-and-forget: run the upgrade asynchronously
+  runUpgrade(upgrade.id, instance.basePath, instance.slug, options).catch((err) => {
+    logger.error(`[upgrade] Upgrade orchestration failed for ${instance.slug}: ${err}`);
+  });
+
+  return upgrade;
+}
+
+/**
+ * Async upgrade runner. Runs upgrade.sh and polls progress.
+ */
+async function runUpgrade(
+  upgradeId: string,
+  basePath: string,
+  slug: string,
+  options?: StartUpgradeOptions
+) {
+  const progressFile = path.join(basePath, 'data', 'upgrade', 'progress.json');
+  const resultFile = path.join(basePath, 'data', 'upgrade', 'result.json');
+  const scriptPath = path.join(basePath, 'scripts', 'upgrade.sh');
+
+  // Ensure data/upgrade directory exists
+  await fs.mkdir(path.join(basePath, 'data', 'upgrade'), { recursive: true });
+
+  // Clean up any stale progress/result files from previous runs
+  await fs.rm(progressFile, { force: true });
+  await fs.rm(resultFile, { force: true });
+
+  // Mark as IN_PROGRESS
+  await prisma.instanceUpgrade.update({
+    where: { id: upgradeId },
+    data: {
+      status: UpgradeStatus.IN_PROGRESS,
+      progressMessage: 'Starting upgrade...',
+    },
+  });
+
+  // Build command flags
+  const flags: string[] = ['--api-mode', '--force'];
+  if (options?.skipBackup) flags.push('--skip-backup');
+  if (options?.useRegistry) flags.push('--use-registry');
+  if (options?.branch) flags.push('--branch', options.branch);
+
+  // Start progress polling
+  let pollTimer: NodeJS.Timeout | null = null;
+  pollTimer = setInterval(async () => {
+    try {
+      const raw = await fs.readFile(progressFile, 'utf-8');
+      const progress = JSON.parse(raw);
+      await prisma.instanceUpgrade.update({
+        where: { id: upgradeId },
+        data: {
+          currentPhase: progress.phase || 0,
+          phaseName: progress.phaseName || null,
+          percentage: progress.percentage || 0,
+          progressMessage: progress.message || null,
+        },
+      });
+    } catch {
+      // progress.json may not exist yet or be mid-write
+    }
+  }, PROGRESS_POLL_INTERVAL);
+
+  try {
+    // Run upgrade.sh
+    await exec(`bash "${scriptPath}" ${flags.join(' ')}`, {
+      cwd: basePath,
+      timeout: UPGRADE_TIMEOUT,
+      maxBuffer: 10 * 1024 * 1024,
+      env: { ...process.env, COMPOSE_ANSI: 'never' },
+    });
+
+    // Read result
+    const result = await readResultFile(resultFile);
+
+    // Read log tail
+    const logTail = await readLatestLogTail(basePath);
+
+    // Get new commit
+    let newCommit: string | null = null;
+    try {
+      const { stdout } = await exec('git rev-parse --short HEAD', { cwd: basePath, timeout: 5_000 });
+      newCommit = stdout.trim();
+    } catch { /* ignore */ }
+
+    // Update the upgrade record
+    await prisma.instanceUpgrade.update({
+      where: { id: upgradeId },
+      data: {
+        status: result.success ? UpgradeStatus.COMPLETED : UpgradeStatus.FAILED,
+        newCommit: result.newCommit || newCommit,
+        commitCount: result.commitCount || 0,
+        percentage: 100,
+        phaseName: 'Complete',
+        progressMessage: result.message || 'Upgrade completed',
+        durationSeconds: result.durationSeconds || null,
+        warnings: result.warnings?.length ? result.warnings : undefined,
+        errorMessage: result.success ? null : (result.message || 'Upgrade failed'),
+        log: logTail,
+        completedAt: new Date(),
+      },
+    });
+
+    // Update instance gitCommit
+    if (newCommit) {
+      await prisma.instance.update({
+        where: { id: (await prisma.instanceUpgrade.findUnique({ where: { id: upgradeId } }))!.instanceId },
+        data: { gitCommit: newCommit },
+      });
+    }
+
+    if (!result.success) {
+      // Create error event
+      const upgrade = await prisma.instanceUpgrade.findUnique({ where: { id: upgradeId } });
+      if (upgrade) {
+        await createEvent(
+          upgrade.instanceId,
+          'ERROR',
+          'upgrade',
+          'Upgrade failed',
+          result.message || 'The upgrade process failed. Check logs for details.',
+          { upgradeId, previousCommit: upgrade.previousCommit, warnings: result.warnings }
+        );
+      }
+    }
+
+    logger.info(`[upgrade] ${slug}: Upgrade ${result.success ? 'completed' : 'failed'}`);
+  } catch (err) {
+    const errorMsg = (err as Error).message;
+    const isTimeout = errorMsg.includes('timed out');
+
+    // Read whatever result/progress we have
+    const result = await readResultFile(resultFile);
+    const logTail = await readLatestLogTail(basePath);
+
+    await prisma.instanceUpgrade.update({
+      where: { id: upgradeId },
+      data: {
+        status: UpgradeStatus.FAILED,
+        errorMessage: isTimeout ? 'Upgrade timed out after 10 minutes' : errorMsg.slice(0, 2000),
+        progressMessage: 'Failed',
+        log: logTail,
+        completedAt: new Date(),
+        durationSeconds: result.durationSeconds || null,
+      },
+    });
+
+    // Create error event
+    const upgrade = await prisma.instanceUpgrade.findUnique({ where: { id: upgradeId } });
+    if (upgrade) {
+      await createEvent(
+        upgrade.instanceId,
+        'ERROR',
+        'upgrade',
+        isTimeout ? 'Upgrade timed out' : 'Upgrade failed',
+        isTimeout ? 'The upgrade process timed out after 10 minutes.' : errorMsg.slice(0, 500),
+        { upgradeId }
+      );
+
+      // Set instance to ERROR state
+      await prisma.instance.update({
+        where: { id: upgrade.instanceId },
+        data: {
+          status: InstanceStatus.ERROR,
+          statusMessage: `Upgrade failed: ${isTimeout ? 'timeout' : errorMsg.slice(0, 200)}`,
+        },
+      });
+    }
+
+    logger.error(`[upgrade] ${slug}: Upgrade failed: ${errorMsg}`);
+  } finally {
+    if (pollTimer) clearInterval(pollTimer);
+  }
+}
+
+// ─── File Readers ─────────────────────────────────────────────────
+
+interface UpgradeResult {
+  success: boolean;
+  message: string;
+  previousCommit?: string;
+  newCommit?: string;
+  commitCount?: number;
+  durationSeconds?: number;
+  warnings?: string[];
+}
+
+async function readResultFile(resultFile: string): Promise<UpgradeResult> {
+  try {
+    const raw = await fs.readFile(resultFile, 'utf-8');
+    return JSON.parse(raw);
+  } catch {
+    return { success: false, message: 'No result file found' };
+  }
+}
+
+async function readLatestLogTail(basePath: string): Promise<string | null> {
+  try {
+    const logDir = path.join(basePath, 'logs');
+    const files = await fs.readdir(logDir);
+    const upgradeLog = files
+      .filter((f) => f.startsWith('upgrade-'))
+      .sort()
+      .pop();
+    if (!upgradeLog) return null;
+
+    const content = await fs.readFile(path.join(logDir, upgradeLog), 'utf-8');
+    // Return last 5000 chars to keep DB storage reasonable
+    return content.slice(-5000);
+  } catch {
+    return null;
+  }
+}
+
+// ─── Query Functions ──────────────────────────────────────────────
+
+/**
+ * Get the current/latest upgrade progress for an instance.
+ */
+export async function getUpgradeStatus(instanceId: string) {
+  return prisma.instanceUpgrade.findFirst({
+    where: { instanceId },
+    orderBy: { startedAt: 'desc' },
+    include: {
+      triggeredBy: { select: { id: true, name: true, email: true } },
+    },
+  });
+}
+
+/**
+ * Get paginated upgrade history for an instance.
+ */
+export async function getUpgradeHistory(instanceId: string, page = 1, limit = 20) {
+  const [data, total] = await Promise.all([
+    prisma.instanceUpgrade.findMany({
+      where: { instanceId },
+      orderBy: { startedAt: 'desc' },
+      skip: (page - 1) * limit,
+      take: limit,
+      include: {
+        triggeredBy: { select: { id: true, name: true, email: true } },
+      },
+    }),
+    prisma.instanceUpgrade.count({ where: { instanceId } }),
+  ]);
+
+  return { data, total, page, limit };
+}