fix(approach-c): full E2E success on marcelle - byte-identical templates + core-only recreate

This session completed Approach C end-to-end on marcelle (status=COMPLETED,
mkdocs untouched, idempotent on re-run). Four fixes landed:

1. template-engine.ts: dropped nginx/conf.d/*.hbs (default, api, services)
   from renderAllTemplates AND renderAllTemplatesInMemory. The new
   prod-style docker-compose.yml.hbs does NOT mount conf.d/ into the
   nginx container ("Note: conf.d is NOT mounted (configs are generated
   at startup from templates)" — nginx confs are baked into the nginx
   Docker image). Writing them was a no-op orphan that showed up as 3
   "modified" lines in preview unnecessarily.
   Same reason removed nginx/nginx.conf from staticFiles.

2. templates/configs/{pangolin/resources.yml,prometheus/prometheus.yml,
   grafana/datasources/datasources.yml}.hbs: synced byte-identical to
   canonical changemaker.lite/configs/*. These ARE mounted into pangolin
   tunnel + prometheus + grafana respectively. Preview now reports
   "unchanged" for them on install.sh tenants.

3. templates/docker-compose.yml.hbs: dropped the CCP-tenant header
   comment, making the template now BYTE-IDENTICAL (58907 bytes) to
   canonical changemaker.lite/docker-compose.prod.yml. Even a 1-byte
   comment difference caused docker compose to compute new config hashes
   for every service, triggering full-stack recreates (including
   ccp-agent — the Phase 6 self-destruct trap from upgrade.sh).

4. upgrade.service.ts:runReleaseUpgrade — composeUp now restricted to
   core app services [api, admin, media-api, nginx] (same set as
   image-upgrade.sh). Unscoped composeUp would recreate ccp-agent
   mid-apply and orphan the runner. Until Approach C inherits the
   deferred-ccp-agent-restart pattern from upgrade.sh, this restriction
   keeps the apply path safe. Limitation: brand-new services in a
   release won't auto-deploy via Approach C alone — operator must
   follow with Approach A (full upgrade.sh) to pick them up.

E2E verification on marcelle:
  - Apply: status=COMPLETED, duration<10s.
  - mkdocs.yml md5 unchanged (38810d9df8b4258ad46a6739232cf88a).
  - mkdocs/docs file count unchanged (242).
  - docker-compose.yml now byte-identical to canonical (58907 bytes).
  - app + api public sites: 200 both.
  - Re-preview: ALL 10 files show "unchanged" — true idempotency.

Phase 6 acceptance gate met. Approach C now fully operational on the
install.sh fleet.

Bunker Admin

changemaker-control-panel/api/src/services/template-engine.ts

@@ -243,12 +243,15 @@ export async function renderAllTemplates(context: TemplateContext, outputDir: st
 
   const templatesDir = path.resolve(__dirname, '../..', 'templates');
 
+  // Templates that produce on-disk files actually consumed by tenant containers.
+  // nginx/conf.d/* templates removed 2026-05-23: the new (prod-style)
+  // docker-compose.yml does NOT mount conf.d/ into the nginx container
+  // ("Note: conf.d is NOT mounted (configs are generated at startup from
+  // templates)" — nginx confs are baked into the nginx Docker image).
+  // Writing them was a no-op orphan.
   const templateFiles = [
     { template: 'docker-compose.yml.hbs', output: 'docker-compose.yml' },
     { template: 'env.hbs', output: '.env' },
-    { template: 'nginx/conf.d/default.conf.hbs', output: 'nginx/conf.d/default.conf' },
-    { template: 'nginx/conf.d/api.conf.hbs', output: 'nginx/conf.d/api.conf' },
-    { template: 'nginx/conf.d/services.conf.hbs', output: 'nginx/conf.d/services.conf' },
     { template: 'configs/pangolin/resources.yml.hbs', output: 'configs/pangolin/resources.yml' },
     { template: 'configs/prometheus/prometheus.yml.hbs', output: 'configs/prometheus/prometheus.yml' },
     { template: 'configs/grafana/datasources/datasources.yml.hbs', output: 'configs/grafana/datasources/datasources.yml' },
@@ -272,9 +275,10 @@ export async function renderAllTemplates(context: TemplateContext, outputDir: st
     logger.debug(`Rendered ${template} → ${outputPath}`);
   }
 
-  // Copy static files (no templating needed)
+  // Copy static files (no templating needed). nginx/nginx.conf removed
+  // 2026-05-23 — same reason as nginx/conf.d/* in templateFiles above
+  // (nginx image bakes its own; on-disk file is an orphan).
   const staticFiles = [
-    'nginx/nginx.conf',
     'configs/prometheus/alerts.yml',
     'configs/alertmanager/alertmanager.yml',
     'configs/grafana/dashboards/dashboards.yml',
@@ -311,12 +315,15 @@ export async function renderAllTemplatesInMemory(
   const templatesDir = path.resolve(__dirname, '../..', 'templates');
   const result: Array<{ relativePath: string; content: string }> = [];
 
+  // Templates that produce on-disk files actually consumed by tenant containers.
+  // nginx/conf.d/* templates removed 2026-05-23: the new (prod-style)
+  // docker-compose.yml does NOT mount conf.d/ into the nginx container
+  // ("Note: conf.d is NOT mounted (configs are generated at startup from
+  // templates)" — nginx confs are baked into the nginx Docker image).
+  // Writing them was a no-op orphan.
   const templateFiles = [
     { template: 'docker-compose.yml.hbs', output: 'docker-compose.yml' },
     { template: 'env.hbs', output: '.env' },
-    { template: 'nginx/conf.d/default.conf.hbs', output: 'nginx/conf.d/default.conf' },
-    { template: 'nginx/conf.d/api.conf.hbs', output: 'nginx/conf.d/api.conf' },
-    { template: 'nginx/conf.d/services.conf.hbs', output: 'nginx/conf.d/services.conf' },
     { template: 'configs/pangolin/resources.yml.hbs', output: 'configs/pangolin/resources.yml' },
     { template: 'configs/prometheus/prometheus.yml.hbs', output: 'configs/prometheus/prometheus.yml' },
     { template: 'configs/grafana/datasources/datasources.yml.hbs', output: 'configs/grafana/datasources/datasources.yml' },
@@ -334,9 +341,9 @@ export async function renderAllTemplatesInMemory(
     result.push({ relativePath: output, content: rendered });
   }
 
-  // Read static files into memory
+  // Read static files into memory. nginx/nginx.conf removed 2026-05-23 —
+  // same orphan reason as in renderAllTemplates above.
   const staticFiles = [
-    'nginx/nginx.conf',
     'configs/prometheus/alerts.yml',
     'configs/alertmanager/alertmanager.yml',
     'configs/grafana/dashboards/dashboards.yml',

changemaker-control-panel/api/src/services/upgrade.service.ts

@@ -651,14 +651,24 @@ async function runReleaseUpgrade(
     });
     await driver.composePull(instance.basePath, instance.composeProject);
 
-    // Phase 4: recreate services
+    // Phase 4: recreate services. Restricted to core app services (api,
+    // admin, media-api, nginx) — same set as scripts/image-upgrade.sh.
+    // Calling unscoped composeUp would recreate ccp-agent too, severing the
+    // CCP↔agent connection mid-apply and orphaning the script (same trap as
+    // upgrade.sh Phase 6 self-destruct fixed in v2.10.2). Until we mirror
+    // upgrade.sh's deferred-ccp-agent-restart pattern here, stick to the
+    // explicit service list. Limitation: Approach C will not pick up
+    // brand-new services in a release until either (a) operator runs full
+    // upgrade.sh (Approach A) afterwards, or (b) this is upgraded to the
+    // deferred-restart pattern.
+    const coreServices = ['api', 'admin', 'media-api', 'nginx'];
     await updateStatus({
       currentPhase: 4,
       phaseName: 'Recreate Services',
       percentage: 80,
-      progressMessage: 'Recreating services with new orchestration...',
+      progressMessage: `Recreating core services (${coreServices.join(', ')})...`,
     });
-    await driver.composeUp(instance.basePath, instance.composeProject);
+    await driver.composeUp(instance.basePath, instance.composeProject, coreServices);
 
     // Phase 5: verify (best-effort; soft warnings only)
     await updateStatus({

changemaker-control-panel/templates/configs/grafana/datasources/datasources.yml.hbs

@@ -4,7 +4,7 @@ datasources:
   - name: Prometheus
     type: prometheus
     access: proxy
-    url: http://{{containerPrefix}}-prometheus:9090
+    url: http://prometheus-changemaker:9090
     isDefault: true
     editable: true
     jsonData:

changemaker-control-panel/templates/configs/pangolin/resources.yml.hbs

@@ -1,116 +1,155 @@
-# Pangolin Resources — Instance: {{name}}
-# All resources route through nginx for consistent subdomain handling
+# Pangolin Resource Definitions
+# All resources route through Nginx (port 80) by default
+# Newt tunnel → Nginx (port 80) → Backend containers (various ports)
+#
+# target_ip: the hostname/IP that Newt sends traffic to (default: nginx)
+# target_port: the port on the target host (default: 80)
 
 resources:
-  # ─── Always-On Resources ──────────────────────────────────
+  # Required services (fail if down)
+  - subdomain: app
+    name: Admin GUI
+    container: changemaker-v2-admin
+    port: 3000
+    target_ip: nginx
+    target_port: 80
+    required: true
 
-  - name: app
-    subdomain: app
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: api
+    name: API Server
+    container: changemaker-v2-api
+    port: 4000
+    target_ip: nginx
+    target_port: 80
+    required: true
 
-  - name: api
-    subdomain: api
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: ""  # Root domain
+    name: Public Site
+    container: mkdocs-site-server-changemaker
+    port: 80
+    target_ip: nginx
+    target_port: 80
+    required: true
 
-  - name: root
-    subdomain: ""
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: true
+  # Optional services (warn and skip if down)
+  - subdomain: db
+    name: NocoDB
+    container: changemaker-v2-nocodb
+    port: 8080
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: docs
-    subdomain: docs
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: docs
+    name: Documentation
+    container: mkdocs-changemaker
+    port: 8000
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: db
-    subdomain: db
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: code
+    name: Code Server
+    container: code-server-changemaker
+    port: 8080
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: mail
-    subdomain: mail
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: n8n
+    name: Workflows
+    container: n8n-changemaker
+    port: 5678
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: qr
-    subdomain: qr
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: git
+    name: Gitea
+    container: gitea-changemaker
+    port: 3000
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  # ─── Conditional Resources ────────────────────────────────
+  - subdomain: home
+    name: Homepage
+    container: homepage-changemaker
+    port: 3000
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableMedia}}
-  - name: media
-    subdomain: media
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: listmonk
+    name: Newsletter
+    container: listmonk-app
+    port: 9000
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableListmonk}}
-  - name: listmonk
-    subdomain: listmonk
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: qr
+    name: Mini QR
+    container: mini-qr
+    port: 8080
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableGancio}}
-  - name: events
-    subdomain: events
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: draw
+    name: Excalidraw
+    container: excalidraw-changemaker
+    port: 80
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableChat}}
-  - name: chat
-    subdomain: chat
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: vault
+    name: Vaultwarden
+    container: vaultwarden-changemaker
+    port: 80
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableMeet}}
-  - name: meet
-    subdomain: meet
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: mail
+    name: MailHog
+    container: mailhog-changemaker
+    port: 8025
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableMonitoring}}
-  - name: grafana
-    subdomain: grafana
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  - subdomain: chat
+    name: Rocket.Chat
+    container: rocketchat-changemaker
+    port: 3000
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-{{#if enableDevTools}}
-  - name: code
-    subdomain: code
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: events
+    name: Gancio Events
+    container: gancio-changemaker
+    port: 13120
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: git
-    subdomain: git
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
+  - subdomain: meet
+    name: Jitsi Meet
+    container: jitsi-web-changemaker
+    port: 80
+    target_ip: nginx
+    target_port: 80
+    required: false
 
-  - name: n8n
-    subdomain: n8n
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-
-  - name: home
-    subdomain: home
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-
-  - name: vault
-    subdomain: vault
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-
-  - name: draw
-    subdomain: draw
-    target: http://{{containerPrefix}}-nginx:80
-    isBaseDomain: false
-{{/if}}
+  # Monitoring services (auto-detect profile)
+  - subdomain: grafana
+    name: Grafana
+    container: grafana-changemaker
+    port: 3000
+    target_ip: nginx
+    target_port: 80
+    required: false
+    profile: monitoring  # Auto-detect if monitoring profile active

changemaker-control-panel/templates/configs/prometheus/prometheus.yml.hbs

@@ -1,66 +1,61 @@
-# Prometheus — Instance: {{name}}
-
 global:
   scrape_interval: 15s
   evaluation_interval: 15s
   external_labels:
-    monitor: '{{composeProject}}'
+    monitor: 'changemaker-lite'
 
-{{#if enableMonitoring}}
+# Alertmanager configuration
 alerting:
   alertmanagers:
     - static_configs:
-        - targets: ['{{containerPrefix}}-alertmanager:9093']
+        - targets: ['alertmanager:9093']
 
+# Load rules once and periodically evaluate them
 rule_files:
   - "alerts.yml"
-{{/if}}
 
+# Scrape configurations
 scrape_configs:
-  - job_name: '{{composeProject}}-api'
+  # V2 Unified API Metrics
+  - job_name: 'changemaker-v2-api'
     static_configs:
-      - targets: ['{{containerPrefix}}-api:4000']
-    metrics_path: '/api/metrics'
+      - targets: ['changemaker-v2-api:4000']
+    metrics_path: '/api/metrics/internal'
     scrape_interval: 10s
     scrape_timeout: 5s
 
-{{#if enableMedia}}
-  - job_name: '{{composeProject}}-media-api'
+  # N8N Metrics (if available)
+  - job_name: 'n8n'
     static_configs:
-      - targets: ['{{containerPrefix}}-media-api:4100']
-    metrics_path: '/api/metrics'
-{{/if}}
+      - targets: ['n8n-changemaker:5678']
+    metrics_path: '/metrics'
+    scrape_interval: 30s
 
-  - job_name: '{{composeProject}}-redis'
+  # Redis Metrics
+  - job_name: 'redis'
     static_configs:
-      - targets: ['{{containerPrefix}}-redis-exporter:9121']
+      - targets: ['redis-exporter:9121']
     scrape_interval: 15s
 
-{{#if enableMonitoring}}
-  - job_name: '{{composeProject}}-cadvisor'
+  # cAdvisor - Docker container metrics
+  - job_name: 'cadvisor'
     static_configs:
-      - targets: ['{{containerPrefix}}-cadvisor:8080']
+      - targets: ['cadvisor:8080']
     scrape_interval: 15s
 
-  - job_name: '{{composeProject}}-node'
+  # Node Exporter - System metrics
+  - job_name: 'node'
     static_configs:
-      - targets: ['{{containerPrefix}}-node-exporter:9100']
+      - targets: ['node-exporter:9100']
     scrape_interval: 15s
 
-  - job_name: '{{composeProject}}-prometheus'
+  # Prometheus self-monitoring
+  - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
 
-  - job_name: '{{composeProject}}-alertmanager'
+  # Alertmanager monitoring
+  - job_name: 'alertmanager'
     static_configs:
-      - targets: ['{{containerPrefix}}-alertmanager:9093']
+      - targets: ['alertmanager:9093']
     scrape_interval: 30s
-{{/if}}
-
-{{#if enableDevTools}}
-  - job_name: '{{composeProject}}-n8n'
-    static_configs:
-      - targets: ['{{containerPrefix}}-n8n:5678']
-    metrics_path: '/metrics'
-    scrape_interval: 30s
-{{/if}}

changemaker-control-panel/templates/docker-compose.yml.hbs

@@ -1,24 +1,8 @@
 ###############################################################################
 ###############################################################################
-# Changemaker Lite v2 — Tenant compose (CCP template)
-# Instance: {{name}} ({{slug}})
-# Compose project: {{composeProject}}
-#
-# This template is a byte-mirror of changemaker.lite/docker-compose.prod.yml
-# (modulo this header comment). Approach C (CCP-driven release upgrade)
-# renders this against the tenant's context and writes the result to the
-# tenant's filesystem.
-#
-# All per-instance variation flows through env-var substitution from the
-# tenant's .env file (rendered by env.hbs for CCP-provisioned tenants;
-# kept as-is for install.sh-registered tenants). The CCP controls image
-# tag selection by writing IMAGE_TAG to the tenant's .env, which the
-# compose's ${IMAGE_TAG:-latest} substitution then picks up at compose-up.
-#
-# To keep this template in sync with canonical docker-compose.prod.yml:
-#   - When a new service is added to changemaker.lite/docker-compose.prod.yml,
-#     copy the same block here verbatim. Handlebars is NOT used in the
-#     compose template itself — all variation is env-var driven.
+# Changemaker Lite v2 — Production Docker Compose
+# Pre-built images only. No source code mounts, no build blocks.
+# Generated from docker-compose.yml by build-release.sh
 ###############################################################################
 ###############################################################################