name: Hetzner Deploy (gluecron.com)

# Default-deny: scope GITHUB_TOKEN to read-only. The deploy uses an SSH key
# (HETZNER_SSH_KEY) to push to the box, not the GitHub token, so read-only
# is sufficient. Individual jobs may escalate if they need write access.
permissions: read-all

# Triggered on every push to main and on manual dispatch.
# Steps:
#   1. Capture the current SHA on the box (for rollback)
#   2. SSH in, git pull, run deploy-crontech.sh (which restarts systemd + reloads Caddy)
#   3. Smoke-test https://gluecron.com/healthz with retries
#   4. On smoke failure: roll back to the previous SHA and restart
#   5. On any failure: have Claude read the last 100 journal lines and post a one-paragraph
#      root-cause analysis to the workflow summary (and optionally a webhook)
#
# Secrets required:
#   HETZNER_HOST      — public IP/hostname of the box (e.g. 178.104.208.252)
#   HETZNER_USER      — ssh user (e.g. root)
#   HETZNER_SSH_KEY   — private deploy key (PEM/OpenSSH format)
#
# Optional:
#   ANTHROPIC_API_KEY      — enables AI failure-diagnosis step
#   DEPLOY_WEBHOOK_URL     — POSTed with JSON deploy status (Slack/Discord/anything)
#   DEPLOY_EVENT_TOKEN     — bearer used to POST deploy timeline events to
#                            ${APP_BASE_URL}/api/events/deploy/{started,finished}.
#                            Block N3: this makes the live site display ITS OWN
#                            deploy in the admin nav + at /admin/deploys.
#   APP_BASE_URL           — base URL of the running site (default https://gluecron.com).

on:
  push:
    branches: [main]
  workflow_dispatch: {}

concurrency:
  group: hetzner-deploy
  cancel-in-progress: false

jobs:
  deploy:
    name: Deploy gluecron.com
    runs-on: ubuntu-latest
    timeout-minutes: 12
    steps:
      - uses: actions/checkout@v4

      # ─── 0. Mark start time + notify the live site (Block N3) ────────────
      # The site has its own admin status pill + /admin/deploys timeline.
      # POSTing here makes the pill say "Deploying… 14s" the moment SSH
      # begins. `--fail` is intentionally NOT set: a 5xx here must never
      # block the deploy itself.
      - name: Record start time
        id: start
        run: |
          echo "epoch=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify deploy started
        if: env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
        run: |
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/started" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"sha\":\"${{ github.sha }}\",\"run_id\":\"${{ github.run_id }}\",\"source\":\"hetzner-deploy\"}" \
            || echo "(deploy-started notify failed — continuing)"

      # ─── 1. Capture pre-deploy SHA so we can rollback ───────────────────
      # R2 — bracket every major step with notify-deploy-step calls so the
      # /admin/deploys modal can stream the workflow live. The composite
      # action is fire-and-forget — a 5xx never blocks the deploy.
      - name: Notify step — setup (in_progress)
        if: env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: in_progress
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Capture pre-deploy SHA
        id: prev
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            cd /opt/gluecron
            sha=$(git rev-parse HEAD)
            echo "Previous SHA: $sha"
            echo "$sha" > /tmp/gluecron_prev_sha
            cat /tmp/gluecron_prev_sha

      - name: Notify step — setup (succeeded)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: succeeded
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Notify step — setup (failed)
        if: failure() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: failed
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      # ─── 2. Deploy: pull main, install/compile/restart ─────────────────
      # Block N2 — speed optimisation:
      #   (a) Cache `bun install` by hashing bun.lock; skip the walk when
      #       the lockfile is unchanged. Saves ~5-15s per "no deps changed"
      #       deploy.
      #   (b) Compile to a single Bun static binary at
      #       /opt/gluecron/.next/gluecron-server. Boot drops from ~500ms
      #       (cold ESM resolve) to <50ms. Compile itself takes ~3-5s.
      #   (c) Rewrite the systemd unit on first run if it lacks Type=notify
      #       (idempotent — diff-then-write, daemon-reload only on change).
      #       Falls back to `bun run src/index.ts` if the compiled binary is
      #       missing for any reason — the deploy MUST stay backward-
      #       compatible.
      #   (d) `systemctl restart` blocks until sd_notify(READY=1) fires
      #       (wired in src/lib/systemd-notify.ts), so we no longer rely on
      #       the curl-with-retries loop downstream.
      - name: Deploy
        id: deploy
        uses: appleboy/ssh-action@v1.2.0
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          GH_RUN_ID: ${{ github.run_id }}
          GH_SHA: ${{ github.sha }}
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          command_timeout: 8m
          script_stop: true
          # R2: stream per-phase progress to /api/events/deploy/step. We
          # share one SSH session across all phases (git pull → install →
          # build → migrate → restart) so the in-script curl posts give us
          # the live timeline a black-box step boundary can't.
          envs: DEPLOY_EVENT_TOKEN,APP_BASE_URL,GH_RUN_ID,GH_SHA
          script: |
            set -euo pipefail
            cd /opt/gluecron

            # R2 helper: POST a single step event (in_progress|succeeded|failed).
            # Never fails — observability must not break deploys.
            notify_step() {
              local NAME="$1" STATUS="$2" DUR="${3:-}"
              if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] || [ -z "${APP_BASE_URL:-}" ]; then
                return 0
              fi
              local DUR_FIELD=""
              if [ -n "$DUR" ]; then
                DUR_FIELD=",\"duration_ms\":$DUR"
              fi
              curl --silent --show-error --max-time 5 \
                -X POST "$APP_BASE_URL/api/events/deploy/step" \
                -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
                -H "content-type: application/json" \
                --data "{\"run_id\":\"$GH_RUN_ID\",\"sha\":\"$GH_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
                >/dev/null 2>&1 || true
            }

            notify_step "git-pull" "in_progress"
            GP_START=$(date +%s)
            git fetch --prune origin main
            git reset --hard origin/main
            new_sha=$(git rev-parse HEAD)
            echo "Deploying SHA: $new_sha"
            notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"

            BUN=/root/.bun/bin/bun
            CACHE_DIR=/opt/gluecron/.cache
            HASH_FILE=$CACHE_DIR/bun-lockfile-hash
            mkdir -p "$CACHE_DIR"

            # ─── (a) Cached deps: skip bun install when bun.lock is unchanged
            notify_step "bun-install" "in_progress"
            BI_START=$(date +%s)
            if [ -f bun.lock ]; then
              new_hash=$(sha256sum bun.lock | awk '{print $1}')
            else
              new_hash="no-lockfile"
            fi
            old_hash=""
            if [ -f "$HASH_FILE" ]; then
              old_hash=$(cat "$HASH_FILE")
            fi
            if [ "$new_hash" = "$old_hash" ] && [ -d node_modules ]; then
              echo "==> bun install: SKIP (lockfile unchanged: $new_hash)"
            else
              echo "==> bun install: hash changed ($old_hash -> $new_hash) — installing"
              "$BUN" install --frozen-lockfile
              echo "$new_hash" > "$HASH_FILE"
            fi
            notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"

            # ─── (b) Compile to a single static binary (best-effort)
            notify_step "build" "in_progress"
            BD_START=$(date +%s)
            mkdir -p .next
            COMPILED=.next/gluecron-server
            COMPILED_TMP=.next/gluecron-server.new
            if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts; then
              mv -f "$COMPILED_TMP" "$COMPILED"
              chmod +x "$COMPILED"
              EXEC_START="/opt/gluecron/.next/gluecron-server"
              echo "==> compiled binary ready: $COMPILED"
              notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
            else
              # Backward-compat: if compile fails, fall back to interpreted Bun
              echo "WARN: bun build --compile failed — falling back to bun run"
              rm -f "$COMPILED_TMP"
              EXEC_START="$BUN run src/index.ts"
              # Treat compile-failure-with-fallback as 'succeeded' for the
              # modal — the deploy itself is still going.
              notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
            fi

            # ─── (c) Idempotent systemd unit rewrite (Type=notify)
            UNIT=/etc/systemd/system/gluecron.service
            DESIRED=$(cat <<UNIT_EOF
            [Unit]
            Description=Gluecron — AI-native code intelligence platform
            After=network-online.target postgresql.service
            Wants=network-online.target

            [Service]
            Type=notify
            NotifyAccess=main
            User=root
            WorkingDirectory=/opt/gluecron
            EnvironmentFile=/etc/gluecron.env
            ExecStart=$EXEC_START
            Restart=always
            RestartSec=5
            TimeoutStartSec=30
            StandardOutput=journal
            StandardError=journal
            SyslogIdentifier=gluecron
            LimitNOFILE=65536

            [Install]
            WantedBy=multi-user.target
            UNIT_EOF
            )
            # Strip the leading indentation from the heredoc (`sed 's/^            //'`)
            # so the rendered unit is column-0 like systemd expects.
            DESIRED=$(printf '%s\n' "$DESIRED" | sed 's/^            //')

            need_rewrite=1
            if [ -f "$UNIT" ] && diff -q <(printf '%s\n' "$DESIRED") "$UNIT" >/dev/null 2>&1; then
              need_rewrite=0
            fi

            if [ "$need_rewrite" = "1" ]; then
              echo "==> rewriting $UNIT (Type=notify, ExecStart=$EXEC_START)"
              printf '%s\n' "$DESIRED" > "$UNIT"
              systemctl daemon-reload
            else
              echo "==> $UNIT already matches desired state — skipping daemon-reload"
            fi

            # ─── DB migrations
            #
            # Block S1 (2026-05-14): migrate.ts MUST be allowed to fail
            # the entire workflow. The previous `|| echo WARN` swallowed
            # the exit code, which is exactly how migrations 0046-0053
            # silently skipped a real deploy and left the live site
            # crashing every request that touched `users.*`. We now:
            #   - run migrate.ts and abort the deploy on non-zero exit
            #   - stream stdout + stderr live to the workflow log
            #   - then read back the applied list from _migrations and
            #     refuse to restart if the latest drizzle/*.sql file
            #     isn't in it.
            notify_step "db-migrate" "in_progress"
            DM_START=$(date +%s)
            set -a; source /etc/gluecron.env; set +a

            echo "==> running migrations (must succeed to proceed)"
            if ! "$BUN" run src/db/migrate.ts; then
              notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
              echo "ERROR: bun run db:migrate failed — aborting deploy" >&2
              exit 1
            fi

            # Verify the LATEST drizzle/*.sql is present in _migrations.
            # This catches the case where migrate.ts thought every file
            # was already applied but the actual list on disk has new
            # entries the runner somehow skipped. We write a tiny ESM
            # verifier into /tmp and call it; this avoids the require()
            # vs ESM mismatch you'd hit using `bun -e` on a "type":"module"
            # package.
            echo "==> verifying latest drizzle/*.sql is recorded in _migrations"
            LATEST_FILE=$(ls drizzle/*.sql 2>/dev/null | sort | tail -1 | xargs -n1 basename || true)
            if [ -z "$LATEST_FILE" ]; then
              echo "WARN: no drizzle/*.sql files found — skipping verification"
            else
              cat > /tmp/verify-migration.mjs <<'VERIFY_EOF'
              import { neon } from "@neondatabase/serverless";
              import postgres from "postgres";
              const url = process.env.DATABASE_URL;
              const target = process.argv[2];
              if (!url || !target) { console.error("verify: missing DATABASE_URL or target"); process.exit(1); }
              const isNeon = /(^|\.)neon\.tech$/i.test(new URL(url).hostname);
              try {
                let rows;
                if (isNeon) {
                  const sql = neon(url);
                  rows = await sql(`SELECT name FROM _migrations WHERE name = $1`, [target]);
                } else {
                  const client = postgres(url, { max: 1, prepare: false });
                  rows = await client`SELECT name FROM _migrations WHERE name = ${target}`;
                  await client.end({ timeout: 5 });
                }
                if (rows.length === 0) {
                  console.error(`verify: ${target} is NOT in _migrations`);
                  process.exit(2);
                }
                console.log(`verify: ${target} is applied`);
              } catch (e) {
                console.error("verify: query failed:", e?.message ?? e);
                process.exit(3);
              }
              VERIFY_EOF
              if ! "$BUN" run /tmp/verify-migration.mjs "$LATEST_FILE"; then
                echo "ERROR: latest migration $LATEST_FILE is NOT recorded in _migrations — aborting deploy" >&2
                notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
                exit 1
              fi
            fi

            notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"

            # ─── (c.5) Pin BUILD_SHA into systemd before restart.
            # Without this, src/routes/pwa.ts falls back to a stable
            # "dev-stable" version string and the browser service worker
            # never invalidates between real deploys. The drop-in survives
            # daemon-reload; old contents are overwritten on every deploy.
            mkdir -p /etc/systemd/system/gluecron.service.d
            cat > /etc/systemd/system/gluecron.service.d/build-sha.conf <<EOF
            [Service]
            Environment="BUILD_SHA=$new_sha"
            Environment="BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
            EOF
            systemctl daemon-reload
            echo "==> pinned BUILD_SHA=${new_sha:0:12} in systemd drop-in"

            # ─── (d) Zero-downtime restart. Blocks until sd_notify(READY=1).
            notify_step "restart-service" "in_progress"
            RS_START=$(date +%s)
            echo "==> systemctl restart gluecron (blocks on sd_notify READY=1)"
            systemctl restart gluecron
            echo "==> restart returned — gluecron signalled ready"
            notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"

      # ─── 3. Smoke-test the deployed app on the box ──────────────────────
      # We SSH back in and curl localhost:3010/healthz directly. This tests
      # the EXACT instance we just deployed, independent of:
      #   - DNS state (gluecron.com may still point at an old box during a
      #     migration)
      #   - Caddy TLS state (cert may not be issued yet for a new domain)
      #   - external network reachability from GH runners
      # If you ALSO want a public-DNS smoke check, add a second step that
      # hits https://gluecron.com after this one succeeds.
      - name: Notify step — smoke-test (in_progress)
        if: env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: in_progress
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Smoke test (localhost on the box)
        id: smoke
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            # Block N2 — `systemctl restart` already blocked on
            # sd_notify(READY=1), so the FIRST curl should succeed. We keep a
            # short retry budget for paranoia: a brief delay between
            # systemd's READY ack and the HTTP listener becoming routable
            # via 127.0.0.1 is theoretically possible (unusual but cheap to
            # tolerate). 3 attempts × 2s = 6s ceiling instead of 8 × 6s = 48s.
            set +e
            for i in 1 2 3; do
              code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
              echo "Attempt $i: /healthz -> $code"
              if [ "$code" = "200" ]; then
                echo "OK: gluecron is healthy on localhost:3010"
                curl -s http://localhost:3010/api/version || true
                exit 0
              fi
              sleep 2
            done
            echo "FAIL: /healthz did not return 200 after 6s"
            systemctl status gluecron --no-pager | head -10 || true
            journalctl -u gluecron -n 30 --no-pager || true
            exit 1

      - name: Notify step — smoke-test (succeeded)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: succeeded
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Notify step — smoke-test (failed)
        if: failure() && steps.smoke.conclusion == 'failure' && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: failed
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      # ─── 3b. Full post-deploy smoke suite (Block S1+S3) ──────────────────
      # `/healthz` alone is NOT enough — it doesn't touch the DB schema, so
      # a broken migration leaves it green while every real page crashes
      # selecting columns that don't exist. The post-deploy-smoke script
      # hits 15 critical endpoints (login renders, /api/version, /demo,
      # /mcp, /sw.js, etc.) and verifies the LATEST drizzle/*.sql is in
      # the running process's reported migrations list. If ANY check
      # fails, the workflow auto-rolls back.
      - name: Notify step — full-smoke (in_progress)
        if: env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: full-smoke
          status: in_progress
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Full post-deploy smoke suite
        id: full_smoke
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            set -e
            cd /opt/gluecron
            BUN=/root/.bun/bin/bun
            export GLUECRON_HOST="http://localhost:3010"
            echo "==> running 15-endpoint smoke suite against $GLUECRON_HOST"
            # The script exits 1 on endpoint failure, 2 on missing
            # migration. We treat both as fatal so the workflow rolls
            # back. stdout/stderr stream live to the GH Actions log.
            "$BUN" run scripts/post-deploy-smoke.ts

      - name: Notify step — full-smoke (succeeded)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: full-smoke
          status: succeeded
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Notify step — full-smoke (failed)
        if: failure() && steps.full_smoke.conclusion == 'failure' && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: full-smoke
          status: failed
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      # ─── 4. Auto-rollback on smoke failure ──────────────────────────────
      # Only rolls back if the workflow was triggered by a normal push.
      # Manual workflow_dispatch runs SKIP rollback so the operator can
      # diagnose the new code on the box before reverting. This stops the
      # pathological case where rollback masks the real failure by reverting
      # to an already-broken previous SHA.
      #
      # S1 (2026-05-14): rollback now also fires when the FULL smoke suite
      # fails (steps.full_smoke), not just the basic /healthz curl. Recursion
      # cap: only ONE rollback attempt per workflow run (the `if:` guard
      # naturally enforces this — the only rollback step in the file).
      - name: Rollback on failure
        id: rollback
        if: failure() && github.event_name == 'push' && (steps.smoke.conclusion == 'failure' || steps.full_smoke.conclusion == 'failure')
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            cd /opt/gluecron
            prev=$(cat /tmp/gluecron_prev_sha 2>/dev/null || true)
            if [ -z "$prev" ]; then
              echo "ROLLBACK SKIPPED: no /tmp/gluecron_prev_sha — human intervention required"
              exit 1
            fi
            echo "ROLLED BACK to $prev because post-deploy smoke failed"
            git reset --hard "$prev"
            # Don't re-run migrations here: rolling back schema is
            # destructive and migrations are forward-only. We just put
            # the code back to where it was and restart. If a migration
            # is the reason the new code is incompatible with the old,
            # the operator must intervene manually.
            BUN=/root/.bun/bin/bun
            if [ -f bun.lock ] && [ -d node_modules ]; then
              echo "==> reusing existing node_modules (lockfile hash check skipped during rollback)"
            else
              "$BUN" install --frozen-lockfile || true
            fi
            systemctl restart gluecron
            # Verify the rollback target itself comes up green.
            sleep 3
            for i in 1 2 3; do
              code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
              echo "Rollback healthz attempt $i: $code"
              if [ "$code" = "200" ]; then
                echo "OK: rolled-back instance is healthy"
                exit 0
              fi
              sleep 2
            done
            echo "WARN: rollback target ALSO failed /healthz — human intervention required"
            systemctl status gluecron --no-pager | head -20 || true
            exit 1

      # ─── 5. Failure diagnostics — captured into a file for summary + AI ──
      - name: Capture failure context
        if: failure()
        id: ctx
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            echo "===== systemd status ====="
            systemctl status gluecron --no-pager 2>&1 | head -40 || true
            echo ""
            echo "===== last 80 journal lines (gluecron) ====="
            journalctl -u gluecron -n 80 --no-pager --output=cat 2>&1 || true
            echo ""
            echo "===== caddy validate ====="
            caddy validate --config /etc/caddy/Caddyfile 2>&1 | head -20 || true
            echo ""
            echo "===== /healthz from inside box ====="
            curl -s -w "\nHTTP %{http_code}\n" http://localhost:3000/healthz 2>&1 || true
            echo ""
            echo "===== port 3000 listener ====="
            ss -tlnp 2>&1 | grep ':3000' || echo '(nothing listening on :3000)'

      # Always post the captured diagnostics to the workflow summary so the
      # owner can read what broke without SSH'ing or grepping log files.
      - name: Post diagnostics to summary
        if: failure() && steps.ctx.outputs.stdout != ''
        env:
          DIAG: ${{ steps.ctx.outputs.stdout }}
        run: |
          {
            echo "## ❌ Deploy failed — diagnostics"
            echo ""
            echo "**Commit:** \`${GITHUB_SHA:0:7}\` — ${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-${GITHUB_SHA:0:7}}"
            echo "**Run:** [#${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
            echo ""
            echo '```'
            echo "$DIAG"
            echo '```'
          } >> $GITHUB_STEP_SUMMARY

      - name: AI root-cause analysis (Claude)
        if: failure() && env.ANTHROPIC_API_KEY != ''
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          DEPLOY_LOGS: ${{ steps.ctx.outputs.stdout }}
          COMMIT_SHA: ${{ github.sha }}
          COMMIT_MSG: ${{ github.event.head_commit.message }}
        run: |
          set +e
          # Build the prompt. Use python3 to JSON-encode every untrusted input
          # (commit message, deploy logs, SHA) so backticks, $(…), quotes, or
          # newlines in a commit message cannot break out of the JSON string
          # or execute on the runner. The previous unquoted heredoc allowed
          # arbitrary command execution via crafted commit messages.
          python3 - <<'PY' > /tmp/prompt.json
          import json, os
          payload = {
              "model": "claude-haiku-4-5-20251001",
              "max_tokens": 600,
              "system": "You are a senior SRE diagnosing a failed deploy. Read the systemd status, journal, and curl output. In 1 short paragraph (under 100 words), identify the most likely root cause and the single fastest fix. Be direct, no preamble.",
              "messages": [{
                  "role": "user",
                  "content": "Commit: " + os.environ.get("COMMIT_SHA", "") +
                             "\nMessage: " + os.environ.get("COMMIT_MSG", "") +
                             "\n\nDeploy logs:\n" + os.environ.get("DEPLOY_LOGS", ""),
              }],
          }
          print(json.dumps(payload))
          PY
          response=$(curl -s https://api.anthropic.com/v1/messages \
            -H "x-api-key: $ANTHROPIC_API_KEY" \
            -H "anthropic-version: 2023-06-01" \
            -H "content-type: application/json" \
            --data @/tmp/prompt.json)
          analysis=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('content',[{}])[0].get('text','(no analysis)'))" 2>/dev/null)
          echo "## 🤖 AI Failure Analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "$analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Commit:** \`${COMMIT_SHA:0:7}\` — $COMMIT_MSG" >> $GITHUB_STEP_SUMMARY

      # ─── 6. Optional webhook notification ───────────────────────────────
      - name: Notify webhook
        if: always() && env.DEPLOY_WEBHOOK_URL != ''
        env:
          DEPLOY_WEBHOOK_URL: ${{ secrets.DEPLOY_WEBHOOK_URL }}
          STATUS: ${{ job.status }}
        run: |
          curl -s -X POST "$DEPLOY_WEBHOOK_URL" \
            -H "content-type: application/json" \
            --data "{\"status\":\"$STATUS\",\"target\":\"gluecron.com\",\"sha\":\"${{ github.sha }}\",\"run\":\"${{ github.run_id }}\"}" || true

      # ─── 7. Workflow summary on success ─────────────────────────────────
      - name: Success summary
        if: success()
        run: |
          echo "## ✅ Deploy succeeded" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Target:** https://gluecron.com" >> $GITHUB_STEP_SUMMARY
          echo "- **SHA:** \`${GITHUB_SHA:0:7}\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Health:** /healthz → 200" >> $GITHUB_STEP_SUMMARY

      # ─── 8. Block N3 — POST deploy-finished event to the live site ───────
      # The site's admin status pill flips to "Deployed Ns ago" or
      # "Deploy failed Nm ago" the instant this lands. `if: always()` so we
      # always report final state (including failure); the inner `if:` flag
      # splits success vs failure for the payload body. We never `--fail` —
      # a 5xx must not retroactively break a deploy that actually succeeded.
      - name: Notify deploy finished (success)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
            || echo "(deploy-finished[succeeded] notify failed — continuing)"

      - name: Notify deploy finished (failure)
        if: failure() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
          DIAG: ${{ steps.ctx.outputs.stdout }}
          ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }}
          SMOKE_OUTCOME: ${{ steps.smoke.conclusion }}
          FULL_SMOKE_OUTCOME: ${{ steps.full_smoke.conclusion }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          # Build a short reason header. S1 (2026-05-14): the deploy-
          # finished payload now records WHICH smoke layer failed and
          # whether the rollback succeeded, so /admin/deploys shows a
          # red pill with the actual cause instead of "deploy failed".
          REASON_HEADER=""
          if [ "$FULL_SMOKE_OUTCOME" = "failure" ]; then
            REASON_HEADER="post-deploy smoke suite failed"
          elif [ "$SMOKE_OUTCOME" = "failure" ]; then
            REASON_HEADER="/healthz smoke failed"
          fi
          if [ -n "$ROLLBACK_OUTCOME" ] && [ "$ROLLBACK_OUTCOME" != "skipped" ]; then
            if [ "$ROLLBACK_OUTCOME" = "success" ]; then
              REASON_HEADER="ROLLED BACK — $REASON_HEADER"
            else
              REASON_HEADER="ROLLBACK FAILED — $REASON_HEADER — human intervention required"
            fi
          fi
          # First 1 KB of diagnostics — keeps the JSON small and the DB row sane.
          ERR_TEXT=$(printf '%s\n\n%s' "${REASON_HEADER:-deploy failed}" "${DIAG:-see workflow logs}" | head -c 1024)
          # jq -Rs '.' is the safest way to JSON-escape arbitrary multi-line text.
          if command -v jq >/dev/null 2>&1; then
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | jq -Rs '.')
          else
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | python3 -c "import sys,json;print(json.dumps(sys.stdin.read()))")
          fi
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
            || echo "(deploy-finished[failed] notify failed — continuing)"