name: Hetzner Deploy (gluecron.com)

# Triggered on every push to main and on manual dispatch.
# Steps:
#   1. Capture the current SHA on the box (for rollback)
#   2. SSH in, git pull, run deploy-crontech.sh (which restarts systemd + reloads Caddy)
#   3. Smoke-test https://gluecron.com/healthz with retries
#   4. On smoke failure: roll back to the previous SHA and restart
#   5. On any failure: have Claude read the last 100 journal lines and post a one-paragraph
#      root-cause analysis to the workflow summary (and optionally a webhook)
#
# Secrets required:
#   HETZNER_HOST      — public IP/hostname of the box (e.g. 178.104.208.252)
#   HETZNER_USER      — ssh user (e.g. root)
#   HETZNER_SSH_KEY   — private deploy key (PEM/OpenSSH format)
#
# Optional:
#   ANTHROPIC_API_KEY      — enables AI failure-diagnosis step
#   DEPLOY_WEBHOOK_URL     — POSTed with JSON deploy status (Slack/Discord/anything)
#   DEPLOY_EVENT_TOKEN     — bearer used to POST deploy timeline events to
#                            ${APP_BASE_URL}/api/events/deploy/{started,finished}.
#                            Block N3: this makes the live site display ITS OWN
#                            deploy in the admin nav + at /admin/deploys.
#   APP_BASE_URL           — base URL of the running site (default https://gluecron.com).

on:
  push:
    branches: [main]
  workflow_dispatch: {}

concurrency:
  group: hetzner-deploy
  cancel-in-progress: false

jobs:
  deploy:
    name: Deploy gluecron.com
    runs-on: ubuntu-latest
    timeout-minutes: 12
    steps:
      - uses: actions/checkout@v4

      # ─── 0. Mark start time + notify the live site (Block N3) ────────────
      # The site has its own admin status pill + /admin/deploys timeline.
      # POSTing here makes the pill say "Deploying… 14s" the moment SSH
      # begins. `--fail` is intentionally NOT set: a 5xx here must never
      # block the deploy itself.
      - name: Record start time
        id: start
        run: |
          echo "epoch=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify deploy started
        if: env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
        run: |
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/started" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"sha\":\"${{ github.sha }}\",\"run_id\":\"${{ github.run_id }}\",\"source\":\"hetzner-deploy\"}" \
            || echo "(deploy-started notify failed — continuing)"

      # ─── 1. Capture pre-deploy SHA so we can rollback ───────────────────
      - name: Capture pre-deploy SHA
        id: prev
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            cd /opt/gluecron
            sha=$(git rev-parse HEAD)
            echo "Previous SHA: $sha"
            echo "$sha" > /tmp/gluecron_prev_sha
            cat /tmp/gluecron_prev_sha

      # ─── 2. Deploy: pull main, install/compile/restart ─────────────────
      # Block N2 — speed optimisation:
      #   (a) Cache `bun install` by hashing bun.lock; skip the walk when
      #       the lockfile is unchanged. Saves ~5-15s per "no deps changed"
      #       deploy.
      #   (b) Compile to a single Bun static binary at
      #       /opt/gluecron/.next/gluecron-server. Boot drops from ~500ms
      #       (cold ESM resolve) to <50ms. Compile itself takes ~3-5s.
      #   (c) Rewrite the systemd unit on first run if it lacks Type=notify
      #       (idempotent — diff-then-write, daemon-reload only on change).
      #       Falls back to `bun run src/index.ts` if the compiled binary is
      #       missing for any reason — the deploy MUST stay backward-
      #       compatible.
      #   (d) `systemctl restart` blocks until sd_notify(READY=1) fires
      #       (wired in src/lib/systemd-notify.ts), so we no longer rely on
      #       the curl-with-retries loop downstream.
      - name: Deploy
        id: deploy
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          command_timeout: 8m
          script_stop: true
          script: |
            set -euo pipefail
            cd /opt/gluecron
            git fetch --prune origin main
            git reset --hard origin/main
            new_sha=$(git rev-parse HEAD)
            echo "Deploying SHA: $new_sha"

            BUN=/root/.bun/bin/bun
            CACHE_DIR=/opt/gluecron/.cache
            HASH_FILE=$CACHE_DIR/bun-lockfile-hash
            mkdir -p "$CACHE_DIR"

            # ─── (a) Cached deps: skip bun install when bun.lock is unchanged
            if [ -f bun.lock ]; then
              new_hash=$(sha256sum bun.lock | awk '{print $1}')
            else
              new_hash="no-lockfile"
            fi
            old_hash=""
            if [ -f "$HASH_FILE" ]; then
              old_hash=$(cat "$HASH_FILE")
            fi
            if [ "$new_hash" = "$old_hash" ] && [ -d node_modules ]; then
              echo "==> bun install: SKIP (lockfile unchanged: $new_hash)"
            else
              echo "==> bun install: hash changed ($old_hash -> $new_hash) — installing"
              "$BUN" install --frozen-lockfile
              echo "$new_hash" > "$HASH_FILE"
            fi

            # ─── (b) Compile to a single static binary (best-effort)
            mkdir -p .next
            COMPILED=.next/gluecron-server
            COMPILED_TMP=.next/gluecron-server.new
            if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts; then
              mv -f "$COMPILED_TMP" "$COMPILED"
              chmod +x "$COMPILED"
              EXEC_START="/opt/gluecron/.next/gluecron-server"
              echo "==> compiled binary ready: $COMPILED"
            else
              # Backward-compat: if compile fails, fall back to interpreted Bun
              echo "WARN: bun build --compile failed — falling back to bun run"
              rm -f "$COMPILED_TMP"
              EXEC_START="$BUN run src/index.ts"
            fi

            # ─── (c) Idempotent systemd unit rewrite (Type=notify)
            UNIT=/etc/systemd/system/gluecron.service
            DESIRED=$(cat <<UNIT_EOF
            [Unit]
            Description=Gluecron — AI-native code intelligence platform
            After=network-online.target postgresql.service
            Wants=network-online.target

            [Service]
            Type=notify
            NotifyAccess=main
            User=root
            WorkingDirectory=/opt/gluecron
            EnvironmentFile=/etc/gluecron.env
            ExecStart=$EXEC_START
            Restart=always
            RestartSec=5
            TimeoutStartSec=30
            StandardOutput=journal
            StandardError=journal
            SyslogIdentifier=gluecron
            LimitNOFILE=65536

            [Install]
            WantedBy=multi-user.target
            UNIT_EOF
            )
            # Strip the leading indentation from the heredoc (`sed 's/^            //'`)
            # so the rendered unit is column-0 like systemd expects.
            DESIRED=$(printf '%s\n' "$DESIRED" | sed 's/^            //')

            need_rewrite=1
            if [ -f "$UNIT" ] && diff -q <(printf '%s\n' "$DESIRED") "$UNIT" >/dev/null 2>&1; then
              need_rewrite=0
            fi

            if [ "$need_rewrite" = "1" ]; then
              echo "==> rewriting $UNIT (Type=notify, ExecStart=$EXEC_START)"
              printf '%s\n' "$DESIRED" > "$UNIT"
              systemctl daemon-reload
            else
              echo "==> $UNIT already matches desired state — skipping daemon-reload"
            fi

            # ─── DB migrations (cheap; safe to always run)
            set -a; source /etc/gluecron.env; set +a
            "$BUN" run src/db/migrate.ts || echo "WARN: migrate failed (may be already-applied)"

            # ─── (d) Zero-downtime restart. Blocks until sd_notify(READY=1).
            echo "==> systemctl restart gluecron (blocks on sd_notify READY=1)"
            systemctl restart gluecron
            echo "==> restart returned — gluecron signalled ready"

      # ─── 3. Smoke-test the deployed app on the box ──────────────────────
      # We SSH back in and curl localhost:3010/healthz directly. This tests
      # the EXACT instance we just deployed, independent of:
      #   - DNS state (gluecron.com may still point at an old box during a
      #     migration)
      #   - Caddy TLS state (cert may not be issued yet for a new domain)
      #   - external network reachability from GH runners
      # If you ALSO want a public-DNS smoke check, add a second step that
      # hits https://gluecron.com after this one succeeds.
      - name: Smoke test (localhost on the box)
        id: smoke
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            # Block N2 — `systemctl restart` already blocked on
            # sd_notify(READY=1), so the FIRST curl should succeed. We keep a
            # short retry budget for paranoia: a brief delay between
            # systemd's READY ack and the HTTP listener becoming routable
            # via 127.0.0.1 is theoretically possible (unusual but cheap to
            # tolerate). 3 attempts × 2s = 6s ceiling instead of 8 × 6s = 48s.
            set +e
            for i in 1 2 3; do
              code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
              echo "Attempt $i: /healthz -> $code"
              if [ "$code" = "200" ]; then
                echo "OK: gluecron is healthy on localhost:3010"
                curl -s http://localhost:3010/api/version || true
                exit 0
              fi
              sleep 2
            done
            echo "FAIL: /healthz did not return 200 after 6s"
            systemctl status gluecron --no-pager | head -10 || true
            journalctl -u gluecron -n 30 --no-pager || true
            exit 1

      # ─── 4. Auto-rollback on smoke failure ──────────────────────────────
      # Only rolls back if the workflow was triggered by a normal push.
      # Manual workflow_dispatch runs SKIP rollback so the operator can
      # diagnose the new code on the box before reverting. This stops the
      # pathological case where rollback masks the real failure by reverting
      # to an already-broken previous SHA.
      - name: Rollback on failure
        if: failure() && steps.smoke.conclusion == 'failure' && github.event_name == 'push'
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            cd /opt/gluecron
            prev=$(cat /tmp/gluecron_prev_sha)
            echo "Rolling back to $prev"
            git reset --hard "$prev"
            systemctl restart gluecron
            sleep 5
            systemctl status gluecron --no-pager | head -20 || true

      # ─── 5. Failure diagnostics — captured into a file for summary + AI ──
      - name: Capture failure context
        if: failure()
        id: ctx
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            echo "===== systemd status ====="
            systemctl status gluecron --no-pager 2>&1 | head -40 || true
            echo ""
            echo "===== last 80 journal lines (gluecron) ====="
            journalctl -u gluecron -n 80 --no-pager --output=cat 2>&1 || true
            echo ""
            echo "===== caddy validate ====="
            caddy validate --config /etc/caddy/Caddyfile 2>&1 | head -20 || true
            echo ""
            echo "===== /healthz from inside box ====="
            curl -s -w "\nHTTP %{http_code}\n" http://localhost:3000/healthz 2>&1 || true
            echo ""
            echo "===== port 3000 listener ====="
            ss -tlnp 2>&1 | grep ':3000' || echo '(nothing listening on :3000)'

      # Always post the captured diagnostics to the workflow summary so the
      # owner can read what broke without SSH'ing or grepping log files.
      - name: Post diagnostics to summary
        if: failure() && steps.ctx.outputs.stdout != ''
        env:
          DIAG: ${{ steps.ctx.outputs.stdout }}
        run: |
          {
            echo "## ❌ Deploy failed — diagnostics"
            echo ""
            echo "**Commit:** \`${GITHUB_SHA:0:7}\` — ${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-${GITHUB_SHA:0:7}}"
            echo "**Run:** [#${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
            echo ""
            echo '```'
            echo "$DIAG"
            echo '```'
          } >> $GITHUB_STEP_SUMMARY

      - name: AI root-cause analysis (Claude)
        if: failure() && env.ANTHROPIC_API_KEY != ''
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          DEPLOY_LOGS: ${{ steps.ctx.outputs.stdout }}
          COMMIT_SHA: ${{ github.sha }}
          COMMIT_MSG: ${{ github.event.head_commit.message }}
        run: |
          set +e
          # Build the prompt
          cat > /tmp/prompt.json <<EOF
          {
            "model": "claude-haiku-4-5-20251001",
            "max_tokens": 600,
            "system": "You are a senior SRE diagnosing a failed deploy. Read the systemd status, journal, and curl output. In 1 short paragraph (under 100 words), identify the most likely root cause and the single fastest fix. Be direct, no preamble.",
            "messages": [{
              "role": "user",
              "content": "Commit: $COMMIT_SHA\nMessage: $COMMIT_MSG\n\nDeploy logs:\n$DEPLOY_LOGS"
            }]
          }
          EOF
          response=$(curl -s https://api.anthropic.com/v1/messages \
            -H "x-api-key: $ANTHROPIC_API_KEY" \
            -H "anthropic-version: 2023-06-01" \
            -H "content-type: application/json" \
            --data @/tmp/prompt.json)
          analysis=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('content',[{}])[0].get('text','(no analysis)'))" 2>/dev/null)
          echo "## 🤖 AI Failure Analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "$analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Commit:** \`${COMMIT_SHA:0:7}\` — $COMMIT_MSG" >> $GITHUB_STEP_SUMMARY

      # ─── 6. Optional webhook notification ───────────────────────────────
      - name: Notify webhook
        if: always() && env.DEPLOY_WEBHOOK_URL != ''
        env:
          DEPLOY_WEBHOOK_URL: ${{ secrets.DEPLOY_WEBHOOK_URL }}
          STATUS: ${{ job.status }}
        run: |
          curl -s -X POST "$DEPLOY_WEBHOOK_URL" \
            -H "content-type: application/json" \
            --data "{\"status\":\"$STATUS\",\"target\":\"gluecron.com\",\"sha\":\"${{ github.sha }}\",\"run\":\"${{ github.run_id }}\"}" || true

      # ─── 7. Workflow summary on success ─────────────────────────────────
      - name: Success summary
        if: success()
        run: |
          echo "## ✅ Deploy succeeded" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Target:** https://gluecron.com" >> $GITHUB_STEP_SUMMARY
          echo "- **SHA:** \`${GITHUB_SHA:0:7}\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Health:** /healthz → 200" >> $GITHUB_STEP_SUMMARY

      # ─── 8. Block N3 — POST deploy-finished event to the live site ───────
      # The site's admin status pill flips to "Deployed Ns ago" or
      # "Deploy failed Nm ago" the instant this lands. `if: always()` so we
      # always report final state (including failure); the inner `if:` flag
      # splits success vs failure for the payload body. We never `--fail` —
      # a 5xx must not retroactively break a deploy that actually succeeded.
      - name: Notify deploy finished (success)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
            || echo "(deploy-finished[succeeded] notify failed — continuing)"

      - name: Notify deploy finished (failure)
        if: failure() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
          DIAG: ${{ steps.ctx.outputs.stdout }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          # First 1 KB of diagnostics — keeps the JSON small and the DB row sane.
          ERR_TEXT=$(printf '%s' "${DIAG:-deploy failed; see workflow logs}" | head -c 1024)
          # jq -Rs '.' is the safest way to JSON-escape arbitrary multi-line text.
          if command -v jq >/dev/null 2>&1; then
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | jq -Rs '.')
          else
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | python3 -c "import sys,json;print(json.dumps(sys.stdin.read()))")
          fi
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
            || echo "(deploy-finished[failed] notify failed — continuing)"