Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
claude/adoring-hopper-5x74bqclaude/affectionate-feynman-ykrf1hclaude/architecture-audit-design-wxprenclaude/build-status-update-3MXsfclaude/charming-meitner-mllb5rclaude/compare-gate-gluecron-s4mFQclaude/confident-faraday-tikcwbclaude/continue-work-XMTlIclaude/crontech-gluecron-deploy-7MIECclaude/crontech-platform-setup-SeKfwclaude/design-2026claude/ecstatic-ptolemy-jMdigclaude/enhance-github-integration-QNHdGclaude/fix-aa-loop-issue-PonMQclaude/fix-actions-and-processclaude/fix-desktop-errors-XqoW8claude/fix-red-workflowsclaude/fix-website-access-6FKJNclaude/gatetest-integration-hardeningclaude/github-audit-improvements-bDFr9claude/gluecron-launch-status-FoMRlclaude/hopeful-lamport-olfCTclaude/issue-to-pr-and-protectionsclaude/jolly-heisenberg-2sg1Qclaude/launch-preparation-QmTb6claude/new-session-xk1l7claude/plan-platform-architecture-kkN4yclaude/platform-analysis-roadmap-1nUGLclaude/platform-launch-assessment-8dWV8claude/polish-platform-release-AeDrUclaude/resume-previous-work-KzyLwclaude/review-crontech-handoff-qYEVqclaude/review-project-completeness-lHhS2claude/review-readme-docs-ulqPKclaude/serene-edison-rj87weclaude/setup-multi-repo-dev-BCwNQclaude/ship-fixes-and-tests-Jvz1cclaude/site-audit-competitive-pctlwgclaude/site-migration-vercel-XstpKclaude/standalone-product-repos-XHFTDcopilot/feat-smart-empty-states-keyboard-first-enhancementcopilot/feat-smart-morning-digest-review-context-restorecopilot/fix-and-process-workflowscopilot/update-ai-powered-code-reviewfeat/debt-mapfeat/push-policy-codeowners-hardeningfeat/smart-digest-contextfeat/stage-impactfeat/t1-secret-migrationfeat/u-polishfeat/w-self-hostfeat/w2-claude-configfix/agent-journey-orphan-sweepgatetest/auto-fix-1776586424172gatetest/auto-fix-1776586534814gatetest/auto-fix-1776590685143gatetest/auto-fix-1776590808199mainops/redeploy-retriggerstyle/dxt-cta-themeworktree-agent-a3377aad30d55da26worktree-agent-a7ef607b7ee1d6c74
hetzner-deploy.yml23.9 KB · 516 lines
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
name: Hetzner Deploy (gluecron.com)

# Triggered on every push to main and on manual dispatch.
# Steps:
#   1. Capture the current SHA on the box (for rollback)
#   2. SSH in, git pull, run deploy-crontech.sh (which restarts systemd + reloads Caddy)
#   3. Smoke-test https://gluecron.com/healthz with retries
#   4. On smoke failure: roll back to the previous SHA and restart
#   5. On any failure: have Claude read the last 100 journal lines and post a one-paragraph
#      root-cause analysis to the workflow summary (and optionally a webhook)
#
# Secrets required:
#   HETZNER_HOST      — public IP/hostname of the box (e.g. 178.104.208.252)
#   HETZNER_USER      — ssh user (e.g. root)
#   HETZNER_SSH_KEY   — private deploy key (PEM/OpenSSH format)
#
# Optional:
#   ANTHROPIC_API_KEY      — enables AI failure-diagnosis step
#   DEPLOY_WEBHOOK_URL     — POSTed with JSON deploy status (Slack/Discord/anything)
#   DEPLOY_EVENT_TOKEN     — bearer used to POST deploy timeline events to
#                            ${APP_BASE_URL}/api/events/deploy/{started,finished}.
#                            Block N3: this makes the live site display ITS OWN
#                            deploy in the admin nav + at /admin/deploys.
#   APP_BASE_URL           — base URL of the running site (default https://gluecron.com).

on:
  push:
    branches: [main]
  workflow_dispatch: {}

concurrency:
  group: hetzner-deploy
  cancel-in-progress: false

jobs:
  deploy:
    name: Deploy gluecron.com
    runs-on: ubuntu-latest
    timeout-minutes: 12
    steps:
      - uses: actions/checkout@v4

      # ─── 0. Mark start time + notify the live site (Block N3) ────────────
      # The site has its own admin status pill + /admin/deploys timeline.
      # POSTing here makes the pill say "Deploying… 14s" the moment SSH
      # begins. `--fail` is intentionally NOT set: a 5xx here must never
      # block the deploy itself.
      - name: Record start time
        id: start
        run: |
          echo "epoch=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify deploy started
        if: env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
        run: |
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/started" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"sha\":\"${{ github.sha }}\",\"run_id\":\"${{ github.run_id }}\",\"source\":\"hetzner-deploy\"}" \
            || echo "(deploy-started notify failed — continuing)"

      # ─── 1. Capture pre-deploy SHA so we can rollback ───────────────────
      # R2 — bracket every major step with notify-deploy-step calls so the
      # /admin/deploys modal can stream the workflow live. The composite
      # action is fire-and-forget — a 5xx never blocks the deploy.
      - name: Notify step  setup (in_progress)
        if: env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: in_progress
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Capture pre-deploy SHA
        id: prev
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            cd /opt/gluecron
            sha=$(git rev-parse HEAD)
            echo "Previous SHA: $sha"
            echo "$sha" > /tmp/gluecron_prev_sha
            cat /tmp/gluecron_prev_sha

      - name: Notify step  setup (succeeded)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: succeeded
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Notify step  setup (failed)
        if: failure() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: setup
          status: failed
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      # ─── 2. Deploy: pull main, install/compile/restart ─────────────────
      # Block N2 — speed optimisation:
      #   (a) Cache `bun install` by hashing bun.lock; skip the walk when
      #       the lockfile is unchanged. Saves ~5-15s per "no deps changed"
      #       deploy.
      #   (b) Compile to a single Bun static binary at
      #       /opt/gluecron/.next/gluecron-server. Boot drops from ~500ms
      #       (cold ESM resolve) to <50ms. Compile itself takes ~3-5s.
      #   (c) Rewrite the systemd unit on first run if it lacks Type=notify
      #       (idempotent — diff-then-write, daemon-reload only on change).
      #       Falls back to `bun run src/index.ts` if the compiled binary is
      #       missing for any reason — the deploy MUST stay backward-
      #       compatible.
      #   (d) `systemctl restart` blocks until sd_notify(READY=1) fires
      #       (wired in src/lib/systemd-notify.ts), so we no longer rely on
      #       the curl-with-retries loop downstream.
      - name: Deploy
        id: deploy
        uses: appleboy/ssh-action@v1.2.0
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          GH_RUN_ID: ${{ github.run_id }}
          GH_SHA: ${{ github.sha }}
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          command_timeout: 8m
          script_stop: true
          # R2: stream per-phase progress to /api/events/deploy/step. We
          # share one SSH session across all phases (git pull → install →
          # build → migrate → restart) so the in-script curl posts give us
          # the live timeline a black-box step boundary can't.
          envs: DEPLOY_EVENT_TOKEN,APP_BASE_URL,GH_RUN_ID,GH_SHA
          script: |
            set -euo pipefail
            cd /opt/gluecron

            # R2 helper: POST a single step event (in_progress|succeeded|failed).
            # Never fails — observability must not break deploys.
            notify_step() {
              local NAME="$1" STATUS="$2" DUR="${3:-}"
              if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] || [ -z "${APP_BASE_URL:-}" ]; then
                return 0
              fi
              local DUR_FIELD=""
              if [ -n "$DUR" ]; then
                DUR_FIELD=",\"duration_ms\":$DUR"
              fi
              curl --silent --show-error --max-time 5 \
                -X POST "$APP_BASE_URL/api/events/deploy/step" \
                -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
                -H "content-type: application/json" \
                --data "{\"run_id\":\"$GH_RUN_ID\",\"sha\":\"$GH_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
                >/dev/null 2>&1 || true
            }

            notify_step "git-pull" "in_progress"
            GP_START=$(date +%s)
            git fetch --prune origin main
            git reset --hard origin/main
            new_sha=$(git rev-parse HEAD)
            echo "Deploying SHA: $new_sha"
            notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"

            BUN=/root/.bun/bin/bun
            CACHE_DIR=/opt/gluecron/.cache
            HASH_FILE=$CACHE_DIR/bun-lockfile-hash
            mkdir -p "$CACHE_DIR"

            # ─── (a) Cached deps: skip bun install when bun.lock is unchanged
            notify_step "bun-install" "in_progress"
            BI_START=$(date +%s)
            if [ -f bun.lock ]; then
              new_hash=$(sha256sum bun.lock | awk '{print $1}')
            else
              new_hash="no-lockfile"
            fi
            old_hash=""
            if [ -f "$HASH_FILE" ]; then
              old_hash=$(cat "$HASH_FILE")
            fi
            if [ "$new_hash" = "$old_hash" ] && [ -d node_modules ]; then
              echo "==> bun install: SKIP (lockfile unchanged: $new_hash)"
            else
              echo "==> bun install: hash changed ($old_hash -> $new_hash) — installing"
              "$BUN" install --frozen-lockfile
              echo "$new_hash" > "$HASH_FILE"
            fi
            notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"

            # ─── (b) Compile to a single static binary (best-effort)
            notify_step "build" "in_progress"
            BD_START=$(date +%s)
            mkdir -p .next
            COMPILED=.next/gluecron-server
            COMPILED_TMP=.next/gluecron-server.new
            if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts; then
              mv -f "$COMPILED_TMP" "$COMPILED"
              chmod +x "$COMPILED"
              EXEC_START="/opt/gluecron/.next/gluecron-server"
              echo "==> compiled binary ready: $COMPILED"
              notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
            else
              # Backward-compat: if compile fails, fall back to interpreted Bun
              echo "WARN: bun build --compile failed — falling back to bun run"
              rm -f "$COMPILED_TMP"
              EXEC_START="$BUN run src/index.ts"
              # Treat compile-failure-with-fallback as 'succeeded' for the
              # modal — the deploy itself is still going.
              notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
            fi

            # ─── (c) Idempotent systemd unit rewrite (Type=notify)
            UNIT=/etc/systemd/system/gluecron.service
            DESIRED=$(cat <<UNIT_EOF
            [Unit]
            Description=Gluecron  AI-native code intelligence platform
            After=network-online.target postgresql.service
            Wants=network-online.target

            [Service]
            Type=notify
            NotifyAccess=main
            User=root
            WorkingDirectory=/opt/gluecron
            EnvironmentFile=/etc/gluecron.env
            ExecStart=$EXEC_START
            Restart=always
            RestartSec=5
            TimeoutStartSec=30
            StandardOutput=journal
            StandardError=journal
            SyslogIdentifier=gluecron
            LimitNOFILE=65536

            [Install]
            WantedBy=multi-user.target
            UNIT_EOF
            )
            # Strip the leading indentation from the heredoc (`sed 's/^            //'`)
            # so the rendered unit is column-0 like systemd expects.
            DESIRED=$(printf '%s\n' "$DESIRED" | sed 's/^            //')

            need_rewrite=1
            if [ -f "$UNIT" ] && diff -q <(printf '%s\n' "$DESIRED") "$UNIT" >/dev/null 2>&1; then
              need_rewrite=0
            fi

            if [ "$need_rewrite" = "1" ]; then
              echo "==> rewriting $UNIT (Type=notify, ExecStart=$EXEC_START)"
              printf '%s\n' "$DESIRED" > "$UNIT"
              systemctl daemon-reload
            else
              echo "==> $UNIT already matches desired state — skipping daemon-reload"
            fi

            # ─── DB migrations (cheap; safe to always run)
            notify_step "db-migrate" "in_progress"
            DM_START=$(date +%s)
            set -a; source /etc/gluecron.env; set +a
            "$BUN" run src/db/migrate.ts || echo "WARN: migrate failed (may be already-applied)"
            notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"

            # ─── (d) Zero-downtime restart. Blocks until sd_notify(READY=1).
            notify_step "restart-service" "in_progress"
            RS_START=$(date +%s)
            echo "==> systemctl restart gluecron (blocks on sd_notify READY=1)"
            systemctl restart gluecron
            echo "==> restart returned — gluecron signalled ready"
            notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"

      # ─── 3. Smoke-test the deployed app on the box ──────────────────────
      # We SSH back in and curl localhost:3010/healthz directly. This tests
      # the EXACT instance we just deployed, independent of:
      #   - DNS state (gluecron.com may still point at an old box during a
      #     migration)
      #   - Caddy TLS state (cert may not be issued yet for a new domain)
      #   - external network reachability from GH runners
      # If you ALSO want a public-DNS smoke check, add a second step that
      # hits https://gluecron.com after this one succeeds.
      - name: Notify step  smoke-test (in_progress)
        if: env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: in_progress
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Smoke test (localhost on the box)
        id: smoke
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: true
          script: |
            # Block N2 — `systemctl restart` already blocked on
            # sd_notify(READY=1), so the FIRST curl should succeed. We keep a
            # short retry budget for paranoia: a brief delay between
            # systemd's READY ack and the HTTP listener becoming routable
            # via 127.0.0.1 is theoretically possible (unusual but cheap to
            # tolerate). 3 attempts × 2s = 6s ceiling instead of 8 × 6s = 48s.
            set +e
            for i in 1 2 3; do
              code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
              echo "Attempt $i: /healthz -> $code"
              if [ "$code" = "200" ]; then
                echo "OK: gluecron is healthy on localhost:3010"
                curl -s http://localhost:3010/api/version || true
                exit 0
              fi
              sleep 2
            done
            echo "FAIL: /healthz did not return 200 after 6s"
            systemctl status gluecron --no-pager | head -10 || true
            journalctl -u gluecron -n 30 --no-pager || true
            exit 1

      - name: Notify step  smoke-test (succeeded)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: succeeded
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      - name: Notify step  smoke-test (failed)
        if: failure() && steps.smoke.conclusion == 'failure' && env.DEPLOY_EVENT_TOKEN != ''
        uses: ./.github/actions/notify-deploy-step
        with:
          step_name: smoke-test
          status: failed
          app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}

      # ─── 4. Auto-rollback on smoke failure ──────────────────────────────
      # Only rolls back if the workflow was triggered by a normal push.
      # Manual workflow_dispatch runs SKIP rollback so the operator can
      # diagnose the new code on the box before reverting. This stops the
      # pathological case where rollback masks the real failure by reverting
      # to an already-broken previous SHA.
      - name: Rollback on failure
        if: failure() && steps.smoke.conclusion == 'failure' && github.event_name == 'push'
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            cd /opt/gluecron
            prev=$(cat /tmp/gluecron_prev_sha)
            echo "Rolling back to $prev"
            git reset --hard "$prev"
            systemctl restart gluecron
            sleep 5
            systemctl status gluecron --no-pager | head -20 || true

      # ─── 5. Failure diagnostics — captured into a file for summary + AI ──
      - name: Capture failure context
        if: failure()
        id: ctx
        uses: appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.HETZNER_HOST }}
          username: ${{ secrets.HETZNER_USER }}
          key: ${{ secrets.HETZNER_SSH_KEY }}
          script_stop: false
          script: |
            echo "===== systemd status ====="
            systemctl status gluecron --no-pager 2>&1 | head -40 || true
            echo ""
            echo "===== last 80 journal lines (gluecron) ====="
            journalctl -u gluecron -n 80 --no-pager --output=cat 2>&1 || true
            echo ""
            echo "===== caddy validate ====="
            caddy validate --config /etc/caddy/Caddyfile 2>&1 | head -20 || true
            echo ""
            echo "===== /healthz from inside box ====="
            curl -s -w "\nHTTP %{http_code}\n" http://localhost:3000/healthz 2>&1 || true
            echo ""
            echo "===== port 3000 listener ====="
            ss -tlnp 2>&1 | grep ':3000' || echo '(nothing listening on :3000)'

      # Always post the captured diagnostics to the workflow summary so the
      # owner can read what broke without SSH'ing or grepping log files.
      - name: Post diagnostics to summary
        if: failure() && steps.ctx.outputs.stdout != ''
        env:
          DIAG: ${{ steps.ctx.outputs.stdout }}
        run: |
          {
            echo "## ❌ Deploy failed — diagnostics"
            echo ""
            echo "**Commit:** \`${GITHUB_SHA:0:7}\` — ${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-${GITHUB_SHA:0:7}}"
            echo "**Run:** [#${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
            echo ""
            echo '```'
            echo "$DIAG"
            echo '```'
          } >> $GITHUB_STEP_SUMMARY

      - name: AI root-cause analysis (Claude)
        if: failure() && env.ANTHROPIC_API_KEY != ''
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          DEPLOY_LOGS: ${{ steps.ctx.outputs.stdout }}
          COMMIT_SHA: ${{ github.sha }}
          COMMIT_MSG: ${{ github.event.head_commit.message }}
        run: |
          set +e
          # Build the prompt
          cat > /tmp/prompt.json <<EOF
          {
            "model": "claude-haiku-4-5-20251001",
            "max_tokens": 600,
            "system": "You are a senior SRE diagnosing a failed deploy. Read the systemd status, journal, and curl output. In 1 short paragraph (under 100 words), identify the most likely root cause and the single fastest fix. Be direct, no preamble.",
            "messages": [{
              "role": "user",
              "content": "Commit: $COMMIT_SHA\nMessage: $COMMIT_MSG\n\nDeploy logs:\n$DEPLOY_LOGS"
            }]
          }
          EOF
          response=$(curl -s https://api.anthropic.com/v1/messages \
            -H "x-api-key: $ANTHROPIC_API_KEY" \
            -H "anthropic-version: 2023-06-01" \
            -H "content-type: application/json" \
            --data @/tmp/prompt.json)
          analysis=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('content',[{}])[0].get('text','(no analysis)'))" 2>/dev/null)
          echo "## 🤖 AI Failure Analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "$analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Commit:** \`${COMMIT_SHA:0:7}\` — $COMMIT_MSG" >> $GITHUB_STEP_SUMMARY

      # ─── 6. Optional webhook notification ───────────────────────────────
      - name: Notify webhook
        if: always() && env.DEPLOY_WEBHOOK_URL != ''
        env:
          DEPLOY_WEBHOOK_URL: ${{ secrets.DEPLOY_WEBHOOK_URL }}
          STATUS: ${{ job.status }}
        run: |
          curl -s -X POST "$DEPLOY_WEBHOOK_URL" \
            -H "content-type: application/json" \
            --data "{\"status\":\"$STATUS\",\"target\":\"gluecron.com\",\"sha\":\"${{ github.sha }}\",\"run\":\"${{ github.run_id }}\"}" || true

      # ─── 7. Workflow summary on success ─────────────────────────────────
      - name: Success summary
        if: success()
        run: |
          echo "## ✅ Deploy succeeded" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Target:** https://gluecron.com" >> $GITHUB_STEP_SUMMARY
          echo "- **SHA:** \`${GITHUB_SHA:0:7}\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Health:** /healthz → 200" >> $GITHUB_STEP_SUMMARY

      # ─── 8. Block N3 — POST deploy-finished event to the live site ───────
      # The site's admin status pill flips to "Deployed Ns ago" or
      # "Deploy failed Nm ago" the instant this lands. `if: always()` so we
      # always report final state (including failure); the inner `if:` flag
      # splits success vs failure for the payload body. We never `--fail` —
      # a 5xx must not retroactively break a deploy that actually succeeded.
      - name: Notify deploy finished (success)
        if: success() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
            || echo "(deploy-finished[succeeded] notify failed — continuing)"

      - name: Notify deploy finished (failure)
        if: failure() && env.DEPLOY_EVENT_TOKEN != ''
        env:
          DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
          APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
          START_EPOCH: ${{ steps.start.outputs.epoch }}
          DIAG: ${{ steps.ctx.outputs.stdout }}
        run: |
          DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
          # First 1 KB of diagnostics — keeps the JSON small and the DB row sane.
          ERR_TEXT=$(printf '%s' "${DIAG:-deploy failed; see workflow logs}" | head -c 1024)
          # jq -Rs '.' is the safest way to JSON-escape arbitrary multi-line text.
          if command -v jq >/dev/null 2>&1; then
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | jq -Rs '.')
          else
            ERR_JSON=$(printf '%s' "$ERR_TEXT" | python3 -c "import sys,json;print(json.dumps(sys.stdin.read()))")
          fi
          curl --silent --show-error --max-time 10 \
            -X POST "$APP_BASE_URL/api/events/deploy/finished" \
            -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
            -H "content-type: application/json" \
            --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
            || echo "(deploy-finished[failed] notify failed — continuing)"