Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

self-deploy.sh

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

self-deploy.shBlame351 lines · 3 contributors
f2c00b4CC LABS App1#!/usr/bin/env bash
2# =============================================================================
3# BLOCK W — Gluecron self-deploy.
4#
5# Fired by:
6# - src/hooks/post-receive.ts when SELF_HOST_REPO matches the pushed repo
7# AND the ref is refs/heads/main
8# - the bare repo's hooks/post-receive (for SSH receive-pack)
9# - the optional .gluecron/workflows/deploy.yml on the workflow runner
10#
11# Contract:
12# - Detaches into the background via systemd-run (or nohup fallback) so
13# the caller's git push returns in ~1 second
14# - Logs every step to /var/log/gluecron-self-deploy.log
15# - Notifies /api/events/deploy/{started,step,finished} so /admin/deploys
16# streams the live timeline (same wire as the GitHub Actions workflow)
17# - Rolls back via git reflog if the post-deploy smoke fails
18#
19# Operator invariants:
20# - /etc/gluecron.env is the source of env truth (DATABASE_URL,
21# DEPLOY_EVENT_TOKEN, APP_BASE_URL, ANTHROPIC_API_KEY, …)
22# - /opt/gluecron is the working tree on the box, with git remote `origin`
23# pointing at https://gluecron.com/<owner>/<repo>.git (NOT GitHub)
24# - /opt/gluecron/.next/gluecron-server is the compiled Bun binary
25# - systemd unit `gluecron` is Type=notify and ExecStart=$EXEC_START
26#
27# TODO(ops): configure /etc/logrotate.d/gluecron-self-deploy for the log.
28# =============================================================================
29
ff4423bClaude30# `set -E` so traps propagate into subshells; `-x` traces every command
31# to stderr (captured into $LOG via the `>>"$LOG" 2>&1` redirects on the
32# detached re-exec line). Reliability sweep 2026-05-16: when this script
33# fails, the trace tells us EXACTLY which line broke instead of leaving
34# us guessing as we did for 17 hours of failed Hetzner deploys.
35set -Eeuxo pipefail
f2c00b4CC LABS App36
37WORKING_DIR="${GLUECRON_WORKING_DIR:-/opt/gluecron}"
38LOG="${GLUECRON_SELF_DEPLOY_LOG:-/var/log/gluecron-self-deploy.log}"
39ENV_FILE="${GLUECRON_ENV_FILE:-/etc/gluecron.env}"
40BUN="${GLUECRON_BUN:-/root/.bun/bin/bun}"
41HEALTHZ_URL="${GLUECRON_HEALTHZ_URL:-http://localhost:3010/healthz}"
42PORT="${GLUECRON_PORT:-3010}"
43DETACHED_FLAG="${1:-}"
44
45# ── helpers ────────────────────────────────────────────────────────────────
46ts() { date +'%Y-%m-%dT%H:%M:%S%z'; }
47log() { echo "[$(ts)] $*" | tee -a "$LOG" >&2; }
48
49notify_step() {
50 local NAME="$1" STATUS="$2" DUR="${3:-}"
51 if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] || [ -z "${APP_BASE_URL:-}" ]; then
52 return 0
53 fi
54 local DUR_FIELD=""
55 if [ -n "$DUR" ]; then DUR_FIELD=",\"duration_ms\":$DUR"; fi
56 curl --silent --show-error --max-time 5 \
57 -X POST "$APP_BASE_URL/api/events/deploy/step" \
58 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
59 -H "content-type: application/json" \
60 --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
61 >/dev/null 2>&1 || true
62}
63
64# ── re-exec into the background unless already detached ──────────────────
65# When the post-receive hook calls this script, git push is blocked on the
66# child's stdout/stderr. We re-exec ourselves through systemd-run so the
67# original SSH/HTTP receive-pack process can return immediately.
68if [ "$DETACHED_FLAG" != "--inline" ] && [ -z "${GLUECRON_SELF_DEPLOY_DETACHED:-}" ]; then
69 export GLUECRON_SELF_DEPLOY_DETACHED=1
70 if command -v systemd-run >/dev/null 2>&1; then
71 systemd-run --quiet --unit="gluecron-self-deploy-$(date +%s)" \
72 --collect --no-block \
73 bash "$0" --inline "$@" || nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
74 else
75 nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
76 disown || true
77 fi
78 exit 0
79fi
80
81# Everything below runs in the detached process.
82mkdir -p "$(dirname "$LOG")" 2>/dev/null || true
83touch "$LOG" 2>/dev/null || true
84
85log "==> gluecron self-deploy starting (pid $$)"
86
87# ── 1. Source env ──────────────────────────────────────────────────────────
88if [ -f "$ENV_FILE" ]; then
89 set -a
90 # shellcheck disable=SC1090
91 source "$ENV_FILE"
92 set +a
93 log " v sourced $ENV_FILE"
94else
95 log " ! $ENV_FILE not found — relying on inherited env"
96fi
97
98cd "$WORKING_DIR"
99
100# ── 2. Capture pre-deploy SHA for rollback ────────────────────────────────
101PREV_SHA="$(git rev-parse HEAD 2>/dev/null || echo '')"
102log " v previous SHA: $PREV_SHA"
103
104# ── 3. Pull latest main ────────────────────────────────────────────────────
105GP_START=$(date +%s)
106notify_step "git-pull" "in_progress"
107git fetch --prune origin main 2>&1 | tee -a "$LOG"
108git reset --hard origin/main 2>&1 | tee -a "$LOG"
109NEW_SHA="$(git rev-parse HEAD)"
110RUN_ID="self-${NEW_SHA:0:12}-$(date +%s)"
111log " v pulled to $NEW_SHA (run_id=$RUN_ID)"
112notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"
113
114# ── 3.5 Notify deploy started (now that we have NEW_SHA + RUN_ID) ─────────
115if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
116 curl --silent --show-error --max-time 10 \
117 -X POST "$APP_BASE_URL/api/events/deploy/started" \
118 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
119 -H "content-type: application/json" \
120 --data "{\"sha\":\"$NEW_SHA\",\"run_id\":\"$RUN_ID\",\"source\":\"self-deploy\"}" \
121 >>"$LOG" 2>&1 || log " ! deploy/started notify failed (non-fatal)"
122fi
123
124START_EPOCH=$(date +%s)
125DEPLOY_FAILED=0
126FAIL_REASON=""
127
141561fClaude128# ── 4. bun install --frozen-lockfile (skipped if lockfile unchanged) ──────
129# Track the last-installed lockfile hash in a stamp file. If it matches the
130# current lockfile we skip — saves ~2-5s every deploy that doesn't touch
131# dependencies (90%+ of deploys). On any error the fallback runs install
132# anyway so we never silently skip a needed install.
133STAMP_DIR="${GLUECRON_DEPLOY_STAMP_DIR:-.next}"
134mkdir -p "$STAMP_DIR" 2>/dev/null || true
135LOCK_STAMP="$STAMP_DIR/bun-install.stamp"
136LOCK_HASH=""
137if [ -f bun.lockb ]; then
138 LOCK_HASH=$(sha256sum bun.lockb 2>/dev/null | awk '{print $1}' || echo "")
139elif [ -f bun.lock ]; then
140 LOCK_HASH=$(sha256sum bun.lock 2>/dev/null | awk '{print $1}' || echo "")
141fi
142LAST_HASH=""
143if [ -f "$LOCK_STAMP" ]; then
144 LAST_HASH=$(cat "$LOCK_STAMP" 2>/dev/null || echo "")
145fi
146
f2c00b4CC LABS App147BI_START=$(date +%s)
148notify_step "bun-install" "in_progress"
141561fClaude149if [ -n "$LOCK_HASH" ] && [ "$LOCK_HASH" = "$LAST_HASH" ] && [ -d node_modules ]; then
150 log " v bun install skipped (lockfile unchanged: ${LOCK_HASH:0:12})"
151 notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
152elif "$BUN" install --frozen-lockfile >>"$LOG" 2>&1; then
153 echo "$LOCK_HASH" > "$LOCK_STAMP" 2>/dev/null || true
f2c00b4CC LABS App154 log " v bun install ok"
155 notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
156else
157 log " x bun install FAILED"
158 DEPLOY_FAILED=1
159 FAIL_REASON="bun install failed"
160 notify_step "bun-install" "failed" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
161fi
162
163# ── 5. DB migrations (fail loud) ───────────────────────────────────────────
164if [ "$DEPLOY_FAILED" = "0" ]; then
165 DM_START=$(date +%s)
166 notify_step "db-migrate" "in_progress"
167 if "$BUN" run src/db/migrate.ts >>"$LOG" 2>&1; then
168 log " v db migrate ok"
169 notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
170 else
171 log " x db migrate FAILED"
172 DEPLOY_FAILED=1
173 FAIL_REASON="bun run db:migrate failed"
174 notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
175 fi
176fi
177
141561fClaude178# ── 6. Build the static binary (skipped if no source files changed) ──────
179# We hash every tracked src/*.ts(x) file and stash the digest in a stamp
180# file. If nothing in src/ changed, the binary from the previous deploy
181# is still valid — saves 5-15s on docs-only / config-only commits.
182# On any error the fallback rebuilds anyway.
f2c00b4CC LABS App183if [ "$DEPLOY_FAILED" = "0" ]; then
184 BD_START=$(date +%s)
185 notify_step "build" "in_progress"
186 mkdir -p .next
187 COMPILED=.next/gluecron-server
188 COMPILED_TMP=.next/gluecron-server.new
141561fClaude189 BUILD_STAMP="$STAMP_DIR/build-src.stamp"
190 SRC_HASH=$(find src -type f \( -name "*.ts" -o -name "*.tsx" \) -print0 2>/dev/null \
191 | sort -z \
192 | xargs -0 sha256sum 2>/dev/null \
193 | sha256sum 2>/dev/null \
194 | awk '{print $1}' || echo "")
195 LAST_BUILD_HASH=""
196 if [ -f "$BUILD_STAMP" ]; then
197 LAST_BUILD_HASH=$(cat "$BUILD_STAMP" 2>/dev/null || echo "")
198 fi
199
200 if [ -n "$SRC_HASH" ] && [ "$SRC_HASH" = "$LAST_BUILD_HASH" ] && [ -x "$COMPILED" ]; then
201 log " v build skipped (src/ unchanged: ${SRC_HASH:0:12})"
202 notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
203 elif "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts >>"$LOG" 2>&1; then
f2c00b4CC LABS App204 mv -f "$COMPILED_TMP" "$COMPILED"
205 chmod +x "$COMPILED"
141561fClaude206 echo "$SRC_HASH" > "$BUILD_STAMP" 2>/dev/null || true
f2c00b4CC LABS App207 log " v compiled $COMPILED"
208 notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
209 else
210 rm -f "$COMPILED_TMP"
211 log " ! bun build --compile failed — systemd will fall back to bun run"
212 notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
213 fi
214fi
215
f85b88aTest User216# ── 6.5 Pin BUILD_SHA into the systemd unit so the running process can
217# report it and the SW versioning rotates exactly per-deploy. Drop-in
218# survives daemon-reload; the OLD file is overwritten on every deploy.
219# Without this, src/routes/pwa.ts falls back to a stable "dev-stable"
220# string and the browser SW never invalidates between real deploys.
221if [ "$DEPLOY_FAILED" = "0" ]; then
222 DROPIN_DIR=/etc/systemd/system/gluecron.service.d
223 mkdir -p "$DROPIN_DIR"
224 cat > "$DROPIN_DIR/build-sha.conf" <<EOF
225[Service]
226Environment="BUILD_SHA=$NEW_SHA"
227Environment="BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
228EOF
229 systemctl daemon-reload >>"$LOG" 2>&1 || log " ! daemon-reload non-fatal warning"
230 log " v pinned BUILD_SHA=${NEW_SHA:0:12} in systemd drop-in"
231fi
232
f2c00b4CC LABS App233# ── 7. systemctl restart (blocks on sd_notify READY=1) ────────────────────
234if [ "$DEPLOY_FAILED" = "0" ]; then
235 RS_START=$(date +%s)
236 notify_step "restart-service" "in_progress"
237 if systemctl restart gluecron >>"$LOG" 2>&1; then
238 log " v systemctl restart gluecron ok"
239 notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
240 else
241 log " x systemctl restart FAILED"
242 DEPLOY_FAILED=1
243 FAIL_REASON="systemctl restart failed"
244 notify_step "restart-service" "failed" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
245 fi
246fi
247
141561fClaude248# ── 8. Wait for /healthz to be green (fast poll, up to ~30s) ─────────────
249# Tighter polling: 500ms × 10 (covers the common case where the service
250# is healthy within 1-2s of restart returning), then 2s × 10 for the slow
251# path. Cuts a typical successful deploy by 1-3s vs the old fixed 2s poll.
f2c00b4CC LABS App252if [ "$DEPLOY_FAILED" = "0" ]; then
253 HZ_START=$(date +%s)
254 notify_step "healthz" "in_progress"
255 green=0
141561fClaude256 for i in 1 2 3 4 5 6 7 8 9 10; do
257 code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$HEALTHZ_URL" || echo "000")
258 if [ "$code" = "200" ]; then green=1; log " healthz fast attempt $i: $code (green)"; break; fi
259 sleep 0.5
f2c00b4CC LABS App260 done
141561fClaude261 if [ "$green" = "0" ]; then
262 for i in 11 12 13 14 15 16 17 18 19 20; do
263 code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$HEALTHZ_URL" || echo "000")
264 log " healthz slow attempt $i: $code"
265 if [ "$code" = "200" ]; then green=1; break; fi
266 sleep 2
267 done
268 fi
f2c00b4CC LABS App269 if [ "$green" = "1" ]; then
270 log " v /healthz green"
271 notify_step "healthz" "succeeded" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
272 else
273 log " x /healthz did not return 200 within 30s"
274 DEPLOY_FAILED=1
275 FAIL_REASON="/healthz timeout"
276 notify_step "healthz" "failed" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
277 fi
278fi
279
280# ── 9. Post-deploy smoke suite ────────────────────────────────────────────
281if [ "$DEPLOY_FAILED" = "0" ]; then
282 PS_START=$(date +%s)
283 notify_step "full-smoke" "in_progress"
284 export GLUECRON_HOST="http://localhost:${PORT}"
285 if "$BUN" run scripts/post-deploy-smoke.ts >>"$LOG" 2>&1; then
286 log " v post-deploy smoke green"
287 notify_step "full-smoke" "succeeded" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
288 else
289 log " x post-deploy smoke FAILED"
290 DEPLOY_FAILED=1
291 FAIL_REASON="post-deploy smoke failed"
292 notify_step "full-smoke" "failed" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
293 fi
294fi
295
296# ── 10. Rollback on failure ───────────────────────────────────────────────
297if [ "$DEPLOY_FAILED" = "1" ] && [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$NEW_SHA" ]; then
298 notify_step "rollback" "in_progress"
299 log " ! rolling back to $PREV_SHA (reason: $FAIL_REASON)"
300 git reset --hard "$PREV_SHA" >>"$LOG" 2>&1 || true
301 systemctl restart gluecron >>"$LOG" 2>&1 || true
302 sleep 3
303 rb_green=0
304 for i in 1 2 3; do
305 code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" || echo "000")
306 log " rollback healthz attempt $i: $code"
307 if [ "$code" = "200" ]; then rb_green=1; break; fi
308 sleep 2
309 done
310 if [ "$rb_green" = "1" ]; then
311 notify_step "rollback" "succeeded"
312 log " v rollback green"
313 else
314 notify_step "rollback" "failed"
315 log " x ROLLBACK FAILED — human intervention required"
316 fi
317fi
318
319# ── 11. Notify deploy finished ────────────────────────────────────────────
320DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
321if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
322 if [ "$DEPLOY_FAILED" = "0" ]; then
323 curl --silent --show-error --max-time 10 \
324 -X POST "$APP_BASE_URL/api/events/deploy/finished" \
325 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
326 -H "content-type: application/json" \
327 --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
328 >>"$LOG" 2>&1 || true
329 else
330 ERR_PAYLOAD="$(printf '%s' "${FAIL_REASON:-deploy failed}" | head -c 512)"
331 if command -v jq >/dev/null 2>&1; then
332 ERR_JSON=$(printf '%s' "$ERR_PAYLOAD" | jq -Rs '.')
333 else
334 ERR_JSON="\"${ERR_PAYLOAD//\"/\\\"}\""
335 fi
336 curl --silent --show-error --max-time 10 \
337 -X POST "$APP_BASE_URL/api/events/deploy/finished" \
338 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
339 -H "content-type: application/json" \
340 --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
341 >>"$LOG" 2>&1 || true
342 fi
343fi
344
345if [ "$DEPLOY_FAILED" = "0" ]; then
346 log "==> gluecron self-deploy SUCCESS in ${DUR_MS}ms (sha=$NEW_SHA)"
347 exit 0
348else
349 log "==> gluecron self-deploy FAILED in ${DUR_MS}ms (reason=$FAIL_REASON)"
350 exit 1
351fi