CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history
self-deploy.sh
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| f2c00b4 | 1 | #!/usr/bin/env bash |
| 2 | # ============================================================================= | |
| 3 | # BLOCK W — Gluecron self-deploy. | |
| 4 | # | |
| 5 | # Fired by: | |
| 6 | # - src/hooks/post-receive.ts when SELF_HOST_REPO matches the pushed repo | |
| 7 | # AND the ref is refs/heads/main | |
| 8 | # - the bare repo's hooks/post-receive (for SSH receive-pack) | |
| 9 | # - the optional .gluecron/workflows/deploy.yml on the workflow runner | |
| 10 | # | |
| 11 | # Contract: | |
| 12 | # - Detaches into the background via systemd-run (or nohup fallback) so | |
| 13 | # the caller's git push returns in ~1 second | |
| 14 | # - Logs every step to /var/log/gluecron-self-deploy.log | |
| 15 | # - Notifies /api/events/deploy/{started,step,finished} so /admin/deploys | |
| 16 | # streams the live timeline (same wire as the GitHub Actions workflow) | |
| 17 | # - Rolls back via git reflog if the post-deploy smoke fails | |
| 18 | # | |
| 19 | # Operator invariants: | |
| 20 | # - /etc/gluecron.env is the source of env truth (DATABASE_URL, | |
| 21 | # DEPLOY_EVENT_TOKEN, APP_BASE_URL, ANTHROPIC_API_KEY, …) | |
| 22 | # - /opt/gluecron is the working tree on the box, with git remote `origin` | |
| 23 | # pointing at https://gluecron.com/<owner>/<repo>.git (NOT GitHub) | |
| 24 | # - /opt/gluecron/.next/gluecron-server is the compiled Bun binary | |
| 25 | # - systemd unit `gluecron` is Type=notify and ExecStart=$EXEC_START | |
| 26 | # | |
| 27 | # TODO(ops): configure /etc/logrotate.d/gluecron-self-deploy for the log. | |
| 28 | # ============================================================================= | |
| 29 | ||
| ff4423b | 30 | # `set -E` so traps propagate into subshells; `-x` traces every command |
| 31 | # to stderr (captured into $LOG via the `>>"$LOG" 2>&1` redirects on the | |
| 32 | # detached re-exec line). Reliability sweep 2026-05-16: when this script | |
| 33 | # fails, the trace tells us EXACTLY which line broke instead of leaving | |
| 34 | # us guessing as we did for 17 hours of failed Hetzner deploys. | |
| 35 | set -Eeuxo pipefail | |
| f2c00b4 | 36 | |
| 37 | WORKING_DIR="${GLUECRON_WORKING_DIR:-/opt/gluecron}" | |
| 38 | LOG="${GLUECRON_SELF_DEPLOY_LOG:-/var/log/gluecron-self-deploy.log}" | |
| 39 | ENV_FILE="${GLUECRON_ENV_FILE:-/etc/gluecron.env}" | |
| 40 | BUN="${GLUECRON_BUN:-/root/.bun/bin/bun}" | |
| 41 | HEALTHZ_URL="${GLUECRON_HEALTHZ_URL:-http://localhost:3010/healthz}" | |
| 42 | PORT="${GLUECRON_PORT:-3010}" | |
| 43 | DETACHED_FLAG="${1:-}" | |
| 44 | ||
| 45 | # ── helpers ──────────────────────────────────────────────────────────────── | |
| 46 | ts() { date +'%Y-%m-%dT%H:%M:%S%z'; } | |
| 47 | log() { echo "[$(ts)] $*" | tee -a "$LOG" >&2; } | |
| 48 | ||
| 49 | notify_step() { | |
| 50 | local NAME="$1" STATUS="$2" DUR="${3:-}" | |
| 51 | if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] || [ -z "${APP_BASE_URL:-}" ]; then | |
| 52 | return 0 | |
| 53 | fi | |
| 54 | local DUR_FIELD="" | |
| 55 | if [ -n "$DUR" ]; then DUR_FIELD=",\"duration_ms\":$DUR"; fi | |
| 56 | curl --silent --show-error --max-time 5 \ | |
| 57 | -X POST "$APP_BASE_URL/api/events/deploy/step" \ | |
| 58 | -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \ | |
| 59 | -H "content-type: application/json" \ | |
| 60 | --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \ | |
| 61 | >/dev/null 2>&1 || true | |
| 62 | } | |
| 63 | ||
| 64 | # ── re-exec into the background unless already detached ────────────────── | |
| 65 | # When the post-receive hook calls this script, git push is blocked on the | |
| 66 | # child's stdout/stderr. We re-exec ourselves through systemd-run so the | |
| 67 | # original SSH/HTTP receive-pack process can return immediately. | |
| 68 | if [ "$DETACHED_FLAG" != "--inline" ] && [ -z "${GLUECRON_SELF_DEPLOY_DETACHED:-}" ]; then | |
| 69 | export GLUECRON_SELF_DEPLOY_DETACHED=1 | |
| 70 | if command -v systemd-run >/dev/null 2>&1; then | |
| 71 | systemd-run --quiet --unit="gluecron-self-deploy-$(date +%s)" \ | |
| 72 | --collect --no-block \ | |
| 73 | bash "$0" --inline "$@" || nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 & | |
| 74 | else | |
| 75 | nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 & | |
| 76 | disown || true | |
| 77 | fi | |
| 78 | exit 0 | |
| 79 | fi | |
| 80 | ||
| 81 | # Everything below runs in the detached process. | |
| 82 | mkdir -p "$(dirname "$LOG")" 2>/dev/null || true | |
| 83 | touch "$LOG" 2>/dev/null || true | |
| 84 | ||
| 85 | log "==> gluecron self-deploy starting (pid $$)" | |
| 86 | ||
| 87 | # ── 1. Source env ────────────────────────────────────────────────────────── | |
| 88 | if [ -f "$ENV_FILE" ]; then | |
| 89 | set -a | |
| 90 | # shellcheck disable=SC1090 | |
| 91 | source "$ENV_FILE" | |
| 92 | set +a | |
| 93 | log " v sourced $ENV_FILE" | |
| 94 | else | |
| 95 | log " ! $ENV_FILE not found — relying on inherited env" | |
| 96 | fi | |
| 97 | ||
| 98 | cd "$WORKING_DIR" | |
| 99 | ||
| 100 | # ── 2. Capture pre-deploy SHA for rollback ──────────────────────────────── | |
| 101 | PREV_SHA="$(git rev-parse HEAD 2>/dev/null || echo '')" | |
| 102 | log " v previous SHA: $PREV_SHA" | |
| 103 | ||
| 104 | # ── 3. Pull latest main ──────────────────────────────────────────────────── | |
| 105 | GP_START=$(date +%s) | |
| 106 | notify_step "git-pull" "in_progress" | |
| 107 | git fetch --prune origin main 2>&1 | tee -a "$LOG" | |
| 108 | git reset --hard origin/main 2>&1 | tee -a "$LOG" | |
| 109 | NEW_SHA="$(git rev-parse HEAD)" | |
| 110 | RUN_ID="self-${NEW_SHA:0:12}-$(date +%s)" | |
| 111 | log " v pulled to $NEW_SHA (run_id=$RUN_ID)" | |
| 112 | notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))" | |
| 113 | ||
| 114 | # ── 3.5 Notify deploy started (now that we have NEW_SHA + RUN_ID) ───────── | |
| 115 | if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then | |
| 116 | curl --silent --show-error --max-time 10 \ | |
| 117 | -X POST "$APP_BASE_URL/api/events/deploy/started" \ | |
| 118 | -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \ | |
| 119 | -H "content-type: application/json" \ | |
| 120 | --data "{\"sha\":\"$NEW_SHA\",\"run_id\":\"$RUN_ID\",\"source\":\"self-deploy\"}" \ | |
| 121 | >>"$LOG" 2>&1 || log " ! deploy/started notify failed (non-fatal)" | |
| 122 | fi | |
| 123 | ||
| 124 | START_EPOCH=$(date +%s) | |
| 125 | DEPLOY_FAILED=0 | |
| 126 | FAIL_REASON="" | |
| 127 | ||
| 128 | # ── 4. bun install --frozen-lockfile ─────────────────────────────────────── | |
| 129 | BI_START=$(date +%s) | |
| 130 | notify_step "bun-install" "in_progress" | |
| 131 | if "$BUN" install --frozen-lockfile >>"$LOG" 2>&1; then | |
| 132 | log " v bun install ok" | |
| 133 | notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))" | |
| 134 | else | |
| 135 | log " x bun install FAILED" | |
| 136 | DEPLOY_FAILED=1 | |
| 137 | FAIL_REASON="bun install failed" | |
| 138 | notify_step "bun-install" "failed" "$(( ( $(date +%s) - BI_START ) * 1000 ))" | |
| 139 | fi | |
| 140 | ||
| 141 | # ── 5. DB migrations (fail loud) ─────────────────────────────────────────── | |
| 142 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 143 | DM_START=$(date +%s) | |
| 144 | notify_step "db-migrate" "in_progress" | |
| 145 | if "$BUN" run src/db/migrate.ts >>"$LOG" 2>&1; then | |
| 146 | log " v db migrate ok" | |
| 147 | notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))" | |
| 148 | else | |
| 149 | log " x db migrate FAILED" | |
| 150 | DEPLOY_FAILED=1 | |
| 151 | FAIL_REASON="bun run db:migrate failed" | |
| 152 | notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))" | |
| 153 | fi | |
| 154 | fi | |
| 155 | ||
| 156 | # ── 6. Build the static binary ───────────────────────────────────────────── | |
| 157 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 158 | BD_START=$(date +%s) | |
| 159 | notify_step "build" "in_progress" | |
| 160 | mkdir -p .next | |
| 161 | COMPILED=.next/gluecron-server | |
| 162 | COMPILED_TMP=.next/gluecron-server.new | |
| 163 | if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts >>"$LOG" 2>&1; then | |
| 164 | mv -f "$COMPILED_TMP" "$COMPILED" | |
| 165 | chmod +x "$COMPILED" | |
| 166 | log " v compiled $COMPILED" | |
| 167 | notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))" | |
| 168 | else | |
| 169 | rm -f "$COMPILED_TMP" | |
| 170 | log " ! bun build --compile failed — systemd will fall back to bun run" | |
| 171 | notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))" | |
| 172 | fi | |
| 173 | fi | |
| 174 | ||
| f85b88a | 175 | # ── 6.5 Pin BUILD_SHA into the systemd unit so the running process can |
| 176 | # report it and the SW versioning rotates exactly per-deploy. Drop-in | |
| 177 | # survives daemon-reload; the OLD file is overwritten on every deploy. | |
| 178 | # Without this, src/routes/pwa.ts falls back to a stable "dev-stable" | |
| 179 | # string and the browser SW never invalidates between real deploys. | |
| 180 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 181 | DROPIN_DIR=/etc/systemd/system/gluecron.service.d | |
| 182 | mkdir -p "$DROPIN_DIR" | |
| 183 | cat > "$DROPIN_DIR/build-sha.conf" <<EOF | |
| 184 | [Service] | |
| 185 | Environment="BUILD_SHA=$NEW_SHA" | |
| 186 | Environment="BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)" | |
| 187 | EOF | |
| 188 | systemctl daemon-reload >>"$LOG" 2>&1 || log " ! daemon-reload non-fatal warning" | |
| 189 | log " v pinned BUILD_SHA=${NEW_SHA:0:12} in systemd drop-in" | |
| 190 | fi | |
| 191 | ||
| f2c00b4 | 192 | # ── 7. systemctl restart (blocks on sd_notify READY=1) ──────────────────── |
| 193 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 194 | RS_START=$(date +%s) | |
| 195 | notify_step "restart-service" "in_progress" | |
| 196 | if systemctl restart gluecron >>"$LOG" 2>&1; then | |
| 197 | log " v systemctl restart gluecron ok" | |
| 198 | notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))" | |
| 199 | else | |
| 200 | log " x systemctl restart FAILED" | |
| 201 | DEPLOY_FAILED=1 | |
| 202 | FAIL_REASON="systemctl restart failed" | |
| 203 | notify_step "restart-service" "failed" "$(( ( $(date +%s) - RS_START ) * 1000 ))" | |
| 204 | fi | |
| 205 | fi | |
| 206 | ||
| 207 | # ── 8. Wait for /healthz to be green (up to 30s) ────────────────────────── | |
| 208 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 209 | HZ_START=$(date +%s) | |
| 210 | notify_step "healthz" "in_progress" | |
| 211 | green=0 | |
| 212 | for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do | |
| 213 | code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" || echo "000") | |
| 214 | log " healthz attempt $i: $code" | |
| 215 | if [ "$code" = "200" ]; then green=1; break; fi | |
| 216 | sleep 2 | |
| 217 | done | |
| 218 | if [ "$green" = "1" ]; then | |
| 219 | log " v /healthz green" | |
| 220 | notify_step "healthz" "succeeded" "$(( ( $(date +%s) - HZ_START ) * 1000 ))" | |
| 221 | else | |
| 222 | log " x /healthz did not return 200 within 30s" | |
| 223 | DEPLOY_FAILED=1 | |
| 224 | FAIL_REASON="/healthz timeout" | |
| 225 | notify_step "healthz" "failed" "$(( ( $(date +%s) - HZ_START ) * 1000 ))" | |
| 226 | fi | |
| 227 | fi | |
| 228 | ||
| 229 | # ── 9. Post-deploy smoke suite ──────────────────────────────────────────── | |
| 230 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 231 | PS_START=$(date +%s) | |
| 232 | notify_step "full-smoke" "in_progress" | |
| 233 | export GLUECRON_HOST="http://localhost:${PORT}" | |
| 234 | if "$BUN" run scripts/post-deploy-smoke.ts >>"$LOG" 2>&1; then | |
| 235 | log " v post-deploy smoke green" | |
| 236 | notify_step "full-smoke" "succeeded" "$(( ( $(date +%s) - PS_START ) * 1000 ))" | |
| 237 | else | |
| 238 | log " x post-deploy smoke FAILED" | |
| 239 | DEPLOY_FAILED=1 | |
| 240 | FAIL_REASON="post-deploy smoke failed" | |
| 241 | notify_step "full-smoke" "failed" "$(( ( $(date +%s) - PS_START ) * 1000 ))" | |
| 242 | fi | |
| 243 | fi | |
| 244 | ||
| 245 | # ── 10. Rollback on failure ─────────────────────────────────────────────── | |
| 246 | if [ "$DEPLOY_FAILED" = "1" ] && [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$NEW_SHA" ]; then | |
| 247 | notify_step "rollback" "in_progress" | |
| 248 | log " ! rolling back to $PREV_SHA (reason: $FAIL_REASON)" | |
| 249 | git reset --hard "$PREV_SHA" >>"$LOG" 2>&1 || true | |
| 250 | systemctl restart gluecron >>"$LOG" 2>&1 || true | |
| 251 | sleep 3 | |
| 252 | rb_green=0 | |
| 253 | for i in 1 2 3; do | |
| 254 | code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" || echo "000") | |
| 255 | log " rollback healthz attempt $i: $code" | |
| 256 | if [ "$code" = "200" ]; then rb_green=1; break; fi | |
| 257 | sleep 2 | |
| 258 | done | |
| 259 | if [ "$rb_green" = "1" ]; then | |
| 260 | notify_step "rollback" "succeeded" | |
| 261 | log " v rollback green" | |
| 262 | else | |
| 263 | notify_step "rollback" "failed" | |
| 264 | log " x ROLLBACK FAILED — human intervention required" | |
| 265 | fi | |
| 266 | fi | |
| 267 | ||
| 268 | # ── 11. Notify deploy finished ──────────────────────────────────────────── | |
| 269 | DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 )) | |
| 270 | if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then | |
| 271 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 272 | curl --silent --show-error --max-time 10 \ | |
| 273 | -X POST "$APP_BASE_URL/api/events/deploy/finished" \ | |
| 274 | -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \ | |
| 275 | -H "content-type: application/json" \ | |
| 276 | --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \ | |
| 277 | >>"$LOG" 2>&1 || true | |
| 278 | else | |
| 279 | ERR_PAYLOAD="$(printf '%s' "${FAIL_REASON:-deploy failed}" | head -c 512)" | |
| 280 | if command -v jq >/dev/null 2>&1; then | |
| 281 | ERR_JSON=$(printf '%s' "$ERR_PAYLOAD" | jq -Rs '.') | |
| 282 | else | |
| 283 | ERR_JSON="\"${ERR_PAYLOAD//\"/\\\"}\"" | |
| 284 | fi | |
| 285 | curl --silent --show-error --max-time 10 \ | |
| 286 | -X POST "$APP_BASE_URL/api/events/deploy/finished" \ | |
| 287 | -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \ | |
| 288 | -H "content-type: application/json" \ | |
| 289 | --data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \ | |
| 290 | >>"$LOG" 2>&1 || true | |
| 291 | fi | |
| 292 | fi | |
| 293 | ||
| 294 | if [ "$DEPLOY_FAILED" = "0" ]; then | |
| 295 | log "==> gluecron self-deploy SUCCESS in ${DUR_MS}ms (sha=$NEW_SHA)" | |
| 296 | exit 0 | |
| 297 | else | |
| 298 | log "==> gluecron self-deploy FAILED in ${DUR_MS}ms (reason=$FAIL_REASON)" | |
| 299 | exit 1 | |
| 300 | fi |