Blame · Line-by-line history

`self-deploy.sh`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

self-deploy.shBlame278 lines · 1 contributor

Normal view Raw

f2c00b4CC LABS App	1	#!/usr/bin/env bash
	2	# =============================================================================
	3	# BLOCK W — Gluecron self-deploy.
	4	#
	5	# Fired by:
	6	# - src/hooks/post-receive.ts when SELF_HOST_REPO matches the pushed repo
	7	# AND the ref is refs/heads/main
	8	# - the bare repo's hooks/post-receive (for SSH receive-pack)
	9	# - the optional .gluecron/workflows/deploy.yml on the workflow runner
	10	#
	11	# Contract:
	12	# - Detaches into the background via systemd-run (or nohup fallback) so
	13	# the caller's git push returns in ~1 second
	14	# - Logs every step to /var/log/gluecron-self-deploy.log
	15	# - Notifies /api/events/deploy/{started,step,finished} so /admin/deploys
	16	# streams the live timeline (same wire as the GitHub Actions workflow)
	17	# - Rolls back via git reflog if the post-deploy smoke fails
	18	#
	19	# Operator invariants:
	20	# - /etc/gluecron.env is the source of env truth (DATABASE_URL,
	21	# DEPLOY_EVENT_TOKEN, APP_BASE_URL, ANTHROPIC_API_KEY, …)
	22	# - /opt/gluecron is the working tree on the box, with git remote `origin`
	23	# pointing at https://gluecron.com/<owner>/<repo>.git (NOT GitHub)
	24	# - /opt/gluecron/.next/gluecron-server is the compiled Bun binary
	25	# - systemd unit `gluecron` is Type=notify and ExecStart=$EXEC_START
	26	#
	27	# TODO(ops): configure /etc/logrotate.d/gluecron-self-deploy for the log.
	28	# =============================================================================
	29
	30	set -euo pipefail
	31
	32	WORKING_DIR="${GLUECRON_WORKING_DIR:-/opt/gluecron}"
	33	LOG="${GLUECRON_SELF_DEPLOY_LOG:-/var/log/gluecron-self-deploy.log}"
	34	ENV_FILE="${GLUECRON_ENV_FILE:-/etc/gluecron.env}"
	35	BUN="${GLUECRON_BUN:-/root/.bun/bin/bun}"
	36	HEALTHZ_URL="${GLUECRON_HEALTHZ_URL:-http://localhost:3010/healthz}"
	37	PORT="${GLUECRON_PORT:-3010}"
	38	DETACHED_FLAG="${1:-}"
	39
	40	# ── helpers ────────────────────────────────────────────────────────────────
	41	ts() { date +'%Y-%m-%dT%H:%M:%S%z'; }
	42	log() { echo "[$(ts)] $*" \| tee -a "$LOG" >&2; }
	43
	44	notify_step() {
	45	local NAME="$1" STATUS="$2" DUR="${3:-}"
	46	if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] \|\| [ -z "${APP_BASE_URL:-}" ]; then
	47	return 0
	48	fi
	49	local DUR_FIELD=""
	50	if [ -n "$DUR" ]; then DUR_FIELD=",\"duration_ms\":$DUR"; fi
	51	curl --silent --show-error --max-time 5 \
	52	-X POST "$APP_BASE_URL/api/events/deploy/step" \
	53	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	54	-H "content-type: application/json" \
	55	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
	56	>/dev/null 2>&1 \|\| true
	57	}
	58
	59	# ── re-exec into the background unless already detached ──────────────────
	60	# When the post-receive hook calls this script, git push is blocked on the
	61	# child's stdout/stderr. We re-exec ourselves through systemd-run so the
	62	# original SSH/HTTP receive-pack process can return immediately.
	63	if [ "$DETACHED_FLAG" != "--inline" ] && [ -z "${GLUECRON_SELF_DEPLOY_DETACHED:-}" ]; then
	64	export GLUECRON_SELF_DEPLOY_DETACHED=1
	65	if command -v systemd-run >/dev/null 2>&1; then
	66	systemd-run --quiet --unit="gluecron-self-deploy-$(date +%s)" \
	67	--collect --no-block \
	68	bash "$0" --inline "$@" \|\| nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	69	else
	70	nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	71	disown \|\| true
	72	fi
	73	exit 0
	74	fi
	75
	76	# Everything below runs in the detached process.
	77	mkdir -p "$(dirname "$LOG")" 2>/dev/null \|\| true
	78	touch "$LOG" 2>/dev/null \|\| true
	79
	80	log "==> gluecron self-deploy starting (pid $$)"
	81
	82	# ── 1. Source env ──────────────────────────────────────────────────────────
	83	if [ -f "$ENV_FILE" ]; then
	84	set -a
	85	# shellcheck disable=SC1090
	86	source "$ENV_FILE"
	87	set +a
	88	log " v sourced $ENV_FILE"
	89	else
	90	log " ! $ENV_FILE not found — relying on inherited env"
	91	fi
	92
	93	cd "$WORKING_DIR"
	94
	95	# ── 2. Capture pre-deploy SHA for rollback ────────────────────────────────
	96	PREV_SHA="$(git rev-parse HEAD 2>/dev/null \|\| echo '')"
	97	log " v previous SHA: $PREV_SHA"
	98
	99	# ── 3. Pull latest main ────────────────────────────────────────────────────
	100	GP_START=$(date +%s)
	101	notify_step "git-pull" "in_progress"
	102	git fetch --prune origin main 2>&1 \| tee -a "$LOG"
	103	git reset --hard origin/main 2>&1 \| tee -a "$LOG"
	104	NEW_SHA="$(git rev-parse HEAD)"
	105	RUN_ID="self-${NEW_SHA:0:12}-$(date +%s)"
	106	log " v pulled to $NEW_SHA (run_id=$RUN_ID)"
	107	notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"
	108
	109	# ── 3.5 Notify deploy started (now that we have NEW_SHA + RUN_ID) ─────────
	110	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
	111	curl --silent --show-error --max-time 10 \
	112	-X POST "$APP_BASE_URL/api/events/deploy/started" \
	113	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	114	-H "content-type: application/json" \
	115	--data "{\"sha\":\"$NEW_SHA\",\"run_id\":\"$RUN_ID\",\"source\":\"self-deploy\"}" \
	116	>>"$LOG" 2>&1 \|\| log " ! deploy/started notify failed (non-fatal)"
	117	fi
	118
	119	START_EPOCH=$(date +%s)
	120	DEPLOY_FAILED=0
	121	FAIL_REASON=""
	122
	123	# ── 4. bun install --frozen-lockfile ───────────────────────────────────────
	124	BI_START=$(date +%s)
	125	notify_step "bun-install" "in_progress"
	126	if "$BUN" install --frozen-lockfile >>"$LOG" 2>&1; then
	127	log " v bun install ok"
	128	notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	129	else
	130	log " x bun install FAILED"
	131	DEPLOY_FAILED=1
	132	FAIL_REASON="bun install failed"
	133	notify_step "bun-install" "failed" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	134	fi
	135
	136	# ── 5. DB migrations (fail loud) ───────────────────────────────────────────
	137	if [ "$DEPLOY_FAILED" = "0" ]; then
	138	DM_START=$(date +%s)
	139	notify_step "db-migrate" "in_progress"
	140	if "$BUN" run src/db/migrate.ts >>"$LOG" 2>&1; then
	141	log " v db migrate ok"
	142	notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	143	else
	144	log " x db migrate FAILED"
	145	DEPLOY_FAILED=1
	146	FAIL_REASON="bun run db:migrate failed"
	147	notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	148	fi
	149	fi
	150
	151	# ── 6. Build the static binary ─────────────────────────────────────────────
	152	if [ "$DEPLOY_FAILED" = "0" ]; then
	153	BD_START=$(date +%s)
	154	notify_step "build" "in_progress"
	155	mkdir -p .next
	156	COMPILED=.next/gluecron-server
	157	COMPILED_TMP=.next/gluecron-server.new
	158	if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts >>"$LOG" 2>&1; then
	159	mv -f "$COMPILED_TMP" "$COMPILED"
	160	chmod +x "$COMPILED"
	161	log " v compiled $COMPILED"
	162	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	163	else
	164	rm -f "$COMPILED_TMP"
	165	log " ! bun build --compile failed — systemd will fall back to bun run"
	166	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	167	fi
	168	fi
	169
	170	# ── 7. systemctl restart (blocks on sd_notify READY=1) ────────────────────
	171	if [ "$DEPLOY_FAILED" = "0" ]; then
	172	RS_START=$(date +%s)
	173	notify_step "restart-service" "in_progress"
	174	if systemctl restart gluecron >>"$LOG" 2>&1; then
	175	log " v systemctl restart gluecron ok"
	176	notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	177	else
	178	log " x systemctl restart FAILED"
	179	DEPLOY_FAILED=1
	180	FAIL_REASON="systemctl restart failed"
	181	notify_step "restart-service" "failed" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	182	fi
	183	fi
	184
	185	# ── 8. Wait for /healthz to be green (up to 30s) ──────────────────────────
	186	if [ "$DEPLOY_FAILED" = "0" ]; then
	187	HZ_START=$(date +%s)
	188	notify_step "healthz" "in_progress"
	189	green=0
	190	for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
	191	code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" \|\| echo "000")
	192	log " healthz attempt $i: $code"
	193	if [ "$code" = "200" ]; then green=1; break; fi
	194	sleep 2
	195	done
	196	if [ "$green" = "1" ]; then
	197	log " v /healthz green"
	198	notify_step "healthz" "succeeded" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	199	else
	200	log " x /healthz did not return 200 within 30s"
	201	DEPLOY_FAILED=1
	202	FAIL_REASON="/healthz timeout"
	203	notify_step "healthz" "failed" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	204	fi
	205	fi
	206
	207	# ── 9. Post-deploy smoke suite ────────────────────────────────────────────
	208	if [ "$DEPLOY_FAILED" = "0" ]; then
	209	PS_START=$(date +%s)
	210	notify_step "full-smoke" "in_progress"
	211	export GLUECRON_HOST="http://localhost:${PORT}"
	212	if "$BUN" run scripts/post-deploy-smoke.ts >>"$LOG" 2>&1; then
	213	log " v post-deploy smoke green"
	214	notify_step "full-smoke" "succeeded" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	215	else
	216	log " x post-deploy smoke FAILED"
	217	DEPLOY_FAILED=1
	218	FAIL_REASON="post-deploy smoke failed"
	219	notify_step "full-smoke" "failed" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	220	fi
	221	fi
	222
	223	# ── 10. Rollback on failure ───────────────────────────────────────────────
	224	if [ "$DEPLOY_FAILED" = "1" ] && [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$NEW_SHA" ]; then
	225	notify_step "rollback" "in_progress"
	226	log " ! rolling back to $PREV_SHA (reason: $FAIL_REASON)"
	227	git reset --hard "$PREV_SHA" >>"$LOG" 2>&1 \|\| true
	228	systemctl restart gluecron >>"$LOG" 2>&1 \|\| true
	229	sleep 3
	230	rb_green=0
	231	for i in 1 2 3; do
	232	code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" \|\| echo "000")
	233	log " rollback healthz attempt $i: $code"
	234	if [ "$code" = "200" ]; then rb_green=1; break; fi
	235	sleep 2
	236	done
	237	if [ "$rb_green" = "1" ]; then
	238	notify_step "rollback" "succeeded"
	239	log " v rollback green"
	240	else
	241	notify_step "rollback" "failed"
	242	log " x ROLLBACK FAILED — human intervention required"
	243	fi
	244	fi
	245
	246	# ── 11. Notify deploy finished ────────────────────────────────────────────
	247	DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
	248	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
	249	if [ "$DEPLOY_FAILED" = "0" ]; then
	250	curl --silent --show-error --max-time 10 \
	251	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	252	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	253	-H "content-type: application/json" \
	254	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
	255	>>"$LOG" 2>&1 \|\| true
	256	else
	257	ERR_PAYLOAD="$(printf '%s' "${FAIL_REASON:-deploy failed}" \| head -c 512)"
	258	if command -v jq >/dev/null 2>&1; then
	259	ERR_JSON=$(printf '%s' "$ERR_PAYLOAD" \| jq -Rs '.')
	260	else
	261	ERR_JSON="\"${ERR_PAYLOAD//\"/\\\"}\""
	262	fi
	263	curl --silent --show-error --max-time 10 \
	264	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	265	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	266	-H "content-type: application/json" \
	267	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
	268	>>"$LOG" 2>&1 \|\| true
	269	fi
	270	fi
	271
	272	if [ "$DEPLOY_FAILED" = "0" ]; then
	273	log "==> gluecron self-deploy SUCCESS in ${DUR_MS}ms (sha=$NEW_SHA)"
	274	exit 0
	275	else
	276	log "==> gluecron self-deploy FAILED in ${DUR_MS}ms (reason=$FAIL_REASON)"
	277	exit 1
	278	fi