Blame · Line-by-line history

`self-deploy.sh`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

self-deploy.shBlame295 lines · 2 contributors

Normal view Raw

f2c00b4CC LABS App	1	#!/usr/bin/env bash
	2	# =============================================================================
	3	# BLOCK W — Gluecron self-deploy.
	4	#
	5	# Fired by:
	6	# - src/hooks/post-receive.ts when SELF_HOST_REPO matches the pushed repo
	7	# AND the ref is refs/heads/main
	8	# - the bare repo's hooks/post-receive (for SSH receive-pack)
	9	# - the optional .gluecron/workflows/deploy.yml on the workflow runner
	10	#
	11	# Contract:
	12	# - Detaches into the background via systemd-run (or nohup fallback) so
	13	# the caller's git push returns in ~1 second
	14	# - Logs every step to /var/log/gluecron-self-deploy.log
	15	# - Notifies /api/events/deploy/{started,step,finished} so /admin/deploys
	16	# streams the live timeline (same wire as the GitHub Actions workflow)
	17	# - Rolls back via git reflog if the post-deploy smoke fails
	18	#
	19	# Operator invariants:
	20	# - /etc/gluecron.env is the source of env truth (DATABASE_URL,
	21	# DEPLOY_EVENT_TOKEN, APP_BASE_URL, ANTHROPIC_API_KEY, …)
	22	# - /opt/gluecron is the working tree on the box, with git remote `origin`
	23	# pointing at https://gluecron.com/<owner>/<repo>.git (NOT GitHub)
	24	# - /opt/gluecron/.next/gluecron-server is the compiled Bun binary
	25	# - systemd unit `gluecron` is Type=notify and ExecStart=$EXEC_START
	26	#
	27	# TODO(ops): configure /etc/logrotate.d/gluecron-self-deploy for the log.
	28	# =============================================================================
	29
	30	set -euo pipefail
	31
	32	WORKING_DIR="${GLUECRON_WORKING_DIR:-/opt/gluecron}"
	33	LOG="${GLUECRON_SELF_DEPLOY_LOG:-/var/log/gluecron-self-deploy.log}"
	34	ENV_FILE="${GLUECRON_ENV_FILE:-/etc/gluecron.env}"
	35	BUN="${GLUECRON_BUN:-/root/.bun/bin/bun}"
	36	HEALTHZ_URL="${GLUECRON_HEALTHZ_URL:-http://localhost:3010/healthz}"
	37	PORT="${GLUECRON_PORT:-3010}"
	38	DETACHED_FLAG="${1:-}"
	39
	40	# ── helpers ────────────────────────────────────────────────────────────────
	41	ts() { date +'%Y-%m-%dT%H:%M:%S%z'; }
	42	log() { echo "[$(ts)] $*" \| tee -a "$LOG" >&2; }
	43
	44	notify_step() {
	45	local NAME="$1" STATUS="$2" DUR="${3:-}"
	46	if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] \|\| [ -z "${APP_BASE_URL:-}" ]; then
	47	return 0
	48	fi
	49	local DUR_FIELD=""
	50	if [ -n "$DUR" ]; then DUR_FIELD=",\"duration_ms\":$DUR"; fi
	51	curl --silent --show-error --max-time 5 \
	52	-X POST "$APP_BASE_URL/api/events/deploy/step" \
	53	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	54	-H "content-type: application/json" \
	55	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
	56	>/dev/null 2>&1 \|\| true
	57	}
	58
	59	# ── re-exec into the background unless already detached ──────────────────
	60	# When the post-receive hook calls this script, git push is blocked on the
	61	# child's stdout/stderr. We re-exec ourselves through systemd-run so the
	62	# original SSH/HTTP receive-pack process can return immediately.
	63	if [ "$DETACHED_FLAG" != "--inline" ] && [ -z "${GLUECRON_SELF_DEPLOY_DETACHED:-}" ]; then
	64	export GLUECRON_SELF_DEPLOY_DETACHED=1
	65	if command -v systemd-run >/dev/null 2>&1; then
	66	systemd-run --quiet --unit="gluecron-self-deploy-$(date +%s)" \
	67	--collect --no-block \
	68	bash "$0" --inline "$@" \|\| nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	69	else
	70	nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	71	disown \|\| true
	72	fi
	73	exit 0
	74	fi
	75
	76	# Everything below runs in the detached process.
	77	mkdir -p "$(dirname "$LOG")" 2>/dev/null \|\| true
	78	touch "$LOG" 2>/dev/null \|\| true
	79
	80	log "==> gluecron self-deploy starting (pid $$)"
	81
	82	# ── 1. Source env ──────────────────────────────────────────────────────────
	83	if [ -f "$ENV_FILE" ]; then
	84	set -a
	85	# shellcheck disable=SC1090
	86	source "$ENV_FILE"
	87	set +a
	88	log " v sourced $ENV_FILE"
	89	else
	90	log " ! $ENV_FILE not found — relying on inherited env"
	91	fi
	92
	93	cd "$WORKING_DIR"
	94
	95	# ── 2. Capture pre-deploy SHA for rollback ────────────────────────────────
	96	PREV_SHA="$(git rev-parse HEAD 2>/dev/null \|\| echo '')"
	97	log " v previous SHA: $PREV_SHA"
	98
	99	# ── 3. Pull latest main ────────────────────────────────────────────────────
	100	GP_START=$(date +%s)
	101	notify_step "git-pull" "in_progress"
	102	git fetch --prune origin main 2>&1 \| tee -a "$LOG"
	103	git reset --hard origin/main 2>&1 \| tee -a "$LOG"
	104	NEW_SHA="$(git rev-parse HEAD)"
	105	RUN_ID="self-${NEW_SHA:0:12}-$(date +%s)"
	106	log " v pulled to $NEW_SHA (run_id=$RUN_ID)"
	107	notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"
	108
	109	# ── 3.5 Notify deploy started (now that we have NEW_SHA + RUN_ID) ─────────
	110	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
	111	curl --silent --show-error --max-time 10 \
	112	-X POST "$APP_BASE_URL/api/events/deploy/started" \
	113	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	114	-H "content-type: application/json" \
	115	--data "{\"sha\":\"$NEW_SHA\",\"run_id\":\"$RUN_ID\",\"source\":\"self-deploy\"}" \
	116	>>"$LOG" 2>&1 \|\| log " ! deploy/started notify failed (non-fatal)"
	117	fi
	118
	119	START_EPOCH=$(date +%s)
	120	DEPLOY_FAILED=0
	121	FAIL_REASON=""
	122
	123	# ── 4. bun install --frozen-lockfile ───────────────────────────────────────
	124	BI_START=$(date +%s)
	125	notify_step "bun-install" "in_progress"
	126	if "$BUN" install --frozen-lockfile >>"$LOG" 2>&1; then
	127	log " v bun install ok"
	128	notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	129	else
	130	log " x bun install FAILED"
	131	DEPLOY_FAILED=1
	132	FAIL_REASON="bun install failed"
	133	notify_step "bun-install" "failed" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	134	fi
	135
	136	# ── 5. DB migrations (fail loud) ───────────────────────────────────────────
	137	if [ "$DEPLOY_FAILED" = "0" ]; then
	138	DM_START=$(date +%s)
	139	notify_step "db-migrate" "in_progress"
	140	if "$BUN" run src/db/migrate.ts >>"$LOG" 2>&1; then
	141	log " v db migrate ok"
	142	notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	143	else
	144	log " x db migrate FAILED"
	145	DEPLOY_FAILED=1
	146	FAIL_REASON="bun run db:migrate failed"
	147	notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	148	fi
	149	fi
	150
	151	# ── 6. Build the static binary ─────────────────────────────────────────────
	152	if [ "$DEPLOY_FAILED" = "0" ]; then
	153	BD_START=$(date +%s)
	154	notify_step "build" "in_progress"
	155	mkdir -p .next
	156	COMPILED=.next/gluecron-server
	157	COMPILED_TMP=.next/gluecron-server.new
	158	if "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts >>"$LOG" 2>&1; then
	159	mv -f "$COMPILED_TMP" "$COMPILED"
	160	chmod +x "$COMPILED"
	161	log " v compiled $COMPILED"
	162	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	163	else
	164	rm -f "$COMPILED_TMP"
	165	log " ! bun build --compile failed — systemd will fall back to bun run"
	166	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	167	fi
	168	fi
	169
f85b88aTest User	170	# ── 6.5 Pin BUILD_SHA into the systemd unit so the running process can
	171	# report it and the SW versioning rotates exactly per-deploy. Drop-in
	172	# survives daemon-reload; the OLD file is overwritten on every deploy.
	173	# Without this, src/routes/pwa.ts falls back to a stable "dev-stable"
	174	# string and the browser SW never invalidates between real deploys.
	175	if [ "$DEPLOY_FAILED" = "0" ]; then
	176	DROPIN_DIR=/etc/systemd/system/gluecron.service.d
	177	mkdir -p "$DROPIN_DIR"
	178	cat > "$DROPIN_DIR/build-sha.conf" <<EOF
	179	[Service]
	180	Environment="BUILD_SHA=$NEW_SHA"
	181	Environment="BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
	182	EOF
	183	systemctl daemon-reload >>"$LOG" 2>&1 \|\| log " ! daemon-reload non-fatal warning"
	184	log " v pinned BUILD_SHA=${NEW_SHA:0:12} in systemd drop-in"
	185	fi
	186
f2c00b4CC LABS App	187	# ── 7. systemctl restart (blocks on sd_notify READY=1) ────────────────────
	188	if [ "$DEPLOY_FAILED" = "0" ]; then
	189	RS_START=$(date +%s)
	190	notify_step "restart-service" "in_progress"
	191	if systemctl restart gluecron >>"$LOG" 2>&1; then
	192	log " v systemctl restart gluecron ok"
	193	notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	194	else
	195	log " x systemctl restart FAILED"
	196	DEPLOY_FAILED=1
	197	FAIL_REASON="systemctl restart failed"
	198	notify_step "restart-service" "failed" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	199	fi
	200	fi
	201
	202	# ── 8. Wait for /healthz to be green (up to 30s) ──────────────────────────
	203	if [ "$DEPLOY_FAILED" = "0" ]; then
	204	HZ_START=$(date +%s)
	205	notify_step "healthz" "in_progress"
	206	green=0
	207	for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
	208	code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" \|\| echo "000")
	209	log " healthz attempt $i: $code"
	210	if [ "$code" = "200" ]; then green=1; break; fi
	211	sleep 2
	212	done
	213	if [ "$green" = "1" ]; then
	214	log " v /healthz green"
	215	notify_step "healthz" "succeeded" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	216	else
	217	log " x /healthz did not return 200 within 30s"
	218	DEPLOY_FAILED=1
	219	FAIL_REASON="/healthz timeout"
	220	notify_step "healthz" "failed" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	221	fi
	222	fi
	223
	224	# ── 9. Post-deploy smoke suite ────────────────────────────────────────────
	225	if [ "$DEPLOY_FAILED" = "0" ]; then
	226	PS_START=$(date +%s)
	227	notify_step "full-smoke" "in_progress"
	228	export GLUECRON_HOST="http://localhost:${PORT}"
	229	if "$BUN" run scripts/post-deploy-smoke.ts >>"$LOG" 2>&1; then
	230	log " v post-deploy smoke green"
	231	notify_step "full-smoke" "succeeded" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	232	else
	233	log " x post-deploy smoke FAILED"
	234	DEPLOY_FAILED=1
	235	FAIL_REASON="post-deploy smoke failed"
	236	notify_step "full-smoke" "failed" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	237	fi
	238	fi
	239
	240	# ── 10. Rollback on failure ───────────────────────────────────────────────
	241	if [ "$DEPLOY_FAILED" = "1" ] && [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$NEW_SHA" ]; then
	242	notify_step "rollback" "in_progress"
	243	log " ! rolling back to $PREV_SHA (reason: $FAIL_REASON)"
	244	git reset --hard "$PREV_SHA" >>"$LOG" 2>&1 \|\| true
	245	systemctl restart gluecron >>"$LOG" 2>&1 \|\| true
	246	sleep 3
	247	rb_green=0
	248	for i in 1 2 3; do
	249	code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" \|\| echo "000")
	250	log " rollback healthz attempt $i: $code"
	251	if [ "$code" = "200" ]; then rb_green=1; break; fi
	252	sleep 2
	253	done
	254	if [ "$rb_green" = "1" ]; then
	255	notify_step "rollback" "succeeded"
	256	log " v rollback green"
	257	else
	258	notify_step "rollback" "failed"
	259	log " x ROLLBACK FAILED — human intervention required"
	260	fi
	261	fi
	262
	263	# ── 11. Notify deploy finished ────────────────────────────────────────────
	264	DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
	265	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
	266	if [ "$DEPLOY_FAILED" = "0" ]; then
	267	curl --silent --show-error --max-time 10 \
	268	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	269	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	270	-H "content-type: application/json" \
	271	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
	272	>>"$LOG" 2>&1 \|\| true
	273	else
	274	ERR_PAYLOAD="$(printf '%s' "${FAIL_REASON:-deploy failed}" \| head -c 512)"
	275	if command -v jq >/dev/null 2>&1; then
	276	ERR_JSON=$(printf '%s' "$ERR_PAYLOAD" \| jq -Rs '.')
	277	else
	278	ERR_JSON="\"${ERR_PAYLOAD//\"/\\\"}\""
	279	fi
	280	curl --silent --show-error --max-time 10 \
	281	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	282	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	283	-H "content-type: application/json" \
	284	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
	285	>>"$LOG" 2>&1 \|\| true
	286	fi
	287	fi
	288
	289	if [ "$DEPLOY_FAILED" = "0" ]; then
	290	log "==> gluecron self-deploy SUCCESS in ${DUR_MS}ms (sha=$NEW_SHA)"
	291	exit 0
	292	else
	293	log "==> gluecron self-deploy FAILED in ${DUR_MS}ms (reason=$FAIL_REASON)"
	294	exit 1
	295	fi