Blame · Line-by-line history

`self-deploy.sh`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

self-deploy.shBlame394 lines · 4 contributors

Normal view Raw

f2c00b4CC LABS App	1	#!/usr/bin/env bash
	2	# =============================================================================
	3	# BLOCK W — Gluecron self-deploy.
	4	#
	5	# Fired by:
	6	# - src/hooks/post-receive.ts when SELF_HOST_REPO matches the pushed repo
	7	# AND the ref is refs/heads/main
	8	# - the bare repo's hooks/post-receive (for SSH receive-pack)
	9	# - the optional .gluecron/workflows/deploy.yml on the workflow runner
	10	#
	11	# Contract:
	12	# - Detaches into the background via systemd-run (or nohup fallback) so
	13	# the caller's git push returns in ~1 second
	14	# - Logs every step to /var/log/gluecron-self-deploy.log
	15	# - Notifies /api/events/deploy/{started,step,finished} so /admin/deploys
	16	# streams the live timeline (same wire as the GitHub Actions workflow)
	17	# - Rolls back via git reflog if the post-deploy smoke fails
	18	#
	19	# Operator invariants:
	20	# - /etc/gluecron.env is the source of env truth (DATABASE_URL,
	21	# DEPLOY_EVENT_TOKEN, APP_BASE_URL, ANTHROPIC_API_KEY, …)
	22	# - /opt/gluecron is the working tree on the box, with git remote `origin`
	23	# pointing at https://gluecron.com/<owner>/<repo>.git (NOT GitHub)
	24	# - /opt/gluecron/.next/gluecron-server is the compiled Bun binary
	25	# - systemd unit `gluecron` is Type=notify and ExecStart=$EXEC_START
	26	#
	27	# TODO(ops): configure /etc/logrotate.d/gluecron-self-deploy for the log.
7de856fccantynz-alt	28	#
	29	# REACHABILITY (verified on the box 2026-07-29): this script is NOT currently
	30	# running. post-receive.ts only dispatches it when process.env.SELF_HOST_REPO
	31	# matches, and while SELF_HOST_REPO is set in the HOST env file it is absent
	32	# from the container the app actually runs in, so the hook never fires. No
	33	# systemd unit invokes it either. The live deploy path is
	34	# gluecron-update.timer -> scripts/auto-update.sh, which is what CLAUDE.md's
	35	# "post-receive fires self-deploy.sh" claim gets wrong. The log's last write
	36	# was 2026-07-13.
	37	#
	38	# It is kept rather than deleted because admin-diagnose and the runbook still
	39	# present it as the self-host path and it may be re-armed. That is exactly
	40	# why the xtrace holes below were worth closing: a dormant script is the one
	41	# most likely to be switched on without a fresh review.
f2c00b4CC LABS App	42	# =============================================================================
	43
ff4423bClaude	44	# `set -E` so traps propagate into subshells; `-x` traces every command
	45	# to stderr (captured into $LOG via the `>>"$LOG" 2>&1` redirects on the
	46	# detached re-exec line). Reliability sweep 2026-05-16: when this script
	47	# fails, the trace tells us EXACTLY which line broke instead of leaving
	48	# us guessing as we did for 17 hours of failed Hetzner deploys.
	49	set -Eeuxo pipefail
f2c00b4CC LABS App	50
	51	WORKING_DIR="${GLUECRON_WORKING_DIR:-/opt/gluecron}"
	52	LOG="${GLUECRON_SELF_DEPLOY_LOG:-/var/log/gluecron-self-deploy.log}"
	53	ENV_FILE="${GLUECRON_ENV_FILE:-/etc/gluecron.env}"
7de856fccantynz-alt	54
	55	# The log receives this script's stderr, which under `set -x` is a trace of
	56	# every command. Create it 0600 before anything writes: on the live box it
	57	# was found at 0644, i.e. readable by any local user. Best-effort — a
	58	# pre-existing log keeps its mode, so tightening that is an operator step.
	59	if [ ! -e "$LOG" ]; then
	60	( umask 077; : >"$LOG" ) 2>/dev/null \|\| true
	61	fi
	62	chmod 600 "$LOG" 2>/dev/null \|\| true
f2c00b4CC LABS App	63	BUN="${GLUECRON_BUN:-/root/.bun/bin/bun}"
	64	HEALTHZ_URL="${GLUECRON_HEALTHZ_URL:-http://localhost:3010/healthz}"
	65	PORT="${GLUECRON_PORT:-3010}"
	66	DETACHED_FLAG="${1:-}"
	67
	68	# ── helpers ────────────────────────────────────────────────────────────────
	69	ts() { date +'%Y-%m-%dT%H:%M:%S%z'; }
	70	log() { echo "[$(ts)] $*" \| tee -a "$LOG" >&2; }
	71
	72	notify_step() {
	73	local NAME="$1" STATUS="$2" DUR="${3:-}"
	74	if [ -z "${DEPLOY_EVENT_TOKEN:-}" ] \|\| [ -z "${APP_BASE_URL:-}" ]; then
	75	return 0
	76	fi
	77	local DUR_FIELD=""
	78	if [ -n "$DUR" ]; then DUR_FIELD=",\"duration_ms\":$DUR"; fi
7de856fccantynz-alt	79	# xtrace OFF: the bearer token is on the command line, so tracing this
	80	# would print DEPLOY_EVENT_TOKEN into $LOG on every single step.
	81	set +x
f2c00b4CC LABS App	82	curl --silent --show-error --max-time 5 \
	83	-X POST "$APP_BASE_URL/api/events/deploy/step" \
	84	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	85	-H "content-type: application/json" \
	86	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"step_name\":\"$NAME\",\"status\":\"$STATUS\"$DUR_FIELD}" \
	87	>/dev/null 2>&1 \|\| true
7de856fccantynz-alt	88	set -x
f2c00b4CC LABS App	89	}
	90
	91	# ── re-exec into the background unless already detached ──────────────────
	92	# When the post-receive hook calls this script, git push is blocked on the
	93	# child's stdout/stderr. We re-exec ourselves through systemd-run so the
	94	# original SSH/HTTP receive-pack process can return immediately.
	95	if [ "$DETACHED_FLAG" != "--inline" ] && [ -z "${GLUECRON_SELF_DEPLOY_DETACHED:-}" ]; then
	96	export GLUECRON_SELF_DEPLOY_DETACHED=1
	97	if command -v systemd-run >/dev/null 2>&1; then
	98	systemd-run --quiet --unit="gluecron-self-deploy-$(date +%s)" \
	99	--collect --no-block \
	100	bash "$0" --inline "$@" \|\| nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	101	else
	102	nohup bash "$0" --inline "$@" >>"$LOG" 2>&1 &
	103	disown \|\| true
	104	fi
	105	exit 0
	106	fi
	107
	108	# Everything below runs in the detached process.
	109	mkdir -p "$(dirname "$LOG")" 2>/dev/null \|\| true
	110	touch "$LOG" 2>/dev/null \|\| true
	111
	112	log "==> gluecron self-deploy starting (pid $$)"
	113
	114	# ── 1. Source env ──────────────────────────────────────────────────────────
	115	if [ -f "$ENV_FILE" ]; then
7de856fccantynz-alt	116	# xtrace OFF across the source. With `set -x` active, bash traces every
	117	# assignment the sourced file performs, so the entire env file — today
	118	# that includes GOOGLE_OAUTH_CLIENT_SECRET — would be echoed to stderr.
	119	# The header above is explicit that stderr is captured into $LOG by the
	120	# detached re-exec, and that log is world-readable, so tracing here writes
	121	# live credentials to disk for any local user to read.
	122	set +x
f2c00b4CC LABS App	123	set -a
	124	# shellcheck disable=SC1090
	125	source "$ENV_FILE"
	126	set +a
7de856fccantynz-alt	127	set -x
f2c00b4CC LABS App	128	log " v sourced $ENV_FILE"
	129	else
	130	log " ! $ENV_FILE not found — relying on inherited env"
	131	fi
	132
	133	cd "$WORKING_DIR"
	134
	135	# ── 2. Capture pre-deploy SHA for rollback ────────────────────────────────
	136	PREV_SHA="$(git rev-parse HEAD 2>/dev/null \|\| echo '')"
	137	log " v previous SHA: $PREV_SHA"
	138
	139	# ── 3. Pull latest main ────────────────────────────────────────────────────
	140	GP_START=$(date +%s)
	141	notify_step "git-pull" "in_progress"
	142	git fetch --prune origin main 2>&1 \| tee -a "$LOG"
	143	git reset --hard origin/main 2>&1 \| tee -a "$LOG"
	144	NEW_SHA="$(git rev-parse HEAD)"
	145	RUN_ID="self-${NEW_SHA:0:12}-$(date +%s)"
	146	log " v pulled to $NEW_SHA (run_id=$RUN_ID)"
	147	notify_step "git-pull" "succeeded" "$(( ( $(date +%s) - GP_START ) * 1000 ))"
	148
	149	# ── 3.5 Notify deploy started (now that we have NEW_SHA + RUN_ID) ─────────
	150	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
7de856fccantynz-alt	151	# xtrace OFF: bearer token on the command line, and this one redirects
	152	# straight into $LOG.
	153	set +x
f2c00b4CC LABS App	154	curl --silent --show-error --max-time 10 \
	155	-X POST "$APP_BASE_URL/api/events/deploy/started" \
	156	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	157	-H "content-type: application/json" \
	158	--data "{\"sha\":\"$NEW_SHA\",\"run_id\":\"$RUN_ID\",\"source\":\"self-deploy\"}" \
	159	>>"$LOG" 2>&1 \|\| log " ! deploy/started notify failed (non-fatal)"
7de856fccantynz-alt	160	set -x
f2c00b4CC LABS App	161	fi
	162
	163	START_EPOCH=$(date +%s)
	164	DEPLOY_FAILED=0
	165	FAIL_REASON=""
	166
141561fClaude	167	# ── 4. bun install --frozen-lockfile (skipped if lockfile unchanged) ──────
	168	# Track the last-installed lockfile hash in a stamp file. If it matches the
	169	# current lockfile we skip — saves ~2-5s every deploy that doesn't touch
	170	# dependencies (90%+ of deploys). On any error the fallback runs install
	171	# anyway so we never silently skip a needed install.
	172	STAMP_DIR="${GLUECRON_DEPLOY_STAMP_DIR:-.next}"
	173	mkdir -p "$STAMP_DIR" 2>/dev/null \|\| true
	174	LOCK_STAMP="$STAMP_DIR/bun-install.stamp"
	175	LOCK_HASH=""
	176	if [ -f bun.lockb ]; then
	177	LOCK_HASH=$(sha256sum bun.lockb 2>/dev/null \| awk '{print $1}' \|\| echo "")
	178	elif [ -f bun.lock ]; then
	179	LOCK_HASH=$(sha256sum bun.lock 2>/dev/null \| awk '{print $1}' \|\| echo "")
	180	fi
	181	LAST_HASH=""
	182	if [ -f "$LOCK_STAMP" ]; then
	183	LAST_HASH=$(cat "$LOCK_STAMP" 2>/dev/null \|\| echo "")
	184	fi
	185
f2c00b4CC LABS App	186	BI_START=$(date +%s)
	187	notify_step "bun-install" "in_progress"
141561fClaude	188	if [ -n "$LOCK_HASH" ] && [ "$LOCK_HASH" = "$LAST_HASH" ] && [ -d node_modules ]; then
	189	log " v bun install skipped (lockfile unchanged: ${LOCK_HASH:0:12})"
	190	notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	191	elif "$BUN" install --frozen-lockfile >>"$LOG" 2>&1; then
	192	echo "$LOCK_HASH" > "$LOCK_STAMP" 2>/dev/null \|\| true
f2c00b4CC LABS App	193	log " v bun install ok"
	194	notify_step "bun-install" "succeeded" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	195	else
	196	log " x bun install FAILED"
	197	DEPLOY_FAILED=1
	198	FAIL_REASON="bun install failed"
	199	notify_step "bun-install" "failed" "$(( ( $(date +%s) - BI_START ) * 1000 ))"
	200	fi
	201
	202	# ── 5. DB migrations (fail loud) ───────────────────────────────────────────
	203	if [ "$DEPLOY_FAILED" = "0" ]; then
	204	DM_START=$(date +%s)
	205	notify_step "db-migrate" "in_progress"
	206	if "$BUN" run src/db/migrate.ts >>"$LOG" 2>&1; then
	207	log " v db migrate ok"
	208	notify_step "db-migrate" "succeeded" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	209	else
	210	log " x db migrate FAILED"
	211	DEPLOY_FAILED=1
	212	FAIL_REASON="bun run db:migrate failed"
	213	notify_step "db-migrate" "failed" "$(( ( $(date +%s) - DM_START ) * 1000 ))"
	214	fi
	215	fi
	216
141561fClaude	217	# ── 6. Build the static binary (skipped if no source files changed) ──────
	218	# We hash every tracked src/*.ts(x) file and stash the digest in a stamp
	219	# file. If nothing in src/ changed, the binary from the previous deploy
	220	# is still valid — saves 5-15s on docs-only / config-only commits.
	221	# On any error the fallback rebuilds anyway.
f2c00b4CC LABS App	222	if [ "$DEPLOY_FAILED" = "0" ]; then
	223	BD_START=$(date +%s)
	224	notify_step "build" "in_progress"
	225	mkdir -p .next
	226	COMPILED=.next/gluecron-server
	227	COMPILED_TMP=.next/gluecron-server.new
141561fClaude	228	BUILD_STAMP="$STAMP_DIR/build-src.stamp"
	229	SRC_HASH=$(find src -type f $ -name ".ts" -o -name ".tsx" $ -print0 2>/dev/null \
	230	\| sort -z \
	231	\| xargs -0 sha256sum 2>/dev/null \
	232	\| sha256sum 2>/dev/null \
	233	\| awk '{print $1}' \|\| echo "")
	234	LAST_BUILD_HASH=""
	235	if [ -f "$BUILD_STAMP" ]; then
	236	LAST_BUILD_HASH=$(cat "$BUILD_STAMP" 2>/dev/null \|\| echo "")
	237	fi
	238
	239	if [ -n "$SRC_HASH" ] && [ "$SRC_HASH" = "$LAST_BUILD_HASH" ] && [ -x "$COMPILED" ]; then
	240	log " v build skipped (src/ unchanged: ${SRC_HASH:0:12})"
	241	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	242	elif "$BUN" build --compile --outfile "$COMPILED_TMP" src/index.ts >>"$LOG" 2>&1; then
f2c00b4CC LABS App	243	mv -f "$COMPILED_TMP" "$COMPILED"
	244	chmod +x "$COMPILED"
141561fClaude	245	echo "$SRC_HASH" > "$BUILD_STAMP" 2>/dev/null \|\| true
f2c00b4CC LABS App	246	log " v compiled $COMPILED"
	247	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	248	else
	249	rm -f "$COMPILED_TMP"
	250	log " ! bun build --compile failed — systemd will fall back to bun run"
	251	notify_step "build" "succeeded" "$(( ( $(date +%s) - BD_START ) * 1000 ))"
	252	fi
	253	fi
	254
f85b88aTest User	255	# ── 6.5 Pin BUILD_SHA into the systemd unit so the running process can
	256	# report it and the SW versioning rotates exactly per-deploy. Drop-in
	257	# survives daemon-reload; the OLD file is overwritten on every deploy.
	258	# Without this, src/routes/pwa.ts falls back to a stable "dev-stable"
	259	# string and the browser SW never invalidates between real deploys.
	260	if [ "$DEPLOY_FAILED" = "0" ]; then
	261	DROPIN_DIR=/etc/systemd/system/gluecron.service.d
	262	mkdir -p "$DROPIN_DIR"
	263	cat > "$DROPIN_DIR/build-sha.conf" <<EOF
	264	[Service]
	265	Environment="BUILD_SHA=$NEW_SHA"
	266	Environment="BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
	267	EOF
	268	systemctl daemon-reload >>"$LOG" 2>&1 \|\| log " ! daemon-reload non-fatal warning"
	269	log " v pinned BUILD_SHA=${NEW_SHA:0:12} in systemd drop-in"
	270	fi
	271
f2c00b4CC LABS App	272	# ── 7. systemctl restart (blocks on sd_notify READY=1) ────────────────────
	273	if [ "$DEPLOY_FAILED" = "0" ]; then
	274	RS_START=$(date +%s)
	275	notify_step "restart-service" "in_progress"
	276	if systemctl restart gluecron >>"$LOG" 2>&1; then
	277	log " v systemctl restart gluecron ok"
	278	notify_step "restart-service" "succeeded" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	279	else
	280	log " x systemctl restart FAILED"
	281	DEPLOY_FAILED=1
	282	FAIL_REASON="systemctl restart failed"
	283	notify_step "restart-service" "failed" "$(( ( $(date +%s) - RS_START ) * 1000 ))"
	284	fi
	285	fi
	286
141561fClaude	287	# ── 8. Wait for /healthz to be green (fast poll, up to ~30s) ─────────────
	288	# Tighter polling: 500ms × 10 (covers the common case where the service
	289	# is healthy within 1-2s of restart returning), then 2s × 10 for the slow
	290	# path. Cuts a typical successful deploy by 1-3s vs the old fixed 2s poll.
f2c00b4CC LABS App	291	if [ "$DEPLOY_FAILED" = "0" ]; then
	292	HZ_START=$(date +%s)
	293	notify_step "healthz" "in_progress"
	294	green=0
141561fClaude	295	for i in 1 2 3 4 5 6 7 8 9 10; do
	296	code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$HEALTHZ_URL" \|\| echo "000")
	297	if [ "$code" = "200" ]; then green=1; log " healthz fast attempt $i: $code (green)"; break; fi
	298	sleep 0.5
f2c00b4CC LABS App	299	done
141561fClaude	300	if [ "$green" = "0" ]; then
	301	for i in 11 12 13 14 15 16 17 18 19 20; do
	302	code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$HEALTHZ_URL" \|\| echo "000")
	303	log " healthz slow attempt $i: $code"
	304	if [ "$code" = "200" ]; then green=1; break; fi
	305	sleep 2
	306	done
	307	fi
f2c00b4CC LABS App	308	if [ "$green" = "1" ]; then
	309	log " v /healthz green"
	310	notify_step "healthz" "succeeded" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	311	else
	312	log " x /healthz did not return 200 within 30s"
	313	DEPLOY_FAILED=1
	314	FAIL_REASON="/healthz timeout"
	315	notify_step "healthz" "failed" "$(( ( $(date +%s) - HZ_START ) * 1000 ))"
	316	fi
	317	fi
	318
	319	# ── 9. Post-deploy smoke suite ────────────────────────────────────────────
	320	if [ "$DEPLOY_FAILED" = "0" ]; then
	321	PS_START=$(date +%s)
	322	notify_step "full-smoke" "in_progress"
	323	export GLUECRON_HOST="http://localhost:${PORT}"
	324	if "$BUN" run scripts/post-deploy-smoke.ts >>"$LOG" 2>&1; then
	325	log " v post-deploy smoke green"
	326	notify_step "full-smoke" "succeeded" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	327	else
	328	log " x post-deploy smoke FAILED"
	329	DEPLOY_FAILED=1
	330	FAIL_REASON="post-deploy smoke failed"
	331	notify_step "full-smoke" "failed" "$(( ( $(date +%s) - PS_START ) * 1000 ))"
	332	fi
	333	fi
	334
	335	# ── 10. Rollback on failure ───────────────────────────────────────────────
	336	if [ "$DEPLOY_FAILED" = "1" ] && [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$NEW_SHA" ]; then
	337	notify_step "rollback" "in_progress"
	338	log " ! rolling back to $PREV_SHA (reason: $FAIL_REASON)"
	339	git reset --hard "$PREV_SHA" >>"$LOG" 2>&1 \|\| true
	340	systemctl restart gluecron >>"$LOG" 2>&1 \|\| true
	341	sleep 3
	342	rb_green=0
	343	for i in 1 2 3; do
	344	code=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTHZ_URL" \|\| echo "000")
	345	log " rollback healthz attempt $i: $code"
	346	if [ "$code" = "200" ]; then rb_green=1; break; fi
	347	sleep 2
	348	done
	349	if [ "$rb_green" = "1" ]; then
	350	notify_step "rollback" "succeeded"
	351	log " v rollback green"
	352	else
	353	notify_step "rollback" "failed"
	354	log " x ROLLBACK FAILED — human intervention required"
	355	fi
	356	fi
	357
	358	# ── 11. Notify deploy finished ────────────────────────────────────────────
	359	DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
	360	if [ -n "${DEPLOY_EVENT_TOKEN:-}" ] && [ -n "${APP_BASE_URL:-}" ]; then
7de856fccantynz-alt	361	# xtrace OFF across both branches: bearer token on the command line, both
	362	# redirecting into $LOG.
	363	set +x
f2c00b4CC LABS App	364	if [ "$DEPLOY_FAILED" = "0" ]; then
	365	curl --silent --show-error --max-time 10 \
	366	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	367	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	368	-H "content-type: application/json" \
	369	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
	370	>>"$LOG" 2>&1 \|\| true
	371	else
	372	ERR_PAYLOAD="$(printf '%s' "${FAIL_REASON:-deploy failed}" \| head -c 512)"
	373	if command -v jq >/dev/null 2>&1; then
	374	ERR_JSON=$(printf '%s' "$ERR_PAYLOAD" \| jq -Rs '.')
	375	else
	376	ERR_JSON="\"${ERR_PAYLOAD//\"/\\\"}\""
	377	fi
	378	curl --silent --show-error --max-time 10 \
	379	-X POST "$APP_BASE_URL/api/events/deploy/finished" \
	380	-H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
	381	-H "content-type: application/json" \
	382	--data "{\"run_id\":\"$RUN_ID\",\"sha\":\"$NEW_SHA\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
	383	>>"$LOG" 2>&1 \|\| true
	384	fi
7de856fccantynz-alt	385	set -x
f2c00b4CC LABS App	386	fi
	387
	388	if [ "$DEPLOY_FAILED" = "0" ]; then
	389	log "==> gluecron self-deploy SUCCESS in ${DUR_MS}ms (sha=$NEW_SHA)"
	390	exit 0
	391	else
	392	log "==> gluecron self-deploy FAILED in ${DUR_MS}ms (reason=$FAIL_REASON)"
	393	exit 1
	394	fi