CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/usr/bin/env bash
#
# Restore drill — proves the latest production (Neon) backup actually restores.
# "A backup you have never restored is not a backup, it is a hope."
#
# Self-contained, zero external dependencies: spins up a THROWAWAY postgres:17
# container, pg_restores the newest dump into it, then compares key row counts
# against the live Neon database. Tears the container down unconditionally.
#
# Exit 0 = restore verified (counts match). Non-zero = drill FAILED — the
# backup is NOT trustworthy; investigate before relying on it.
#
# Usage: bash scripts/restore-drill.sh
set -uo pipefail
BACKUP_DIR="/opt/gluecron/backups"
APP_CONTAINER="gluecron-gluecron-1"
PG_IMAGE="postgres:17-alpine"
DRILL="gluecron-restore-drill-$$"
log() { echo "$(date -Is) [drill] $*"; }
cleanup() { docker rm -f "$DRILL" >/dev/null 2>&1 || true; }
trap cleanup EXIT
dump="$(ls -t "$BACKUP_DIR"/gluecron-db-*.dump 2>/dev/null | head -1)"
if [ -z "$dump" ]; then
log "FAIL: no backup found in $BACKUP_DIR"
exit 1
fi
log "restoring: $dump"
raw_url="$(docker exec "$APP_CONTAINER" printenv DATABASE_URL)"
clean_url="${raw_url%%DATABASE_URL=*}"
# Throwaway target instance.
docker run -d --name "$DRILL" -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill \
"$PG_IMAGE" >/dev/null
for _ in $(seq 1 30); do
docker exec "$DRILL" pg_isready -U postgres -d drill >/dev/null 2>&1 && break
sleep 1
done
docker cp "$dump" "$DRILL":/tmp/d.dump
# Ignore benign restore noise (extensions, comments); we judge by row counts.
docker exec "$DRILL" pg_restore --no-owner --no-privileges -U postgres -d drill /tmp/d.dump \
>/dev/null 2>&1 || true
fails=0
checked=0
for tbl in users repositories issues pull_requests oauth_apps; do
restored="$(docker exec "$DRILL" psql -At -U postgres -d drill \
-c "SELECT count(*) FROM $tbl" 2>/dev/null | tr -d '[:space:]')"
# $U/$TBL expand inside the container (host-side would be empty).
live="$(docker run --rm -e U="$clean_url" -e TBL="$tbl" "$PG_IMAGE" \
sh -c 'psql -At "$U" -c "SELECT count(*) FROM $TBL"' 2>/dev/null | tr -d '[:space:]')"
if [ -z "$restored" ] || [ -z "$live" ]; then
log "WARN: $tbl — could not read a count (restored='$restored' live='$live')"
fails=$((fails + 1))
continue
fi
checked=$((checked + 1))
if [ "$restored" = "$live" ]; then
log "OK $tbl: $restored rows (matches live)"
else
log "FAIL $tbl: restored=$restored live=$live (MISMATCH)"
fails=$((fails + 1))
fi
done
if [ "$checked" -eq 0 ]; then
log "FAIL: could not verify any table"
exit 1
fi
if [ "$fails" -ne 0 ]; then
log "FAIL: $fails check(s) failed — backup is NOT trustworthy"
exit 1
fi
# Record a success marker + metric for monitoring (optional consumers).
echo "$(date +%s)" > "$BACKUP_DIR/.last-restore-drill-ok"
log "PASS: restore verified against live Neon ($checked tables). Backup is good."
|