CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history
restore-drill.sh
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 41d6ea3 | 1 | #!/usr/bin/env bash |
| 2 | # | |
| 3 | # Restore drill — proves the latest production (Neon) backup actually restores. | |
| 4 | # "A backup you have never restored is not a backup, it is a hope." | |
| 5 | # | |
| 6 | # Self-contained, zero external dependencies: spins up a THROWAWAY postgres:17 | |
| 7 | # container, pg_restores the newest dump into it, then compares key row counts | |
| 8 | # against the live Neon database. Tears the container down unconditionally. | |
| 9 | # | |
| 10 | # Exit 0 = restore verified (counts match). Non-zero = drill FAILED — the | |
| 11 | # backup is NOT trustworthy; investigate before relying on it. | |
| 12 | # | |
| 13 | # Usage: bash scripts/restore-drill.sh | |
| 14 | set -uo pipefail | |
| 15 | ||
| 16 | BACKUP_DIR="/opt/gluecron/backups" | |
| 17 | APP_CONTAINER="gluecron-gluecron-1" | |
| 18 | PG_IMAGE="postgres:17-alpine" | |
| 19 | DRILL="gluecron-restore-drill-$$" | |
| 20 | ||
| 21 | log() { echo "$(date -Is) [drill] $*"; } | |
| 22 | cleanup() { docker rm -f "$DRILL" >/dev/null 2>&1 || true; } | |
| 23 | trap cleanup EXIT | |
| 24 | ||
| 25 | dump="$(ls -t "$BACKUP_DIR"/gluecron-db-*.dump 2>/dev/null | head -1)" | |
| 26 | if [ -z "$dump" ]; then | |
| 27 | log "FAIL: no backup found in $BACKUP_DIR" | |
| 28 | exit 1 | |
| 29 | fi | |
| 30 | log "restoring: $dump" | |
| 31 | ||
| 32 | raw_url="$(docker exec "$APP_CONTAINER" printenv DATABASE_URL)" | |
| 33 | clean_url="${raw_url%%DATABASE_URL=*}" | |
| 34 | ||
| 35 | # Throwaway target instance. | |
| 36 | docker run -d --name "$DRILL" -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill \ | |
| 37 | "$PG_IMAGE" >/dev/null | |
| 38 | for _ in $(seq 1 30); do | |
| 39 | docker exec "$DRILL" pg_isready -U postgres -d drill >/dev/null 2>&1 && break | |
| 40 | sleep 1 | |
| 41 | done | |
| 42 | ||
| 43 | docker cp "$dump" "$DRILL":/tmp/d.dump | |
| 44 | # Ignore benign restore noise (extensions, comments); we judge by row counts. | |
| 45 | docker exec "$DRILL" pg_restore --no-owner --no-privileges -U postgres -d drill /tmp/d.dump \ | |
| 46 | >/dev/null 2>&1 || true | |
| 47 | ||
| 48 | fails=0 | |
| 49 | checked=0 | |
| 50 | for tbl in users repositories issues pull_requests oauth_apps; do | |
| 51 | restored="$(docker exec "$DRILL" psql -At -U postgres -d drill \ | |
| 52 | -c "SELECT count(*) FROM $tbl" 2>/dev/null | tr -d '[:space:]')" | |
| 53 | # $U/$TBL expand inside the container (host-side would be empty). | |
| 54 | live="$(docker run --rm -e U="$clean_url" -e TBL="$tbl" "$PG_IMAGE" \ | |
| 55 | sh -c 'psql -At "$U" -c "SELECT count(*) FROM $TBL"' 2>/dev/null | tr -d '[:space:]')" | |
| 56 | if [ -z "$restored" ] || [ -z "$live" ]; then | |
| 57 | log "WARN: $tbl — could not read a count (restored='$restored' live='$live')" | |
| 58 | fails=$((fails + 1)) | |
| 59 | continue | |
| 60 | fi | |
| 61 | checked=$((checked + 1)) | |
| 62 | if [ "$restored" = "$live" ]; then | |
| 63 | log "OK $tbl: $restored rows (matches live)" | |
| 64 | else | |
| 65 | log "FAIL $tbl: restored=$restored live=$live (MISMATCH)" | |
| 66 | fails=$((fails + 1)) | |
| 67 | fi | |
| 68 | done | |
| 69 | ||
| 70 | if [ "$checked" -eq 0 ]; then | |
| 71 | log "FAIL: could not verify any table" | |
| 72 | exit 1 | |
| 73 | fi | |
| 74 | if [ "$fails" -ne 0 ]; then | |
| 75 | log "FAIL: $fails check(s) failed — backup is NOT trustworthy" | |
| 76 | exit 1 | |
| 77 | fi | |
| 78 | ||
| 79 | # Record a success marker + metric for monitoring (optional consumers). | |
| 80 | echo "$(date +%s)" > "$BACKUP_DIR/.last-restore-drill-ok" | |
| 81 | log "PASS: restore verified against live Neon ($checked tables). Backup is good." |