Blame · Line-by-line history
backup-offsite.sh
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 1a07e02 | 1 | #!/usr/bin/env bash |
| 2 | # | |
| 3 | # Durable offsite backup for the self-hosted VPS (systemd + Neon, NOT docker). | |
| 4 | # | |
| 5 | # This is the backup that lets Gluecron drop the GitHub mirror safely: it is | |
| 6 | # the ONLY off-box copy of the bare git repos once GitHub is gone. It captures | |
| 7 | # both halves of the platform's state: | |
| 8 | # | |
| 9 | # 1. the bare git repos under $GIT_REPOS_PATH (the actual repo objects — | |
| 10 | # the DB only stores metadata and CANNOT reconstruct these) | |
| 11 | # 2. the Neon Postgres database via pg_dump over $DATABASE_URL | |
| 12 | # | |
| 13 | # ...then pushes both to an offsite bucket with rclone and pings a | |
| 14 | # dead-man's-switch so a silently-missed backup pages you. | |
| 15 | # | |
| 16 | # Install as a systemd timer (see docs/CUTOVER_RUNBOOK.md). Run daily. | |
| 17 | # | |
| 18 | # Required env (set in /etc/gluecron.env): | |
| 19 | # DATABASE_URL Neon connection string (postgresql://...) | |
| 20 | # BACKUP_RCLONE_REMOTE e.g. r2:gluecron-backups (rclone must be configured) | |
| 392689d | 21 | # GIT_REPOS_PATH bare repos dir. REQUIRED IN PRACTICE: the default |
| 22 | # below (/opt/gluecron/repos) does NOT exist on the | |
| 23 | # live host, where the repos are in a docker volume at | |
| 24 | # /var/lib/docker/volumes/gluecron_git-repos/_data. | |
| 25 | # The app reads this from /opt/gluecron/.env, but a | |
| 26 | # systemd unit for THIS script reads | |
| 27 | # /etc/gluecron.env, so it must be set there too. | |
| 1a07e02 | 28 | # Optional env: |
| 29 | # BACKUP_DIR local staging dir (default: /opt/gluecron/backups) | |
| 30 | # RETAIN_DAYS local + offsite retention (default: 14) | |
| 31 | # HEALTHCHECK_PING_URL https://hc-ping.com/<uuid> (healthchecks.io) | |
| 392689d | 32 | # |
| 33 | # Escape hatches — all default OFF, because each one makes the backup weaker | |
| 34 | # than it looks. Set deliberately, never to quiet a failure: | |
| 35 | # ALLOW_DB_ONLY_BACKUP=1 proceed with no repos snapshot | |
| 36 | # ALLOW_EMPTY_REPOS_BACKUP=1 proceed when the repos dir holds no repos | |
| 37 | # ALLOW_LOCAL_ONLY_BACKUP=1 proceed with no offsite destination | |
| 1a07e02 | 38 | set -euo pipefail |
| 39 | ||
| 40 | GIT_REPOS_PATH="${GIT_REPOS_PATH:-/opt/gluecron/repos}" | |
| 41 | BACKUP_DIR="${BACKUP_DIR:-/opt/gluecron/backups}" | |
| 42 | RETAIN_DAYS="${RETAIN_DAYS:-14}" | |
| 43 | ||
| 44 | log() { echo "$(date -Is) [backup-offsite] $*"; } | |
| 45 | ||
| 46 | if [ -z "${DATABASE_URL:-}" ]; then | |
| 47 | log "FATAL: DATABASE_URL is not set — cannot back up the database." | |
| 48 | exit 1 | |
| 49 | fi | |
| 50 | ||
| 51 | mkdir -p "$BACKUP_DIR" | |
| 52 | ts=$(date +%Y%m%d-%H%M%S) | |
| 392689d | 53 | repos_included=0 |
| 1a07e02 | 54 | repos_out="$BACKUP_DIR/repos-$ts.tar.gz" |
| 55 | db_out="$BACKUP_DIR/db-$ts.dump" | |
| 56 | ||
| 57 | # 1. Bare git repos — the irreplaceable half. tar the whole tree; bare repos | |
| 58 | # are self-contained so a plain tar is a valid, restorable snapshot. | |
| 392689d | 59 | # |
| 60 | # A missing or empty repos path is FATAL, not a warning. It used to log a | |
| 61 | # WARN and carry on: the tarball was then never created, the `-f` guard on | |
| 62 | # the upload below skipped it silently, and the run still pinged the | |
| 63 | # dead-man's-switch. The result was a green healthcheck next to a DB-only | |
| 64 | # backup — and the DB stores metadata ONLY, so it cannot reconstruct a | |
| 65 | # single repository. That is the exact failure this script exists to | |
| 66 | # prevent, and it would have looked like success indefinitely. | |
| 67 | # | |
| 68 | # Not hypothetical: the default below is /opt/gluecron/repos, which does | |
| 69 | # not exist on the live host — the repos live in a docker volume. The | |
| 70 | # correct GIT_REPOS_PATH is set in /opt/gluecron/.env, but this script's | |
| 71 | # own header tells you to configure it in /etc/gluecron.env, where it is | |
| 72 | # absent. Installing the timer as documented would have produced exactly | |
| 73 | # the silent DB-only backup described above. | |
| 74 | if [ ! -d "$GIT_REPOS_PATH" ]; then | |
| 75 | if [ "${ALLOW_DB_ONLY_BACKUP:-0}" = "1" ]; then | |
| 76 | log "WARN: GIT_REPOS_PATH ($GIT_REPOS_PATH) does not exist — DB-only backup (ALLOW_DB_ONLY_BACKUP=1)" | |
| 77 | repos_included=0 | |
| 78 | else | |
| 79 | log "FATAL: GIT_REPOS_PATH ($GIT_REPOS_PATH) does not exist." | |
| 80 | log " The database alone CANNOT reconstruct the repositories." | |
| 81 | log " Set GIT_REPOS_PATH, or set ALLOW_DB_ONLY_BACKUP=1 to accept a DB-only backup." | |
| 82 | exit 1 | |
| 83 | fi | |
| 1a07e02 | 84 | else |
| 392689d | 85 | # An existing but WRONG path is the nastier case: tar succeeds and uploads a |
| 86 | # valid, near-empty archive, which looks healthier than an outright failure. | |
| 87 | # Bare repos are laid out as <owner>/<repo>.git, so count those. | |
| 88 | repo_count=$(find "$GIT_REPOS_PATH" -maxdepth 2 -type d -name '*.git' 2>/dev/null | wc -l | tr -d ' ') | |
| 89 | if [ "$repo_count" -eq 0 ]; then | |
| 90 | # Fall back to detecting bare repos by their HEAD file, in case a repo is | |
| 91 | # stored without the .git suffix. | |
| 92 | repo_count=$(find "$GIT_REPOS_PATH" -maxdepth 3 -type f -name HEAD 2>/dev/null | wc -l | tr -d ' ') | |
| 93 | fi | |
| 94 | if [ "$repo_count" -eq 0 ] && [ "${ALLOW_EMPTY_REPOS_BACKUP:-0}" != "1" ]; then | |
| 95 | log "FATAL: no bare repositories found under $GIT_REPOS_PATH." | |
| 96 | log " This is almost always the wrong path rather than a genuinely" | |
| 97 | log " empty host — on the live box the repos are in a docker volume." | |
| 98 | log " Set ALLOW_EMPTY_REPOS_BACKUP=1 if this host really has none." | |
| 99 | exit 1 | |
| 100 | fi | |
| 101 | tar -czf "$repos_out" -C "$(dirname "$GIT_REPOS_PATH")" "$(basename "$GIT_REPOS_PATH")" | |
| 102 | log "repos snapshot: $repos_out ($(du -h "$repos_out" | cut -f1), $repo_count repos)" | |
| 103 | repos_included=1 | |
| 1a07e02 | 104 | fi |
| 105 | ||
| 106 | # 2. Database — custom format so pg_restore can do selective/parallel restore. | |
| 107 | pg_dump --format=custom --no-owner --no-privileges "$DATABASE_URL" > "$db_out" | |
| 108 | log "db dump: $db_out ($(du -h "$db_out" | cut -f1))" | |
| 109 | ||
| 110 | # 3. Offsite copy (both artifacts). Fail LOUD if configured but the copy fails — | |
| 111 | # a backup that only lives on the same box it protects is not a backup. | |
| 112 | if [ -n "${BACKUP_RCLONE_REMOTE:-}" ]; then | |
| 113 | if ! command -v rclone >/dev/null 2>&1; then | |
| 114 | log "FATAL: BACKUP_RCLONE_REMOTE set but rclone is not installed" | |
| 115 | exit 1 | |
| 116 | fi | |
| 117 | rclone copy "$db_out" "$BACKUP_RCLONE_REMOTE/db/" \ | |
| 118 | && log "offsite: db -> $BACKUP_RCLONE_REMOTE/db/" \ | |
| 119 | || { log "FATAL: offsite db copy failed"; exit 1; } | |
| 392689d | 120 | if [ "$repos_included" = "1" ]; then |
| 121 | # Explicit flag rather than a bare `-f` test: a missing tarball we EXPECTED | |
| 122 | # to have is a bug, and the old `-f` guard turned exactly that into a | |
| 123 | # silent skip that still reported success. | |
| 124 | if [ ! -f "$repos_out" ]; then | |
| 125 | log "FATAL: repos snapshot $repos_out is missing after being created" | |
| 126 | exit 1 | |
| 127 | fi | |
| 1a07e02 | 128 | rclone copy "$repos_out" "$BACKUP_RCLONE_REMOTE/repos/" \ |
| 129 | && log "offsite: repos -> $BACKUP_RCLONE_REMOTE/repos/" \ | |
| 130 | || { log "FATAL: offsite repos copy failed"; exit 1; } | |
| 131 | fi | |
| 132 | # Prune offsite copies older than retention (best-effort). | |
| 133 | rclone delete --min-age "${RETAIN_DAYS}d" "$BACKUP_RCLONE_REMOTE/db/" 2>/dev/null || true | |
| 134 | rclone delete --min-age "${RETAIN_DAYS}d" "$BACKUP_RCLONE_REMOTE/repos/" 2>/dev/null || true | |
| 392689d | 135 | elif [ "${ALLOW_LOCAL_ONLY_BACKUP:-0}" = "1" ]; then |
| 136 | log "WARN: BACKUP_RCLONE_REMOTE not set — backup is LOCAL ONLY (ALLOW_LOCAL_ONLY_BACKUP=1)" | |
| 1a07e02 | 137 | else |
| 392689d | 138 | # Same class of lie as the skipped repos snapshot: a backup sitting on the |
| 139 | # box it is meant to protect is not a backup, and pinging the | |
| 140 | # dead-man's-switch for one tells you durability is fine when it is not. | |
| 141 | log "FATAL: BACKUP_RCLONE_REMOTE is not set — this backup would be LOCAL ONLY." | |
| 142 | log " Set it, or set ALLOW_LOCAL_ONLY_BACKUP=1 to accept a local-only run." | |
| 143 | exit 1 | |
| 1a07e02 | 144 | fi |
| 145 | ||
| 146 | # 4. Local retention. | |
| 147 | find "$BACKUP_DIR" -name 'repos-*.tar.gz' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null || true | |
| 148 | find "$BACKUP_DIR" -name 'db-*.dump' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null || true | |
| 149 | ||
| 150 | # 5. Dead-man's-switch: only ping on full success (reached the end of the script). | |
| 151 | if [ -n "${HEALTHCHECK_PING_URL:-}" ]; then | |
| 152 | curl -fsS -m 10 "$HEALTHCHECK_PING_URL" >/dev/null 2>&1 || true | |
| 153 | fi | |
| 154 | ||
| 155 | log "done." |