Commit5dc0e94
fix(backup): include bare git repos in the daily offsite backup
fix(backup): include bare git repos in the daily offsite backup The daily backup timer only ever dumped the Neon database — the bare git repos (the irreplaceable half; the DB can't reconstruct repo objects) had zero offsite copy. Losing the VPS meant losing every hosted repo, including this platform's own source. Tars the gluecron_git-repos docker volume alongside the existing DB dump and ships both through the same BACKUP_SCP_TARGET path. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1 file changed+53−135dc0e94394de36f131aedf5e4e5d6138bef84ff8
1 changed file+53−13
Modifiedscripts/backup.sh+53−13View fileUnifiedSplit
@@ -1,28 +1,39 @@
11
22#
3# Daily production-database backup for the standalone/Coolify box.
3# Daily production backup for the standalone/Coolify box.
44# Installed as a systemd timer by scripts/standalone-deploy.sh.
55#
6# WHAT IT BACKS UP (corrected 2026-07-14): the REAL production database, which
7# is **Neon** — reached via the app container's own DATABASE_URL. The compose
8# also runs an unused local `postgres` container; the previous version of this
9# script dumped THAT by mistake, so daily backups were capturing empty/stale
10# data. Do not reintroduce `$COMPOSE exec postgres pg_dump`.
6# WHAT IT BACKS UP:
7# 1. the REAL production database, which is **Neon** — reached via the app
8# container's own DATABASE_URL. The compose also runs an unused local
9# `postgres` container; an earlier version of this script dumped THAT by
10# mistake (corrected 2026-07-14). Do not reintroduce
11# `$COMPOSE exec postgres pg_dump`.
12# 2. the bare git repos themselves (added 2026-07-15) — these live in the
13# `gluecron_git-repos` docker volume, NOT a plain host directory, and are
14# the irreplaceable half: the DB only stores metadata and cannot
15# reconstruct repo objects. Losing the box without this half means
16# losing every hosted repo, full stop.
1117#
12# HOW: Neon runs Postgres 17, so we dump with an ephemeral `postgres:17`
18# HOW (DB): Neon runs Postgres 17, so we dump with an ephemeral `postgres:17`
1319# container (the box's local client is 16 and refuses a newer server). The
1420# app's DATABASE_URL on this box is malformed (two connstrings mashed on one
1521# line — a fragile latent bug), so we take the clean Neon URL up to the second
1622# "DATABASE_URL=" marker. `${URL%%DATABASE_URL=*}` is a no-op once the .env is
1723# fixed, so this stays correct either way.
1824#
19# Output: custom-format (`-Fc`) dumps — compressed + restorable selectively via
20# pg_restore. Compatible with scripts/restore.sh and scripts/backup-restore-drill.sh.
25# HOW (repos): tar the docker volume's mountpoint directly on the host (bare
26# repos are self-contained, so a plain tar is a valid, restorable snapshot).
27#
28# Output: custom-format (`-Fc`) DB dumps — compressed + restorable selectively
29# via pg_restore. Compatible with scripts/restore.sh and
30# scripts/backup-restore-drill.sh. Repos ship as a gzip tarball.
2131#
2232# Optional env (set in /opt/gluecron/.env):
2333# BACKUP_RCLONE_REMOTE e.g. r2:gluecron-backups (needs rclone configured)
2434# BACKUP_SCP_TARGET e.g. user@100.x.y.z:/backups (self-owned offsite over Tailscale)
2535# HEALTHCHECK_PING_URL e.g. https://hc-ping.com/<uuid> (dead-man's-switch)
36# GIT_REPOS_VOLUME docker volume name for bare repos (default: gluecron_git-repos)
2637set -euo pipefail
2738
2839REPO_DIR="/opt/gluecron"
@@ -30,11 +41,13 @@ BACKUP_DIR="$REPO_DIR/backups"
3041RETAIN_DAYS=14
3142APP_CONTAINER="gluecron-gluecron-1"
3243PG_IMAGE="postgres:17-alpine"
44GIT_REPOS_VOLUME="${GIT_REPOS_VOLUME:-gluecron_git-repos}"
3345
3446cd "$REPO_DIR"
3547mkdir -p "$BACKUP_DIR"
3648ts=$(date +%Y%m%d-%H%M%S)
3749out="$BACKUP_DIR/gluecron-db-$ts.dump"
50repos_out="$BACKUP_DIR/gluecron-repos-$ts.tar.gz"
3851
3952# Resolve the live DATABASE_URL from the running app, then clean it. Reading it
4053# from the container (not .env) means the backup always matches what the app
@@ -73,19 +86,46 @@ fi
7386
7487size="$(du -h "$out" | cut -f1)"
7588
89# Bare git repos — resolve the volume's real host mountpoint (not a fixed
90# path: docker owns where volumes actually live) and tar it in place.
91repos_mountpoint="$(docker volume inspect "$GIT_REPOS_VOLUME" --format '{{ .Mountpoint }}' 2>/dev/null || true)"
92if [ -n "$repos_mountpoint" ] && [ -d "$repos_mountpoint" ]; then
93 tar -czf "$repos_out" -C "$(dirname "$repos_mountpoint")" "$(basename "$repos_mountpoint")"
94 repos_size="$(du -h "$repos_out" | cut -f1)"
95 repos_file_count="$(tar -tzf "$repos_out" | wc -l)"
96 if [ "$repos_file_count" -lt 5 ]; then
97 echo "$(date -Is) FATAL: repos tarball looks empty ($repos_file_count entries) — refusing to keep it" >&2
98 rm -f "$repos_out"
99 exit 1
100 fi
101 echo "$(date -Is) repos snapshot: $repos_out ($repos_size, $repos_file_count entries)"
102else
103 echo "$(date -Is) FATAL: could not resolve docker volume '$GIT_REPOS_VOLUME' — repos NOT backed up this run" >&2
104 exit 1
105fi
106
76107# Retention — keep RETAIN_DAYS of daily dumps.
77108find "$BACKUP_DIR" -name 'gluecron-db-*.dump' -mtime +$RETAIN_DAYS -delete
109find "$BACKUP_DIR" -name 'gluecron-repos-*.tar.gz' -mtime +$RETAIN_DAYS -delete
78110
79111# Optional self-owned offsite copy over Tailscale (preferred — no third party).
80112if [ -n "${BACKUP_SCP_TARGET:-}" ]; then
81113 scp -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$out" "$BACKUP_SCP_TARGET/" \
82 && echo "$(date -Is) offsite scp ok -> $BACKUP_SCP_TARGET" \
83 || echo "$(date -Is) WARN: offsite scp failed"
114 && echo "$(date -Is) offsite scp ok -> $BACKUP_SCP_TARGET (db)" \
115 || echo "$(date -Is) WARN: offsite scp failed (db)"
116 if [ -f "$repos_out" ]; then
117 scp -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$repos_out" "$BACKUP_SCP_TARGET/" \
118 && echo "$(date -Is) offsite scp ok -> $BACKUP_SCP_TARGET (repos)" \
119 || echo "$(date -Is) WARN: offsite scp failed (repos)"
120 fi
84121fi
85122
86123# Optional rclone offsite copy (if you do use a bucket).
87124if [ -n "${BACKUP_RCLONE_REMOTE:-}" ] && command -v rclone >/dev/null 2>&1; then
88 rclone copy "$out" "$BACKUP_RCLONE_REMOTE" || echo "$(date -Is) WARN: rclone copy failed"
125 rclone copy "$out" "$BACKUP_RCLONE_REMOTE" || echo "$(date -Is) WARN: rclone copy failed (db)"
126 if [ -f "$repos_out" ]; then
127 rclone copy "$repos_out" "$BACKUP_RCLONE_REMOTE" || echo "$(date -Is) WARN: rclone copy failed (repos)"
128 fi
89129fi
90130
91131# Optional dead-man's-switch heartbeat (alerts if a backup is ever missed).
@@ -93,4 +133,4 @@ if [ -n "${HEALTHCHECK_PING_URL:-}" ]; then
93133 curl -fsS -m 10 "$HEALTHCHECK_PING_URL" >/dev/null 2>&1 || true
94134fi
95135
96echo "$(date -Is) backup written: $out ($size, $obj_count objects)"
136echo "$(date -Is) backup written: $out ($size, $obj_count objects); $repos_out ($repos_size, $repos_file_count entries)"
97137