CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | #!/usr/bin/env bash
#
# Daily production backup for the standalone/Coolify box.
# Installed as a systemd timer by scripts/standalone-deploy.sh.
#
# WHAT IT BACKS UP:
# 1. the REAL production database, which is **Neon** — reached via the app
# container's own DATABASE_URL. The compose also runs an unused local
# `postgres` container; an earlier version of this script dumped THAT by
# mistake (corrected 2026-07-14). Do not reintroduce
# `$COMPOSE exec postgres pg_dump`.
# 2. the bare git repos themselves (added 2026-07-15) — these live in the
# `gluecron_git-repos` docker volume, NOT a plain host directory, and are
# the irreplaceable half: the DB only stores metadata and cannot
# reconstruct repo objects. Losing the box without this half means
# losing every hosted repo, full stop.
#
# HOW (DB): Neon runs Postgres 17, so we dump with an ephemeral `postgres:17`
# container (the box's local client is 16 and refuses a newer server). The
# app's DATABASE_URL on this box is malformed (two connstrings mashed on one
# line — a fragile latent bug), so we take the clean Neon URL up to the second
# "DATABASE_URL=" marker. `${URL%%DATABASE_URL=*}` is a no-op once the .env is
# fixed, so this stays correct either way.
#
# HOW (repos): tar the docker volume's mountpoint directly on the host (bare
# repos are self-contained, so a plain tar is a valid, restorable snapshot).
#
# Output: custom-format (`-Fc`) DB dumps — compressed + restorable selectively
# via pg_restore. Compatible with scripts/restore.sh and
# scripts/backup-restore-drill.sh. Repos ship as a gzip tarball.
#
# Optional env (set in /opt/gluecron/.env):
# BACKUP_RCLONE_REMOTE e.g. r2:gluecron-backups (needs rclone configured)
# BACKUP_SCP_TARGET e.g. user@100.x.y.z:/backups (self-owned offsite over Tailscale)
# HEALTHCHECK_PING_URL e.g. https://hc-ping.com/<uuid> (dead-man's-switch)
# GIT_REPOS_VOLUME docker volume name for bare repos (default: gluecron_git-repos)
set -euo pipefail
REPO_DIR="/opt/gluecron"
BACKUP_DIR="$REPO_DIR/backups"
RETAIN_DAYS=14
APP_CONTAINER="gluecron-gluecron-1"
PG_IMAGE="postgres:17-alpine"
GIT_REPOS_VOLUME="${GIT_REPOS_VOLUME:-gluecron_git-repos}"
cd "$REPO_DIR"
mkdir -p "$BACKUP_DIR"
ts=$(date +%Y%m%d-%H%M%S)
out="$BACKUP_DIR/gluecron-db-$ts.dump"
repos_out="$BACKUP_DIR/gluecron-repos-$ts.tar.gz"
# Resolve the live DATABASE_URL from the running app, then clean it. Reading it
# from the container (not .env) means the backup always matches what the app
# actually connects to.
raw_url="$(docker exec "$APP_CONTAINER" printenv DATABASE_URL)"
clean_url="${raw_url%%DATABASE_URL=*}"
if [ -z "$clean_url" ]; then
echo "$(date -Is) FATAL: could not resolve DATABASE_URL from $APP_CONTAINER" >&2
exit 1
fi
# Dump Neon with a matching-major-version client. Custom format, no owner/ACLs
# (portable across restore targets). Password + URL travel only in container
# env; the pg_dump runs via `sh -c` so $NEONURL expands INSIDE the container
# (host-side expansion would be empty and trip `set -u`).
docker run --rm -e NEONURL="$clean_url" -e OUTFILE="/backups/$(basename "$out")" \
-v "$BACKUP_DIR:/backups" "$PG_IMAGE" \
sh -c 'pg_dump "$NEONURL" --format=custom --no-owner --no-privileges -f "$OUTFILE"' \
2>/tmp/backup-err.$$ || {
echo "$(date -Is) FATAL: pg_dump failed:" >&2
cat /tmp/backup-err.$$ >&2
rm -f /tmp/backup-err.$$ "$out"
exit 1
}
rm -f /tmp/backup-err.$$
# Sanity: a valid custom-format dump must list objects. Guards against a
# 0-byte/corrupt file being counted as a good backup.
obj_count="$(docker run --rm -e F="/backups/$(basename "$out")" -v "$BACKUP_DIR:/backups" "$PG_IMAGE" \
sh -c 'pg_restore --list "$F"' 2>/dev/null | grep -cE 'TABLE DATA|TABLE|SEQUENCE' || true)"
if [ "${obj_count:-0}" -lt 20 ]; then
echo "$(date -Is) FATAL: dump looks empty ($obj_count objects) — refusing to keep it" >&2
rm -f "$out"
exit 1
fi
size="$(du -h "$out" | cut -f1)"
# Bare git repos — resolve the volume's real host mountpoint (not a fixed
# path: docker owns where volumes actually live) and tar it in place.
repos_mountpoint="$(docker volume inspect "$GIT_REPOS_VOLUME" --format '{{ .Mountpoint }}' 2>/dev/null || true)"
if [ -n "$repos_mountpoint" ] && [ -d "$repos_mountpoint" ]; then
tar -czf "$repos_out" -C "$(dirname "$repos_mountpoint")" "$(basename "$repos_mountpoint")"
repos_size="$(du -h "$repos_out" | cut -f1)"
repos_file_count="$(tar -tzf "$repos_out" | wc -l)"
if [ "$repos_file_count" -lt 5 ]; then
echo "$(date -Is) FATAL: repos tarball looks empty ($repos_file_count entries) — refusing to keep it" >&2
rm -f "$repos_out"
exit 1
fi
echo "$(date -Is) repos snapshot: $repos_out ($repos_size, $repos_file_count entries)"
else
echo "$(date -Is) FATAL: could not resolve docker volume '$GIT_REPOS_VOLUME' — repos NOT backed up this run" >&2
exit 1
fi
# Retention — keep RETAIN_DAYS of daily dumps.
find "$BACKUP_DIR" -name 'gluecron-db-*.dump' -mtime +$RETAIN_DAYS -delete
find "$BACKUP_DIR" -name 'gluecron-repos-*.tar.gz' -mtime +$RETAIN_DAYS -delete
# Optional self-owned offsite copy over Tailscale (preferred — no third party).
if [ -n "${BACKUP_SCP_TARGET:-}" ]; then
scp -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$out" "$BACKUP_SCP_TARGET/" \
&& echo "$(date -Is) offsite scp ok -> $BACKUP_SCP_TARGET (db)" \
|| echo "$(date -Is) WARN: offsite scp failed (db)"
if [ -f "$repos_out" ]; then
scp -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$repos_out" "$BACKUP_SCP_TARGET/" \
&& echo "$(date -Is) offsite scp ok -> $BACKUP_SCP_TARGET (repos)" \
|| echo "$(date -Is) WARN: offsite scp failed (repos)"
fi
fi
# Optional rclone offsite copy (if you do use a bucket).
if [ -n "${BACKUP_RCLONE_REMOTE:-}" ] && command -v rclone >/dev/null 2>&1; then
rclone copy "$out" "$BACKUP_RCLONE_REMOTE" || echo "$(date -Is) WARN: rclone copy failed (db)"
if [ -f "$repos_out" ]; then
rclone copy "$repos_out" "$BACKUP_RCLONE_REMOTE" || echo "$(date -Is) WARN: rclone copy failed (repos)"
fi
fi
# Optional dead-man's-switch heartbeat (alerts if a backup is ever missed).
if [ -n "${HEALTHCHECK_PING_URL:-}" ]; then
curl -fsS -m 10 "$HEALTHCHECK_PING_URL" >/dev/null 2>&1 || true
fi
echo "$(date -Is) backup written: $out ($size, $obj_count objects); $repos_out ($repos_size, $repos_file_count entries)"
|