homelabstack/scripts/migration-backup.sh

#!/usr/bin/env bash
#
# Phase 0 migration backup — valhalla (Dell R510 -> R730XD / Proxmox move).
#
# Captures everything that does NOT travel on the /storage1 ZFS pool disks:
#   * logical dumps of all Postgres + MariaDB databases (clean, version-portable)
#   * the /config app-state tree (app configs, vault, portainer, couchdb, raw datadirs)
#   * native Plex "Application Support" (watch history, collections, metadata)
#
# NOT copied here (they live on /storage1/labdata and ride along on the disks):
#   MinIO blobs (famapp/resume/plane), gitea repos+LFS, docker registry, media.
#
# Best-effort by design: a failed individual DB dump logs a WARNING and is skipped
# (its raw datadir is still inside config.tar) — it never blocks the /config/Plex tars.
#
# Run ON valhalla as ginnoir:
#   bash migration-backup.sh           # live insurance backup (no downtime)
#   bash migration-backup.sh --stop    # consistent backup: stop app containers first,
#                                       #   then restart them when done (rehearsal-safe)
#
# Output: /storage1/migration-backup/<timestamp>/  (+ SHA256SUMS)
#
set -uo pipefail   # NOTE: deliberately no `set -e` — we want best-effort completion.

DEST="/storage1/migration-backup"
STAMP="$(date +%Y%m%d-%H%M%S)"
OUT="$DEST/$STAMP"
STOP=0
[[ "${1:-}" == "--stop" ]] && STOP=1

mkdir -p "$OUT/db" "$OUT/config" "$OUT/plex"
LOG="$OUT/backup.log"
exec > >(tee -a "$LOG") 2>&1
log(){ printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
running(){ docker ps --format '{{.Names}}' | grep -qx "$1"; }

# Resolve a working Postgres superuser for a container (env POSTGRES_USER, then
# common fallbacks). Echoes the user on success; non-zero if none connect.
pg_super(){
  local c="$1" cand
  for cand in "$(docker exec "$c" printenv POSTGRES_USER 2>/dev/null)" \
              "$(docker exec "$c" printenv POSTGRES_DB   2>/dev/null)" \
              immich postgres; do
    [ -z "$cand" ] && continue
    if docker exec "$c" psql -U "$cand" -tAc 'select 1' >/dev/null 2>&1; then
      echo "$cand"; return 0
    fi
  done
  return 1
}

log "migration backup -> $OUT (stop=$STOP)"

PG_CONTAINERS="postgres_authentik postgres_famapp postgres_gitea postgres_plane postgres_resume nextcloud-postgres immich-postgres"
MARIA_CONTAINERS="owncloud_mariadb romm-db mariadb_bookstack"

# --- 1. logical DB dumps (best-effort; authoritative restore path) ---
for c in $PG_CONTAINERS; do
  running "$c" || { log "SKIP $c (not running)"; continue; }
  if u="$(pg_super "$c")"; then
    log "pg_dumpall $c (user=$u)"
    if docker exec "$c" pg_dumpall --clean --if-exists -U "$u" | gzip > "$OUT/db/${c}.sql.gz"; then
      :
    else
      log "WARN: pg_dumpall $c FAILED — raw pgdata remains in config.tar"; rm -f "$OUT/db/${c}.sql.gz"
    fi
  else
    log "WARN: no working superuser found for $c — raw pgdata remains in config.tar"
  fi
done

for c in $MARIA_CONTAINERS; do
  running "$c" || { log "SKIP $c (not running)"; continue; }
  log "mariadb-dump $c"
  pw="$(docker exec "$c" sh -c 'printf %s "${MYSQL_ROOT_PASSWORD:-${MARIADB_ROOT_PASSWORD:-}}"' 2>/dev/null)"
  if docker exec -e MYSQL_PWD="$pw" "$c" sh -c \
       'mariadb-dump -uroot --all-databases --single-transaction --routines --triggers --events 2>/dev/null \
        || mysqldump  -uroot --all-databases --single-transaction --routines --triggers --events' \
       | gzip > "$OUT/db/${c}.sql.gz"; then
    :
  else
    log "WARN: mariadb-dump $c FAILED — raw datadir remains in config.tar"; rm -f "$OUT/db/${c}.sql.gz"
  fi
done

# --- 2. optionally stop app containers for a fully-consistent /config snapshot ---
STOPPED=""
if [[ "$STOP" == "1" ]]; then
  log "stopping app containers for a consistent /config snapshot (portainer left up)"
  KEEP="portainer"
  for c in $(docker ps --format '{{.Names}}'); do
    case " $KEEP " in *" $c "*) continue;; esac
    docker stop "$c" >/dev/null && STOPPED="$STOPPED $c"
  done
  log "stopped:$STOPPED"
fi

# --- 3. /config tree (app state, vault, couchdb, raw datadirs) ---
log "tar /config (the big one, ~100GB)"
sudo tar --warning=no-file-changed -cf "$OUT/config/config.tar" -C / config || log "WARN: /config tar returned non-zero"

# --- 4. native Plex Application Support ---
log "tar Plex Application Support"
tar --warning=no-file-changed -cf "$OUT/plex/plex-appsupport.tar" \
  -C "/var/lib/plexmediaserver/Library/Application Support" "Plex Media Server" || log "WARN: Plex tar returned non-zero"

# --- 5. restart anything we stopped (leave the box as we found it) ---
if [[ -n "$STOPPED" ]]; then
  log "restarting stopped containers"
  # shellcheck disable=SC2086
  docker start $STOPPED >/dev/null
fi

# --- 6. inventory + checksums ---
log "writing inventory + checksums"
docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Label "com.docker.compose.project"}}' > "$OUT/inventory-containers.txt"
docker network ls > "$OUT/inventory-networks.txt"
cp /config/caddy/Caddyfile                          "$OUT/Caddyfile"             2>/dev/null || true
cp /home/ginnoir/valhalla-lab/portainer-compose.yml "$OUT/portainer-compose.yml" 2>/dev/null || true
cp /home/ginnoir/valhalla-lab/.env                  "$OUT/valhalla-lab.env"      2>/dev/null || true
( cd "$OUT" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS )

log "sizes:"
du -sh "$OUT"/db "$OUT"/config "$OUT"/plex 2>/dev/null || true
du -sh "$OUT" 2>/dev/null || true
echo "=== db dumps ==="; ls -lh "$OUT/db"
log "DONE -> $OUT"
echo
echo "NOTE: MinIO blobs, gitea repos/LFS, the registry, and all media are on /storage1"
echo "      and travel with the pool disks; they are intentionally NOT in this backup."