chore: add migration helper scripts + cutover plan doc
R510 -> R730XD/Proxmox migration helpers (consistent DB dumps, B2/Kopia snapshot refresh, cutover final backup) plus the previously-untracked gitea-portainer-registry cutover plan under docs/. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ginnoir
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
4ede8c483a
commit
c5d487effe
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Phase 0 migration backup — valhalla (Dell R510 -> R730XD / Proxmox move).
|
||||
#
|
||||
# Captures everything that does NOT travel on the /storage1 ZFS pool disks:
|
||||
# * logical dumps of all Postgres + MariaDB databases (clean, version-portable)
|
||||
# * the /config app-state tree (app configs, vault, portainer, couchdb, raw datadirs)
|
||||
# * native Plex "Application Support" (watch history, collections, metadata)
|
||||
#
|
||||
# NOT copied here (they live on /storage1/labdata and ride along on the disks):
|
||||
# MinIO blobs (famapp/resume/plane), gitea repos+LFS, docker registry, media.
|
||||
#
|
||||
# Best-effort by design: a failed individual DB dump logs a WARNING and is skipped
|
||||
# (its raw datadir is still inside config.tar) — it never blocks the /config/Plex tars.
|
||||
#
|
||||
# Run ON valhalla as ginnoir:
|
||||
# bash migration-backup.sh # live insurance backup (no downtime)
|
||||
# bash migration-backup.sh --stop # consistent backup: stop app containers first,
|
||||
# # then restart them when done (rehearsal-safe)
|
||||
#
|
||||
# Output: /storage1/migration-backup/<timestamp>/ (+ SHA256SUMS)
|
||||
#
|
||||
set -uo pipefail # NOTE: deliberately no `set -e` — we want best-effort completion.
|
||||
|
||||
DEST="/storage1/migration-backup"
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
OUT="$DEST/$STAMP"
|
||||
STOP=0
|
||||
[[ "${1:-}" == "--stop" ]] && STOP=1
|
||||
|
||||
mkdir -p "$OUT/db" "$OUT/config" "$OUT/plex"
|
||||
LOG="$OUT/backup.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
log(){ printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
running(){ docker ps --format '{{.Names}}' | grep -qx "$1"; }
|
||||
|
||||
# Resolve a working Postgres superuser for a container (env POSTGRES_USER, then
|
||||
# common fallbacks). Echoes the user on success; non-zero if none connect.
|
||||
pg_super(){
|
||||
local c="$1" cand
|
||||
for cand in "$(docker exec "$c" printenv POSTGRES_USER 2>/dev/null)" \
|
||||
"$(docker exec "$c" printenv POSTGRES_DB 2>/dev/null)" \
|
||||
immich postgres; do
|
||||
[ -z "$cand" ] && continue
|
||||
if docker exec "$c" psql -U "$cand" -tAc 'select 1' >/dev/null 2>&1; then
|
||||
echo "$cand"; return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
log "migration backup -> $OUT (stop=$STOP)"
|
||||
|
||||
PG_CONTAINERS="postgres_authentik postgres_famapp postgres_gitea postgres_plane postgres_resume nextcloud-postgres immich-postgres"
|
||||
MARIA_CONTAINERS="owncloud_mariadb romm-db mariadb_bookstack"
|
||||
|
||||
# --- 1. logical DB dumps (best-effort; authoritative restore path) ---
|
||||
for c in $PG_CONTAINERS; do
|
||||
running "$c" || { log "SKIP $c (not running)"; continue; }
|
||||
if u="$(pg_super "$c")"; then
|
||||
log "pg_dumpall $c (user=$u)"
|
||||
if docker exec "$c" pg_dumpall --clean --if-exists -U "$u" | gzip > "$OUT/db/${c}.sql.gz"; then
|
||||
:
|
||||
else
|
||||
log "WARN: pg_dumpall $c FAILED — raw pgdata remains in config.tar"; rm -f "$OUT/db/${c}.sql.gz"
|
||||
fi
|
||||
else
|
||||
log "WARN: no working superuser found for $c — raw pgdata remains in config.tar"
|
||||
fi
|
||||
done
|
||||
|
||||
for c in $MARIA_CONTAINERS; do
|
||||
running "$c" || { log "SKIP $c (not running)"; continue; }
|
||||
log "mariadb-dump $c"
|
||||
pw="$(docker exec "$c" sh -c 'printf %s "${MYSQL_ROOT_PASSWORD:-${MARIADB_ROOT_PASSWORD:-}}"' 2>/dev/null)"
|
||||
if docker exec -e MYSQL_PWD="$pw" "$c" sh -c \
|
||||
'mariadb-dump -uroot --all-databases --single-transaction --routines --triggers --events 2>/dev/null \
|
||||
|| mysqldump -uroot --all-databases --single-transaction --routines --triggers --events' \
|
||||
| gzip > "$OUT/db/${c}.sql.gz"; then
|
||||
:
|
||||
else
|
||||
log "WARN: mariadb-dump $c FAILED — raw datadir remains in config.tar"; rm -f "$OUT/db/${c}.sql.gz"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- 2. optionally stop app containers for a fully-consistent /config snapshot ---
|
||||
STOPPED=""
|
||||
if [[ "$STOP" == "1" ]]; then
|
||||
log "stopping app containers for a consistent /config snapshot (portainer left up)"
|
||||
KEEP="portainer"
|
||||
for c in $(docker ps --format '{{.Names}}'); do
|
||||
case " $KEEP " in *" $c "*) continue;; esac
|
||||
docker stop "$c" >/dev/null && STOPPED="$STOPPED $c"
|
||||
done
|
||||
log "stopped:$STOPPED"
|
||||
fi
|
||||
|
||||
# --- 3. /config tree (app state, vault, couchdb, raw datadirs) ---
|
||||
log "tar /config (the big one, ~100GB)"
|
||||
sudo tar --warning=no-file-changed -cf "$OUT/config/config.tar" -C / config || log "WARN: /config tar returned non-zero"
|
||||
|
||||
# --- 4. native Plex Application Support ---
|
||||
log "tar Plex Application Support"
|
||||
tar --warning=no-file-changed -cf "$OUT/plex/plex-appsupport.tar" \
|
||||
-C "/var/lib/plexmediaserver/Library/Application Support" "Plex Media Server" || log "WARN: Plex tar returned non-zero"
|
||||
|
||||
# --- 5. restart anything we stopped (leave the box as we found it) ---
|
||||
if [[ -n "$STOPPED" ]]; then
|
||||
log "restarting stopped containers"
|
||||
# shellcheck disable=SC2086
|
||||
docker start $STOPPED >/dev/null
|
||||
fi
|
||||
|
||||
# --- 6. inventory + checksums ---
|
||||
log "writing inventory + checksums"
|
||||
docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Label "com.docker.compose.project"}}' > "$OUT/inventory-containers.txt"
|
||||
docker network ls > "$OUT/inventory-networks.txt"
|
||||
cp /config/caddy/Caddyfile "$OUT/Caddyfile" 2>/dev/null || true
|
||||
cp /home/ginnoir/valhalla-lab/portainer-compose.yml "$OUT/portainer-compose.yml" 2>/dev/null || true
|
||||
cp /home/ginnoir/valhalla-lab/.env "$OUT/valhalla-lab.env" 2>/dev/null || true
|
||||
( cd "$OUT" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS )
|
||||
|
||||
log "sizes:"
|
||||
du -sh "$OUT"/db "$OUT"/config "$OUT"/plex 2>/dev/null || true
|
||||
du -sh "$OUT" 2>/dev/null || true
|
||||
echo "=== db dumps ==="; ls -lh "$OUT/db"
|
||||
log "DONE -> $OUT"
|
||||
echo
|
||||
echo "NOTE: MinIO blobs, gitea repos/LFS, the registry, and all media are on /storage1"
|
||||
echo " and travel with the pool disks; they are intentionally NOT in this backup."
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# CUTOVER final backup — valhalla R510 (run as ginnoir, right before zpool export).
|
||||
#
|
||||
# Takes the FINAL consistent backup of everything not already on the pool, lands it on
|
||||
# /storage1 (which travels with the disks), then leaves all services STOPPED.
|
||||
# Does NOT restart services and does NOT export the pool (that's a separate confirmed step).
|
||||
#
|
||||
# Order matters: dump DBs while their containers are still up, THEN stop everything, THEN
|
||||
# tar /config (consistent), THEN stop Plex + delta-rsync it.
|
||||
#
|
||||
set -uo pipefail
|
||||
DEST=/storage1/migration-backup
|
||||
STAMP=$(date +%Y%m%d-%H%M%S)
|
||||
OUT="$DEST/cutover-$STAMP"
|
||||
mkdir -p "$OUT/db" "$OUT/config"
|
||||
LOG="$OUT/cutover.log"; exec > >(tee -a "$LOG") 2>&1
|
||||
log(){ printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
running(){ docker ps --format '{{.Names}}' | grep -qx "$1"; }
|
||||
pg_super(){ local c="$1" cand; for cand in "$(docker exec "$c" printenv POSTGRES_USER 2>/dev/null)" immich postgres; do [ -z "$cand" ] && continue; docker exec "$c" psql -U "$cand" -tAc 'select 1' >/dev/null 2>&1 && { echo "$cand"; return 0; }; done; return 1; }
|
||||
|
||||
log "===== CUTOVER FINAL BACKUP -> $OUT ====="
|
||||
|
||||
# --- 1. dump all databases (containers still running) ---
|
||||
for c in postgres_authentik postgres_famapp postgres_gitea postgres_plane postgres_resume nextcloud-postgres immich-postgres; do
|
||||
running "$c" || { log "skip $c (down)"; continue; }
|
||||
if u=$(pg_super "$c"); then
|
||||
docker exec "$c" pg_dumpall --clean --if-exists -U "$u" | gzip > "$OUT/db/$c.sql.gz" && log "pg $c (u=$u) ok" || { log "WARN pg $c failed"; rm -f "$OUT/db/$c.sql.gz"; }
|
||||
else log "WARN no superuser for $c"; fi
|
||||
done
|
||||
# owncloud + romm: root w/ env password, named DB
|
||||
dump_maria_pw(){ local c="$1" db="$2" pw; pw=$(docker exec "$c" sh -c 'printf %s "${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-}}"' 2>/dev/null); docker exec -e MYSQL_PWD="$pw" "$c" mariadb-dump -uroot --single-transaction --routines --triggers "$db" 2>/dev/null | gzip > "$OUT/db/$c.sql.gz" && log "maria $c ok" || { log "WARN maria $c failed"; rm -f "$OUT/db/$c.sql.gz"; }; }
|
||||
running owncloud_mariadb && dump_maria_pw owncloud_mariadb owncloud
|
||||
running romm-db && dump_maria_pw romm-db romm
|
||||
# bookstack: root via unix_socket, all DBs
|
||||
running mariadb_bookstack && { docker exec mariadb_bookstack mariadb-dump -uroot --single-transaction --all-databases 2>/dev/null | gzip > "$OUT/db/mariadb_bookstack.sql.gz" && log "maria bookstack ok" || log "WARN bookstack failed"; }
|
||||
log "db dumps:"; ls -lh "$OUT/db"
|
||||
|
||||
# --- 2. stop ALL containers (homelab goes down) ---
|
||||
log "stopping all containers..."
|
||||
docker stop $(docker ps -q) >/dev/null 2>&1; sleep 2
|
||||
log "running containers now: $(docker ps -q | wc -l) (expect 0)"
|
||||
|
||||
# --- 3. tar /config (now fully consistent) ---
|
||||
log "tar /config (~112G, the long step)"
|
||||
sudo tar --warning=no-file-changed -cf "$OUT/config/config.tar" -C / config && log "config.tar done" || log "WARN config tar non-zero"
|
||||
|
||||
# --- 4. Plex: stop + final delta rsync ---
|
||||
log "stopping Plex + final delta rsync"
|
||||
sudo systemctl stop plexmediaserver
|
||||
sudo rsync -aHX "/var/lib/plexmediaserver/Library/Application Support/Plex Media Server" /storage1/labdata/plex/ && log "plex delta synced" || log "WARN plex rsync"
|
||||
|
||||
# --- 5. checksums + sizes ---
|
||||
log "checksums"
|
||||
( cd "$OUT" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS )
|
||||
du -sh "$OUT"/db "$OUT"/config 2>/dev/null
|
||||
log "===== CUTOVER BACKUP COMPLETE. Services stopped. Ready to release /storage1 + export. ====="
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Pre-migration: make the Kopia -> B2 snapshots current AND complete.
|
||||
#
|
||||
# 1) waits for the running migration-backup.sh to finish (avoid /config read contention)
|
||||
# 2) supplements the logical DB dumps the nightly job omits (nextcloud, immich) and
|
||||
# replaces the empty bookstack dump, using the complete set from migration-backup.sh
|
||||
# 3) runs the existing Kopia snapshot pipeline -> fresh B2 snapshot of /config + dumps + labdata
|
||||
#
|
||||
# Run ON valhalla as ginnoir: nohup bash ~/refresh-b2-snapshot.sh > ~/refresh-b2-snapshot.out 2>&1 &
|
||||
#
|
||||
set -uo pipefail
|
||||
log(){ printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
|
||||
# 1. wait for the migration backup to complete
|
||||
log "waiting for migration-backup.sh to finish..."
|
||||
while pgrep -f 'migration-backup.sh' >/dev/null 2>&1; do sleep 15; done
|
||||
log "migration backup finished"
|
||||
|
||||
# locate newest migration backup db dir and newest official dump dir
|
||||
MIG_DB="$(ls -1dt /storage1/migration-backup/*/db 2>/dev/null | head -1)"
|
||||
DUMPS="$(ls -1dt /config/backup/dumps/*/ 2>/dev/null | head -1)"
|
||||
log "migration db dir: ${MIG_DB:-<none>}"
|
||||
log "official dump dir: ${DUMPS:-<none>}"
|
||||
|
||||
# 2. supplement the dumps the nightly job misses (raw datadirs already cover them in /config,
|
||||
# but this gives B2 a complete logical-dump layer too)
|
||||
if [ -n "${MIG_DB:-}" ] && [ -n "${DUMPS:-}" ]; then
|
||||
for f in nextcloud-postgres.sql.gz immich-postgres.sql.gz mariadb_bookstack.sql.gz; do
|
||||
if [ -s "$MIG_DB/$f" ]; then
|
||||
sudo cp -f "$MIG_DB/$f" "$DUMPS/$f" && log "supplemented $f ($(du -h "$DUMPS/$f" | cut -f1))"
|
||||
else
|
||||
log "WARN: $MIG_DB/$f missing/empty — not supplemented"
|
||||
fi
|
||||
done
|
||||
log "dump dir now:"; ls -lh "$DUMPS"
|
||||
fi
|
||||
|
||||
# 3. fresh Kopia -> B2 snapshot (same pipeline cron uses)
|
||||
log "running Kopia snapshot to B2..."
|
||||
docker exec backup_scheduler sh /scripts/backup-run-snapshots.sh 2>&1
|
||||
|
||||
log "DONE — verify with: docker exec kopia sh -c 'KOPIA_PASSWORD=\$KOPIA_REPOSITORY_PASSWORD kopia snapshot list --all' | tail"
|
||||
Reference in New Issue
Block a user