ops: add integration secret rotation and offsite backup alerting
This commit is contained in:
Austin A
163
infra/deploy/db-backup-replicate-offsite.sh
Normal file
163
infra/deploy/db-backup-replicate-offsite.sh
Normal file
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/proxpanel}"
|
||||
SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
|
||||
BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-replicate-offsite.sh [--backup-dir <dir>]
|
||||
|
||||
Reads replication settings from /opt/proxpanel/.backup.env.
|
||||
Required keys for S3-compatible replication:
|
||||
OFFSITE_BACKUP_ENABLED=true
|
||||
OFFSITE_S3_BUCKET=<bucket-name>
|
||||
OFFSITE_S3_ACCESS_KEY_ID=<access-key>
|
||||
OFFSITE_S3_SECRET_ACCESS_KEY=<secret-key>
|
||||
|
||||
Optional:
|
||||
OFFSITE_S3_REGION=us-east-1
|
||||
OFFSITE_S3_PREFIX=proxpanel/db
|
||||
OFFSITE_S3_ENDPOINT_URL=https://s3.us-west-1.wasabisys.com
|
||||
OFFSITE_S3_SSE=AES256
|
||||
OFFSITE_REPLICA_RETENTION_DAYS=30
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf '[ERROR] %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_file() {
|
||||
[[ -f "$1" ]] || die "Missing required file: $1"
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
|
||||
}
|
||||
|
||||
find_latest_backup_dir() {
|
||||
find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1
|
||||
}
|
||||
|
||||
prune_remote_retention() {
|
||||
local bucket="$1"
|
||||
local prefix="$2"
|
||||
local endpoint_flag=()
|
||||
if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then
|
||||
endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL")
|
||||
fi
|
||||
|
||||
local days="${OFFSITE_REPLICA_RETENTION_DAYS:-}"
|
||||
if [[ -z "$days" || ! "$days" =~ ^[0-9]+$ || "$days" -le 0 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local cutoff
|
||||
cutoff="$(date -u -d "-${days} days" +%Y%m%d-%H%M%S)"
|
||||
log "Applying offsite retention policy (${days} days; cutoff=${cutoff})"
|
||||
|
||||
local listing prefixes old_prefix
|
||||
listing="$(aws "${endpoint_flag[@]}" s3 ls "s3://${bucket}/${prefix}/" || true)"
|
||||
prefixes="$(printf '%s\n' "$listing" | awk '/PRE [0-9]{8}-[0-9]{6}\// {print $2}' | tr -d '/')"
|
||||
|
||||
while IFS= read -r old_prefix; do
|
||||
[[ -n "$old_prefix" ]] || continue
|
||||
if [[ "$old_prefix" < "$cutoff" ]]; then
|
||||
log "Pruning remote backup prefix ${old_prefix}"
|
||||
aws "${endpoint_flag[@]}" s3 rm "s3://${bucket}/${prefix}/${old_prefix}/" --recursive >/dev/null
|
||||
fi
|
||||
done <<<"$prefixes"
|
||||
}
|
||||
|
||||
main() {
|
||||
local forced_backup_dir=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--backup-dir)
|
||||
forced_backup_dir="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "Unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
require_file "$SECRET_FILE"
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$SECRET_FILE"
|
||||
|
||||
if [[ "${OFFSITE_BACKUP_ENABLED:-false}" != "true" ]]; then
|
||||
log "Offsite replication disabled (OFFSITE_BACKUP_ENABLED != true)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_command aws
|
||||
|
||||
[[ -n "${OFFSITE_S3_BUCKET:-}" ]] || die "OFFSITE_S3_BUCKET is required."
|
||||
[[ -n "${OFFSITE_S3_ACCESS_KEY_ID:-}" ]] || die "OFFSITE_S3_ACCESS_KEY_ID is required."
|
||||
[[ -n "${OFFSITE_S3_SECRET_ACCESS_KEY:-}" ]] || die "OFFSITE_S3_SECRET_ACCESS_KEY is required."
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$OFFSITE_S3_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$OFFSITE_S3_SECRET_ACCESS_KEY"
|
||||
if [[ -n "${OFFSITE_S3_SESSION_TOKEN:-}" ]]; then
|
||||
export AWS_SESSION_TOKEN="$OFFSITE_S3_SESSION_TOKEN"
|
||||
fi
|
||||
export AWS_DEFAULT_REGION="${OFFSITE_S3_REGION:-us-east-1}"
|
||||
|
||||
local backup_dir
|
||||
if [[ -n "$forced_backup_dir" ]]; then
|
||||
backup_dir="$forced_backup_dir"
|
||||
else
|
||||
backup_dir="$(find_latest_backup_dir)"
|
||||
fi
|
||||
|
||||
[[ -n "$backup_dir" && -d "$backup_dir" ]] || die "Unable to locate backup directory."
|
||||
|
||||
local encrypted_file checksum_file stamp prefix endpoint_flag sse_flag destination
|
||||
encrypted_file="${backup_dir}/proxpanel.sql.enc"
|
||||
checksum_file="${encrypted_file}.sha256"
|
||||
require_file "$encrypted_file"
|
||||
require_file "$checksum_file"
|
||||
|
||||
stamp="$(basename "$backup_dir")"
|
||||
prefix="${OFFSITE_S3_PREFIX:-proxpanel/db}"
|
||||
destination="s3://${OFFSITE_S3_BUCKET}/${prefix}/${stamp}/"
|
||||
|
||||
endpoint_flag=()
|
||||
if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then
|
||||
endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL")
|
||||
fi
|
||||
|
||||
sse_flag=()
|
||||
if [[ -n "${OFFSITE_S3_SSE:-}" ]]; then
|
||||
sse_flag=(--sse "$OFFSITE_S3_SSE")
|
||||
fi
|
||||
|
||||
log "Replicating encrypted backup to ${destination}"
|
||||
aws "${endpoint_flag[@]}" s3 cp "$encrypted_file" "${destination}proxpanel.sql.enc" "${sse_flag[@]}" --only-show-errors
|
||||
aws "${endpoint_flag[@]}" s3 cp "$checksum_file" "${destination}proxpanel.sql.enc.sha256" "${sse_flag[@]}" --only-show-errors
|
||||
|
||||
log "Verifying offsite object presence"
|
||||
aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc" >/dev/null
|
||||
aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc.sha256" >/dev/null
|
||||
|
||||
prune_remote_retention "$OFFSITE_S3_BUCKET" "$prefix"
|
||||
log "Offsite replication completed successfully."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user