ops: add production hardening automation for secrets, backups, and rollback
This commit is contained in:
Austin A
186
infra/deploy/rollback-blue-green.sh
Executable file
186
infra/deploy/rollback-blue-green.sh
Executable file
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/proxpanel}"
|
||||
BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
|
||||
GREEN_DIR="${GREEN_DIR:-/opt/proxpanel-green}"
|
||||
GREEN_PROJECT="${GREEN_PROJECT:-proxpanel_green}"
|
||||
GREEN_FRONTEND_PORT="${GREEN_FRONTEND_PORT:-18081}"
|
||||
GREEN_BACKEND_PORT="${GREEN_BACKEND_PORT:-18080}"
|
||||
NGINX_SITE="${NGINX_SITE:-/etc/nginx/sites-available/proxpanel.conf}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
sudo bash infra/deploy/rollback-blue-green.sh --backup-dir /opt/proxpanel-backups/<timestamp> [--cutover]
|
||||
|
||||
Behavior:
|
||||
1) Stages backup app/env into /opt/proxpanel-green
|
||||
2) Boots green stack on ports 18081/18080
|
||||
3) Restores DB dump from selected backup into green postgres
|
||||
4) Verifies green API health
|
||||
5) If --cutover is provided, switches Nginx upstream to green frontend port
|
||||
|
||||
Notes:
|
||||
- This script requires backup directory format created by deployment steps:
|
||||
<backup-dir>/app/
|
||||
<backup-dir>/env.production.bak (or .env.production.bak)
|
||||
<backup-dir>/db_pre_*.sql
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf '\n[ERROR] %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_root() {
|
||||
[[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Run as root (or with sudo)."
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
BACKUP_DIR=""
|
||||
CUTOVER="false"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--backup-dir)
|
||||
BACKUP_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--cutover)
|
||||
CUTOVER="true"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "Unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$BACKUP_DIR" ]] || die "--backup-dir is required"
|
||||
}
|
||||
|
||||
pick_db_dump() {
|
||||
find "$BACKUP_DIR" -maxdepth 1 -type f -name 'db_pre*.sql' | sort | tail -n 1
|
||||
}
|
||||
|
||||
pick_env_backup() {
|
||||
if [[ -f "$BACKUP_DIR/env.production.bak" ]]; then
|
||||
printf '%s' "$BACKUP_DIR/env.production.bak"
|
||||
return
|
||||
fi
|
||||
if [[ -f "$BACKUP_DIR/.env.production.bak" ]]; then
|
||||
printf '%s' "$BACKUP_DIR/.env.production.bak"
|
||||
return
|
||||
fi
|
||||
printf ''
|
||||
}
|
||||
|
||||
wait_for_green_health() {
|
||||
local i
|
||||
for i in $(seq 1 90); do
|
||||
if curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
die "Green backend health check failed."
|
||||
}
|
||||
|
||||
main() {
|
||||
require_root
|
||||
parse_args "$@"
|
||||
|
||||
[[ -d "$BACKUP_DIR/app" ]] || die "Missing backup app directory: $BACKUP_DIR/app"
|
||||
local env_backup
|
||||
env_backup="$(pick_env_backup)"
|
||||
[[ -n "$env_backup" ]] || die "Missing env backup in $BACKUP_DIR (.env.production.bak or env.production.bak)"
|
||||
local db_dump
|
||||
db_dump="$(pick_db_dump)"
|
||||
[[ -n "$db_dump" ]] || die "Missing DB dump in backup dir: $BACKUP_DIR"
|
||||
|
||||
log "Preparing green staging directory at $GREEN_DIR"
|
||||
rm -rf "$GREEN_DIR"
|
||||
mkdir -p "$GREEN_DIR"
|
||||
rsync -a "$BACKUP_DIR/app/" "$GREEN_DIR/"
|
||||
cp "$env_backup" "$GREEN_DIR/.env.production"
|
||||
|
||||
# Override ports for green stack and keep CORS aligned to production hostname.
|
||||
if grep -q '^FRONTEND_PORT=' "$GREEN_DIR/.env.production"; then
|
||||
sed -i "s/^FRONTEND_PORT=.*/FRONTEND_PORT=${GREEN_FRONTEND_PORT}/" "$GREEN_DIR/.env.production"
|
||||
else
|
||||
echo "FRONTEND_PORT=${GREEN_FRONTEND_PORT}" >> "$GREEN_DIR/.env.production"
|
||||
fi
|
||||
if grep -q '^BACKEND_PORT=' "$GREEN_DIR/.env.production"; then
|
||||
sed -i "s/^BACKEND_PORT=.*/BACKEND_PORT=${GREEN_BACKEND_PORT}/" "$GREEN_DIR/.env.production"
|
||||
else
|
||||
echo "BACKEND_PORT=${GREEN_BACKEND_PORT}" >> "$GREEN_DIR/.env.production"
|
||||
fi
|
||||
|
||||
# Remove fixed container_name lines for project-isolated blue/green operation.
|
||||
local compose_green
|
||||
compose_green="$GREEN_DIR/infra/deploy/docker-compose.green.yml"
|
||||
awk '!/^[[:space:]]*container_name:/' "$GREEN_DIR/infra/deploy/docker-compose.production.yml" > "$compose_green"
|
||||
|
||||
log "Stopping previous green project (if any)"
|
||||
(
|
||||
cd "$GREEN_DIR"
|
||||
docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" down -v || true
|
||||
)
|
||||
|
||||
log "Starting green stack on ${GREEN_FRONTEND_PORT}/${GREEN_BACKEND_PORT}"
|
||||
(
|
||||
cd "$GREEN_DIR"
|
||||
docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" up -d --build
|
||||
)
|
||||
|
||||
log "Restoring backup DB into green postgres"
|
||||
local green_postgres_id green_pg_user green_pg_db
|
||||
green_postgres_id="$(
|
||||
cd "$GREEN_DIR" && docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" ps -q postgres
|
||||
)"
|
||||
[[ -n "$green_postgres_id" ]] || die "Could not locate green postgres container."
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$GREEN_DIR/.env.production"
|
||||
green_pg_user="${POSTGRES_USER:-proxpanel}"
|
||||
green_pg_db="${POSTGRES_DB:-proxpanel}"
|
||||
|
||||
docker exec "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'
|
||||
|
||||
cat "$db_dump" | docker exec -i "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 >/dev/null
|
||||
|
||||
log "Validating green health"
|
||||
wait_for_green_health
|
||||
curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null
|
||||
|
||||
if [[ "$CUTOVER" == "true" ]]; then
|
||||
log "Cutting over Nginx upstream to green frontend port ${GREEN_FRONTEND_PORT}"
|
||||
[[ -f "$NGINX_SITE" ]] || die "Missing nginx site config: $NGINX_SITE"
|
||||
cp "$NGINX_SITE" "${NGINX_SITE}.pre-green.$(date -u +%Y%m%d-%H%M%S).bak"
|
||||
sed -i -E "s#proxy_pass http://127\\.0\\.0\\.1:[0-9]+;#proxy_pass http://127.0.0.1:${GREEN_FRONTEND_PORT};#g" "$NGINX_SITE"
|
||||
nginx -t
|
||||
systemctl reload nginx
|
||||
log "Nginx cutover complete."
|
||||
fi
|
||||
|
||||
log "Blue/green rollback prep complete."
|
||||
printf '\nGreen frontend: http://127.0.0.1:%s\n' "$GREEN_FRONTEND_PORT"
|
||||
printf 'Green backend : http://127.0.0.1:%s/api/health\n' "$GREEN_BACKEND_PORT"
|
||||
if [[ "$CUTOVER" == "true" ]]; then
|
||||
printf 'Traffic switched to green.\n'
|
||||
else
|
||||
printf 'Traffic still on blue. Re-run with --cutover when ready.\n'
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user