From 95633a6722624692421e7eba2e3f82c5d52d7459 Mon Sep 17 00:00:00 2001 From: Austin A Date: Sat, 18 Apr 2026 09:13:13 +0100 Subject: [PATCH] ops: add production hardening automation for secrets, backups, and rollback --- infra/deploy/SECRET_ROTATION_CHECKLIST.md | 68 +++++++ infra/deploy/configure-db-backup-cron.sh | 87 +++++++++ infra/deploy/db-backup-encrypted.sh | 74 ++++++++ infra/deploy/db-restore-test.sh | 114 ++++++++++++ infra/deploy/rollback-blue-green.sh | 186 +++++++++++++++++++ infra/deploy/rotate-production-secrets.sh | 209 ++++++++++++++++++++++ 6 files changed, 738 insertions(+) create mode 100644 infra/deploy/SECRET_ROTATION_CHECKLIST.md create mode 100755 infra/deploy/configure-db-backup-cron.sh create mode 100755 infra/deploy/db-backup-encrypted.sh create mode 100755 infra/deploy/db-restore-test.sh create mode 100755 infra/deploy/rollback-blue-green.sh create mode 100755 infra/deploy/rotate-production-secrets.sh diff --git a/infra/deploy/SECRET_ROTATION_CHECKLIST.md b/infra/deploy/SECRET_ROTATION_CHECKLIST.md new file mode 100644 index 0000000..7105869 --- /dev/null +++ b/infra/deploy/SECRET_ROTATION_CHECKLIST.md @@ -0,0 +1,68 @@ +# Secret Rotation Checklist (Production) + +Target: `my.votcloud.com` +Host: `102.69.243.167` + +## Scope + +Rotate the following regularly (monthly/quarterly or after any incident): + +1. `JWT_SECRET` +2. `JWT_REFRESH_SECRET` +3. `POSTGRES_PASSWORD` +4. `ADMIN_PASSWORD` + +Recommended: + +1. `BACKUP_ENCRYPTION_KEY` (with controlled key migration plan) +2. Proxmox API token secret +3. Payment/webhook secrets + +## Runbook (Safe Order) + +1. Create timestamped app/env/db backup. +2. Rotate env secrets in `.env.production`. +3. Apply DB password rotation (`ALTER USER ... WITH PASSWORD ...`). +4. Restart app stack with new env. +5. Re-seed admin (`npm run prisma:seed`) to sync rotated admin password. +6. Revoke all active sessions (`AuthSession`) to invalidate old sessions. +7. Verify: + - `GET /api/health` + - Admin login + - Core pages (`/rbac`, `/profile`, `/system`, `/audit-logs`) +8. Save secure summary with new admin credentials under `/root/`. + +## Automation Script + +Use: + +```bash +sudo bash /opt/proxpanel/infra/deploy/rotate-production-secrets.sh +``` + +Script guarantees: + +1. Backup directory created in `/opt/proxpanel-backups/-secret-rotation/` +2. DB pre-rotation dump created +3. Post-rotation health + login verified +4. Summary written to `/root/proxpanel-secret-rotation-.txt` + +## Rollback Plan + +If post-rotation checks fail: + +1. Restore `.env.production` from backup. +2. Restore previous app files if needed. +3. Restore DB dump if schema/state corruption occurred. +4. Recreate containers: + - `docker compose --env-file .env.production -f infra/deploy/docker-compose.production.yml up -d --build` + +## Audit Trail + +Store: + +1. Rotation timestamp +2. Operator identity +3. Backup directory used +4. Health verification evidence +5. Any rollback events diff --git a/infra/deploy/configure-db-backup-cron.sh b/infra/deploy/configure-db-backup-cron.sh new file mode 100755 index 0000000..d50a16b --- /dev/null +++ b/infra/deploy/configure-db-backup-cron.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" +CRON_FILE="${CRON_FILE:-/etc/cron.d/proxpanel-db-backup}" +BACKUP_LOG="${BACKUP_LOG:-/var/log/proxpanel-db-backup.log}" +RESTORE_LOG="${RESTORE_LOG:-/var/log/proxpanel-db-restore-test.log}" + +usage() { + cat <<'EOF' +Usage: + sudo bash infra/deploy/configure-db-backup-cron.sh [--run-now] + +Default schedule (UTC): + - 02:15 daily: encrypted DB backup + - 02:45 daily: restore test against latest encrypted backup +EOF +} + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +random_key() { + openssl rand -hex 64 | tr -d '\n' +} + +main() { + local run_now="false" + if [[ "${1:-}" == "--run-now" ]]; then + run_now="true" + elif [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 + elif [[ -n "${1:-}" ]]; then + die "Unknown argument: $1" + fi + + [[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Run as root (or with sudo)." + [[ -d "$APP_DIR" ]] || die "Missing app dir: $APP_DIR" + + if [[ ! -f "$SECRET_FILE" ]]; then + log "Creating $SECRET_FILE" + umask 077 + cat >"$SECRET_FILE" <"$CRON_FILE" <<'EOF' +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Encrypted PostgreSQL backup every day at 02:15 UTC +15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-encrypted.sh >> /var/log/proxpanel-db-backup.log 2>&1 + +# Restore test every day at 02:45 UTC +45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test.sh >> /var/log/proxpanel-db-restore-test.log 2>&1 +EOF + chmod 644 "$CRON_FILE" + + touch "$BACKUP_LOG" "$RESTORE_LOG" + chmod 640 "$BACKUP_LOG" "$RESTORE_LOG" + + if [[ "$run_now" == "true" ]]; then + log "Running immediate backup + restore test" + APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-encrypted.sh" + APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test.sh" + fi + + log "DB backup/restore cron configured successfully." +} + +main "$@" diff --git a/infra/deploy/db-backup-encrypted.sh b/infra/deploy/db-backup-encrypted.sh new file mode 100755 index 0000000..7dea745 --- /dev/null +++ b/infra/deploy/db-backup-encrypted.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}" +RETENTION_DAYS="${RETENTION_DAYS:-14}" + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_file() { + [[ -f "$1" ]] || die "Missing required file: $1" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +main() { + require_command docker + require_command openssl + require_command sha256sum + require_file "$ENV_FILE" + require_file "$SECRET_FILE" + + # shellcheck disable=SC1090 + source "$ENV_FILE" + # shellcheck disable=SC1090 + source "$SECRET_FILE" + + [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]] || die "BACKUP_ENCRYPTION_KEY is empty in $SECRET_FILE" + export BACKUP_ENCRYPTION_KEY + [[ -n "${POSTGRES_USER:-}" ]] || die "POSTGRES_USER missing in $ENV_FILE" + [[ -n "${POSTGRES_DB:-}" ]] || die "POSTGRES_DB missing in $ENV_FILE" + + local ts backup_dir plain_sql encrypted_sql + ts="$(date -u +%Y%m%d-%H%M%S)" + backup_dir="${BACKUP_ROOT}/${ts}" + plain_sql="${backup_dir}/proxpanel.sql" + encrypted_sql="${plain_sql}.enc" + + mkdir -p "$backup_dir" + chmod 700 "$backup_dir" + + log "Creating plaintext dump" + docker exec proxpanel-postgres pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists >"$plain_sql" + + log "Encrypting dump" + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \ + -in "$plain_sql" \ + -out "$encrypted_sql" \ + -pass env:BACKUP_ENCRYPTION_KEY + + log "Writing checksum" + sha256sum "$encrypted_sql" >"${encrypted_sql}.sha256" + + rm -f "$plain_sql" + chmod 600 "$encrypted_sql" "${encrypted_sql}.sha256" + + log "Applying retention policy (${RETENTION_DAYS} days)" + find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +"$RETENTION_DAYS" -exec rm -rf {} + + + log "Encrypted backup created: $encrypted_sql" +} + +main "$@" diff --git a/infra/deploy/db-restore-test.sh b/infra/deploy/db-restore-test.sh new file mode 100755 index 0000000..a294615 --- /dev/null +++ b/infra/deploy/db-restore-test.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}" +TMP_ROOT="${TMP_ROOT:-/tmp/proxpanel-restore-test}" +TEST_CONTAINER="${TEST_CONTAINER:-proxpanel-restore-test}" +PG_IMAGE="${PG_IMAGE:-postgres:16-alpine}" +PG_USER="${PG_USER:-proxpanel}" +PG_PASSWORD="${PG_PASSWORD:-restoretestpass}" +PG_DB="${PG_DB:-proxpanel_restore}" + +cleanup() { + docker rm -f "$TEST_CONTAINER" >/dev/null 2>&1 || true + rm -rf "$TMP_ROOT" +} +trap cleanup EXIT + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_file() { + [[ -f "$1" ]] || die "Missing required file: $1" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +find_latest_encrypted_backup() { + find "$BACKUP_ROOT" -mindepth 2 -maxdepth 2 -type f -name 'proxpanel.sql.enc' | sort | tail -n 1 +} + +wait_pg_ready() { + local tries=60 + local i + for ((i=1; i<=tries; i++)); do + if docker exec "$TEST_CONTAINER" pg_isready -U "$PG_USER" -d "$PG_DB" >/dev/null 2>&1; then + return + fi + sleep 1 + done + die "Restore test postgres did not become ready." +} + +main() { + require_command docker + require_command openssl + require_command sha256sum + require_file "$SECRET_FILE" + + # shellcheck disable=SC1090 + source "$SECRET_FILE" + [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]] || die "BACKUP_ENCRYPTION_KEY is empty in $SECRET_FILE" + export BACKUP_ENCRYPTION_KEY + + local encrypted_backup checksum_file latest_dir decrypted_sql + encrypted_backup="$(find_latest_encrypted_backup)" + [[ -n "$encrypted_backup" ]] || die "No encrypted backup found in $BACKUP_ROOT" + checksum_file="${encrypted_backup}.sha256" + require_file "$checksum_file" + + latest_dir="$(dirname "$encrypted_backup")" + mkdir -p "$TMP_ROOT" + chmod 700 "$TMP_ROOT" + decrypted_sql="${TMP_ROOT}/restore.sql" + + log "Verifying checksum for $encrypted_backup" + (cd "$latest_dir" && sha256sum -c "$(basename "$checksum_file")") + + log "Decrypting latest backup" + openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 \ + -in "$encrypted_backup" \ + -out "$decrypted_sql" \ + -pass env:BACKUP_ENCRYPTION_KEY + + log "Starting isolated restore-test postgres container" + docker rm -f "$TEST_CONTAINER" >/dev/null 2>&1 || true + docker run -d --name "$TEST_CONTAINER" \ + -e POSTGRES_USER="$PG_USER" \ + -e POSTGRES_PASSWORD="$PG_PASSWORD" \ + -e POSTGRES_DB="$PG_DB" \ + "$PG_IMAGE" >/dev/null + + wait_pg_ready + + log "Applying restored SQL into test DB" + cat "$decrypted_sql" | docker exec -i "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" >/dev/null + + log "Running restore sanity checks" + local table_count required_table_count + table_count="$( + docker exec "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -Atc \ + "select count(*) from information_schema.tables where table_schema='public';" + )" + required_table_count="$( + docker exec "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -Atc \ + "select count(*) from information_schema.tables where table_schema='public' and table_name in ('User','Tenant','AuditLog');" + )" + + [[ "${table_count:-0}" -ge 10 ]] || die "Restore sanity check failed (unexpected table count: $table_count)" + [[ "${required_table_count:-0}" -eq 3 ]] || die "Restore sanity check failed (required tables missing)" + + log "Restore test passed (tables=$table_count)" +} + +main "$@" diff --git a/infra/deploy/rollback-blue-green.sh b/infra/deploy/rollback-blue-green.sh new file mode 100755 index 0000000..0ce4c59 --- /dev/null +++ b/infra/deploy/rollback-blue-green.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}" +GREEN_DIR="${GREEN_DIR:-/opt/proxpanel-green}" +GREEN_PROJECT="${GREEN_PROJECT:-proxpanel_green}" +GREEN_FRONTEND_PORT="${GREEN_FRONTEND_PORT:-18081}" +GREEN_BACKEND_PORT="${GREEN_BACKEND_PORT:-18080}" +NGINX_SITE="${NGINX_SITE:-/etc/nginx/sites-available/proxpanel.conf}" + +usage() { + cat <<'EOF' +Usage: + sudo bash infra/deploy/rollback-blue-green.sh --backup-dir /opt/proxpanel-backups/ [--cutover] + +Behavior: + 1) Stages backup app/env into /opt/proxpanel-green + 2) Boots green stack on ports 18081/18080 + 3) Restores DB dump from selected backup into green postgres + 4) Verifies green API health + 5) If --cutover is provided, switches Nginx upstream to green frontend port + +Notes: + - This script requires backup directory format created by deployment steps: + /app/ + /env.production.bak (or .env.production.bak) + /db_pre_*.sql +EOF +} + +log() { + printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '\n[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_root() { + [[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Run as root (or with sudo)." +} + +parse_args() { + BACKUP_DIR="" + CUTOVER="false" + while [[ $# -gt 0 ]]; do + case "$1" in + --backup-dir) + BACKUP_DIR="${2:-}" + shift 2 + ;; + --cutover) + CUTOVER="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "Unknown argument: $1" + ;; + esac + done + + [[ -n "$BACKUP_DIR" ]] || die "--backup-dir is required" +} + +pick_db_dump() { + find "$BACKUP_DIR" -maxdepth 1 -type f -name 'db_pre*.sql' | sort | tail -n 1 +} + +pick_env_backup() { + if [[ -f "$BACKUP_DIR/env.production.bak" ]]; then + printf '%s' "$BACKUP_DIR/env.production.bak" + return + fi + if [[ -f "$BACKUP_DIR/.env.production.bak" ]]; then + printf '%s' "$BACKUP_DIR/.env.production.bak" + return + fi + printf '' +} + +wait_for_green_health() { + local i + for i in $(seq 1 90); do + if curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null 2>&1; then + return + fi + sleep 2 + done + die "Green backend health check failed." +} + +main() { + require_root + parse_args "$@" + + [[ -d "$BACKUP_DIR/app" ]] || die "Missing backup app directory: $BACKUP_DIR/app" + local env_backup + env_backup="$(pick_env_backup)" + [[ -n "$env_backup" ]] || die "Missing env backup in $BACKUP_DIR (.env.production.bak or env.production.bak)" + local db_dump + db_dump="$(pick_db_dump)" + [[ -n "$db_dump" ]] || die "Missing DB dump in backup dir: $BACKUP_DIR" + + log "Preparing green staging directory at $GREEN_DIR" + rm -rf "$GREEN_DIR" + mkdir -p "$GREEN_DIR" + rsync -a "$BACKUP_DIR/app/" "$GREEN_DIR/" + cp "$env_backup" "$GREEN_DIR/.env.production" + + # Override ports for green stack and keep CORS aligned to production hostname. + if grep -q '^FRONTEND_PORT=' "$GREEN_DIR/.env.production"; then + sed -i "s/^FRONTEND_PORT=.*/FRONTEND_PORT=${GREEN_FRONTEND_PORT}/" "$GREEN_DIR/.env.production" + else + echo "FRONTEND_PORT=${GREEN_FRONTEND_PORT}" >> "$GREEN_DIR/.env.production" + fi + if grep -q '^BACKEND_PORT=' "$GREEN_DIR/.env.production"; then + sed -i "s/^BACKEND_PORT=.*/BACKEND_PORT=${GREEN_BACKEND_PORT}/" "$GREEN_DIR/.env.production" + else + echo "BACKEND_PORT=${GREEN_BACKEND_PORT}" >> "$GREEN_DIR/.env.production" + fi + + # Remove fixed container_name lines for project-isolated blue/green operation. + local compose_green + compose_green="$GREEN_DIR/infra/deploy/docker-compose.green.yml" + awk '!/^[[:space:]]*container_name:/' "$GREEN_DIR/infra/deploy/docker-compose.production.yml" > "$compose_green" + + log "Stopping previous green project (if any)" + ( + cd "$GREEN_DIR" + docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" down -v || true + ) + + log "Starting green stack on ${GREEN_FRONTEND_PORT}/${GREEN_BACKEND_PORT}" + ( + cd "$GREEN_DIR" + docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" up -d --build + ) + + log "Restoring backup DB into green postgres" + local green_postgres_id green_pg_user green_pg_db + green_postgres_id="$( + cd "$GREEN_DIR" && docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" ps -q postgres + )" + [[ -n "$green_postgres_id" ]] || die "Could not locate green postgres container." + + # shellcheck disable=SC1090 + source "$GREEN_DIR/.env.production" + green_pg_user="${POSTGRES_USER:-proxpanel}" + green_pg_db="${POSTGRES_DB:-proxpanel}" + + docker exec "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 \ + -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' + + cat "$db_dump" | docker exec -i "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 >/dev/null + + log "Validating green health" + wait_for_green_health + curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null + + if [[ "$CUTOVER" == "true" ]]; then + log "Cutting over Nginx upstream to green frontend port ${GREEN_FRONTEND_PORT}" + [[ -f "$NGINX_SITE" ]] || die "Missing nginx site config: $NGINX_SITE" + cp "$NGINX_SITE" "${NGINX_SITE}.pre-green.$(date -u +%Y%m%d-%H%M%S).bak" + sed -i -E "s#proxy_pass http://127\\.0\\.0\\.1:[0-9]+;#proxy_pass http://127.0.0.1:${GREEN_FRONTEND_PORT};#g" "$NGINX_SITE" + nginx -t + systemctl reload nginx + log "Nginx cutover complete." + fi + + log "Blue/green rollback prep complete." + printf '\nGreen frontend: http://127.0.0.1:%s\n' "$GREEN_FRONTEND_PORT" + printf 'Green backend : http://127.0.0.1:%s/api/health\n' "$GREEN_BACKEND_PORT" + if [[ "$CUTOVER" == "true" ]]; then + printf 'Traffic switched to green.\n' + else + printf 'Traffic still on blue. Re-run with --cutover when ready.\n' + fi +} + +main "$@" diff --git a/infra/deploy/rotate-production-secrets.sh b/infra/deploy/rotate-production-secrets.sh new file mode 100755 index 0000000..8fb641b --- /dev/null +++ b/infra/deploy/rotate-production-secrets.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}" +COMPOSE_FILE="${COMPOSE_FILE:-$APP_DIR/infra/deploy/docker-compose.production.yml}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}" + +usage() { + cat <<'EOF' +Usage: + sudo bash infra/deploy/rotate-production-secrets.sh + +What this rotates: + - JWT_SECRET + - JWT_REFRESH_SECRET + - POSTGRES_PASSWORD + - ADMIN_PASSWORD + +Safety: + - Creates backup in /opt/proxpanel-backups/-secret-rotation/ + - Dumps DB before password change + - Verifies API health and admin login after rotation +EOF +} + +log() { + printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '\n[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_root() { + if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then + die "Run as root (or with sudo)." + fi +} + +require_file() { + local file="$1" + [[ -f "$file" ]] || die "Missing required file: $file" +} + +require_command() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || die "Missing required command: $cmd" +} + +escape_sed_replacement() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//&/\\&}" + printf '%s' "$value" +} + +update_env_value() { + local key="$1" + local value="$2" + local escaped + escaped="$(escape_sed_replacement "$value")" + + if grep -q "^${key}=" "$ENV_FILE"; then + sed -i "s/^${key}=.*/${key}=${escaped}/" "$ENV_FILE" + else + printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE" + fi +} + +random_password() { + openssl rand -base64 36 | tr -d '\n' | tr '/+' 'ab' +} + +random_hex() { + openssl rand -hex 32 | tr -d '\n' +} + +wait_for_health() { + local max_tries=60 + local i + for ((i=1; i<=max_tries; i++)); do + if curl -fsS "http://127.0.0.1:${BACKEND_PORT}/api/health" >/dev/null 2>&1; then + return + fi + sleep 2 + done + die "Backend health check failed after secret rotation." +} + +main() { + if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 + fi + + require_root + require_command docker + require_command curl + require_command openssl + require_command rsync + require_file "$ENV_FILE" + require_file "$COMPOSE_FILE" + + # shellcheck disable=SC1090 + source "$ENV_FILE" + + local ts backup_dir compose_args + ts="$(date -u +%Y%m%d-%H%M%S)" + backup_dir="${BACKUP_ROOT}/${ts}-secret-rotation" + compose_args=(--env-file "$ENV_FILE" -f "$COMPOSE_FILE") + + local old_admin_email old_admin_password old_postgres_user old_postgres_db old_backend_port + old_admin_email="${ADMIN_EMAIL:-admin@proxpanel.local}" + old_admin_password="${ADMIN_PASSWORD:-}" + old_postgres_user="${POSTGRES_USER:-proxpanel}" + old_postgres_db="${POSTGRES_DB:-proxpanel}" + old_backend_port="${BACKEND_PORT:-8080}" + + local new_jwt_secret new_jwt_refresh_secret new_postgres_password new_admin_password + new_jwt_secret="$(random_password)$(random_password)" + new_jwt_refresh_secret="$(random_password)$(random_password)" + new_postgres_password="$(random_hex)" + new_admin_password="$(random_password)A9!" + + log "Creating pre-rotation backups in $backup_dir" + mkdir -p "$backup_dir" + cp "$ENV_FILE" "$backup_dir/.env.production.bak" + rsync -a "$APP_DIR/" "$backup_dir/app/" + docker exec proxpanel-postgres pg_dump -U "$old_postgres_user" -d "$old_postgres_db" >"$backup_dir/db_pre_rotation.sql" + + log "Updating env file with rotated secrets" + update_env_value "JWT_SECRET" "$new_jwt_secret" + update_env_value "JWT_REFRESH_SECRET" "$new_jwt_refresh_secret" + update_env_value "POSTGRES_PASSWORD" "$new_postgres_password" + update_env_value "ADMIN_PASSWORD" "$new_admin_password" + + # Re-load env values after edits. + # shellcheck disable=SC1090 + source "$ENV_FILE" + + log "Applying Postgres password rotation" + docker exec proxpanel-postgres psql -U "$old_postgres_user" -d "$old_postgres_db" -v ON_ERROR_STOP=1 \ + -c "ALTER USER \"$old_postgres_user\" WITH PASSWORD '$new_postgres_password';" + + log "Restarting stack with new secrets" + ( + cd "$APP_DIR" + docker compose "${compose_args[@]}" up -d + ) + + BACKEND_PORT="${BACKEND_PORT:-$old_backend_port}" + wait_for_health + + log "Re-seeding admin credential to match rotated ADMIN_PASSWORD" + ( + cd "$APP_DIR" + docker compose "${compose_args[@]}" exec -T backend npm run prisma:seed + ) + + log "Revoking all active auth sessions after JWT/password rotation" + docker exec proxpanel-postgres psql -U "${POSTGRES_USER:-$old_postgres_user}" -d "${POSTGRES_DB:-$old_postgres_db}" \ + -v ON_ERROR_STOP=1 -c 'TRUNCATE TABLE "AuthSession" RESTART IDENTITY;' + + log "Verifying post-rotation login" + local login_status + login_status="$( + curl -sS -o /tmp/proxpanel-rotate-login.json -w "%{http_code}" \ + -X POST "http://127.0.0.1:${BACKEND_PORT}/api/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"${ADMIN_EMAIL}\",\"password\":\"${ADMIN_PASSWORD}\"}" + )" + [[ "$login_status" == "200" ]] || die "Admin login failed after secret rotation (status $login_status)." + + local summary_file + summary_file="/root/proxpanel-secret-rotation-${ts}.txt" + cat >"$summary_file" <