ops: add production hardening automation for secrets, backups, and rollback

infra/deploy/SECRET_ROTATION_CHECKLIST.md

@@ -0,0 +1,68 @@
+# Secret Rotation Checklist (Production)
+
+Target: `my.votcloud.com`  
+Host: `102.69.243.167`
+
+## Scope
+
+Rotate the following regularly (monthly/quarterly or after any incident):
+
+1. `JWT_SECRET`
+2. `JWT_REFRESH_SECRET`
+3. `POSTGRES_PASSWORD`
+4. `ADMIN_PASSWORD`
+
+Recommended:
+
+1. `BACKUP_ENCRYPTION_KEY` (with controlled key migration plan)
+2. Proxmox API token secret
+3. Payment/webhook secrets
+
+## Runbook (Safe Order)
+
+1. Create timestamped app/env/db backup.
+2. Rotate env secrets in `.env.production`.
+3. Apply DB password rotation (`ALTER USER ... WITH PASSWORD ...`).
+4. Restart app stack with new env.
+5. Re-seed admin (`npm run prisma:seed`) to sync rotated admin password.
+6. Revoke all active sessions (`AuthSession`) to invalidate old sessions.
+7. Verify:
+   - `GET /api/health`
+   - Admin login
+   - Core pages (`/rbac`, `/profile`, `/system`, `/audit-logs`)
+8. Save secure summary with new admin credentials under `/root/`.
+
+## Automation Script
+
+Use:
+
+```bash
+sudo bash /opt/proxpanel/infra/deploy/rotate-production-secrets.sh
+```
+
+Script guarantees:
+
+1. Backup directory created in `/opt/proxpanel-backups/<timestamp>-secret-rotation/`
+2. DB pre-rotation dump created
+3. Post-rotation health + login verified
+4. Summary written to `/root/proxpanel-secret-rotation-<timestamp>.txt`
+
+## Rollback Plan
+
+If post-rotation checks fail:
+
+1. Restore `.env.production` from backup.
+2. Restore previous app files if needed.
+3. Restore DB dump if schema/state corruption occurred.
+4. Recreate containers:
+   - `docker compose --env-file .env.production -f infra/deploy/docker-compose.production.yml up -d --build`
+
+## Audit Trail
+
+Store:
+
+1. Rotation timestamp
+2. Operator identity
+3. Backup directory used
+4. Health verification evidence
+5. Any rollback events

infra/deploy/configure-db-backup-cron.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+CRON_FILE="${CRON_FILE:-/etc/cron.d/proxpanel-db-backup}"
+BACKUP_LOG="${BACKUP_LOG:-/var/log/proxpanel-db-backup.log}"
+RESTORE_LOG="${RESTORE_LOG:-/var/log/proxpanel-db-restore-test.log}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  sudo bash infra/deploy/configure-db-backup-cron.sh [--run-now]
+
+Default schedule (UTC):
+  - 02:15 daily: encrypted DB backup
+  - 02:45 daily: restore test against latest encrypted backup
+EOF
+}
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+random_key() {
+  openssl rand -hex 64 | tr -d '\n'
+}
+
+main() {
+  local run_now="false"
+  if [[ "${1:-}" == "--run-now" ]]; then
+    run_now="true"
+  elif [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+    usage
+    exit 0
+  elif [[ -n "${1:-}" ]]; then
+    die "Unknown argument: $1"
+  fi
+
+  [[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Run as root (or with sudo)."
+  [[ -d "$APP_DIR" ]] || die "Missing app dir: $APP_DIR"
+
+  if [[ ! -f "$SECRET_FILE" ]]; then
+    log "Creating $SECRET_FILE"
+    umask 077
+    cat >"$SECRET_FILE" <<EOF
+BACKUP_ENCRYPTION_KEY=$(random_key)
+EOF
+  fi
+  chmod 600 "$SECRET_FILE"
+
+  log "Making scripts executable"
+  chmod +x \
+    "$APP_DIR/infra/deploy/db-backup-encrypted.sh" \
+    "$APP_DIR/infra/deploy/db-restore-test.sh"
+
+  log "Installing cron schedule at $CRON_FILE"
+  cat >"$CRON_FILE" <<'EOF'
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Encrypted PostgreSQL backup every day at 02:15 UTC
+15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-encrypted.sh >> /var/log/proxpanel-db-backup.log 2>&1
+
+# Restore test every day at 02:45 UTC
+45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test.sh >> /var/log/proxpanel-db-restore-test.log 2>&1
+EOF
+  chmod 644 "$CRON_FILE"
+
+  touch "$BACKUP_LOG" "$RESTORE_LOG"
+  chmod 640 "$BACKUP_LOG" "$RESTORE_LOG"
+
+  if [[ "$run_now" == "true" ]]; then
+    log "Running immediate backup + restore test"
+    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-encrypted.sh"
+    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test.sh"
+  fi
+
+  log "DB backup/restore cron configured successfully."
+}
+
+main "$@"

infra/deploy/db-backup-encrypted.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}"
+RETENTION_DAYS="${RETENTION_DAYS:-14}"
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_file() {
+  [[ -f "$1" ]] || die "Missing required file: $1"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
+}
+
+main() {
+  require_command docker
+  require_command openssl
+  require_command sha256sum
+  require_file "$ENV_FILE"
+  require_file "$SECRET_FILE"
+
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+  # shellcheck disable=SC1090
+  source "$SECRET_FILE"
+
+  [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]] || die "BACKUP_ENCRYPTION_KEY is empty in $SECRET_FILE"
+  export BACKUP_ENCRYPTION_KEY
+  [[ -n "${POSTGRES_USER:-}" ]] || die "POSTGRES_USER missing in $ENV_FILE"
+  [[ -n "${POSTGRES_DB:-}" ]] || die "POSTGRES_DB missing in $ENV_FILE"
+
+  local ts backup_dir plain_sql encrypted_sql
+  ts="$(date -u +%Y%m%d-%H%M%S)"
+  backup_dir="${BACKUP_ROOT}/${ts}"
+  plain_sql="${backup_dir}/proxpanel.sql"
+  encrypted_sql="${plain_sql}.enc"
+
+  mkdir -p "$backup_dir"
+  chmod 700 "$backup_dir"
+
+  log "Creating plaintext dump"
+  docker exec proxpanel-postgres pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists >"$plain_sql"
+
+  log "Encrypting dump"
+  openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \
+    -in "$plain_sql" \
+    -out "$encrypted_sql" \
+    -pass env:BACKUP_ENCRYPTION_KEY
+
+  log "Writing checksum"
+  sha256sum "$encrypted_sql" >"${encrypted_sql}.sha256"
+
+  rm -f "$plain_sql"
+  chmod 600 "$encrypted_sql" "${encrypted_sql}.sha256"
+
+  log "Applying retention policy (${RETENTION_DAYS} days)"
+  find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +"$RETENTION_DAYS" -exec rm -rf {} +
+
+  log "Encrypted backup created: $encrypted_sql"
+}
+
+main "$@"

infra/deploy/db-restore-test.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}"
+TMP_ROOT="${TMP_ROOT:-/tmp/proxpanel-restore-test}"
+TEST_CONTAINER="${TEST_CONTAINER:-proxpanel-restore-test}"
+PG_IMAGE="${PG_IMAGE:-postgres:16-alpine}"
+PG_USER="${PG_USER:-proxpanel}"
+PG_PASSWORD="${PG_PASSWORD:-restoretestpass}"
+PG_DB="${PG_DB:-proxpanel_restore}"
+
+cleanup() {
+  docker rm -f "$TEST_CONTAINER" >/dev/null 2>&1 || true
+  rm -rf "$TMP_ROOT"
+}
+trap cleanup EXIT
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_file() {
+  [[ -f "$1" ]] || die "Missing required file: $1"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
+}
+
+find_latest_encrypted_backup() {
+  find "$BACKUP_ROOT" -mindepth 2 -maxdepth 2 -type f -name 'proxpanel.sql.enc' | sort | tail -n 1
+}
+
+wait_pg_ready() {
+  local tries=60
+  local i
+  for ((i=1; i<=tries; i++)); do
+    if docker exec "$TEST_CONTAINER" pg_isready -U "$PG_USER" -d "$PG_DB" >/dev/null 2>&1; then
+      return
+    fi
+    sleep 1
+  done
+  die "Restore test postgres did not become ready."
+}
+
+main() {
+  require_command docker
+  require_command openssl
+  require_command sha256sum
+  require_file "$SECRET_FILE"
+
+  # shellcheck disable=SC1090
+  source "$SECRET_FILE"
+  [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]] || die "BACKUP_ENCRYPTION_KEY is empty in $SECRET_FILE"
+  export BACKUP_ENCRYPTION_KEY
+
+  local encrypted_backup checksum_file latest_dir decrypted_sql
+  encrypted_backup="$(find_latest_encrypted_backup)"
+  [[ -n "$encrypted_backup" ]] || die "No encrypted backup found in $BACKUP_ROOT"
+  checksum_file="${encrypted_backup}.sha256"
+  require_file "$checksum_file"
+
+  latest_dir="$(dirname "$encrypted_backup")"
+  mkdir -p "$TMP_ROOT"
+  chmod 700 "$TMP_ROOT"
+  decrypted_sql="${TMP_ROOT}/restore.sql"
+
+  log "Verifying checksum for $encrypted_backup"
+  (cd "$latest_dir" && sha256sum -c "$(basename "$checksum_file")")
+
+  log "Decrypting latest backup"
+  openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 \
+    -in "$encrypted_backup" \
+    -out "$decrypted_sql" \
+    -pass env:BACKUP_ENCRYPTION_KEY
+
+  log "Starting isolated restore-test postgres container"
+  docker rm -f "$TEST_CONTAINER" >/dev/null 2>&1 || true
+  docker run -d --name "$TEST_CONTAINER" \
+    -e POSTGRES_USER="$PG_USER" \
+    -e POSTGRES_PASSWORD="$PG_PASSWORD" \
+    -e POSTGRES_DB="$PG_DB" \
+    "$PG_IMAGE" >/dev/null
+
+  wait_pg_ready
+
+  log "Applying restored SQL into test DB"
+  cat "$decrypted_sql" | docker exec -i "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" >/dev/null
+
+  log "Running restore sanity checks"
+  local table_count required_table_count
+  table_count="$(
+    docker exec "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -Atc \
+      "select count(*) from information_schema.tables where table_schema='public';"
+  )"
+  required_table_count="$(
+    docker exec "$TEST_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -Atc \
+      "select count(*) from information_schema.tables where table_schema='public' and table_name in ('User','Tenant','AuditLog');"
+  )"
+
+  [[ "${table_count:-0}" -ge 10 ]] || die "Restore sanity check failed (unexpected table count: $table_count)"
+  [[ "${required_table_count:-0}" -eq 3 ]] || die "Restore sanity check failed (required tables missing)"
+
+  log "Restore test passed (tables=$table_count)"
+}
+
+main "$@"

infra/deploy/rollback-blue-green.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
+GREEN_DIR="${GREEN_DIR:-/opt/proxpanel-green}"
+GREEN_PROJECT="${GREEN_PROJECT:-proxpanel_green}"
+GREEN_FRONTEND_PORT="${GREEN_FRONTEND_PORT:-18081}"
+GREEN_BACKEND_PORT="${GREEN_BACKEND_PORT:-18080}"
+NGINX_SITE="${NGINX_SITE:-/etc/nginx/sites-available/proxpanel.conf}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  sudo bash infra/deploy/rollback-blue-green.sh --backup-dir /opt/proxpanel-backups/<timestamp> [--cutover]
+
+Behavior:
+  1) Stages backup app/env into /opt/proxpanel-green
+  2) Boots green stack on ports 18081/18080
+  3) Restores DB dump from selected backup into green postgres
+  4) Verifies green API health
+  5) If --cutover is provided, switches Nginx upstream to green frontend port
+
+Notes:
+  - This script requires backup directory format created by deployment steps:
+      <backup-dir>/app/
+      <backup-dir>/env.production.bak (or .env.production.bak)
+      <backup-dir>/db_pre_*.sql
+EOF
+}
+
+log() {
+  printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '\n[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_root() {
+  [[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Run as root (or with sudo)."
+}
+
+parse_args() {
+  BACKUP_DIR=""
+  CUTOVER="false"
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --backup-dir)
+        BACKUP_DIR="${2:-}"
+        shift 2
+        ;;
+      --cutover)
+        CUTOVER="true"
+        shift
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        die "Unknown argument: $1"
+        ;;
+    esac
+  done
+
+  [[ -n "$BACKUP_DIR" ]] || die "--backup-dir is required"
+}
+
+pick_db_dump() {
+  find "$BACKUP_DIR" -maxdepth 1 -type f -name 'db_pre*.sql' | sort | tail -n 1
+}
+
+pick_env_backup() {
+  if [[ -f "$BACKUP_DIR/env.production.bak" ]]; then
+    printf '%s' "$BACKUP_DIR/env.production.bak"
+    return
+  fi
+  if [[ -f "$BACKUP_DIR/.env.production.bak" ]]; then
+    printf '%s' "$BACKUP_DIR/.env.production.bak"
+    return
+  fi
+  printf ''
+}
+
+wait_for_green_health() {
+  local i
+  for i in $(seq 1 90); do
+    if curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null 2>&1; then
+      return
+    fi
+    sleep 2
+  done
+  die "Green backend health check failed."
+}
+
+main() {
+  require_root
+  parse_args "$@"
+
+  [[ -d "$BACKUP_DIR/app" ]] || die "Missing backup app directory: $BACKUP_DIR/app"
+  local env_backup
+  env_backup="$(pick_env_backup)"
+  [[ -n "$env_backup" ]] || die "Missing env backup in $BACKUP_DIR (.env.production.bak or env.production.bak)"
+  local db_dump
+  db_dump="$(pick_db_dump)"
+  [[ -n "$db_dump" ]] || die "Missing DB dump in backup dir: $BACKUP_DIR"
+
+  log "Preparing green staging directory at $GREEN_DIR"
+  rm -rf "$GREEN_DIR"
+  mkdir -p "$GREEN_DIR"
+  rsync -a "$BACKUP_DIR/app/" "$GREEN_DIR/"
+  cp "$env_backup" "$GREEN_DIR/.env.production"
+
+  # Override ports for green stack and keep CORS aligned to production hostname.
+  if grep -q '^FRONTEND_PORT=' "$GREEN_DIR/.env.production"; then
+    sed -i "s/^FRONTEND_PORT=.*/FRONTEND_PORT=${GREEN_FRONTEND_PORT}/" "$GREEN_DIR/.env.production"
+  else
+    echo "FRONTEND_PORT=${GREEN_FRONTEND_PORT}" >> "$GREEN_DIR/.env.production"
+  fi
+  if grep -q '^BACKEND_PORT=' "$GREEN_DIR/.env.production"; then
+    sed -i "s/^BACKEND_PORT=.*/BACKEND_PORT=${GREEN_BACKEND_PORT}/" "$GREEN_DIR/.env.production"
+  else
+    echo "BACKEND_PORT=${GREEN_BACKEND_PORT}" >> "$GREEN_DIR/.env.production"
+  fi
+
+  # Remove fixed container_name lines for project-isolated blue/green operation.
+  local compose_green
+  compose_green="$GREEN_DIR/infra/deploy/docker-compose.green.yml"
+  awk '!/^[[:space:]]*container_name:/' "$GREEN_DIR/infra/deploy/docker-compose.production.yml" > "$compose_green"
+
+  log "Stopping previous green project (if any)"
+  (
+    cd "$GREEN_DIR"
+    docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" down -v || true
+  )
+
+  log "Starting green stack on ${GREEN_FRONTEND_PORT}/${GREEN_BACKEND_PORT}"
+  (
+    cd "$GREEN_DIR"
+    docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" up -d --build
+  )
+
+  log "Restoring backup DB into green postgres"
+  local green_postgres_id green_pg_user green_pg_db
+  green_postgres_id="$(
+    cd "$GREEN_DIR" && docker compose -p "$GREEN_PROJECT" --env-file .env.production -f "$compose_green" ps -q postgres
+  )"
+  [[ -n "$green_postgres_id" ]] || die "Could not locate green postgres container."
+
+  # shellcheck disable=SC1090
+  source "$GREEN_DIR/.env.production"
+  green_pg_user="${POSTGRES_USER:-proxpanel}"
+  green_pg_db="${POSTGRES_DB:-proxpanel}"
+
+  docker exec "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 \
+    -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'
+
+  cat "$db_dump" | docker exec -i "$green_postgres_id" psql -U "$green_pg_user" -d "$green_pg_db" -v ON_ERROR_STOP=1 >/dev/null
+
+  log "Validating green health"
+  wait_for_green_health
+  curl -fsS "http://127.0.0.1:${GREEN_BACKEND_PORT}/api/health" >/dev/null
+
+  if [[ "$CUTOVER" == "true" ]]; then
+    log "Cutting over Nginx upstream to green frontend port ${GREEN_FRONTEND_PORT}"
+    [[ -f "$NGINX_SITE" ]] || die "Missing nginx site config: $NGINX_SITE"
+    cp "$NGINX_SITE" "${NGINX_SITE}.pre-green.$(date -u +%Y%m%d-%H%M%S).bak"
+    sed -i -E "s#proxy_pass http://127\\.0\\.0\\.1:[0-9]+;#proxy_pass http://127.0.0.1:${GREEN_FRONTEND_PORT};#g" "$NGINX_SITE"
+    nginx -t
+    systemctl reload nginx
+    log "Nginx cutover complete."
+  fi
+
+  log "Blue/green rollback prep complete."
+  printf '\nGreen frontend: http://127.0.0.1:%s\n' "$GREEN_FRONTEND_PORT"
+  printf 'Green backend : http://127.0.0.1:%s/api/health\n' "$GREEN_BACKEND_PORT"
+  if [[ "$CUTOVER" == "true" ]]; then
+    printf 'Traffic switched to green.\n'
+  else
+    printf 'Traffic still on blue. Re-run with --cutover when ready.\n'
+  fi
+}
+
+main "$@"

infra/deploy/rotate-production-secrets.sh

@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}"
+COMPOSE_FILE="${COMPOSE_FILE:-$APP_DIR/infra/deploy/docker-compose.production.yml}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  sudo bash infra/deploy/rotate-production-secrets.sh
+
+What this rotates:
+  - JWT_SECRET
+  - JWT_REFRESH_SECRET
+  - POSTGRES_PASSWORD
+  - ADMIN_PASSWORD
+
+Safety:
+  - Creates backup in /opt/proxpanel-backups/<timestamp>-secret-rotation/
+  - Dumps DB before password change
+  - Verifies API health and admin login after rotation
+EOF
+}
+
+log() {
+  printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '\n[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_root() {
+  if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then
+    die "Run as root (or with sudo)."
+  fi
+}
+
+require_file() {
+  local file="$1"
+  [[ -f "$file" ]] || die "Missing required file: $file"
+}
+
+require_command() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || die "Missing required command: $cmd"
+}
+
+escape_sed_replacement() {
+  local value="$1"
+  value="${value//\\/\\\\}"
+  value="${value//&/\\&}"
+  printf '%s' "$value"
+}
+
+update_env_value() {
+  local key="$1"
+  local value="$2"
+  local escaped
+  escaped="$(escape_sed_replacement "$value")"
+
+  if grep -q "^${key}=" "$ENV_FILE"; then
+    sed -i "s/^${key}=.*/${key}=${escaped}/" "$ENV_FILE"
+  else
+    printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
+  fi
+}
+
+random_password() {
+  openssl rand -base64 36 | tr -d '\n' | tr '/+' 'ab'
+}
+
+random_hex() {
+  openssl rand -hex 32 | tr -d '\n'
+}
+
+wait_for_health() {
+  local max_tries=60
+  local i
+  for ((i=1; i<=max_tries; i++)); do
+    if curl -fsS "http://127.0.0.1:${BACKEND_PORT}/api/health" >/dev/null 2>&1; then
+      return
+    fi
+    sleep 2
+  done
+  die "Backend health check failed after secret rotation."
+}
+
+main() {
+  if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+    usage
+    exit 0
+  fi
+
+  require_root
+  require_command docker
+  require_command curl
+  require_command openssl
+  require_command rsync
+  require_file "$ENV_FILE"
+  require_file "$COMPOSE_FILE"
+
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+
+  local ts backup_dir compose_args
+  ts="$(date -u +%Y%m%d-%H%M%S)"
+  backup_dir="${BACKUP_ROOT}/${ts}-secret-rotation"
+  compose_args=(--env-file "$ENV_FILE" -f "$COMPOSE_FILE")
+
+  local old_admin_email old_admin_password old_postgres_user old_postgres_db old_backend_port
+  old_admin_email="${ADMIN_EMAIL:-admin@proxpanel.local}"
+  old_admin_password="${ADMIN_PASSWORD:-}"
+  old_postgres_user="${POSTGRES_USER:-proxpanel}"
+  old_postgres_db="${POSTGRES_DB:-proxpanel}"
+  old_backend_port="${BACKEND_PORT:-8080}"
+
+  local new_jwt_secret new_jwt_refresh_secret new_postgres_password new_admin_password
+  new_jwt_secret="$(random_password)$(random_password)"
+  new_jwt_refresh_secret="$(random_password)$(random_password)"
+  new_postgres_password="$(random_hex)"
+  new_admin_password="$(random_password)A9!"
+
+  log "Creating pre-rotation backups in $backup_dir"
+  mkdir -p "$backup_dir"
+  cp "$ENV_FILE" "$backup_dir/.env.production.bak"
+  rsync -a "$APP_DIR/" "$backup_dir/app/"
+  docker exec proxpanel-postgres pg_dump -U "$old_postgres_user" -d "$old_postgres_db" >"$backup_dir/db_pre_rotation.sql"
+
+  log "Updating env file with rotated secrets"
+  update_env_value "JWT_SECRET" "$new_jwt_secret"
+  update_env_value "JWT_REFRESH_SECRET" "$new_jwt_refresh_secret"
+  update_env_value "POSTGRES_PASSWORD" "$new_postgres_password"
+  update_env_value "ADMIN_PASSWORD" "$new_admin_password"
+
+  # Re-load env values after edits.
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+
+  log "Applying Postgres password rotation"
+  docker exec proxpanel-postgres psql -U "$old_postgres_user" -d "$old_postgres_db" -v ON_ERROR_STOP=1 \
+    -c "ALTER USER \"$old_postgres_user\" WITH PASSWORD '$new_postgres_password';"
+
+  log "Restarting stack with new secrets"
+  (
+    cd "$APP_DIR"
+    docker compose "${compose_args[@]}" up -d
+  )
+
+  BACKEND_PORT="${BACKEND_PORT:-$old_backend_port}"
+  wait_for_health
+
+  log "Re-seeding admin credential to match rotated ADMIN_PASSWORD"
+  (
+    cd "$APP_DIR"
+    docker compose "${compose_args[@]}" exec -T backend npm run prisma:seed
+  )
+
+  log "Revoking all active auth sessions after JWT/password rotation"
+  docker exec proxpanel-postgres psql -U "${POSTGRES_USER:-$old_postgres_user}" -d "${POSTGRES_DB:-$old_postgres_db}" \
+    -v ON_ERROR_STOP=1 -c 'TRUNCATE TABLE "AuthSession" RESTART IDENTITY;'
+
+  log "Verifying post-rotation login"
+  local login_status
+  login_status="$(
+    curl -sS -o /tmp/proxpanel-rotate-login.json -w "%{http_code}" \
+      -X POST "http://127.0.0.1:${BACKEND_PORT}/api/auth/login" \
+      -H "Content-Type: application/json" \
+      -d "{\"email\":\"${ADMIN_EMAIL}\",\"password\":\"${ADMIN_PASSWORD}\"}"
+  )"
+  [[ "$login_status" == "200" ]] || die "Admin login failed after secret rotation (status $login_status)."
+
+  local summary_file
+  summary_file="/root/proxpanel-secret-rotation-${ts}.txt"
+  cat >"$summary_file" <<EOF
+ProxPanel secret rotation completed at $(date -u +'%Y-%m-%d %H:%M:%S UTC')
+
+Backup directory:
+  $backup_dir
+
+Rotated secrets:
+  JWT_SECRET
+  JWT_REFRESH_SECRET
+  POSTGRES_PASSWORD
+  ADMIN_PASSWORD
+
+Admin credentials:
+  ADMIN_EMAIL=${ADMIN_EMAIL}
+  ADMIN_PASSWORD=${ADMIN_PASSWORD}
+
+Post-rotation checks:
+  curl -fsS http://127.0.0.1:${BACKEND_PORT}/api/health
+  curl -X POST http://127.0.0.1:${BACKEND_PORT}/api/auth/login ...
+
+Important:
+  Change ADMIN_PASSWORD again from Profile page after login.
+EOF
+  chmod 600 "$summary_file"
+
+  log "Rotation complete."
+  printf '\nNew admin email: %s\n' "${ADMIN_EMAIL}"
+  printf 'New admin password: %s\n' "${ADMIN_PASSWORD}"
+  printf 'Summary: %s\n' "$summary_file"
+}
+
+main "$@"