ops: add production hardening automation for secrets, backups, and rollback
This commit is contained in:
Austin A
209
infra/deploy/rotate-production-secrets.sh
Executable file
209
infra/deploy/rotate-production-secrets.sh
Executable file
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/proxpanel}"
|
||||
ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-$APP_DIR/infra/deploy/docker-compose.production.yml}"
|
||||
BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
sudo bash infra/deploy/rotate-production-secrets.sh
|
||||
|
||||
What this rotates:
|
||||
- JWT_SECRET
|
||||
- JWT_REFRESH_SECRET
|
||||
- POSTGRES_PASSWORD
|
||||
- ADMIN_PASSWORD
|
||||
|
||||
Safety:
|
||||
- Creates backup in /opt/proxpanel-backups/<timestamp>-secret-rotation/
|
||||
- Dumps DB before password change
|
||||
- Verifies API health and admin login after rotation
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf '\n[ERROR] %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_root() {
|
||||
if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then
|
||||
die "Run as root (or with sudo)."
|
||||
fi
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local file="$1"
|
||||
[[ -f "$file" ]] || die "Missing required file: $file"
|
||||
}
|
||||
|
||||
require_command() {
|
||||
local cmd="$1"
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "Missing required command: $cmd"
|
||||
}
|
||||
|
||||
escape_sed_replacement() {
|
||||
local value="$1"
|
||||
value="${value//\\/\\\\}"
|
||||
value="${value//&/\\&}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
update_env_value() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
local escaped
|
||||
escaped="$(escape_sed_replacement "$value")"
|
||||
|
||||
if grep -q "^${key}=" "$ENV_FILE"; then
|
||||
sed -i "s/^${key}=.*/${key}=${escaped}/" "$ENV_FILE"
|
||||
else
|
||||
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
random_password() {
|
||||
openssl rand -base64 36 | tr -d '\n' | tr '/+' 'ab'
|
||||
}
|
||||
|
||||
random_hex() {
|
||||
openssl rand -hex 32 | tr -d '\n'
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local max_tries=60
|
||||
local i
|
||||
for ((i=1; i<=max_tries; i++)); do
|
||||
if curl -fsS "http://127.0.0.1:${BACKEND_PORT}/api/health" >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
die "Backend health check failed after secret rotation."
|
||||
}
|
||||
|
||||
main() {
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_root
|
||||
require_command docker
|
||||
require_command curl
|
||||
require_command openssl
|
||||
require_command rsync
|
||||
require_file "$ENV_FILE"
|
||||
require_file "$COMPOSE_FILE"
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
|
||||
local ts backup_dir compose_args
|
||||
ts="$(date -u +%Y%m%d-%H%M%S)"
|
||||
backup_dir="${BACKUP_ROOT}/${ts}-secret-rotation"
|
||||
compose_args=(--env-file "$ENV_FILE" -f "$COMPOSE_FILE")
|
||||
|
||||
local old_admin_email old_admin_password old_postgres_user old_postgres_db old_backend_port
|
||||
old_admin_email="${ADMIN_EMAIL:-admin@proxpanel.local}"
|
||||
old_admin_password="${ADMIN_PASSWORD:-}"
|
||||
old_postgres_user="${POSTGRES_USER:-proxpanel}"
|
||||
old_postgres_db="${POSTGRES_DB:-proxpanel}"
|
||||
old_backend_port="${BACKEND_PORT:-8080}"
|
||||
|
||||
local new_jwt_secret new_jwt_refresh_secret new_postgres_password new_admin_password
|
||||
new_jwt_secret="$(random_password)$(random_password)"
|
||||
new_jwt_refresh_secret="$(random_password)$(random_password)"
|
||||
new_postgres_password="$(random_hex)"
|
||||
new_admin_password="$(random_password)A9!"
|
||||
|
||||
log "Creating pre-rotation backups in $backup_dir"
|
||||
mkdir -p "$backup_dir"
|
||||
cp "$ENV_FILE" "$backup_dir/.env.production.bak"
|
||||
rsync -a "$APP_DIR/" "$backup_dir/app/"
|
||||
docker exec proxpanel-postgres pg_dump -U "$old_postgres_user" -d "$old_postgres_db" >"$backup_dir/db_pre_rotation.sql"
|
||||
|
||||
log "Updating env file with rotated secrets"
|
||||
update_env_value "JWT_SECRET" "$new_jwt_secret"
|
||||
update_env_value "JWT_REFRESH_SECRET" "$new_jwt_refresh_secret"
|
||||
update_env_value "POSTGRES_PASSWORD" "$new_postgres_password"
|
||||
update_env_value "ADMIN_PASSWORD" "$new_admin_password"
|
||||
|
||||
# Re-load env values after edits.
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
|
||||
log "Applying Postgres password rotation"
|
||||
docker exec proxpanel-postgres psql -U "$old_postgres_user" -d "$old_postgres_db" -v ON_ERROR_STOP=1 \
|
||||
-c "ALTER USER \"$old_postgres_user\" WITH PASSWORD '$new_postgres_password';"
|
||||
|
||||
log "Restarting stack with new secrets"
|
||||
(
|
||||
cd "$APP_DIR"
|
||||
docker compose "${compose_args[@]}" up -d
|
||||
)
|
||||
|
||||
BACKEND_PORT="${BACKEND_PORT:-$old_backend_port}"
|
||||
wait_for_health
|
||||
|
||||
log "Re-seeding admin credential to match rotated ADMIN_PASSWORD"
|
||||
(
|
||||
cd "$APP_DIR"
|
||||
docker compose "${compose_args[@]}" exec -T backend npm run prisma:seed
|
||||
)
|
||||
|
||||
log "Revoking all active auth sessions after JWT/password rotation"
|
||||
docker exec proxpanel-postgres psql -U "${POSTGRES_USER:-$old_postgres_user}" -d "${POSTGRES_DB:-$old_postgres_db}" \
|
||||
-v ON_ERROR_STOP=1 -c 'TRUNCATE TABLE "AuthSession" RESTART IDENTITY;'
|
||||
|
||||
log "Verifying post-rotation login"
|
||||
local login_status
|
||||
login_status="$(
|
||||
curl -sS -o /tmp/proxpanel-rotate-login.json -w "%{http_code}" \
|
||||
-X POST "http://127.0.0.1:${BACKEND_PORT}/api/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"email\":\"${ADMIN_EMAIL}\",\"password\":\"${ADMIN_PASSWORD}\"}"
|
||||
)"
|
||||
[[ "$login_status" == "200" ]] || die "Admin login failed after secret rotation (status $login_status)."
|
||||
|
||||
local summary_file
|
||||
summary_file="/root/proxpanel-secret-rotation-${ts}.txt"
|
||||
cat >"$summary_file" <<EOF
|
||||
ProxPanel secret rotation completed at $(date -u +'%Y-%m-%d %H:%M:%S UTC')
|
||||
|
||||
Backup directory:
|
||||
$backup_dir
|
||||
|
||||
Rotated secrets:
|
||||
JWT_SECRET
|
||||
JWT_REFRESH_SECRET
|
||||
POSTGRES_PASSWORD
|
||||
ADMIN_PASSWORD
|
||||
|
||||
Admin credentials:
|
||||
ADMIN_EMAIL=${ADMIN_EMAIL}
|
||||
ADMIN_PASSWORD=${ADMIN_PASSWORD}
|
||||
|
||||
Post-rotation checks:
|
||||
curl -fsS http://127.0.0.1:${BACKEND_PORT}/api/health
|
||||
curl -X POST http://127.0.0.1:${BACKEND_PORT}/api/auth/login ...
|
||||
|
||||
Important:
|
||||
Change ADMIN_PASSWORD again from Profile page after login.
|
||||
EOF
|
||||
chmod 600 "$summary_file"
|
||||
|
||||
log "Rotation complete."
|
||||
printf '\nNew admin email: %s\n' "${ADMIN_EMAIL}"
|
||||
printf 'New admin password: %s\n' "${ADMIN_PASSWORD}"
|
||||
printf 'Summary: %s\n' "$summary_file"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user