#!/usr/bin/env bash set -Eeuo pipefail APP_DIR="${APP_DIR:-/opt/proxpanel}" ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}" BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}" BACKEND_URL="${BACKEND_URL:-}" ADMIN_EMAIL_INPUT="" ADMIN_PASSWORD_INPUT="" NEW_PROXMOX_HOST="" NEW_PROXMOX_PORT="" NEW_PROXMOX_USERNAME="" NEW_PROXMOX_TOKEN_ID="" NEW_PROXMOX_TOKEN_SECRET="" NEW_PROXMOX_VERIFY_SSL="" NEW_PAYSTACK_SECRET="" NEW_FLUTTERWAVE_SECRET="" NEW_FLUTTERWAVE_WEBHOOK_HASH="" FINALIZE_PAYMENT_GRACE="false" NEW_MONITORING_WEBHOOK_URL="" NEW_ALERT_WEBHOOK_URL="" NEW_NOTIFICATION_EMAIL_WEBHOOK="" NEW_OPS_EMAIL="" API_TOKEN="" TS="" BACKUP_DIR="" UPDATED_PROXMOX="false" UPDATED_PAYMENT="false" UPDATED_NOTIFICATIONS="false" ORIGINAL_PROXMOX_JSON="" ORIGINAL_PAYMENT_JSON="" ORIGINAL_NOTIFICATIONS_JSON="" ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="" usage() { cat <<'EOF' Usage: sudo bash infra/deploy/rotate-integration-secrets.sh [options] Options: --backend-url Backend URL (default: http://127.0.0.1:${BACKEND_PORT}/api) --admin-email Admin email (default from .env.production) --admin-password Admin password (default from .env.production) # Proxmox token rotation (zero-downtime by preflight + post-sync validation): --proxmox-token-secret New Proxmox token secret --proxmox-token-id Optional new token id --proxmox-username Optional new token username --proxmox-host Optional host override --proxmox-port Optional port override --proxmox-verify-ssl Optional SSL verify override # Payment/webhook secret rotation (supports grace window): --paystack-secret New Paystack secret (stores old value as paystack_secret_previous) --flutterwave-secret New Flutterwave secret (stores old value as flutterwave_secret_previous) --flutterwave-webhook-hash New Flutterwave webhook hash (stores old value as flutterwave_webhook_hash_previous) --finalize-payment-webhook-grace Clears *_previous fields after provider cutover is complete # Alerting webhook endpoints (optional): --monitoring-webhook-url --alert-webhook-url --notification-email-webhook --ops-email -h, --help Show help Examples: sudo bash infra/deploy/rotate-integration-secrets.sh \ --proxmox-token-secret 'NEW_SECRET' \ --paystack-secret 'sk_live_new' \ --flutterwave-webhook-hash 'new_hash' sudo bash infra/deploy/rotate-integration-secrets.sh \ --finalize-payment-webhook-grace EOF } log() { printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" } die() { printf '\n[ERROR] %s\n' "$*" >&2 exit 1 } require_root() { if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then die "Run as root (or with sudo)." fi } require_command() { command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" } require_file() { [[ -f "$1" ]] || die "Missing required file: $1" } parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --backend-url) BACKEND_URL="${2:-}" shift 2 ;; --admin-email) ADMIN_EMAIL_INPUT="${2:-}" shift 2 ;; --admin-password) ADMIN_PASSWORD_INPUT="${2:-}" shift 2 ;; --proxmox-host) NEW_PROXMOX_HOST="${2:-}" shift 2 ;; --proxmox-port) NEW_PROXMOX_PORT="${2:-}" shift 2 ;; --proxmox-username) NEW_PROXMOX_USERNAME="${2:-}" shift 2 ;; --proxmox-token-id) NEW_PROXMOX_TOKEN_ID="${2:-}" shift 2 ;; --proxmox-token-secret) NEW_PROXMOX_TOKEN_SECRET="${2:-}" shift 2 ;; --proxmox-verify-ssl) NEW_PROXMOX_VERIFY_SSL="${2:-}" shift 2 ;; --paystack-secret) NEW_PAYSTACK_SECRET="${2:-}" shift 2 ;; --flutterwave-secret) NEW_FLUTTERWAVE_SECRET="${2:-}" shift 2 ;; --flutterwave-webhook-hash) NEW_FLUTTERWAVE_WEBHOOK_HASH="${2:-}" shift 2 ;; --finalize-payment-webhook-grace) FINALIZE_PAYMENT_GRACE="true" shift ;; --monitoring-webhook-url) NEW_MONITORING_WEBHOOK_URL="${2:-}" shift 2 ;; --alert-webhook-url) NEW_ALERT_WEBHOOK_URL="${2:-}" shift 2 ;; --notification-email-webhook) NEW_NOTIFICATION_EMAIL_WEBHOOK="${2:-}" shift 2 ;; --ops-email) NEW_OPS_EMAIL="${2:-}" shift 2 ;; -h|--help) usage exit 0 ;; *) die "Unknown argument: $1" ;; esac done } json_escape() { printf '%s' "$1" | jq -Rs . } json_canonical() { printf '%s' "$1" | jq -cS . } sanitize_notifications_payload() { printf '%s' "$1" | jq ' if (.monitoring_webhook_url // "") == "" then del(.monitoring_webhook_url) else . end | if (.alert_webhook_url // "") == "" then del(.alert_webhook_url) else . end | if (.email_gateway_url // "") == "" then del(.email_gateway_url) else . end | if (.notification_email_webhook // "") == "" then del(.notification_email_webhook) else . end | if (.ops_email // "") == "" then del(.ops_email) else . end ' } api_get() { local path="$1" curl -fsS \ -H "Authorization: Bearer ${API_TOKEN}" \ "${BACKEND_URL}${path}" } api_put() { local path="$1" local payload="$2" curl -fsS \ -X PUT \ -H "Authorization: Bearer ${API_TOKEN}" \ -H "Content-Type: application/json" \ -d "$payload" \ "${BACKEND_URL}${path}" >/dev/null } preflight_proxmox_candidate() { local payload_json="$1" local host port username token_id token_secret verify_ssl auth_header host="$(printf '%s' "$payload_json" | jq -r '.host // empty')" port="$(printf '%s' "$payload_json" | jq -r '.port // 8006')" username="$(printf '%s' "$payload_json" | jq -r '.username // empty')" token_id="$(printf '%s' "$payload_json" | jq -r '.token_id // empty')" token_secret="$(printf '%s' "$payload_json" | jq -r '.token_secret // empty')" verify_ssl="$(printf '%s' "$payload_json" | jq -r '.verify_ssl // true')" [[ -n "$host" && -n "$username" && -n "$token_id" && -n "$token_secret" ]] || { die "Candidate Proxmox settings are incomplete." } auth_header="PVEAPIToken=${username}!${token_id}=${token_secret}" log "Preflight: validating candidate Proxmox token directly against ${host}:${port}" if [[ "$verify_ssl" == "false" ]]; then curl -ksfS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null else curl -fsS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null fi } rollback_if_needed() { if [[ -z "$API_TOKEN" ]]; then return fi if [[ "$UPDATED_NOTIFICATIONS" == "true" && -n "$ORIGINAL_NOTIFICATIONS_JSON" ]]; then log "Rollback: restoring notifications settings" api_put "/settings/notifications" "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED" || true fi if [[ "$UPDATED_PAYMENT" == "true" && -n "$ORIGINAL_PAYMENT_JSON" ]]; then log "Rollback: restoring payment settings" api_put "/settings/payment" "$ORIGINAL_PAYMENT_JSON" || true fi if [[ "$UPDATED_PROXMOX" == "true" && -n "$ORIGINAL_PROXMOX_JSON" ]]; then log "Rollback: restoring proxmox settings" api_put "/settings/proxmox" "$ORIGINAL_PROXMOX_JSON" || true fi } on_error() { local line="$1" printf '\n[ERROR] Rotation failed at line %s. Attempting rollback where possible.\n' "$line" >&2 rollback_if_needed } build_backend_url() { if [[ -n "$BACKEND_URL" ]]; then return fi local backend_port backend_port="${BACKEND_PORT:-8080}" BACKEND_URL="http://127.0.0.1:${backend_port}/api" } main() { parse_args "$@" require_root require_command curl require_command jq require_file "$ENV_FILE" # shellcheck disable=SC1090 source "$ENV_FILE" build_backend_url local admin_email admin_password admin_email="${ADMIN_EMAIL_INPUT:-${ADMIN_EMAIL:-}}" admin_password="${ADMIN_PASSWORD_INPUT:-${ADMIN_PASSWORD:-}}" [[ -n "$admin_email" && -n "$admin_password" ]] || { die "Admin credentials are required (provide --admin-email/--admin-password or set ADMIN_EMAIL/ADMIN_PASSWORD in env)." } local wants_rotation="false" if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" || -n "$NEW_PROXMOX_TOKEN_ID" || -n "$NEW_PROXMOX_USERNAME" || -n "$NEW_PROXMOX_HOST" || -n "$NEW_PROXMOX_PORT" || -n "$NEW_PROXMOX_VERIFY_SSL" || -n "$NEW_PAYSTACK_SECRET" || -n "$NEW_FLUTTERWAVE_SECRET" || -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" || -n "$NEW_MONITORING_WEBHOOK_URL" || -n "$NEW_ALERT_WEBHOOK_URL" || -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" || -n "$NEW_OPS_EMAIL" || "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then wants_rotation="true" fi [[ "$wants_rotation" == "true" ]] || die "No rotation inputs supplied. Run with --help for options." trap 'on_error $LINENO' ERR TS="$(date -u +%Y%m%d-%H%M%S)" BACKUP_DIR="${BACKUP_ROOT}/${TS}-integration-secret-rotation" mkdir -p "$BACKUP_DIR" chmod 700 "$BACKUP_DIR" log "Authenticating against backend API" local login_response login_response="$( curl -fsS -X POST "${BACKEND_URL}/auth/login" \ -H "Content-Type: application/json" \ -d "{\"email\":$(json_escape "$admin_email"),\"password\":$(json_escape "$admin_password")}" )" API_TOKEN="$(printf '%s' "$login_response" | jq -r '.token // empty')" [[ -n "$API_TOKEN" ]] || die "Failed to obtain auth token from backend login." log "Fetching current integration settings" ORIGINAL_PROXMOX_JSON="$(api_get "/settings/proxmox")" ORIGINAL_PAYMENT_JSON="$(api_get "/settings/payment")" ORIGINAL_NOTIFICATIONS_JSON="$(api_get "/settings/notifications")" ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="$(sanitize_notifications_payload "$ORIGINAL_NOTIFICATIONS_JSON")" printf '%s\n' "$ORIGINAL_PROXMOX_JSON" >"${BACKUP_DIR}/proxmox.before.json" printf '%s\n' "$ORIGINAL_PAYMENT_JSON" >"${BACKUP_DIR}/payment.before.json" printf '%s\n' "$ORIGINAL_NOTIFICATIONS_JSON" >"${BACKUP_DIR}/notifications.before.json" cp "$ENV_FILE" "${BACKUP_DIR}/.env.production.bak" local proxmox_payload payment_payload notifications_payload proxmox_payload="$ORIGINAL_PROXMOX_JSON" payment_payload="$ORIGINAL_PAYMENT_JSON" notifications_payload="$ORIGINAL_NOTIFICATIONS_JSON" if [[ -n "$NEW_PROXMOX_HOST" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_HOST" '.host = $v')" fi if [[ -n "$NEW_PROXMOX_PORT" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --argjson v "$NEW_PROXMOX_PORT" '.port = $v')" fi if [[ -n "$NEW_PROXMOX_USERNAME" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_USERNAME" '.username = $v')" fi if [[ -n "$NEW_PROXMOX_TOKEN_ID" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_ID" '.token_id = $v')" fi if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_SECRET" '.token_secret = $v')" fi if [[ -n "$NEW_PROXMOX_VERIFY_SSL" ]]; then proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "${NEW_PROXMOX_VERIFY_SSL,,}" '.verify_ssl = ($v == "true")')" fi if [[ "$(json_canonical "$proxmox_payload")" != "$(json_canonical "$ORIGINAL_PROXMOX_JSON")" ]]; then preflight_proxmox_candidate "$proxmox_payload" log "Applying Proxmox settings rotation" api_put "/settings/proxmox" "$proxmox_payload" UPDATED_PROXMOX="true" log "Post-rotation validation: Proxmox sync" curl -fsS -X POST "${BACKEND_URL}/proxmox/sync" -H "Authorization: Bearer ${API_TOKEN}" >/dev/null fi if [[ -n "$NEW_PAYSTACK_SECRET" ]]; then payment_payload="$( printf '%s' "$payment_payload" | jq --arg v "$NEW_PAYSTACK_SECRET" ' if (.paystack_secret // "") == $v then . else .paystack_secret_previous = (.paystack_secret // "") | .paystack_secret = $v end ' )" fi if [[ -n "$NEW_FLUTTERWAVE_SECRET" ]]; then payment_payload="$( printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_SECRET" ' if (.flutterwave_secret // "") == $v then . else .flutterwave_secret_previous = (.flutterwave_secret // "") | .flutterwave_secret = $v end ' )" fi if [[ -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ]]; then payment_payload="$( printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ' if (.flutterwave_webhook_hash // "") == $v then . else .flutterwave_webhook_hash_previous = (.flutterwave_webhook_hash // "") | .flutterwave_webhook_hash = $v end ' )" fi if [[ "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then payment_payload="$( printf '%s' "$payment_payload" | jq ' .paystack_secret_previous = "" | .flutterwave_secret_previous = "" | .flutterwave_webhook_hash_previous = "" ' )" fi if [[ "$(json_canonical "$payment_payload")" != "$(json_canonical "$ORIGINAL_PAYMENT_JSON")" ]]; then log "Applying payment/webhook rotation payload" api_put "/settings/payment" "$payment_payload" UPDATED_PAYMENT="true" fi if [[ -n "$NEW_MONITORING_WEBHOOK_URL" ]]; then notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_MONITORING_WEBHOOK_URL" '.monitoring_webhook_url = $v')" fi if [[ -n "$NEW_ALERT_WEBHOOK_URL" ]]; then notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_ALERT_WEBHOOK_URL" '.alert_webhook_url = $v')" fi if [[ -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" ]]; then notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_NOTIFICATION_EMAIL_WEBHOOK" '.notification_email_webhook = $v')" fi if [[ -n "$NEW_OPS_EMAIL" ]]; then notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_OPS_EMAIL" '.ops_email = $v')" fi notifications_payload="$(sanitize_notifications_payload "$notifications_payload")" if [[ "$(json_canonical "$notifications_payload")" != "$(json_canonical "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED")" ]]; then log "Applying alerting destination updates" api_put "/settings/notifications" "$notifications_payload" UPDATED_NOTIFICATIONS="true" fi local summary_file summary_file="/root/proxpanel-integration-rotation-${TS}.txt" cat >"$summary_file" <