ops: add integration secret rotation and offsite backup alerting

This commit is contained in:
Austin A
2026-04-18 09:33:17 +01:00
parent 95633a6722
commit 81be9c5e42
13 changed files with 1105 additions and 16 deletions

View File

@@ -0,0 +1,469 @@
#!/usr/bin/env bash
set -Eeuo pipefail
APP_DIR="${APP_DIR:-/opt/proxpanel}"
ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}"
BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
BACKEND_URL="${BACKEND_URL:-}"
ADMIN_EMAIL_INPUT=""
ADMIN_PASSWORD_INPUT=""
NEW_PROXMOX_HOST=""
NEW_PROXMOX_PORT=""
NEW_PROXMOX_USERNAME=""
NEW_PROXMOX_TOKEN_ID=""
NEW_PROXMOX_TOKEN_SECRET=""
NEW_PROXMOX_VERIFY_SSL=""
NEW_PAYSTACK_SECRET=""
NEW_FLUTTERWAVE_SECRET=""
NEW_FLUTTERWAVE_WEBHOOK_HASH=""
FINALIZE_PAYMENT_GRACE="false"
NEW_MONITORING_WEBHOOK_URL=""
NEW_ALERT_WEBHOOK_URL=""
NEW_NOTIFICATION_EMAIL_WEBHOOK=""
NEW_OPS_EMAIL=""
API_TOKEN=""
TS=""
BACKUP_DIR=""
UPDATED_PROXMOX="false"
UPDATED_PAYMENT="false"
UPDATED_NOTIFICATIONS="false"
ORIGINAL_PROXMOX_JSON=""
ORIGINAL_PAYMENT_JSON=""
ORIGINAL_NOTIFICATIONS_JSON=""
ORIGINAL_NOTIFICATIONS_JSON_SANITIZED=""
usage() {
cat <<'EOF'
Usage:
sudo bash infra/deploy/rotate-integration-secrets.sh [options]
Options:
--backend-url <url> Backend URL (default: http://127.0.0.1:${BACKEND_PORT}/api)
--admin-email <email> Admin email (default from .env.production)
--admin-password <password> Admin password (default from .env.production)
# Proxmox token rotation (zero-downtime by preflight + post-sync validation):
--proxmox-token-secret <secret> New Proxmox token secret
--proxmox-token-id <token-id> Optional new token id
--proxmox-username <user@realm> Optional new token username
--proxmox-host <host> Optional host override
--proxmox-port <port> Optional port override
--proxmox-verify-ssl <true|false> Optional SSL verify override
# Payment/webhook secret rotation (supports grace window):
--paystack-secret <secret> New Paystack secret (stores old value as paystack_secret_previous)
--flutterwave-secret <secret> New Flutterwave secret (stores old value as flutterwave_secret_previous)
--flutterwave-webhook-hash <hash> New Flutterwave webhook hash (stores old value as flutterwave_webhook_hash_previous)
--finalize-payment-webhook-grace Clears *_previous fields after provider cutover is complete
# Alerting webhook endpoints (optional):
--monitoring-webhook-url <url>
--alert-webhook-url <url>
--notification-email-webhook <url>
--ops-email <email>
-h, --help Show help
Examples:
sudo bash infra/deploy/rotate-integration-secrets.sh \
--proxmox-token-secret 'NEW_SECRET' \
--paystack-secret 'sk_live_new' \
--flutterwave-webhook-hash 'new_hash'
sudo bash infra/deploy/rotate-integration-secrets.sh \
--finalize-payment-webhook-grace
EOF
}
log() {
printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
}
die() {
printf '\n[ERROR] %s\n' "$*" >&2
exit 1
}
require_root() {
if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then
die "Run as root (or with sudo)."
fi
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
}
require_file() {
[[ -f "$1" ]] || die "Missing required file: $1"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--backend-url)
BACKEND_URL="${2:-}"
shift 2
;;
--admin-email)
ADMIN_EMAIL_INPUT="${2:-}"
shift 2
;;
--admin-password)
ADMIN_PASSWORD_INPUT="${2:-}"
shift 2
;;
--proxmox-host)
NEW_PROXMOX_HOST="${2:-}"
shift 2
;;
--proxmox-port)
NEW_PROXMOX_PORT="${2:-}"
shift 2
;;
--proxmox-username)
NEW_PROXMOX_USERNAME="${2:-}"
shift 2
;;
--proxmox-token-id)
NEW_PROXMOX_TOKEN_ID="${2:-}"
shift 2
;;
--proxmox-token-secret)
NEW_PROXMOX_TOKEN_SECRET="${2:-}"
shift 2
;;
--proxmox-verify-ssl)
NEW_PROXMOX_VERIFY_SSL="${2:-}"
shift 2
;;
--paystack-secret)
NEW_PAYSTACK_SECRET="${2:-}"
shift 2
;;
--flutterwave-secret)
NEW_FLUTTERWAVE_SECRET="${2:-}"
shift 2
;;
--flutterwave-webhook-hash)
NEW_FLUTTERWAVE_WEBHOOK_HASH="${2:-}"
shift 2
;;
--finalize-payment-webhook-grace)
FINALIZE_PAYMENT_GRACE="true"
shift
;;
--monitoring-webhook-url)
NEW_MONITORING_WEBHOOK_URL="${2:-}"
shift 2
;;
--alert-webhook-url)
NEW_ALERT_WEBHOOK_URL="${2:-}"
shift 2
;;
--notification-email-webhook)
NEW_NOTIFICATION_EMAIL_WEBHOOK="${2:-}"
shift 2
;;
--ops-email)
NEW_OPS_EMAIL="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "Unknown argument: $1"
;;
esac
done
}
json_escape() {
printf '%s' "$1" | jq -Rs .
}
json_canonical() {
printf '%s' "$1" | jq -cS .
}
sanitize_notifications_payload() {
printf '%s' "$1" | jq '
if (.monitoring_webhook_url // "") == "" then del(.monitoring_webhook_url) else . end
| if (.alert_webhook_url // "") == "" then del(.alert_webhook_url) else . end
| if (.email_gateway_url // "") == "" then del(.email_gateway_url) else . end
| if (.notification_email_webhook // "") == "" then del(.notification_email_webhook) else . end
| if (.ops_email // "") == "" then del(.ops_email) else . end
'
}
api_get() {
local path="$1"
curl -fsS \
-H "Authorization: Bearer ${API_TOKEN}" \
"${BACKEND_URL}${path}"
}
api_put() {
local path="$1"
local payload="$2"
curl -fsS \
-X PUT \
-H "Authorization: Bearer ${API_TOKEN}" \
-H "Content-Type: application/json" \
-d "$payload" \
"${BACKEND_URL}${path}" >/dev/null
}
preflight_proxmox_candidate() {
local payload_json="$1"
local host port username token_id token_secret verify_ssl auth_header
host="$(printf '%s' "$payload_json" | jq -r '.host // empty')"
port="$(printf '%s' "$payload_json" | jq -r '.port // 8006')"
username="$(printf '%s' "$payload_json" | jq -r '.username // empty')"
token_id="$(printf '%s' "$payload_json" | jq -r '.token_id // empty')"
token_secret="$(printf '%s' "$payload_json" | jq -r '.token_secret // empty')"
verify_ssl="$(printf '%s' "$payload_json" | jq -r '.verify_ssl // true')"
[[ -n "$host" && -n "$username" && -n "$token_id" && -n "$token_secret" ]] || {
die "Candidate Proxmox settings are incomplete."
}
auth_header="PVEAPIToken=${username}!${token_id}=${token_secret}"
log "Preflight: validating candidate Proxmox token directly against ${host}:${port}"
if [[ "$verify_ssl" == "false" ]]; then
curl -ksfS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null
else
curl -fsS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null
fi
}
rollback_if_needed() {
if [[ -z "$API_TOKEN" ]]; then
return
fi
if [[ "$UPDATED_NOTIFICATIONS" == "true" && -n "$ORIGINAL_NOTIFICATIONS_JSON" ]]; then
log "Rollback: restoring notifications settings"
api_put "/settings/notifications" "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED" || true
fi
if [[ "$UPDATED_PAYMENT" == "true" && -n "$ORIGINAL_PAYMENT_JSON" ]]; then
log "Rollback: restoring payment settings"
api_put "/settings/payment" "$ORIGINAL_PAYMENT_JSON" || true
fi
if [[ "$UPDATED_PROXMOX" == "true" && -n "$ORIGINAL_PROXMOX_JSON" ]]; then
log "Rollback: restoring proxmox settings"
api_put "/settings/proxmox" "$ORIGINAL_PROXMOX_JSON" || true
fi
}
on_error() {
local line="$1"
printf '\n[ERROR] Rotation failed at line %s. Attempting rollback where possible.\n' "$line" >&2
rollback_if_needed
}
build_backend_url() {
if [[ -n "$BACKEND_URL" ]]; then
return
fi
local backend_port
backend_port="${BACKEND_PORT:-8080}"
BACKEND_URL="http://127.0.0.1:${backend_port}/api"
}
main() {
parse_args "$@"
require_root
require_command curl
require_command jq
require_file "$ENV_FILE"
# shellcheck disable=SC1090
source "$ENV_FILE"
build_backend_url
local admin_email admin_password
admin_email="${ADMIN_EMAIL_INPUT:-${ADMIN_EMAIL:-}}"
admin_password="${ADMIN_PASSWORD_INPUT:-${ADMIN_PASSWORD:-}}"
[[ -n "$admin_email" && -n "$admin_password" ]] || {
die "Admin credentials are required (provide --admin-email/--admin-password or set ADMIN_EMAIL/ADMIN_PASSWORD in env)."
}
local wants_rotation="false"
if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" || -n "$NEW_PROXMOX_TOKEN_ID" || -n "$NEW_PROXMOX_USERNAME" || -n "$NEW_PROXMOX_HOST" || -n "$NEW_PROXMOX_PORT" || -n "$NEW_PROXMOX_VERIFY_SSL" || -n "$NEW_PAYSTACK_SECRET" || -n "$NEW_FLUTTERWAVE_SECRET" || -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" || -n "$NEW_MONITORING_WEBHOOK_URL" || -n "$NEW_ALERT_WEBHOOK_URL" || -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" || -n "$NEW_OPS_EMAIL" || "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then
wants_rotation="true"
fi
[[ "$wants_rotation" == "true" ]] || die "No rotation inputs supplied. Run with --help for options."
trap 'on_error $LINENO' ERR
TS="$(date -u +%Y%m%d-%H%M%S)"
BACKUP_DIR="${BACKUP_ROOT}/${TS}-integration-secret-rotation"
mkdir -p "$BACKUP_DIR"
chmod 700 "$BACKUP_DIR"
log "Authenticating against backend API"
local login_response
login_response="$(
curl -fsS -X POST "${BACKEND_URL}/auth/login" \
-H "Content-Type: application/json" \
-d "{\"email\":$(json_escape "$admin_email"),\"password\":$(json_escape "$admin_password")}"
)"
API_TOKEN="$(printf '%s' "$login_response" | jq -r '.token // empty')"
[[ -n "$API_TOKEN" ]] || die "Failed to obtain auth token from backend login."
log "Fetching current integration settings"
ORIGINAL_PROXMOX_JSON="$(api_get "/settings/proxmox")"
ORIGINAL_PAYMENT_JSON="$(api_get "/settings/payment")"
ORIGINAL_NOTIFICATIONS_JSON="$(api_get "/settings/notifications")"
ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="$(sanitize_notifications_payload "$ORIGINAL_NOTIFICATIONS_JSON")"
printf '%s\n' "$ORIGINAL_PROXMOX_JSON" >"${BACKUP_DIR}/proxmox.before.json"
printf '%s\n' "$ORIGINAL_PAYMENT_JSON" >"${BACKUP_DIR}/payment.before.json"
printf '%s\n' "$ORIGINAL_NOTIFICATIONS_JSON" >"${BACKUP_DIR}/notifications.before.json"
cp "$ENV_FILE" "${BACKUP_DIR}/.env.production.bak"
local proxmox_payload payment_payload notifications_payload
proxmox_payload="$ORIGINAL_PROXMOX_JSON"
payment_payload="$ORIGINAL_PAYMENT_JSON"
notifications_payload="$ORIGINAL_NOTIFICATIONS_JSON"
if [[ -n "$NEW_PROXMOX_HOST" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_HOST" '.host = $v')"
fi
if [[ -n "$NEW_PROXMOX_PORT" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --argjson v "$NEW_PROXMOX_PORT" '.port = $v')"
fi
if [[ -n "$NEW_PROXMOX_USERNAME" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_USERNAME" '.username = $v')"
fi
if [[ -n "$NEW_PROXMOX_TOKEN_ID" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_ID" '.token_id = $v')"
fi
if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_SECRET" '.token_secret = $v')"
fi
if [[ -n "$NEW_PROXMOX_VERIFY_SSL" ]]; then
proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "${NEW_PROXMOX_VERIFY_SSL,,}" '.verify_ssl = ($v == "true")')"
fi
if [[ "$(json_canonical "$proxmox_payload")" != "$(json_canonical "$ORIGINAL_PROXMOX_JSON")" ]]; then
preflight_proxmox_candidate "$proxmox_payload"
log "Applying Proxmox settings rotation"
api_put "/settings/proxmox" "$proxmox_payload"
UPDATED_PROXMOX="true"
log "Post-rotation validation: Proxmox sync"
curl -fsS -X POST "${BACKEND_URL}/proxmox/sync" -H "Authorization: Bearer ${API_TOKEN}" >/dev/null
fi
if [[ -n "$NEW_PAYSTACK_SECRET" ]]; then
payment_payload="$(
printf '%s' "$payment_payload" | jq --arg v "$NEW_PAYSTACK_SECRET" '
if (.paystack_secret // "") == $v then . else .paystack_secret_previous = (.paystack_secret // "") | .paystack_secret = $v end
'
)"
fi
if [[ -n "$NEW_FLUTTERWAVE_SECRET" ]]; then
payment_payload="$(
printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_SECRET" '
if (.flutterwave_secret // "") == $v then . else .flutterwave_secret_previous = (.flutterwave_secret // "") | .flutterwave_secret = $v end
'
)"
fi
if [[ -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ]]; then
payment_payload="$(
printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_WEBHOOK_HASH" '
if (.flutterwave_webhook_hash // "") == $v then .
else .flutterwave_webhook_hash_previous = (.flutterwave_webhook_hash // "") | .flutterwave_webhook_hash = $v end
'
)"
fi
if [[ "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then
payment_payload="$(
printf '%s' "$payment_payload" | jq '
.paystack_secret_previous = "" |
.flutterwave_secret_previous = "" |
.flutterwave_webhook_hash_previous = ""
'
)"
fi
if [[ "$(json_canonical "$payment_payload")" != "$(json_canonical "$ORIGINAL_PAYMENT_JSON")" ]]; then
log "Applying payment/webhook rotation payload"
api_put "/settings/payment" "$payment_payload"
UPDATED_PAYMENT="true"
fi
if [[ -n "$NEW_MONITORING_WEBHOOK_URL" ]]; then
notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_MONITORING_WEBHOOK_URL" '.monitoring_webhook_url = $v')"
fi
if [[ -n "$NEW_ALERT_WEBHOOK_URL" ]]; then
notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_ALERT_WEBHOOK_URL" '.alert_webhook_url = $v')"
fi
if [[ -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" ]]; then
notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_NOTIFICATION_EMAIL_WEBHOOK" '.notification_email_webhook = $v')"
fi
if [[ -n "$NEW_OPS_EMAIL" ]]; then
notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_OPS_EMAIL" '.ops_email = $v')"
fi
notifications_payload="$(sanitize_notifications_payload "$notifications_payload")"
if [[ "$(json_canonical "$notifications_payload")" != "$(json_canonical "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED")" ]]; then
log "Applying alerting destination updates"
api_put "/settings/notifications" "$notifications_payload"
UPDATED_NOTIFICATIONS="true"
fi
local summary_file
summary_file="/root/proxpanel-integration-rotation-${TS}.txt"
cat >"$summary_file" <<EOF
ProxPanel integration secret rotation completed at $(date -u +'%Y-%m-%d %H:%M:%S UTC')
Backend URL:
${BACKEND_URL}
Backup directory:
${BACKUP_DIR}
Changed blocks:
Proxmox settings: ${UPDATED_PROXMOX}
Payment/webhook settings: ${UPDATED_PAYMENT}
Notification destinations: ${UPDATED_NOTIFICATIONS}
Grace mode:
finalize_payment_webhook_grace=${FINALIZE_PAYMENT_GRACE}
Post checks:
GET ${BACKEND_URL}/health
POST ${BACKEND_URL}/proxmox/sync
Next:
If webhook grace fields were populated, finalize later with:
sudo bash ${APP_DIR}/infra/deploy/rotate-integration-secrets.sh --finalize-payment-webhook-grace
EOF
chmod 600 "$summary_file"
trap - ERR
log "Integration secret rotation completed successfully."
printf 'Summary: %s\n' "$summary_file"
}
main "$@"