ops: add integration secret rotation and offsite backup alerting

2026-04-18 09:33:17 +01:00
parent 95633a6722
commit 81be9c5e42
13 changed files with 1105 additions and 16 deletions
--- a/backend/src/routes/settings.routes.ts
+++ b/backend/src/routes/settings.routes.ts
@@ -22,9 +22,12 @@ const paymentSchema = z.object({
  default_provider: z.enum(["paystack", "flutterwave", "manual"]).default("paystack"),
  paystack_public: z.string().optional(),
  paystack_secret: z.string().optional(),
+  paystack_secret_previous: z.string().optional(),
  flutterwave_public: z.string().optional(),
  flutterwave_secret: z.string().optional(),
+  flutterwave_secret_previous: z.string().optional(),
  flutterwave_webhook_hash: z.string().optional(),
+  flutterwave_webhook_hash_previous: z.string().optional(),
  callback_url: z.string().optional()
 });

--- a/backend/src/services/payment.service.ts
+++ b/backend/src/services/payment.service.ts
@@ -10,9 +10,12 @@ type PaymentSettings = {
  default_provider?: "paystack" | "flutterwave" | "manual";
  paystack_public?: string;
  paystack_secret?: string;
+  paystack_secret_previous?: string;
  flutterwave_public?: string;
  flutterwave_secret?: string;
+  flutterwave_secret_previous?: string;
  flutterwave_webhook_hash?: string;
+  flutterwave_webhook_hash_previous?: string;
  callback_url?: string;
 };

@@ -135,18 +138,24 @@ export async function handleManualInvoicePayment(invoiceId: string, reference: s
 export async function verifyPaystackSignature(signature: string | undefined, rawBody: string | undefined) {
  if (!signature || !rawBody) return false;
  const settings = await getPaymentSettings();
-  if (!settings.paystack_secret) return false;
-  const expected = crypto
-    .createHmac("sha512", settings.paystack_secret)
-    .update(rawBody)
-    .digest("hex");
-  return expected === signature;
+  const secrets = [settings.paystack_secret, settings.paystack_secret_previous].filter(
+    (value): value is string => typeof value === "string" && value.trim().length > 0
+  );
+  if (secrets.length === 0) return false;
+
+  return secrets.some((secret) => {
+    const expected = crypto.createHmac("sha512", secret).update(rawBody).digest("hex");
+    return expected === signature;
+  });
 }

 export async function verifyFlutterwaveSignature(signature: string | undefined) {
  const settings = await getPaymentSettings();
-  if (!settings.flutterwave_webhook_hash) return false;
-  return settings.flutterwave_webhook_hash === signature;
+  const validHashes = [settings.flutterwave_webhook_hash, settings.flutterwave_webhook_hash_previous].filter(
+    (value): value is string => typeof value === "string" && value.trim().length > 0
+  );
+  if (validHashes.length === 0 || !signature) return false;
+  return validHashes.includes(signature);
 }

 export async function processPaystackWebhook(payload: any) {
--- a/infra/deploy/.backup.env.example
+++ b/infra/deploy/.backup.env.example
@@ -0,0 +1,20 @@
+BACKUP_ENCRYPTION_KEY=replace_with_128_hex_chars
+
+# Offsite replication (S3-compatible: AWS S3, Backblaze B2 S3, Wasabi)
+OFFSITE_BACKUP_ENABLED=false
+OFFSITE_S3_BUCKET=
+OFFSITE_S3_REGION=us-east-1
+OFFSITE_S3_PREFIX=proxpanel/db
+OFFSITE_S3_ENDPOINT_URL=
+OFFSITE_S3_ACCESS_KEY_ID=
+OFFSITE_S3_SECRET_ACCESS_KEY=
+OFFSITE_S3_SESSION_TOKEN=
+OFFSITE_S3_SSE=
+OFFSITE_REPLICA_RETENTION_DAYS=30
+
+# Alerting for backup / restore failures
+BACKUP_ALERT_WEBHOOK_URL=
+BACKUP_ALERT_EMAIL_WEBHOOK_URL=
+BACKUP_ALERT_EMAIL_TO=
+BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel_Backup]
+BACKUP_ALERT_SEND_SUCCESS=false
--- a/infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md
+++ b/infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md
@@ -0,0 +1,82 @@
+# Offsite Backup Replication + Failure Alerting
+
+This runbook configures:
+
+1. Encrypted local DB backups
+2. Replication to S3-compatible object storage (AWS S3, Backblaze B2 S3, Wasabi)
+3. Webhook/email alerting on backup or restore-test failure
+
+## 1) Configure secrets file
+
+Create or edit:
+
+`/opt/proxpanel/.backup.env`
+
+Reference template:
+
+`/opt/proxpanel/infra/deploy/.backup.env.example`
+
+## 2) Example provider configs
+
+### AWS S3
+
+```bash
+OFFSITE_BACKUP_ENABLED=true
+OFFSITE_S3_BUCKET=my-proxpanel-backups
+OFFSITE_S3_REGION=us-east-1
+OFFSITE_S3_PREFIX=proxpanel/db
+OFFSITE_S3_ACCESS_KEY_ID=AKIA...
+OFFSITE_S3_SECRET_ACCESS_KEY=...
+OFFSITE_S3_ENDPOINT_URL=
+```
+
+### Wasabi
+
+```bash
+OFFSITE_BACKUP_ENABLED=true
+OFFSITE_S3_BUCKET=my-proxpanel-backups
+OFFSITE_S3_REGION=us-east-1
+OFFSITE_S3_PREFIX=proxpanel/db
+OFFSITE_S3_ENDPOINT_URL=https://s3.us-east-1.wasabisys.com
+OFFSITE_S3_ACCESS_KEY_ID=...
+OFFSITE_S3_SECRET_ACCESS_KEY=...
+```
+
+### Backblaze B2 (S3 Compatible)
+
+```bash
+OFFSITE_BACKUP_ENABLED=true
+OFFSITE_S3_BUCKET=my-proxpanel-backups
+OFFSITE_S3_REGION=us-west-002
+OFFSITE_S3_PREFIX=proxpanel/db
+OFFSITE_S3_ENDPOINT_URL=https://s3.us-west-002.backblazeb2.com
+OFFSITE_S3_ACCESS_KEY_ID=...
+OFFSITE_S3_SECRET_ACCESS_KEY=...
+```
+
+## 3) Configure alerting
+
+Set one or both:
+
+```bash
+BACKUP_ALERT_WEBHOOK_URL=https://hooks.example.com/proxpanel-backup
+BACKUP_ALERT_EMAIL_WEBHOOK_URL=https://mailer.example.com/send
+BACKUP_ALERT_EMAIL_TO=ops@votcloud.com
+```
+
+## 4) Apply cron schedule
+
+```bash
+sudo bash /opt/proxpanel/infra/deploy/configure-db-backup-cron.sh --run-now
+```
+
+## 5) Verification
+
+1. Check local encrypted backup exists in `/opt/proxpanel-backups/daily/<timestamp>/`.
+2. Check offsite files:
+   - `proxpanel.sql.enc`
+   - `proxpanel.sql.enc.sha256`
+3. Check logs:
+   - `/var/log/proxpanel-db-backup.log`
+   - `/var/log/proxpanel-db-restore-test.log`
+4. Trigger controlled failure and confirm alert received (webhook/email).
--- a/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md
+++ b/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md
@@ -69,7 +69,21 @@ Rollback entrypoint:
 6. System Management page can load branding/policy/CMS data.
 7. Proxmox sync returns success (or actionable credential/SSL error message).

-## 6) Incident Rollback Criteria
+## 6) Backup Hardening (Offsite + Alerts)
+
+1. Configure `/opt/proxpanel/.backup.env`:
+   - `OFFSITE_BACKUP_ENABLED=true`
+   - `OFFSITE_S3_BUCKET`, `OFFSITE_S3_ACCESS_KEY_ID`, `OFFSITE_S3_SECRET_ACCESS_KEY`
+   - `OFFSITE_S3_ENDPOINT_URL` (required for Wasabi/B2 S3)
+   - `BACKUP_ALERT_WEBHOOK_URL` and/or `BACKUP_ALERT_EMAIL_WEBHOOK_URL`
+2. Apply cron wiring:
+   - `sudo bash /opt/proxpanel/infra/deploy/configure-db-backup-cron.sh --run-now`
+3. Validate offsite object upload:
+   - `aws s3 ls s3://<bucket>/<prefix>/<timestamp>/`
+4. Validate restore-test success and alert pipeline:
+   - force a controlled failure and verify webhook/email delivery
+
+## 7) Incident Rollback Criteria

 Rollback immediately if any of the following persists > 10 minutes:

--- a/infra/deploy/SECRET_ROTATION_CHECKLIST.md
+++ b/infra/deploy/SECRET_ROTATION_CHECKLIST.md
@@ -18,6 +18,11 @@ Recommended:
 2. Proxmox API token secret
 3. Payment/webhook secrets

+Enterprise hardening:
+
+1. Keep one grace window for webhook secret rotation (`*_previous`) to avoid dropped payment events during provider cutover.
+2. Validate new Proxmox token directly against `/api2/json/version` before applying it in panel settings.
+
 ## Runbook (Safe Order)

 1. Create timestamped app/env/db backup.
@@ -47,6 +52,22 @@ Script guarantees:
 3. Post-rotation health + login verified
 4. Summary written to `/root/proxpanel-secret-rotation-<timestamp>.txt`

+For integration secrets (Proxmox + payment/webhook + alerting endpoints), use:
+
+```bash
+sudo bash /opt/proxpanel/infra/deploy/rotate-integration-secrets.sh \
+  --proxmox-token-secret 'new_token_secret' \
+  --paystack-secret 'new_paystack_secret' \
+  --flutterwave-webhook-hash 'new_hash'
+```
+
+After external provider cutover is confirmed, clear grace secrets:
+
+```bash
+sudo bash /opt/proxpanel/infra/deploy/rotate-integration-secrets.sh \
+  --finalize-payment-webhook-grace
+```
+
 ## Rollback Plan

 If post-rotation checks fail:
--- a/infra/deploy/configure-db-backup-cron.sh
+++ b/infra/deploy/configure-db-backup-cron.sh
@@ -13,8 +13,11 @@ Usage:
  sudo bash infra/deploy/configure-db-backup-cron.sh [--run-now]

 Default schedule (UTC):
-  - 02:15 daily: encrypted DB backup
+  - 02:15 daily: encrypted DB backup + optional offsite replication
  - 02:45 daily: restore test against latest encrypted backup
+
+Alerting:
+  - backup/restore failures dispatch webhook/email alerts if configured in /opt/proxpanel/.backup.env
 EOF
 }

@@ -31,6 +34,14 @@ random_key() {
  openssl rand -hex 64 | tr -d '\n'
 }

+ensure_secret_setting() {
+  local key="$1"
+  local value="$2"
+  if ! grep -q "^${key}=" "$SECRET_FILE"; then
+    printf '%s=%s\n' "$key" "$value" >>"$SECRET_FILE"
+  fi
+}
+
 main() {
  local run_now="false"
  if [[ "${1:-}" == "--run-now" ]]; then
@@ -50,14 +61,49 @@ main() {
    umask 077
    cat >"$SECRET_FILE" <<EOF
 BACKUP_ENCRYPTION_KEY=$(random_key)
+OFFSITE_BACKUP_ENABLED=false
+OFFSITE_S3_BUCKET=
+OFFSITE_S3_REGION=us-east-1
+OFFSITE_S3_PREFIX=proxpanel/db
+OFFSITE_S3_ENDPOINT_URL=
+OFFSITE_S3_ACCESS_KEY_ID=
+OFFSITE_S3_SECRET_ACCESS_KEY=
+OFFSITE_S3_SESSION_TOKEN=
+OFFSITE_S3_SSE=
+OFFSITE_REPLICA_RETENTION_DAYS=30
+BACKUP_ALERT_WEBHOOK_URL=
+BACKUP_ALERT_EMAIL_WEBHOOK_URL=
+BACKUP_ALERT_EMAIL_TO=
+BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel_Backup]
+BACKUP_ALERT_SEND_SUCCESS=false
 EOF
  fi
  chmod 600 "$SECRET_FILE"

+  ensure_secret_setting "OFFSITE_BACKUP_ENABLED" "false"
+  ensure_secret_setting "OFFSITE_S3_BUCKET" ""
+  ensure_secret_setting "OFFSITE_S3_REGION" "us-east-1"
+  ensure_secret_setting "OFFSITE_S3_PREFIX" "proxpanel/db"
+  ensure_secret_setting "OFFSITE_S3_ENDPOINT_URL" ""
+  ensure_secret_setting "OFFSITE_S3_ACCESS_KEY_ID" ""
+  ensure_secret_setting "OFFSITE_S3_SECRET_ACCESS_KEY" ""
+  ensure_secret_setting "OFFSITE_S3_SESSION_TOKEN" ""
+  ensure_secret_setting "OFFSITE_S3_SSE" ""
+  ensure_secret_setting "OFFSITE_REPLICA_RETENTION_DAYS" "30"
+  ensure_secret_setting "BACKUP_ALERT_WEBHOOK_URL" ""
+  ensure_secret_setting "BACKUP_ALERT_EMAIL_WEBHOOK_URL" ""
+  ensure_secret_setting "BACKUP_ALERT_EMAIL_TO" ""
+  ensure_secret_setting "BACKUP_ALERT_SUBJECT_PREFIX" "[ProxPanel_Backup]"
+  ensure_secret_setting "BACKUP_ALERT_SEND_SUCCESS" "false"
+
  log "Making scripts executable"
  chmod +x \
    "$APP_DIR/infra/deploy/db-backup-encrypted.sh" \
-    "$APP_DIR/infra/deploy/db-restore-test.sh"
+    "$APP_DIR/infra/deploy/db-restore-test.sh" \
+    "$APP_DIR/infra/deploy/db-backup-replicate-offsite.sh" \
+    "$APP_DIR/infra/deploy/notify-backup-alert.sh" \
+    "$APP_DIR/infra/deploy/db-backup-job.sh" \
+    "$APP_DIR/infra/deploy/db-restore-test-job.sh"

  log "Installing cron schedule at $CRON_FILE"
  cat >"$CRON_FILE" <<'EOF'
@@ -65,10 +111,10 @@ SHELL=/bin/bash
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

 # Encrypted PostgreSQL backup every day at 02:15 UTC
-15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-encrypted.sh >> /var/log/proxpanel-db-backup.log 2>&1
+15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-job.sh >> /var/log/proxpanel-db-backup.log 2>&1

 # Restore test every day at 02:45 UTC
-45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test.sh >> /var/log/proxpanel-db-restore-test.log 2>&1
+45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test-job.sh >> /var/log/proxpanel-db-restore-test.log 2>&1
 EOF
  chmod 644 "$CRON_FILE"

@@ -77,8 +123,8 @@ EOF

  if [[ "$run_now" == "true" ]]; then
    log "Running immediate backup + restore test"
-    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-encrypted.sh"
-    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test.sh"
+    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-job.sh"
+    APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test-job.sh"
  fi

  log "DB backup/restore cron configured successfully."
--- a/infra/deploy/db-backup-job.sh
+++ b/infra/deploy/db-backup-job.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+
+BACKUP_SCRIPT="${APP_DIR}/infra/deploy/db-backup-encrypted.sh"
+REPLICATE_SCRIPT="${APP_DIR}/infra/deploy/db-backup-replicate-offsite.sh"
+NOTIFY_SCRIPT="${APP_DIR}/infra/deploy/notify-backup-alert.sh"
+
+job_failed() {
+  local line="$1"
+  local message="Daily backup job failed (line ${line}) on host $(hostname -f 2>/dev/null || hostname)"
+  APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+    --event backup_failed \
+    --severity critical \
+    --status failed \
+    --source db-backup-job \
+    --message "$message" \
+    --context-json "{\"line\":${line}}"
+}
+
+main() {
+  trap 'job_failed $LINENO' ERR
+
+  APP_DIR="$APP_DIR" "$BACKUP_SCRIPT"
+  APP_DIR="$APP_DIR" "$REPLICATE_SCRIPT"
+
+  local send_success="false"
+  if [[ -f "$SECRET_FILE" ]]; then
+    # shellcheck disable=SC1090
+    source "$SECRET_FILE"
+    send_success="${BACKUP_ALERT_SEND_SUCCESS:-false}"
+  fi
+
+  if [[ "$send_success" == "true" ]]; then
+    APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+      --event backup_success \
+      --severity info \
+      --status ok \
+      --source db-backup-job \
+      --message "Daily backup + offsite replication completed successfully"
+  fi
+
+  trap - ERR
+}
+
+main "$@"
--- a/infra/deploy/db-backup-replicate-offsite.sh
+++ b/infra/deploy/db-backup-replicate-offsite.sh
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-replicate-offsite.sh [--backup-dir <dir>]
+
+Reads replication settings from /opt/proxpanel/.backup.env.
+Required keys for S3-compatible replication:
+  OFFSITE_BACKUP_ENABLED=true
+  OFFSITE_S3_BUCKET=<bucket-name>
+  OFFSITE_S3_ACCESS_KEY_ID=<access-key>
+  OFFSITE_S3_SECRET_ACCESS_KEY=<secret-key>
+
+Optional:
+  OFFSITE_S3_REGION=us-east-1
+  OFFSITE_S3_PREFIX=proxpanel/db
+  OFFSITE_S3_ENDPOINT_URL=https://s3.us-west-1.wasabisys.com
+  OFFSITE_S3_SSE=AES256
+  OFFSITE_REPLICA_RETENTION_DAYS=30
+EOF
+}
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_file() {
+  [[ -f "$1" ]] || die "Missing required file: $1"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
+}
+
+find_latest_backup_dir() {
+  find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1
+}
+
+prune_remote_retention() {
+  local bucket="$1"
+  local prefix="$2"
+  local endpoint_flag=()
+  if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then
+    endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL")
+  fi
+
+  local days="${OFFSITE_REPLICA_RETENTION_DAYS:-}"
+  if [[ -z "$days" || ! "$days" =~ ^[0-9]+$ || "$days" -le 0 ]]; then
+    return
+  fi
+
+  local cutoff
+  cutoff="$(date -u -d "-${days} days" +%Y%m%d-%H%M%S)"
+  log "Applying offsite retention policy (${days} days; cutoff=${cutoff})"
+
+  local listing prefixes old_prefix
+  listing="$(aws "${endpoint_flag[@]}" s3 ls "s3://${bucket}/${prefix}/" || true)"
+  prefixes="$(printf '%s\n' "$listing" | awk '/PRE [0-9]{8}-[0-9]{6}\// {print $2}' | tr -d '/')"
+
+  while IFS= read -r old_prefix; do
+    [[ -n "$old_prefix" ]] || continue
+    if [[ "$old_prefix" < "$cutoff" ]]; then
+      log "Pruning remote backup prefix ${old_prefix}"
+      aws "${endpoint_flag[@]}" s3 rm "s3://${bucket}/${prefix}/${old_prefix}/" --recursive >/dev/null
+    fi
+  done <<<"$prefixes"
+}
+
+main() {
+  local forced_backup_dir=""
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --backup-dir)
+        forced_backup_dir="${2:-}"
+        shift 2
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        die "Unknown argument: $1"
+        ;;
+    esac
+  done
+
+  require_file "$SECRET_FILE"
+
+  # shellcheck disable=SC1090
+  source "$SECRET_FILE"
+
+  if [[ "${OFFSITE_BACKUP_ENABLED:-false}" != "true" ]]; then
+    log "Offsite replication disabled (OFFSITE_BACKUP_ENABLED != true)."
+    exit 0
+  fi
+
+  require_command aws
+
+  [[ -n "${OFFSITE_S3_BUCKET:-}" ]] || die "OFFSITE_S3_BUCKET is required."
+  [[ -n "${OFFSITE_S3_ACCESS_KEY_ID:-}" ]] || die "OFFSITE_S3_ACCESS_KEY_ID is required."
+  [[ -n "${OFFSITE_S3_SECRET_ACCESS_KEY:-}" ]] || die "OFFSITE_S3_SECRET_ACCESS_KEY is required."
+
+  export AWS_ACCESS_KEY_ID="$OFFSITE_S3_ACCESS_KEY_ID"
+  export AWS_SECRET_ACCESS_KEY="$OFFSITE_S3_SECRET_ACCESS_KEY"
+  if [[ -n "${OFFSITE_S3_SESSION_TOKEN:-}" ]]; then
+    export AWS_SESSION_TOKEN="$OFFSITE_S3_SESSION_TOKEN"
+  fi
+  export AWS_DEFAULT_REGION="${OFFSITE_S3_REGION:-us-east-1}"
+
+  local backup_dir
+  if [[ -n "$forced_backup_dir" ]]; then
+    backup_dir="$forced_backup_dir"
+  else
+    backup_dir="$(find_latest_backup_dir)"
+  fi
+
+  [[ -n "$backup_dir" && -d "$backup_dir" ]] || die "Unable to locate backup directory."
+
+  local encrypted_file checksum_file stamp prefix endpoint_flag sse_flag destination
+  encrypted_file="${backup_dir}/proxpanel.sql.enc"
+  checksum_file="${encrypted_file}.sha256"
+  require_file "$encrypted_file"
+  require_file "$checksum_file"
+
+  stamp="$(basename "$backup_dir")"
+  prefix="${OFFSITE_S3_PREFIX:-proxpanel/db}"
+  destination="s3://${OFFSITE_S3_BUCKET}/${prefix}/${stamp}/"
+
+  endpoint_flag=()
+  if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then
+    endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL")
+  fi
+
+  sse_flag=()
+  if [[ -n "${OFFSITE_S3_SSE:-}" ]]; then
+    sse_flag=(--sse "$OFFSITE_S3_SSE")
+  fi
+
+  log "Replicating encrypted backup to ${destination}"
+  aws "${endpoint_flag[@]}" s3 cp "$encrypted_file" "${destination}proxpanel.sql.enc" "${sse_flag[@]}" --only-show-errors
+  aws "${endpoint_flag[@]}" s3 cp "$checksum_file" "${destination}proxpanel.sql.enc.sha256" "${sse_flag[@]}" --only-show-errors
+
+  log "Verifying offsite object presence"
+  aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc" >/dev/null
+  aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc.sha256" >/dev/null
+
+  prune_remote_retention "$OFFSITE_S3_BUCKET" "$prefix"
+  log "Offsite replication completed successfully."
+}
+
+main "$@"
--- a/infra/deploy/db-restore-test-job.sh
+++ b/infra/deploy/db-restore-test-job.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+
+RESTORE_SCRIPT="${APP_DIR}/infra/deploy/db-restore-test.sh"
+NOTIFY_SCRIPT="${APP_DIR}/infra/deploy/notify-backup-alert.sh"
+
+job_failed() {
+  local line="$1"
+  local message="Daily restore test failed (line ${line}) on host $(hostname -f 2>/dev/null || hostname)"
+  APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+    --event restore_test_failed \
+    --severity critical \
+    --status failed \
+    --source db-restore-test-job \
+    --message "$message" \
+    --context-json "{\"line\":${line}}"
+}
+
+main() {
+  trap 'job_failed $LINENO' ERR
+
+  APP_DIR="$APP_DIR" "$RESTORE_SCRIPT"
+
+  local send_success="false"
+  if [[ -f "$SECRET_FILE" ]]; then
+    # shellcheck disable=SC1090
+    source "$SECRET_FILE"
+    send_success="${BACKUP_ALERT_SEND_SUCCESS:-false}"
+  fi
+
+  if [[ "$send_success" == "true" ]]; then
+    APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+      --event restore_test_success \
+      --severity info \
+      --status ok \
+      --source db-restore-test-job \
+      --message "Daily restore test completed successfully"
+  fi
+
+  trap - ERR
+}
+
+main "$@"
--- a/infra/deploy/install-proxpanel.sh
+++ b/infra/deploy/install-proxpanel.sh
@@ -121,7 +121,7 @@ install_prereqs() {
  log "Installing OS prerequisites..."
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -y
-  apt-get install -y ca-certificates curl git openssl jq rsync
+  apt-get install -y ca-certificates curl git openssl jq rsync awscli
 }

 install_docker_if_needed() {
--- a/infra/deploy/notify-backup-alert.sh
+++ b/infra/deploy/notify-backup-alert.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+
+EVENT="${EVENT:-}"
+SEVERITY="${SEVERITY:-warning}"
+STATUS="${STATUS:-failed}"
+MESSAGE="${MESSAGE:-}"
+SOURCE="${SOURCE:-backup-jobs}"
+CONTEXT_JSON="${CONTEXT_JSON:-{}}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/notify-backup-alert.sh \
+    --event <backup_failed|restore_test_failed|...> \
+    --severity <info|warning|critical> \
+    --status <ok|failed> \
+    --message "<human-readable-message>" \
+    [--source backup-cron] \
+    [--context-json '{"key":"value"}']
+
+Alert destinations are read from /opt/proxpanel/.backup.env:
+  BACKUP_ALERT_WEBHOOK_URL=
+  BACKUP_ALERT_EMAIL_WEBHOOK_URL=
+  BACKUP_ALERT_EMAIL_TO=
+  BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel Backup]
+EOF
+}
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    printf '[WARN] Missing command: %s\n' "$1" >&2
+    return 1
+  }
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --event)
+        EVENT="${2:-}"
+        shift 2
+        ;;
+      --severity)
+        SEVERITY="${2:-}"
+        shift 2
+        ;;
+      --status)
+        STATUS="${2:-}"
+        shift 2
+        ;;
+      --message)
+        MESSAGE="${2:-}"
+        shift 2
+        ;;
+      --source)
+        SOURCE="${2:-}"
+        shift 2
+        ;;
+      --context-json)
+        CONTEXT_JSON="${2:-{}}"
+        shift 2
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        printf '[WARN] Ignoring unknown argument: %s\n' "$1" >&2
+        shift
+        ;;
+    esac
+  done
+}
+
+main() {
+  parse_args "$@"
+  require_command curl || exit 0
+  require_command jq || exit 0
+
+  if [[ -f "$SECRET_FILE" ]]; then
+    # shellcheck disable=SC1090
+    source "$SECRET_FILE"
+  fi
+
+  [[ -n "$EVENT" ]] || EVENT="backup_job_event"
+  [[ -n "$MESSAGE" ]] || MESSAGE="Backup alert event raised"
+
+  local context
+  context="$(printf '%s' "$CONTEXT_JSON" | jq -c '.' 2>/dev/null || printf '{}')"
+
+  local payload now
+  now="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+  payload="$(
+    jq -n \
+      --arg type "backup.alert" \
+      --arg event "$EVENT" \
+      --arg severity "$SEVERITY" \
+      --arg status "$STATUS" \
+      --arg message "$MESSAGE" \
+      --arg source "$SOURCE" \
+      --arg timestamp "$now" \
+      --argjson context "$context" \
+      '{
+        type: $type,
+        event: $event,
+        severity: $severity,
+        status: $status,
+        source: $source,
+        message: $message,
+        timestamp: $timestamp,
+        context: $context
+      }'
+  )"
+
+  local webhook_url email_webhook email_to subject_prefix subject status_webhook status_email
+  webhook_url="${BACKUP_ALERT_WEBHOOK_URL:-}"
+  email_webhook="${BACKUP_ALERT_EMAIL_WEBHOOK_URL:-}"
+  email_to="${BACKUP_ALERT_EMAIL_TO:-${OPS_EMAIL:-}}"
+  subject_prefix="${BACKUP_ALERT_SUBJECT_PREFIX:-[ProxPanel Backup]}"
+  subject="${subject_prefix} ${EVENT} (${SEVERITY})"
+  status_webhook="skipped"
+  status_email="skipped"
+
+  if [[ -n "$webhook_url" ]]; then
+    if curl -fsS -X POST "$webhook_url" -H "Content-Type: application/json" -d "$payload" >/dev/null; then
+      status_webhook="sent"
+    else
+      status_webhook="failed"
+    fi
+  fi
+
+  if [[ -n "$email_webhook" && -n "$email_to" ]]; then
+    local email_payload
+    email_payload="$(
+      jq -n \
+        --arg type "backup.alert.email" \
+        --arg to "$email_to" \
+        --arg subject "$subject" \
+        --arg message "$MESSAGE" \
+        --argjson payload "$payload" \
+        '{
+          type: $type,
+          to: $to,
+          subject: $subject,
+          message: $message,
+          payload: $payload
+        }'
+    )"
+
+    if curl -fsS -X POST "$email_webhook" -H "Content-Type: application/json" -d "$email_payload" >/dev/null; then
+      status_email="sent"
+    else
+      status_email="failed"
+    fi
+  fi
+
+  log "Alert dispatch result: webhook=${status_webhook}, email=${status_email}"
+}
+
+main "$@"
--- a/infra/deploy/rotate-integration-secrets.sh
+++ b/infra/deploy/rotate-integration-secrets.sh
@@ -0,0 +1,469 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}"
+BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}"
+BACKEND_URL="${BACKEND_URL:-}"
+
+ADMIN_EMAIL_INPUT=""
+ADMIN_PASSWORD_INPUT=""
+
+NEW_PROXMOX_HOST=""
+NEW_PROXMOX_PORT=""
+NEW_PROXMOX_USERNAME=""
+NEW_PROXMOX_TOKEN_ID=""
+NEW_PROXMOX_TOKEN_SECRET=""
+NEW_PROXMOX_VERIFY_SSL=""
+
+NEW_PAYSTACK_SECRET=""
+NEW_FLUTTERWAVE_SECRET=""
+NEW_FLUTTERWAVE_WEBHOOK_HASH=""
+FINALIZE_PAYMENT_GRACE="false"
+
+NEW_MONITORING_WEBHOOK_URL=""
+NEW_ALERT_WEBHOOK_URL=""
+NEW_NOTIFICATION_EMAIL_WEBHOOK=""
+NEW_OPS_EMAIL=""
+
+API_TOKEN=""
+TS=""
+BACKUP_DIR=""
+
+UPDATED_PROXMOX="false"
+UPDATED_PAYMENT="false"
+UPDATED_NOTIFICATIONS="false"
+
+ORIGINAL_PROXMOX_JSON=""
+ORIGINAL_PAYMENT_JSON=""
+ORIGINAL_NOTIFICATIONS_JSON=""
+ORIGINAL_NOTIFICATIONS_JSON_SANITIZED=""
+
+usage() {
+  cat <<'EOF'
+Usage:
+  sudo bash infra/deploy/rotate-integration-secrets.sh [options]
+
+Options:
+  --backend-url <url>                     Backend URL (default: http://127.0.0.1:${BACKEND_PORT}/api)
+  --admin-email <email>                   Admin email (default from .env.production)
+  --admin-password <password>             Admin password (default from .env.production)
+
+  # Proxmox token rotation (zero-downtime by preflight + post-sync validation):
+  --proxmox-token-secret <secret>         New Proxmox token secret
+  --proxmox-token-id <token-id>           Optional new token id
+  --proxmox-username <user@realm>         Optional new token username
+  --proxmox-host <host>                   Optional host override
+  --proxmox-port <port>                   Optional port override
+  --proxmox-verify-ssl <true|false>       Optional SSL verify override
+
+  # Payment/webhook secret rotation (supports grace window):
+  --paystack-secret <secret>              New Paystack secret (stores old value as paystack_secret_previous)
+  --flutterwave-secret <secret>           New Flutterwave secret (stores old value as flutterwave_secret_previous)
+  --flutterwave-webhook-hash <hash>       New Flutterwave webhook hash (stores old value as flutterwave_webhook_hash_previous)
+  --finalize-payment-webhook-grace        Clears *_previous fields after provider cutover is complete
+
+  # Alerting webhook endpoints (optional):
+  --monitoring-webhook-url <url>
+  --alert-webhook-url <url>
+  --notification-email-webhook <url>
+  --ops-email <email>
+
+  -h, --help                              Show help
+
+Examples:
+  sudo bash infra/deploy/rotate-integration-secrets.sh \
+    --proxmox-token-secret 'NEW_SECRET' \
+    --paystack-secret 'sk_live_new' \
+    --flutterwave-webhook-hash 'new_hash'
+
+  sudo bash infra/deploy/rotate-integration-secrets.sh \
+    --finalize-payment-webhook-grace
+EOF
+}
+
+log() {
+  printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+die() {
+  printf '\n[ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_root() {
+  if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then
+    die "Run as root (or with sudo)."
+  fi
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
+}
+
+require_file() {
+  [[ -f "$1" ]] || die "Missing required file: $1"
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --backend-url)
+        BACKEND_URL="${2:-}"
+        shift 2
+        ;;
+      --admin-email)
+        ADMIN_EMAIL_INPUT="${2:-}"
+        shift 2
+        ;;
+      --admin-password)
+        ADMIN_PASSWORD_INPUT="${2:-}"
+        shift 2
+        ;;
+      --proxmox-host)
+        NEW_PROXMOX_HOST="${2:-}"
+        shift 2
+        ;;
+      --proxmox-port)
+        NEW_PROXMOX_PORT="${2:-}"
+        shift 2
+        ;;
+      --proxmox-username)
+        NEW_PROXMOX_USERNAME="${2:-}"
+        shift 2
+        ;;
+      --proxmox-token-id)
+        NEW_PROXMOX_TOKEN_ID="${2:-}"
+        shift 2
+        ;;
+      --proxmox-token-secret)
+        NEW_PROXMOX_TOKEN_SECRET="${2:-}"
+        shift 2
+        ;;
+      --proxmox-verify-ssl)
+        NEW_PROXMOX_VERIFY_SSL="${2:-}"
+        shift 2
+        ;;
+      --paystack-secret)
+        NEW_PAYSTACK_SECRET="${2:-}"
+        shift 2
+        ;;
+      --flutterwave-secret)
+        NEW_FLUTTERWAVE_SECRET="${2:-}"
+        shift 2
+        ;;
+      --flutterwave-webhook-hash)
+        NEW_FLUTTERWAVE_WEBHOOK_HASH="${2:-}"
+        shift 2
+        ;;
+      --finalize-payment-webhook-grace)
+        FINALIZE_PAYMENT_GRACE="true"
+        shift
+        ;;
+      --monitoring-webhook-url)
+        NEW_MONITORING_WEBHOOK_URL="${2:-}"
+        shift 2
+        ;;
+      --alert-webhook-url)
+        NEW_ALERT_WEBHOOK_URL="${2:-}"
+        shift 2
+        ;;
+      --notification-email-webhook)
+        NEW_NOTIFICATION_EMAIL_WEBHOOK="${2:-}"
+        shift 2
+        ;;
+      --ops-email)
+        NEW_OPS_EMAIL="${2:-}"
+        shift 2
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        die "Unknown argument: $1"
+        ;;
+    esac
+  done
+}
+
+json_escape() {
+  printf '%s' "$1" | jq -Rs .
+}
+
+json_canonical() {
+  printf '%s' "$1" | jq -cS .
+}
+
+sanitize_notifications_payload() {
+  printf '%s' "$1" | jq '
+    if (.monitoring_webhook_url // "") == "" then del(.monitoring_webhook_url) else . end
+    | if (.alert_webhook_url // "") == "" then del(.alert_webhook_url) else . end
+    | if (.email_gateway_url // "") == "" then del(.email_gateway_url) else . end
+    | if (.notification_email_webhook // "") == "" then del(.notification_email_webhook) else . end
+    | if (.ops_email // "") == "" then del(.ops_email) else . end
+  '
+}
+
+api_get() {
+  local path="$1"
+  curl -fsS \
+    -H "Authorization: Bearer ${API_TOKEN}" \
+    "${BACKEND_URL}${path}"
+}
+
+api_put() {
+  local path="$1"
+  local payload="$2"
+  curl -fsS \
+    -X PUT \
+    -H "Authorization: Bearer ${API_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "$payload" \
+    "${BACKEND_URL}${path}" >/dev/null
+}
+
+preflight_proxmox_candidate() {
+  local payload_json="$1"
+  local host port username token_id token_secret verify_ssl auth_header
+
+  host="$(printf '%s' "$payload_json" | jq -r '.host // empty')"
+  port="$(printf '%s' "$payload_json" | jq -r '.port // 8006')"
+  username="$(printf '%s' "$payload_json" | jq -r '.username // empty')"
+  token_id="$(printf '%s' "$payload_json" | jq -r '.token_id // empty')"
+  token_secret="$(printf '%s' "$payload_json" | jq -r '.token_secret // empty')"
+  verify_ssl="$(printf '%s' "$payload_json" | jq -r '.verify_ssl // true')"
+
+  [[ -n "$host" && -n "$username" && -n "$token_id" && -n "$token_secret" ]] || {
+    die "Candidate Proxmox settings are incomplete."
+  }
+
+  auth_header="PVEAPIToken=${username}!${token_id}=${token_secret}"
+  log "Preflight: validating candidate Proxmox token directly against ${host}:${port}"
+  if [[ "$verify_ssl" == "false" ]]; then
+    curl -ksfS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null
+  else
+    curl -fsS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null
+  fi
+}
+
+rollback_if_needed() {
+  if [[ -z "$API_TOKEN" ]]; then
+    return
+  fi
+
+  if [[ "$UPDATED_NOTIFICATIONS" == "true" && -n "$ORIGINAL_NOTIFICATIONS_JSON" ]]; then
+    log "Rollback: restoring notifications settings"
+    api_put "/settings/notifications" "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED" || true
+  fi
+
+  if [[ "$UPDATED_PAYMENT" == "true" && -n "$ORIGINAL_PAYMENT_JSON" ]]; then
+    log "Rollback: restoring payment settings"
+    api_put "/settings/payment" "$ORIGINAL_PAYMENT_JSON" || true
+  fi
+
+  if [[ "$UPDATED_PROXMOX" == "true" && -n "$ORIGINAL_PROXMOX_JSON" ]]; then
+    log "Rollback: restoring proxmox settings"
+    api_put "/settings/proxmox" "$ORIGINAL_PROXMOX_JSON" || true
+  fi
+}
+
+on_error() {
+  local line="$1"
+  printf '\n[ERROR] Rotation failed at line %s. Attempting rollback where possible.\n' "$line" >&2
+  rollback_if_needed
+}
+
+build_backend_url() {
+  if [[ -n "$BACKEND_URL" ]]; then
+    return
+  fi
+  local backend_port
+  backend_port="${BACKEND_PORT:-8080}"
+  BACKEND_URL="http://127.0.0.1:${backend_port}/api"
+}
+
+main() {
+  parse_args "$@"
+  require_root
+  require_command curl
+  require_command jq
+  require_file "$ENV_FILE"
+
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+  build_backend_url
+
+  local admin_email admin_password
+  admin_email="${ADMIN_EMAIL_INPUT:-${ADMIN_EMAIL:-}}"
+  admin_password="${ADMIN_PASSWORD_INPUT:-${ADMIN_PASSWORD:-}}"
+
+  [[ -n "$admin_email" && -n "$admin_password" ]] || {
+    die "Admin credentials are required (provide --admin-email/--admin-password or set ADMIN_EMAIL/ADMIN_PASSWORD in env)."
+  }
+
+  local wants_rotation="false"
+  if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" || -n "$NEW_PROXMOX_TOKEN_ID" || -n "$NEW_PROXMOX_USERNAME" || -n "$NEW_PROXMOX_HOST" || -n "$NEW_PROXMOX_PORT" || -n "$NEW_PROXMOX_VERIFY_SSL" || -n "$NEW_PAYSTACK_SECRET" || -n "$NEW_FLUTTERWAVE_SECRET" || -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" || -n "$NEW_MONITORING_WEBHOOK_URL" || -n "$NEW_ALERT_WEBHOOK_URL" || -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" || -n "$NEW_OPS_EMAIL" || "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then
+    wants_rotation="true"
+  fi
+  [[ "$wants_rotation" == "true" ]] || die "No rotation inputs supplied. Run with --help for options."
+
+  trap 'on_error $LINENO' ERR
+
+  TS="$(date -u +%Y%m%d-%H%M%S)"
+  BACKUP_DIR="${BACKUP_ROOT}/${TS}-integration-secret-rotation"
+  mkdir -p "$BACKUP_DIR"
+  chmod 700 "$BACKUP_DIR"
+
+  log "Authenticating against backend API"
+  local login_response
+  login_response="$(
+    curl -fsS -X POST "${BACKEND_URL}/auth/login" \
+      -H "Content-Type: application/json" \
+      -d "{\"email\":$(json_escape "$admin_email"),\"password\":$(json_escape "$admin_password")}"
+  )"
+  API_TOKEN="$(printf '%s' "$login_response" | jq -r '.token // empty')"
+  [[ -n "$API_TOKEN" ]] || die "Failed to obtain auth token from backend login."
+
+  log "Fetching current integration settings"
+  ORIGINAL_PROXMOX_JSON="$(api_get "/settings/proxmox")"
+  ORIGINAL_PAYMENT_JSON="$(api_get "/settings/payment")"
+  ORIGINAL_NOTIFICATIONS_JSON="$(api_get "/settings/notifications")"
+  ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="$(sanitize_notifications_payload "$ORIGINAL_NOTIFICATIONS_JSON")"
+
+  printf '%s\n' "$ORIGINAL_PROXMOX_JSON" >"${BACKUP_DIR}/proxmox.before.json"
+  printf '%s\n' "$ORIGINAL_PAYMENT_JSON" >"${BACKUP_DIR}/payment.before.json"
+  printf '%s\n' "$ORIGINAL_NOTIFICATIONS_JSON" >"${BACKUP_DIR}/notifications.before.json"
+  cp "$ENV_FILE" "${BACKUP_DIR}/.env.production.bak"
+
+  local proxmox_payload payment_payload notifications_payload
+  proxmox_payload="$ORIGINAL_PROXMOX_JSON"
+  payment_payload="$ORIGINAL_PAYMENT_JSON"
+  notifications_payload="$ORIGINAL_NOTIFICATIONS_JSON"
+
+  if [[ -n "$NEW_PROXMOX_HOST" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_HOST" '.host = $v')"
+  fi
+  if [[ -n "$NEW_PROXMOX_PORT" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --argjson v "$NEW_PROXMOX_PORT" '.port = $v')"
+  fi
+  if [[ -n "$NEW_PROXMOX_USERNAME" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_USERNAME" '.username = $v')"
+  fi
+  if [[ -n "$NEW_PROXMOX_TOKEN_ID" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_ID" '.token_id = $v')"
+  fi
+  if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_SECRET" '.token_secret = $v')"
+  fi
+  if [[ -n "$NEW_PROXMOX_VERIFY_SSL" ]]; then
+    proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "${NEW_PROXMOX_VERIFY_SSL,,}" '.verify_ssl = ($v == "true")')"
+  fi
+
+  if [[ "$(json_canonical "$proxmox_payload")" != "$(json_canonical "$ORIGINAL_PROXMOX_JSON")" ]]; then
+    preflight_proxmox_candidate "$proxmox_payload"
+    log "Applying Proxmox settings rotation"
+    api_put "/settings/proxmox" "$proxmox_payload"
+    UPDATED_PROXMOX="true"
+
+    log "Post-rotation validation: Proxmox sync"
+    curl -fsS -X POST "${BACKEND_URL}/proxmox/sync" -H "Authorization: Bearer ${API_TOKEN}" >/dev/null
+  fi
+
+  if [[ -n "$NEW_PAYSTACK_SECRET" ]]; then
+    payment_payload="$(
+      printf '%s' "$payment_payload" | jq --arg v "$NEW_PAYSTACK_SECRET" '
+        if (.paystack_secret // "") == $v then . else .paystack_secret_previous = (.paystack_secret // "") | .paystack_secret = $v end
+      '
+    )"
+  fi
+
+  if [[ -n "$NEW_FLUTTERWAVE_SECRET" ]]; then
+    payment_payload="$(
+      printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_SECRET" '
+        if (.flutterwave_secret // "") == $v then . else .flutterwave_secret_previous = (.flutterwave_secret // "") | .flutterwave_secret = $v end
+      '
+    )"
+  fi
+
+  if [[ -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ]]; then
+    payment_payload="$(
+      printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_WEBHOOK_HASH" '
+        if (.flutterwave_webhook_hash // "") == $v then .
+        else .flutterwave_webhook_hash_previous = (.flutterwave_webhook_hash // "") | .flutterwave_webhook_hash = $v end
+      '
+    )"
+  fi
+
+  if [[ "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then
+    payment_payload="$(
+      printf '%s' "$payment_payload" | jq '
+        .paystack_secret_previous = "" |
+        .flutterwave_secret_previous = "" |
+        .flutterwave_webhook_hash_previous = ""
+      '
+    )"
+  fi
+
+  if [[ "$(json_canonical "$payment_payload")" != "$(json_canonical "$ORIGINAL_PAYMENT_JSON")" ]]; then
+    log "Applying payment/webhook rotation payload"
+    api_put "/settings/payment" "$payment_payload"
+    UPDATED_PAYMENT="true"
+  fi
+
+  if [[ -n "$NEW_MONITORING_WEBHOOK_URL" ]]; then
+    notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_MONITORING_WEBHOOK_URL" '.monitoring_webhook_url = $v')"
+  fi
+  if [[ -n "$NEW_ALERT_WEBHOOK_URL" ]]; then
+    notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_ALERT_WEBHOOK_URL" '.alert_webhook_url = $v')"
+  fi
+  if [[ -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" ]]; then
+    notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_NOTIFICATION_EMAIL_WEBHOOK" '.notification_email_webhook = $v')"
+  fi
+  if [[ -n "$NEW_OPS_EMAIL" ]]; then
+    notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_OPS_EMAIL" '.ops_email = $v')"
+  fi
+
+  notifications_payload="$(sanitize_notifications_payload "$notifications_payload")"
+
+  if [[ "$(json_canonical "$notifications_payload")" != "$(json_canonical "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED")" ]]; then
+    log "Applying alerting destination updates"
+    api_put "/settings/notifications" "$notifications_payload"
+    UPDATED_NOTIFICATIONS="true"
+  fi
+
+  local summary_file
+  summary_file="/root/proxpanel-integration-rotation-${TS}.txt"
+  cat >"$summary_file" <<EOF
+ProxPanel integration secret rotation completed at $(date -u +'%Y-%m-%d %H:%M:%S UTC')
+
+Backend URL:
+  ${BACKEND_URL}
+
+Backup directory:
+  ${BACKUP_DIR}
+
+Changed blocks:
+  Proxmox settings: ${UPDATED_PROXMOX}
+  Payment/webhook settings: ${UPDATED_PAYMENT}
+  Notification destinations: ${UPDATED_NOTIFICATIONS}
+
+Grace mode:
+  finalize_payment_webhook_grace=${FINALIZE_PAYMENT_GRACE}
+
+Post checks:
+  GET ${BACKEND_URL}/health
+  POST ${BACKEND_URL}/proxmox/sync
+
+Next:
+  If webhook grace fields were populated, finalize later with:
+  sudo bash ${APP_DIR}/infra/deploy/rotate-integration-secrets.sh --finalize-payment-webhook-grace
+EOF
+  chmod 600 "$summary_file"
+
+  trap - ERR
+  log "Integration secret rotation completed successfully."
+  printf 'Summary: %s\n' "$summary_file"
+}
+
+main "$@"