From 81be9c5e427b0e19c5be8b3c5089e3fe99d63146 Mon Sep 17 00:00:00 2001 From: Austin A Date: Sat, 18 Apr 2026 09:33:17 +0100 Subject: [PATCH] ops: add integration secret rotation and offsite backup alerting --- backend/src/routes/settings.routes.ts | 3 + backend/src/services/payment.service.ts | 25 +- infra/deploy/.backup.env.example | 20 + infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md | 82 +++ .../PRODUCTION_CHECKLIST_my.votcloud.com.md | 16 +- infra/deploy/SECRET_ROTATION_CHECKLIST.md | 21 + infra/deploy/configure-db-backup-cron.sh | 58 ++- infra/deploy/db-backup-job.sh | 48 ++ infra/deploy/db-backup-replicate-offsite.sh | 163 ++++++ infra/deploy/db-restore-test-job.sh | 46 ++ infra/deploy/install-proxpanel.sh | 2 +- infra/deploy/notify-backup-alert.sh | 168 +++++++ infra/deploy/rotate-integration-secrets.sh | 469 ++++++++++++++++++ 13 files changed, 1105 insertions(+), 16 deletions(-) create mode 100644 infra/deploy/.backup.env.example create mode 100644 infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md create mode 100644 infra/deploy/db-backup-job.sh create mode 100644 infra/deploy/db-backup-replicate-offsite.sh create mode 100644 infra/deploy/db-restore-test-job.sh create mode 100644 infra/deploy/notify-backup-alert.sh create mode 100644 infra/deploy/rotate-integration-secrets.sh diff --git a/backend/src/routes/settings.routes.ts b/backend/src/routes/settings.routes.ts index e626eab..2a386c2 100644 --- a/backend/src/routes/settings.routes.ts +++ b/backend/src/routes/settings.routes.ts @@ -22,9 +22,12 @@ const paymentSchema = z.object({ default_provider: z.enum(["paystack", "flutterwave", "manual"]).default("paystack"), paystack_public: z.string().optional(), paystack_secret: z.string().optional(), + paystack_secret_previous: z.string().optional(), flutterwave_public: z.string().optional(), flutterwave_secret: z.string().optional(), + flutterwave_secret_previous: z.string().optional(), flutterwave_webhook_hash: z.string().optional(), + flutterwave_webhook_hash_previous: z.string().optional(), callback_url: z.string().optional() }); diff --git a/backend/src/services/payment.service.ts b/backend/src/services/payment.service.ts index ce1025c..3d50c6b 100644 --- a/backend/src/services/payment.service.ts +++ b/backend/src/services/payment.service.ts @@ -10,9 +10,12 @@ type PaymentSettings = { default_provider?: "paystack" | "flutterwave" | "manual"; paystack_public?: string; paystack_secret?: string; + paystack_secret_previous?: string; flutterwave_public?: string; flutterwave_secret?: string; + flutterwave_secret_previous?: string; flutterwave_webhook_hash?: string; + flutterwave_webhook_hash_previous?: string; callback_url?: string; }; @@ -135,18 +138,24 @@ export async function handleManualInvoicePayment(invoiceId: string, reference: s export async function verifyPaystackSignature(signature: string | undefined, rawBody: string | undefined) { if (!signature || !rawBody) return false; const settings = await getPaymentSettings(); - if (!settings.paystack_secret) return false; - const expected = crypto - .createHmac("sha512", settings.paystack_secret) - .update(rawBody) - .digest("hex"); - return expected === signature; + const secrets = [settings.paystack_secret, settings.paystack_secret_previous].filter( + (value): value is string => typeof value === "string" && value.trim().length > 0 + ); + if (secrets.length === 0) return false; + + return secrets.some((secret) => { + const expected = crypto.createHmac("sha512", secret).update(rawBody).digest("hex"); + return expected === signature; + }); } export async function verifyFlutterwaveSignature(signature: string | undefined) { const settings = await getPaymentSettings(); - if (!settings.flutterwave_webhook_hash) return false; - return settings.flutterwave_webhook_hash === signature; + const validHashes = [settings.flutterwave_webhook_hash, settings.flutterwave_webhook_hash_previous].filter( + (value): value is string => typeof value === "string" && value.trim().length > 0 + ); + if (validHashes.length === 0 || !signature) return false; + return validHashes.includes(signature); } export async function processPaystackWebhook(payload: any) { diff --git a/infra/deploy/.backup.env.example b/infra/deploy/.backup.env.example new file mode 100644 index 0000000..69922ab --- /dev/null +++ b/infra/deploy/.backup.env.example @@ -0,0 +1,20 @@ +BACKUP_ENCRYPTION_KEY=replace_with_128_hex_chars + +# Offsite replication (S3-compatible: AWS S3, Backblaze B2 S3, Wasabi) +OFFSITE_BACKUP_ENABLED=false +OFFSITE_S3_BUCKET= +OFFSITE_S3_REGION=us-east-1 +OFFSITE_S3_PREFIX=proxpanel/db +OFFSITE_S3_ENDPOINT_URL= +OFFSITE_S3_ACCESS_KEY_ID= +OFFSITE_S3_SECRET_ACCESS_KEY= +OFFSITE_S3_SESSION_TOKEN= +OFFSITE_S3_SSE= +OFFSITE_REPLICA_RETENTION_DAYS=30 + +# Alerting for backup / restore failures +BACKUP_ALERT_WEBHOOK_URL= +BACKUP_ALERT_EMAIL_WEBHOOK_URL= +BACKUP_ALERT_EMAIL_TO= +BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel_Backup] +BACKUP_ALERT_SEND_SUCCESS=false diff --git a/infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md b/infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md new file mode 100644 index 0000000..6aebc31 --- /dev/null +++ b/infra/deploy/OFFSITE_BACKUP_AND_ALERTING.md @@ -0,0 +1,82 @@ +# Offsite Backup Replication + Failure Alerting + +This runbook configures: + +1. Encrypted local DB backups +2. Replication to S3-compatible object storage (AWS S3, Backblaze B2 S3, Wasabi) +3. Webhook/email alerting on backup or restore-test failure + +## 1) Configure secrets file + +Create or edit: + +`/opt/proxpanel/.backup.env` + +Reference template: + +`/opt/proxpanel/infra/deploy/.backup.env.example` + +## 2) Example provider configs + +### AWS S3 + +```bash +OFFSITE_BACKUP_ENABLED=true +OFFSITE_S3_BUCKET=my-proxpanel-backups +OFFSITE_S3_REGION=us-east-1 +OFFSITE_S3_PREFIX=proxpanel/db +OFFSITE_S3_ACCESS_KEY_ID=AKIA... +OFFSITE_S3_SECRET_ACCESS_KEY=... +OFFSITE_S3_ENDPOINT_URL= +``` + +### Wasabi + +```bash +OFFSITE_BACKUP_ENABLED=true +OFFSITE_S3_BUCKET=my-proxpanel-backups +OFFSITE_S3_REGION=us-east-1 +OFFSITE_S3_PREFIX=proxpanel/db +OFFSITE_S3_ENDPOINT_URL=https://s3.us-east-1.wasabisys.com +OFFSITE_S3_ACCESS_KEY_ID=... +OFFSITE_S3_SECRET_ACCESS_KEY=... +``` + +### Backblaze B2 (S3 Compatible) + +```bash +OFFSITE_BACKUP_ENABLED=true +OFFSITE_S3_BUCKET=my-proxpanel-backups +OFFSITE_S3_REGION=us-west-002 +OFFSITE_S3_PREFIX=proxpanel/db +OFFSITE_S3_ENDPOINT_URL=https://s3.us-west-002.backblazeb2.com +OFFSITE_S3_ACCESS_KEY_ID=... +OFFSITE_S3_SECRET_ACCESS_KEY=... +``` + +## 3) Configure alerting + +Set one or both: + +```bash +BACKUP_ALERT_WEBHOOK_URL=https://hooks.example.com/proxpanel-backup +BACKUP_ALERT_EMAIL_WEBHOOK_URL=https://mailer.example.com/send +BACKUP_ALERT_EMAIL_TO=ops@votcloud.com +``` + +## 4) Apply cron schedule + +```bash +sudo bash /opt/proxpanel/infra/deploy/configure-db-backup-cron.sh --run-now +``` + +## 5) Verification + +1. Check local encrypted backup exists in `/opt/proxpanel-backups/daily//`. +2. Check offsite files: + - `proxpanel.sql.enc` + - `proxpanel.sql.enc.sha256` +3. Check logs: + - `/var/log/proxpanel-db-backup.log` + - `/var/log/proxpanel-db-restore-test.log` +4. Trigger controlled failure and confirm alert received (webhook/email). diff --git a/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md b/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md index 6a18c89..ed4043f 100644 --- a/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md +++ b/infra/deploy/PRODUCTION_CHECKLIST_my.votcloud.com.md @@ -69,7 +69,21 @@ Rollback entrypoint: 6. System Management page can load branding/policy/CMS data. 7. Proxmox sync returns success (or actionable credential/SSL error message). -## 6) Incident Rollback Criteria +## 6) Backup Hardening (Offsite + Alerts) + +1. Configure `/opt/proxpanel/.backup.env`: + - `OFFSITE_BACKUP_ENABLED=true` + - `OFFSITE_S3_BUCKET`, `OFFSITE_S3_ACCESS_KEY_ID`, `OFFSITE_S3_SECRET_ACCESS_KEY` + - `OFFSITE_S3_ENDPOINT_URL` (required for Wasabi/B2 S3) + - `BACKUP_ALERT_WEBHOOK_URL` and/or `BACKUP_ALERT_EMAIL_WEBHOOK_URL` +2. Apply cron wiring: + - `sudo bash /opt/proxpanel/infra/deploy/configure-db-backup-cron.sh --run-now` +3. Validate offsite object upload: + - `aws s3 ls s3://///` +4. Validate restore-test success and alert pipeline: + - force a controlled failure and verify webhook/email delivery + +## 7) Incident Rollback Criteria Rollback immediately if any of the following persists > 10 minutes: diff --git a/infra/deploy/SECRET_ROTATION_CHECKLIST.md b/infra/deploy/SECRET_ROTATION_CHECKLIST.md index 7105869..8a4e775 100644 --- a/infra/deploy/SECRET_ROTATION_CHECKLIST.md +++ b/infra/deploy/SECRET_ROTATION_CHECKLIST.md @@ -18,6 +18,11 @@ Recommended: 2. Proxmox API token secret 3. Payment/webhook secrets +Enterprise hardening: + +1. Keep one grace window for webhook secret rotation (`*_previous`) to avoid dropped payment events during provider cutover. +2. Validate new Proxmox token directly against `/api2/json/version` before applying it in panel settings. + ## Runbook (Safe Order) 1. Create timestamped app/env/db backup. @@ -47,6 +52,22 @@ Script guarantees: 3. Post-rotation health + login verified 4. Summary written to `/root/proxpanel-secret-rotation-.txt` +For integration secrets (Proxmox + payment/webhook + alerting endpoints), use: + +```bash +sudo bash /opt/proxpanel/infra/deploy/rotate-integration-secrets.sh \ + --proxmox-token-secret 'new_token_secret' \ + --paystack-secret 'new_paystack_secret' \ + --flutterwave-webhook-hash 'new_hash' +``` + +After external provider cutover is confirmed, clear grace secrets: + +```bash +sudo bash /opt/proxpanel/infra/deploy/rotate-integration-secrets.sh \ + --finalize-payment-webhook-grace +``` + ## Rollback Plan If post-rotation checks fail: diff --git a/infra/deploy/configure-db-backup-cron.sh b/infra/deploy/configure-db-backup-cron.sh index d50a16b..b3de3db 100755 --- a/infra/deploy/configure-db-backup-cron.sh +++ b/infra/deploy/configure-db-backup-cron.sh @@ -13,8 +13,11 @@ Usage: sudo bash infra/deploy/configure-db-backup-cron.sh [--run-now] Default schedule (UTC): - - 02:15 daily: encrypted DB backup + - 02:15 daily: encrypted DB backup + optional offsite replication - 02:45 daily: restore test against latest encrypted backup + +Alerting: + - backup/restore failures dispatch webhook/email alerts if configured in /opt/proxpanel/.backup.env EOF } @@ -31,6 +34,14 @@ random_key() { openssl rand -hex 64 | tr -d '\n' } +ensure_secret_setting() { + local key="$1" + local value="$2" + if ! grep -q "^${key}=" "$SECRET_FILE"; then + printf '%s=%s\n' "$key" "$value" >>"$SECRET_FILE" + fi +} + main() { local run_now="false" if [[ "${1:-}" == "--run-now" ]]; then @@ -50,14 +61,49 @@ main() { umask 077 cat >"$SECRET_FILE" <"$CRON_FILE" <<'EOF' @@ -65,10 +111,10 @@ SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin # Encrypted PostgreSQL backup every day at 02:15 UTC -15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-encrypted.sh >> /var/log/proxpanel-db-backup.log 2>&1 +15 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-job.sh >> /var/log/proxpanel-db-backup.log 2>&1 # Restore test every day at 02:45 UTC -45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test.sh >> /var/log/proxpanel-db-restore-test.log 2>&1 +45 2 * * * root APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-restore-test-job.sh >> /var/log/proxpanel-db-restore-test.log 2>&1 EOF chmod 644 "$CRON_FILE" @@ -77,8 +123,8 @@ EOF if [[ "$run_now" == "true" ]]; then log "Running immediate backup + restore test" - APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-encrypted.sh" - APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test.sh" + APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-backup-job.sh" + APP_DIR="$APP_DIR" "$APP_DIR/infra/deploy/db-restore-test-job.sh" fi log "DB backup/restore cron configured successfully." diff --git a/infra/deploy/db-backup-job.sh b/infra/deploy/db-backup-job.sh new file mode 100644 index 0000000..0cf9ad3 --- /dev/null +++ b/infra/deploy/db-backup-job.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" + +BACKUP_SCRIPT="${APP_DIR}/infra/deploy/db-backup-encrypted.sh" +REPLICATE_SCRIPT="${APP_DIR}/infra/deploy/db-backup-replicate-offsite.sh" +NOTIFY_SCRIPT="${APP_DIR}/infra/deploy/notify-backup-alert.sh" + +job_failed() { + local line="$1" + local message="Daily backup job failed (line ${line}) on host $(hostname -f 2>/dev/null || hostname)" + APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \ + --event backup_failed \ + --severity critical \ + --status failed \ + --source db-backup-job \ + --message "$message" \ + --context-json "{\"line\":${line}}" +} + +main() { + trap 'job_failed $LINENO' ERR + + APP_DIR="$APP_DIR" "$BACKUP_SCRIPT" + APP_DIR="$APP_DIR" "$REPLICATE_SCRIPT" + + local send_success="false" + if [[ -f "$SECRET_FILE" ]]; then + # shellcheck disable=SC1090 + source "$SECRET_FILE" + send_success="${BACKUP_ALERT_SEND_SUCCESS:-false}" + fi + + if [[ "$send_success" == "true" ]]; then + APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \ + --event backup_success \ + --severity info \ + --status ok \ + --source db-backup-job \ + --message "Daily backup + offsite replication completed successfully" + fi + + trap - ERR +} + +main "$@" diff --git a/infra/deploy/db-backup-replicate-offsite.sh b/infra/deploy/db-backup-replicate-offsite.sh new file mode 100644 index 0000000..bbb74c0 --- /dev/null +++ b/infra/deploy/db-backup-replicate-offsite.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups/daily}" + +usage() { + cat <<'EOF' +Usage: + APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/db-backup-replicate-offsite.sh [--backup-dir ] + +Reads replication settings from /opt/proxpanel/.backup.env. +Required keys for S3-compatible replication: + OFFSITE_BACKUP_ENABLED=true + OFFSITE_S3_BUCKET= + OFFSITE_S3_ACCESS_KEY_ID= + OFFSITE_S3_SECRET_ACCESS_KEY= + +Optional: + OFFSITE_S3_REGION=us-east-1 + OFFSITE_S3_PREFIX=proxpanel/db + OFFSITE_S3_ENDPOINT_URL=https://s3.us-west-1.wasabisys.com + OFFSITE_S3_SSE=AES256 + OFFSITE_REPLICA_RETENTION_DAYS=30 +EOF +} + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_file() { + [[ -f "$1" ]] || die "Missing required file: $1" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +find_latest_backup_dir() { + find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1 +} + +prune_remote_retention() { + local bucket="$1" + local prefix="$2" + local endpoint_flag=() + if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then + endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL") + fi + + local days="${OFFSITE_REPLICA_RETENTION_DAYS:-}" + if [[ -z "$days" || ! "$days" =~ ^[0-9]+$ || "$days" -le 0 ]]; then + return + fi + + local cutoff + cutoff="$(date -u -d "-${days} days" +%Y%m%d-%H%M%S)" + log "Applying offsite retention policy (${days} days; cutoff=${cutoff})" + + local listing prefixes old_prefix + listing="$(aws "${endpoint_flag[@]}" s3 ls "s3://${bucket}/${prefix}/" || true)" + prefixes="$(printf '%s\n' "$listing" | awk '/PRE [0-9]{8}-[0-9]{6}\// {print $2}' | tr -d '/')" + + while IFS= read -r old_prefix; do + [[ -n "$old_prefix" ]] || continue + if [[ "$old_prefix" < "$cutoff" ]]; then + log "Pruning remote backup prefix ${old_prefix}" + aws "${endpoint_flag[@]}" s3 rm "s3://${bucket}/${prefix}/${old_prefix}/" --recursive >/dev/null + fi + done <<<"$prefixes" +} + +main() { + local forced_backup_dir="" + + while [[ $# -gt 0 ]]; do + case "$1" in + --backup-dir) + forced_backup_dir="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "Unknown argument: $1" + ;; + esac + done + + require_file "$SECRET_FILE" + + # shellcheck disable=SC1090 + source "$SECRET_FILE" + + if [[ "${OFFSITE_BACKUP_ENABLED:-false}" != "true" ]]; then + log "Offsite replication disabled (OFFSITE_BACKUP_ENABLED != true)." + exit 0 + fi + + require_command aws + + [[ -n "${OFFSITE_S3_BUCKET:-}" ]] || die "OFFSITE_S3_BUCKET is required." + [[ -n "${OFFSITE_S3_ACCESS_KEY_ID:-}" ]] || die "OFFSITE_S3_ACCESS_KEY_ID is required." + [[ -n "${OFFSITE_S3_SECRET_ACCESS_KEY:-}" ]] || die "OFFSITE_S3_SECRET_ACCESS_KEY is required." + + export AWS_ACCESS_KEY_ID="$OFFSITE_S3_ACCESS_KEY_ID" + export AWS_SECRET_ACCESS_KEY="$OFFSITE_S3_SECRET_ACCESS_KEY" + if [[ -n "${OFFSITE_S3_SESSION_TOKEN:-}" ]]; then + export AWS_SESSION_TOKEN="$OFFSITE_S3_SESSION_TOKEN" + fi + export AWS_DEFAULT_REGION="${OFFSITE_S3_REGION:-us-east-1}" + + local backup_dir + if [[ -n "$forced_backup_dir" ]]; then + backup_dir="$forced_backup_dir" + else + backup_dir="$(find_latest_backup_dir)" + fi + + [[ -n "$backup_dir" && -d "$backup_dir" ]] || die "Unable to locate backup directory." + + local encrypted_file checksum_file stamp prefix endpoint_flag sse_flag destination + encrypted_file="${backup_dir}/proxpanel.sql.enc" + checksum_file="${encrypted_file}.sha256" + require_file "$encrypted_file" + require_file "$checksum_file" + + stamp="$(basename "$backup_dir")" + prefix="${OFFSITE_S3_PREFIX:-proxpanel/db}" + destination="s3://${OFFSITE_S3_BUCKET}/${prefix}/${stamp}/" + + endpoint_flag=() + if [[ -n "${OFFSITE_S3_ENDPOINT_URL:-}" ]]; then + endpoint_flag=(--endpoint-url "$OFFSITE_S3_ENDPOINT_URL") + fi + + sse_flag=() + if [[ -n "${OFFSITE_S3_SSE:-}" ]]; then + sse_flag=(--sse "$OFFSITE_S3_SSE") + fi + + log "Replicating encrypted backup to ${destination}" + aws "${endpoint_flag[@]}" s3 cp "$encrypted_file" "${destination}proxpanel.sql.enc" "${sse_flag[@]}" --only-show-errors + aws "${endpoint_flag[@]}" s3 cp "$checksum_file" "${destination}proxpanel.sql.enc.sha256" "${sse_flag[@]}" --only-show-errors + + log "Verifying offsite object presence" + aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc" >/dev/null + aws "${endpoint_flag[@]}" s3 ls "${destination}proxpanel.sql.enc.sha256" >/dev/null + + prune_remote_retention "$OFFSITE_S3_BUCKET" "$prefix" + log "Offsite replication completed successfully." +} + +main "$@" diff --git a/infra/deploy/db-restore-test-job.sh b/infra/deploy/db-restore-test-job.sh new file mode 100644 index 0000000..17695b1 --- /dev/null +++ b/infra/deploy/db-restore-test-job.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" + +RESTORE_SCRIPT="${APP_DIR}/infra/deploy/db-restore-test.sh" +NOTIFY_SCRIPT="${APP_DIR}/infra/deploy/notify-backup-alert.sh" + +job_failed() { + local line="$1" + local message="Daily restore test failed (line ${line}) on host $(hostname -f 2>/dev/null || hostname)" + APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \ + --event restore_test_failed \ + --severity critical \ + --status failed \ + --source db-restore-test-job \ + --message "$message" \ + --context-json "{\"line\":${line}}" +} + +main() { + trap 'job_failed $LINENO' ERR + + APP_DIR="$APP_DIR" "$RESTORE_SCRIPT" + + local send_success="false" + if [[ -f "$SECRET_FILE" ]]; then + # shellcheck disable=SC1090 + source "$SECRET_FILE" + send_success="${BACKUP_ALERT_SEND_SUCCESS:-false}" + fi + + if [[ "$send_success" == "true" ]]; then + APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \ + --event restore_test_success \ + --severity info \ + --status ok \ + --source db-restore-test-job \ + --message "Daily restore test completed successfully" + fi + + trap - ERR +} + +main "$@" diff --git a/infra/deploy/install-proxpanel.sh b/infra/deploy/install-proxpanel.sh index 2bc2972..8bf6355 100644 --- a/infra/deploy/install-proxpanel.sh +++ b/infra/deploy/install-proxpanel.sh @@ -121,7 +121,7 @@ install_prereqs() { log "Installing OS prerequisites..." export DEBIAN_FRONTEND=noninteractive apt-get update -y - apt-get install -y ca-certificates curl git openssl jq rsync + apt-get install -y ca-certificates curl git openssl jq rsync awscli } install_docker_if_needed() { diff --git a/infra/deploy/notify-backup-alert.sh b/infra/deploy/notify-backup-alert.sh new file mode 100644 index 0000000..fad6dfd --- /dev/null +++ b/infra/deploy/notify-backup-alert.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}" + +EVENT="${EVENT:-}" +SEVERITY="${SEVERITY:-warning}" +STATUS="${STATUS:-failed}" +MESSAGE="${MESSAGE:-}" +SOURCE="${SOURCE:-backup-jobs}" +CONTEXT_JSON="${CONTEXT_JSON:-{}}" + +usage() { + cat <<'EOF' +Usage: + APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/notify-backup-alert.sh \ + --event \ + --severity \ + --status \ + --message "" \ + [--source backup-cron] \ + [--context-json '{"key":"value"}'] + +Alert destinations are read from /opt/proxpanel/.backup.env: + BACKUP_ALERT_WEBHOOK_URL= + BACKUP_ALERT_EMAIL_WEBHOOK_URL= + BACKUP_ALERT_EMAIL_TO= + BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel Backup] +EOF +} + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + printf '[WARN] Missing command: %s\n' "$1" >&2 + return 1 + } +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --event) + EVENT="${2:-}" + shift 2 + ;; + --severity) + SEVERITY="${2:-}" + shift 2 + ;; + --status) + STATUS="${2:-}" + shift 2 + ;; + --message) + MESSAGE="${2:-}" + shift 2 + ;; + --source) + SOURCE="${2:-}" + shift 2 + ;; + --context-json) + CONTEXT_JSON="${2:-{}}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf '[WARN] Ignoring unknown argument: %s\n' "$1" >&2 + shift + ;; + esac + done +} + +main() { + parse_args "$@" + require_command curl || exit 0 + require_command jq || exit 0 + + if [[ -f "$SECRET_FILE" ]]; then + # shellcheck disable=SC1090 + source "$SECRET_FILE" + fi + + [[ -n "$EVENT" ]] || EVENT="backup_job_event" + [[ -n "$MESSAGE" ]] || MESSAGE="Backup alert event raised" + + local context + context="$(printf '%s' "$CONTEXT_JSON" | jq -c '.' 2>/dev/null || printf '{}')" + + local payload now + now="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + payload="$( + jq -n \ + --arg type "backup.alert" \ + --arg event "$EVENT" \ + --arg severity "$SEVERITY" \ + --arg status "$STATUS" \ + --arg message "$MESSAGE" \ + --arg source "$SOURCE" \ + --arg timestamp "$now" \ + --argjson context "$context" \ + '{ + type: $type, + event: $event, + severity: $severity, + status: $status, + source: $source, + message: $message, + timestamp: $timestamp, + context: $context + }' + )" + + local webhook_url email_webhook email_to subject_prefix subject status_webhook status_email + webhook_url="${BACKUP_ALERT_WEBHOOK_URL:-}" + email_webhook="${BACKUP_ALERT_EMAIL_WEBHOOK_URL:-}" + email_to="${BACKUP_ALERT_EMAIL_TO:-${OPS_EMAIL:-}}" + subject_prefix="${BACKUP_ALERT_SUBJECT_PREFIX:-[ProxPanel Backup]}" + subject="${subject_prefix} ${EVENT} (${SEVERITY})" + status_webhook="skipped" + status_email="skipped" + + if [[ -n "$webhook_url" ]]; then + if curl -fsS -X POST "$webhook_url" -H "Content-Type: application/json" -d "$payload" >/dev/null; then + status_webhook="sent" + else + status_webhook="failed" + fi + fi + + if [[ -n "$email_webhook" && -n "$email_to" ]]; then + local email_payload + email_payload="$( + jq -n \ + --arg type "backup.alert.email" \ + --arg to "$email_to" \ + --arg subject "$subject" \ + --arg message "$MESSAGE" \ + --argjson payload "$payload" \ + '{ + type: $type, + to: $to, + subject: $subject, + message: $message, + payload: $payload + }' + )" + + if curl -fsS -X POST "$email_webhook" -H "Content-Type: application/json" -d "$email_payload" >/dev/null; then + status_email="sent" + else + status_email="failed" + fi + fi + + log "Alert dispatch result: webhook=${status_webhook}, email=${status_email}" +} + +main "$@" diff --git a/infra/deploy/rotate-integration-secrets.sh b/infra/deploy/rotate-integration-secrets.sh new file mode 100644 index 0000000..fe450d3 --- /dev/null +++ b/infra/deploy/rotate-integration-secrets.sh @@ -0,0 +1,469 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +APP_DIR="${APP_DIR:-/opt/proxpanel}" +ENV_FILE="${ENV_FILE:-$APP_DIR/.env.production}" +BACKUP_ROOT="${BACKUP_ROOT:-/opt/proxpanel-backups}" +BACKEND_URL="${BACKEND_URL:-}" + +ADMIN_EMAIL_INPUT="" +ADMIN_PASSWORD_INPUT="" + +NEW_PROXMOX_HOST="" +NEW_PROXMOX_PORT="" +NEW_PROXMOX_USERNAME="" +NEW_PROXMOX_TOKEN_ID="" +NEW_PROXMOX_TOKEN_SECRET="" +NEW_PROXMOX_VERIFY_SSL="" + +NEW_PAYSTACK_SECRET="" +NEW_FLUTTERWAVE_SECRET="" +NEW_FLUTTERWAVE_WEBHOOK_HASH="" +FINALIZE_PAYMENT_GRACE="false" + +NEW_MONITORING_WEBHOOK_URL="" +NEW_ALERT_WEBHOOK_URL="" +NEW_NOTIFICATION_EMAIL_WEBHOOK="" +NEW_OPS_EMAIL="" + +API_TOKEN="" +TS="" +BACKUP_DIR="" + +UPDATED_PROXMOX="false" +UPDATED_PAYMENT="false" +UPDATED_NOTIFICATIONS="false" + +ORIGINAL_PROXMOX_JSON="" +ORIGINAL_PAYMENT_JSON="" +ORIGINAL_NOTIFICATIONS_JSON="" +ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="" + +usage() { + cat <<'EOF' +Usage: + sudo bash infra/deploy/rotate-integration-secrets.sh [options] + +Options: + --backend-url Backend URL (default: http://127.0.0.1:${BACKEND_PORT}/api) + --admin-email Admin email (default from .env.production) + --admin-password Admin password (default from .env.production) + + # Proxmox token rotation (zero-downtime by preflight + post-sync validation): + --proxmox-token-secret New Proxmox token secret + --proxmox-token-id Optional new token id + --proxmox-username Optional new token username + --proxmox-host Optional host override + --proxmox-port Optional port override + --proxmox-verify-ssl Optional SSL verify override + + # Payment/webhook secret rotation (supports grace window): + --paystack-secret New Paystack secret (stores old value as paystack_secret_previous) + --flutterwave-secret New Flutterwave secret (stores old value as flutterwave_secret_previous) + --flutterwave-webhook-hash New Flutterwave webhook hash (stores old value as flutterwave_webhook_hash_previous) + --finalize-payment-webhook-grace Clears *_previous fields after provider cutover is complete + + # Alerting webhook endpoints (optional): + --monitoring-webhook-url + --alert-webhook-url + --notification-email-webhook + --ops-email + + -h, --help Show help + +Examples: + sudo bash infra/deploy/rotate-integration-secrets.sh \ + --proxmox-token-secret 'NEW_SECRET' \ + --paystack-secret 'sk_live_new' \ + --flutterwave-webhook-hash 'new_hash' + + sudo bash infra/deploy/rotate-integration-secrets.sh \ + --finalize-payment-webhook-grace +EOF +} + +log() { + printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*" +} + +die() { + printf '\n[ERROR] %s\n' "$*" >&2 + exit 1 +} + +require_root() { + if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then + die "Run as root (or with sudo)." + fi +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +require_file() { + [[ -f "$1" ]] || die "Missing required file: $1" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --backend-url) + BACKEND_URL="${2:-}" + shift 2 + ;; + --admin-email) + ADMIN_EMAIL_INPUT="${2:-}" + shift 2 + ;; + --admin-password) + ADMIN_PASSWORD_INPUT="${2:-}" + shift 2 + ;; + --proxmox-host) + NEW_PROXMOX_HOST="${2:-}" + shift 2 + ;; + --proxmox-port) + NEW_PROXMOX_PORT="${2:-}" + shift 2 + ;; + --proxmox-username) + NEW_PROXMOX_USERNAME="${2:-}" + shift 2 + ;; + --proxmox-token-id) + NEW_PROXMOX_TOKEN_ID="${2:-}" + shift 2 + ;; + --proxmox-token-secret) + NEW_PROXMOX_TOKEN_SECRET="${2:-}" + shift 2 + ;; + --proxmox-verify-ssl) + NEW_PROXMOX_VERIFY_SSL="${2:-}" + shift 2 + ;; + --paystack-secret) + NEW_PAYSTACK_SECRET="${2:-}" + shift 2 + ;; + --flutterwave-secret) + NEW_FLUTTERWAVE_SECRET="${2:-}" + shift 2 + ;; + --flutterwave-webhook-hash) + NEW_FLUTTERWAVE_WEBHOOK_HASH="${2:-}" + shift 2 + ;; + --finalize-payment-webhook-grace) + FINALIZE_PAYMENT_GRACE="true" + shift + ;; + --monitoring-webhook-url) + NEW_MONITORING_WEBHOOK_URL="${2:-}" + shift 2 + ;; + --alert-webhook-url) + NEW_ALERT_WEBHOOK_URL="${2:-}" + shift 2 + ;; + --notification-email-webhook) + NEW_NOTIFICATION_EMAIL_WEBHOOK="${2:-}" + shift 2 + ;; + --ops-email) + NEW_OPS_EMAIL="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "Unknown argument: $1" + ;; + esac + done +} + +json_escape() { + printf '%s' "$1" | jq -Rs . +} + +json_canonical() { + printf '%s' "$1" | jq -cS . +} + +sanitize_notifications_payload() { + printf '%s' "$1" | jq ' + if (.monitoring_webhook_url // "") == "" then del(.monitoring_webhook_url) else . end + | if (.alert_webhook_url // "") == "" then del(.alert_webhook_url) else . end + | if (.email_gateway_url // "") == "" then del(.email_gateway_url) else . end + | if (.notification_email_webhook // "") == "" then del(.notification_email_webhook) else . end + | if (.ops_email // "") == "" then del(.ops_email) else . end + ' +} + +api_get() { + local path="$1" + curl -fsS \ + -H "Authorization: Bearer ${API_TOKEN}" \ + "${BACKEND_URL}${path}" +} + +api_put() { + local path="$1" + local payload="$2" + curl -fsS \ + -X PUT \ + -H "Authorization: Bearer ${API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + "${BACKEND_URL}${path}" >/dev/null +} + +preflight_proxmox_candidate() { + local payload_json="$1" + local host port username token_id token_secret verify_ssl auth_header + + host="$(printf '%s' "$payload_json" | jq -r '.host // empty')" + port="$(printf '%s' "$payload_json" | jq -r '.port // 8006')" + username="$(printf '%s' "$payload_json" | jq -r '.username // empty')" + token_id="$(printf '%s' "$payload_json" | jq -r '.token_id // empty')" + token_secret="$(printf '%s' "$payload_json" | jq -r '.token_secret // empty')" + verify_ssl="$(printf '%s' "$payload_json" | jq -r '.verify_ssl // true')" + + [[ -n "$host" && -n "$username" && -n "$token_id" && -n "$token_secret" ]] || { + die "Candidate Proxmox settings are incomplete." + } + + auth_header="PVEAPIToken=${username}!${token_id}=${token_secret}" + log "Preflight: validating candidate Proxmox token directly against ${host}:${port}" + if [[ "$verify_ssl" == "false" ]]; then + curl -ksfS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null + else + curl -fsS -H "Authorization: ${auth_header}" "https://${host}:${port}/api2/json/version" >/dev/null + fi +} + +rollback_if_needed() { + if [[ -z "$API_TOKEN" ]]; then + return + fi + + if [[ "$UPDATED_NOTIFICATIONS" == "true" && -n "$ORIGINAL_NOTIFICATIONS_JSON" ]]; then + log "Rollback: restoring notifications settings" + api_put "/settings/notifications" "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED" || true + fi + + if [[ "$UPDATED_PAYMENT" == "true" && -n "$ORIGINAL_PAYMENT_JSON" ]]; then + log "Rollback: restoring payment settings" + api_put "/settings/payment" "$ORIGINAL_PAYMENT_JSON" || true + fi + + if [[ "$UPDATED_PROXMOX" == "true" && -n "$ORIGINAL_PROXMOX_JSON" ]]; then + log "Rollback: restoring proxmox settings" + api_put "/settings/proxmox" "$ORIGINAL_PROXMOX_JSON" || true + fi +} + +on_error() { + local line="$1" + printf '\n[ERROR] Rotation failed at line %s. Attempting rollback where possible.\n' "$line" >&2 + rollback_if_needed +} + +build_backend_url() { + if [[ -n "$BACKEND_URL" ]]; then + return + fi + local backend_port + backend_port="${BACKEND_PORT:-8080}" + BACKEND_URL="http://127.0.0.1:${backend_port}/api" +} + +main() { + parse_args "$@" + require_root + require_command curl + require_command jq + require_file "$ENV_FILE" + + # shellcheck disable=SC1090 + source "$ENV_FILE" + build_backend_url + + local admin_email admin_password + admin_email="${ADMIN_EMAIL_INPUT:-${ADMIN_EMAIL:-}}" + admin_password="${ADMIN_PASSWORD_INPUT:-${ADMIN_PASSWORD:-}}" + + [[ -n "$admin_email" && -n "$admin_password" ]] || { + die "Admin credentials are required (provide --admin-email/--admin-password or set ADMIN_EMAIL/ADMIN_PASSWORD in env)." + } + + local wants_rotation="false" + if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" || -n "$NEW_PROXMOX_TOKEN_ID" || -n "$NEW_PROXMOX_USERNAME" || -n "$NEW_PROXMOX_HOST" || -n "$NEW_PROXMOX_PORT" || -n "$NEW_PROXMOX_VERIFY_SSL" || -n "$NEW_PAYSTACK_SECRET" || -n "$NEW_FLUTTERWAVE_SECRET" || -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" || -n "$NEW_MONITORING_WEBHOOK_URL" || -n "$NEW_ALERT_WEBHOOK_URL" || -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" || -n "$NEW_OPS_EMAIL" || "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then + wants_rotation="true" + fi + [[ "$wants_rotation" == "true" ]] || die "No rotation inputs supplied. Run with --help for options." + + trap 'on_error $LINENO' ERR + + TS="$(date -u +%Y%m%d-%H%M%S)" + BACKUP_DIR="${BACKUP_ROOT}/${TS}-integration-secret-rotation" + mkdir -p "$BACKUP_DIR" + chmod 700 "$BACKUP_DIR" + + log "Authenticating against backend API" + local login_response + login_response="$( + curl -fsS -X POST "${BACKEND_URL}/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":$(json_escape "$admin_email"),\"password\":$(json_escape "$admin_password")}" + )" + API_TOKEN="$(printf '%s' "$login_response" | jq -r '.token // empty')" + [[ -n "$API_TOKEN" ]] || die "Failed to obtain auth token from backend login." + + log "Fetching current integration settings" + ORIGINAL_PROXMOX_JSON="$(api_get "/settings/proxmox")" + ORIGINAL_PAYMENT_JSON="$(api_get "/settings/payment")" + ORIGINAL_NOTIFICATIONS_JSON="$(api_get "/settings/notifications")" + ORIGINAL_NOTIFICATIONS_JSON_SANITIZED="$(sanitize_notifications_payload "$ORIGINAL_NOTIFICATIONS_JSON")" + + printf '%s\n' "$ORIGINAL_PROXMOX_JSON" >"${BACKUP_DIR}/proxmox.before.json" + printf '%s\n' "$ORIGINAL_PAYMENT_JSON" >"${BACKUP_DIR}/payment.before.json" + printf '%s\n' "$ORIGINAL_NOTIFICATIONS_JSON" >"${BACKUP_DIR}/notifications.before.json" + cp "$ENV_FILE" "${BACKUP_DIR}/.env.production.bak" + + local proxmox_payload payment_payload notifications_payload + proxmox_payload="$ORIGINAL_PROXMOX_JSON" + payment_payload="$ORIGINAL_PAYMENT_JSON" + notifications_payload="$ORIGINAL_NOTIFICATIONS_JSON" + + if [[ -n "$NEW_PROXMOX_HOST" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_HOST" '.host = $v')" + fi + if [[ -n "$NEW_PROXMOX_PORT" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --argjson v "$NEW_PROXMOX_PORT" '.port = $v')" + fi + if [[ -n "$NEW_PROXMOX_USERNAME" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_USERNAME" '.username = $v')" + fi + if [[ -n "$NEW_PROXMOX_TOKEN_ID" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_ID" '.token_id = $v')" + fi + if [[ -n "$NEW_PROXMOX_TOKEN_SECRET" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "$NEW_PROXMOX_TOKEN_SECRET" '.token_secret = $v')" + fi + if [[ -n "$NEW_PROXMOX_VERIFY_SSL" ]]; then + proxmox_payload="$(printf '%s' "$proxmox_payload" | jq --arg v "${NEW_PROXMOX_VERIFY_SSL,,}" '.verify_ssl = ($v == "true")')" + fi + + if [[ "$(json_canonical "$proxmox_payload")" != "$(json_canonical "$ORIGINAL_PROXMOX_JSON")" ]]; then + preflight_proxmox_candidate "$proxmox_payload" + log "Applying Proxmox settings rotation" + api_put "/settings/proxmox" "$proxmox_payload" + UPDATED_PROXMOX="true" + + log "Post-rotation validation: Proxmox sync" + curl -fsS -X POST "${BACKEND_URL}/proxmox/sync" -H "Authorization: Bearer ${API_TOKEN}" >/dev/null + fi + + if [[ -n "$NEW_PAYSTACK_SECRET" ]]; then + payment_payload="$( + printf '%s' "$payment_payload" | jq --arg v "$NEW_PAYSTACK_SECRET" ' + if (.paystack_secret // "") == $v then . else .paystack_secret_previous = (.paystack_secret // "") | .paystack_secret = $v end + ' + )" + fi + + if [[ -n "$NEW_FLUTTERWAVE_SECRET" ]]; then + payment_payload="$( + printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_SECRET" ' + if (.flutterwave_secret // "") == $v then . else .flutterwave_secret_previous = (.flutterwave_secret // "") | .flutterwave_secret = $v end + ' + )" + fi + + if [[ -n "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ]]; then + payment_payload="$( + printf '%s' "$payment_payload" | jq --arg v "$NEW_FLUTTERWAVE_WEBHOOK_HASH" ' + if (.flutterwave_webhook_hash // "") == $v then . + else .flutterwave_webhook_hash_previous = (.flutterwave_webhook_hash // "") | .flutterwave_webhook_hash = $v end + ' + )" + fi + + if [[ "$FINALIZE_PAYMENT_GRACE" == "true" ]]; then + payment_payload="$( + printf '%s' "$payment_payload" | jq ' + .paystack_secret_previous = "" | + .flutterwave_secret_previous = "" | + .flutterwave_webhook_hash_previous = "" + ' + )" + fi + + if [[ "$(json_canonical "$payment_payload")" != "$(json_canonical "$ORIGINAL_PAYMENT_JSON")" ]]; then + log "Applying payment/webhook rotation payload" + api_put "/settings/payment" "$payment_payload" + UPDATED_PAYMENT="true" + fi + + if [[ -n "$NEW_MONITORING_WEBHOOK_URL" ]]; then + notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_MONITORING_WEBHOOK_URL" '.monitoring_webhook_url = $v')" + fi + if [[ -n "$NEW_ALERT_WEBHOOK_URL" ]]; then + notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_ALERT_WEBHOOK_URL" '.alert_webhook_url = $v')" + fi + if [[ -n "$NEW_NOTIFICATION_EMAIL_WEBHOOK" ]]; then + notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_NOTIFICATION_EMAIL_WEBHOOK" '.notification_email_webhook = $v')" + fi + if [[ -n "$NEW_OPS_EMAIL" ]]; then + notifications_payload="$(printf '%s' "$notifications_payload" | jq --arg v "$NEW_OPS_EMAIL" '.ops_email = $v')" + fi + + notifications_payload="$(sanitize_notifications_payload "$notifications_payload")" + + if [[ "$(json_canonical "$notifications_payload")" != "$(json_canonical "$ORIGINAL_NOTIFICATIONS_JSON_SANITIZED")" ]]; then + log "Applying alerting destination updates" + api_put "/settings/notifications" "$notifications_payload" + UPDATED_NOTIFICATIONS="true" + fi + + local summary_file + summary_file="/root/proxpanel-integration-rotation-${TS}.txt" + cat >"$summary_file" <