ops: add integration secret rotation and offsite backup alerting

2026-04-18 09:33:17 +01:00
parent 95633a6722
commit 81be9c5e42
13 changed files with 1105 additions and 16 deletions
--- a/infra/deploy/notify-backup-alert.sh
+++ b/infra/deploy/notify-backup-alert.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+
+EVENT="${EVENT:-}"
+SEVERITY="${SEVERITY:-warning}"
+STATUS="${STATUS:-failed}"
+MESSAGE="${MESSAGE:-}"
+SOURCE="${SOURCE:-backup-jobs}"
+CONTEXT_JSON="${CONTEXT_JSON:-{}}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  APP_DIR=/opt/proxpanel /opt/proxpanel/infra/deploy/notify-backup-alert.sh \
+    --event <backup_failed|restore_test_failed|...> \
+    --severity <info|warning|critical> \
+    --status <ok|failed> \
+    --message "<human-readable-message>" \
+    [--source backup-cron] \
+    [--context-json '{"key":"value"}']
+
+Alert destinations are read from /opt/proxpanel/.backup.env:
+  BACKUP_ALERT_WEBHOOK_URL=
+  BACKUP_ALERT_EMAIL_WEBHOOK_URL=
+  BACKUP_ALERT_EMAIL_TO=
+  BACKUP_ALERT_SUBJECT_PREFIX=[ProxPanel Backup]
+EOF
+}
+
+log() {
+  printf '[%s] %s\n' "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" "$*"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    printf '[WARN] Missing command: %s\n' "$1" >&2
+    return 1
+  }
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --event)
+        EVENT="${2:-}"
+        shift 2
+        ;;
+      --severity)
+        SEVERITY="${2:-}"
+        shift 2
+        ;;
+      --status)
+        STATUS="${2:-}"
+        shift 2
+        ;;
+      --message)
+        MESSAGE="${2:-}"
+        shift 2
+        ;;
+      --source)
+        SOURCE="${2:-}"
+        shift 2
+        ;;
+      --context-json)
+        CONTEXT_JSON="${2:-{}}"
+        shift 2
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        printf '[WARN] Ignoring unknown argument: %s\n' "$1" >&2
+        shift
+        ;;
+    esac
+  done
+}
+
+main() {
+  parse_args "$@"
+  require_command curl || exit 0
+  require_command jq || exit 0
+
+  if [[ -f "$SECRET_FILE" ]]; then
+    # shellcheck disable=SC1090
+    source "$SECRET_FILE"
+  fi
+
+  [[ -n "$EVENT" ]] || EVENT="backup_job_event"
+  [[ -n "$MESSAGE" ]] || MESSAGE="Backup alert event raised"
+
+  local context
+  context="$(printf '%s' "$CONTEXT_JSON" | jq -c '.' 2>/dev/null || printf '{}')"
+
+  local payload now
+  now="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+  payload="$(
+    jq -n \
+      --arg type "backup.alert" \
+      --arg event "$EVENT" \
+      --arg severity "$SEVERITY" \
+      --arg status "$STATUS" \
+      --arg message "$MESSAGE" \
+      --arg source "$SOURCE" \
+      --arg timestamp "$now" \
+      --argjson context "$context" \
+      '{
+        type: $type,
+        event: $event,
+        severity: $severity,
+        status: $status,
+        source: $source,
+        message: $message,
+        timestamp: $timestamp,
+        context: $context
+      }'
+  )"
+
+  local webhook_url email_webhook email_to subject_prefix subject status_webhook status_email
+  webhook_url="${BACKUP_ALERT_WEBHOOK_URL:-}"
+  email_webhook="${BACKUP_ALERT_EMAIL_WEBHOOK_URL:-}"
+  email_to="${BACKUP_ALERT_EMAIL_TO:-${OPS_EMAIL:-}}"
+  subject_prefix="${BACKUP_ALERT_SUBJECT_PREFIX:-[ProxPanel Backup]}"
+  subject="${subject_prefix} ${EVENT} (${SEVERITY})"
+  status_webhook="skipped"
+  status_email="skipped"
+
+  if [[ -n "$webhook_url" ]]; then
+    if curl -fsS -X POST "$webhook_url" -H "Content-Type: application/json" -d "$payload" >/dev/null; then
+      status_webhook="sent"
+    else
+      status_webhook="failed"
+    fi
+  fi
+
+  if [[ -n "$email_webhook" && -n "$email_to" ]]; then
+    local email_payload
+    email_payload="$(
+      jq -n \
+        --arg type "backup.alert.email" \
+        --arg to "$email_to" \
+        --arg subject "$subject" \
+        --arg message "$MESSAGE" \
+        --argjson payload "$payload" \
+        '{
+          type: $type,
+          to: $to,
+          subject: $subject,
+          message: $message,
+          payload: $payload
+        }'
+    )"
+
+    if curl -fsS -X POST "$email_webhook" -H "Content-Type: application/json" -d "$email_payload" >/dev/null; then
+      status_email="sent"
+    else
+      status_email="failed"
+    fi
+  fi
+
+  log "Alert dispatch result: webhook=${status_webhook}, email=${status_email}"
+}
+
+main "$@"