ops: add integration secret rotation and offsite backup alerting

infra/deploy/db-restore-test-job.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+APP_DIR="${APP_DIR:-/opt/proxpanel}"
+SECRET_FILE="${SECRET_FILE:-$APP_DIR/.backup.env}"
+
+RESTORE_SCRIPT="${APP_DIR}/infra/deploy/db-restore-test.sh"
+NOTIFY_SCRIPT="${APP_DIR}/infra/deploy/notify-backup-alert.sh"
+
+job_failed() {
+  local line="$1"
+  local message="Daily restore test failed (line ${line}) on host $(hostname -f 2>/dev/null || hostname)"
+  APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+    --event restore_test_failed \
+    --severity critical \
+    --status failed \
+    --source db-restore-test-job \
+    --message "$message" \
+    --context-json "{\"line\":${line}}"
+}
+
+main() {
+  trap 'job_failed $LINENO' ERR
+
+  APP_DIR="$APP_DIR" "$RESTORE_SCRIPT"
+
+  local send_success="false"
+  if [[ -f "$SECRET_FILE" ]]; then
+    # shellcheck disable=SC1090
+    source "$SECRET_FILE"
+    send_success="${BACKUP_ALERT_SEND_SUCCESS:-false}"
+  fi
+
+  if [[ "$send_success" == "true" ]]; then
+    APP_DIR="$APP_DIR" "$NOTIFY_SCRIPT" \
+      --event restore_test_success \
+      --severity info \
+      --status ok \
+      --source db-restore-test-job \
+      --message "Daily restore test completed successfully"
+  fi
+
+  trap - ERR
+}
+
+main "$@"