1087 lines
28 KiB
TypeScript
1087 lines
28 KiB
TypeScript
import {
|
|
BackupRestoreMode,
|
|
BackupRestoreStatus,
|
|
BackupSchedule,
|
|
BackupSource,
|
|
BackupStatus,
|
|
BackupType,
|
|
Prisma,
|
|
SnapshotFrequency
|
|
} from "@prisma/client";
|
|
import { prisma } from "../lib/prisma";
|
|
import { HttpError } from "../lib/http-error";
|
|
|
|
type PolicyShape = {
|
|
id?: string;
|
|
max_files: number;
|
|
max_total_size_mb: number;
|
|
max_protected_files: number;
|
|
allow_file_restore: boolean;
|
|
allow_cross_vm_restore: boolean;
|
|
allow_pbs_restore: boolean;
|
|
};
|
|
|
|
const DEFAULT_POLICY: PolicyShape = {
|
|
max_files: 20,
|
|
max_total_size_mb: 102400,
|
|
max_protected_files: 5,
|
|
allow_file_restore: true,
|
|
allow_cross_vm_restore: true,
|
|
allow_pbs_restore: true
|
|
};
|
|
|
|
function toPositiveInt(value: number, fallback: number) {
|
|
if (!Number.isFinite(value)) return fallback;
|
|
const rounded = Math.floor(value);
|
|
return rounded > 0 ? rounded : fallback;
|
|
}
|
|
|
|
function toScheduleForSnapshot(frequency: SnapshotFrequency): BackupSchedule {
|
|
if (frequency === SnapshotFrequency.HOURLY) return BackupSchedule.DAILY;
|
|
if (frequency === SnapshotFrequency.DAILY) return BackupSchedule.DAILY;
|
|
return BackupSchedule.WEEKLY;
|
|
}
|
|
|
|
function nextBackupRunDate(schedule: BackupSchedule, fromDate = new Date()) {
|
|
const now = new Date(fromDate);
|
|
if (schedule === BackupSchedule.DAILY) return new Date(now.getTime() + 24 * 60 * 60 * 1000);
|
|
if (schedule === BackupSchedule.WEEKLY) return new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000);
|
|
if (schedule === BackupSchedule.MONTHLY) return new Date(now.getTime() + 30 * 24 * 60 * 60 * 1000);
|
|
return null;
|
|
}
|
|
|
|
function nextSnapshotRunAt(job: {
|
|
frequency: SnapshotFrequency;
|
|
interval: number;
|
|
day_of_week: number | null;
|
|
hour_utc: number;
|
|
minute_utc: number;
|
|
}, fromDate = new Date()) {
|
|
const now = new Date(fromDate);
|
|
const interval = Math.max(1, job.interval);
|
|
|
|
if (job.frequency === SnapshotFrequency.HOURLY) {
|
|
const candidate = new Date(now);
|
|
candidate.setUTCSeconds(0, 0);
|
|
candidate.setUTCMinutes(job.minute_utc);
|
|
if (candidate <= now) {
|
|
candidate.setUTCHours(candidate.getUTCHours() + interval);
|
|
}
|
|
return candidate;
|
|
}
|
|
|
|
if (job.frequency === SnapshotFrequency.DAILY) {
|
|
const candidate = new Date(now);
|
|
candidate.setUTCSeconds(0, 0);
|
|
candidate.setUTCHours(job.hour_utc, job.minute_utc, 0, 0);
|
|
if (candidate <= now) {
|
|
candidate.setUTCDate(candidate.getUTCDate() + interval);
|
|
}
|
|
return candidate;
|
|
}
|
|
|
|
const dayOfWeek = job.day_of_week ?? 0;
|
|
const candidate = new Date(now);
|
|
candidate.setUTCSeconds(0, 0);
|
|
candidate.setUTCHours(job.hour_utc, job.minute_utc, 0, 0);
|
|
|
|
const currentDow = candidate.getUTCDay();
|
|
const deltaDays = (dayOfWeek - currentDow + 7) % 7;
|
|
candidate.setUTCDate(candidate.getUTCDate() + deltaDays);
|
|
|
|
if (candidate <= now) {
|
|
candidate.setUTCDate(candidate.getUTCDate() + interval * 7);
|
|
}
|
|
|
|
return candidate;
|
|
}
|
|
|
|
function estimateBackupSizeMb(input: { requestedSizeMb?: number; diskGb: number; type: BackupType }) {
|
|
if (typeof input.requestedSizeMb === "number" && Number.isFinite(input.requestedSizeMb) && input.requestedSizeMb > 0) {
|
|
return input.requestedSizeMb;
|
|
}
|
|
|
|
const base = Math.max(256, input.diskGb * 1024 * 0.35);
|
|
if (input.type === BackupType.SNAPSHOT) {
|
|
return Math.max(128, base * 0.4);
|
|
}
|
|
if (input.type === BackupType.INCREMENTAL) {
|
|
return Math.max(128, base * 0.6);
|
|
}
|
|
return base;
|
|
}
|
|
|
|
async function effectiveBackupPolicy(tenantId: string, billingPlanId?: string | null): Promise<PolicyShape> {
|
|
const policies = await prisma.backupPolicy.findMany({
|
|
where: {
|
|
OR: [
|
|
{ tenant_id: tenantId, billing_plan_id: billingPlanId ?? null },
|
|
{ tenant_id: tenantId, billing_plan_id: null },
|
|
{ tenant_id: null, billing_plan_id: billingPlanId ?? null },
|
|
{ tenant_id: null, billing_plan_id: null }
|
|
]
|
|
}
|
|
});
|
|
|
|
const score = (item: (typeof policies)[number]) => {
|
|
let value = 0;
|
|
if (item.tenant_id === tenantId) value += 8;
|
|
if (item.billing_plan_id && billingPlanId && item.billing_plan_id === billingPlanId) value += 4;
|
|
if (!item.tenant_id) value += 1;
|
|
if (!item.billing_plan_id) value += 1;
|
|
return value;
|
|
};
|
|
|
|
const selected = policies.sort((a, b) => score(b) - score(a))[0];
|
|
if (!selected) return DEFAULT_POLICY;
|
|
|
|
return {
|
|
id: selected.id,
|
|
max_files: selected.max_files,
|
|
max_total_size_mb: selected.max_total_size_mb,
|
|
max_protected_files: selected.max_protected_files,
|
|
allow_file_restore: selected.allow_file_restore,
|
|
allow_cross_vm_restore: selected.allow_cross_vm_restore,
|
|
allow_pbs_restore: selected.allow_pbs_restore
|
|
};
|
|
}
|
|
|
|
async function tenantBackupUsage(tenantId: string) {
|
|
const backups = await prisma.backup.findMany({
|
|
where: {
|
|
OR: [{ tenant_id: tenantId }, { vm: { tenant_id: tenantId } }],
|
|
status: { in: [BackupStatus.PENDING, BackupStatus.RUNNING, BackupStatus.COMPLETED] }
|
|
},
|
|
select: {
|
|
id: true,
|
|
size_mb: true,
|
|
is_protected: true
|
|
}
|
|
});
|
|
|
|
const totalSize = backups.reduce((sum, item) => sum + (item.size_mb ?? 0), 0);
|
|
const protectedCount = backups.filter((item) => item.is_protected).length;
|
|
|
|
return {
|
|
count: backups.length,
|
|
totalSizeMb: totalSize,
|
|
protectedCount
|
|
};
|
|
}
|
|
|
|
async function enforceBackupPolicyLimits(input: {
|
|
tenantId: string;
|
|
billingPlanId?: string | null;
|
|
predictedSizeMb: number;
|
|
protected: boolean;
|
|
}) {
|
|
const [policy, usage] = await Promise.all([
|
|
effectiveBackupPolicy(input.tenantId, input.billingPlanId),
|
|
tenantBackupUsage(input.tenantId)
|
|
]);
|
|
|
|
if (usage.count >= policy.max_files) {
|
|
throw new HttpError(
|
|
400,
|
|
`Backup limit reached (${policy.max_files} files). Remove old backups or raise policy limits.`,
|
|
"BACKUP_LIMIT_FILES"
|
|
);
|
|
}
|
|
|
|
if (usage.totalSizeMb + input.predictedSizeMb > policy.max_total_size_mb) {
|
|
throw new HttpError(
|
|
400,
|
|
`Backup storage limit exceeded (${policy.max_total_size_mb} MB).`,
|
|
"BACKUP_LIMIT_SIZE"
|
|
);
|
|
}
|
|
|
|
if (input.protected && usage.protectedCount >= policy.max_protected_files) {
|
|
throw new HttpError(
|
|
400,
|
|
`Protected backup limit reached (${policy.max_protected_files}).`,
|
|
"BACKUP_LIMIT_PROTECTED"
|
|
);
|
|
}
|
|
|
|
return policy;
|
|
}
|
|
|
|
export async function listBackups(input: {
|
|
tenantId?: string;
|
|
vmId?: string;
|
|
status?: BackupStatus;
|
|
limit?: number;
|
|
offset?: number;
|
|
}) {
|
|
const where: Prisma.BackupWhereInput = {};
|
|
if (input.vmId) where.vm_id = input.vmId;
|
|
if (input.status) where.status = input.status;
|
|
if (input.tenantId) {
|
|
where.OR = [{ tenant_id: input.tenantId }, { vm: { tenant_id: input.tenantId } }];
|
|
}
|
|
|
|
const limit = Math.min(Math.max(input.limit ?? 50, 1), 200);
|
|
const offset = Math.max(input.offset ?? 0, 0);
|
|
|
|
const [data, total] = await Promise.all([
|
|
prisma.backup.findMany({
|
|
where,
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
node: true,
|
|
tenant_id: true
|
|
}
|
|
},
|
|
snapshot_job: {
|
|
select: {
|
|
id: true,
|
|
name: true
|
|
}
|
|
}
|
|
},
|
|
orderBy: [{ is_protected: "desc" }, { created_at: "desc" }],
|
|
take: limit,
|
|
skip: offset
|
|
}),
|
|
prisma.backup.count({ where })
|
|
]);
|
|
|
|
return {
|
|
data,
|
|
meta: {
|
|
total,
|
|
limit,
|
|
offset
|
|
}
|
|
};
|
|
}
|
|
|
|
export async function createBackup(input: {
|
|
vmId: string;
|
|
type?: BackupType;
|
|
source?: BackupSource;
|
|
schedule?: BackupSchedule;
|
|
retentionDays?: number;
|
|
storage?: string;
|
|
routeKey?: string;
|
|
isProtected?: boolean;
|
|
notes?: string;
|
|
requestedSizeMb?: number;
|
|
createdBy?: string;
|
|
}) {
|
|
const vm = await prisma.virtualMachine.findUnique({
|
|
where: { id: input.vmId },
|
|
include: {
|
|
billing_plan: {
|
|
select: {
|
|
id: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
if (!vm) {
|
|
throw new HttpError(404, "VM not found", "VM_NOT_FOUND");
|
|
}
|
|
|
|
const type = input.type ?? BackupType.FULL;
|
|
const source = input.source ?? BackupSource.LOCAL;
|
|
const schedule = input.schedule ?? BackupSchedule.MANUAL;
|
|
const retentionDays = toPositiveInt(input.retentionDays ?? 7, 7);
|
|
const sizeMb = estimateBackupSizeMb({
|
|
requestedSizeMb: input.requestedSizeMb,
|
|
diskGb: vm.disk_gb,
|
|
type
|
|
});
|
|
const keepProtected = Boolean(input.isProtected);
|
|
|
|
await enforceBackupPolicyLimits({
|
|
tenantId: vm.tenant_id,
|
|
billingPlanId: vm.billing_plan_id,
|
|
predictedSizeMb: sizeMb,
|
|
protected: keepProtected
|
|
});
|
|
|
|
const now = new Date();
|
|
const expiresAt = new Date(now.getTime() + retentionDays * 24 * 60 * 60 * 1000);
|
|
const nextRunAt = schedule === BackupSchedule.MANUAL ? null : nextBackupRunDate(schedule, now);
|
|
|
|
const backup = await prisma.backup.create({
|
|
data: {
|
|
vm_id: vm.id,
|
|
vm_name: vm.name,
|
|
tenant_id: vm.tenant_id,
|
|
node: vm.node,
|
|
status: schedule === BackupSchedule.MANUAL ? BackupStatus.COMPLETED : BackupStatus.PENDING,
|
|
type,
|
|
source,
|
|
size_mb: sizeMb,
|
|
storage: input.storage ?? (source === BackupSource.PBS ? "pbs-datastore" : "local-lvm"),
|
|
backup_path: source === BackupSource.PBS ? null : `backup/${vm.node}/${vm.vmid}/${Date.now()}`,
|
|
pbs_snapshot_id: source === BackupSource.PBS ? `pbs/${vm.node}/${vm.vmid}/${Date.now()}` : null,
|
|
route_key: input.routeKey,
|
|
is_protected: keepProtected,
|
|
restore_enabled: true,
|
|
total_files: Math.max(1, Math.floor(sizeMb / 64)),
|
|
schedule,
|
|
retention_days: retentionDays,
|
|
started_at: schedule === BackupSchedule.MANUAL ? now : null,
|
|
completed_at: schedule === BackupSchedule.MANUAL ? now : null,
|
|
next_run_at: nextRunAt,
|
|
expires_at: expiresAt,
|
|
notes: input.notes
|
|
}
|
|
});
|
|
|
|
return backup;
|
|
}
|
|
|
|
export async function toggleBackupProtection(backupId: string, protect: boolean) {
|
|
const backup = await prisma.backup.findUnique({
|
|
where: { id: backupId },
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
tenant_id: true,
|
|
billing_plan_id: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
if (!backup) {
|
|
throw new HttpError(404, "Backup not found", "BACKUP_NOT_FOUND");
|
|
}
|
|
|
|
if (protect) {
|
|
await enforceBackupPolicyLimits({
|
|
tenantId: backup.vm.tenant_id,
|
|
billingPlanId: backup.vm.billing_plan_id,
|
|
predictedSizeMb: 0,
|
|
protected: true
|
|
});
|
|
}
|
|
|
|
return prisma.backup.update({
|
|
where: { id: backup.id },
|
|
data: {
|
|
is_protected: protect
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function deleteBackup(backupId: string, force = false) {
|
|
const backup = await prisma.backup.findUnique({ where: { id: backupId } });
|
|
if (!backup) {
|
|
throw new HttpError(404, "Backup not found", "BACKUP_NOT_FOUND");
|
|
}
|
|
|
|
if (backup.is_protected && !force) {
|
|
throw new HttpError(400, "Protected backup cannot be deleted without force", "BACKUP_PROTECTED");
|
|
}
|
|
|
|
return prisma.backup.delete({
|
|
where: { id: backup.id }
|
|
});
|
|
}
|
|
|
|
async function executeRestoreTask(taskId: string) {
|
|
const task = await prisma.backupRestoreTask.findUnique({
|
|
where: { id: taskId },
|
|
include: {
|
|
backup: true,
|
|
source_vm: true,
|
|
target_vm: true
|
|
}
|
|
});
|
|
|
|
if (!task) {
|
|
throw new HttpError(404, "Restore task not found", "RESTORE_TASK_NOT_FOUND");
|
|
}
|
|
|
|
await prisma.backupRestoreTask.update({
|
|
where: { id: task.id },
|
|
data: {
|
|
status: BackupRestoreStatus.RUNNING,
|
|
started_at: new Date()
|
|
}
|
|
});
|
|
|
|
let resultMessage = "Restore completed";
|
|
if (task.mode === BackupRestoreMode.FILES || task.mode === BackupRestoreMode.SINGLE_FILE) {
|
|
const files = Array.isArray(task.requested_files) ? task.requested_files.length : 0;
|
|
resultMessage = `File restore completed (${files} item${files === 1 ? "" : "s"})`;
|
|
}
|
|
|
|
if (task.pbs_enabled) {
|
|
resultMessage = `${resultMessage} via PBS`;
|
|
}
|
|
|
|
return prisma.backupRestoreTask.update({
|
|
where: { id: task.id },
|
|
data: {
|
|
status: BackupRestoreStatus.COMPLETED,
|
|
completed_at: new Date(),
|
|
result_message: resultMessage
|
|
},
|
|
include: {
|
|
backup: true,
|
|
source_vm: true,
|
|
target_vm: true
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function createRestoreTask(input: {
|
|
backupId: string;
|
|
targetVmId?: string;
|
|
mode: BackupRestoreMode;
|
|
requestedFiles?: string[];
|
|
pbsEnabled?: boolean;
|
|
createdBy?: string;
|
|
runImmediately?: boolean;
|
|
}) {
|
|
const backup = await prisma.backup.findUnique({
|
|
where: { id: input.backupId },
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
tenant_id: true,
|
|
billing_plan_id: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
if (!backup) {
|
|
throw new HttpError(404, "Backup not found", "BACKUP_NOT_FOUND");
|
|
}
|
|
|
|
if (!backup.restore_enabled) {
|
|
throw new HttpError(400, "Restore is disabled for this backup", "RESTORE_DISABLED");
|
|
}
|
|
|
|
const targetVmId = input.targetVmId ?? backup.vm_id;
|
|
const targetVm = await prisma.virtualMachine.findUnique({
|
|
where: { id: targetVmId },
|
|
select: {
|
|
id: true,
|
|
tenant_id: true
|
|
}
|
|
});
|
|
|
|
if (!targetVm) {
|
|
throw new HttpError(404, "Target VM not found", "TARGET_VM_NOT_FOUND");
|
|
}
|
|
|
|
if (targetVm.tenant_id !== backup.vm.tenant_id) {
|
|
throw new HttpError(403, "Cross-tenant restore is not allowed", "CROSS_TENANT_RESTORE_DENIED");
|
|
}
|
|
|
|
const policy = await effectiveBackupPolicy(backup.vm.tenant_id, backup.vm.billing_plan_id);
|
|
const isCrossVm = targetVmId !== backup.vm_id;
|
|
if (isCrossVm && !policy.allow_cross_vm_restore) {
|
|
throw new HttpError(400, "Cross-VM restore is disabled by policy", "CROSS_VM_RESTORE_DISABLED");
|
|
}
|
|
|
|
if (input.mode !== BackupRestoreMode.FULL_VM && !policy.allow_file_restore) {
|
|
throw new HttpError(400, "File-level restore is disabled by policy", "FILE_RESTORE_DISABLED");
|
|
}
|
|
|
|
const pbsEnabled = Boolean(input.pbsEnabled);
|
|
if (pbsEnabled && !policy.allow_pbs_restore) {
|
|
throw new HttpError(400, "PBS restore is disabled by policy", "PBS_RESTORE_DISABLED");
|
|
}
|
|
if (pbsEnabled && backup.source !== BackupSource.PBS) {
|
|
throw new HttpError(400, "PBS restore requires a PBS backup source", "PBS_SOURCE_REQUIRED");
|
|
}
|
|
|
|
const requestedFiles = input.requestedFiles ?? [];
|
|
if (input.mode === BackupRestoreMode.SINGLE_FILE && requestedFiles.length !== 1) {
|
|
throw new HttpError(400, "SINGLE_FILE restore requires exactly one file path", "INVALID_RESTORE_FILES");
|
|
}
|
|
|
|
if ((input.mode === BackupRestoreMode.FILES || input.mode === BackupRestoreMode.SINGLE_FILE) && requestedFiles.length === 0) {
|
|
throw new HttpError(400, "Requested files are required for file-level restore", "INVALID_RESTORE_FILES");
|
|
}
|
|
|
|
const task = await prisma.backupRestoreTask.create({
|
|
data: {
|
|
backup_id: backup.id,
|
|
source_vm_id: backup.vm_id,
|
|
target_vm_id: targetVmId,
|
|
mode: input.mode,
|
|
requested_files: requestedFiles,
|
|
pbs_enabled: pbsEnabled,
|
|
created_by: input.createdBy
|
|
}
|
|
});
|
|
|
|
if (input.runImmediately === false) {
|
|
return prisma.backupRestoreTask.findUnique({
|
|
where: { id: task.id },
|
|
include: {
|
|
backup: true,
|
|
source_vm: true,
|
|
target_vm: true
|
|
}
|
|
});
|
|
}
|
|
|
|
return executeRestoreTask(task.id);
|
|
}
|
|
|
|
export async function runRestoreTaskNow(taskId: string) {
|
|
const task = await prisma.backupRestoreTask.findUnique({ where: { id: taskId } });
|
|
if (!task) {
|
|
throw new HttpError(404, "Restore task not found", "RESTORE_TASK_NOT_FOUND");
|
|
}
|
|
if (task.status === BackupRestoreStatus.COMPLETED) {
|
|
return prisma.backupRestoreTask.findUnique({
|
|
where: { id: task.id },
|
|
include: { backup: true, source_vm: true, target_vm: true }
|
|
});
|
|
}
|
|
return executeRestoreTask(task.id);
|
|
}
|
|
|
|
export async function listRestoreTasks(input: {
|
|
tenantId?: string;
|
|
status?: BackupRestoreStatus;
|
|
limit?: number;
|
|
offset?: number;
|
|
}) {
|
|
const where: Prisma.BackupRestoreTaskWhereInput = {};
|
|
if (input.status) where.status = input.status;
|
|
if (input.tenantId) {
|
|
where.source_vm = { tenant_id: input.tenantId };
|
|
}
|
|
|
|
const limit = Math.min(Math.max(input.limit ?? 50, 1), 200);
|
|
const offset = Math.max(input.offset ?? 0, 0);
|
|
|
|
const [data, total] = await Promise.all([
|
|
prisma.backupRestoreTask.findMany({
|
|
where,
|
|
include: {
|
|
backup: true,
|
|
source_vm: { select: { id: true, name: true, tenant_id: true } },
|
|
target_vm: { select: { id: true, name: true, tenant_id: true } }
|
|
},
|
|
orderBy: { created_at: "desc" },
|
|
take: limit,
|
|
skip: offset
|
|
}),
|
|
prisma.backupRestoreTask.count({ where })
|
|
]);
|
|
|
|
return {
|
|
data,
|
|
meta: {
|
|
total,
|
|
limit,
|
|
offset
|
|
}
|
|
};
|
|
}
|
|
|
|
async function enforceSnapshotRetention(jobId: string, retention: number) {
|
|
const backups = await prisma.backup.findMany({
|
|
where: {
|
|
snapshot_job_id: jobId,
|
|
type: BackupType.SNAPSHOT
|
|
},
|
|
orderBy: {
|
|
created_at: "desc"
|
|
}
|
|
});
|
|
|
|
const removable = backups.filter((item, index) => index >= retention && !item.is_protected);
|
|
if (removable.length === 0) {
|
|
return 0;
|
|
}
|
|
|
|
await prisma.backup.deleteMany({
|
|
where: {
|
|
id: {
|
|
in: removable.map((item) => item.id)
|
|
}
|
|
}
|
|
});
|
|
|
|
return removable.length;
|
|
}
|
|
|
|
async function createSnapshotBackupFromJob(job: {
|
|
id: string;
|
|
vm_id: string;
|
|
vm: {
|
|
id: string;
|
|
name: string;
|
|
node: string;
|
|
tenant_id: string;
|
|
billing_plan_id: string | null;
|
|
disk_gb: number;
|
|
vmid: number;
|
|
};
|
|
retention: number;
|
|
frequency: SnapshotFrequency;
|
|
}) {
|
|
const predictedSizeMb = estimateBackupSizeMb({
|
|
diskGb: job.vm.disk_gb,
|
|
type: BackupType.SNAPSHOT
|
|
});
|
|
|
|
await enforceBackupPolicyLimits({
|
|
tenantId: job.vm.tenant_id,
|
|
billingPlanId: job.vm.billing_plan_id,
|
|
predictedSizeMb,
|
|
protected: false
|
|
});
|
|
|
|
const now = new Date();
|
|
return prisma.backup.create({
|
|
data: {
|
|
vm_id: job.vm_id,
|
|
vm_name: job.vm.name,
|
|
tenant_id: job.vm.tenant_id,
|
|
node: job.vm.node,
|
|
status: BackupStatus.COMPLETED,
|
|
type: BackupType.SNAPSHOT,
|
|
source: BackupSource.LOCAL,
|
|
size_mb: predictedSizeMb,
|
|
storage: "local-lvm",
|
|
backup_path: `snapshot/${job.vm.node}/${job.vm.vmid}/${Date.now()}`,
|
|
schedule: toScheduleForSnapshot(job.frequency),
|
|
retention_days: Math.max(1, job.retention),
|
|
snapshot_job_id: job.id,
|
|
started_at: now,
|
|
completed_at: now,
|
|
expires_at: new Date(now.getTime() + Math.max(1, job.retention) * 24 * 60 * 60 * 1000),
|
|
notes: "Created by snapshot policy scheduler"
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function listSnapshotJobs(input: { tenantId?: string }) {
|
|
const where: Prisma.SnapshotJobWhereInput = {};
|
|
if (input.tenantId) {
|
|
where.vm = { tenant_id: input.tenantId };
|
|
}
|
|
|
|
return prisma.snapshotJob.findMany({
|
|
where,
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
tenant_id: true,
|
|
node: true
|
|
}
|
|
}
|
|
},
|
|
orderBy: [{ enabled: "desc" }, { next_run_at: "asc" }, { created_at: "desc" }]
|
|
});
|
|
}
|
|
|
|
export async function createSnapshotJob(input: {
|
|
vmId: string;
|
|
name: string;
|
|
frequency: SnapshotFrequency;
|
|
interval?: number;
|
|
dayOfWeek?: number;
|
|
hourUtc?: number;
|
|
minuteUtc?: number;
|
|
retention?: number;
|
|
enabled?: boolean;
|
|
createdBy?: string;
|
|
}) {
|
|
const vm = await prisma.virtualMachine.findUnique({
|
|
where: { id: input.vmId },
|
|
select: { id: true }
|
|
});
|
|
if (!vm) {
|
|
throw new HttpError(404, "VM not found", "VM_NOT_FOUND");
|
|
}
|
|
|
|
const interval = toPositiveInt(input.interval ?? 1, 1);
|
|
const retention = toPositiveInt(input.retention ?? 7, 7);
|
|
const hourUtc = Math.max(0, Math.min(23, Math.floor(input.hourUtc ?? 2)));
|
|
const minuteUtc = Math.max(0, Math.min(59, Math.floor(input.minuteUtc ?? 0)));
|
|
|
|
if (input.frequency === SnapshotFrequency.WEEKLY) {
|
|
const day = input.dayOfWeek ?? 0;
|
|
if (day < 0 || day > 6) {
|
|
throw new HttpError(400, "dayOfWeek must be between 0 and 6", "INVALID_DAY_OF_WEEK");
|
|
}
|
|
}
|
|
|
|
const nextRun = nextSnapshotRunAt({
|
|
frequency: input.frequency,
|
|
interval,
|
|
day_of_week: input.dayOfWeek ?? null,
|
|
hour_utc: hourUtc,
|
|
minute_utc: minuteUtc
|
|
});
|
|
|
|
return prisma.snapshotJob.create({
|
|
data: {
|
|
vm_id: input.vmId,
|
|
name: input.name,
|
|
frequency: input.frequency,
|
|
interval,
|
|
day_of_week: input.frequency === SnapshotFrequency.WEEKLY ? input.dayOfWeek ?? 0 : null,
|
|
hour_utc: hourUtc,
|
|
minute_utc: minuteUtc,
|
|
retention,
|
|
enabled: input.enabled ?? true,
|
|
next_run_at: input.enabled === false ? null : nextRun,
|
|
created_by: input.createdBy
|
|
},
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
tenant_id: true,
|
|
node: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function updateSnapshotJob(
|
|
jobId: string,
|
|
input: Partial<{
|
|
name: string;
|
|
frequency: SnapshotFrequency;
|
|
interval: number;
|
|
dayOfWeek: number | null;
|
|
hourUtc: number;
|
|
minuteUtc: number;
|
|
retention: number;
|
|
enabled: boolean;
|
|
}>
|
|
) {
|
|
const existing = await prisma.snapshotJob.findUnique({ where: { id: jobId } });
|
|
if (!existing) {
|
|
throw new HttpError(404, "Snapshot job not found", "SNAPSHOT_JOB_NOT_FOUND");
|
|
}
|
|
|
|
const frequency = input.frequency ?? existing.frequency;
|
|
const interval = toPositiveInt(input.interval ?? existing.interval, existing.interval);
|
|
const hourUtc = Math.max(0, Math.min(23, Math.floor(input.hourUtc ?? existing.hour_utc)));
|
|
const minuteUtc = Math.max(0, Math.min(59, Math.floor(input.minuteUtc ?? existing.minute_utc)));
|
|
const retention = toPositiveInt(input.retention ?? existing.retention, existing.retention);
|
|
const dayOfWeek =
|
|
frequency === SnapshotFrequency.WEEKLY
|
|
? (input.dayOfWeek ?? existing.day_of_week ?? 0)
|
|
: null;
|
|
|
|
if (frequency === SnapshotFrequency.WEEKLY && (dayOfWeek === null || dayOfWeek < 0 || dayOfWeek > 6)) {
|
|
throw new HttpError(400, "dayOfWeek must be between 0 and 6", "INVALID_DAY_OF_WEEK");
|
|
}
|
|
|
|
const enabled = input.enabled ?? existing.enabled;
|
|
const nextRun = enabled
|
|
? nextSnapshotRunAt({
|
|
frequency,
|
|
interval,
|
|
day_of_week: dayOfWeek,
|
|
hour_utc: hourUtc,
|
|
minute_utc: minuteUtc
|
|
})
|
|
: null;
|
|
|
|
return prisma.snapshotJob.update({
|
|
where: { id: existing.id },
|
|
data: {
|
|
name: input.name,
|
|
frequency,
|
|
interval,
|
|
day_of_week: dayOfWeek,
|
|
hour_utc: hourUtc,
|
|
minute_utc: minuteUtc,
|
|
retention,
|
|
enabled,
|
|
next_run_at: nextRun
|
|
},
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
tenant_id: true,
|
|
node: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function deleteSnapshotJob(jobId: string) {
|
|
return prisma.snapshotJob.delete({ where: { id: jobId } });
|
|
}
|
|
|
|
export async function runSnapshotJobNow(jobId: string) {
|
|
const job = await prisma.snapshotJob.findUnique({
|
|
where: { id: jobId },
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
node: true,
|
|
vmid: true,
|
|
disk_gb: true,
|
|
tenant_id: true,
|
|
billing_plan_id: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
if (!job) {
|
|
throw new HttpError(404, "Snapshot job not found", "SNAPSHOT_JOB_NOT_FOUND");
|
|
}
|
|
|
|
const backup = await createSnapshotBackupFromJob({
|
|
id: job.id,
|
|
vm_id: job.vm_id,
|
|
vm: job.vm,
|
|
retention: job.retention,
|
|
frequency: job.frequency
|
|
});
|
|
|
|
const pruned = await enforceSnapshotRetention(job.id, Math.max(1, job.retention));
|
|
const now = new Date();
|
|
const nextRun = job.enabled
|
|
? nextSnapshotRunAt({
|
|
frequency: job.frequency,
|
|
interval: job.interval,
|
|
day_of_week: job.day_of_week,
|
|
hour_utc: job.hour_utc,
|
|
minute_utc: job.minute_utc
|
|
}, now)
|
|
: null;
|
|
|
|
await prisma.snapshotJob.update({
|
|
where: { id: job.id },
|
|
data: {
|
|
last_run_at: now,
|
|
next_run_at: nextRun
|
|
}
|
|
});
|
|
|
|
return { backup, pruned };
|
|
}
|
|
|
|
export async function processDueSnapshotJobs() {
|
|
const now = new Date();
|
|
const dueJobs = await prisma.snapshotJob.findMany({
|
|
where: {
|
|
enabled: true,
|
|
next_run_at: {
|
|
lte: now
|
|
}
|
|
},
|
|
include: {
|
|
vm: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
node: true,
|
|
vmid: true,
|
|
disk_gb: true,
|
|
tenant_id: true,
|
|
billing_plan_id: true
|
|
}
|
|
}
|
|
},
|
|
orderBy: {
|
|
next_run_at: "asc"
|
|
},
|
|
take: 100
|
|
});
|
|
|
|
let executed = 0;
|
|
let failed = 0;
|
|
let pruned = 0;
|
|
let skipped = 0;
|
|
|
|
for (const job of dueJobs) {
|
|
const nextRun = nextSnapshotRunAt({
|
|
frequency: job.frequency,
|
|
interval: job.interval,
|
|
day_of_week: job.day_of_week,
|
|
hour_utc: job.hour_utc,
|
|
minute_utc: job.minute_utc
|
|
}, now);
|
|
|
|
const claim = await prisma.snapshotJob.updateMany({
|
|
where: {
|
|
id: job.id,
|
|
enabled: true,
|
|
next_run_at: {
|
|
lte: now
|
|
}
|
|
},
|
|
data: {
|
|
last_run_at: now,
|
|
next_run_at: nextRun
|
|
}
|
|
});
|
|
|
|
if (claim.count === 0) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
await createSnapshotBackupFromJob({
|
|
id: job.id,
|
|
vm_id: job.vm_id,
|
|
vm: job.vm,
|
|
retention: job.retention,
|
|
frequency: job.frequency
|
|
});
|
|
executed += 1;
|
|
pruned += await enforceSnapshotRetention(job.id, Math.max(1, job.retention));
|
|
} catch {
|
|
failed += 1;
|
|
}
|
|
}
|
|
|
|
return {
|
|
scanned: dueJobs.length,
|
|
executed,
|
|
failed,
|
|
pruned,
|
|
skipped
|
|
};
|
|
}
|
|
|
|
export async function processPendingBackups() {
|
|
const now = new Date();
|
|
const pending = await prisma.backup.findMany({
|
|
where: {
|
|
status: BackupStatus.PENDING
|
|
},
|
|
orderBy: {
|
|
created_at: "asc"
|
|
},
|
|
take: 200
|
|
});
|
|
|
|
let completed = 0;
|
|
let skipped = 0;
|
|
for (const backup of pending) {
|
|
const claim = await prisma.backup.updateMany({
|
|
where: {
|
|
id: backup.id,
|
|
status: BackupStatus.PENDING
|
|
},
|
|
data: {
|
|
status: BackupStatus.RUNNING,
|
|
started_at: backup.started_at ?? now
|
|
}
|
|
});
|
|
|
|
if (claim.count === 0) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
const retentionDays = Math.max(1, backup.retention_days);
|
|
await prisma.backup.update({
|
|
where: { id: backup.id },
|
|
data: {
|
|
status: BackupStatus.COMPLETED,
|
|
completed_at: now,
|
|
expires_at: new Date(now.getTime() + retentionDays * 24 * 60 * 60 * 1000),
|
|
next_run_at: backup.schedule === BackupSchedule.MANUAL ? null : nextBackupRunDate(backup.schedule, now),
|
|
pbs_snapshot_id:
|
|
backup.source === BackupSource.PBS
|
|
? backup.pbs_snapshot_id ?? `pbs/${backup.node ?? "node"}/${backup.vm_id}/${Date.now()}`
|
|
: backup.pbs_snapshot_id
|
|
}
|
|
});
|
|
completed += 1;
|
|
}
|
|
|
|
return {
|
|
scanned: pending.length,
|
|
completed,
|
|
skipped
|
|
};
|
|
}
|
|
|
|
export async function listBackupPolicies() {
|
|
return prisma.backupPolicy.findMany({
|
|
include: {
|
|
tenant: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
slug: true
|
|
}
|
|
},
|
|
billing_plan: {
|
|
select: {
|
|
id: true,
|
|
name: true,
|
|
slug: true
|
|
}
|
|
}
|
|
},
|
|
orderBy: [{ tenant_id: "asc" }, { billing_plan_id: "asc" }, { created_at: "desc" }]
|
|
});
|
|
}
|
|
|
|
export async function upsertBackupPolicy(input: {
|
|
policyId?: string;
|
|
tenantId?: string;
|
|
billingPlanId?: string;
|
|
maxFiles?: number;
|
|
maxTotalSizeMb?: number;
|
|
maxProtectedFiles?: number;
|
|
allowFileRestore?: boolean;
|
|
allowCrossVmRestore?: boolean;
|
|
allowPbsRestore?: boolean;
|
|
}) {
|
|
const basePayload = {
|
|
tenant_id: input.tenantId,
|
|
billing_plan_id: input.billingPlanId,
|
|
max_files: toPositiveInt(input.maxFiles ?? DEFAULT_POLICY.max_files, DEFAULT_POLICY.max_files),
|
|
max_total_size_mb:
|
|
typeof input.maxTotalSizeMb === "number" && Number.isFinite(input.maxTotalSizeMb) && input.maxTotalSizeMb > 0
|
|
? input.maxTotalSizeMb
|
|
: DEFAULT_POLICY.max_total_size_mb,
|
|
max_protected_files: toPositiveInt(
|
|
input.maxProtectedFiles ?? DEFAULT_POLICY.max_protected_files,
|
|
DEFAULT_POLICY.max_protected_files
|
|
),
|
|
allow_file_restore: input.allowFileRestore ?? DEFAULT_POLICY.allow_file_restore,
|
|
allow_cross_vm_restore: input.allowCrossVmRestore ?? DEFAULT_POLICY.allow_cross_vm_restore,
|
|
allow_pbs_restore: input.allowPbsRestore ?? DEFAULT_POLICY.allow_pbs_restore
|
|
};
|
|
|
|
if (input.policyId) {
|
|
return prisma.backupPolicy.update({
|
|
where: { id: input.policyId },
|
|
data: basePayload
|
|
});
|
|
}
|
|
|
|
return prisma.backupPolicy.create({
|
|
data: basePayload
|
|
});
|
|
}
|