459 lines
15 KiB
TypeScript
459 lines
15 KiB
TypeScript
|
|
/**
|
||
|
|
* Admin-configurable backup destinations — service layer.
|
||
|
|
* See docs/superpowers/specs/2026-06-04-backup-destinations-design.md.
|
||
|
|
*
|
||
|
|
* Responsibilities:
|
||
|
|
* - CRUD over `backup_destinations` with secret encryption at rest + masking on
|
||
|
|
* read (mirrors the send-from-accounts pattern: API returns only `*IsSet`).
|
||
|
|
* - test / manual-push / prune to a destination.
|
||
|
|
* - scheduled push to all enabled destinations, with failure alerting.
|
||
|
|
*
|
||
|
|
* Every push transports the exact SHA-verified tar from `createFullBackupTar()`
|
||
|
|
* — the same bundle admins download — optionally AES-256 encrypted first.
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { unlink } from 'node:fs/promises';
|
||
|
|
|
||
|
|
import { and, eq, isNull } from 'drizzle-orm';
|
||
|
|
|
||
|
|
import { createAuditLog } from '@/lib/audit';
|
||
|
|
import { db } from '@/lib/db';
|
||
|
|
import {
|
||
|
|
backupDestinations,
|
||
|
|
backupJobs,
|
||
|
|
systemSettings,
|
||
|
|
type BackupDestination,
|
||
|
|
} from '@/lib/db/schema/system';
|
||
|
|
import { userPortRoles, userProfiles } from '@/lib/db/schema/users';
|
||
|
|
import { logger } from '@/lib/logger';
|
||
|
|
import { createNotification } from '@/lib/services/notifications.service';
|
||
|
|
import { createFullBackupTar } from '@/lib/services/backup-export.service';
|
||
|
|
import { decrypt, encrypt } from '@/lib/utils/encryption';
|
||
|
|
|
||
|
|
import {
|
||
|
|
buildTransport,
|
||
|
|
type BackupDestinationType,
|
||
|
|
type BackupTransport,
|
||
|
|
} from './backup-destinations';
|
||
|
|
import { encryptFileToFile } from './backup-destinations/bundle-encryption';
|
||
|
|
|
||
|
|
// ─── secret config handling ─────────────────────────────────────────────────
|
||
|
|
|
||
|
|
const SECRET_FIELDS: Record<BackupDestinationType, string[]> = {
|
||
|
|
sftp: ['password', 'privateKey', 'passphrase'],
|
||
|
|
s3: ['secretKey'],
|
||
|
|
filesystem: [],
|
||
|
|
};
|
||
|
|
|
||
|
|
type Cfg = Record<string, unknown>;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Prepare an incoming config for storage: encrypt every secret field that
|
||
|
|
* carries a new non-empty value; for blank/absent secret fields, carry over the
|
||
|
|
* already-encrypted value from `existing` (so "leave unchanged" works on edit).
|
||
|
|
* Non-secret fields are taken from `incoming` (falling back to `existing`).
|
||
|
|
*/
|
||
|
|
export function serializeConfig(type: BackupDestinationType, incoming: Cfg, existing?: Cfg): Cfg {
|
||
|
|
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||
|
|
const out: Cfg = {};
|
||
|
|
// Non-secret fields from incoming (or carry existing if omitted).
|
||
|
|
for (const [k, v] of Object.entries(incoming)) {
|
||
|
|
if (!secrets.has(k)) out[k] = v;
|
||
|
|
}
|
||
|
|
for (const field of secrets) {
|
||
|
|
const incomingVal = incoming[field];
|
||
|
|
if (typeof incomingVal === 'string' && incomingVal.length > 0) {
|
||
|
|
out[field] = encrypt(incomingVal);
|
||
|
|
} else if (existing && typeof existing[field] === 'string') {
|
||
|
|
out[field] = existing[field];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Decrypt the secret fields of a stored config back to plaintext for transport use. */
|
||
|
|
export function decryptConfig(type: BackupDestinationType, stored: Cfg): Cfg {
|
||
|
|
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||
|
|
const out: Cfg = { ...stored };
|
||
|
|
for (const field of secrets) {
|
||
|
|
const v = stored[field];
|
||
|
|
if (typeof v === 'string' && v.length > 0) {
|
||
|
|
try {
|
||
|
|
out[field] = decrypt(v);
|
||
|
|
} catch (err) {
|
||
|
|
logger.error({ err, field }, 'Failed to decrypt backup destination secret');
|
||
|
|
delete out[field];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Strip secret fields from a stored config and expose `<field>IsSet` markers. */
|
||
|
|
export function maskConfig(type: BackupDestinationType, stored: Cfg): Cfg {
|
||
|
|
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||
|
|
const out: Cfg = {};
|
||
|
|
for (const [k, v] of Object.entries(stored)) {
|
||
|
|
if (!secrets.has(k)) out[k] = v;
|
||
|
|
}
|
||
|
|
for (const field of secrets) {
|
||
|
|
out[`${field}IsSet`] =
|
||
|
|
typeof stored[field] === 'string' && (stored[field] as string).length > 0;
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
export interface MaskedDestination {
|
||
|
|
id: string;
|
||
|
|
name: string;
|
||
|
|
type: BackupDestinationType;
|
||
|
|
enabled: boolean;
|
||
|
|
config: Cfg;
|
||
|
|
retentionCount: number | null;
|
||
|
|
encryptBundle: boolean;
|
||
|
|
encryptionKeyIsSet: boolean;
|
||
|
|
lastRunAt: Date | null;
|
||
|
|
lastStatus: string | null;
|
||
|
|
lastError: string | null;
|
||
|
|
lastBackupBytes: number | null;
|
||
|
|
createdAt: Date;
|
||
|
|
updatedAt: Date;
|
||
|
|
}
|
||
|
|
|
||
|
|
function mask(row: BackupDestination): MaskedDestination {
|
||
|
|
const type = row.type as BackupDestinationType;
|
||
|
|
return {
|
||
|
|
id: row.id,
|
||
|
|
name: row.name,
|
||
|
|
type,
|
||
|
|
enabled: row.enabled,
|
||
|
|
config: maskConfig(type, (row.config ?? {}) as Cfg),
|
||
|
|
retentionCount: row.retentionCount,
|
||
|
|
encryptBundle: row.encryptBundle,
|
||
|
|
encryptionKeyIsSet: Boolean(row.encryptionKeyEncrypted),
|
||
|
|
lastRunAt: row.lastRunAt,
|
||
|
|
lastStatus: row.lastStatus,
|
||
|
|
lastError: row.lastError,
|
||
|
|
lastBackupBytes: row.lastBackupBytes,
|
||
|
|
createdAt: row.createdAt,
|
||
|
|
updatedAt: row.updatedAt,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── schedule ────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
export type BackupSchedule = 'off' | 'daily' | 'weekly';
|
||
|
|
|
||
|
|
/** Whether a scheduled push should run for `date` under `schedule`. */
|
||
|
|
export function isScheduleDue(schedule: BackupSchedule, date: Date): boolean {
|
||
|
|
if (schedule === 'off') return false;
|
||
|
|
if (schedule === 'daily') return true;
|
||
|
|
return date.getUTCDay() === 0; // weekly → Sundays
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function getSchedule(): Promise<BackupSchedule> {
|
||
|
|
const [row] = await db
|
||
|
|
.select()
|
||
|
|
.from(systemSettings)
|
||
|
|
.where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
|
||
|
|
const v = row?.value;
|
||
|
|
return v === 'daily' || v === 'weekly' ? v : 'off';
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function setSchedule(value: BackupSchedule, userId: string): Promise<void> {
|
||
|
|
const existing = await db.query.systemSettings.findFirst({
|
||
|
|
where: and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)),
|
||
|
|
});
|
||
|
|
if (existing) {
|
||
|
|
await db
|
||
|
|
.update(systemSettings)
|
||
|
|
.set({ value, updatedBy: userId, updatedAt: new Date() })
|
||
|
|
.where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
|
||
|
|
} else {
|
||
|
|
await db
|
||
|
|
.insert(systemSettings)
|
||
|
|
.values({ key: 'backup_schedule', value, portId: null, updatedBy: userId });
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── CRUD ──────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
export async function listDestinations(): Promise<MaskedDestination[]> {
|
||
|
|
const rows = await db.query.backupDestinations.findMany({
|
||
|
|
orderBy: (d, { asc }) => [asc(d.createdAt)],
|
||
|
|
});
|
||
|
|
return rows.map(mask);
|
||
|
|
}
|
||
|
|
|
||
|
|
export interface DestinationInput {
|
||
|
|
name: string;
|
||
|
|
type: BackupDestinationType;
|
||
|
|
enabled?: boolean;
|
||
|
|
config: Cfg;
|
||
|
|
retentionCount?: number | null;
|
||
|
|
encryptBundle?: boolean;
|
||
|
|
/** Plaintext bundle passphrase; encrypted at rest. Blank = leave unchanged. */
|
||
|
|
encryptionKey?: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function createDestination(input: DestinationInput): Promise<MaskedDestination> {
|
||
|
|
const [row] = await db
|
||
|
|
.insert(backupDestinations)
|
||
|
|
.values({
|
||
|
|
name: input.name,
|
||
|
|
type: input.type,
|
||
|
|
enabled: input.enabled ?? false,
|
||
|
|
config: serializeConfig(input.type, input.config),
|
||
|
|
retentionCount: input.retentionCount ?? null,
|
||
|
|
encryptBundle: input.encryptBundle ?? false,
|
||
|
|
encryptionKeyEncrypted:
|
||
|
|
input.encryptionKey && input.encryptionKey.length > 0 ? encrypt(input.encryptionKey) : null,
|
||
|
|
})
|
||
|
|
.returning();
|
||
|
|
if (!row) throw new Error('Failed to create backup destination');
|
||
|
|
return mask(row);
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function updateDestination(
|
||
|
|
id: string,
|
||
|
|
input: DestinationInput,
|
||
|
|
): Promise<MaskedDestination> {
|
||
|
|
const existing = await db.query.backupDestinations.findFirst({
|
||
|
|
where: eq(backupDestinations.id, id),
|
||
|
|
});
|
||
|
|
if (!existing) throw new Error('Backup destination not found');
|
||
|
|
|
||
|
|
const [row] = await db
|
||
|
|
.update(backupDestinations)
|
||
|
|
.set({
|
||
|
|
name: input.name,
|
||
|
|
type: input.type,
|
||
|
|
enabled: input.enabled ?? existing.enabled,
|
||
|
|
config: serializeConfig(input.type, input.config, (existing.config ?? {}) as Cfg),
|
||
|
|
retentionCount: input.retentionCount ?? null,
|
||
|
|
encryptBundle: input.encryptBundle ?? false,
|
||
|
|
encryptionKeyEncrypted:
|
||
|
|
input.encryptionKey && input.encryptionKey.length > 0
|
||
|
|
? encrypt(input.encryptionKey)
|
||
|
|
: existing.encryptionKeyEncrypted,
|
||
|
|
updatedAt: new Date(),
|
||
|
|
})
|
||
|
|
.where(eq(backupDestinations.id, id))
|
||
|
|
.returning();
|
||
|
|
if (!row) throw new Error('Failed to update backup destination');
|
||
|
|
return mask(row);
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function deleteDestination(id: string): Promise<void> {
|
||
|
|
await db.delete(backupDestinations).where(eq(backupDestinations.id, id));
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── transport helpers ────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
function transportFor(row: BackupDestination): BackupTransport {
|
||
|
|
const type = row.type as BackupDestinationType;
|
||
|
|
return buildTransport(type, decryptConfig(type, (row.config ?? {}) as Cfg));
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function testDestination(id: string): Promise<void> {
|
||
|
|
const row = await db.query.backupDestinations.findFirst({
|
||
|
|
where: eq(backupDestinations.id, id),
|
||
|
|
});
|
||
|
|
if (!row) throw new Error('Backup destination not found');
|
||
|
|
await transportFor(row).test();
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── push ─────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
interface PushOpts {
|
||
|
|
/** Reuse an already-assembled tar (scheduled push assembles once for all). */
|
||
|
|
tarPath?: string;
|
||
|
|
filename?: string;
|
||
|
|
trigger: 'manual' | 'cron';
|
||
|
|
triggeredBy?: string | null;
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function pushBackupToDestination(
|
||
|
|
id: string,
|
||
|
|
opts: PushOpts,
|
||
|
|
): Promise<{
|
||
|
|
bytes: number;
|
||
|
|
remoteRef: string;
|
||
|
|
}> {
|
||
|
|
const row = await db.query.backupDestinations.findFirst({
|
||
|
|
where: eq(backupDestinations.id, id),
|
||
|
|
});
|
||
|
|
if (!row) throw new Error('Backup destination not found');
|
||
|
|
|
||
|
|
const transport = transportFor(row);
|
||
|
|
let tarPath = opts.tarPath;
|
||
|
|
let filename = opts.filename;
|
||
|
|
let ownTar: (() => Promise<void>) | null = null;
|
||
|
|
let encPath: string | null = null;
|
||
|
|
|
||
|
|
try {
|
||
|
|
if (!tarPath || !filename) {
|
||
|
|
const made = await createFullBackupTar();
|
||
|
|
tarPath = made.tarPath;
|
||
|
|
filename = made.filename;
|
||
|
|
ownTar = made.cleanup;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Optional client-side encryption before the bytes leave this server.
|
||
|
|
let uploadPath = tarPath;
|
||
|
|
let remoteName = filename;
|
||
|
|
if (row.encryptBundle) {
|
||
|
|
if (!row.encryptionKeyEncrypted) {
|
||
|
|
throw new Error('Destination has encryption enabled but no passphrase configured');
|
||
|
|
}
|
||
|
|
const passphrase = decrypt(row.encryptionKeyEncrypted);
|
||
|
|
encPath = `${tarPath}.enc`;
|
||
|
|
await encryptFileToFile(tarPath, encPath, passphrase);
|
||
|
|
uploadPath = encPath;
|
||
|
|
remoteName = `${filename}.enc`;
|
||
|
|
}
|
||
|
|
|
||
|
|
const { bytes, remoteRef } = await transport.push(uploadPath, remoteName);
|
||
|
|
await transport.prune(row.retentionCount).catch((err) => {
|
||
|
|
logger.warn({ err, destinationId: id }, 'Backup prune failed (push succeeded)');
|
||
|
|
});
|
||
|
|
|
||
|
|
await db
|
||
|
|
.update(backupDestinations)
|
||
|
|
.set({
|
||
|
|
lastRunAt: new Date(),
|
||
|
|
lastStatus: 'ok',
|
||
|
|
lastError: null,
|
||
|
|
lastBackupBytes: bytes,
|
||
|
|
})
|
||
|
|
.where(eq(backupDestinations.id, id));
|
||
|
|
|
||
|
|
await db.insert(backupJobs).values({
|
||
|
|
status: 'completed',
|
||
|
|
trigger: opts.trigger,
|
||
|
|
triggeredBy: opts.triggeredBy ?? null,
|
||
|
|
sizeBytes: bytes,
|
||
|
|
storagePath: remoteRef,
|
||
|
|
completedAt: new Date(),
|
||
|
|
});
|
||
|
|
|
||
|
|
logger.info({ destinationId: id, bytes, remoteRef }, 'Backup pushed to destination');
|
||
|
|
return { bytes, remoteRef };
|
||
|
|
} catch (err) {
|
||
|
|
const message = err instanceof Error ? err.message : 'unknown';
|
||
|
|
await db
|
||
|
|
.update(backupDestinations)
|
||
|
|
.set({ lastRunAt: new Date(), lastStatus: 'failed', lastError: message })
|
||
|
|
.where(eq(backupDestinations.id, id));
|
||
|
|
await db
|
||
|
|
.insert(backupJobs)
|
||
|
|
.values({
|
||
|
|
status: 'failed',
|
||
|
|
trigger: opts.trigger,
|
||
|
|
triggeredBy: opts.triggeredBy ?? null,
|
||
|
|
errorMessage: `[${row.name}] ${message}`,
|
||
|
|
completedAt: new Date(),
|
||
|
|
})
|
||
|
|
.catch(() => {});
|
||
|
|
await notifyBackupFailure(row.name, message, opts.trigger);
|
||
|
|
throw err;
|
||
|
|
} finally {
|
||
|
|
if (encPath) await unlink(encPath).catch(() => {});
|
||
|
|
if (ownTar) await ownTar();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Scheduled push: assemble the bundle ONCE and fan it out to every enabled
|
||
|
|
* destination. Per-destination failures are isolated (one bad server doesn't
|
||
|
|
* skip the others) and alerted.
|
||
|
|
*/
|
||
|
|
export async function runScheduledBackupPush(now = new Date()): Promise<{
|
||
|
|
ran: boolean;
|
||
|
|
pushed: number;
|
||
|
|
failed: number;
|
||
|
|
}> {
|
||
|
|
const schedule = await getSchedule();
|
||
|
|
if (!isScheduleDue(schedule, now)) {
|
||
|
|
logger.info({ schedule }, 'Scheduled backup not due');
|
||
|
|
return { ran: false, pushed: 0, failed: 0 };
|
||
|
|
}
|
||
|
|
|
||
|
|
const enabled = await db.query.backupDestinations.findMany({
|
||
|
|
where: eq(backupDestinations.enabled, true),
|
||
|
|
});
|
||
|
|
if (enabled.length === 0) {
|
||
|
|
logger.warn('Backup schedule is on but no destinations are enabled');
|
||
|
|
return { ran: false, pushed: 0, failed: 0 };
|
||
|
|
}
|
||
|
|
|
||
|
|
const bundle = await createFullBackupTar();
|
||
|
|
let pushed = 0;
|
||
|
|
let failed = 0;
|
||
|
|
try {
|
||
|
|
for (const dest of enabled) {
|
||
|
|
try {
|
||
|
|
await pushBackupToDestination(dest.id, {
|
||
|
|
tarPath: bundle.tarPath,
|
||
|
|
filename: bundle.filename,
|
||
|
|
trigger: 'cron',
|
||
|
|
});
|
||
|
|
pushed += 1;
|
||
|
|
} catch (err) {
|
||
|
|
failed += 1;
|
||
|
|
logger.error({ err, destinationId: dest.id }, 'Scheduled push to destination failed');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} finally {
|
||
|
|
await bundle.cleanup();
|
||
|
|
}
|
||
|
|
logger.info({ pushed, failed, total: enabled.length }, 'Scheduled backup push complete');
|
||
|
|
return { ran: true, pushed, failed };
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── failure alerting ────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
async function notifyBackupFailure(
|
||
|
|
destinationName: string,
|
||
|
|
message: string,
|
||
|
|
trigger: 'manual' | 'cron',
|
||
|
|
): Promise<void> {
|
||
|
|
// Guaranteed signal: an error-severity audit row (visible in /admin/audit).
|
||
|
|
await createAuditLog({
|
||
|
|
userId: null,
|
||
|
|
portId: null,
|
||
|
|
action: 'job_failed',
|
||
|
|
entityType: 'backup_destination',
|
||
|
|
entityId: destinationName,
|
||
|
|
severity: 'error',
|
||
|
|
source: trigger === 'cron' ? 'cron' : 'job',
|
||
|
|
metadata: { destination: destinationName, error: message },
|
||
|
|
});
|
||
|
|
|
||
|
|
// Best-effort: in-app system alert to every super-admin (per their port).
|
||
|
|
try {
|
||
|
|
const admins = await db
|
||
|
|
.select({ userId: userProfiles.userId, portId: userPortRoles.portId })
|
||
|
|
.from(userProfiles)
|
||
|
|
.innerJoin(userPortRoles, eq(userPortRoles.userId, userProfiles.userId))
|
||
|
|
.where(eq(userProfiles.isSuperAdmin, true));
|
||
|
|
const seen = new Set<string>();
|
||
|
|
for (const a of admins) {
|
||
|
|
const key = `${a.userId}:${a.portId}`;
|
||
|
|
if (seen.has(key)) continue;
|
||
|
|
seen.add(key);
|
||
|
|
await createNotification({
|
||
|
|
portId: a.portId,
|
||
|
|
userId: a.userId,
|
||
|
|
type: 'system_alert',
|
||
|
|
title: 'Backup push failed',
|
||
|
|
description: `Backup to "${destinationName}" failed: ${message}`,
|
||
|
|
dedupeKey: `backup-fail:${destinationName}`,
|
||
|
|
cooldownMs: 60 * 60 * 1000,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
} catch (err) {
|
||
|
|
logger.error({ err }, 'Failed to notify super-admins of backup failure');
|
||
|
|
}
|
||
|
|
}
|