feat(backup): full DR bundle export + admin-configurable offsite destinations

Backend-agnostic disaster-recovery backup engine that runs on the current storage backend (no storage cutover required): - Full-bundle export: db.dump (pg_dump custom) + every storage blob + manifest.json with per-object SHA-256, streamed as a tar. Entry points: admin UI download, GET /api/v1/admin/backup/export, scripts/create-full-backup.ts. - Admin-configurable push destinations (backup_destinations table, migration 0091): SFTP/SSH, S3-compatible (reuses the minio client), and mounted path/NAS behind one transport interface (test/push/prune). Secrets AES-GCM at rest; API returns only *IsSet markers. - Opt-in per-destination AES-256 bundle encryption (scrypt KDF, streamed) + scripts/decrypt-backup.ts for restore. - Wired the previously-dead database-backup cron to runScheduledBackupPush (push to enabled destinations, prune to retention, alert super-admins on failure). Tests: 1608 unit/integration pass; tsc + lint clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-04 11:23:42 +02:00
parent 05950ae0b6
commit fe863a588e
35 changed files with 3125 additions and 15 deletions
--- a/src/lib/services/backup-destinations.service.ts
+++ b/src/lib/services/backup-destinations.service.ts
@@ -0,0 +1,458 @@
+/**
+ * Admin-configurable backup destinations — service layer.
+ * See docs/superpowers/specs/2026-06-04-backup-destinations-design.md.
+ *
+ * Responsibilities:
+ *  - CRUD over `backup_destinations` with secret encryption at rest + masking on
+ *    read (mirrors the send-from-accounts pattern: API returns only `*IsSet`).
+ *  - test / manual-push / prune to a destination.
+ *  - scheduled push to all enabled destinations, with failure alerting.
+ *
+ * Every push transports the exact SHA-verified tar from `createFullBackupTar()`
+ * — the same bundle admins download — optionally AES-256 encrypted first.
+ */
+
+import { unlink } from 'node:fs/promises';
+
+import { and, eq, isNull } from 'drizzle-orm';
+
+import { createAuditLog } from '@/lib/audit';
+import { db } from '@/lib/db';
+import {
+  backupDestinations,
+  backupJobs,
+  systemSettings,
+  type BackupDestination,
+} from '@/lib/db/schema/system';
+import { userPortRoles, userProfiles } from '@/lib/db/schema/users';
+import { logger } from '@/lib/logger';
+import { createNotification } from '@/lib/services/notifications.service';
+import { createFullBackupTar } from '@/lib/services/backup-export.service';
+import { decrypt, encrypt } from '@/lib/utils/encryption';
+
+import {
+  buildTransport,
+  type BackupDestinationType,
+  type BackupTransport,
+} from './backup-destinations';
+import { encryptFileToFile } from './backup-destinations/bundle-encryption';
+
+// ─── secret config handling ─────────────────────────────────────────────────
+
+const SECRET_FIELDS: Record<BackupDestinationType, string[]> = {
+  sftp: ['password', 'privateKey', 'passphrase'],
+  s3: ['secretKey'],
+  filesystem: [],
+};
+
+type Cfg = Record<string, unknown>;
+
+/**
+ * Prepare an incoming config for storage: encrypt every secret field that
+ * carries a new non-empty value; for blank/absent secret fields, carry over the
+ * already-encrypted value from `existing` (so "leave unchanged" works on edit).
+ * Non-secret fields are taken from `incoming` (falling back to `existing`).
+ */
+export function serializeConfig(type: BackupDestinationType, incoming: Cfg, existing?: Cfg): Cfg {
+  const secrets = new Set(SECRET_FIELDS[type] ?? []);
+  const out: Cfg = {};
+  // Non-secret fields from incoming (or carry existing if omitted).
+  for (const [k, v] of Object.entries(incoming)) {
+    if (!secrets.has(k)) out[k] = v;
+  }
+  for (const field of secrets) {
+    const incomingVal = incoming[field];
+    if (typeof incomingVal === 'string' && incomingVal.length > 0) {
+      out[field] = encrypt(incomingVal);
+    } else if (existing && typeof existing[field] === 'string') {
+      out[field] = existing[field];
+    }
+  }
+  return out;
+}
+
+/** Decrypt the secret fields of a stored config back to plaintext for transport use. */
+export function decryptConfig(type: BackupDestinationType, stored: Cfg): Cfg {
+  const secrets = new Set(SECRET_FIELDS[type] ?? []);
+  const out: Cfg = { ...stored };
+  for (const field of secrets) {
+    const v = stored[field];
+    if (typeof v === 'string' && v.length > 0) {
+      try {
+        out[field] = decrypt(v);
+      } catch (err) {
+        logger.error({ err, field }, 'Failed to decrypt backup destination secret');
+        delete out[field];
+      }
+    }
+  }
+  return out;
+}
+
+/** Strip secret fields from a stored config and expose `<field>IsSet` markers. */
+export function maskConfig(type: BackupDestinationType, stored: Cfg): Cfg {
+  const secrets = new Set(SECRET_FIELDS[type] ?? []);
+  const out: Cfg = {};
+  for (const [k, v] of Object.entries(stored)) {
+    if (!secrets.has(k)) out[k] = v;
+  }
+  for (const field of secrets) {
+    out[`${field}IsSet`] =
+      typeof stored[field] === 'string' && (stored[field] as string).length > 0;
+  }
+  return out;
+}
+
+export interface MaskedDestination {
+  id: string;
+  name: string;
+  type: BackupDestinationType;
+  enabled: boolean;
+  config: Cfg;
+  retentionCount: number | null;
+  encryptBundle: boolean;
+  encryptionKeyIsSet: boolean;
+  lastRunAt: Date | null;
+  lastStatus: string | null;
+  lastError: string | null;
+  lastBackupBytes: number | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+function mask(row: BackupDestination): MaskedDestination {
+  const type = row.type as BackupDestinationType;
+  return {
+    id: row.id,
+    name: row.name,
+    type,
+    enabled: row.enabled,
+    config: maskConfig(type, (row.config ?? {}) as Cfg),
+    retentionCount: row.retentionCount,
+    encryptBundle: row.encryptBundle,
+    encryptionKeyIsSet: Boolean(row.encryptionKeyEncrypted),
+    lastRunAt: row.lastRunAt,
+    lastStatus: row.lastStatus,
+    lastError: row.lastError,
+    lastBackupBytes: row.lastBackupBytes,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
+// ─── schedule ────────────────────────────────────────────────────────────────
+
+export type BackupSchedule = 'off' | 'daily' | 'weekly';
+
+/** Whether a scheduled push should run for `date` under `schedule`. */
+export function isScheduleDue(schedule: BackupSchedule, date: Date): boolean {
+  if (schedule === 'off') return false;
+  if (schedule === 'daily') return true;
+  return date.getUTCDay() === 0; // weekly → Sundays
+}
+
+export async function getSchedule(): Promise<BackupSchedule> {
+  const [row] = await db
+    .select()
+    .from(systemSettings)
+    .where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
+  const v = row?.value;
+  return v === 'daily' || v === 'weekly' ? v : 'off';
+}
+
+export async function setSchedule(value: BackupSchedule, userId: string): Promise<void> {
+  const existing = await db.query.systemSettings.findFirst({
+    where: and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)),
+  });
+  if (existing) {
+    await db
+      .update(systemSettings)
+      .set({ value, updatedBy: userId, updatedAt: new Date() })
+      .where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
+  } else {
+    await db
+      .insert(systemSettings)
+      .values({ key: 'backup_schedule', value, portId: null, updatedBy: userId });
+  }
+}
+
+// ─── CRUD ──────────────────────────────────────────────────────────────────
+
+export async function listDestinations(): Promise<MaskedDestination[]> {
+  const rows = await db.query.backupDestinations.findMany({
+    orderBy: (d, { asc }) => [asc(d.createdAt)],
+  });
+  return rows.map(mask);
+}
+
+export interface DestinationInput {
+  name: string;
+  type: BackupDestinationType;
+  enabled?: boolean;
+  config: Cfg;
+  retentionCount?: number | null;
+  encryptBundle?: boolean;
+  /** Plaintext bundle passphrase; encrypted at rest. Blank = leave unchanged. */
+  encryptionKey?: string;
+}
+
+export async function createDestination(input: DestinationInput): Promise<MaskedDestination> {
+  const [row] = await db
+    .insert(backupDestinations)
+    .values({
+      name: input.name,
+      type: input.type,
+      enabled: input.enabled ?? false,
+      config: serializeConfig(input.type, input.config),
+      retentionCount: input.retentionCount ?? null,
+      encryptBundle: input.encryptBundle ?? false,
+      encryptionKeyEncrypted:
+        input.encryptionKey && input.encryptionKey.length > 0 ? encrypt(input.encryptionKey) : null,
+    })
+    .returning();
+  if (!row) throw new Error('Failed to create backup destination');
+  return mask(row);
+}
+
+export async function updateDestination(
+  id: string,
+  input: DestinationInput,
+): Promise<MaskedDestination> {
+  const existing = await db.query.backupDestinations.findFirst({
+    where: eq(backupDestinations.id, id),
+  });
+  if (!existing) throw new Error('Backup destination not found');
+
+  const [row] = await db
+    .update(backupDestinations)
+    .set({
+      name: input.name,
+      type: input.type,
+      enabled: input.enabled ?? existing.enabled,
+      config: serializeConfig(input.type, input.config, (existing.config ?? {}) as Cfg),
+      retentionCount: input.retentionCount ?? null,
+      encryptBundle: input.encryptBundle ?? false,
+      encryptionKeyEncrypted:
+        input.encryptionKey && input.encryptionKey.length > 0
+          ? encrypt(input.encryptionKey)
+          : existing.encryptionKeyEncrypted,
+      updatedAt: new Date(),
+    })
+    .where(eq(backupDestinations.id, id))
+    .returning();
+  if (!row) throw new Error('Failed to update backup destination');
+  return mask(row);
+}
+
+export async function deleteDestination(id: string): Promise<void> {
+  await db.delete(backupDestinations).where(eq(backupDestinations.id, id));
+}
+
+// ─── transport helpers ────────────────────────────────────────────────────
+
+function transportFor(row: BackupDestination): BackupTransport {
+  const type = row.type as BackupDestinationType;
+  return buildTransport(type, decryptConfig(type, (row.config ?? {}) as Cfg));
+}
+
+export async function testDestination(id: string): Promise<void> {
+  const row = await db.query.backupDestinations.findFirst({
+    where: eq(backupDestinations.id, id),
+  });
+  if (!row) throw new Error('Backup destination not found');
+  await transportFor(row).test();
+}
+
+// ─── push ─────────────────────────────────────────────────────────────────
+
+interface PushOpts {
+  /** Reuse an already-assembled tar (scheduled push assembles once for all). */
+  tarPath?: string;
+  filename?: string;
+  trigger: 'manual' | 'cron';
+  triggeredBy?: string | null;
+}
+
+export async function pushBackupToDestination(
+  id: string,
+  opts: PushOpts,
+): Promise<{
+  bytes: number;
+  remoteRef: string;
+}> {
+  const row = await db.query.backupDestinations.findFirst({
+    where: eq(backupDestinations.id, id),
+  });
+  if (!row) throw new Error('Backup destination not found');
+
+  const transport = transportFor(row);
+  let tarPath = opts.tarPath;
+  let filename = opts.filename;
+  let ownTar: (() => Promise<void>) | null = null;
+  let encPath: string | null = null;
+
+  try {
+    if (!tarPath || !filename) {
+      const made = await createFullBackupTar();
+      tarPath = made.tarPath;
+      filename = made.filename;
+      ownTar = made.cleanup;
+    }
+
+    // Optional client-side encryption before the bytes leave this server.
+    let uploadPath = tarPath;
+    let remoteName = filename;
+    if (row.encryptBundle) {
+      if (!row.encryptionKeyEncrypted) {
+        throw new Error('Destination has encryption enabled but no passphrase configured');
+      }
+      const passphrase = decrypt(row.encryptionKeyEncrypted);
+      encPath = `${tarPath}.enc`;
+      await encryptFileToFile(tarPath, encPath, passphrase);
+      uploadPath = encPath;
+      remoteName = `${filename}.enc`;
+    }
+
+    const { bytes, remoteRef } = await transport.push(uploadPath, remoteName);
+    await transport.prune(row.retentionCount).catch((err) => {
+      logger.warn({ err, destinationId: id }, 'Backup prune failed (push succeeded)');
+    });
+
+    await db
+      .update(backupDestinations)
+      .set({
+        lastRunAt: new Date(),
+        lastStatus: 'ok',
+        lastError: null,
+        lastBackupBytes: bytes,
+      })
+      .where(eq(backupDestinations.id, id));
+
+    await db.insert(backupJobs).values({
+      status: 'completed',
+      trigger: opts.trigger,
+      triggeredBy: opts.triggeredBy ?? null,
+      sizeBytes: bytes,
+      storagePath: remoteRef,
+      completedAt: new Date(),
+    });
+
+    logger.info({ destinationId: id, bytes, remoteRef }, 'Backup pushed to destination');
+    return { bytes, remoteRef };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'unknown';
+    await db
+      .update(backupDestinations)
+      .set({ lastRunAt: new Date(), lastStatus: 'failed', lastError: message })
+      .where(eq(backupDestinations.id, id));
+    await db
+      .insert(backupJobs)
+      .values({
+        status: 'failed',
+        trigger: opts.trigger,
+        triggeredBy: opts.triggeredBy ?? null,
+        errorMessage: `[${row.name}] ${message}`,
+        completedAt: new Date(),
+      })
+      .catch(() => {});
+    await notifyBackupFailure(row.name, message, opts.trigger);
+    throw err;
+  } finally {
+    if (encPath) await unlink(encPath).catch(() => {});
+    if (ownTar) await ownTar();
+  }
+}
+
+/**
+ * Scheduled push: assemble the bundle ONCE and fan it out to every enabled
+ * destination. Per-destination failures are isolated (one bad server doesn't
+ * skip the others) and alerted.
+ */
+export async function runScheduledBackupPush(now = new Date()): Promise<{
+  ran: boolean;
+  pushed: number;
+  failed: number;
+}> {
+  const schedule = await getSchedule();
+  if (!isScheduleDue(schedule, now)) {
+    logger.info({ schedule }, 'Scheduled backup not due');
+    return { ran: false, pushed: 0, failed: 0 };
+  }
+
+  const enabled = await db.query.backupDestinations.findMany({
+    where: eq(backupDestinations.enabled, true),
+  });
+  if (enabled.length === 0) {
+    logger.warn('Backup schedule is on but no destinations are enabled');
+    return { ran: false, pushed: 0, failed: 0 };
+  }
+
+  const bundle = await createFullBackupTar();
+  let pushed = 0;
+  let failed = 0;
+  try {
+    for (const dest of enabled) {
+      try {
+        await pushBackupToDestination(dest.id, {
+          tarPath: bundle.tarPath,
+          filename: bundle.filename,
+          trigger: 'cron',
+        });
+        pushed += 1;
+      } catch (err) {
+        failed += 1;
+        logger.error({ err, destinationId: dest.id }, 'Scheduled push to destination failed');
+      }
+    }
+  } finally {
+    await bundle.cleanup();
+  }
+  logger.info({ pushed, failed, total: enabled.length }, 'Scheduled backup push complete');
+  return { ran: true, pushed, failed };
+}
+
+// ─── failure alerting ────────────────────────────────────────────────────
+
+async function notifyBackupFailure(
+  destinationName: string,
+  message: string,
+  trigger: 'manual' | 'cron',
+): Promise<void> {
+  // Guaranteed signal: an error-severity audit row (visible in /admin/audit).
+  await createAuditLog({
+    userId: null,
+    portId: null,
+    action: 'job_failed',
+    entityType: 'backup_destination',
+    entityId: destinationName,
+    severity: 'error',
+    source: trigger === 'cron' ? 'cron' : 'job',
+    metadata: { destination: destinationName, error: message },
+  });
+
+  // Best-effort: in-app system alert to every super-admin (per their port).
+  try {
+    const admins = await db
+      .select({ userId: userProfiles.userId, portId: userPortRoles.portId })
+      .from(userProfiles)
+      .innerJoin(userPortRoles, eq(userPortRoles.userId, userProfiles.userId))
+      .where(eq(userProfiles.isSuperAdmin, true));
+    const seen = new Set<string>();
+    for (const a of admins) {
+      const key = `${a.userId}:${a.portId}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      await createNotification({
+        portId: a.portId,
+        userId: a.userId,
+        type: 'system_alert',
+        title: 'Backup push failed',
+        description: `Backup to "${destinationName}" failed: ${message}`,
+        dedupeKey: `backup-fail:${destinationName}`,
+        cooldownMs: 60 * 60 * 1000,
+      });
+    }
+  } catch (err) {
+    logger.error({ err }, 'Failed to notify super-admins of backup failure');
+  }
+}