feat(backup): full DR bundle export + admin-configurable offsite destinations

Backend-agnostic disaster-recovery backup engine that runs on the current
storage backend (no storage cutover required):

- Full-bundle export: db.dump (pg_dump custom) + every storage blob +
  manifest.json with per-object SHA-256, streamed as a tar. Entry points:
  admin UI download, GET /api/v1/admin/backup/export, scripts/create-full-backup.ts.
- Admin-configurable push destinations (backup_destinations table, migration
  0091): SFTP/SSH, S3-compatible (reuses the minio client), and mounted
  path/NAS behind one transport interface (test/push/prune). Secrets AES-GCM
  at rest; API returns only *IsSet markers.
- Opt-in per-destination AES-256 bundle encryption (scrypt KDF, streamed) +
  scripts/decrypt-backup.ts for restore.
- Wired the previously-dead database-backup cron to runScheduledBackupPush
  (push to enabled destinations, prune to retention, alert super-admins on
  failure).

Tests: 1608 unit/integration pass; tsc + lint clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/unit/services/backup-bundle-encryption.test.ts

@@ -0,0 +1,75 @@
+/**
+ * Unit test for opt-in backup-bundle encryption
+ * (`src/lib/services/backup-destinations/bundle-encryption.ts`).
+ *
+ * Contract: AES-256-GCM streaming encryption with a scrypt-derived key.
+ * - round-trips arbitrary bytes (small + multi-chunk),
+ * - rejects the wrong passphrase (GCM auth failure),
+ * - rejects tampered ciphertext (GCM auth failure).
+ */
+
+import { createHash, randomBytes } from 'node:crypto';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import {
+  decryptFileToFile,
+  encryptFileToFile,
+} from '@/lib/services/backup-destinations/bundle-encryption';
+
+function sha(p: string): string {
+  return createHash('sha256').update(readFileSync(p)).digest('hex');
+}
+
+describe('backup bundle encryption', () => {
+  let dir: string;
+  beforeEach(() => {
+    dir = mkdtempSync(path.join(tmpdir(), 'pn-enc-'));
+  });
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('round-trips a multi-chunk file with the correct passphrase', async () => {
+    const plain = path.join(dir, 'bundle.tar');
+    // ~400 KB of pseudo-random bytes → multiple cipher chunks.
+    writeFileSync(plain, randomBytes(400 * 1024));
+    const enc = path.join(dir, 'bundle.tar.enc');
+    const dec = path.join(dir, 'bundle.roundtrip.tar');
+
+    await encryptFileToFile(plain, enc, 'correct horse battery staple');
+    // Ciphertext must differ from plaintext and not be empty.
+    expect(statSync(enc).size).toBeGreaterThan(0);
+    expect(sha(enc)).not.toBe(sha(plain));
+
+    await decryptFileToFile(enc, dec, 'correct horse battery staple');
+    expect(sha(dec)).toBe(sha(plain));
+  });
+
+  it('rejects the wrong passphrase', async () => {
+    const plain = path.join(dir, 'b.tar');
+    writeFileSync(plain, Buffer.from('top secret contract bytes'));
+    const enc = path.join(dir, 'b.tar.enc');
+    await encryptFileToFile(plain, enc, 'right-pass');
+
+    await expect(decryptFileToFile(enc, path.join(dir, 'out.tar'), 'WRONG-pass')).rejects.toThrow();
+  });
+
+  it('rejects tampered ciphertext', async () => {
+    const plain = path.join(dir, 'c.tar');
+    writeFileSync(plain, randomBytes(64 * 1024));
+    const enc = path.join(dir, 'c.tar.enc');
+    await encryptFileToFile(plain, enc, 'pw');
+
+    // Flip a byte in the middle of the ciphertext region.
+    const buf = readFileSync(enc);
+    const mid = Math.floor(buf.length / 2);
+    buf[mid] = (buf[mid] ?? 0) ^ 0xff;
+    writeFileSync(enc, buf);
+
+    await expect(decryptFileToFile(enc, path.join(dir, 'out.tar'), 'pw')).rejects.toThrow();
+  });
+});

tests/unit/services/backup-destinations-service.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Unit tests for the pure helpers of the backup-destinations service:
+ * schedule-due logic + secret config (serialize → encrypt at rest →
+ * decrypt for use, mask for API). DB-backed CRUD is covered by the e2e
+ * verification, not here.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  decryptConfig,
+  isScheduleDue,
+  maskConfig,
+  serializeConfig,
+} from '@/lib/services/backup-destinations.service';
+
+describe('isScheduleDue', () => {
+  // 2026-06-07 is a Sunday; 2026-06-08 a Monday.
+  const sunday = new Date('2026-06-07T02:00:00Z');
+  const monday = new Date('2026-06-08T02:00:00Z');
+
+  it('off is never due', () => {
+    expect(isScheduleDue('off', sunday)).toBe(false);
+    expect(isScheduleDue('off', monday)).toBe(false);
+  });
+  it('daily is always due', () => {
+    expect(isScheduleDue('daily', sunday)).toBe(true);
+    expect(isScheduleDue('daily', monday)).toBe(true);
+  });
+  it('weekly is due only on Sunday', () => {
+    expect(isScheduleDue('weekly', sunday)).toBe(true);
+    expect(isScheduleDue('weekly', monday)).toBe(false);
+  });
+});
+
+describe('secret config handling', () => {
+  it('serialize encrypts secrets, decrypt restores them, mask hides them', () => {
+    const incoming = {
+      host: 'box.example.com',
+      username: 'crm',
+      password: 'hunter2',
+      remoteDir: '/backups',
+    };
+    const stored = serializeConfig('sftp', incoming);
+
+    // Stored password must not be the plaintext.
+    expect(stored.password).not.toBe('hunter2');
+    expect(stored.host).toBe('box.example.com');
+
+    // Decrypt restores the plaintext for transport use.
+    expect(decryptConfig('sftp', stored)).toMatchObject({
+      host: 'box.example.com',
+      username: 'crm',
+      password: 'hunter2',
+      remoteDir: '/backups',
+    });
+
+    // Mask hides the secret and exposes only a *IsSet marker.
+    const masked = maskConfig('sftp', stored);
+    expect(masked.password).toBeUndefined();
+    expect(masked.passwordIsSet).toBe(true);
+    expect(masked.host).toBe('box.example.com');
+  });
+
+  it('update with a blank secret keeps the existing encrypted value', () => {
+    const original = serializeConfig('s3', {
+      endpoint: 'https://s3.example.com',
+      bucket: 'b',
+      accessKey: 'AK',
+      secretKey: 'SUPERSECRET',
+    });
+    // Admin edits the bucket but leaves the secret key blank (unchanged).
+    const updated = serializeConfig(
+      's3',
+      { endpoint: 'https://s3.example.com', bucket: 'b2', accessKey: 'AK', secretKey: '' },
+      original,
+    );
+    expect(updated.bucket).toBe('b2');
+    expect(decryptConfig('s3', updated).secretKey).toBe('SUPERSECRET');
+  });
+
+  it('filesystem has no secrets to mask', () => {
+    const stored = serializeConfig('filesystem', { directory: '/mnt/nas/backups' });
+    expect(maskConfig('filesystem', stored)).toEqual({ directory: '/mnt/nas/backups' });
+  });
+});

tests/unit/services/backup-filesystem-transport.test.ts

@@ -0,0 +1,72 @@
+/**
+ * Unit test for the filesystem (mounted-path / NAS) backup transport
+ * (`src/lib/services/backup-destinations/filesystem.ts`).
+ */
+
+import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import { FilesystemTransport } from '@/lib/services/backup-destinations/filesystem';
+import { BACKUP_NAME_PREFIX } from '@/lib/services/backup-destinations/types';
+
+describe('FilesystemTransport', () => {
+  let work: string;
+  let destDir: string;
+  beforeEach(() => {
+    work = mkdtempSync(path.join(tmpdir(), 'pn-fstr-'));
+    destDir = path.join(work, 'backups');
+    mkdirSync(destDir);
+  });
+  afterEach(() => {
+    rmSync(work, { recursive: true, force: true });
+  });
+
+  it('test() rejects when the directory does not exist', async () => {
+    const t = new FilesystemTransport({ directory: path.join(work, 'nope') });
+    await expect(t.test()).rejects.toThrow();
+  });
+
+  it('push() copies the bundle into the destination directory', async () => {
+    const local = path.join(work, `${BACKUP_NAME_PREFIX}2026-06-04.tar`);
+    writeFileSync(local, Buffer.from('BUNDLE-BYTES'));
+    const t = new FilesystemTransport({ directory: destDir });
+
+    const res = await t.push(local, `${BACKUP_NAME_PREFIX}2026-06-04.tar`);
+
+    expect(res.bytes).toBe('BUNDLE-BYTES'.length);
+    const landed = readFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-04.tar`), 'utf8');
+    expect(landed).toBe('BUNDLE-BYTES');
+  });
+
+  it('prune() keeps the N newest bundles and ignores unrelated files', async () => {
+    // Five backups (timestamp-in-name sorts chronologically) + an unrelated file.
+    for (const d of ['01', '02', '03', '04', '05']) {
+      writeFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-${d}.tar`), 'x');
+    }
+    writeFileSync(path.join(destDir, 'unrelated-keepme.txt'), 'y');
+
+    const t = new FilesystemTransport({ directory: destDir });
+    const { deleted } = await t.prune(2);
+
+    expect(deleted).toBe(3);
+    const remaining = readdirSync(destDir).sort();
+    expect(remaining).toEqual([
+      `${BACKUP_NAME_PREFIX}2026-06-04.tar`,
+      `${BACKUP_NAME_PREFIX}2026-06-05.tar`,
+      'unrelated-keepme.txt',
+    ]);
+  });
+
+  it('prune(null) keeps everything', async () => {
+    for (const d of ['01', '02', '03']) {
+      writeFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-${d}.tar`), 'x');
+    }
+    const t = new FilesystemTransport({ directory: destDir });
+    const { deleted } = await t.prune(null);
+    expect(deleted).toBe(0);
+    expect(readdirSync(destDir).length).toBe(3);
+  });
+});

tests/unit/services/backup-transport-factory.test.ts

@@ -0,0 +1,63 @@
+/**
+ * Unit test for the transport factory + S3 endpoint parser
+ * (`src/lib/services/backup-destinations/`).
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  buildTransport,
+  FilesystemTransport,
+  parseS3Endpoint,
+  S3Transport,
+  SftpTransport,
+} from '@/lib/services/backup-destinations';
+
+describe('buildTransport', () => {
+  it('builds the right transport per type', () => {
+    expect(buildTransport('filesystem', { directory: '/x' })).toBeInstanceOf(FilesystemTransport);
+    expect(buildTransport('sftp', { host: 'h', username: 'u', remoteDir: '/d' })).toBeInstanceOf(
+      SftpTransport,
+    );
+    expect(
+      buildTransport('s3', { endpoint: 'h', bucket: 'b', accessKey: 'a', secretKey: 's' }),
+    ).toBeInstanceOf(S3Transport);
+  });
+
+  it('throws on an unknown type', () => {
+    // @ts-expect-error testing the runtime guard
+    expect(() => buildTransport('ftp', {})).toThrow(/Unknown backup destination type/);
+  });
+});
+
+describe('parseS3Endpoint', () => {
+  it('parses an https URL into host + ssl', () => {
+    expect(parseS3Endpoint('https://s3.eu-central.example.com', {})).toEqual({
+      endPoint: 's3.eu-central.example.com',
+      useSSL: true,
+    });
+  });
+
+  it('parses an http URL with a port', () => {
+    expect(parseS3Endpoint('http://localhost:9000', {})).toEqual({
+      endPoint: 'localhost',
+      port: 9000,
+      useSSL: false,
+    });
+  });
+
+  it('treats a bare host as ssl-by-default', () => {
+    expect(parseS3Endpoint('s3.amazonaws.com', {})).toEqual({
+      endPoint: 's3.amazonaws.com',
+      useSSL: true,
+    });
+  });
+
+  it('honours explicit useSSL=false on a bare host', () => {
+    expect(parseS3Endpoint('minio.internal', { useSSL: false, port: 9000 })).toEqual({
+      endPoint: 'minio.internal',
+      port: 9000,
+      useSSL: false,
+    });
+  });
+});

tests/unit/services/pg-dump-runner.test.ts

@@ -0,0 +1,63 @@
+/**
+ * Unit test for `runPgDump` in backup.service.ts.
+ *
+ * Regression: the dump promise must resolve once BOTH (a) the child process
+ * exits 0 and (b) the output file is fully flushed — regardless of which event
+ * fires first. The original implementation attached the file's `finish`
+ * listener *inside* the child `close` handler, but `stdout.pipe(out)`
+ * auto-ends the file when the child's stdout closes, so `finish` frequently
+ * fired before the listener was attached → the promise hung forever (observed
+ * end-to-end against a real pg_dump).
+ *
+ * We drive the spawn with `node` instead of `pg_dump` (injected via opts) so
+ * the test is deterministic and needs no database.
+ */
+
+import { readFileSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import { runPgDump } from '@/lib/services/backup.service';
+
+describe('runPgDump', () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(path.join(tmpdir(), 'pn-pgdump-'));
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('resolves and writes the child stdout to the output file on exit 0', async () => {
+    const out = path.join(dir, 'a.dump');
+    await runPgDump('ignored://url', out, {
+      command: process.execPath,
+      buildArgs: () => ['-e', 'process.stdout.write("DUMP-CONTENTS-1234")'],
+    });
+    expect(readFileSync(out, 'utf8')).toBe('DUMP-CONTENTS-1234');
+  });
+
+  it('rejects with the captured stderr when the child exits non-zero', async () => {
+    const out = path.join(dir, 'b.dump');
+    await expect(
+      runPgDump('ignored://url', out, {
+        command: process.execPath,
+        buildArgs: () => ['-e', 'process.stderr.write("kaboom"); process.exit(3)'],
+      }),
+    ).rejects.toThrow(/exited 3.*kaboom/s);
+  });
+
+  it('rejects when the command cannot be spawned', async () => {
+    const out = path.join(dir, 'c.dump');
+    await expect(
+      runPgDump('ignored://url', out, {
+        command: '/nonexistent/definitely-not-a-real-binary-xyz',
+        buildArgs: () => [],
+      }),
+    ).rejects.toBeTruthy();
+  });
+});