feat(backup): full DR bundle export + admin-configurable offsite destinations
All checks were successful
Build & Push Docker Images / lint (push) Successful in 2m52s
Build & Push Docker Images / build-and-push (push) Successful in 11m59s

Backend-agnostic disaster-recovery backup engine that runs on the current
storage backend (no storage cutover required):

- Full-bundle export: db.dump (pg_dump custom) + every storage blob +
  manifest.json with per-object SHA-256, streamed as a tar. Entry points:
  admin UI download, GET /api/v1/admin/backup/export, scripts/create-full-backup.ts.
- Admin-configurable push destinations (backup_destinations table, migration
  0091): SFTP/SSH, S3-compatible (reuses the minio client), and mounted
  path/NAS behind one transport interface (test/push/prune). Secrets AES-GCM
  at rest; API returns only *IsSet markers.
- Opt-in per-destination AES-256 bundle encryption (scrypt KDF, streamed) +
  scripts/decrypt-backup.ts for restore.
- Wired the previously-dead database-backup cron to runScheduledBackupPush
  (push to enabled destinations, prune to retention, alert super-admins on
  failure).

Tests: 1608 unit/integration pass; tsc + lint clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-04 11:23:42 +02:00
parent 05950ae0b6
commit fe863a588e
35 changed files with 3125 additions and 15 deletions

View File

@@ -0,0 +1,75 @@
/**
* Unit test for opt-in backup-bundle encryption
* (`src/lib/services/backup-destinations/bundle-encryption.ts`).
*
* Contract: AES-256-GCM streaming encryption with a scrypt-derived key.
* - round-trips arbitrary bytes (small + multi-chunk),
* - rejects the wrong passphrase (GCM auth failure),
* - rejects tampered ciphertext (GCM auth failure).
*/
import { createHash, randomBytes } from 'node:crypto';
import { mkdtempSync, readFileSync, rmSync, writeFileSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
decryptFileToFile,
encryptFileToFile,
} from '@/lib/services/backup-destinations/bundle-encryption';
function sha(p: string): string {
return createHash('sha256').update(readFileSync(p)).digest('hex');
}
describe('backup bundle encryption', () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(path.join(tmpdir(), 'pn-enc-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it('round-trips a multi-chunk file with the correct passphrase', async () => {
const plain = path.join(dir, 'bundle.tar');
// ~400 KB of pseudo-random bytes → multiple cipher chunks.
writeFileSync(plain, randomBytes(400 * 1024));
const enc = path.join(dir, 'bundle.tar.enc');
const dec = path.join(dir, 'bundle.roundtrip.tar');
await encryptFileToFile(plain, enc, 'correct horse battery staple');
// Ciphertext must differ from plaintext and not be empty.
expect(statSync(enc).size).toBeGreaterThan(0);
expect(sha(enc)).not.toBe(sha(plain));
await decryptFileToFile(enc, dec, 'correct horse battery staple');
expect(sha(dec)).toBe(sha(plain));
});
it('rejects the wrong passphrase', async () => {
const plain = path.join(dir, 'b.tar');
writeFileSync(plain, Buffer.from('top secret contract bytes'));
const enc = path.join(dir, 'b.tar.enc');
await encryptFileToFile(plain, enc, 'right-pass');
await expect(decryptFileToFile(enc, path.join(dir, 'out.tar'), 'WRONG-pass')).rejects.toThrow();
});
it('rejects tampered ciphertext', async () => {
const plain = path.join(dir, 'c.tar');
writeFileSync(plain, randomBytes(64 * 1024));
const enc = path.join(dir, 'c.tar.enc');
await encryptFileToFile(plain, enc, 'pw');
// Flip a byte in the middle of the ciphertext region.
const buf = readFileSync(enc);
const mid = Math.floor(buf.length / 2);
buf[mid] = (buf[mid] ?? 0) ^ 0xff;
writeFileSync(enc, buf);
await expect(decryptFileToFile(enc, path.join(dir, 'out.tar'), 'pw')).rejects.toThrow();
});
});

View File

@@ -0,0 +1,86 @@
/**
* Unit tests for the pure helpers of the backup-destinations service:
* schedule-due logic + secret config (serialize → encrypt at rest →
* decrypt for use, mask for API). DB-backed CRUD is covered by the e2e
* verification, not here.
*/
import { describe, expect, it } from 'vitest';
import {
decryptConfig,
isScheduleDue,
maskConfig,
serializeConfig,
} from '@/lib/services/backup-destinations.service';
describe('isScheduleDue', () => {
// 2026-06-07 is a Sunday; 2026-06-08 a Monday.
const sunday = new Date('2026-06-07T02:00:00Z');
const monday = new Date('2026-06-08T02:00:00Z');
it('off is never due', () => {
expect(isScheduleDue('off', sunday)).toBe(false);
expect(isScheduleDue('off', monday)).toBe(false);
});
it('daily is always due', () => {
expect(isScheduleDue('daily', sunday)).toBe(true);
expect(isScheduleDue('daily', monday)).toBe(true);
});
it('weekly is due only on Sunday', () => {
expect(isScheduleDue('weekly', sunday)).toBe(true);
expect(isScheduleDue('weekly', monday)).toBe(false);
});
});
describe('secret config handling', () => {
it('serialize encrypts secrets, decrypt restores them, mask hides them', () => {
const incoming = {
host: 'box.example.com',
username: 'crm',
password: 'hunter2',
remoteDir: '/backups',
};
const stored = serializeConfig('sftp', incoming);
// Stored password must not be the plaintext.
expect(stored.password).not.toBe('hunter2');
expect(stored.host).toBe('box.example.com');
// Decrypt restores the plaintext for transport use.
expect(decryptConfig('sftp', stored)).toMatchObject({
host: 'box.example.com',
username: 'crm',
password: 'hunter2',
remoteDir: '/backups',
});
// Mask hides the secret and exposes only a *IsSet marker.
const masked = maskConfig('sftp', stored);
expect(masked.password).toBeUndefined();
expect(masked.passwordIsSet).toBe(true);
expect(masked.host).toBe('box.example.com');
});
it('update with a blank secret keeps the existing encrypted value', () => {
const original = serializeConfig('s3', {
endpoint: 'https://s3.example.com',
bucket: 'b',
accessKey: 'AK',
secretKey: 'SUPERSECRET',
});
// Admin edits the bucket but leaves the secret key blank (unchanged).
const updated = serializeConfig(
's3',
{ endpoint: 'https://s3.example.com', bucket: 'b2', accessKey: 'AK', secretKey: '' },
original,
);
expect(updated.bucket).toBe('b2');
expect(decryptConfig('s3', updated).secretKey).toBe('SUPERSECRET');
});
it('filesystem has no secrets to mask', () => {
const stored = serializeConfig('filesystem', { directory: '/mnt/nas/backups' });
expect(maskConfig('filesystem', stored)).toEqual({ directory: '/mnt/nas/backups' });
});
});

View File

@@ -0,0 +1,72 @@
/**
* Unit test for the filesystem (mounted-path / NAS) backup transport
* (`src/lib/services/backup-destinations/filesystem.ts`).
*/
import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { FilesystemTransport } from '@/lib/services/backup-destinations/filesystem';
import { BACKUP_NAME_PREFIX } from '@/lib/services/backup-destinations/types';
describe('FilesystemTransport', () => {
let work: string;
let destDir: string;
beforeEach(() => {
work = mkdtempSync(path.join(tmpdir(), 'pn-fstr-'));
destDir = path.join(work, 'backups');
mkdirSync(destDir);
});
afterEach(() => {
rmSync(work, { recursive: true, force: true });
});
it('test() rejects when the directory does not exist', async () => {
const t = new FilesystemTransport({ directory: path.join(work, 'nope') });
await expect(t.test()).rejects.toThrow();
});
it('push() copies the bundle into the destination directory', async () => {
const local = path.join(work, `${BACKUP_NAME_PREFIX}2026-06-04.tar`);
writeFileSync(local, Buffer.from('BUNDLE-BYTES'));
const t = new FilesystemTransport({ directory: destDir });
const res = await t.push(local, `${BACKUP_NAME_PREFIX}2026-06-04.tar`);
expect(res.bytes).toBe('BUNDLE-BYTES'.length);
const landed = readFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-04.tar`), 'utf8');
expect(landed).toBe('BUNDLE-BYTES');
});
it('prune() keeps the N newest bundles and ignores unrelated files', async () => {
// Five backups (timestamp-in-name sorts chronologically) + an unrelated file.
for (const d of ['01', '02', '03', '04', '05']) {
writeFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-${d}.tar`), 'x');
}
writeFileSync(path.join(destDir, 'unrelated-keepme.txt'), 'y');
const t = new FilesystemTransport({ directory: destDir });
const { deleted } = await t.prune(2);
expect(deleted).toBe(3);
const remaining = readdirSync(destDir).sort();
expect(remaining).toEqual([
`${BACKUP_NAME_PREFIX}2026-06-04.tar`,
`${BACKUP_NAME_PREFIX}2026-06-05.tar`,
'unrelated-keepme.txt',
]);
});
it('prune(null) keeps everything', async () => {
for (const d of ['01', '02', '03']) {
writeFileSync(path.join(destDir, `${BACKUP_NAME_PREFIX}2026-06-${d}.tar`), 'x');
}
const t = new FilesystemTransport({ directory: destDir });
const { deleted } = await t.prune(null);
expect(deleted).toBe(0);
expect(readdirSync(destDir).length).toBe(3);
});
});

View File

@@ -0,0 +1,63 @@
/**
* Unit test for the transport factory + S3 endpoint parser
* (`src/lib/services/backup-destinations/`).
*/
import { describe, expect, it } from 'vitest';
import {
buildTransport,
FilesystemTransport,
parseS3Endpoint,
S3Transport,
SftpTransport,
} from '@/lib/services/backup-destinations';
describe('buildTransport', () => {
it('builds the right transport per type', () => {
expect(buildTransport('filesystem', { directory: '/x' })).toBeInstanceOf(FilesystemTransport);
expect(buildTransport('sftp', { host: 'h', username: 'u', remoteDir: '/d' })).toBeInstanceOf(
SftpTransport,
);
expect(
buildTransport('s3', { endpoint: 'h', bucket: 'b', accessKey: 'a', secretKey: 's' }),
).toBeInstanceOf(S3Transport);
});
it('throws on an unknown type', () => {
// @ts-expect-error testing the runtime guard
expect(() => buildTransport('ftp', {})).toThrow(/Unknown backup destination type/);
});
});
describe('parseS3Endpoint', () => {
it('parses an https URL into host + ssl', () => {
expect(parseS3Endpoint('https://s3.eu-central.example.com', {})).toEqual({
endPoint: 's3.eu-central.example.com',
useSSL: true,
});
});
it('parses an http URL with a port', () => {
expect(parseS3Endpoint('http://localhost:9000', {})).toEqual({
endPoint: 'localhost',
port: 9000,
useSSL: false,
});
});
it('treats a bare host as ssl-by-default', () => {
expect(parseS3Endpoint('s3.amazonaws.com', {})).toEqual({
endPoint: 's3.amazonaws.com',
useSSL: true,
});
});
it('honours explicit useSSL=false on a bare host', () => {
expect(parseS3Endpoint('minio.internal', { useSSL: false, port: 9000 })).toEqual({
endPoint: 'minio.internal',
port: 9000,
useSSL: false,
});
});
});

View File

@@ -0,0 +1,63 @@
/**
* Unit test for `runPgDump` in backup.service.ts.
*
* Regression: the dump promise must resolve once BOTH (a) the child process
* exits 0 and (b) the output file is fully flushed — regardless of which event
* fires first. The original implementation attached the file's `finish`
* listener *inside* the child `close` handler, but `stdout.pipe(out)`
* auto-ends the file when the child's stdout closes, so `finish` frequently
* fired before the listener was attached → the promise hung forever (observed
* end-to-end against a real pg_dump).
*
* We drive the spawn with `node` instead of `pg_dump` (injected via opts) so
* the test is deterministic and needs no database.
*/
import { readFileSync, mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { runPgDump } from '@/lib/services/backup.service';
describe('runPgDump', () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(path.join(tmpdir(), 'pn-pgdump-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it('resolves and writes the child stdout to the output file on exit 0', async () => {
const out = path.join(dir, 'a.dump');
await runPgDump('ignored://url', out, {
command: process.execPath,
buildArgs: () => ['-e', 'process.stdout.write("DUMP-CONTENTS-1234")'],
});
expect(readFileSync(out, 'utf8')).toBe('DUMP-CONTENTS-1234');
});
it('rejects with the captured stderr when the child exits non-zero', async () => {
const out = path.join(dir, 'b.dump');
await expect(
runPgDump('ignored://url', out, {
command: process.execPath,
buildArgs: () => ['-e', 'process.stderr.write("kaboom"); process.exit(3)'],
}),
).rejects.toThrow(/exited 3.*kaboom/s);
});
it('rejects when the command cannot be spawned', async () => {
const out = path.join(dir, 'c.dump');
await expect(
runPgDump('ignored://url', out, {
command: '/nonexistent/definitely-not-a-real-binary-xyz',
buildArgs: () => [],
}),
).rejects.toBeTruthy();
});
});

View File

@@ -0,0 +1,208 @@
/**
* Unit test for the full-bundle backup tar assembler
* (`assembleBackupTar` in `src/lib/services/backup-export.service.ts`).
*
* Phase 4a of docs/storage-migration-and-backup-plan.md: assemble a single
* tar containing `db.dump` (a pre-produced pg_dump file) + `blobs/<key>` for
* every blob-bearing row + a `manifest.json` describing the bundle with a
* sha256 per object so a restore can verify integrity.
*
* Uses an in-memory storage backend (no MinIO) and a synthetic dump file
* (no pg_dump). The produced tar is read back with the system `tar` CLI so we
* assert against the real archive bytes, not archiver internals.
*/
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { mkdtempSync, readFileSync, rmSync, writeFileSync, readdirSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { Readable } from 'node:stream';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { assembleBackupTar } from '@/lib/services/backup-export.service';
import type { PresignOpts, PutOpts, StorageBackend } from '@/lib/storage';
class InMemoryBackend implements StorageBackend {
readonly name = 's3' as const;
readonly store = new Map<string, { body: Buffer; contentType: string }>();
async put(
key: string,
body: Buffer | NodeJS.ReadableStream,
opts: PutOpts,
): Promise<{ key: string; sizeBytes: number; sha256: string }> {
const buffer = Buffer.isBuffer(body) ? body : await streamToBuffer(body);
this.store.set(key, { body: buffer, contentType: opts.contentType });
return {
key,
sizeBytes: buffer.length,
sha256: createHash('sha256').update(buffer).digest('hex'),
};
}
async get(key: string): Promise<NodeJS.ReadableStream> {
const r = this.store.get(key);
if (!r) throw new Error(`not found: ${key}`);
// Chunk the body so multi-chunk streaming is exercised.
const chunks: Buffer[] = [];
for (let i = 0; i < r.body.length; i += 64 * 1024) {
chunks.push(r.body.subarray(i, i + 64 * 1024));
}
return Readable.from(chunks.length ? chunks : [Buffer.alloc(0)]);
}
async head(key: string) {
const r = this.store.get(key);
if (!r) return null;
return { sizeBytes: r.body.length, contentType: r.contentType };
}
async delete(key: string): Promise<void> {
this.store.delete(key);
}
async listByPrefix(prefix: string): Promise<string[]> {
return [...this.store.keys()].filter((k) => k.startsWith(prefix));
}
async presignUpload(_key: string, _opts: PresignOpts) {
return { url: 'mem://upload', method: 'PUT' as const };
}
async presignDownload(_key: string, _opts: PresignOpts) {
return { url: 'mem://download', expiresAt: new Date(Date.now() + 1000) };
}
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const c of stream) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c as string));
return Buffer.concat(chunks);
}
function sha256(buf: Buffer): string {
return createHash('sha256').update(buf).digest('hex');
}
describe('assembleBackupTar', () => {
let workDir: string;
beforeEach(() => {
workDir = mkdtempSync(path.join(tmpdir(), 'pn-backup-test-'));
});
afterEach(() => {
rmSync(workDir, { recursive: true, force: true });
});
it('bundles the db dump + every blob with a verifiable manifest', async () => {
const backend = new InMemoryBackend();
// A small blob and a larger multi-chunk blob (exercises streaming).
const smallBody = Buffer.from('the quick brown fox');
const bigBody = Buffer.alloc(300 * 1024);
for (let i = 0; i < bigBody.length; i++) bigBody[i] = (i * 7) % 256;
await backend.put('port-nimara/files/a.bin', smallBody, { contentType: 'application/pdf' });
await backend.put('port-nimara/berths/big.pdf', bigBody, { contentType: 'application/pdf' });
// Synthetic pg_dump file.
const dumpBody = Buffer.from('PGDMP-FAKE-CUSTOM-FORMAT-CONTENTS\n'.repeat(1000));
const dumpPath = path.join(workDir, 'db.dump');
writeFileSync(dumpPath, dumpBody);
const outPath = path.join(workDir, 'bundle.tar');
const now = new Date('2026-06-04T12:00:00.000Z');
const manifest = await assembleBackupTar({
backend,
dumpFilePath: dumpPath,
blobRefs: [
{ tableName: 'files', pk: 'f1', key: 'port-nimara/files/a.bin' },
{ tableName: 'berth_pdf_versions', pk: 'b1', key: 'port-nimara/berths/big.pdf' },
],
outFilePath: outPath,
storageBackendName: 's3',
now,
});
// ── manifest shape ──────────────────────────────────────────────────────
expect(manifest.formatVersion).toBe(1);
expect(manifest.createdAt).toBe('2026-06-04T12:00:00.000Z');
expect(manifest.storageBackend).toBe('s3');
expect(manifest.database.file).toBe('db.dump');
expect(manifest.database.sizeBytes).toBe(dumpBody.length);
expect(manifest.database.sha256).toBe(sha256(dumpBody));
expect(manifest.counts.blobs).toBe(2);
expect(manifest.counts.blobBytes).toBe(smallBody.length + bigBody.length);
expect(manifest.counts.skipped).toBe(0);
const smallEntry = manifest.blobs.find((b) => b.key === 'port-nimara/files/a.bin');
expect(smallEntry).toMatchObject({
table: 'files',
pk: 'f1',
sizeBytes: smallBody.length,
sha256: sha256(smallBody),
});
// ── extract the real tar and verify bytes ────────────────────────────────
const extractDir = path.join(workDir, 'extract');
execFileSync('mkdir', ['-p', extractDir]);
execFileSync('tar', ['-xf', outPath, '-C', extractDir]);
const extractedDump = readFileSync(path.join(extractDir, 'db.dump'));
expect(extractedDump.equals(dumpBody)).toBe(true);
const extractedSmall = readFileSync(path.join(extractDir, 'blobs/port-nimara/files/a.bin'));
expect(extractedSmall.equals(smallBody)).toBe(true);
const extractedBig = readFileSync(path.join(extractDir, 'blobs/port-nimara/berths/big.pdf'));
expect(extractedBig.equals(bigBody)).toBe(true);
// The on-disk manifest matches the returned one and verifies the bytes.
const onDiskManifest = JSON.parse(readFileSync(path.join(extractDir, 'manifest.json'), 'utf8'));
expect(onDiskManifest).toEqual(manifest);
for (const entry of manifest.blobs) {
const bytes = readFileSync(path.join(extractDir, 'blobs', entry.key));
expect(sha256(bytes)).toBe(entry.sha256);
expect(bytes.length).toBe(entry.sizeBytes);
}
});
it('records referenced-but-missing blobs as skipped instead of failing', async () => {
const backend = new InMemoryBackend();
await backend.put('port-nimara/files/present.bin', Buffer.from('here'), {
contentType: 'application/octet-stream',
});
const dumpPath = path.join(workDir, 'db.dump');
writeFileSync(dumpPath, Buffer.from('dump'));
const outPath = path.join(workDir, 'bundle.tar');
const manifest = await assembleBackupTar({
backend,
dumpFilePath: dumpPath,
blobRefs: [
{ tableName: 'files', pk: 'f1', key: 'port-nimara/files/present.bin' },
{ tableName: 'files', pk: 'f2', key: 'port-nimara/files/GONE.bin' },
],
outFilePath: outPath,
storageBackendName: 's3',
now: new Date('2026-06-04T12:00:00.000Z'),
});
expect(manifest.counts.blobs).toBe(1);
expect(manifest.counts.skipped).toBe(1);
expect(manifest.skipped).toEqual([
expect.objectContaining({ table: 'files', pk: 'f2', key: 'port-nimara/files/GONE.bin' }),
]);
// The missing blob must NOT appear in the archive.
const extractDir = path.join(workDir, 'extract');
execFileSync('mkdir', ['-p', extractDir]);
execFileSync('tar', ['-xf', outPath, '-C', extractDir]);
const blobFiles = readdirSync(path.join(extractDir, 'blobs', 'port-nimara', 'files'));
expect(blobFiles).toEqual(['present.bin']);
});
});