feat(backup): full DR bundle export + admin-configurable offsite destinations
Backend-agnostic disaster-recovery backup engine that runs on the current storage backend (no storage cutover required): - Full-bundle export: db.dump (pg_dump custom) + every storage blob + manifest.json with per-object SHA-256, streamed as a tar. Entry points: admin UI download, GET /api/v1/admin/backup/export, scripts/create-full-backup.ts. - Admin-configurable push destinations (backup_destinations table, migration 0091): SFTP/SSH, S3-compatible (reuses the minio client), and mounted path/NAS behind one transport interface (test/push/prune). Secrets AES-GCM at rest; API returns only *IsSet markers. - Opt-in per-destination AES-256 bundle encryption (scrypt KDF, streamed) + scripts/decrypt-backup.ts for restore. - Wired the previously-dead database-backup cron to runScheduledBackupPush (push to enabled destinations, prune to retention, alert super-admins on failure). Tests: 1608 unit/integration pass; tsc + lint clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,10 @@ export type AuditAction =
|
||||
// Branding (port logo upload pipeline).
|
||||
| 'branding.logo.uploaded'
|
||||
| 'branding.logo.archived'
|
||||
// Full-bundle backup export (DB dump + every blob) downloaded by an
|
||||
// operator. A cross-tenant data egress — logged at warning severity so the
|
||||
// audit filter surfaces it distinctly from routine reads.
|
||||
| 'backup_export'
|
||||
// System / background events.
|
||||
| 'webhook_delivered'
|
||||
| 'webhook_failed'
|
||||
|
||||
23
src/lib/db/migrations/0091_backup_destinations.sql
Normal file
23
src/lib/db/migrations/0091_backup_destinations.sql
Normal file
@@ -0,0 +1,23 @@
|
||||
-- Admin-configurable backup destinations (Phase 4b).
|
||||
-- Each row is a place scheduled/manual full-bundle backups are pushed to.
|
||||
-- Secrets inside `config` are AES-GCM-encrypted by the application layer.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "backup_destinations" (
|
||||
"id" text PRIMARY KEY NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"type" text NOT NULL,
|
||||
"enabled" boolean DEFAULT false NOT NULL,
|
||||
"config" jsonb DEFAULT '{}'::jsonb NOT NULL,
|
||||
"retention_count" integer,
|
||||
"encrypt_bundle" boolean DEFAULT false NOT NULL,
|
||||
"encryption_key_encrypted" text,
|
||||
"last_run_at" timestamptz,
|
||||
"last_status" text,
|
||||
"last_error" text,
|
||||
"last_backup_bytes" bigint,
|
||||
"created_at" timestamptz DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamptz DEFAULT now() NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_backup_destinations_enabled"
|
||||
ON "backup_destinations" ("enabled");
|
||||
@@ -371,3 +371,43 @@ export const backupJobs = pgTable(
|
||||
|
||||
export type BackupJob = typeof backupJobs.$inferSelect;
|
||||
export type NewBackupJob = typeof backupJobs.$inferInsert;
|
||||
|
||||
/**
|
||||
* Admin-configurable destinations that scheduled/manual backups are pushed to.
|
||||
* Each row transports the exact full-bundle tar produced by
|
||||
* `createFullBackupTar()` (db.dump + blobs + manifest) — see
|
||||
* docs/superpowers/specs/2026-06-04-backup-destinations-design.md.
|
||||
*
|
||||
* `config` holds the type-specific connection settings; any secret inside it
|
||||
* (SFTP password / private key, S3 secret key) is AES-GCM-encrypted via
|
||||
* `@/lib/utils/encryption` before storage and never returned raw (the API
|
||||
* surfaces only `*IsSet` markers, mirroring the send-from-accounts pattern).
|
||||
*/
|
||||
export const backupDestinations = pgTable(
|
||||
'backup_destinations',
|
||||
{
|
||||
id: text('id')
|
||||
.primaryKey()
|
||||
.$defaultFn(() => crypto.randomUUID()),
|
||||
name: text('name').notNull(),
|
||||
type: text('type').notNull(), // 'sftp' | 's3' | 'filesystem'
|
||||
enabled: boolean('enabled').notNull().default(false),
|
||||
config: jsonb('config').notNull().default({}),
|
||||
/** Keep last N bundles at this destination; null = keep all. */
|
||||
retentionCount: integer('retention_count'),
|
||||
/** Opt-in client-side AES-256 encryption of the bundle before push. */
|
||||
encryptBundle: boolean('encrypt_bundle').notNull().default(false),
|
||||
/** The bundle passphrase, itself AES-GCM-encrypted at rest. */
|
||||
encryptionKeyEncrypted: text('encryption_key_encrypted'),
|
||||
lastRunAt: timestamp('last_run_at', { withTimezone: true }),
|
||||
lastStatus: text('last_status'), // 'ok' | 'failed'
|
||||
lastError: text('last_error'),
|
||||
lastBackupBytes: bigint('last_backup_bytes', { mode: 'number' }),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(table) => [index('idx_backup_destinations_enabled').on(table.enabled)],
|
||||
);
|
||||
|
||||
export type BackupDestination = typeof backupDestinations.$inferSelect;
|
||||
export type NewBackupDestination = typeof backupDestinations.$inferInsert;
|
||||
|
||||
@@ -38,6 +38,17 @@ export const maintenanceWorker = new Worker(
|
||||
await refreshRates();
|
||||
break;
|
||||
}
|
||||
case 'database-backup': {
|
||||
// Scheduled full-bundle backup pushed to every enabled destination.
|
||||
// No-op until an admin turns the schedule on AND enables a destination
|
||||
// (`backup_schedule` setting + `backup_destinations`). Replaces the
|
||||
// previous silent no-op (this case did not exist before).
|
||||
const { runScheduledBackupPush } =
|
||||
await import('@/lib/services/backup-destinations.service');
|
||||
const summary = await runScheduledBackupPush();
|
||||
logger.info(summary, 'Scheduled backup push complete');
|
||||
break;
|
||||
}
|
||||
case 'form-expiry-check': {
|
||||
const result = await db
|
||||
.update(formSubmissions)
|
||||
|
||||
458
src/lib/services/backup-destinations.service.ts
Normal file
458
src/lib/services/backup-destinations.service.ts
Normal file
@@ -0,0 +1,458 @@
|
||||
/**
|
||||
* Admin-configurable backup destinations — service layer.
|
||||
* See docs/superpowers/specs/2026-06-04-backup-destinations-design.md.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - CRUD over `backup_destinations` with secret encryption at rest + masking on
|
||||
* read (mirrors the send-from-accounts pattern: API returns only `*IsSet`).
|
||||
* - test / manual-push / prune to a destination.
|
||||
* - scheduled push to all enabled destinations, with failure alerting.
|
||||
*
|
||||
* Every push transports the exact SHA-verified tar from `createFullBackupTar()`
|
||||
* — the same bundle admins download — optionally AES-256 encrypted first.
|
||||
*/
|
||||
|
||||
import { unlink } from 'node:fs/promises';
|
||||
|
||||
import { and, eq, isNull } from 'drizzle-orm';
|
||||
|
||||
import { createAuditLog } from '@/lib/audit';
|
||||
import { db } from '@/lib/db';
|
||||
import {
|
||||
backupDestinations,
|
||||
backupJobs,
|
||||
systemSettings,
|
||||
type BackupDestination,
|
||||
} from '@/lib/db/schema/system';
|
||||
import { userPortRoles, userProfiles } from '@/lib/db/schema/users';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { createNotification } from '@/lib/services/notifications.service';
|
||||
import { createFullBackupTar } from '@/lib/services/backup-export.service';
|
||||
import { decrypt, encrypt } from '@/lib/utils/encryption';
|
||||
|
||||
import {
|
||||
buildTransport,
|
||||
type BackupDestinationType,
|
||||
type BackupTransport,
|
||||
} from './backup-destinations';
|
||||
import { encryptFileToFile } from './backup-destinations/bundle-encryption';
|
||||
|
||||
// ─── secret config handling ─────────────────────────────────────────────────
|
||||
|
||||
const SECRET_FIELDS: Record<BackupDestinationType, string[]> = {
|
||||
sftp: ['password', 'privateKey', 'passphrase'],
|
||||
s3: ['secretKey'],
|
||||
filesystem: [],
|
||||
};
|
||||
|
||||
type Cfg = Record<string, unknown>;
|
||||
|
||||
/**
|
||||
* Prepare an incoming config for storage: encrypt every secret field that
|
||||
* carries a new non-empty value; for blank/absent secret fields, carry over the
|
||||
* already-encrypted value from `existing` (so "leave unchanged" works on edit).
|
||||
* Non-secret fields are taken from `incoming` (falling back to `existing`).
|
||||
*/
|
||||
export function serializeConfig(type: BackupDestinationType, incoming: Cfg, existing?: Cfg): Cfg {
|
||||
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||||
const out: Cfg = {};
|
||||
// Non-secret fields from incoming (or carry existing if omitted).
|
||||
for (const [k, v] of Object.entries(incoming)) {
|
||||
if (!secrets.has(k)) out[k] = v;
|
||||
}
|
||||
for (const field of secrets) {
|
||||
const incomingVal = incoming[field];
|
||||
if (typeof incomingVal === 'string' && incomingVal.length > 0) {
|
||||
out[field] = encrypt(incomingVal);
|
||||
} else if (existing && typeof existing[field] === 'string') {
|
||||
out[field] = existing[field];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Decrypt the secret fields of a stored config back to plaintext for transport use. */
|
||||
export function decryptConfig(type: BackupDestinationType, stored: Cfg): Cfg {
|
||||
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||||
const out: Cfg = { ...stored };
|
||||
for (const field of secrets) {
|
||||
const v = stored[field];
|
||||
if (typeof v === 'string' && v.length > 0) {
|
||||
try {
|
||||
out[field] = decrypt(v);
|
||||
} catch (err) {
|
||||
logger.error({ err, field }, 'Failed to decrypt backup destination secret');
|
||||
delete out[field];
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Strip secret fields from a stored config and expose `<field>IsSet` markers. */
|
||||
export function maskConfig(type: BackupDestinationType, stored: Cfg): Cfg {
|
||||
const secrets = new Set(SECRET_FIELDS[type] ?? []);
|
||||
const out: Cfg = {};
|
||||
for (const [k, v] of Object.entries(stored)) {
|
||||
if (!secrets.has(k)) out[k] = v;
|
||||
}
|
||||
for (const field of secrets) {
|
||||
out[`${field}IsSet`] =
|
||||
typeof stored[field] === 'string' && (stored[field] as string).length > 0;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface MaskedDestination {
|
||||
id: string;
|
||||
name: string;
|
||||
type: BackupDestinationType;
|
||||
enabled: boolean;
|
||||
config: Cfg;
|
||||
retentionCount: number | null;
|
||||
encryptBundle: boolean;
|
||||
encryptionKeyIsSet: boolean;
|
||||
lastRunAt: Date | null;
|
||||
lastStatus: string | null;
|
||||
lastError: string | null;
|
||||
lastBackupBytes: number | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
function mask(row: BackupDestination): MaskedDestination {
|
||||
const type = row.type as BackupDestinationType;
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
type,
|
||||
enabled: row.enabled,
|
||||
config: maskConfig(type, (row.config ?? {}) as Cfg),
|
||||
retentionCount: row.retentionCount,
|
||||
encryptBundle: row.encryptBundle,
|
||||
encryptionKeyIsSet: Boolean(row.encryptionKeyEncrypted),
|
||||
lastRunAt: row.lastRunAt,
|
||||
lastStatus: row.lastStatus,
|
||||
lastError: row.lastError,
|
||||
lastBackupBytes: row.lastBackupBytes,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── schedule ────────────────────────────────────────────────────────────────
|
||||
|
||||
export type BackupSchedule = 'off' | 'daily' | 'weekly';
|
||||
|
||||
/** Whether a scheduled push should run for `date` under `schedule`. */
|
||||
export function isScheduleDue(schedule: BackupSchedule, date: Date): boolean {
|
||||
if (schedule === 'off') return false;
|
||||
if (schedule === 'daily') return true;
|
||||
return date.getUTCDay() === 0; // weekly → Sundays
|
||||
}
|
||||
|
||||
export async function getSchedule(): Promise<BackupSchedule> {
|
||||
const [row] = await db
|
||||
.select()
|
||||
.from(systemSettings)
|
||||
.where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
|
||||
const v = row?.value;
|
||||
return v === 'daily' || v === 'weekly' ? v : 'off';
|
||||
}
|
||||
|
||||
export async function setSchedule(value: BackupSchedule, userId: string): Promise<void> {
|
||||
const existing = await db.query.systemSettings.findFirst({
|
||||
where: and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)),
|
||||
});
|
||||
if (existing) {
|
||||
await db
|
||||
.update(systemSettings)
|
||||
.set({ value, updatedBy: userId, updatedAt: new Date() })
|
||||
.where(and(eq(systemSettings.key, 'backup_schedule'), isNull(systemSettings.portId)));
|
||||
} else {
|
||||
await db
|
||||
.insert(systemSettings)
|
||||
.values({ key: 'backup_schedule', value, portId: null, updatedBy: userId });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── CRUD ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function listDestinations(): Promise<MaskedDestination[]> {
|
||||
const rows = await db.query.backupDestinations.findMany({
|
||||
orderBy: (d, { asc }) => [asc(d.createdAt)],
|
||||
});
|
||||
return rows.map(mask);
|
||||
}
|
||||
|
||||
export interface DestinationInput {
|
||||
name: string;
|
||||
type: BackupDestinationType;
|
||||
enabled?: boolean;
|
||||
config: Cfg;
|
||||
retentionCount?: number | null;
|
||||
encryptBundle?: boolean;
|
||||
/** Plaintext bundle passphrase; encrypted at rest. Blank = leave unchanged. */
|
||||
encryptionKey?: string;
|
||||
}
|
||||
|
||||
export async function createDestination(input: DestinationInput): Promise<MaskedDestination> {
|
||||
const [row] = await db
|
||||
.insert(backupDestinations)
|
||||
.values({
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
enabled: input.enabled ?? false,
|
||||
config: serializeConfig(input.type, input.config),
|
||||
retentionCount: input.retentionCount ?? null,
|
||||
encryptBundle: input.encryptBundle ?? false,
|
||||
encryptionKeyEncrypted:
|
||||
input.encryptionKey && input.encryptionKey.length > 0 ? encrypt(input.encryptionKey) : null,
|
||||
})
|
||||
.returning();
|
||||
if (!row) throw new Error('Failed to create backup destination');
|
||||
return mask(row);
|
||||
}
|
||||
|
||||
export async function updateDestination(
|
||||
id: string,
|
||||
input: DestinationInput,
|
||||
): Promise<MaskedDestination> {
|
||||
const existing = await db.query.backupDestinations.findFirst({
|
||||
where: eq(backupDestinations.id, id),
|
||||
});
|
||||
if (!existing) throw new Error('Backup destination not found');
|
||||
|
||||
const [row] = await db
|
||||
.update(backupDestinations)
|
||||
.set({
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
enabled: input.enabled ?? existing.enabled,
|
||||
config: serializeConfig(input.type, input.config, (existing.config ?? {}) as Cfg),
|
||||
retentionCount: input.retentionCount ?? null,
|
||||
encryptBundle: input.encryptBundle ?? false,
|
||||
encryptionKeyEncrypted:
|
||||
input.encryptionKey && input.encryptionKey.length > 0
|
||||
? encrypt(input.encryptionKey)
|
||||
: existing.encryptionKeyEncrypted,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(backupDestinations.id, id))
|
||||
.returning();
|
||||
if (!row) throw new Error('Failed to update backup destination');
|
||||
return mask(row);
|
||||
}
|
||||
|
||||
export async function deleteDestination(id: string): Promise<void> {
|
||||
await db.delete(backupDestinations).where(eq(backupDestinations.id, id));
|
||||
}
|
||||
|
||||
// ─── transport helpers ────────────────────────────────────────────────────
|
||||
|
||||
function transportFor(row: BackupDestination): BackupTransport {
|
||||
const type = row.type as BackupDestinationType;
|
||||
return buildTransport(type, decryptConfig(type, (row.config ?? {}) as Cfg));
|
||||
}
|
||||
|
||||
export async function testDestination(id: string): Promise<void> {
|
||||
const row = await db.query.backupDestinations.findFirst({
|
||||
where: eq(backupDestinations.id, id),
|
||||
});
|
||||
if (!row) throw new Error('Backup destination not found');
|
||||
await transportFor(row).test();
|
||||
}
|
||||
|
||||
// ─── push ─────────────────────────────────────────────────────────────────
|
||||
|
||||
interface PushOpts {
|
||||
/** Reuse an already-assembled tar (scheduled push assembles once for all). */
|
||||
tarPath?: string;
|
||||
filename?: string;
|
||||
trigger: 'manual' | 'cron';
|
||||
triggeredBy?: string | null;
|
||||
}
|
||||
|
||||
export async function pushBackupToDestination(
|
||||
id: string,
|
||||
opts: PushOpts,
|
||||
): Promise<{
|
||||
bytes: number;
|
||||
remoteRef: string;
|
||||
}> {
|
||||
const row = await db.query.backupDestinations.findFirst({
|
||||
where: eq(backupDestinations.id, id),
|
||||
});
|
||||
if (!row) throw new Error('Backup destination not found');
|
||||
|
||||
const transport = transportFor(row);
|
||||
let tarPath = opts.tarPath;
|
||||
let filename = opts.filename;
|
||||
let ownTar: (() => Promise<void>) | null = null;
|
||||
let encPath: string | null = null;
|
||||
|
||||
try {
|
||||
if (!tarPath || !filename) {
|
||||
const made = await createFullBackupTar();
|
||||
tarPath = made.tarPath;
|
||||
filename = made.filename;
|
||||
ownTar = made.cleanup;
|
||||
}
|
||||
|
||||
// Optional client-side encryption before the bytes leave this server.
|
||||
let uploadPath = tarPath;
|
||||
let remoteName = filename;
|
||||
if (row.encryptBundle) {
|
||||
if (!row.encryptionKeyEncrypted) {
|
||||
throw new Error('Destination has encryption enabled but no passphrase configured');
|
||||
}
|
||||
const passphrase = decrypt(row.encryptionKeyEncrypted);
|
||||
encPath = `${tarPath}.enc`;
|
||||
await encryptFileToFile(tarPath, encPath, passphrase);
|
||||
uploadPath = encPath;
|
||||
remoteName = `${filename}.enc`;
|
||||
}
|
||||
|
||||
const { bytes, remoteRef } = await transport.push(uploadPath, remoteName);
|
||||
await transport.prune(row.retentionCount).catch((err) => {
|
||||
logger.warn({ err, destinationId: id }, 'Backup prune failed (push succeeded)');
|
||||
});
|
||||
|
||||
await db
|
||||
.update(backupDestinations)
|
||||
.set({
|
||||
lastRunAt: new Date(),
|
||||
lastStatus: 'ok',
|
||||
lastError: null,
|
||||
lastBackupBytes: bytes,
|
||||
})
|
||||
.where(eq(backupDestinations.id, id));
|
||||
|
||||
await db.insert(backupJobs).values({
|
||||
status: 'completed',
|
||||
trigger: opts.trigger,
|
||||
triggeredBy: opts.triggeredBy ?? null,
|
||||
sizeBytes: bytes,
|
||||
storagePath: remoteRef,
|
||||
completedAt: new Date(),
|
||||
});
|
||||
|
||||
logger.info({ destinationId: id, bytes, remoteRef }, 'Backup pushed to destination');
|
||||
return { bytes, remoteRef };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'unknown';
|
||||
await db
|
||||
.update(backupDestinations)
|
||||
.set({ lastRunAt: new Date(), lastStatus: 'failed', lastError: message })
|
||||
.where(eq(backupDestinations.id, id));
|
||||
await db
|
||||
.insert(backupJobs)
|
||||
.values({
|
||||
status: 'failed',
|
||||
trigger: opts.trigger,
|
||||
triggeredBy: opts.triggeredBy ?? null,
|
||||
errorMessage: `[${row.name}] ${message}`,
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.catch(() => {});
|
||||
await notifyBackupFailure(row.name, message, opts.trigger);
|
||||
throw err;
|
||||
} finally {
|
||||
if (encPath) await unlink(encPath).catch(() => {});
|
||||
if (ownTar) await ownTar();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scheduled push: assemble the bundle ONCE and fan it out to every enabled
|
||||
* destination. Per-destination failures are isolated (one bad server doesn't
|
||||
* skip the others) and alerted.
|
||||
*/
|
||||
export async function runScheduledBackupPush(now = new Date()): Promise<{
|
||||
ran: boolean;
|
||||
pushed: number;
|
||||
failed: number;
|
||||
}> {
|
||||
const schedule = await getSchedule();
|
||||
if (!isScheduleDue(schedule, now)) {
|
||||
logger.info({ schedule }, 'Scheduled backup not due');
|
||||
return { ran: false, pushed: 0, failed: 0 };
|
||||
}
|
||||
|
||||
const enabled = await db.query.backupDestinations.findMany({
|
||||
where: eq(backupDestinations.enabled, true),
|
||||
});
|
||||
if (enabled.length === 0) {
|
||||
logger.warn('Backup schedule is on but no destinations are enabled');
|
||||
return { ran: false, pushed: 0, failed: 0 };
|
||||
}
|
||||
|
||||
const bundle = await createFullBackupTar();
|
||||
let pushed = 0;
|
||||
let failed = 0;
|
||||
try {
|
||||
for (const dest of enabled) {
|
||||
try {
|
||||
await pushBackupToDestination(dest.id, {
|
||||
tarPath: bundle.tarPath,
|
||||
filename: bundle.filename,
|
||||
trigger: 'cron',
|
||||
});
|
||||
pushed += 1;
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
logger.error({ err, destinationId: dest.id }, 'Scheduled push to destination failed');
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await bundle.cleanup();
|
||||
}
|
||||
logger.info({ pushed, failed, total: enabled.length }, 'Scheduled backup push complete');
|
||||
return { ran: true, pushed, failed };
|
||||
}
|
||||
|
||||
// ─── failure alerting ────────────────────────────────────────────────────
|
||||
|
||||
async function notifyBackupFailure(
|
||||
destinationName: string,
|
||||
message: string,
|
||||
trigger: 'manual' | 'cron',
|
||||
): Promise<void> {
|
||||
// Guaranteed signal: an error-severity audit row (visible in /admin/audit).
|
||||
await createAuditLog({
|
||||
userId: null,
|
||||
portId: null,
|
||||
action: 'job_failed',
|
||||
entityType: 'backup_destination',
|
||||
entityId: destinationName,
|
||||
severity: 'error',
|
||||
source: trigger === 'cron' ? 'cron' : 'job',
|
||||
metadata: { destination: destinationName, error: message },
|
||||
});
|
||||
|
||||
// Best-effort: in-app system alert to every super-admin (per their port).
|
||||
try {
|
||||
const admins = await db
|
||||
.select({ userId: userProfiles.userId, portId: userPortRoles.portId })
|
||||
.from(userProfiles)
|
||||
.innerJoin(userPortRoles, eq(userPortRoles.userId, userProfiles.userId))
|
||||
.where(eq(userProfiles.isSuperAdmin, true));
|
||||
const seen = new Set<string>();
|
||||
for (const a of admins) {
|
||||
const key = `${a.userId}:${a.portId}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
await createNotification({
|
||||
portId: a.portId,
|
||||
userId: a.userId,
|
||||
type: 'system_alert',
|
||||
title: 'Backup push failed',
|
||||
description: `Backup to "${destinationName}" failed: ${message}`,
|
||||
dedupeKey: `backup-fail:${destinationName}`,
|
||||
cooldownMs: 60 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err }, 'Failed to notify super-admins of backup failure');
|
||||
}
|
||||
}
|
||||
109
src/lib/services/backup-destinations/bundle-encryption.ts
Normal file
109
src/lib/services/backup-destinations/bundle-encryption.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* Opt-in client-side encryption for backup bundles
|
||||
* (docs/superpowers/specs/2026-06-04-backup-destinations-design.md).
|
||||
*
|
||||
* When a destination has `encryptBundle` on, the tar is encrypted to
|
||||
* `<name>.tar.enc` before it leaves this server, so a compromised destination
|
||||
* (untrusted SFTP host, third-party bucket) never holds raw signed contracts +
|
||||
* GDPR data.
|
||||
*
|
||||
* Format (AES-256-GCM, scrypt KDF):
|
||||
*
|
||||
* ┌────────┬──────────┬──────────┬──────────────┬──────────┐
|
||||
* │ magic │ salt │ iv │ ciphertext … │ authTag │
|
||||
* │ 5 bytes│ 16 bytes │ 12 bytes │ (streamed) │ 16 bytes │
|
||||
* └────────┴──────────┴──────────┴──────────────┴──────────┘
|
||||
*
|
||||
* Streaming throughout (memory stays O(chunk)). The auth tag is written last
|
||||
* because GCM only produces it after the final block; decryption reads it from
|
||||
* the file tail first, then streams the ciphertext through the decipher.
|
||||
*/
|
||||
|
||||
import { createCipheriv, createDecipheriv, randomBytes, scrypt as scryptCb } from 'node:crypto';
|
||||
import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { open, stat } from 'node:fs/promises';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const scrypt = promisify(scryptCb);
|
||||
|
||||
const MAGIC = Buffer.from('PNBK1', 'ascii'); // 5 bytes
|
||||
const SALT_LEN = 16;
|
||||
const IV_LEN = 12;
|
||||
const TAG_LEN = 16;
|
||||
const HEADER_LEN = MAGIC.length + SALT_LEN + IV_LEN; // 33
|
||||
|
||||
async function deriveKey(passphrase: string, salt: Buffer): Promise<Buffer> {
|
||||
return (await scrypt(passphrase, salt, 32)) as Buffer;
|
||||
}
|
||||
|
||||
/** Encrypt `srcPath` → `destPath` with a passphrase-derived AES-256-GCM key. */
|
||||
export async function encryptFileToFile(
|
||||
srcPath: string,
|
||||
destPath: string,
|
||||
passphrase: string,
|
||||
): Promise<void> {
|
||||
const salt = randomBytes(SALT_LEN);
|
||||
const iv = randomBytes(IV_LEN);
|
||||
const key = await deriveKey(passphrase, salt);
|
||||
|
||||
const cipher = createCipheriv('aes-256-gcm', key, iv);
|
||||
const out = createWriteStream(destPath);
|
||||
out.write(Buffer.concat([MAGIC, salt, iv]));
|
||||
|
||||
// Pipe plaintext → cipher → file, writing to `out` by hand (rather than
|
||||
// letting pipeline end it) so we can append the auth tag once the cipher has
|
||||
// flushed its final block.
|
||||
await pipeline(createReadStream(srcPath), cipher, async (source) => {
|
||||
for await (const chunk of source) {
|
||||
if (!out.write(chunk as Buffer)) {
|
||||
await new Promise<void>((resolve) => out.once('drain', () => resolve()));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
out.write(cipher.getAuthTag());
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
out.end((err?: Error | null) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
}
|
||||
|
||||
/** Decrypt a file produced by {@link encryptFileToFile}. Throws on wrong key / tamper. */
|
||||
export async function decryptFileToFile(
|
||||
srcPath: string,
|
||||
destPath: string,
|
||||
passphrase: string,
|
||||
): Promise<void> {
|
||||
const { size } = await stat(srcPath);
|
||||
if (size < HEADER_LEN + TAG_LEN) {
|
||||
throw new Error('Encrypted backup is too small / not a PNBK1 bundle');
|
||||
}
|
||||
|
||||
// Read the fixed header + the trailing auth tag.
|
||||
const fh = await open(srcPath, 'r');
|
||||
try {
|
||||
const header = Buffer.alloc(HEADER_LEN);
|
||||
await fh.read(header, 0, HEADER_LEN, 0);
|
||||
if (!header.subarray(0, MAGIC.length).equals(MAGIC)) {
|
||||
throw new Error('Not a PNBK1 encrypted backup (bad magic)');
|
||||
}
|
||||
const salt = header.subarray(MAGIC.length, MAGIC.length + SALT_LEN);
|
||||
const iv = header.subarray(MAGIC.length + SALT_LEN, HEADER_LEN);
|
||||
|
||||
const tag = Buffer.alloc(TAG_LEN);
|
||||
await fh.read(tag, 0, TAG_LEN, size - TAG_LEN);
|
||||
|
||||
const key = await deriveKey(passphrase, salt);
|
||||
const decipher = createDecipheriv('aes-256-gcm', key, iv);
|
||||
decipher.setAuthTag(tag);
|
||||
|
||||
// Stream only the ciphertext region [HEADER_LEN, size - TAG_LEN).
|
||||
const cipherStream = createReadStream(srcPath, {
|
||||
start: HEADER_LEN,
|
||||
end: size - TAG_LEN - 1,
|
||||
});
|
||||
await pipeline(cipherStream, decipher, createWriteStream(destPath));
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
47
src/lib/services/backup-destinations/filesystem.ts
Normal file
47
src/lib/services/backup-destinations/filesystem.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Filesystem backup transport — pushes the bundle to a configured directory
|
||||
* (a mounted volume / NAS share). The simplest destination: no network, just a
|
||||
* path the app can write to.
|
||||
*/
|
||||
|
||||
import { constants } from 'node:fs';
|
||||
import { access, copyFile, readdir, stat, unlink } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
BACKUP_NAME_PREFIX,
|
||||
sortBundlesNewestFirst,
|
||||
type BackupTransport,
|
||||
type FilesystemDestConfig,
|
||||
} from './types';
|
||||
|
||||
export class FilesystemTransport implements BackupTransport {
|
||||
constructor(private readonly cfg: FilesystemDestConfig) {}
|
||||
|
||||
async test(): Promise<void> {
|
||||
if (!this.cfg.directory) throw new Error('No directory configured');
|
||||
await access(this.cfg.directory, constants.W_OK).catch(() => {
|
||||
throw new Error(`Directory not writable or does not exist: ${this.cfg.directory}`);
|
||||
});
|
||||
const s = await stat(this.cfg.directory);
|
||||
if (!s.isDirectory()) throw new Error(`Not a directory: ${this.cfg.directory}`);
|
||||
}
|
||||
|
||||
async push(localPath: string, remoteName: string): Promise<{ remoteRef: string; bytes: number }> {
|
||||
const dest = path.join(this.cfg.directory, remoteName);
|
||||
await copyFile(localPath, dest);
|
||||
const s = await stat(dest);
|
||||
return { remoteRef: dest, bytes: s.size };
|
||||
}
|
||||
|
||||
async prune(retentionCount: number | null): Promise<{ deleted: number }> {
|
||||
if (retentionCount === null || retentionCount < 0) return { deleted: 0 };
|
||||
const entries = await readdir(this.cfg.directory);
|
||||
const bundles = sortBundlesNewestFirst(entries.filter((n) => n.startsWith(BACKUP_NAME_PREFIX)));
|
||||
const toDelete = bundles.slice(retentionCount);
|
||||
for (const name of toDelete) {
|
||||
await unlink(path.join(this.cfg.directory, name)).catch(() => {});
|
||||
}
|
||||
return { deleted: toDelete.length };
|
||||
}
|
||||
}
|
||||
36
src/lib/services/backup-destinations/index.ts
Normal file
36
src/lib/services/backup-destinations/index.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Backup destination transport factory. Given a destination type + its
|
||||
* *decrypted* runtime config, build the matching transport.
|
||||
*/
|
||||
|
||||
import { FilesystemTransport } from './filesystem';
|
||||
import { S3Transport } from './s3';
|
||||
import { SftpTransport } from './sftp';
|
||||
import type {
|
||||
BackupDestinationType,
|
||||
BackupTransport,
|
||||
FilesystemDestConfig,
|
||||
S3DestConfig,
|
||||
SftpDestConfig,
|
||||
} from './types';
|
||||
|
||||
export function buildTransport(
|
||||
type: BackupDestinationType,
|
||||
config: Record<string, unknown>,
|
||||
): BackupTransport {
|
||||
switch (type) {
|
||||
case 'filesystem':
|
||||
return new FilesystemTransport(config as unknown as FilesystemDestConfig);
|
||||
case 'sftp':
|
||||
return new SftpTransport(config as unknown as SftpDestConfig);
|
||||
case 's3':
|
||||
return new S3Transport(config as unknown as S3DestConfig);
|
||||
default:
|
||||
throw new Error(`Unknown backup destination type: ${String(type)}`);
|
||||
}
|
||||
}
|
||||
|
||||
export { FilesystemTransport } from './filesystem';
|
||||
export { SftpTransport } from './sftp';
|
||||
export { S3Transport, parseS3Endpoint } from './s3';
|
||||
export * from './types';
|
||||
104
src/lib/services/backup-destinations/s3.ts
Normal file
104
src/lib/services/backup-destinations/s3.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* S3-compatible backup transport — pushes the bundle to any S3 API endpoint
|
||||
* (AWS S3, Backblaze B2, Wasabi, Cloudflare R2, MinIO). Reuses the `minio`
|
||||
* client the storage backend already depends on, so no new SDK.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
import { Client as MinioClient } from 'minio';
|
||||
|
||||
import {
|
||||
BACKUP_NAME_PREFIX,
|
||||
sortBundlesNewestFirst,
|
||||
type BackupTransport,
|
||||
type S3DestConfig,
|
||||
} from './types';
|
||||
|
||||
/** Split a configured endpoint (host or URL) into minio's endPoint/port/useSSL. */
|
||||
export function parseS3Endpoint(
|
||||
endpoint: string,
|
||||
cfg: { useSSL?: boolean; port?: number },
|
||||
): { endPoint: string; port?: number; useSSL: boolean } {
|
||||
let host = endpoint.trim();
|
||||
let useSSL = cfg.useSSL ?? true;
|
||||
let port = cfg.port;
|
||||
const m = /^(https?):\/\/([^/:]+)(?::(\d+))?/i.exec(host);
|
||||
if (m) {
|
||||
useSSL = m[1]!.toLowerCase() === 'https';
|
||||
host = m[2]!;
|
||||
if (m[3]) port = Number(m[3]);
|
||||
} else {
|
||||
host = host.replace(/\/.*$/, '');
|
||||
}
|
||||
return { endPoint: host, ...(port ? { port } : {}), useSSL };
|
||||
}
|
||||
|
||||
export class S3Transport implements BackupTransport {
|
||||
private readonly prefix: string;
|
||||
|
||||
constructor(private readonly cfg: S3DestConfig) {
|
||||
// Normalise prefix to "" or "dir/".
|
||||
const p = (cfg.prefix ?? '').replace(/^\/+|\/+$/g, '');
|
||||
this.prefix = p ? `${p}/` : '';
|
||||
}
|
||||
|
||||
private client(): MinioClient {
|
||||
const { endPoint, port, useSSL } = parseS3Endpoint(this.cfg.endpoint, {
|
||||
useSSL: this.cfg.useSSL,
|
||||
port: this.cfg.port,
|
||||
});
|
||||
return new MinioClient({
|
||||
endPoint,
|
||||
...(port ? { port } : {}),
|
||||
useSSL,
|
||||
accessKey: this.cfg.accessKey,
|
||||
secretKey: this.cfg.secretKey,
|
||||
...(this.cfg.region ? { region: this.cfg.region } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async test(): Promise<void> {
|
||||
const exists = await this.client().bucketExists(this.cfg.bucket);
|
||||
if (!exists) throw new Error(`Bucket not found or not accessible: ${this.cfg.bucket}`);
|
||||
}
|
||||
|
||||
async push(localPath: string, remoteName: string): Promise<{ remoteRef: string; bytes: number }> {
|
||||
const key = `${this.prefix}${remoteName}`;
|
||||
await this.client().fPutObject(this.cfg.bucket, key, localPath, {
|
||||
'Content-Type': 'application/x-tar',
|
||||
});
|
||||
const { stat } = await import('node:fs/promises');
|
||||
const s = await stat(localPath);
|
||||
return { remoteRef: `s3://${this.cfg.bucket}/${key}`, bytes: s.size };
|
||||
}
|
||||
|
||||
async prune(retentionCount: number | null): Promise<{ deleted: number }> {
|
||||
if (retentionCount === null || retentionCount < 0) return { deleted: 0 };
|
||||
const client = this.client();
|
||||
const names = await this.listBundleKeys(client);
|
||||
const sorted = sortBundlesNewestFirst(names.map((k) => path.posix.basename(k)));
|
||||
const keepBasenames = new Set(sorted.slice(0, retentionCount));
|
||||
const toDelete = names.filter(
|
||||
(k) =>
|
||||
path.posix.basename(k).startsWith(BACKUP_NAME_PREFIX) &&
|
||||
!keepBasenames.has(path.posix.basename(k)),
|
||||
);
|
||||
for (const key of toDelete) await client.removeObject(this.cfg.bucket, key);
|
||||
return { deleted: toDelete.length };
|
||||
}
|
||||
|
||||
private listBundleKeys(client: MinioClient): Promise<string[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const keys: string[] = [];
|
||||
const stream = client.listObjectsV2(this.cfg.bucket, this.prefix, true);
|
||||
stream.on('data', (obj) => {
|
||||
if (obj.name && path.posix.basename(obj.name).startsWith(BACKUP_NAME_PREFIX)) {
|
||||
keys.push(obj.name);
|
||||
}
|
||||
});
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(keys));
|
||||
});
|
||||
}
|
||||
}
|
||||
102
src/lib/services/backup-destinations/sftp.ts
Normal file
102
src/lib/services/backup-destinations/sftp.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* SFTP/SSH backup transport — pushes the bundle to a remote server over SFTP.
|
||||
* This is the "separate server" destination most deployments will use.
|
||||
*
|
||||
* Host-key handling: when `hostFingerprint` is set, the server's key is verified
|
||||
* against it (sha256, colons/whitespace ignored) and the connection is rejected
|
||||
* on mismatch — defends against MITM. With no fingerprint configured we accept
|
||||
* on first use (TOFU); admins should pin the fingerprint for untrusted networks.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
|
||||
import SftpClient from 'ssh2-sftp-client';
|
||||
|
||||
import { logger } from '@/lib/logger';
|
||||
|
||||
import {
|
||||
BACKUP_NAME_PREFIX,
|
||||
sortBundlesNewestFirst,
|
||||
type BackupTransport,
|
||||
type SftpDestConfig,
|
||||
} from './types';
|
||||
|
||||
function normalizeFingerprint(fp: string): string {
|
||||
return fp
|
||||
.replace(/^sha256:/i, '')
|
||||
.replace(/[:\s]/g, '')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export class SftpTransport implements BackupTransport {
|
||||
constructor(private readonly cfg: SftpDestConfig) {}
|
||||
|
||||
private connectOptions(): SftpClient.ConnectOptions {
|
||||
const expected = this.cfg.hostFingerprint
|
||||
? normalizeFingerprint(this.cfg.hostFingerprint)
|
||||
: null;
|
||||
return {
|
||||
host: this.cfg.host,
|
||||
port: this.cfg.port ?? 22,
|
||||
username: this.cfg.username,
|
||||
...(this.cfg.password ? { password: this.cfg.password } : {}),
|
||||
...(this.cfg.privateKey ? { privateKey: this.cfg.privateKey } : {}),
|
||||
...(this.cfg.passphrase ? { passphrase: this.cfg.passphrase } : {}),
|
||||
// ssh2 calls this with the server's host key; hash + compare to the pin.
|
||||
hostVerifier: (key: Buffer): boolean => {
|
||||
if (!expected) return true;
|
||||
const actual = createHash('sha256').update(key).digest('hex');
|
||||
const ok = actual === expected;
|
||||
if (!ok) logger.error({ host: this.cfg.host }, 'SFTP host-key fingerprint mismatch');
|
||||
return ok;
|
||||
},
|
||||
} as SftpClient.ConnectOptions;
|
||||
}
|
||||
|
||||
private async withClient<T>(fn: (c: SftpClient) => Promise<T>): Promise<T> {
|
||||
const client = new SftpClient();
|
||||
try {
|
||||
await client.connect(this.connectOptions());
|
||||
return await fn(client);
|
||||
} finally {
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async test(): Promise<void> {
|
||||
await this.withClient(async (c) => {
|
||||
// Ensure the remote dir exists (create recursively if needed) and is usable.
|
||||
const exists = await c.exists(this.cfg.remoteDir);
|
||||
if (!exists) await c.mkdir(this.cfg.remoteDir, true);
|
||||
});
|
||||
}
|
||||
|
||||
async push(localPath: string, remoteName: string): Promise<{ remoteRef: string; bytes: number }> {
|
||||
return this.withClient(async (c) => {
|
||||
const exists = await c.exists(this.cfg.remoteDir);
|
||||
if (!exists) await c.mkdir(this.cfg.remoteDir, true);
|
||||
const remotePath = path.posix.join(this.cfg.remoteDir, remoteName);
|
||||
await c.fastPut(localPath, remotePath);
|
||||
const s = await c.stat(remotePath);
|
||||
return { remoteRef: `sftp://${this.cfg.host}${remotePath}`, bytes: s.size };
|
||||
});
|
||||
}
|
||||
|
||||
async prune(retentionCount: number | null): Promise<{ deleted: number }> {
|
||||
if (retentionCount === null || retentionCount < 0) return { deleted: 0 };
|
||||
return this.withClient(async (c) => {
|
||||
const list = await c.list(this.cfg.remoteDir);
|
||||
const bundles = sortBundlesNewestFirst(
|
||||
list
|
||||
.filter((e) => e.type === '-' && e.name.startsWith(BACKUP_NAME_PREFIX))
|
||||
.map((e) => e.name),
|
||||
);
|
||||
const toDelete = bundles.slice(retentionCount);
|
||||
for (const name of toDelete) {
|
||||
await c.delete(path.posix.join(this.cfg.remoteDir, name)).catch(() => {});
|
||||
}
|
||||
return { deleted: toDelete.length };
|
||||
});
|
||||
}
|
||||
}
|
||||
60
src/lib/services/backup-destinations/types.ts
Normal file
60
src/lib/services/backup-destinations/types.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Backup destination transport contract + per-type config shapes.
|
||||
* See docs/superpowers/specs/2026-06-04-backup-destinations-design.md.
|
||||
*
|
||||
* Config shapes here are the *decrypted, runtime* form. Secrets are stored
|
||||
* AES-GCM-encrypted in the `backup_destinations.config` jsonb and decrypted by
|
||||
* the service layer before a transport is constructed.
|
||||
*/
|
||||
|
||||
export type BackupDestinationType = 'sftp' | 's3' | 'filesystem';
|
||||
|
||||
/** Filename prefix every full-bundle tar carries (`createFullBackupTar`). */
|
||||
export const BACKUP_NAME_PREFIX = 'pn-crm-backup-';
|
||||
|
||||
export interface BackupTransport {
|
||||
/** Verify the destination is reachable + writable. Throws on failure. */
|
||||
test(): Promise<void>;
|
||||
/** Upload `localPath` to the destination as `remoteName`. */
|
||||
push(localPath: string, remoteName: string): Promise<{ remoteRef: string; bytes: number }>;
|
||||
/** Keep the `retentionCount` newest bundles; null = keep all. */
|
||||
prune(retentionCount: number | null): Promise<{ deleted: number }>;
|
||||
}
|
||||
|
||||
export interface FilesystemDestConfig {
|
||||
/** Absolute path to a mounted volume / NAS directory the app can write to. */
|
||||
directory: string;
|
||||
}
|
||||
|
||||
export interface SftpDestConfig {
|
||||
host: string;
|
||||
port?: number;
|
||||
username: string;
|
||||
/** One of password / privateKey is required. */
|
||||
password?: string;
|
||||
privateKey?: string;
|
||||
/** Passphrase for an encrypted private key. */
|
||||
passphrase?: string;
|
||||
remoteDir: string;
|
||||
/** Optional pinned host-key fingerprint (sha256 hex). When set, the
|
||||
* connection is rejected unless the server's key matches. */
|
||||
hostFingerprint?: string;
|
||||
}
|
||||
|
||||
export interface S3DestConfig {
|
||||
endpoint: string;
|
||||
region?: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
/** Key prefix within the bucket (e.g. "crm-backups/"). */
|
||||
prefix?: string;
|
||||
/** Default true; set false only for pl-text local MinIO test endpoints. */
|
||||
useSSL?: boolean;
|
||||
port?: number;
|
||||
}
|
||||
|
||||
/** Sort backup bundle filenames newest-first (timestamp-in-name sorts lexically). */
|
||||
export function sortBundlesNewestFirst(names: string[]): string[] {
|
||||
return names.filter((n) => n.startsWith(BACKUP_NAME_PREFIX)).sort((a, b) => b.localeCompare(a));
|
||||
}
|
||||
297
src/lib/services/backup-export.service.ts
Normal file
297
src/lib/services/backup-export.service.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
/**
|
||||
* Full-bundle backup export (Phase 4a — docs/storage-migration-and-backup-plan.md).
|
||||
*
|
||||
* Today's `runBackup()` (backup.service.ts) dumps ONLY the database, buffers the
|
||||
* whole dump in memory, and writes it back to the SAME storage backend it would
|
||||
* lose if storage died. This module produces a *complete*, backend-agnostic
|
||||
* disaster-recovery bundle:
|
||||
*
|
||||
* bundle.tar
|
||||
* ├── db.dump (pg_dump --format=custom of the live DB)
|
||||
* ├── blobs/<storage_key> (every blob referenced by a DB row)
|
||||
* └── manifest.json (sha256 + size per object, for restore-side verify)
|
||||
*
|
||||
* Streaming is mandatory: blobs are piped backend → tar with a known size
|
||||
* (`stats.size`, which is what makes archiver stream instead of buffering the
|
||||
* whole entry into memory) so total memory stays O(largest chunk), not
|
||||
* O(total bytes). The tar is assembled to a temp file first, then handed to the
|
||||
* caller to stream to the operator — so a mid-assembly failure surfaces as a
|
||||
* clean error rather than a truncated download.
|
||||
*
|
||||
* The assembler (`assembleBackupTar`) is pure w.r.t. its inputs (a storage
|
||||
* backend, a pre-produced dump file, a blob-ref list) so it is unit-tested with
|
||||
* an in-memory backend; the orchestrator (`createFullBackupTar`) wires in
|
||||
* pg_dump + the live blob inventory.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { stat, unlink } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Transform } from 'node:stream';
|
||||
|
||||
import archiver from 'archiver';
|
||||
|
||||
import { env } from '@/lib/env';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { getStorageBackend, type StorageBackend } from '@/lib/storage';
|
||||
import { collectStorageRefs } from '@/lib/storage/migrate';
|
||||
import { runPgDump } from '@/lib/services/backup.service';
|
||||
|
||||
/** A blob the bundle should attempt to include. */
|
||||
export interface BackupBlobRef {
|
||||
tableName: string;
|
||||
pk: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export interface BackupManifestBlobEntry {
|
||||
table: string;
|
||||
pk: string;
|
||||
key: string;
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface BackupSkippedEntry {
|
||||
table: string;
|
||||
pk: string;
|
||||
key: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface BackupManifest {
|
||||
formatVersion: number;
|
||||
createdAt: string;
|
||||
storageBackend: string;
|
||||
database: {
|
||||
file: string;
|
||||
format: string;
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
};
|
||||
blobs: BackupManifestBlobEntry[];
|
||||
skipped: BackupSkippedEntry[];
|
||||
counts: {
|
||||
blobs: number;
|
||||
blobBytes: number;
|
||||
skipped: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipe `source` into the archive under `name`, computing the sha256 and byte
|
||||
* count of exactly the bytes that pass through. Resolves once the entry has
|
||||
* been fully consumed by archiver.
|
||||
*
|
||||
* `stats.size` MUST be supplied: archiver's tar plugin streams the entry only
|
||||
* when `data.stats` is present (otherwise it buffers the whole stream into
|
||||
* memory via `collectStream` to discover the size — the exact behaviour we're
|
||||
* avoiding for multi-GB blob sets).
|
||||
*/
|
||||
function appendHashedStream(
|
||||
archive: archiver.Archiver,
|
||||
source: NodeJS.ReadableStream,
|
||||
name: string,
|
||||
declaredSize: number,
|
||||
now: Date,
|
||||
): Promise<{ sha256: string; bytes: number }> {
|
||||
const hash = createHash('sha256');
|
||||
let bytes = 0;
|
||||
const tee = new Transform({
|
||||
transform(chunk: Buffer, _enc, cb) {
|
||||
hash.update(chunk);
|
||||
bytes += chunk.length;
|
||||
cb(null, chunk);
|
||||
},
|
||||
});
|
||||
|
||||
const done = new Promise<{ sha256: string; bytes: number }>((resolve, reject) => {
|
||||
source.on('error', (err) => tee.destroy(err instanceof Error ? err : new Error(String(err))));
|
||||
tee.on('error', reject);
|
||||
tee.on('end', () => resolve({ sha256: hash.digest('hex'), bytes }));
|
||||
});
|
||||
|
||||
source.pipe(tee);
|
||||
archive.append(tee, {
|
||||
name,
|
||||
date: now,
|
||||
// A minimal fs.Stats-like object. `size` engages archiver's streaming
|
||||
// tar path; `mode`/`mtime` keep the header deterministic.
|
||||
stats: {
|
||||
size: declaredSize,
|
||||
mode: 0o644,
|
||||
mtime: now,
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
} as unknown as import('node:fs').Stats,
|
||||
});
|
||||
|
||||
return done;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble a backup tar at `outFilePath` from a pre-produced pg_dump file and a
|
||||
* list of blob references. Returns the manifest describing the bundle.
|
||||
*/
|
||||
export async function assembleBackupTar(opts: {
|
||||
backend: StorageBackend;
|
||||
dumpFilePath: string;
|
||||
blobRefs: BackupBlobRef[];
|
||||
outFilePath: string;
|
||||
storageBackendName: string;
|
||||
now: Date;
|
||||
}): Promise<BackupManifest> {
|
||||
const { backend, dumpFilePath, blobRefs, outFilePath, storageBackendName, now } = opts;
|
||||
|
||||
const archive = archiver('tar');
|
||||
const output = createWriteStream(outFilePath);
|
||||
|
||||
const finished = new Promise<void>((resolve, reject) => {
|
||||
output.on('close', () => resolve());
|
||||
output.on('error', reject);
|
||||
archive.on('error', reject);
|
||||
archive.on('warning', (err: Error & { code?: string }) => {
|
||||
// Non-fatal (e.g. ENOENT on a vanished file) — log and keep going.
|
||||
logger.warn({ err }, 'archiver warning during backup export');
|
||||
});
|
||||
});
|
||||
|
||||
archive.pipe(output);
|
||||
|
||||
// 1. db.dump
|
||||
const dumpStat = await stat(dumpFilePath);
|
||||
const dump = await appendHashedStream(
|
||||
archive,
|
||||
createReadStream(dumpFilePath),
|
||||
'db.dump',
|
||||
dumpStat.size,
|
||||
now,
|
||||
);
|
||||
|
||||
// 2. blobs (one at a time so memory stays bounded)
|
||||
const blobs: BackupManifestBlobEntry[] = [];
|
||||
const skipped: BackupSkippedEntry[] = [];
|
||||
for (const ref of blobRefs) {
|
||||
const head = await backend.head(ref.key);
|
||||
if (!head) {
|
||||
skipped.push({
|
||||
table: ref.tableName,
|
||||
pk: ref.pk,
|
||||
key: ref.key,
|
||||
reason: 'missing-in-storage',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let source: NodeJS.ReadableStream;
|
||||
try {
|
||||
source = await backend.get(ref.key);
|
||||
} catch (err) {
|
||||
skipped.push({
|
||||
table: ref.tableName,
|
||||
pk: ref.pk,
|
||||
key: ref.key,
|
||||
reason: `unreadable: ${err instanceof Error ? err.message : 'unknown'}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const { sha256, bytes } = await appendHashedStream(
|
||||
archive,
|
||||
source,
|
||||
`blobs/${ref.key}`,
|
||||
head.sizeBytes,
|
||||
now,
|
||||
);
|
||||
blobs.push({ table: ref.tableName, pk: ref.pk, key: ref.key, sizeBytes: bytes, sha256 });
|
||||
}
|
||||
|
||||
// 3. manifest.json (last — it carries the sha256 of every prior entry)
|
||||
const manifest: BackupManifest = {
|
||||
formatVersion: 1,
|
||||
createdAt: now.toISOString(),
|
||||
storageBackend: storageBackendName,
|
||||
database: {
|
||||
file: 'db.dump',
|
||||
format: 'pg_dump-custom',
|
||||
sizeBytes: dump.bytes,
|
||||
sha256: dump.sha256,
|
||||
},
|
||||
blobs,
|
||||
skipped,
|
||||
counts: {
|
||||
blobs: blobs.length,
|
||||
blobBytes: blobs.reduce((acc, b) => acc + b.sizeBytes, 0),
|
||||
skipped: skipped.length,
|
||||
},
|
||||
};
|
||||
archive.append(Buffer.from(JSON.stringify(manifest, null, 2)), {
|
||||
name: 'manifest.json',
|
||||
date: now,
|
||||
});
|
||||
|
||||
await archive.finalize();
|
||||
await finished;
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export interface FullBackupResult {
|
||||
tarPath: string;
|
||||
filename: string;
|
||||
manifest: BackupManifest;
|
||||
/** Removes the assembled tar. The intermediate dump is removed eagerly. */
|
||||
cleanup: () => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate a full backup: pg_dump the live DB, inventory every blob, and
|
||||
* assemble the bundle to a temp tar. The caller streams `tarPath` to the
|
||||
* operator and invokes `cleanup()` when the download finishes.
|
||||
*
|
||||
* `backup_jobs` blobs (prior pg_dump artefacts) are excluded so a full export
|
||||
* doesn't recursively bundle previous backups.
|
||||
*/
|
||||
export async function createFullBackupTar(): Promise<FullBackupResult> {
|
||||
const now = new Date();
|
||||
const id = crypto.randomUUID();
|
||||
const dumpPath = path.join(tmpdir(), `pn-fullbackup-${id}.dump`);
|
||||
const tarPath = path.join(tmpdir(), `pn-fullbackup-${id}.tar`);
|
||||
|
||||
try {
|
||||
await runPgDump(env.DATABASE_URL, dumpPath);
|
||||
const backend = await getStorageBackend();
|
||||
const refs = await collectStorageRefs({ excludeTables: ['backup_jobs'] });
|
||||
const blobRefs: BackupBlobRef[] = refs.map((r) => ({
|
||||
tableName: r.tableName,
|
||||
pk: r.pk,
|
||||
key: r.key,
|
||||
}));
|
||||
|
||||
const manifest = await assembleBackupTar({
|
||||
backend,
|
||||
dumpFilePath: dumpPath,
|
||||
blobRefs,
|
||||
outFilePath: tarPath,
|
||||
storageBackendName: backend.name,
|
||||
now,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
{ blobs: manifest.counts.blobs, skipped: manifest.counts.skipped },
|
||||
'Full backup bundle assembled',
|
||||
);
|
||||
|
||||
const stamp = now.toISOString().replace(/[:.]/g, '-');
|
||||
return {
|
||||
tarPath,
|
||||
filename: `pn-crm-backup-${stamp}.tar`,
|
||||
manifest,
|
||||
cleanup: async () => {
|
||||
await unlink(tarPath).catch(() => {});
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
// The dump is already inside the tar (or assembly failed) — drop it now.
|
||||
await unlink(dumpPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -88,24 +88,63 @@ export async function runBackup({ trigger, triggeredBy }: RunBackupArgs): Promis
|
||||
}
|
||||
}
|
||||
|
||||
function runPgDump(databaseUrl: string, outFile: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('pg_dump', ['--format=custom', '--no-owner', databaseUrl]);
|
||||
const out = createWriteStream(outFile);
|
||||
child.stdout.pipe(out);
|
||||
export interface RunPgDumpOpts {
|
||||
/** Override the binary (tests inject `node`). Defaults to `pg_dump`. */
|
||||
command?: string;
|
||||
/** Build the argv from the connection URL. Defaults to a custom-format dump. */
|
||||
buildArgs?: (databaseUrl: string) => string[];
|
||||
}
|
||||
|
||||
export function runPgDump(
|
||||
databaseUrl: string,
|
||||
outFile: string,
|
||||
opts: RunPgDumpOpts = {},
|
||||
): Promise<void> {
|
||||
const command = opts.command ?? 'pg_dump';
|
||||
const args = (opts.buildArgs ?? ((url) => ['--format=custom', '--no-owner', url]))(databaseUrl);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args);
|
||||
const out = createWriteStream(outFile);
|
||||
// `stdout.pipe(out)` auto-ends `out` when the child's stdout closes, so the
|
||||
// file's `finish` event can fire *before* the process `close` event. Gate
|
||||
// resolution on BOTH having happened rather than attaching the `finish`
|
||||
// listener inside the `close` handler (which raced and hung when `finish`
|
||||
// had already fired).
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
let exitCode: number | null = null;
|
||||
let fileFlushed = false;
|
||||
|
||||
const fail = (err: Error): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(err);
|
||||
};
|
||||
const maybeResolve = (): void => {
|
||||
if (settled || exitCode === null || !fileFlushed) return;
|
||||
if (exitCode === 0) {
|
||||
settled = true;
|
||||
resolve();
|
||||
} else {
|
||||
fail(new Error(`pg_dump exited ${exitCode}: ${stderr}`));
|
||||
}
|
||||
};
|
||||
|
||||
child.stderr.on('data', (b) => {
|
||||
stderr += b.toString();
|
||||
});
|
||||
child.on('error', (err) => reject(err));
|
||||
child.on('close', (code) => {
|
||||
out.end();
|
||||
out.on('finish', () => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`pg_dump exited ${code}: ${stderr}`));
|
||||
});
|
||||
child.on('error', fail);
|
||||
out.on('error', fail);
|
||||
out.on('finish', () => {
|
||||
fileFlushed = true;
|
||||
maybeResolve();
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
exitCode = code;
|
||||
maybeResolve();
|
||||
});
|
||||
child.stdout.pipe(out);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -136,7 +136,7 @@ async function markRowMigrated(
|
||||
`);
|
||||
}
|
||||
|
||||
interface RowRef {
|
||||
export interface RowRef {
|
||||
tableName: string;
|
||||
pk: string;
|
||||
key: string;
|
||||
@@ -164,6 +164,22 @@ async function listKeysFor(tbl: StorageKeyTable): Promise<RowRef[]> {
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Inventory every blob reference across all blob-bearing tables. Used by the
|
||||
* full-backup exporter (Phase 4a) to enumerate what to bundle. `excludeTables`
|
||||
* lets the exporter drop `backup_jobs` so a full export doesn't recursively
|
||||
* include prior backup artefacts.
|
||||
*/
|
||||
export async function collectStorageRefs(opts?: { excludeTables?: string[] }): Promise<RowRef[]> {
|
||||
const exclude = new Set(opts?.excludeTables ?? []);
|
||||
const all: RowRef[] = [];
|
||||
for (const tbl of TABLES_WITH_STORAGE_KEYS) {
|
||||
if (exclude.has(tbl.table)) continue;
|
||||
all.push(...(await listKeysFor(tbl)));
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
// ─── streaming + sha256 verify ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
66
src/lib/validators/backup-destinations.ts
Normal file
66
src/lib/validators/backup-destinations.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/** Per-type connection config. Secret fields are optional so an edit can leave
|
||||
* them blank to keep the stored (encrypted) value; create-time omission is
|
||||
* surfaced by the destination's "Test connection" instead. */
|
||||
const filesystemConfigSchema = z.object({
|
||||
directory: z.string().min(1, 'Directory is required'),
|
||||
});
|
||||
|
||||
const sftpConfigSchema = z.object({
|
||||
host: z.string().min(1, 'Host is required'),
|
||||
port: z.number().int().positive().max(65535).optional(),
|
||||
username: z.string().min(1, 'Username is required'),
|
||||
password: z.string().optional(),
|
||||
privateKey: z.string().optional(),
|
||||
passphrase: z.string().optional(),
|
||||
remoteDir: z.string().min(1, 'Remote directory is required'),
|
||||
hostFingerprint: z.string().optional(),
|
||||
});
|
||||
|
||||
const s3ConfigSchema = z.object({
|
||||
endpoint: z.string().min(1, 'Endpoint is required'),
|
||||
region: z.string().optional(),
|
||||
bucket: z.string().min(1, 'Bucket is required'),
|
||||
accessKey: z.string().min(1, 'Access key is required'),
|
||||
secretKey: z.string().optional(),
|
||||
prefix: z.string().optional(),
|
||||
useSSL: z.boolean().optional(),
|
||||
port: z.number().int().positive().max(65535).optional(),
|
||||
});
|
||||
|
||||
const CONFIG_SCHEMA_BY_TYPE = {
|
||||
filesystem: filesystemConfigSchema,
|
||||
sftp: sftpConfigSchema,
|
||||
s3: s3ConfigSchema,
|
||||
} as const;
|
||||
|
||||
export const backupDestinationSchema = z
|
||||
.object({
|
||||
name: z.string().min(1).max(120),
|
||||
type: z.enum(['sftp', 's3', 'filesystem']),
|
||||
enabled: z.boolean().optional(),
|
||||
config: z.record(z.string(), z.unknown()),
|
||||
retentionCount: z.number().int().min(0).max(10000).nullable().optional(),
|
||||
encryptBundle: z.boolean().optional(),
|
||||
encryptionKey: z.string().optional(),
|
||||
})
|
||||
.superRefine((val, ctx) => {
|
||||
const schema = CONFIG_SCHEMA_BY_TYPE[val.type];
|
||||
const result = schema.safeParse(val.config);
|
||||
if (!result.success) {
|
||||
for (const issue of result.error.issues) {
|
||||
ctx.addIssue({ ...issue, path: ['config', ...issue.path] });
|
||||
}
|
||||
}
|
||||
if (val.encryptBundle && !val.encryptionKey) {
|
||||
// Allowed on update (keeps existing key); the route/service enforces a key
|
||||
// exists before a push. Surfaced here only as a soft hint via no error.
|
||||
}
|
||||
});
|
||||
|
||||
export const backupScheduleSchema = z.object({
|
||||
schedule: z.enum(['off', 'daily', 'weekly']),
|
||||
});
|
||||
|
||||
export type BackupDestinationInput = z.infer<typeof backupDestinationSchema>;
|
||||
Reference in New Issue
Block a user