feat(storage): pluggable s3-or-filesystem backend + migration CLI + admin UI
Phase 6a from docs/berth-recommender-and-pdf-plan.md §4.7a + §14.9a. Lays
the storage groundwork for Phase 6b/7 file-bearing schemas (per-berth PDFs,
brochures) without touching those domains yet.
New files:
- src/lib/storage/index.ts StorageBackend interface + per-process
factory keyed on system_settings.
- src/lib/storage/s3.ts S3-compatible backend (MinIO/AWS/B2/R2/
Wasabi/Tigris) wrapping the existing minio
JS client. Includes a healthCheck() used
by the admin "Test connection" button.
- src/lib/storage/filesystem.ts Local filesystem backend with all §14.9a
mitigations baked in.
- src/lib/storage/migrate.ts Shared migration core — pg_advisory_lock,
per-row resumable progress markers,
sha256 round-trip verification, atomic
storage_backend flip on success.
- scripts/migrate-storage.ts Thin CLI shim around runMigration().
- src/app/api/storage/[token]/route.ts
Filesystem proxy GET. Verifies HMAC,
enforces single-use replay protection
via Redis SET NX, streams via NextResponse
ReadableStream with explicit Content-Type
+ Content-Disposition. Node runtime only.
- src/app/api/v1/admin/storage/route.ts
GET status + POST connection test.
- src/app/api/v1/admin/storage/migrate/route.ts
Super-admin-only POST that runs the
exact same runMigration() as the CLI.
- src/app/(dashboard)/[portSlug]/admin/storage/page.tsx
Super-admin admin UI (current backend,
capacity stats, switch button with
dry-run, test connection, backup hint).
- src/components/admin/storage-admin-panel.tsx
Client component for the page above.
§14.9a critical mitigations implemented:
- Path-traversal: storage keys validated against ^[a-zA-Z0-9/_.-]+$;
`..`, `.`, `//`, leading `/`, and overlength keys rejected.
- Realpath: storage root realpath'd at create time, every per-key
resolution checked against the realpath'd prefix.
- Storage root created (or chmod'd) to 0o700.
- Multi-node refusal: FilesystemBackend.create() throws when
MULTI_NODE_DEPLOYMENT=true.
- HMAC token: sha256-HMAC over the (key, expiry, nonce, filename,
content-type) payload. Verified with timingSafeEqual; bad sig,
expired, or invalid-key payloads all return 403.
- Single-use replay: token body cached in Redis SET NX EX 1800s.
- sha256 round-trip: copyAndVerify() re-fetches from the target after
put() and aborts the migration on any mismatch.
- Free-disk pre-flight: when migrating to filesystem, sums byte counts
via source.head() and aborts if free space < total * 1.2.
- pg_advisory_lock(0xc7000a01) prevents concurrent migrations.
- Resumable: per-row progress markers in _storage_migration_progress.
system_settings keys read by the factory (jsonb, no schema change):
storage_backend, storage_s3_endpoint, storage_s3_region,
storage_s3_bucket, storage_s3_access_key,
storage_s3_secret_key_encrypted, storage_s3_force_path_style,
storage_filesystem_root, storage_proxy_hmac_secret_encrypted.
Defaults: storage_backend=`s3`, storage_filesystem_root=`./storage`
(./storage added to .gitignore).
Tests added (34 tests, all green):
- tests/unit/storage/filesystem-backend.test.ts — key validation
allow/reject matrix, realpath escape, 0o700 perms, multi-node
refusal, HMAC token sign/verify/tamper/expire/invalid-key.
- tests/unit/storage/copy-and-verify.test.ts — sha256 mismatch on
round-trip aborts the migration.
- tests/integration/storage/proxy-route.test.ts — happy path, wrong
HMAC secret, expired token, replay rejection.
Phase 6a ships zero file-bearing tables — TABLES_WITH_STORAGE_KEYS is
intentionally empty. berth_pdf_versions and brochure_versions land in
Phase 6b and join the list there. Existing s3_key columns: only
gdpr_export_jobs.storage_key, already named correctly — no rename needed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 03:15:59 +02:00
|
|
|
/**
|
|
|
|
|
* Local filesystem backend. Stores files at `${root}/<key>` on disk and serves
|
|
|
|
|
* downloads via a CRM-internal proxy route (`/api/storage/[token]`) that
|
|
|
|
|
* verifies an HMAC token before streaming the bytes. Used for single-VPS
|
|
|
|
|
* deployments where running MinIO is overkill.
|
|
|
|
|
*
|
|
|
|
|
* §14.9a critical mitigations:
|
|
|
|
|
*
|
|
|
|
|
* - Storage keys are validated against `^[a-zA-Z0-9/_.-]+$`. Anything containing
|
|
|
|
|
* `..` or that resolves to an absolute path is rejected.
|
|
|
|
|
* - The resolved path is checked with `path.resolve` against the resolved
|
|
|
|
|
* storage root (realpath form) — symlink escapes are blocked.
|
|
|
|
|
* - The storage root is created with mode `0o700` (owner only).
|
|
|
|
|
* - Refuses to start when `MULTI_NODE_DEPLOYMENT === 'true'` — multi-node
|
|
|
|
|
* deployments must use an S3-compatible store.
|
|
|
|
|
* - Proxy download URLs carry an HMAC-signed payload (key + expiry); the
|
|
|
|
|
* route refuses to stream a key whose token doesn't verify.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
import { createHash, createHmac, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
|
|
|
import { createReadStream } from 'node:fs';
|
|
|
|
|
import * as fs from 'node:fs/promises';
|
|
|
|
|
import * as path from 'node:path';
|
|
|
|
|
import { Readable } from 'node:stream';
|
|
|
|
|
|
|
|
|
|
import { env } from '@/lib/env';
|
|
|
|
|
import { NotFoundError, ValidationError } from '@/lib/errors';
|
|
|
|
|
import { logger } from '@/lib/logger';
|
|
|
|
|
import { decrypt } from '@/lib/utils/encryption';
|
|
|
|
|
|
|
|
|
|
import type { PresignOpts, PutOpts, StorageBackend } from './index';
|
|
|
|
|
|
|
|
|
|
// ─── key validation ─────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
const VALID_KEY_RE = /^[a-zA-Z0-9/_.-]+$/;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Validate a storage key. Rejects:
|
|
|
|
|
* - empty / whitespace
|
|
|
|
|
* - characters outside `[a-zA-Z0-9/_.-]`
|
|
|
|
|
* - traversal segments (`..`, `/..`, `../`)
|
|
|
|
|
* - absolute paths (leading `/`)
|
|
|
|
|
* - segments starting with `.` (hidden files / dotfiles other than the
|
|
|
|
|
* intentional dot-extension at the end)
|
|
|
|
|
*
|
|
|
|
|
* Use this both at write time AND at read time — a key fed back from the DB
|
|
|
|
|
* could in theory have been tampered with at rest.
|
|
|
|
|
*/
|
|
|
|
|
export function validateStorageKey(key: string): void {
|
|
|
|
|
if (typeof key !== 'string' || key.length === 0) {
|
|
|
|
|
throw new ValidationError('Storage key must be a non-empty string');
|
|
|
|
|
}
|
|
|
|
|
if (key.length > 1024) {
|
|
|
|
|
throw new ValidationError('Storage key exceeds 1024 chars');
|
|
|
|
|
}
|
|
|
|
|
if (key.startsWith('/') || key.startsWith('\\')) {
|
|
|
|
|
throw new ValidationError('Storage key must not be absolute');
|
|
|
|
|
}
|
|
|
|
|
if (!VALID_KEY_RE.test(key)) {
|
|
|
|
|
throw new ValidationError('Storage key contains forbidden characters');
|
|
|
|
|
}
|
|
|
|
|
// Reject any traversal segment in any normalized form.
|
|
|
|
|
const segments = key.split('/');
|
|
|
|
|
for (const seg of segments) {
|
|
|
|
|
if (seg === '..' || seg === '.' || seg === '') {
|
|
|
|
|
throw new ValidationError('Storage key has empty or traversal segment');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── HMAC token helpers ─────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
interface ProxyTokenPayload {
|
|
|
|
|
/** Storage key (validated). */
|
|
|
|
|
k: string;
|
|
|
|
|
/** Expiry epoch seconds. */
|
|
|
|
|
e: number;
|
|
|
|
|
/** Random nonce so two URLs for the same (key, expiry) differ. */
|
|
|
|
|
n: string;
|
|
|
|
|
/** Optional download filename. */
|
|
|
|
|
f?: string;
|
|
|
|
|
/** Optional content-type override. */
|
|
|
|
|
c?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function b64urlEncode(buf: Buffer): string {
|
|
|
|
|
return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function b64urlDecode(s: string): Buffer {
|
|
|
|
|
const pad = s.length % 4 === 0 ? '' : '='.repeat(4 - (s.length % 4));
|
|
|
|
|
return Buffer.from(s.replace(/-/g, '+').replace(/_/g, '/') + pad, 'base64');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function signProxyToken(payload: ProxyTokenPayload, secret: string): string {
|
|
|
|
|
const json = JSON.stringify(payload);
|
|
|
|
|
const body = b64urlEncode(Buffer.from(json, 'utf8'));
|
|
|
|
|
const sig = createHmac('sha256', secret).update(body).digest();
|
|
|
|
|
return `${body}.${b64urlEncode(sig)}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function verifyProxyToken(
|
|
|
|
|
token: string,
|
|
|
|
|
secret: string,
|
|
|
|
|
): { ok: true; payload: ProxyTokenPayload } | { ok: false; reason: string } {
|
|
|
|
|
if (typeof token !== 'string' || !token.includes('.')) {
|
|
|
|
|
return { ok: false, reason: 'malformed' };
|
|
|
|
|
}
|
|
|
|
|
const [body, sigB64] = token.split('.', 2);
|
|
|
|
|
if (!body || !sigB64) return { ok: false, reason: 'malformed' };
|
|
|
|
|
|
|
|
|
|
const expected = createHmac('sha256', secret).update(body).digest();
|
|
|
|
|
let provided: Buffer;
|
|
|
|
|
try {
|
|
|
|
|
provided = b64urlDecode(sigB64);
|
|
|
|
|
} catch {
|
|
|
|
|
return { ok: false, reason: 'malformed' };
|
|
|
|
|
}
|
|
|
|
|
if (provided.length !== expected.length) return { ok: false, reason: 'sig-mismatch' };
|
|
|
|
|
if (!timingSafeEqual(provided, expected)) return { ok: false, reason: 'sig-mismatch' };
|
|
|
|
|
|
|
|
|
|
let payload: ProxyTokenPayload;
|
|
|
|
|
try {
|
|
|
|
|
payload = JSON.parse(b64urlDecode(body).toString('utf8')) as ProxyTokenPayload;
|
|
|
|
|
} catch {
|
|
|
|
|
return { ok: false, reason: 'malformed-payload' };
|
|
|
|
|
}
|
|
|
|
|
|
2026-05-05 04:07:03 +02:00
|
|
|
// `Number.isFinite` catches NaN / ±Infinity that a tampered token could
|
|
|
|
|
// otherwise smuggle past the `< Date.now()` comparison (NaN compares
|
|
|
|
|
// false against any number, which would treat the token as eternally
|
|
|
|
|
// valid). Reject non-finite expiries outright.
|
|
|
|
|
if (!Number.isFinite(payload.e) || payload.e * 1000 < Date.now()) {
|
feat(storage): pluggable s3-or-filesystem backend + migration CLI + admin UI
Phase 6a from docs/berth-recommender-and-pdf-plan.md §4.7a + §14.9a. Lays
the storage groundwork for Phase 6b/7 file-bearing schemas (per-berth PDFs,
brochures) without touching those domains yet.
New files:
- src/lib/storage/index.ts StorageBackend interface + per-process
factory keyed on system_settings.
- src/lib/storage/s3.ts S3-compatible backend (MinIO/AWS/B2/R2/
Wasabi/Tigris) wrapping the existing minio
JS client. Includes a healthCheck() used
by the admin "Test connection" button.
- src/lib/storage/filesystem.ts Local filesystem backend with all §14.9a
mitigations baked in.
- src/lib/storage/migrate.ts Shared migration core — pg_advisory_lock,
per-row resumable progress markers,
sha256 round-trip verification, atomic
storage_backend flip on success.
- scripts/migrate-storage.ts Thin CLI shim around runMigration().
- src/app/api/storage/[token]/route.ts
Filesystem proxy GET. Verifies HMAC,
enforces single-use replay protection
via Redis SET NX, streams via NextResponse
ReadableStream with explicit Content-Type
+ Content-Disposition. Node runtime only.
- src/app/api/v1/admin/storage/route.ts
GET status + POST connection test.
- src/app/api/v1/admin/storage/migrate/route.ts
Super-admin-only POST that runs the
exact same runMigration() as the CLI.
- src/app/(dashboard)/[portSlug]/admin/storage/page.tsx
Super-admin admin UI (current backend,
capacity stats, switch button with
dry-run, test connection, backup hint).
- src/components/admin/storage-admin-panel.tsx
Client component for the page above.
§14.9a critical mitigations implemented:
- Path-traversal: storage keys validated against ^[a-zA-Z0-9/_.-]+$;
`..`, `.`, `//`, leading `/`, and overlength keys rejected.
- Realpath: storage root realpath'd at create time, every per-key
resolution checked against the realpath'd prefix.
- Storage root created (or chmod'd) to 0o700.
- Multi-node refusal: FilesystemBackend.create() throws when
MULTI_NODE_DEPLOYMENT=true.
- HMAC token: sha256-HMAC over the (key, expiry, nonce, filename,
content-type) payload. Verified with timingSafeEqual; bad sig,
expired, or invalid-key payloads all return 403.
- Single-use replay: token body cached in Redis SET NX EX 1800s.
- sha256 round-trip: copyAndVerify() re-fetches from the target after
put() and aborts the migration on any mismatch.
- Free-disk pre-flight: when migrating to filesystem, sums byte counts
via source.head() and aborts if free space < total * 1.2.
- pg_advisory_lock(0xc7000a01) prevents concurrent migrations.
- Resumable: per-row progress markers in _storage_migration_progress.
system_settings keys read by the factory (jsonb, no schema change):
storage_backend, storage_s3_endpoint, storage_s3_region,
storage_s3_bucket, storage_s3_access_key,
storage_s3_secret_key_encrypted, storage_s3_force_path_style,
storage_filesystem_root, storage_proxy_hmac_secret_encrypted.
Defaults: storage_backend=`s3`, storage_filesystem_root=`./storage`
(./storage added to .gitignore).
Tests added (34 tests, all green):
- tests/unit/storage/filesystem-backend.test.ts — key validation
allow/reject matrix, realpath escape, 0o700 perms, multi-node
refusal, HMAC token sign/verify/tamper/expire/invalid-key.
- tests/unit/storage/copy-and-verify.test.ts — sha256 mismatch on
round-trip aborts the migration.
- tests/integration/storage/proxy-route.test.ts — happy path, wrong
HMAC secret, expired token, replay rejection.
Phase 6a ships zero file-bearing tables — TABLES_WITH_STORAGE_KEYS is
intentionally empty. berth_pdf_versions and brochure_versions land in
Phase 6b and join the list there. Existing s3_key columns: only
gdpr_export_jobs.storage_key, already named correctly — no rename needed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 03:15:59 +02:00
|
|
|
return { ok: false, reason: 'expired' };
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
validateStorageKey(payload.k);
|
|
|
|
|
} catch {
|
|
|
|
|
return { ok: false, reason: 'invalid-key' };
|
|
|
|
|
}
|
|
|
|
|
return { ok: true, payload };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── backend ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
interface FilesystemConfig {
|
|
|
|
|
root: string;
|
|
|
|
|
/** AES-GCM-encrypted HMAC secret. When absent, falls back to a derived secret. */
|
|
|
|
|
proxyHmacSecretEncrypted: string | null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export class FilesystemBackend implements StorageBackend {
|
|
|
|
|
readonly name = 'filesystem' as const;
|
|
|
|
|
|
|
|
|
|
private rootResolved: string;
|
|
|
|
|
private hmacSecret: string;
|
|
|
|
|
|
|
|
|
|
private constructor(rootResolved: string, hmacSecret: string) {
|
|
|
|
|
this.rootResolved = rootResolved;
|
|
|
|
|
this.hmacSecret = hmacSecret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Throws if multi-node mode is set or the root isn't writable. */
|
|
|
|
|
static async create(cfg: FilesystemConfig): Promise<FilesystemBackend> {
|
|
|
|
|
if (process.env.MULTI_NODE_DEPLOYMENT === 'true') {
|
|
|
|
|
throw new Error(
|
|
|
|
|
'FilesystemBackend cannot start when MULTI_NODE_DEPLOYMENT=true. ' +
|
|
|
|
|
'Use an S3-compatible backend for multi-node deployments.',
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
const rootInput = cfg.root || './storage';
|
|
|
|
|
const rootAbs = path.isAbsolute(rootInput) ? rootInput : path.resolve(process.cwd(), rootInput);
|
|
|
|
|
await fs.mkdir(rootAbs, { recursive: true, mode: 0o700 });
|
|
|
|
|
// Defensive: re-chmod even if it already existed.
|
|
|
|
|
await fs.chmod(rootAbs, 0o700).catch(() => undefined);
|
|
|
|
|
// Use realpath so symlinked roots are flattened to their actual location;
|
|
|
|
|
// we then compare every per-key resolution against this exact prefix.
|
|
|
|
|
const rootResolved = await fs.realpath(rootAbs);
|
|
|
|
|
|
|
|
|
|
const hmacSecret = resolveHmacSecret(cfg.proxyHmacSecretEncrypted);
|
|
|
|
|
logger.info({ root: rootResolved }, 'FilesystemBackend ready');
|
|
|
|
|
return new FilesystemBackend(rootResolved, hmacSecret);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Resolve a (validated) storage key to an absolute path under the root.
|
|
|
|
|
* Throws if the resolved path escapes the storage root via symlink/.. tricks.
|
|
|
|
|
*/
|
|
|
|
|
private resolveKey(key: string): string {
|
|
|
|
|
validateStorageKey(key);
|
|
|
|
|
const joined = path.join(this.rootResolved, key);
|
|
|
|
|
const resolved = path.resolve(joined);
|
|
|
|
|
if (resolved !== this.rootResolved && !resolved.startsWith(this.rootResolved + path.sep)) {
|
|
|
|
|
throw new ValidationError('Storage key escapes storage root');
|
|
|
|
|
}
|
|
|
|
|
return resolved;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async put(
|
|
|
|
|
key: string,
|
|
|
|
|
body: Buffer | NodeJS.ReadableStream,
|
|
|
|
|
opts: PutOpts,
|
|
|
|
|
): Promise<{ key: string; sizeBytes: number; sha256: string }> {
|
|
|
|
|
const target = this.resolveKey(key);
|
|
|
|
|
await fs.mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
|
|
|
|
|
|
|
|
const buffer = Buffer.isBuffer(body) ? body : await streamToBuffer(body);
|
|
|
|
|
const sha256 = opts.sha256 ?? createHash('sha256').update(buffer).digest('hex');
|
|
|
|
|
|
|
|
|
|
// Atomic write via temp + rename so partial writes don't leave half-files.
|
|
|
|
|
const tmp = `${target}.${randomUUID()}.tmp`;
|
|
|
|
|
await fs.writeFile(tmp, buffer, { mode: 0o600 });
|
|
|
|
|
// realpath the temp to make sure the final-rename target resolves correctly
|
|
|
|
|
// even if some segment of the path is a symlink we just created.
|
|
|
|
|
await fs.rename(tmp, target);
|
|
|
|
|
|
|
|
|
|
return { key, sizeBytes: buffer.length, sha256 };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async get(key: string): Promise<NodeJS.ReadableStream> {
|
|
|
|
|
const target = this.resolveKey(key);
|
|
|
|
|
try {
|
|
|
|
|
const stat = await fs.stat(target);
|
|
|
|
|
if (!stat.isFile()) throw new NotFoundError(`Storage object ${key}`);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const code = (err as NodeJS.ErrnoException).code;
|
|
|
|
|
if (code === 'ENOENT') throw new NotFoundError(`Storage object ${key}`);
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
return createReadStream(target);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async head(key: string): Promise<{ sizeBytes: number; contentType: string } | null> {
|
|
|
|
|
const target = this.resolveKey(key);
|
|
|
|
|
try {
|
|
|
|
|
const stat = await fs.stat(target);
|
|
|
|
|
if (!stat.isFile()) return null;
|
|
|
|
|
// Filesystem doesn't track content-type. Caller should consult the DB
|
|
|
|
|
// (or sniff via ext) — we return application/octet-stream as a default.
|
|
|
|
|
return { sizeBytes: stat.size, contentType: extToContentType(target) };
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const code = (err as NodeJS.ErrnoException).code;
|
|
|
|
|
if (code === 'ENOENT') return null;
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async delete(key: string): Promise<void> {
|
|
|
|
|
const target = this.resolveKey(key);
|
|
|
|
|
try {
|
|
|
|
|
await fs.unlink(target);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const code = (err as NodeJS.ErrnoException).code;
|
|
|
|
|
if (code === 'ENOENT') return;
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Filesystem mode never exposes a direct upload URL. The CRM-internal proxy
|
|
|
|
|
* accepts uploads via the regular API surface (multipart POST to /api/v1/...
|
|
|
|
|
* or PUT to /api/storage/[token]). We return a placeholder PUT URL pointing
|
|
|
|
|
* at the proxy so the contract stays uniform.
|
|
|
|
|
*/
|
|
|
|
|
async presignUpload(
|
|
|
|
|
key: string,
|
|
|
|
|
opts: PresignOpts,
|
|
|
|
|
): Promise<{ url: string; method: 'PUT' | 'POST' }> {
|
|
|
|
|
validateStorageKey(key);
|
|
|
|
|
const expiresAt = Math.floor(Date.now() / 1000) + (opts.expirySeconds ?? 900);
|
|
|
|
|
const token = signProxyToken(
|
|
|
|
|
{ k: key, e: expiresAt, n: randomUUID(), c: opts.contentType },
|
|
|
|
|
this.hmacSecret,
|
|
|
|
|
);
|
|
|
|
|
return { url: `/api/storage/${token}`, method: 'PUT' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async presignDownload(key: string, opts: PresignOpts): Promise<{ url: string; expiresAt: Date }> {
|
|
|
|
|
validateStorageKey(key);
|
|
|
|
|
const expirySec = opts.expirySeconds ?? 900;
|
|
|
|
|
const expiresAtSec = Math.floor(Date.now() / 1000) + expirySec;
|
|
|
|
|
const token = signProxyToken(
|
|
|
|
|
{ k: key, e: expiresAtSec, n: randomUUID(), f: opts.filename, c: opts.contentType },
|
|
|
|
|
this.hmacSecret,
|
|
|
|
|
);
|
2026-05-05 04:07:03 +02:00
|
|
|
// ABSOLUTE URL: send-out emails interpolate this verbatim into the
|
|
|
|
|
// recipient's inbox. A relative path is unreachable from a mail
|
|
|
|
|
// client. APP_URL strips any trailing slash to keep the join clean.
|
|
|
|
|
const origin = env.APP_URL.replace(/\/$/, '');
|
feat(storage): pluggable s3-or-filesystem backend + migration CLI + admin UI
Phase 6a from docs/berth-recommender-and-pdf-plan.md §4.7a + §14.9a. Lays
the storage groundwork for Phase 6b/7 file-bearing schemas (per-berth PDFs,
brochures) without touching those domains yet.
New files:
- src/lib/storage/index.ts StorageBackend interface + per-process
factory keyed on system_settings.
- src/lib/storage/s3.ts S3-compatible backend (MinIO/AWS/B2/R2/
Wasabi/Tigris) wrapping the existing minio
JS client. Includes a healthCheck() used
by the admin "Test connection" button.
- src/lib/storage/filesystem.ts Local filesystem backend with all §14.9a
mitigations baked in.
- src/lib/storage/migrate.ts Shared migration core — pg_advisory_lock,
per-row resumable progress markers,
sha256 round-trip verification, atomic
storage_backend flip on success.
- scripts/migrate-storage.ts Thin CLI shim around runMigration().
- src/app/api/storage/[token]/route.ts
Filesystem proxy GET. Verifies HMAC,
enforces single-use replay protection
via Redis SET NX, streams via NextResponse
ReadableStream with explicit Content-Type
+ Content-Disposition. Node runtime only.
- src/app/api/v1/admin/storage/route.ts
GET status + POST connection test.
- src/app/api/v1/admin/storage/migrate/route.ts
Super-admin-only POST that runs the
exact same runMigration() as the CLI.
- src/app/(dashboard)/[portSlug]/admin/storage/page.tsx
Super-admin admin UI (current backend,
capacity stats, switch button with
dry-run, test connection, backup hint).
- src/components/admin/storage-admin-panel.tsx
Client component for the page above.
§14.9a critical mitigations implemented:
- Path-traversal: storage keys validated against ^[a-zA-Z0-9/_.-]+$;
`..`, `.`, `//`, leading `/`, and overlength keys rejected.
- Realpath: storage root realpath'd at create time, every per-key
resolution checked against the realpath'd prefix.
- Storage root created (or chmod'd) to 0o700.
- Multi-node refusal: FilesystemBackend.create() throws when
MULTI_NODE_DEPLOYMENT=true.
- HMAC token: sha256-HMAC over the (key, expiry, nonce, filename,
content-type) payload. Verified with timingSafeEqual; bad sig,
expired, or invalid-key payloads all return 403.
- Single-use replay: token body cached in Redis SET NX EX 1800s.
- sha256 round-trip: copyAndVerify() re-fetches from the target after
put() and aborts the migration on any mismatch.
- Free-disk pre-flight: when migrating to filesystem, sums byte counts
via source.head() and aborts if free space < total * 1.2.
- pg_advisory_lock(0xc7000a01) prevents concurrent migrations.
- Resumable: per-row progress markers in _storage_migration_progress.
system_settings keys read by the factory (jsonb, no schema change):
storage_backend, storage_s3_endpoint, storage_s3_region,
storage_s3_bucket, storage_s3_access_key,
storage_s3_secret_key_encrypted, storage_s3_force_path_style,
storage_filesystem_root, storage_proxy_hmac_secret_encrypted.
Defaults: storage_backend=`s3`, storage_filesystem_root=`./storage`
(./storage added to .gitignore).
Tests added (34 tests, all green):
- tests/unit/storage/filesystem-backend.test.ts — key validation
allow/reject matrix, realpath escape, 0o700 perms, multi-node
refusal, HMAC token sign/verify/tamper/expire/invalid-key.
- tests/unit/storage/copy-and-verify.test.ts — sha256 mismatch on
round-trip aborts the migration.
- tests/integration/storage/proxy-route.test.ts — happy path, wrong
HMAC secret, expired token, replay rejection.
Phase 6a ships zero file-bearing tables — TABLES_WITH_STORAGE_KEYS is
intentionally empty. berth_pdf_versions and brochure_versions land in
Phase 6b and join the list there. Existing s3_key columns: only
gdpr_export_jobs.storage_key, already named correctly — no rename needed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 03:15:59 +02:00
|
|
|
return {
|
2026-05-05 04:07:03 +02:00
|
|
|
url: `${origin}/api/storage/${token}`,
|
feat(storage): pluggable s3-or-filesystem backend + migration CLI + admin UI
Phase 6a from docs/berth-recommender-and-pdf-plan.md §4.7a + §14.9a. Lays
the storage groundwork for Phase 6b/7 file-bearing schemas (per-berth PDFs,
brochures) without touching those domains yet.
New files:
- src/lib/storage/index.ts StorageBackend interface + per-process
factory keyed on system_settings.
- src/lib/storage/s3.ts S3-compatible backend (MinIO/AWS/B2/R2/
Wasabi/Tigris) wrapping the existing minio
JS client. Includes a healthCheck() used
by the admin "Test connection" button.
- src/lib/storage/filesystem.ts Local filesystem backend with all §14.9a
mitigations baked in.
- src/lib/storage/migrate.ts Shared migration core — pg_advisory_lock,
per-row resumable progress markers,
sha256 round-trip verification, atomic
storage_backend flip on success.
- scripts/migrate-storage.ts Thin CLI shim around runMigration().
- src/app/api/storage/[token]/route.ts
Filesystem proxy GET. Verifies HMAC,
enforces single-use replay protection
via Redis SET NX, streams via NextResponse
ReadableStream with explicit Content-Type
+ Content-Disposition. Node runtime only.
- src/app/api/v1/admin/storage/route.ts
GET status + POST connection test.
- src/app/api/v1/admin/storage/migrate/route.ts
Super-admin-only POST that runs the
exact same runMigration() as the CLI.
- src/app/(dashboard)/[portSlug]/admin/storage/page.tsx
Super-admin admin UI (current backend,
capacity stats, switch button with
dry-run, test connection, backup hint).
- src/components/admin/storage-admin-panel.tsx
Client component for the page above.
§14.9a critical mitigations implemented:
- Path-traversal: storage keys validated against ^[a-zA-Z0-9/_.-]+$;
`..`, `.`, `//`, leading `/`, and overlength keys rejected.
- Realpath: storage root realpath'd at create time, every per-key
resolution checked against the realpath'd prefix.
- Storage root created (or chmod'd) to 0o700.
- Multi-node refusal: FilesystemBackend.create() throws when
MULTI_NODE_DEPLOYMENT=true.
- HMAC token: sha256-HMAC over the (key, expiry, nonce, filename,
content-type) payload. Verified with timingSafeEqual; bad sig,
expired, or invalid-key payloads all return 403.
- Single-use replay: token body cached in Redis SET NX EX 1800s.
- sha256 round-trip: copyAndVerify() re-fetches from the target after
put() and aborts the migration on any mismatch.
- Free-disk pre-flight: when migrating to filesystem, sums byte counts
via source.head() and aborts if free space < total * 1.2.
- pg_advisory_lock(0xc7000a01) prevents concurrent migrations.
- Resumable: per-row progress markers in _storage_migration_progress.
system_settings keys read by the factory (jsonb, no schema change):
storage_backend, storage_s3_endpoint, storage_s3_region,
storage_s3_bucket, storage_s3_access_key,
storage_s3_secret_key_encrypted, storage_s3_force_path_style,
storage_filesystem_root, storage_proxy_hmac_secret_encrypted.
Defaults: storage_backend=`s3`, storage_filesystem_root=`./storage`
(./storage added to .gitignore).
Tests added (34 tests, all green):
- tests/unit/storage/filesystem-backend.test.ts — key validation
allow/reject matrix, realpath escape, 0o700 perms, multi-node
refusal, HMAC token sign/verify/tamper/expire/invalid-key.
- tests/unit/storage/copy-and-verify.test.ts — sha256 mismatch on
round-trip aborts the migration.
- tests/integration/storage/proxy-route.test.ts — happy path, wrong
HMAC secret, expired token, replay rejection.
Phase 6a ships zero file-bearing tables — TABLES_WITH_STORAGE_KEYS is
intentionally empty. berth_pdf_versions and brochure_versions land in
Phase 6b and join the list there. Existing s3_key columns: only
gdpr_export_jobs.storage_key, already named correctly — no rename needed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 03:15:59 +02:00
|
|
|
expiresAt: new Date(expiresAtSec * 1000),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Used by the proxy route — returns the validated absolute path. */
|
|
|
|
|
resolveKeyForProxy(key: string): string {
|
|
|
|
|
return this.resolveKey(key);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Used by the proxy route — same HMAC secret as presignDownload. */
|
|
|
|
|
getHmacSecret(): string {
|
|
|
|
|
return this.hmacSecret;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── helpers ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
function resolveHmacSecret(encryptedSecret: string | null): string {
|
|
|
|
|
if (encryptedSecret) {
|
|
|
|
|
try {
|
|
|
|
|
return decrypt(encryptedSecret);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
logger.error({ err }, 'Failed to decrypt storage_proxy_hmac_secret_encrypted');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Derive a stable per-process secret from BETTER_AUTH_SECRET so dev mode
|
|
|
|
|
// works without explicit configuration. In production the admin UI writes
|
|
|
|
|
// an encrypted random secret.
|
|
|
|
|
const seed = process.env.BETTER_AUTH_SECRET ?? env.BETTER_AUTH_SECRET ?? 'storage-default';
|
|
|
|
|
return createHash('sha256').update(`storage-proxy:${seed}`).digest('hex');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
|
|
|
|
const chunks: Buffer[] = [];
|
|
|
|
|
for await (const chunk of stream as Readable) {
|
|
|
|
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string));
|
|
|
|
|
}
|
|
|
|
|
return Buffer.concat(chunks);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function extToContentType(filename: string): string {
|
|
|
|
|
const ext = path.extname(filename).toLowerCase();
|
|
|
|
|
switch (ext) {
|
|
|
|
|
case '.pdf':
|
|
|
|
|
return 'application/pdf';
|
|
|
|
|
case '.png':
|
|
|
|
|
return 'image/png';
|
|
|
|
|
case '.jpg':
|
|
|
|
|
case '.jpeg':
|
|
|
|
|
return 'image/jpeg';
|
|
|
|
|
case '.json':
|
|
|
|
|
return 'application/json';
|
|
|
|
|
case '.txt':
|
|
|
|
|
return 'text/plain';
|
|
|
|
|
case '.csv':
|
|
|
|
|
return 'text/csv';
|
|
|
|
|
case '.zip':
|
|
|
|
|
return 'application/zip';
|
|
|
|
|
default:
|
|
|
|
|
return 'application/octet-stream';
|
|
|
|
|
}
|
|
|
|
|
}
|