From cf430d70c3373db6475ff9093ef872ae5c5cc32e Mon Sep 17 00:00:00 2001 From: Matt Ciaccio Date: Tue, 5 May 2026 18:41:02 +0200 Subject: [PATCH] fix(storage): route every file op through getStorageBackend() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes 12 direct minioClient.{put,get,remove}Object call sites that bypassed the pluggable storage abstraction. Filesystem-mode deploys (MULTI_NODE_DEPLOYMENT=false, storage_backend=filesystem) silently broke at every site: GDPR export, invoice PDF, EOI generation, portal download, file upload, folder create/rename/delete, signed PDF land, maintenance cleanup, etc. Each site now resolves the active backend and uses its put/get/delete + the new presignDownloadUrl() helper. Folder marker objects in /files/folders/* keep the same on-the-wire shape but route through the backend. A future refactor should move folder bookkeeping to a DB-backed virtual-folder table (see audit HIGH §3 follow-up note in the route file). Sites left untouched: src/lib/services/system-monitoring.service.ts and src/app/api/ready/route.ts use minioClient.bucketExists as an S3- specific health probe — those are correctly mode-aware and stay. Refs: docs/audit-comprehensive-2026-05-05.md HIGH §3 (auditor-D Issue 1) Co-Authored-By: Claude Opus 4.7 (1M context) --- .../api/v1/files/folders/[...path]/route.ts | 16 ++++---- src/app/api/v1/files/folders/route.ts | 16 +++++--- src/lib/queue/workers/maintenance.ts | 5 +-- src/lib/services/document-templates.ts | 41 +++++++++++-------- src/lib/services/documents.service.ts | 25 ++++++----- src/lib/services/email-compose.service.ts | 4 +- src/lib/services/files.ts | 18 ++++---- src/lib/services/gdpr-export.service.ts | 15 +++---- src/lib/services/invoices.ts | 16 ++++---- src/lib/services/portal.service.ts | 4 +- src/lib/services/reports.service.ts | 29 ++++++------- src/lib/storage/index.ts | 18 ++++++++ 12 files changed, 121 insertions(+), 86 deletions(-) diff --git a/src/app/api/v1/files/folders/[...path]/route.ts b/src/app/api/v1/files/folders/[...path]/route.ts index 4fcc63c..dd22dbf 100644 --- a/src/app/api/v1/files/folders/[...path]/route.ts +++ b/src/app/api/v1/files/folders/[...path]/route.ts @@ -4,8 +4,7 @@ import { z } from 'zod'; import { withAuth, withPermission } from '@/lib/api/helpers'; import { parseBody } from '@/lib/api/route-helpers'; import { errorResponse, ValidationError } from '@/lib/errors'; -import { minioClient } from '@/lib/minio'; -import { env } from '@/lib/env'; +import { getStorageBackend } from '@/lib/storage'; const renameFolderSchema = z.object({ newPath: z.string().min(1).max(500), @@ -38,11 +37,14 @@ export const PATCH = withAuth( const oldKey = `${ctx.portSlug}/${safeCurrent}${safeCurrent.endsWith('/') ? '' : '/'}`; const newKey = `${ctx.portSlug}/${safeNew}${safeNew.endsWith('/') ? '' : '/'}`; - // Create new marker, remove old - await minioClient.putObject(env.MINIO_BUCKET, newKey, Buffer.alloc(0), 0, { - 'Content-Type': 'application/x-directory', + // Create new marker, remove old. Routed through the active backend + // so filesystem mode doesn't crash. + const backend = await getStorageBackend(); + await backend.put(newKey, Buffer.alloc(0), { + contentType: 'application/x-directory', + sizeBytes: 0, }); - await minioClient.removeObject(env.MINIO_BUCKET, oldKey); + await backend.delete(oldKey); return NextResponse.json({ data: { path: newKey } }); } catch (error) { @@ -65,7 +67,7 @@ export const DELETE = withAuth( } const folderKey = `${ctx.portSlug}/${safePath}${safePath.endsWith('/') ? '' : '/'}`; - await minioClient.removeObject(env.MINIO_BUCKET, folderKey); + await (await getStorageBackend()).delete(folderKey); return new NextResponse(null, { status: 204 }); } catch (error) { diff --git a/src/app/api/v1/files/folders/route.ts b/src/app/api/v1/files/folders/route.ts index c48c5db..fef2b46 100644 --- a/src/app/api/v1/files/folders/route.ts +++ b/src/app/api/v1/files/folders/route.ts @@ -4,8 +4,7 @@ import { z } from 'zod'; import { withAuth, withPermission } from '@/lib/api/helpers'; import { parseBody } from '@/lib/api/route-helpers'; import { errorResponse, ValidationError } from '@/lib/errors'; -import { minioClient } from '@/lib/minio'; -import { env } from '@/lib/env'; +import { getStorageBackend } from '@/lib/storage'; const createFolderSchema = z.object({ path: z.string().min(1).max(500), @@ -29,9 +28,16 @@ export const POST = withAuth( const folderKey = `${ctx.portSlug}/${safePath}${safePath.endsWith('/') ? '' : '/'}`; - // Create zero-byte marker object in MinIO - await minioClient.putObject(env.MINIO_BUCKET, folderKey, Buffer.alloc(0), 0, { - 'Content-Type': 'application/x-directory', + // Zero-byte marker through the active storage backend. S3 stores it + // as an empty object; the filesystem backend currently materializes + // it as an empty file (a future refactor should move folder + // bookkeeping to a DB-backed virtual-folder table — see + // docs/audit-comprehensive-2026-05-05.md HIGH §3 follow-up). + await ( + await getStorageBackend() + ).put(folderKey, Buffer.alloc(0), { + contentType: 'application/x-directory', + sizeBytes: 0, }); return NextResponse.json({ data: { path: folderKey } }, { status: 201 }); diff --git a/src/lib/queue/workers/maintenance.ts b/src/lib/queue/workers/maintenance.ts index 42cc72c..b196e54 100644 --- a/src/lib/queue/workers/maintenance.ts +++ b/src/lib/queue/workers/maintenance.ts @@ -6,9 +6,8 @@ import { db } from '@/lib/db'; import { formSubmissions } from '@/lib/db/schema/documents'; import { gdprExports } from '@/lib/db/schema/gdpr'; import { aiUsageLedger } from '@/lib/db/schema/ai-usage'; -import { env } from '@/lib/env'; import { logger } from '@/lib/logger'; -import { minioClient } from '@/lib/minio'; +import { getStorageBackend } from '@/lib/storage'; import { QUEUE_CONFIGS } from '@/lib/queue'; /** AI usage rows older than this are deleted by the retention job. */ @@ -87,7 +86,7 @@ export const maintenanceWorker = new Worker( for (const row of expired) { try { if (row.storageKey) { - await minioClient.removeObject(env.MINIO_BUCKET, row.storageKey); + await (await getStorageBackend()).delete(row.storageKey); } await db.delete(gdprExports).where(eq(gdprExports.id, row.id)); removed++; diff --git a/src/lib/services/document-templates.ts b/src/lib/services/document-templates.ts index c38b892..a3a64da 100644 --- a/src/lib/services/document-templates.ts +++ b/src/lib/services/document-templates.ts @@ -13,7 +13,8 @@ import { createAuditLog, type AuditMeta } from '@/lib/audit'; import { diffEntity } from '@/lib/entity-diff'; import { NotFoundError, ValidationError } from '@/lib/errors'; import { emitToRoom } from '@/lib/socket/server'; -import { minioClient, buildStoragePath } from '@/lib/minio'; +import { buildStoragePath } from '@/lib/minio'; +import { getStorageBackend } from '@/lib/storage'; import { env } from '@/lib/env'; import { logger } from '@/lib/logger'; import { generatePdf } from '@/lib/pdf/generate'; @@ -525,13 +526,14 @@ export async function generateFromTemplate( 'pdf', ); - await minioClient.putObject( - env.MINIO_BUCKET, - storagePath, - Buffer.from(pdfBytes), - pdfBytes.byteLength, - { 'Content-Type': 'application/pdf' }, - ); + { + const buffer = Buffer.from(pdfBytes); + const backend = await getStorageBackend(); + await backend.put(storagePath, buffer, { + contentType: 'application/pdf', + sizeBytes: buffer.length, + }); + } // Create file record const [fileRecord] = await db @@ -661,13 +663,14 @@ async function generateEoiFromSourcePdf( 'pdf', ); - await minioClient.putObject( - env.MINIO_BUCKET, - storagePath, - Buffer.from(pdfBytes), - pdfBytes.byteLength, - { 'Content-Type': 'application/pdf' }, - ); + { + const buffer = Buffer.from(pdfBytes); + const backend = await getStorageBackend(); + await backend.put(storagePath, buffer, { + contentType: 'application/pdf', + sizeBytes: buffer.length, + }); + } const [fileRecord] = await db .insert(files) @@ -809,11 +812,13 @@ async function generateAndSignViaInApp( file: DbFile; }); - // Fetch PDF bytes from MinIO to send to Documenso - const pdfStream = await minioClient.getObject(env.MINIO_BUCKET, file.storagePath); + // Fetch PDF bytes from the active storage backend to send to Documenso. + const pdfStream = await (await getStorageBackend()).get(file.storagePath); const chunks: Buffer[] = []; for await (const chunk of pdfStream) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBuffer)); + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); } const pdfBase64 = Buffer.concat(chunks).toString('base64'); diff --git a/src/lib/services/documents.service.ts b/src/lib/services/documents.service.ts index fee677b..43ddcf5 100644 --- a/src/lib/services/documents.service.ts +++ b/src/lib/services/documents.service.ts @@ -20,7 +20,8 @@ import { createAuditLog, type AuditMeta } from '@/lib/audit'; import { diffEntity } from '@/lib/entity-diff'; import { NotFoundError, ValidationError, ConflictError } from '@/lib/errors'; import { emitToRoom } from '@/lib/socket/server'; -import { minioClient, buildStoragePath } from '@/lib/minio'; +import { buildStoragePath } from '@/lib/minio'; +import { getStorageBackend } from '@/lib/storage'; import { env } from '@/lib/env'; import { logger } from '@/lib/logger'; import { evaluateRule } from '@/lib/services/berth-rules-engine'; @@ -549,7 +550,7 @@ export async function sendForSigning(documentId: string, portId: string, meta: A const fileRecord = await db.query.files.findFirst({ where: eq(files.id, doc.fileId) }); if (!fileRecord) throw new NotFoundError('File'); - const fileStream = await minioClient.getObject(env.MINIO_BUCKET, fileRecord.storagePath); + const fileStream = await (await getStorageBackend()).get(fileRecord.storagePath); const chunks: Buffer[] = []; for await (const chunk of fileStream) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); @@ -659,8 +660,11 @@ export async function uploadSignedManually( const fileId = crypto.randomUUID(); const storagePath = buildStoragePath(port.slug, 'eoi-signed', documentId, fileId, 'pdf'); - await minioClient.putObject(env.MINIO_BUCKET, storagePath, fileData.buffer, fileData.size, { - 'Content-Type': fileData.mimeType, + await ( + await getStorageBackend() + ).put(storagePath, fileData.buffer, { + contentType: fileData.mimeType, + sizeBytes: fileData.size, }); const [fileRecord] = await db @@ -860,13 +864,12 @@ export async function handleDocumentCompleted(eventData: { documentId: string }) const fileId = crypto.randomUUID(); const storagePath = buildStoragePath(port.slug, 'eoi-signed', doc.id, fileId, 'pdf'); - await minioClient.putObject( - env.MINIO_BUCKET, - storagePath, - signedPdfBuffer, - signedPdfBuffer.length, - { 'Content-Type': 'application/pdf' }, - ); + await ( + await getStorageBackend() + ).put(storagePath, signedPdfBuffer, { + contentType: 'application/pdf', + sizeBytes: signedPdfBuffer.length, + }); const [fileRecord] = await db .insert(files) diff --git a/src/lib/services/email-compose.service.ts b/src/lib/services/email-compose.service.ts index 0541c4c..a7c4b01 100644 --- a/src/lib/services/email-compose.service.ts +++ b/src/lib/services/email-compose.service.ts @@ -114,8 +114,8 @@ export async function sendEmail( where: eq(files.id, ref.fileId), }); if (!file) throw new NotFoundError('File'); - const { minioClient } = await import('@/lib/minio'); - const stream = await minioClient.getObject(file.storageBucket, file.storagePath); + const { getStorageBackend } = await import('@/lib/storage'); + const stream = await (await getStorageBackend()).get(file.storagePath); const chunks: Buffer[] = []; for await (const chunk of stream) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); diff --git a/src/lib/services/files.ts b/src/lib/services/files.ts index 90f3daf..d78229f 100644 --- a/src/lib/services/files.ts +++ b/src/lib/services/files.ts @@ -7,7 +7,7 @@ import { berthMaintenanceLog } from '@/lib/db/schema/berths'; import { createAuditLog, type AuditMeta } from '@/lib/audit'; import { ConflictError, NotFoundError, ValidationError } from '@/lib/errors'; import { emitToRoom } from '@/lib/socket/server'; -import { minioClient, getPresignedUrl } from '@/lib/minio'; +import { getStorageBackend, presignDownloadUrl } from '@/lib/storage'; import { buildListQuery } from '@/lib/db/query-builder'; import { env } from '@/lib/env'; import { @@ -50,8 +50,10 @@ export async function uploadFile( const sanitizedOriginal = sanitizeFilename(file.originalName); const sanitizedFilename = sanitizeFilename(data.filename); - await minioClient.putObject(env.MINIO_BUCKET, storagePath, file.buffer, file.size, { - 'Content-Type': file.mimeType, + const backend = await getStorageBackend(); + await backend.put(storagePath, file.buffer, { + contentType: file.mimeType, + sizeBytes: file.size, }); const [record] = await db @@ -93,7 +95,7 @@ export async function uploadFile( export async function getDownloadUrl(id: string, portId: string) { const file = await getFileById(id, portId); - const url = await getPresignedUrl(file.storagePath); + const url = await presignDownloadUrl(file.storagePath); return { url, filename: file.filename }; } @@ -104,7 +106,7 @@ export async function getPreviewUrl(id: string, portId: string) { throw new ValidationError('This file type cannot be previewed'); } - const url = await getPresignedUrl(file.storagePath); + const url = await presignDownloadUrl(file.storagePath); return { url, mimeType: file.mimeType }; } @@ -183,8 +185,10 @@ export async function deleteFile(id: string, portId: string, meta: AuditMeta) { throw new ConflictError('File cannot be deleted because it is referenced by other records'); } - // Delete from MinIO first, then DB - await minioClient.removeObject(env.MINIO_BUCKET, existing.storagePath); + // Delete the blob first, then DB. The storage backend's delete is + // idempotent, so a partial replay (worker crashed mid-delete) does not + // throw on the missing-object retry. + await (await getStorageBackend()).delete(existing.storagePath); await db.delete(files).where(and(eq(files.id, id), eq(files.portId, portId))); diff --git a/src/lib/services/gdpr-export.service.ts b/src/lib/services/gdpr-export.service.ts index 4cf1e43..7392cc2 100644 --- a/src/lib/services/gdpr-export.service.ts +++ b/src/lib/services/gdpr-export.service.ts @@ -18,10 +18,9 @@ import { db } from '@/lib/db'; import { gdprExports, type GdprExport } from '@/lib/db/schema/gdpr'; import { clients, clientContacts } from '@/lib/db/schema/clients'; import { ports } from '@/lib/db/schema/ports'; -import { env } from '@/lib/env'; import { NotFoundError, ValidationError } from '@/lib/errors'; import { logger } from '@/lib/logger'; -import { minioClient, getPresignedUrl } from '@/lib/minio'; +import { getStorageBackend, presignDownloadUrl } from '@/lib/storage'; import { getQueue } from '@/lib/queue'; import { createAuditLog } from '@/lib/audit'; import { buildClientBundle, renderBundleHtml } from '@/lib/services/gdpr-bundle-builder'; @@ -163,9 +162,11 @@ export async function processGdprExportJob(input: ProcessJobInput): Promise { const portSlug = port?.slug ?? 'port'; // 6. Build inputs (pass portName) - const inputs = (config.buildInputs as (data: unknown, portName: string) => Record[])(data, portName); + const inputs = ( + config.buildInputs as (data: unknown, portName: string) => Record[] + )(data, portName); // 7. Generate PDF const pdfBytes = await generatePdf(config.template, inputs); @@ -217,15 +216,13 @@ export async function generateReport(reportJobId: string): Promise { const fileId = crypto.randomUUID(); const storagePath = buildStoragePath(portSlug, 'reports', reportJobId, fileId, 'pdf'); - // 9. Upload PDF to MinIO + // 9. Upload PDF via the active storage backend (filesystem or s3) const buffer = Buffer.from(pdfBytes); - await minioClient.putObject( - env.MINIO_BUCKET, - storagePath, - buffer, - buffer.length, - { 'Content-Type': 'application/pdf', 'report-type': reportType }, - ); + const backend = await getStorageBackend(); + await backend.put(storagePath, buffer, { + contentType: 'application/pdf', + sizeBytes: buffer.length, + }); // 10. Insert into files table const [fileRecord] = await db diff --git a/src/lib/storage/index.ts b/src/lib/storage/index.ts index a8b96d7..be4587e 100644 --- a/src/lib/storage/index.ts +++ b/src/lib/storage/index.ts @@ -194,6 +194,24 @@ async function buildBackend(cfg: StorageConfigSnapshot): Promise }); } +// ─── url helpers ──────────────────────────────────────────────────────────── + +/** + * Convenience wrapper that returns just the presigned download URL — the most + * common need at call sites that don't track expiry. Mirrors the legacy + * `getPresignedUrl(key)` helper in `@/lib/minio` but routes through the + * active backend so filesystem-mode deployments work too. + */ +export async function presignDownloadUrl( + key: string, + expirySeconds = 900, + filename?: string, +): Promise { + const backend = await getStorageBackend(); + const { url } = await backend.presignDownload(key, { expirySeconds, filename }); + return url; +} + // ─── re-exports ───────────────────────────────────────────────────────────── export { S3Backend } from './s3';