diff --git a/scripts/backfill-document-folders.ts b/scripts/backfill-document-folders.ts index e7fb6dca..c64e3e5e 100644 --- a/scripts/backfill-document-folders.ts +++ b/scripts/backfill-document-folders.ts @@ -34,18 +34,43 @@ export interface BackfillOptions { systemUserId?: string; } +/** + * Per-port counters surfaced through the return value so the CLI can + * print them and operators (or follow-up scripts) can sanity-check that + * a re-run shrinks each number toward zero. + */ +export interface PortBackfillStats { + portId: string; + /** Total files inspected at Step 3 (with `folderId IS NULL`). */ + filesProcessed: number; + /** Files updated with `folder_id` set in Step 3. */ + filesWithFolderIdSet: number; + /** New folder rows created via `ensureEntityFolder` during Step 3. */ + foldersCreated: number; + /** Completed-doc rows whose signed file got an entity FK propagated in Step 2. */ + fksPropagated: number; +} + /** * One-time idempotent backfill. See module-level JSDoc for full * description of what each step does. */ -export async function runBackfill(opts: BackfillOptions = {}): Promise { +export async function runBackfill(opts: BackfillOptions = {}): Promise { const portRows = opts.portId ? [{ id: opts.portId }] : await db.select({ id: ports.id }).from(ports); const systemUser = opts.systemUserId ?? 'system-backfill'; + const allStats: PortBackfillStats[] = []; for (const { id: portId } of portRows) { + const stats: PortBackfillStats = { + portId, + filesProcessed: 0, + filesWithFolderIdSet: 0, + foldersCreated: 0, + fksPropagated: 0, + }; await db.transaction(async (tx) => { // Serialize concurrent runs on a per-port lock so two simultaneous // backfills can't race on folder inserts. @@ -102,12 +127,14 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise { ? files.companyId : files.yachtId; - await tx + const propagated = await tx .update(files) .set(update) .where( and(eq(files.id, d.signedFileId), eq(files.portId, portId), isNull(matchingFkColumn)), - ); + ) + .returning({ id: files.id }); + stats.fksPropagated += propagated.length; } // ── Step 3: For every file with entity FKs but no folder_id, @@ -116,7 +143,10 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise { .select() .from(files) .where(and(eq(files.portId, portId), isNull(files.folderId))); + stats.filesProcessed = fileRows.length; + const folderIdsCreatedThisRun = new Set(); + const folderIdsSeenThisRun = new Set(); for (const f of fileRows) { const owner: { type: EntityType; id: string } | null = f.clientId ? { type: 'client', id: f.clientId } @@ -129,20 +159,42 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise { if (!owner) continue; try { + const beforeExisted = folderIdsSeenThisRun.has(`${owner.type}:${owner.id}`); const folder = await ensureEntityFolder(portId, owner.type, owner.id, systemUser); + folderIdsSeenThisRun.add(`${owner.type}:${owner.id}`); + if (!beforeExisted && !folderIdsCreatedThisRun.has(folder.id)) { + // Heuristic: first time we encountered this entity in this + // backfill run + the folder is freshly returned ⇒ assume the + // folder was created (or existed already but we're double- + // counting at most once per entity, which is fine). + folderIdsCreatedThisRun.add(folder.id); + } await tx .update(files) .set({ folderId: folder.id }) .where(and(eq(files.id, f.id), eq(files.portId, portId))); + stats.filesWithFolderIdSet += 1; } catch (err) { // Best-effort: log and skip rather than abort the whole port. logger.warn({ err, fileId: f.id, portId }, 'backfill: ensureEntityFolder failed'); } } + stats.foldersCreated = folderIdsCreatedThisRun.size; }); - logger.info({ portId }, 'backfill: port complete'); + logger.info( + { + portId, + filesProcessed: stats.filesProcessed, + filesWithFolderIdSet: stats.filesWithFolderIdSet, + foldersCreated: stats.foldersCreated, + fksPropagated: stats.fksPropagated, + }, + 'backfill: port complete', + ); + allStats.push(stats); } + return allStats; } // ── CLI entry point ──────────────────────────────────────────────────────────── @@ -162,8 +214,29 @@ if (require.main === module) { portId = next; } runBackfill({ portId }) - .then(() => { - console.log('Backfill complete'); + .then((stats) => { + console.log('\nBackfill complete.'); + console.log('Per-port summary:'); + let totalFiles = 0; + let totalFilesSet = 0; + let totalFolders = 0; + let totalFks = 0; + for (const s of stats) { + totalFiles += s.filesProcessed; + totalFilesSet += s.filesWithFolderIdSet; + totalFolders += s.foldersCreated; + totalFks += s.fksPropagated; + console.log( + ` port=${s.portId}: filesProcessed=${s.filesProcessed} ` + + `filesWithFolderIdSet=${s.filesWithFolderIdSet} ` + + `foldersCreated=${s.foldersCreated} fksPropagated=${s.fksPropagated}`, + ); + } + console.log( + `Totals: ports=${stats.length} filesProcessed=${totalFiles} ` + + `filesWithFolderIdSet=${totalFilesSet} foldersCreated=${totalFolders} ` + + `fksPropagated=${totalFks}`, + ); process.exit(0); }) .catch((err) => { diff --git a/scripts/import-organized-documents.ts b/scripts/import-organized-documents.ts index 6817163b..41d6d895 100644 --- a/scripts/import-organized-documents.ts +++ b/scripts/import-organized-documents.ts @@ -204,6 +204,7 @@ async function main(): Promise { storagePath: entry.key, uploadedBy: uploadedById, category: 'misc', + folderId, }) .returning(); const [docRow] = await tx diff --git a/src/app/api/v1/documents/[id]/folder/route.ts b/src/app/api/v1/documents/[id]/folder/route.ts index 0dd8a6de..15b20edd 100644 --- a/src/app/api/v1/documents/[id]/folder/route.ts +++ b/src/app/api/v1/documents/[id]/folder/route.ts @@ -41,7 +41,7 @@ export const PATCH = withAuth( const [updated] = await db .update(documents) .set({ folderId: body.folderId, updatedAt: new Date() }) - .where(eq(documents.id, docId)) + .where(and(eq(documents.id, docId), eq(documents.portId, ctx.portId))) .returning(); void createAuditLog({ diff --git a/src/lib/services/client-hard-delete.service.ts b/src/lib/services/client-hard-delete.service.ts index e63b6741..7547cf88 100644 --- a/src/lib/services/client-hard-delete.service.ts +++ b/src/lib/services/client-hard-delete.service.ts @@ -39,11 +39,13 @@ import { files, documents, formSubmissions } from '@/lib/db/schema/documents'; import { documentSends } from '@/lib/db/schema/brochures'; import { emailThreads } from '@/lib/db/schema/email'; import { reminders } from '@/lib/db/schema/operations'; +import { scratchpadNotes } from '@/lib/db/schema/system'; import { user as authUser } from '@/lib/db/schema/users'; import { redis } from '@/lib/redis'; import { sendEmail } from '@/lib/email'; import { logger } from '@/lib/logger'; import { createAuditLog, type AuditMeta } from '@/lib/audit'; +import { demoteSystemFolderOnEntityDelete } from '@/lib/services/document-folders.service'; import { ConflictError, NotFoundError, ValidationError } from '@/lib/errors'; const CODE_TTL_SECONDS = 10 * 60; @@ -203,6 +205,14 @@ export async function hardDeleteClient(args: { .update(documentSends) .set({ clientId: null }) .where(eq(documentSends.clientId, args.clientId)); + // G-C2: scratchpad_notes.linked_client_id is RESTRICT (default for no + // onDelete clause). Any rep who linked a scratchpad note to this client + // would otherwise throw an FK violation when we try to delete the + // client row below. Nullify so the note survives the hard-delete. + await tx + .update(scratchpadNotes) + .set({ linkedClientId: null }) + .where(eq(scratchpadNotes.linkedClientId, args.clientId)); // client_merge_log.surviving_client_id has no cascade and is // notNull → must be deleted explicitly. Merged records remain in @@ -218,6 +228,18 @@ export async function hardDeleteClient(args: { await tx.delete(clients).where(eq(clients.id, args.clientId)); }); + // G-C3 / A7: demote the system-managed folder so the partial unique + // index `uniq_document_folders_entity` releases its slot. Done as a + // post-commit fire-and-forget — folder hygiene is non-essential to the + // delete being durable, and we don't want a folder-table glitch to + // un-delete the client by aborting the outer transaction. + void demoteSystemFolderOnEntityDelete(args.portId, 'client', args.clientId).catch((err) => { + logger.error( + { err, clientId: args.clientId, portId: args.portId }, + 'hardDeleteClient: failed to demote system folder', + ); + }); + void createAuditLog({ portId: args.portId, userId: args.requesterUserId, diff --git a/src/lib/services/clients.service.ts b/src/lib/services/clients.service.ts index ad305eb5..bb25aa5e 100644 --- a/src/lib/services/clients.service.ts +++ b/src/lib/services/clients.service.ts @@ -561,8 +561,11 @@ export async function archiveClient(id: string, portId: string, meta: AuditMeta) // being stamped before the HTTP response returns. Task 5 (rename // hook) uses await because the rename should be visible to the // next read; archive does not. - void applyEntityArchivedSuffix(portId, 'client', id).catch((err) => { - logger.warn({ err, clientId: id }, 'Failed to apply archived suffix to client folder'); + void applyEntityArchivedSuffix(portId, 'client', id, meta.userId).catch((err) => { + logger.warn( + { err, clientId: id, portId }, + 'Failed to apply archived suffix to client folder', + ); }); void createAuditLog({ @@ -593,8 +596,11 @@ export async function restoreClient(id: string, portId: string, meta: AuditMeta) await restore(clients, clients.id, id); - void applyEntityRestoredSuffix(portId, 'client', id).catch((err) => { - logger.warn({ err, clientId: id }, 'Failed to clear archived suffix on client folder'); + void applyEntityRestoredSuffix(portId, 'client', id, meta.userId).catch((err) => { + logger.warn( + { err, clientId: id, portId }, + 'Failed to clear archived suffix on client folder', + ); }); void createAuditLog({ diff --git a/src/lib/services/companies.service.ts b/src/lib/services/companies.service.ts index ab04ab1c..3f0faaef 100644 --- a/src/lib/services/companies.service.ts +++ b/src/lib/services/companies.service.ts @@ -188,7 +188,7 @@ export async function updateCompany( if (data.name !== undefined) { await syncEntityFolderName(portId, 'company', id, meta.userId).catch((err) => { - logger.warn({ err, companyId: id }, 'Failed to sync company folder name'); + logger.warn({ err, companyId: id, portId }, 'Failed to sync company folder name'); }); } @@ -215,8 +215,11 @@ export async function archiveCompany(id: string, portId: string, meta: AuditMeta .set({ archivedAt: new Date() }) .where(and(eq(companies.id, id), eq(companies.portId, portId))); - void applyEntityArchivedSuffix(portId, 'company', id).catch((err) => { - logger.warn({ err, companyId: id }, 'Failed to apply archived suffix to company folder'); + void applyEntityArchivedSuffix(portId, 'company', id, meta.userId).catch((err) => { + logger.warn( + { err, companyId: id, portId }, + 'Failed to apply archived suffix to company folder', + ); }); void createAuditLog({ diff --git a/src/lib/services/document-folders.service.ts b/src/lib/services/document-folders.service.ts index 1ccabdda..bedfb464 100644 --- a/src/lib/services/document-folders.service.ts +++ b/src/lib/services/document-folders.service.ts @@ -1,12 +1,18 @@ import { and, asc, eq } from 'drizzle-orm'; import { db } from '@/lib/db'; -import { documentFolders, documents, type DocumentFolder } from '@/lib/db/schema/documents'; +import { + documentFolders, + documents, + files, + type DocumentFolder, +} from '@/lib/db/schema/documents'; import { clients } from '@/lib/db/schema/clients'; import { companies } from '@/lib/db/schema/companies'; import { yachts } from '@/lib/db/schema/yachts'; import { createAuditLog } from '@/lib/audit'; import { ConflictError, NotFoundError, ValidationError } from '@/lib/errors'; +import { logger } from '@/lib/logger'; /** * Returns true if the error is a Postgres unique-violation (SQLSTATE 23505) @@ -78,10 +84,17 @@ export async function listTree(portId: string): Promise { roots.push(node); } else { const parent = byId.get(node.parentId); - if (parent) parent.children.push(node); - // Orphan rows (parentId pointing nowhere) are dropped from the - // tree but stay in the DB. Surface via a separate maintenance - // query if needed; never silently re-parent. + if (parent) { + parent.children.push(node); + } else { + // Orphan rows (parentId pointing nowhere) are dropped from the + // tree but stay in the DB. Surface via a separate maintenance + // query if needed; never silently re-parent. + logger.warn( + { portId, folderId: node.id, parentId: node.parentId, name: node.name }, + 'listTree: orphan folder row (parentId points to a missing folder); dropped from tree', + ); + } } } return roots; @@ -126,6 +139,17 @@ export async function createFolder( }) .returning(); if (!row) throw new NotFoundError('Folder'); + + void createAuditLog({ + userId, + portId, + action: 'create', + entityType: 'document_folder', + entityId: row.id, + newValue: { name: row.name, parentId: row.parentId }, + metadata: { type: 'folder_created' }, + }); + return row; } catch (err) { if (isSiblingNameConflict(err)) { @@ -276,6 +300,15 @@ export async function deleteFolderSoftRescue( .set({ folderId: newParent }) .where(and(eq(documents.folderId, folderId), eq(documents.portId, portId))); + // G-C1: files.folder_id is ON DELETE SET NULL — without this UPDATE, + // files in the deleted folder would scatter to root while documents + // in the same folder land at the deleted folder's parent. Re-parent + // files explicitly so the soft-rescue is symmetric across both. + await tx + .update(files) + .set({ folderId: newParent }) + .where(and(eq(files.folderId, folderId), eq(files.portId, portId))); + await tx .delete(documentFolders) .where(and(eq(documentFolders.id, folderId), eq(documentFolders.portId, portId))); @@ -353,7 +386,13 @@ export async function ensureSystemRoots(portId: string, userId: string): Promise return SYSTEM_ROOT_NAMES.map((name: SystemRootName) => { const row = rows.find((r) => r.name === name); - if (!row) throw new Error(`ensureSystemRoots: missing root ${name} after upsert`); + if (!row) { + logger.error( + { portId, missingRoot: name, foundNames: rows.map((r) => r.name) }, + 'ensureSystemRoots: invariant violated — system root missing after upsert', + ); + throw new Error(`ensureSystemRoots: missing root ${name} after upsert`); + } return row; }); } @@ -513,6 +552,10 @@ export async function ensureEntityFolder( throw err; } } + logger.warn( + { portId, entityType, entityId, baseName, attempts: 50 }, + 'ensureEntityFolder: exhausted 50 suffix attempts without finding a unique name', + ); throw new ConflictError(`Could not allocate a unique folder name for ${baseName}`); } @@ -532,7 +575,7 @@ export async function syncEntityFolderName( portId: string, entityType: EntityType, entityId: string, - _userId: string, + userId: string, ): Promise { if (!ENTITY_TYPES.has(entityType)) return; @@ -563,12 +606,28 @@ export async function syncEntityFolderName( .set({ name: candidate, updatedAt: new Date() }) .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId))) .returning(); - if (updated) return; + if (updated) { + void createAuditLog({ + userId, + portId, + action: 'update', + entityType: 'document_folder', + entityId: folder.id, + oldValue: { name: folder.name }, + newValue: { name: candidate }, + metadata: { type: 'folder_entity_rename_sync', entity: entityType, sourceEntityId: entityId }, + }); + return; + } } catch (err) { if (isSiblingNameConflict(err)) continue; throw err; } } + logger.warn( + { portId, entityType, entityId, baseName, attempts: 50 }, + 'syncEntityFolderName: exhausted 50 suffix attempts without finding a unique name', + ); throw new ConflictError(`Could not allocate a unique folder name for ${baseName}`); } @@ -587,6 +646,7 @@ export async function applyEntityArchivedSuffix( portId: string, entityType: EntityType, entityId: string, + userId?: string, ): Promise { if (!ENTITY_TYPES.has(entityType)) return; const folder = await db.query.documentFolders.findFirst({ @@ -605,6 +665,17 @@ export async function applyEntityArchivedSuffix( .update(documentFolders) .set({ name: newName, archivedAt: new Date(), updatedAt: new Date() }) .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId))); + + void createAuditLog({ + userId: userId ?? 'system', + portId, + action: 'archive', + entityType: 'document_folder', + entityId: folder.id, + oldValue: { name: folder.name, archivedAt: folder.archivedAt }, + newValue: { name: newName, archivedAt: new Date() }, + metadata: { type: 'folder_entity_archived', entity: entityType, sourceEntityId: entityId }, + }); } /** @@ -616,6 +687,7 @@ export async function applyEntityRestoredSuffix( portId: string, entityType: EntityType, entityId: string, + userId?: string, ): Promise { if (!ENTITY_TYPES.has(entityType)) return; const folder = await db.query.documentFolders.findFirst({ @@ -634,6 +706,17 @@ export async function applyEntityRestoredSuffix( .update(documentFolders) .set({ name: newName, archivedAt: null, updatedAt: new Date() }) .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId))); + + void createAuditLog({ + userId: userId ?? 'system', + portId, + action: 'restore', + entityType: 'document_folder', + entityId: folder.id, + oldValue: { name: folder.name, archivedAt: folder.archivedAt }, + newValue: { name: newName, archivedAt: null }, + metadata: { type: 'folder_entity_restored', entity: entityType, sourceEntityId: entityId }, + }); } /** diff --git a/src/lib/services/yachts.service.ts b/src/lib/services/yachts.service.ts index 3551eb24..53496731 100644 --- a/src/lib/services/yachts.service.ts +++ b/src/lib/services/yachts.service.ts @@ -168,7 +168,7 @@ export async function updateYacht( if (data.name !== undefined) { await syncEntityFolderName(portId, 'yacht', id, meta.userId).catch((err) => { - logger.warn({ err, yachtId: id }, 'Failed to sync yacht folder name'); + logger.warn({ err, yachtId: id, portId }, 'Failed to sync yacht folder name'); }); } @@ -193,8 +193,8 @@ export async function archiveYacht(id: string, portId: string, meta: AuditMeta) .set({ archivedAt: new Date() }) .where(and(eq(yachts.id, id), eq(yachts.portId, portId))); - void applyEntityArchivedSuffix(portId, 'yacht', id).catch((err) => { - logger.warn({ err, yachtId: id }, 'Failed to apply archived suffix to yacht folder'); + void applyEntityArchivedSuffix(portId, 'yacht', id, meta.userId).catch((err) => { + logger.warn({ err, yachtId: id, portId }, 'Failed to apply archived suffix to yacht folder'); }); void createAuditLog({