diff --git a/scripts/backfill-document-folders.ts b/scripts/backfill-document-folders.ts new file mode 100644 index 00000000..5ac1610e --- /dev/null +++ b/scripts/backfill-document-folders.ts @@ -0,0 +1,175 @@ +/** + * Idempotent backfill: ensure every port has the three system roots + * (Clients / Companies / Yachts), every entity with attached files + * has a per-entity subfolder, every file with entity FKs has + * `folder_id` set, and every signed file from a completed workflow + * has the workflow's entity FKs propagated onto it. + * + * Safe to re-run: all writes target only rows where the relevant + * column is NULL. Per-port `pg_advisory_xact_lock` serializes + * concurrent runs. + * + * Usage: + * pnpm tsx scripts/backfill-document-folders.ts + * pnpm tsx scripts/backfill-document-folders.ts --port + */ + +import 'dotenv/config'; +import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm'; + +import { db } from '@/lib/db'; +import { ports } from '@/lib/db/schema/ports'; +import { files, documents } from '@/lib/db/schema/documents'; +import { + ensureSystemRoots, + ensureEntityFolder, + type EntityType, +} from '@/lib/services/document-folders.service'; +import { logger } from '@/lib/logger'; + +export interface BackfillOptions { + /** When provided, only backfill this port. Otherwise all ports. */ + portId?: string; + /** User ID recorded in `created_by` for any folders created. */ + systemUserId?: string; +} + +/** + * One-time idempotent backfill. See module-level JSDoc for full + * description of what each step does. + */ +export async function runBackfill(opts: BackfillOptions = {}): Promise { + const portRows = opts.portId + ? [{ id: opts.portId }] + : await db.select({ id: ports.id }).from(ports); + + const systemUser = opts.systemUserId ?? 'system-backfill'; + + for (const { id: portId } of portRows) { + await db.transaction(async (tx) => { + // Serialize concurrent runs on a per-port lock so two simultaneous + // backfills can't race on folder inserts. + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${portId})::bigint)`, + ); + + // ── Step 1: Ensure system roots exist for this port ────────────────── + await ensureSystemRoots(portId, systemUser); + + // ── Step 2: Propagate entity FKs from completed workflows onto their + // signed file rows (pre-auto-deposit legacy completions). ── + const completedDocs = await tx + .select({ + id: documents.id, + signedFileId: documents.signedFileId, + clientId: documents.clientId, + companyId: documents.companyId, + yachtId: documents.yachtId, + }) + .from(documents) + .where( + and( + eq(documents.portId, portId), + eq(documents.status, 'completed'), + isNotNull(documents.signedFileId), + ), + ); + + for (const d of completedDocs) { + if (!d.signedFileId) continue; + + const owner: { type: EntityType; id: string } | null = d.clientId + ? { type: 'client', id: d.clientId } + : d.companyId + ? { type: 'company', id: d.companyId } + : d.yachtId + ? { type: 'yacht', id: d.yachtId } + : null; + + if (!owner) continue; + + // Build the update object with ONLY the matching FK column so we + // never pass column references to .set() (Drizzle syntax bug fix). + const update = + owner.type === 'client' + ? { clientId: owner.id } + : owner.type === 'company' + ? { companyId: owner.id } + : { yachtId: owner.id }; + + const matchingFkColumn = + owner.type === 'client' + ? files.clientId + : owner.type === 'company' + ? files.companyId + : files.yachtId; + + await tx + .update(files) + .set(update) + .where( + and( + eq(files.id, d.signedFileId), + eq(files.portId, portId), + isNull(matchingFkColumn), + ), + ); + } + + // ── Step 3: For every file with entity FKs but no folder_id, + // create the entity subfolder and set folder_id. ────────── + const fileRows = await tx + .select() + .from(files) + .where(and(eq(files.portId, portId), isNull(files.folderId))); + + for (const f of fileRows) { + const owner: { type: EntityType; id: string } | null = f.clientId + ? { type: 'client', id: f.clientId } + : f.companyId + ? { type: 'company', id: f.companyId } + : f.yachtId + ? { type: 'yacht', id: f.yachtId } + : null; + + if (!owner) continue; + + try { + const folder = await ensureEntityFolder(portId, owner.type, owner.id, systemUser); + await tx + .update(files) + .set({ folderId: folder.id }) + .where(and(eq(files.id, f.id), eq(files.portId, portId))); + } catch (err) { + // Best-effort: log and skip rather than abort the whole port. + logger.warn({ err, fileId: f.id, portId }, 'backfill: ensureEntityFolder failed'); + } + } + }); + + logger.info({ portId }, 'backfill: port complete'); + } +} + +// ── CLI entry point ──────────────────────────────────────────────────────────── +// tsx compiles TypeScript to CJS at runtime, so `require.main === module` +// is the standard guard. The test suite imports `runBackfill` as a named +// export; the CLI invocation hits this block and runs main(). + +async function main(): Promise { + const portIdArg = process.argv.indexOf('--port'); + const portId = portIdArg !== -1 ? process.argv[portIdArg + 1] : undefined; + + await runBackfill({ portId }); + // eslint-disable-next-line no-console + console.log('Backfill complete'); + process.exit(0); +} + +// eslint-disable-next-line @typescript-eslint/no-require-imports +if (require.main === module) { + main().catch((err) => { + logger.error({ err }, 'Backfill failed'); + process.exit(1); + }); +} diff --git a/tests/integration/backfill-document-folders.test.ts b/tests/integration/backfill-document-folders.test.ts new file mode 100644 index 00000000..ac999125 --- /dev/null +++ b/tests/integration/backfill-document-folders.test.ts @@ -0,0 +1,225 @@ +/** + * Task 11 — backfill-document-folders integration tests. + * + * Five cases: + * 1. Creates system roots and entity subfolders. + * 2. Sets files.folder_id from entity FKs. + * 3. Copies entity FKs from completed workflows onto signed files. + * 4. Idempotent — second run produces the same result. + * 5. Port isolation — does not touch other ports. + */ + +import { describe, it, expect, beforeAll, beforeEach } from 'vitest'; +import { and, eq } from 'drizzle-orm'; + +import { db } from '@/lib/db'; +import { documentFolders, files, documents } from '@/lib/db/schema/documents'; +import { user } from '@/lib/db/schema/users'; +import { runBackfill } from '../../scripts/backfill-document-folders'; +import { makePort, makeClient } from '../helpers/factories'; + +let TEST_USER_ID = ''; + +beforeAll(async () => { + const [u] = await db.select({ id: user.id }).from(user).limit(1); + if (!u) throw new Error('No user available; run pnpm db:seed first'); + TEST_USER_ID = u.id; +}); + +describe('backfill-document-folders · runBackfill', () => { + let portId: string; + + beforeEach(async () => { + const port = await makePort(); + portId = port.id; + // Clean up any folders left by a prior test on this port. + await db.delete(documentFolders).where(eq(documentFolders.portId, portId)); + }); + + // ── Test 1: Creates system roots and entity subfolders ───────────────────── + + it('creates the three system roots and a client entity subfolder', async () => { + const client = await makeClient({ portId }); + + // Insert a file linked to the client so the backfill creates the subfolder. + await db.insert(files).values({ + portId, + clientId: client.id, + filename: 'test.pdf', + originalName: 'test.pdf', + storagePath: `${portId}/test.pdf`, + storageBucket: 'crm-files', + uploadedBy: TEST_USER_ID, + }); + + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const roots = await db + .select() + .from(documentFolders) + .where(and(eq(documentFolders.portId, portId), eq(documentFolders.entityType, 'root'))); + + expect(roots).toHaveLength(3); + const rootNames = roots.map((r) => r.name).sort(); + expect(rootNames).toEqual(['Clients', 'Companies', 'Yachts']); + + const entityFolder = await db.query.documentFolders.findFirst({ + where: and( + eq(documentFolders.portId, portId), + eq(documentFolders.entityType, 'client'), + eq(documentFolders.entityId, client.id), + ), + }); + expect(entityFolder).toBeDefined(); + expect(entityFolder?.entityType).toBe('client'); + expect(entityFolder?.entityId).toBe(client.id); + }); + + // ── Test 2: Sets files.folder_id from entity FKs ──────────────────────────── + + it('sets files.folder_id for files that have entity FKs but no folder_id', async () => { + const client = await makeClient({ portId }); + + const [fileRow] = await db + .insert(files) + .values({ + portId, + clientId: client.id, + filename: 'contract.pdf', + originalName: 'contract.pdf', + storagePath: `${portId}/contract.pdf`, + storageBucket: 'crm-files', + uploadedBy: TEST_USER_ID, + // folderId intentionally omitted → null + }) + .returning(); + + expect(fileRow!.folderId).toBeNull(); + + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const updated = await db.query.files.findFirst({ + where: and(eq(files.id, fileRow!.id), eq(files.portId, portId)), + }); + + expect(updated!.folderId).not.toBeNull(); + + // The assigned folder must be the client's entity subfolder. + const entityFolder = await db.query.documentFolders.findFirst({ + where: and( + eq(documentFolders.portId, portId), + eq(documentFolders.entityType, 'client'), + eq(documentFolders.entityId, client.id), + ), + }); + expect(updated!.folderId).toBe(entityFolder!.id); + }); + + // ── Test 3: Copies entity FKs from completed workflows onto signed files ──── + + it('propagates entity FKs from completed workflow onto signed file and sets folder_id', async () => { + const client = await makeClient({ portId }); + + // File row with no entity FK (simulating a legacy completed-before-auto-deposit). + const [signedFile] = await db + .insert(files) + .values({ + portId, + // No clientId set — simulates legacy completion before entity FK auto-propagation. + filename: 'signed-eoi.pdf', + originalName: 'signed-eoi.pdf', + storagePath: `${portId}/signed-eoi.pdf`, + storageBucket: 'crm-files', + uploadedBy: TEST_USER_ID, + }) + .returning(); + + expect(signedFile!.clientId).toBeNull(); + + // Completed workflow document pointing at the orphaned file. + await db.insert(documents).values({ + portId, + documentType: 'eoi', + title: 'EOI for client', + status: 'completed', + signedFileId: signedFile!.id, + clientId: client.id, + createdBy: TEST_USER_ID, + }); + + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const updatedFile = await db.query.files.findFirst({ + where: and(eq(files.id, signedFile!.id), eq(files.portId, portId)), + }); + + // The backfill should have set the clientId on the signed file. + expect(updatedFile!.clientId).toBe(client.id); + // And then assigned it to the client's entity subfolder. + expect(updatedFile!.folderId).not.toBeNull(); + }); + + // ── Test 4: Idempotent ──────────────────────────────────────────────────────── + + it('is idempotent — running twice produces the same number of folder rows', async () => { + const client = await makeClient({ portId }); + + await db.insert(files).values({ + portId, + clientId: client.id, + filename: 'idempotent.pdf', + originalName: 'idempotent.pdf', + storagePath: `${portId}/idempotent.pdf`, + storageBucket: 'crm-files', + uploadedBy: TEST_USER_ID, + }); + + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const countAfterFirst = await db + .select() + .from(documentFolders) + .where(eq(documentFolders.portId, portId)); + + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const countAfterSecond = await db + .select() + .from(documentFolders) + .where(eq(documentFolders.portId, portId)); + + expect(countAfterSecond).toHaveLength(countAfterFirst.length); + }); + + // ── Test 5: Port isolation ──────────────────────────────────────────────────── + + it('does not create folders for a different port when only portId is supplied', async () => { + const otherPort = await makePort(); + await db + .delete(documentFolders) + .where(eq(documentFolders.portId, otherPort.id)); + + const otherClient = await makeClient({ portId: otherPort.id }); + + await db.insert(files).values({ + portId: otherPort.id, + clientId: otherClient.id, + filename: 'other-port.pdf', + originalName: 'other-port.pdf', + storagePath: `${otherPort.id}/other-port.pdf`, + storageBucket: 'crm-files', + uploadedBy: TEST_USER_ID, + }); + + // Run backfill only for the main portId, NOT the otherPort. + await runBackfill({ portId, systemUserId: TEST_USER_ID }); + + const otherPortFolders = await db + .select() + .from(documentFolders) + .where(eq(documentFolders.portId, otherPort.id)); + + // The other port should have zero folders — the backfill was not run for it. + expect(otherPortFolders).toHaveLength(0); + }); +});