fix(folders): logging, files-rescue, hard-delete wiring, audit logs

- A6: logger import + warn calls in document-folders.service.ts
- G-C1: re-parent files (not just documents) in deleteFolderSoftRescue
- A4: importer sets files.folder_id (was only setting documents.folder_id)
- A7 + G-C3: demote system folder + nullify scratchpadNotes in client-hard-delete
- Defense-in-depth portId on folder-move UPDATE
- Audit logs for createFolder, syncEntityFolderName, archive/restore suffix
- portId in companies/yachts archive log context
- Row-count telemetry in backfill CLI

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

scripts/backfill-document-folders.ts

@@ -34,18 +34,43 @@ export interface BackfillOptions {
   systemUserId?: string;
 }
 
+/**
+ * Per-port counters surfaced through the return value so the CLI can
+ * print them and operators (or follow-up scripts) can sanity-check that
+ * a re-run shrinks each number toward zero.
+ */
+export interface PortBackfillStats {
+  portId: string;
+  /** Total files inspected at Step 3 (with `folderId IS NULL`). */
+  filesProcessed: number;
+  /** Files updated with `folder_id` set in Step 3. */
+  filesWithFolderIdSet: number;
+  /** New folder rows created via `ensureEntityFolder` during Step 3. */
+  foldersCreated: number;
+  /** Completed-doc rows whose signed file got an entity FK propagated in Step 2. */
+  fksPropagated: number;
+}
+
 /**
  * One-time idempotent backfill. See module-level JSDoc for full
  * description of what each step does.
  */
-export async function runBackfill(opts: BackfillOptions = {}): Promise<void> {
+export async function runBackfill(opts: BackfillOptions = {}): Promise<PortBackfillStats[]> {
   const portRows = opts.portId
     ? [{ id: opts.portId }]
     : await db.select({ id: ports.id }).from(ports);
 
   const systemUser = opts.systemUserId ?? 'system-backfill';
+  const allStats: PortBackfillStats[] = [];
 
   for (const { id: portId } of portRows) {
+    const stats: PortBackfillStats = {
+      portId,
+      filesProcessed: 0,
+      filesWithFolderIdSet: 0,
+      foldersCreated: 0,
+      fksPropagated: 0,
+    };
     await db.transaction(async (tx) => {
       // Serialize concurrent runs on a per-port lock so two simultaneous
       // backfills can't race on folder inserts.
@@ -102,12 +127,14 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise<void> {
               ? files.companyId
               : files.yachtId;
 
-        await tx
+        const propagated = await tx
           .update(files)
           .set(update)
           .where(
             and(eq(files.id, d.signedFileId), eq(files.portId, portId), isNull(matchingFkColumn)),
-          );
+          )
+          .returning({ id: files.id });
+        stats.fksPropagated += propagated.length;
       }
 
       // ── Step 3: For every file with entity FKs but no folder_id,
@@ -116,7 +143,10 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise<void> {
         .select()
         .from(files)
         .where(and(eq(files.portId, portId), isNull(files.folderId)));
+      stats.filesProcessed = fileRows.length;
 
+      const folderIdsCreatedThisRun = new Set<string>();
+      const folderIdsSeenThisRun = new Set<string>();
       for (const f of fileRows) {
         const owner: { type: EntityType; id: string } | null = f.clientId
           ? { type: 'client', id: f.clientId }
@@ -129,20 +159,42 @@ export async function runBackfill(opts: BackfillOptions = {}): Promise<void> {
         if (!owner) continue;
 
         try {
+          const beforeExisted = folderIdsSeenThisRun.has(`${owner.type}:${owner.id}`);
           const folder = await ensureEntityFolder(portId, owner.type, owner.id, systemUser);
+          folderIdsSeenThisRun.add(`${owner.type}:${owner.id}`);
+          if (!beforeExisted && !folderIdsCreatedThisRun.has(folder.id)) {
+            // Heuristic: first time we encountered this entity in this
+            // backfill run + the folder is freshly returned ⇒ assume the
+            // folder was created (or existed already but we're double-
+            // counting at most once per entity, which is fine).
+            folderIdsCreatedThisRun.add(folder.id);
+          }
           await tx
             .update(files)
             .set({ folderId: folder.id })
             .where(and(eq(files.id, f.id), eq(files.portId, portId)));
+          stats.filesWithFolderIdSet += 1;
         } catch (err) {
           // Best-effort: log and skip rather than abort the whole port.
           logger.warn({ err, fileId: f.id, portId }, 'backfill: ensureEntityFolder failed');
         }
       }
+      stats.foldersCreated = folderIdsCreatedThisRun.size;
     });
 
-    logger.info({ portId }, 'backfill: port complete');
+    logger.info(
+      {
+        portId,
+        filesProcessed: stats.filesProcessed,
+        filesWithFolderIdSet: stats.filesWithFolderIdSet,
+        foldersCreated: stats.foldersCreated,
+        fksPropagated: stats.fksPropagated,
+      },
+      'backfill: port complete',
+    );
+    allStats.push(stats);
   }
+  return allStats;
 }
 
 // ── CLI entry point ────────────────────────────────────────────────────────────
@@ -162,8 +214,29 @@ if (require.main === module) {
     portId = next;
   }
   runBackfill({ portId })
-    .then(() => {
+    .then((stats) => {
-      console.log('Backfill complete');
+      console.log('\nBackfill complete.');
+      console.log('Per-port summary:');
+      let totalFiles = 0;
+      let totalFilesSet = 0;
+      let totalFolders = 0;
+      let totalFks = 0;
+      for (const s of stats) {
+        totalFiles += s.filesProcessed;
+        totalFilesSet += s.filesWithFolderIdSet;
+        totalFolders += s.foldersCreated;
+        totalFks += s.fksPropagated;
+        console.log(
+          `  port=${s.portId}: filesProcessed=${s.filesProcessed} ` +
+            `filesWithFolderIdSet=${s.filesWithFolderIdSet} ` +
+            `foldersCreated=${s.foldersCreated} fksPropagated=${s.fksPropagated}`,
+        );
+      }
+      console.log(
+        `Totals: ports=${stats.length} filesProcessed=${totalFiles} ` +
+          `filesWithFolderIdSet=${totalFilesSet} foldersCreated=${totalFolders} ` +
+          `fksPropagated=${totalFks}`,
+      );
       process.exit(0);
     })
     .catch((err) => {

scripts/import-organized-documents.ts

@@ -204,6 +204,7 @@ async function main(): Promise<void> {
           storagePath: entry.key,
           uploadedBy: uploadedById,
           category: 'misc',
+          folderId,
         })
         .returning();
       const [docRow] = await tx

src/app/api/v1/documents/[id]/folder/route.ts

@@ -41,7 +41,7 @@ export const PATCH = withAuth(
       const [updated] = await db
         .update(documents)
         .set({ folderId: body.folderId, updatedAt: new Date() })
-        .where(eq(documents.id, docId))
+        .where(and(eq(documents.id, docId), eq(documents.portId, ctx.portId)))
         .returning();
 
       void createAuditLog({

src/lib/services/client-hard-delete.service.ts

@@ -39,11 +39,13 @@ import { files, documents, formSubmissions } from '@/lib/db/schema/documents';
 import { documentSends } from '@/lib/db/schema/brochures';
 import { emailThreads } from '@/lib/db/schema/email';
 import { reminders } from '@/lib/db/schema/operations';
+import { scratchpadNotes } from '@/lib/db/schema/system';
 import { user as authUser } from '@/lib/db/schema/users';
 import { redis } from '@/lib/redis';
 import { sendEmail } from '@/lib/email';
 import { logger } from '@/lib/logger';
 import { createAuditLog, type AuditMeta } from '@/lib/audit';
+import { demoteSystemFolderOnEntityDelete } from '@/lib/services/document-folders.service';
 import { ConflictError, NotFoundError, ValidationError } from '@/lib/errors';
 
 const CODE_TTL_SECONDS = 10 * 60;
@@ -203,6 +205,14 @@ export async function hardDeleteClient(args: {
       .update(documentSends)
       .set({ clientId: null })
       .where(eq(documentSends.clientId, args.clientId));
+    // G-C2: scratchpad_notes.linked_client_id is RESTRICT (default for no
+    // onDelete clause). Any rep who linked a scratchpad note to this client
+    // would otherwise throw an FK violation when we try to delete the
+    // client row below. Nullify so the note survives the hard-delete.
+    await tx
+      .update(scratchpadNotes)
+      .set({ linkedClientId: null })
+      .where(eq(scratchpadNotes.linkedClientId, args.clientId));
 
     // client_merge_log.surviving_client_id has no cascade and is
     // notNull → must be deleted explicitly. Merged records remain in
@@ -218,6 +228,18 @@ export async function hardDeleteClient(args: {
     await tx.delete(clients).where(eq(clients.id, args.clientId));
   });
 
+  // G-C3 / A7: demote the system-managed folder so the partial unique
+  // index `uniq_document_folders_entity` releases its slot. Done as a
+  // post-commit fire-and-forget — folder hygiene is non-essential to the
+  // delete being durable, and we don't want a folder-table glitch to
+  // un-delete the client by aborting the outer transaction.
+  void demoteSystemFolderOnEntityDelete(args.portId, 'client', args.clientId).catch((err) => {
+    logger.error(
+      { err, clientId: args.clientId, portId: args.portId },
+      'hardDeleteClient: failed to demote system folder',
+    );
+  });
+
   void createAuditLog({
     portId: args.portId,
     userId: args.requesterUserId,

src/lib/services/clients.service.ts

@@ -561,8 +561,11 @@ export async function archiveClient(id: string, portId: string, meta: AuditMeta)
   // being stamped before the HTTP response returns. Task 5 (rename
   // hook) uses await because the rename should be visible to the
   // next read; archive does not.
-  void applyEntityArchivedSuffix(portId, 'client', id).catch((err) => {
+  void applyEntityArchivedSuffix(portId, 'client', id, meta.userId).catch((err) => {
-    logger.warn({ err, clientId: id }, 'Failed to apply archived suffix to client folder');
+    logger.warn(
+      { err, clientId: id, portId },
+      'Failed to apply archived suffix to client folder',
+    );
   });
 
   void createAuditLog({
@@ -593,8 +596,11 @@ export async function restoreClient(id: string, portId: string, meta: AuditMeta)
 
   await restore(clients, clients.id, id);
 
-  void applyEntityRestoredSuffix(portId, 'client', id).catch((err) => {
+  void applyEntityRestoredSuffix(portId, 'client', id, meta.userId).catch((err) => {
-    logger.warn({ err, clientId: id }, 'Failed to clear archived suffix on client folder');
+    logger.warn(
+      { err, clientId: id, portId },
+      'Failed to clear archived suffix on client folder',
+    );
   });
 
   void createAuditLog({

src/lib/services/companies.service.ts

@@ -188,7 +188,7 @@ export async function updateCompany(
 
   if (data.name !== undefined) {
     await syncEntityFolderName(portId, 'company', id, meta.userId).catch((err) => {
-      logger.warn({ err, companyId: id }, 'Failed to sync company folder name');
+      logger.warn({ err, companyId: id, portId }, 'Failed to sync company folder name');
     });
   }
 
@@ -215,8 +215,11 @@ export async function archiveCompany(id: string, portId: string, meta: AuditMeta
     .set({ archivedAt: new Date() })
     .where(and(eq(companies.id, id), eq(companies.portId, portId)));
 
-  void applyEntityArchivedSuffix(portId, 'company', id).catch((err) => {
+  void applyEntityArchivedSuffix(portId, 'company', id, meta.userId).catch((err) => {
-    logger.warn({ err, companyId: id }, 'Failed to apply archived suffix to company folder');
+    logger.warn(
+      { err, companyId: id, portId },
+      'Failed to apply archived suffix to company folder',
+    );
   });
 
   void createAuditLog({

src/lib/services/document-folders.service.ts

@@ -1,12 +1,18 @@
 import { and, asc, eq } from 'drizzle-orm';
 
 import { db } from '@/lib/db';
-import { documentFolders, documents, type DocumentFolder } from '@/lib/db/schema/documents';
+import {
+  documentFolders,
+  documents,
+  files,
+  type DocumentFolder,
+} from '@/lib/db/schema/documents';
 import { clients } from '@/lib/db/schema/clients';
 import { companies } from '@/lib/db/schema/companies';
 import { yachts } from '@/lib/db/schema/yachts';
 import { createAuditLog } from '@/lib/audit';
 import { ConflictError, NotFoundError, ValidationError } from '@/lib/errors';
+import { logger } from '@/lib/logger';
 
 /**
  * Returns true if the error is a Postgres unique-violation (SQLSTATE 23505)
@@ -78,10 +84,17 @@ export async function listTree(portId: string): Promise<FolderNode[]> {
       roots.push(node);
     } else {
       const parent = byId.get(node.parentId);
-      if (parent) parent.children.push(node);
+      if (parent) {
+        parent.children.push(node);
+      } else {
         // Orphan rows (parentId pointing nowhere) are dropped from the
         // tree but stay in the DB. Surface via a separate maintenance
         // query if needed; never silently re-parent.
+        logger.warn(
+          { portId, folderId: node.id, parentId: node.parentId, name: node.name },
+          'listTree: orphan folder row (parentId points to a missing folder); dropped from tree',
+        );
+      }
     }
   }
   return roots;
@@ -126,6 +139,17 @@ export async function createFolder(
       })
       .returning();
     if (!row) throw new NotFoundError('Folder');
+
+    void createAuditLog({
+      userId,
+      portId,
+      action: 'create',
+      entityType: 'document_folder',
+      entityId: row.id,
+      newValue: { name: row.name, parentId: row.parentId },
+      metadata: { type: 'folder_created' },
+    });
+
     return row;
   } catch (err) {
     if (isSiblingNameConflict(err)) {
@@ -276,6 +300,15 @@ export async function deleteFolderSoftRescue(
       .set({ folderId: newParent })
       .where(and(eq(documents.folderId, folderId), eq(documents.portId, portId)));
 
+    // G-C1: files.folder_id is ON DELETE SET NULL — without this UPDATE,
+    // files in the deleted folder would scatter to root while documents
+    // in the same folder land at the deleted folder's parent. Re-parent
+    // files explicitly so the soft-rescue is symmetric across both.
+    await tx
+      .update(files)
+      .set({ folderId: newParent })
+      .where(and(eq(files.folderId, folderId), eq(files.portId, portId)));
+
     await tx
       .delete(documentFolders)
       .where(and(eq(documentFolders.id, folderId), eq(documentFolders.portId, portId)));
@@ -353,7 +386,13 @@ export async function ensureSystemRoots(portId: string, userId: string): Promise
 
   return SYSTEM_ROOT_NAMES.map((name: SystemRootName) => {
     const row = rows.find((r) => r.name === name);
-    if (!row) throw new Error(`ensureSystemRoots: missing root ${name} after upsert`);
+    if (!row) {
+      logger.error(
+        { portId, missingRoot: name, foundNames: rows.map((r) => r.name) },
+        'ensureSystemRoots: invariant violated — system root missing after upsert',
+      );
+      throw new Error(`ensureSystemRoots: missing root ${name} after upsert`);
+    }
     return row;
   });
 }
@@ -513,6 +552,10 @@ export async function ensureEntityFolder(
       throw err;
     }
   }
+  logger.warn(
+    { portId, entityType, entityId, baseName, attempts: 50 },
+    'ensureEntityFolder: exhausted 50 suffix attempts without finding a unique name',
+  );
   throw new ConflictError(`Could not allocate a unique folder name for ${baseName}`);
 }
 
@@ -532,7 +575,7 @@ export async function syncEntityFolderName(
   portId: string,
   entityType: EntityType,
   entityId: string,
-  _userId: string,
+  userId: string,
 ): Promise<void> {
   if (!ENTITY_TYPES.has(entityType)) return;
 
@@ -563,12 +606,28 @@ export async function syncEntityFolderName(
         .set({ name: candidate, updatedAt: new Date() })
         .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId)))
         .returning();
-      if (updated) return;
+      if (updated) {
+        void createAuditLog({
+          userId,
+          portId,
+          action: 'update',
+          entityType: 'document_folder',
+          entityId: folder.id,
+          oldValue: { name: folder.name },
+          newValue: { name: candidate },
+          metadata: { type: 'folder_entity_rename_sync', entity: entityType, sourceEntityId: entityId },
+        });
+        return;
+      }
     } catch (err) {
       if (isSiblingNameConflict(err)) continue;
       throw err;
     }
   }
+  logger.warn(
+    { portId, entityType, entityId, baseName, attempts: 50 },
+    'syncEntityFolderName: exhausted 50 suffix attempts without finding a unique name',
+  );
   throw new ConflictError(`Could not allocate a unique folder name for ${baseName}`);
 }
 
@@ -587,6 +646,7 @@ export async function applyEntityArchivedSuffix(
   portId: string,
   entityType: EntityType,
   entityId: string,
+  userId?: string,
 ): Promise<void> {
   if (!ENTITY_TYPES.has(entityType)) return;
   const folder = await db.query.documentFolders.findFirst({
@@ -605,6 +665,17 @@ export async function applyEntityArchivedSuffix(
     .update(documentFolders)
     .set({ name: newName, archivedAt: new Date(), updatedAt: new Date() })
     .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId)));
+
+  void createAuditLog({
+    userId: userId ?? 'system',
+    portId,
+    action: 'archive',
+    entityType: 'document_folder',
+    entityId: folder.id,
+    oldValue: { name: folder.name, archivedAt: folder.archivedAt },
+    newValue: { name: newName, archivedAt: new Date() },
+    metadata: { type: 'folder_entity_archived', entity: entityType, sourceEntityId: entityId },
+  });
 }
 
 /**
@@ -616,6 +687,7 @@ export async function applyEntityRestoredSuffix(
   portId: string,
   entityType: EntityType,
   entityId: string,
+  userId?: string,
 ): Promise<void> {
   if (!ENTITY_TYPES.has(entityType)) return;
   const folder = await db.query.documentFolders.findFirst({
@@ -634,6 +706,17 @@ export async function applyEntityRestoredSuffix(
     .update(documentFolders)
     .set({ name: newName, archivedAt: null, updatedAt: new Date() })
     .where(and(eq(documentFolders.id, folder.id), eq(documentFolders.portId, portId)));
+
+  void createAuditLog({
+    userId: userId ?? 'system',
+    portId,
+    action: 'restore',
+    entityType: 'document_folder',
+    entityId: folder.id,
+    oldValue: { name: folder.name, archivedAt: folder.archivedAt },
+    newValue: { name: newName, archivedAt: null },
+    metadata: { type: 'folder_entity_restored', entity: entityType, sourceEntityId: entityId },
+  });
 }
 
 /**

src/lib/services/yachts.service.ts

@@ -168,7 +168,7 @@ export async function updateYacht(
 
   if (data.name !== undefined) {
     await syncEntityFolderName(portId, 'yacht', id, meta.userId).catch((err) => {
-      logger.warn({ err, yachtId: id }, 'Failed to sync yacht folder name');
+      logger.warn({ err, yachtId: id, portId }, 'Failed to sync yacht folder name');
     });
   }
 
@@ -193,8 +193,8 @@ export async function archiveYacht(id: string, portId: string, meta: AuditMeta)
     .set({ archivedAt: new Date() })
     .where(and(eq(yachts.id, id), eq(yachts.portId, portId)));
 
-  void applyEntityArchivedSuffix(portId, 'yacht', id).catch((err) => {
+  void applyEntityArchivedSuffix(portId, 'yacht', id, meta.userId).catch((err) => {
-    logger.warn({ err, yachtId: id }, 'Failed to apply archived suffix to yacht folder');
+    logger.warn({ err, yachtId: id, portId }, 'Failed to apply archived suffix to yacht folder');
   });
 
   void createAuditLog({