test(e2e): add Initiative 4 end-to-end + integration specs

Sales-process coverage (launch-readiness Initiative 4):
- exhaustive: full 7-stage sales journey + illegal-skip rejection + deposit
  total + tenancy/berth-sold; multi-berth EOI berth-range; EOI pathway parity
  (in-app vs Documenso, shared EoiContext); mobile-viewport journey.
- realapi (Documenso-gated, opt-in): generate-and-sign + post-EOI stages.
- integration: Documenso DOCUMENT_COMPLETED webhook idempotency (3x replay ->
  single file/audit write); storage backend swap (s3 <-> filesystem) with a
  real on-disk filesystem round-trip.
- visual: Reports UI snapshot cases (baselines captured separately).

1615 unit/integration pass; tsc + lint clean. Test-only change (specs are not
bundled into the app image) - no app behavior modified.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/integration/documenso-webhook-completion-idempotency.test.ts

@@ -0,0 +1,265 @@
+/**
+ * Initiative 4 (e2e testing) — Documenso DOCUMENT_COMPLETED idempotency.
+ *
+ * CLAUDE.md states `handleDocumentCompleted` is idempotent: it early-returns
+ * when `status='completed' && signedFileId` is already set, so Documenso's
+ * 5xx-retry storm (and the reconciling poll worker) can't double-write the
+ * signed PDF, re-clobber `documents.signedFileId`, or leak the first blob.
+ *
+ * Existing coverage gap (checked 2026-06-04):
+ *  - `documents-completion-auto-deposit.test.ts` calls `handleDocumentCompleted`
+ *    exactly ONCE per case (asserts the deposit folder/FK wiring), never a replay.
+ *  - `documenso-webhook-route.test.ts` exercises the route, and its dedup case
+ *    only covers the *route-level* `signatureHash` replay guard for an
+ *    OPENED event with an identical body — NOT the *handler-level* idempotency
+ *    gate inside `handleDocumentCompleted` (line ~1464 of documents.service.ts).
+ *
+ * This file fills that gap: replay DOCUMENT_COMPLETED 3× for the same document
+ * and assert exactly one `files` row is minted by completion (single
+ * `signedFileId` pointer, stable across replays) and exactly one
+ * `audit_logs` provenance row (`action='create'`, `entityType='file'`,
+ * `newValue.source='documenso_completion'`). Mocks all externals (Documenso
+ * download + storage backend) per the integration-test convention.
+ */
+
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { and, eq, sql } from 'drizzle-orm';
+
+import { db } from '@/lib/db';
+import { documents, files, documentFolders, documentEvents } from '@/lib/db/schema/documents';
+import { auditLogs } from '@/lib/db/schema/system';
+import { user } from '@/lib/db/schema/users';
+import { handleDocumentCompleted } from '@/lib/services/documents.service';
+import { ensureSystemRoots } from '@/lib/services/document-folders.service';
+import { makeClient, makePort } from '../helpers/factories';
+
+// Module-scoped counters the mock factories write into. Plain mutable
+// containers (not vi.fn) so they can be safely referenced from the hoisted
+// `vi.mock` factory below without the "cannot access before init" trap.
+const storagePuts: string[] = [];
+const downloadCalls: { count: number } = { count: 0 };
+
+// Stub the Documenso signed-PDF download so we never hit the network. A
+// fresh non-empty buffer keeps the 0-byte guard in the handler happy.
+vi.mock('@/lib/services/documenso-client', async (importOriginal) => {
+  const real = await importOriginal<typeof import('@/lib/services/documenso-client')>();
+  return {
+    ...real,
+    downloadSignedPdf: async () => {
+      downloadCalls.count += 1;
+      return Buffer.from('%PDF-1.4 idempotency stub\n');
+    },
+  };
+});
+
+// Stub storage so no MinIO is required. We count `put` calls to prove the
+// idempotency gate short-circuits BEFORE re-uploading on the 2nd/3rd replay.
+vi.mock('@/lib/storage', async (importOriginal) => {
+  const real = await importOriginal<typeof import('@/lib/storage')>();
+  const blobs = new Map<string, Buffer>();
+  return {
+    ...real,
+    getStorageBackend: async () => ({
+      name: 's3' as const,
+      put: async (key: string, body: Buffer | NodeJS.ReadableStream) => {
+        const buf = Buffer.isBuffer(body) ? body : Buffer.from('');
+        blobs.set(key, buf);
+        storagePuts.push(key);
+        return { key, sizeBytes: buf.length, sha256: 'stub'.padEnd(64, '0') };
+      },
+      get: async (key: string) => {
+        const { Readable } = await import('node:stream');
+        return Readable.from([blobs.get(key) ?? Buffer.alloc(0)]);
+      },
+      head: async (key: string) => {
+        const buf = blobs.get(key);
+        return buf ? { sizeBytes: buf.length, contentType: 'application/pdf' } : null;
+      },
+      delete: async (key: string) => {
+        blobs.delete(key);
+      },
+      presignUpload: async () => ({ url: 'http://stub', method: 'PUT' as const }),
+      presignDownload: async () => ({ url: 'http://stub', expiresAt: new Date(Date.now() + 1000) }),
+      listByPrefix: async (prefix: string) => [...blobs.keys()].filter((k) => k.startsWith(prefix)),
+    }),
+  };
+});
+
+/**
+ * `handleDocumentCompleted` fires its audit-log write as `void createAuditLog(...)`
+ * (fire-and-forget). Give the microtask + the single DB insert a beat to land
+ * before we assert on `audit_logs`.
+ */
+async function flushAudit(): Promise<void> {
+  await new Promise((r) => setTimeout(r, 150));
+}
+
+let TEST_USER_ID = '';
+
+beforeAll(async () => {
+  const [u] = await db.select({ id: user.id }).from(user).limit(1);
+  if (!u) throw new Error('No user available; run pnpm db:seed first');
+  TEST_USER_ID = u.id;
+});
+
+afterEach(() => {
+  storagePuts.length = 0;
+  downloadCalls.count = 0;
+});
+
+describe('handleDocumentCompleted · idempotency on webhook replay', () => {
+  let portId: string;
+  let clientId: string;
+
+  beforeEach(async () => {
+    storagePuts.length = 0;
+    downloadCalls.count = 0;
+    const port = await makePort();
+    portId = port.id;
+
+    await db.delete(documentFolders).where(eq(documentFolders.portId, portId));
+    await ensureSystemRoots(portId, TEST_USER_ID);
+
+    const client = await makeClient({ portId });
+    clientId = client.id;
+  });
+
+  it('replaying DOCUMENT_COMPLETED 3× writes exactly one signed file + one audit row', async () => {
+    const documensoId = `docu-idem-${Date.now()}`;
+    const [doc] = await db
+      .insert(documents)
+      .values({
+        portId,
+        clientId,
+        documentType: 'eoi',
+        title: 'Idempotency replay EOI',
+        status: 'partially_signed',
+        documensoId,
+        createdBy: 'seed',
+      })
+      .returning();
+
+    // Three identical deliveries, exactly as Documenso would retry on a 5xx.
+    await handleDocumentCompleted({ documentId: documensoId, portId });
+    await handleDocumentCompleted({ documentId: documensoId, portId });
+    await handleDocumentCompleted({ documentId: documensoId, portId });
+
+    // ── Gate 1: storage.put fired exactly once. The 2nd/3rd replay must
+    //    short-circuit at the `status==='completed' && signedFileId` guard
+    //    BEFORE re-downloading + re-uploading. ──────────────────────────────
+    expect(storagePuts).toHaveLength(1);
+    expect(downloadCalls.count).toBe(1);
+
+    // ── Gate 2: the document points at one stable signedFileId. ────────────
+    const updatedDoc = await db.query.documents.findFirst({
+      where: eq(documents.id, doc!.id),
+    });
+    expect(updatedDoc?.status).toBe('completed');
+    expect(updatedDoc?.signedFileId).not.toBeNull();
+    const signedFileId = updatedDoc!.signedFileId!;
+
+    // ── Gate 3: exactly one `files` row exists for this completion. A
+    //    double-write would leave a 2nd files row orphaned (no DB pointer). ──
+    const fileRows = await db
+      .select({ id: files.id, folderId: files.folderId })
+      .from(files)
+      .where(eq(files.portId, portId));
+    expect(fileRows).toHaveLength(1);
+    expect(fileRows[0]!.id).toBe(signedFileId);
+    // And it was deposited into the client entity subfolder (folder_id set once).
+    expect(fileRows[0]!.folderId).not.toBeNull();
+
+    // ── Gate 4: exactly one audit_logs provenance row for the file create. ──
+    await flushAudit();
+    const auditRows = await db
+      .select({ id: auditLogs.id, entityId: auditLogs.entityId })
+      .from(auditLogs)
+      .where(
+        and(
+          eq(auditLogs.portId, portId),
+          eq(auditLogs.action, 'create'),
+          eq(auditLogs.entityType, 'file'),
+          sql`${auditLogs.newValue}->>'source' = 'documenso_completion'`,
+        ),
+      );
+    expect(auditRows).toHaveLength(1);
+    expect(auditRows[0]!.entityId).toBe(signedFileId);
+
+    // ── Gate 5: only one 'completed' documentEvents row (the handler writes
+    //    one per non-short-circuited pass). ──────────────────────────────────
+    const completedEvents = await db
+      .select({ id: documentEvents.id })
+      .from(documentEvents)
+      .where(
+        and(eq(documentEvents.documentId, doc!.id), eq(documentEvents.eventType, 'completed')),
+      );
+    expect(completedEvents).toHaveLength(1);
+  });
+
+  it('a single delivery and a 3× replay converge on the same end state', async () => {
+    // Control: prove the replay leaves the DB in the identical shape a single
+    // delivery would — same signedFileId, same file count, same audit count.
+    const singleId = `docu-idem-single-${Date.now()}`;
+    const replayId = `docu-idem-replay-${Date.now()}`;
+
+    const client2 = await makeClient({ portId });
+
+    const [docSingle] = await db
+      .insert(documents)
+      .values({
+        portId,
+        clientId,
+        documentType: 'eoi',
+        title: 'Single delivery',
+        status: 'partially_signed',
+        documensoId: singleId,
+        createdBy: 'seed',
+      })
+      .returning();
+    const [docReplay] = await db
+      .insert(documents)
+      .values({
+        portId,
+        clientId: client2.id,
+        documentType: 'eoi',
+        title: 'Replayed delivery',
+        status: 'partially_signed',
+        documensoId: replayId,
+        createdBy: 'seed',
+      })
+      .returning();
+
+    await handleDocumentCompleted({ documentId: singleId, portId });
+
+    await handleDocumentCompleted({ documentId: replayId, portId });
+    await handleDocumentCompleted({ documentId: replayId, portId });
+    await handleDocumentCompleted({ documentId: replayId, portId });
+
+    const single = await db.query.documents.findFirst({ where: eq(documents.id, docSingle!.id) });
+    const replay = await db.query.documents.findFirst({ where: eq(documents.id, docReplay!.id) });
+
+    expect(single?.status).toBe('completed');
+    expect(replay?.status).toBe('completed');
+    expect(single?.signedFileId).not.toBeNull();
+    expect(replay?.signedFileId).not.toBeNull();
+
+    // Two completions across the two docs → exactly two file rows in the port.
+    const fileCount = await db.select({ id: files.id }).from(files).where(eq(files.portId, portId));
+    expect(fileCount).toHaveLength(2);
+
+    await flushAudit();
+    const auditRows = await db
+      .select({ id: auditLogs.id })
+      .from(auditLogs)
+      .where(
+        and(
+          eq(auditLogs.portId, portId),
+          eq(auditLogs.action, 'create'),
+          eq(auditLogs.entityType, 'file'),
+          sql`${auditLogs.newValue}->>'source' = 'documenso_completion'`,
+        ),
+      );
+    // One provenance row per completed document — the 3× replay added zero extra.
+    expect(auditRows).toHaveLength(2);
+  });
+});

tests/integration/storage-backend-swap.test.ts

@@ -0,0 +1,251 @@
+/**
+ * Initiative 4 (e2e testing) — storage backend swap.
+ *
+ * Exercises switching the deployment's storage backend between `s3` and
+ * `filesystem` through the canonical `getStorageBackend()` abstraction
+ * (`src/lib/storage/index.ts`), driven by the global
+ * `system_settings.storage_backend` row (CLAUDE.md "Storage" section).
+ *
+ * Two concerns are covered:
+ *
+ *  1. Factory selection + cache invalidation — flipping the global setting
+ *     between `filesystem` and `s3` makes `getStorageBackend()` resolve the
+ *     matching backend, and the per-process fingerprint cache re-resolves on
+ *     change (never returns a stale backend). The two concrete backends are
+ *     mocked at the `S3Backend.create` / `FilesystemBackend.create` boundary
+ *     so no MinIO/network is required — we're testing the *swap wiring*, not
+ *     the S3 wire protocol (the remote MinIO endpoint is non-deterministic in
+ *     this env).
+ *
+ *  2. Blob durability across a swap — a blob written under one backend stays
+ *     resolvable from that backend after the global setting flips and back,
+ *     mirroring a real migration window where the old store still answers
+ *     reads. A second block round-trips a real on-disk blob through the
+ *     genuine `FilesystemBackend` so the "written under one, resolvable"
+ *     guarantee is proven against real I/O, not just the in-memory fake.
+ *
+ * Per the abstraction contract we NEVER import the S3 SDK directly.
+ */
+
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import * as path from 'node:path';
+import { Readable } from 'node:stream';
+
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { and, eq, isNull } from 'drizzle-orm';
+
+import type { PresignOpts, PutOpts, StorageBackend, StorageBackendName } from '@/lib/storage';
+
+// ── In-memory fake backends, tagged by name. The factory under test picks
+//    one based on the global `storage_backend` setting; we record which
+//    `create` ran and keep the bytes so a post-swap read still resolves. ──
+const blobsByBackend: Record<StorageBackendName, Map<string, Buffer>> = {
+  s3: new Map(),
+  filesystem: new Map(),
+};
+const createCalls: StorageBackendName[] = [];
+
+function makeFake(name: StorageBackendName): StorageBackend {
+  const store = blobsByBackend[name];
+  return {
+    name,
+    async put(key: string, body: Buffer | NodeJS.ReadableStream, _opts: PutOpts) {
+      const buf = Buffer.isBuffer(body) ? body : Buffer.alloc(0);
+      store.set(key, buf);
+      return { key, sizeBytes: buf.length, sha256: 'fake'.padEnd(64, '0') };
+    },
+    async get(key: string) {
+      const buf = store.get(key);
+      if (!buf) throw new Error(`not found: ${key}`);
+      return Readable.from([buf]);
+    },
+    async head(key: string) {
+      const buf = store.get(key);
+      return buf ? { sizeBytes: buf.length, contentType: 'application/octet-stream' } : null;
+    },
+    async delete(key: string) {
+      store.delete(key);
+    },
+    async presignUpload(_key: string, _opts: PresignOpts) {
+      return { url: `mem://${name}/upload`, method: 'PUT' as const };
+    },
+    async presignDownload(_key: string, _opts: PresignOpts) {
+      return { url: `mem://${name}/download`, expiresAt: new Date(Date.now() + 1000) };
+    },
+    async listByPrefix(prefix: string) {
+      return [...store.keys()].filter((k) => k.startsWith(prefix));
+    },
+  };
+}
+
+// Replace the two concrete backend constructors with fakes. The real
+// `getStorageBackend()` factory (selection + fingerprint cache) is left
+// untouched and is the actual subject under test.
+vi.mock('@/lib/storage/s3', () => ({
+  S3Backend: {
+    create: vi.fn(async () => {
+      createCalls.push('s3');
+      return makeFake('s3');
+    }),
+  },
+}));
+vi.mock('@/lib/storage/filesystem', async (importOriginal) => {
+  // Keep the real path-validation + HMAC exports intact; only override the
+  // backend constructor so the swap test stays network/disk-free here. The
+  // genuine FilesystemBackend is exercised separately below via a direct
+  // import in its own describe block (different module instance).
+  const real = await importOriginal<typeof import('@/lib/storage/filesystem')>();
+  return {
+    ...real,
+    FilesystemBackend: {
+      create: vi.fn(async () => {
+        createCalls.push('filesystem');
+        return makeFake('filesystem');
+      }),
+    },
+  };
+});
+
+// Imported AFTER the mocks so the factory binds to the fakes.
+const { getStorageBackend, resetStorageBackendCache } = await import('@/lib/storage');
+const { db } = await import('@/lib/db');
+const { systemSettings } = await import('@/lib/db/schema/system');
+
+const STORAGE_KEYS = ['storage_backend', 'storage_filesystem_root'] as const;
+
+/** Snapshot of the pre-existing global storage rows so we can restore them. */
+let snapshot: { key: string; value: unknown }[] = [];
+
+async function setGlobal(key: string, value: unknown): Promise<void> {
+  await db
+    .delete(systemSettings)
+    .where(and(eq(systemSettings.key, key), isNull(systemSettings.portId)));
+  await db.insert(systemSettings).values({ key, value, portId: null });
+  // The settings-write hook normally calls this; do it explicitly so the
+  // factory re-resolves on the next getStorageBackend().
+  resetStorageBackendCache();
+}
+
+beforeAll(async () => {
+  snapshot = await db
+    .select({ key: systemSettings.key, value: systemSettings.value })
+    .from(systemSettings)
+    .where(and(isNull(systemSettings.portId)));
+  snapshot = snapshot.filter((r) => (STORAGE_KEYS as readonly string[]).includes(r.key));
+});
+
+afterEach(() => {
+  vi.clearAllMocks();
+  createCalls.length = 0;
+});
+
+afterAll(async () => {
+  // Restore the global storage settings exactly as we found them.
+  for (const key of STORAGE_KEYS) {
+    await db
+      .delete(systemSettings)
+      .where(and(eq(systemSettings.key, key), isNull(systemSettings.portId)));
+    const original = snapshot.find((r) => r.key === key);
+    if (original) {
+      await db.insert(systemSettings).values({ key, value: original.value, portId: null });
+    }
+  }
+  resetStorageBackendCache();
+  blobsByBackend.s3.clear();
+  blobsByBackend.filesystem.clear();
+});
+
+describe('getStorageBackend · backend swap selection', () => {
+  it('resolves filesystem then s3 as the global setting flips', async () => {
+    await setGlobal('storage_backend', 'filesystem');
+    const fs = await getStorageBackend();
+    expect(fs.name).toBe('filesystem');
+
+    await setGlobal('storage_backend', 's3');
+    const s3 = await getStorageBackend();
+    expect(s3.name).toBe('s3');
+
+    // Both concrete constructors ran exactly once across the two resolutions.
+    expect(createCalls.filter((c) => c === 'filesystem')).toHaveLength(1);
+    expect(createCalls.filter((c) => c === 's3')).toHaveLength(1);
+  });
+
+  it('any non-"filesystem" value falls back to s3 (factory default)', async () => {
+    await setGlobal('storage_backend', 'totally-unknown-value');
+    const backend = await getStorageBackend();
+    expect(backend.name).toBe('s3');
+  });
+
+  it('caches within a config and re-resolves only after a change', async () => {
+    await setGlobal('storage_backend', 'filesystem');
+    await getStorageBackend();
+    await getStorageBackend();
+    await getStorageBackend();
+    // Cache hit: the constructor ran once despite three resolutions.
+    expect(createCalls.filter((c) => c === 'filesystem')).toHaveLength(1);
+
+    // Changing the setting (here: the fs root) invalidates the fingerprint.
+    await setGlobal('storage_filesystem_root', '/tmp/pn-swap-test-root');
+    await getStorageBackend();
+    expect(createCalls.filter((c) => c === 'filesystem')).toHaveLength(2);
+  });
+
+  it('a blob written under one backend stays resolvable after a swap and back', async () => {
+    // Write under filesystem.
+    await setGlobal('storage_backend', 'filesystem');
+    const fsBackend = await getStorageBackend();
+    const key = 'test-port/swap/blob.bin';
+    const payload = Buffer.from('swap-durability-payload');
+    await fsBackend.put(key, payload, { contentType: 'application/octet-stream' });
+
+    // Swap to s3 — the filesystem store still holds the bytes (old store keeps
+    // answering reads during a migration window).
+    await setGlobal('storage_backend', 's3');
+    const s3Backend = await getStorageBackend();
+    expect(s3Backend.name).toBe('s3');
+    expect(blobsByBackend.s3.has(key)).toBe(false); // not migrated by a mere flip
+
+    // Swap back to filesystem and confirm the original blob round-trips.
+    await setGlobal('storage_backend', 'filesystem');
+    const fsAgain = await getStorageBackend();
+    const stream = await fsAgain.get(key);
+    const chunks: Buffer[] = [];
+    for await (const c of stream) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c as string));
+    expect(Buffer.concat(chunks).equals(payload)).toBe(true);
+  });
+});
+
+describe('FilesystemBackend · real on-disk round-trip (abstraction contract)', () => {
+  it('writes and resolves a real blob via the genuine filesystem backend', async () => {
+    // Use the REAL FilesystemBackend directly (separate module instance from
+    // the mocked one above — vi.mock is scoped to the factory's import graph).
+    // EMAIL_CREDENTIAL_KEY / BETTER_AUTH_SECRET are required by the key
+    // validator at module load; the vitest env provides them.
+    const { FilesystemBackend } = await vi.importActual<typeof import('@/lib/storage/filesystem')>(
+      '@/lib/storage/filesystem',
+    );
+
+    const root = await mkdtemp(path.join(tmpdir(), 'pn-swap-fs-'));
+    try {
+      const backend = await FilesystemBackend.create({ root, proxyHmacSecretEncrypted: null });
+      expect(backend.name).toBe('filesystem');
+
+      const key = 'sub/dir/durable.bin';
+      const payload = Buffer.from('real-on-disk-bytes');
+      const put = await backend.put(key, payload, { contentType: 'application/octet-stream' });
+      expect(put.sizeBytes).toBe(payload.length);
+      expect(put.sha256).toMatch(/^[0-9a-f]{64}$/);
+
+      const head = await backend.head(key);
+      expect(head?.sizeBytes).toBe(payload.length);
+
+      const stream = await backend.get(key);
+      const chunks: Buffer[] = [];
+      for await (const c of stream) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c as string));
+      expect(Buffer.concat(chunks).equals(payload)).toBe(true);
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+});