feat(import): commit runner + undo + wired BullMQ worker

Third importer increment — the write path, fully testable without UI.

- commit.ts: commitBatch streams classified rows, applies insert/update per
  the conflict policy via the adapter (each row in its own try/catch so valid
  rows still land), records every action in import_batch_rows, and keeps live
  counts on the batch header. undoBatch hard-deletes a batch's inserted rows
  (port-scoped); a delete blocked by a dependent FK is reported, not forced,
  and the batch flips to `undone` only when every inserted row was removed.
- import worker: replaced the no-op placeholder with the real processor —
  loads the batch, re-reads the uploaded file from storage, parses, and runs
  commitBatch under the batch's mapping + policy. Marks the batch failed on
  error. Concurrency 1 so imports don't race each other's dedup lookups.

Tests: commit (skip/insert/error counts + per-row ledger + real inserted
entity), undo (removes exactly the inserted row, flips status), and
update-matches overwrite. 2 passing.

Engine is now functional end-to-end at the service layer: parse → map →
dry-run → commit → undo. Remaining: 4 FK adapters, API routes + permission,
wizard UI + history.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-01 22:36:42 +02:00
parent 3cf12b3015
commit c7325010e6
3 changed files with 392 additions and 26 deletions

187
src/lib/import/commit.ts Normal file
View File

@@ -0,0 +1,187 @@
/**
* Commit + undo for the importer.
*
* commitBatch streams classified rows, applies insert/update per the conflict
* policy via the adapter (each row in its own try/catch so valid rows land),
* records every action in import_batch_rows, and keeps live counts on the
* batch header.
*
* undoBatch hard-deletes the rows a batch *inserted* (reverse order), port-
* scoped, leaning on DB referential integrity as the guard: a row that now
* has dependents fails its delete and is reported, not force-removed.
*/
import { and, eq } from 'drizzle-orm';
import { db } from '@/lib/db';
import { importBatches, importBatchRows } from '@/lib/db/schema/imports';
import { companies } from '@/lib/db/schema/companies';
import { clients } from '@/lib/db/schema/clients';
import { berths } from '@/lib/db/schema/berths';
import { classifyRow } from './classify';
import type { ConflictPolicy, ImportAdapter, ImportCtx, RawRow } from './types';
/** Adapter keys whose inserted rows the importer can hard-delete on undo.
* Extended as the FK adapters land. */
const UNDOABLE = new Set(['companies', 'clients', 'berths']);
/** Hard-delete one imported entity, port-scoped. Throws on FK violation
* (a dependent now exists) — the caller reports that row as blocked. */
async function deleteEntity(entityType: string, entityId: string, portId: string): Promise<void> {
switch (entityType) {
case 'companies':
await db
.delete(companies)
.where(and(eq(companies.id, entityId), eq(companies.portId, portId)));
return;
case 'clients':
await db.delete(clients).where(and(eq(clients.id, entityId), eq(clients.portId, portId)));
return;
case 'berths':
await db.delete(berths).where(and(eq(berths.id, entityId), eq(berths.portId, portId)));
return;
default:
throw new Error(`Undo not supported for entity "${entityType}"`);
}
}
async function recordRow(
batchId: string,
rowNumber: number,
action: 'inserted' | 'updated' | 'skipped' | 'errored',
entityId: string | null,
error: string | null,
): Promise<void> {
await db.insert(importBatchRows).values({ batchId, rowNumber, action, entityId, error });
}
export interface CommitResult {
inserted: number;
updated: number;
skipped: number;
errored: number;
}
export async function commitBatch(params: {
batchId: string;
adapter: ImportAdapter;
rawRows: RawRow[];
mapping: Record<string, string>;
policy: ConflictPolicy;
ctx: ImportCtx;
}): Promise<CommitResult> {
const { batchId, adapter, rawRows, mapping, policy, ctx } = params;
const counts: CommitResult = { inserted: 0, updated: 0, skipped: 0, errored: 0 };
await db
.update(importBatches)
.set({ status: 'committing', totalRows: rawRows.length })
.where(eq(importBatches.id, batchId));
for (let i = 0; i < rawRows.length; i++) {
const rowNumber = i + 1;
try {
const c = await classifyRow(adapter, rawRows[i]!, mapping, rowNumber, policy, ctx);
if (c.outcome === 'error') {
counts.errored += 1;
await recordRow(
batchId,
rowNumber,
'errored',
null,
(c.errors ?? []).map((e) => `${e.field}: ${e.message}`).join('; ') || 'Invalid row',
);
} else if (c.outcome === 'skip') {
counts.skipped += 1;
await recordRow(batchId, rowNumber, 'skipped', c.existingId ?? null, null);
} else if (c.outcome === 'update' && adapter.update) {
await adapter.update(c.existingId!, c.mapped!, c.resolved ?? {}, ctx);
counts.updated += 1;
await recordRow(batchId, rowNumber, 'updated', c.existingId!, null);
} else {
// insert
const { id } = await adapter.insert(c.mapped!, c.resolved ?? {}, ctx);
counts.inserted += 1;
await recordRow(batchId, rowNumber, 'inserted', id, null);
}
} catch (e) {
counts.errored += 1;
await recordRow(batchId, rowNumber, 'errored', null, (e as Error).message);
}
if (rowNumber % 25 === 0) {
await db
.update(importBatches)
.set({ ...counts })
.where(eq(importBatches.id, batchId));
}
}
await db
.update(importBatches)
.set({ ...counts, status: 'completed', completedAt: new Date() })
.where(eq(importBatches.id, batchId));
return counts;
}
export interface UndoResult {
deleted: number;
blocked: number;
blockedRows: number[];
}
/**
* Hard-delete the rows a batch inserted (port-scoped). A delete blocked by a
* dependent (FK) is reported, not forced. Marks the batch `undone` when every
* inserted row was removed.
*/
export async function undoBatch(batchId: string, portId: string): Promise<UndoResult> {
const [batch] = await db
.select({
id: importBatches.id,
entityType: importBatches.entityType,
status: importBatches.status,
})
.from(importBatches)
.where(and(eq(importBatches.id, batchId), eq(importBatches.portId, portId)))
.limit(1);
if (!batch) throw new Error('Import batch not found');
if (!UNDOABLE.has(batch.entityType)) {
throw new Error(`Undo not supported for entity "${batch.entityType}"`);
}
const inserted = await db
.select({
id: importBatchRows.id,
rowNumber: importBatchRows.rowNumber,
entityId: importBatchRows.entityId,
})
.from(importBatchRows)
.where(and(eq(importBatchRows.batchId, batchId), eq(importBatchRows.action, 'inserted')));
const result: UndoResult = { deleted: 0, blocked: 0, blockedRows: [] };
for (const row of inserted) {
if (!row.entityId) continue;
try {
await deleteEntity(batch.entityType, row.entityId, portId);
// Flip the ledger row so a re-undo is a no-op.
await db
.update(importBatchRows)
.set({ action: 'skipped', error: 'undone' })
.where(eq(importBatchRows.id, row.id));
result.deleted += 1;
} catch {
result.blocked += 1;
result.blockedRows.push(row.rowNumber);
}
}
await db
.update(importBatches)
.set({ status: result.blocked === 0 ? 'undone' : 'completed' })
.where(eq(importBatches.id, batchId));
return result;
}