From 3cf12b3015cc329cc23f8cc5041b8da60e154e65 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 1 Jun 2026 22:32:19 +0200 Subject: [PATCH] feat(import): engine core + companies/clients/berths adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second importer increment: the generic engine + the three no-FK adapters, fully unit + integration tested. - types: ImportAdapter contract (targetFields, matchKey, findExisting, resolveForeignKeys, insert/update) + engine types. - mapping: fuzzy header → target-field auto-suggest (exact / alias / edit distance, one header per field) + applyMapping (drops empty cells). - classify: per-field zod + cross-field extraValidate, FK resolution hook, natural-key dedup, and the conflict-policy matrix (skip-matches / update-matches / error-on-match) → row outcomes + summary. - engine: CSV (papaparse) + XLSX (ExcelJS) parse into a uniform {headers, rows} of trimmed strings. - adapters (delegating to existing create/update services for audit + validation): companies (name dedup, update), clients (flat email/phone → contacts[], email-or-phone dedup, insert-only), berths (canonical mooring dedup, numeric coercion, update). - registry: implemented adapters in dependency order. Tests: 11 unit (mapping/validation/matchKey/parse) + 3 integration (dedup + all three conflict policies on a seeded DB). 14 passing. Next increments: FK adapters (yachts/interests/tenancies/expenses), commit runner + worker, API routes + permission, wizard UI + undo. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/import/adapters/berths.ts | 100 +++++++++++++++++ src/lib/import/adapters/clients.ts | 125 ++++++++++++++++++++++ src/lib/import/adapters/companies.ts | 87 +++++++++++++++ src/lib/import/classify.ts | 108 +++++++++++++++++++ src/lib/import/engine.ts | 77 +++++++++++++ src/lib/import/mapping.ts | 83 ++++++++++++++ src/lib/import/registry.ts | 28 +++++ src/lib/import/types.ts | 103 ++++++++++++++++++ tests/integration/import-classify.test.ts | 67 ++++++++++++ tests/unit/import/engine.test.ts | 91 ++++++++++++++++ 10 files changed, 869 insertions(+) create mode 100644 src/lib/import/adapters/berths.ts create mode 100644 src/lib/import/adapters/clients.ts create mode 100644 src/lib/import/adapters/companies.ts create mode 100644 src/lib/import/classify.ts create mode 100644 src/lib/import/engine.ts create mode 100644 src/lib/import/mapping.ts create mode 100644 src/lib/import/registry.ts create mode 100644 src/lib/import/types.ts create mode 100644 tests/integration/import-classify.test.ts create mode 100644 tests/unit/import/engine.test.ts diff --git a/src/lib/import/adapters/berths.ts b/src/lib/import/adapters/berths.ts new file mode 100644 index 00000000..bc395b41 --- /dev/null +++ b/src/lib/import/adapters/berths.ts @@ -0,0 +1,100 @@ +import { and, eq } from 'drizzle-orm'; +import { z } from 'zod'; + +import { db } from '@/lib/db'; +import { berths } from '@/lib/db/schema/berths'; +import { createBerth, updateBerth } from '@/lib/services/berths.service'; + +import type { ImportAdapter, MappedRow } from '../types'; + +/** Canonicalize a mooring to the unified `^[A-Z]+\d+$` form ("A1", "D32"): + * uppercase letters, drop a hyphen + leading zeros on the number. */ +function canonMoo(raw: string): string { + const m = /^([A-Za-z]+)-?0*(\d+)$/.exec(raw.trim()); + return m ? `${m[1]!.toUpperCase()}${parseInt(m[2]!, 10)}` : raw.trim().toUpperCase(); +} + +const num = (s: string | undefined): number | undefined => + s === undefined || s === '' ? undefined : Number(s); + +export const berthsAdapter: ImportAdapter = { + key: 'berths', + label: 'Berths', + order: 4, + dependsOn: [], + targetFields: [ + { + key: 'mooringNumber', + label: 'Mooring number', + required: true, + aliases: ['mooring', 'berth', 'berthnumber'], + zod: z.string().regex(/^[A-Za-z]+-?0*\d+$/, 'Use a form like A1, B12, E18'), + }, + { + key: 'area', + label: 'Area', + required: true, + aliases: ['dock', 'zone'], + zod: z.string().min(1), + }, + { key: 'lengthFt', label: 'Length (ft)', required: false, zod: z.coerce.number() }, + { key: 'widthFt', label: 'Width (ft)', required: false, zod: z.coerce.number() }, + { key: 'draftFt', label: 'Draft (ft)', required: false, zod: z.coerce.number() }, + { key: 'price', label: 'Price', required: false, zod: z.coerce.number() }, + { + key: 'priceCurrency', + label: 'Currency', + required: false, + zod: z.string().length(3), + }, + { + key: 'status', + label: 'Status', + required: false, + zod: z.enum(['available', 'under_offer', 'sold']), + }, + ], + matchKey: (row) => (row.mooringNumber ? canonMoo(row.mooringNumber) : null), + findExisting: async (portId, key) => { + const row = await db.query.berths.findFirst({ + where: and(eq(berths.portId, portId), eq(berths.mooringNumber, key)), + columns: { id: true }, + }); + return row ?? null; + }, + insert: async (row, _resolved, ctx) => { + const b = await createBerth(ctx.portId, buildBerthInput(row), ctx.meta); + return { id: b.id }; + }, + update: async (id, row, _resolved, ctx) => { + const full = buildBerthInput(row); + // Update spec fields only — mooring is the match key, and status has its + // own dedicated endpoint (not part of UpdateBerthInput). + await updateBerth( + id, + ctx.portId, + { + area: full.area, + lengthFt: full.lengthFt, + widthFt: full.widthFt, + draftFt: full.draftFt, + price: full.price, + priceCurrency: full.priceCurrency, + }, + ctx.meta, + ); + }, +}; + +function buildBerthInput(row: MappedRow) { + return { + mooringNumber: canonMoo(row.mooringNumber!), + area: row.area!, // required field — present after validation + lengthFt: num(row.lengthFt), + widthFt: num(row.widthFt), + draftFt: num(row.draftFt), + price: num(row.price), + priceCurrency: row.priceCurrency, + status: (row.status as 'available' | 'under_offer' | 'sold' | undefined) ?? 'available', + }; +} diff --git a/src/lib/import/adapters/clients.ts b/src/lib/import/adapters/clients.ts new file mode 100644 index 00000000..2389d536 --- /dev/null +++ b/src/lib/import/adapters/clients.ts @@ -0,0 +1,125 @@ +import { and, eq, sql } from 'drizzle-orm'; +import { z } from 'zod'; + +import { db } from '@/lib/db'; +import { clients, clientContacts } from '@/lib/db/schema/clients'; +import { createClient } from '@/lib/services/clients.service'; +import { normalizeEmail, normalizePhone } from '@/lib/dedup/normalize'; + +import type { ImportAdapter, MappedRow } from '../types'; + +/** Build the contacts array createClient requires from the flat email/phone + * columns. Phone carries its E.164 form (the contact schema requires it). */ +function buildContacts(row: MappedRow) { + const contacts: Array<{ + channel: 'email' | 'phone'; + value: string; + valueE164?: string; + valueCountry?: string; + isPrimary: boolean; + }> = []; + if (row.email) contacts.push({ channel: 'email', value: row.email, isPrimary: true }); + if (row.phone) { + const ph = normalizePhone(row.phone); + contacts.push({ + channel: 'phone', + value: row.phone, + valueE164: ph?.e164 ?? undefined, + valueCountry: ph?.country ?? undefined, + isPrimary: !row.email, + }); + } + return contacts; +} + +export const clientsAdapter: ImportAdapter = { + key: 'clients', + label: 'Clients', + order: 2, + dependsOn: [], + targetFields: [ + { + key: 'fullName', + label: 'Full name', + required: true, + aliases: ['name', 'client', 'clientname'], + zod: z.string().min(1), + }, + { key: 'email', label: 'Email', required: false, zod: z.string().email() }, + { key: 'phone', label: 'Phone', required: false, aliases: ['mobile', 'tel'], zod: z.string() }, + { + key: 'nationalityIso', + label: 'Nationality (ISO-2)', + required: false, + aliases: ['nationality', 'country'], + zod: z.string().length(2), + }, + { + key: 'preferredContactMethod', + label: 'Preferred contact', + required: false, + zod: z.enum(['email', 'phone', 'whatsapp']), + }, + { + key: 'source', + label: 'Source', + required: false, + zod: z.enum(['website', 'manual', 'referral', 'broker', 'other']), + }, + { key: 'sourceDetails', label: 'Source details', required: false, zod: z.string() }, + ], + extraValidate: (row) => + row.email || row.phone ? [] : [{ field: 'email', message: 'An email or phone is required' }], + matchKey: (row) => { + const email = normalizeEmail(row.email ?? null); + if (email) return `email:${email}`; + const ph = normalizePhone(row.phone ?? null); + return ph?.e164 ? `phone:${ph.e164}` : null; + }, + findExisting: async (portId, key) => { + const idx = key.indexOf(':'); + const channel = key.slice(0, idx); + const value = key.slice(idx + 1); + const base = db + .select({ id: clients.id }) + .from(clientContacts) + .innerJoin(clients, eq(clients.id, clientContacts.clientId)); + const rows = + channel === 'email' + ? await base + .where( + and( + eq(clients.portId, portId), + eq(clientContacts.channel, 'email'), + sql`lower(${clientContacts.value}) = ${value}`, + ), + ) + .limit(1) + : await base + .where(and(eq(clients.portId, portId), eq(clientContacts.valueE164, value))) + .limit(1); + return rows[0] ?? null; + }, + insert: async (row, _resolved, ctx) => { + const c = await createClient( + ctx.portId, + { + fullName: row.fullName!, + contacts: buildContacts(row), + nationalityIso: row.nationalityIso, + preferredContactMethod: row.preferredContactMethod as + | 'email' + | 'phone' + | 'whatsapp' + | undefined, + source: row.source as 'website' | 'manual' | 'referral' | 'broker' | 'other' | undefined, + sourceDetails: row.sourceDetails, + tagIds: [], + }, + ctx.meta, + ); + return { id: c.id }; + }, + // Insert-only under the importer (v1): a client's contacts/addresses are a + // graph the flat update can't safely overwrite. update-matches → skip. +}; diff --git a/src/lib/import/adapters/companies.ts b/src/lib/import/adapters/companies.ts new file mode 100644 index 00000000..f399d130 --- /dev/null +++ b/src/lib/import/adapters/companies.ts @@ -0,0 +1,87 @@ +import { and, eq, sql } from 'drizzle-orm'; +import { z } from 'zod'; + +import { db } from '@/lib/db'; +import { companies } from '@/lib/db/schema/companies'; +import { createCompany, updateCompany } from '@/lib/services/companies.service'; + +import type { ImportAdapter } from '../types'; + +export const companiesAdapter: ImportAdapter = { + key: 'companies', + label: 'Companies', + order: 1, + dependsOn: [], + targetFields: [ + { + key: 'name', + label: 'Name', + required: true, + aliases: ['company', 'companyname'], + zod: z.string().min(1), + }, + { key: 'legalName', label: 'Legal name', required: false, zod: z.string() }, + { + key: 'taxId', + label: 'Tax ID', + required: false, + aliases: ['vat', 'vatnumber'], + zod: z.string(), + }, + { key: 'registrationNumber', label: 'Registration number', required: false, zod: z.string() }, + { + key: 'billingEmail', + label: 'Billing email', + required: false, + aliases: ['email'], + zod: z.string().email(), + }, + { + key: 'status', + label: 'Status', + required: false, + zod: z.enum(['active', 'dissolved']), + }, + { key: 'notes', label: 'Notes', required: false, zod: z.string() }, + ], + matchKey: (row) => (row.name ? row.name.trim().toLowerCase() : null), + findExisting: async (portId, key) => { + const row = await db.query.companies.findFirst({ + where: and(eq(companies.portId, portId), sql`lower(${companies.name}) = ${key}`), + columns: { id: true }, + }); + return row ?? null; + }, + insert: async (row, _resolved, ctx) => { + const c = await createCompany( + ctx.portId, + { + name: row.name!, + legalName: row.legalName, + taxId: row.taxId, + registrationNumber: row.registrationNumber, + billingEmail: row.billingEmail, + status: (row.status as 'active' | 'dissolved' | undefined) ?? 'active', + notes: row.notes, + tagIds: [], + }, + ctx.meta, + ); + return { id: c.id }; + }, + update: async (id, row, _resolved, ctx) => { + await updateCompany( + id, + ctx.portId, + { + legalName: row.legalName, + taxId: row.taxId, + registrationNumber: row.registrationNumber, + billingEmail: row.billingEmail, + status: row.status as 'active' | 'dissolved' | undefined, + notes: row.notes, + }, + ctx.meta, + ); + }, +}; diff --git a/src/lib/import/classify.ts b/src/lib/import/classify.ts new file mode 100644 index 00000000..d1e39408 --- /dev/null +++ b/src/lib/import/classify.ts @@ -0,0 +1,108 @@ +/** + * Row classification — the heart of both the dry-run preview and the commit. + * + * For one mapped row: validate (required + per-field zod) → resolve FKs → + * dedup by natural key → resolve to an outcome under the chosen conflict + * policy. Pure with respect to writes; the commit step acts on the outcome. + */ +import { applyMapping } from './mapping'; +import type { + ClassifiedRow, + ConflictPolicy, + ImportAdapter, + ImportCtx, + MappedRow, + RawRow, + RowError, +} from './types'; + +/** Required-field + per-field zod validation. Never throws; returns all errors. */ +export function validateRow(adapter: ImportAdapter, mapped: MappedRow): RowError[] { + const errors: RowError[] = []; + for (const field of adapter.targetFields) { + const value = mapped[field.key]; + if (value === undefined) { + if (field.required) errors.push({ field: field.key, message: `${field.label} is required` }); + continue; + } + const res = field.zod.safeParse(value); + if (!res.success) { + const first = res.error.issues[0]; + errors.push({ field: field.key, message: first?.message ?? 'Invalid value' }); + } + } + if (errors.length === 0 && adapter.extraValidate) errors.push(...adapter.extraValidate(mapped)); + return errors; +} + +export async function classifyRow( + adapter: ImportAdapter, + raw: RawRow, + mapping: Record, + rowNumber: number, + policy: ConflictPolicy, + ctx: ImportCtx, +): Promise { + const mapped = applyMapping(raw, mapping); + + const errors = validateRow(adapter, mapped); + if (errors.length > 0) return { rowNumber, outcome: 'error', errors }; + + // Foreign keys (by natural key) — resolution failures are row errors. + let resolved: Record = {}; + if (adapter.resolveForeignKeys) { + const fk = await adapter.resolveForeignKeys(mapped, ctx); + if (!fk.ok) return { rowNumber, outcome: 'error', errors: fk.errors }; + resolved = fk.resolved; + } + + // Dedup by natural key. + const key = adapter.matchKey(mapped); + const existing = key ? await adapter.findExisting(ctx.portId, key) : null; + + if (existing) { + if (policy === 'error-on-match') { + return { + rowNumber, + outcome: 'error', + existingId: existing.id, + errors: [{ field: '*', message: 'Matches an existing record' }], + }; + } + if (policy === 'update-matches' && adapter.update) { + return { rowNumber, outcome: 'update', existingId: existing.id, mapped, resolved }; + } + // skip-matches, or update requested on an insert-only adapter. + return { rowNumber, outcome: 'skip', existingId: existing.id, mapped, resolved }; + } + + return { rowNumber, outcome: 'insert', mapped, resolved }; +} + +export interface DryRunSummary { + total: number; + insert: number; + update: number; + skip: number; + error: number; + rows: ClassifiedRow[]; +} + +/** Classify a whole file. Sequential to keep FK lookups + dedup ordering + * deterministic (migration files are small). */ +export async function classifyRows( + adapter: ImportAdapter, + rawRows: RawRow[], + mapping: Record, + policy: ConflictPolicy, + ctx: ImportCtx, +): Promise { + const rows: ClassifiedRow[] = []; + const summary = { total: rawRows.length, insert: 0, update: 0, skip: 0, error: 0 }; + for (let i = 0; i < rawRows.length; i++) { + const c = await classifyRow(adapter, rawRows[i]!, mapping, i + 1, policy, ctx); + rows.push(c); + summary[c.outcome] += 1; + } + return { ...summary, rows }; +} diff --git a/src/lib/import/engine.ts b/src/lib/import/engine.ts new file mode 100644 index 00000000..0362f5ea --- /dev/null +++ b/src/lib/import/engine.ts @@ -0,0 +1,77 @@ +/** + * File parsing for the importer — CSV (papaparse) and XLSX (ExcelJS) into a + * uniform `{ headers, rows }` shape where every cell is a trimmed string. + */ +import ExcelJS from 'exceljs'; +import Papa from 'papaparse'; + +import type { RawRow } from './types'; + +export interface ParsedFile { + headers: string[]; + rows: RawRow[]; +} + +function cellToString(v: unknown): string { + if (v === null || v === undefined) return ''; + if (typeof v === 'object') { + // ExcelJS rich-text / hyperlink / formula cells. + const o = v as { text?: unknown; result?: unknown; hyperlink?: unknown }; + if (typeof o.text === 'string') return o.text; + if (o.result !== undefined) return String(o.result); + if (typeof o.hyperlink === 'string') return o.hyperlink; + } + return String(v); +} + +export function parseCsv(content: string): ParsedFile { + const res = Papa.parse>(content, { + header: true, + skipEmptyLines: 'greedy', + transformHeader: (h) => h.trim(), + }); + const headers = (res.meta.fields ?? []).map((h) => h.trim()).filter(Boolean); + const rows: RawRow[] = res.data.map((r) => { + const out: RawRow = {}; + for (const h of headers) out[h] = cellToString(r[h]).trim(); + return out; + }); + return { headers, rows }; +} + +export async function parseXlsx(buffer: Buffer): Promise { + const wb = new ExcelJS.Workbook(); + await wb.xlsx.load(buffer as unknown as ArrayBuffer); + const ws = wb.worksheets[0]; + if (!ws) return { headers: [], rows: [] }; + + const headerRow = ws.getRow(1); + const headers: string[] = []; + const colIndex: number[] = []; + headerRow.eachCell((cell, col) => { + const h = cellToString(cell.value).trim(); + if (h) { + headers.push(h); + colIndex.push(col); + } + }); + + const rows: RawRow[] = []; + for (let r = 2; r <= ws.rowCount; r++) { + const row = ws.getRow(r); + const out: RawRow = {}; + let nonEmpty = false; + headers.forEach((h, i) => { + const v = cellToString(row.getCell(colIndex[i]!).value).trim(); + out[h] = v; + if (v) nonEmpty = true; + }); + if (nonEmpty) rows.push(out); + } + return { headers, rows }; +} + +export async function parseImportFile(filename: string, buffer: Buffer): Promise { + if (/\.xlsx?$/i.test(filename)) return parseXlsx(buffer); + return parseCsv(buffer.toString('utf8')); +} diff --git a/src/lib/import/mapping.ts b/src/lib/import/mapping.ts new file mode 100644 index 00000000..1a99f434 --- /dev/null +++ b/src/lib/import/mapping.ts @@ -0,0 +1,83 @@ +/** + * Column mapping: auto-suggest target-field → source-header pairings by fuzzy + * header match, and apply a chosen mapping to a raw row. + */ +import type { ImportField, MappedRow, RawRow } from './types'; + +/** lowercase, strip everything but a-z0-9 → comparable token. */ +function norm(s: string): string { + return s.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function lev(a: string, b: string): number { + const m = a.length; + const n = b.length; + if (!m) return n; + if (!n) return m; + let prev = Array.from({ length: n + 1 }, (_, i) => i); + for (let i = 1; i <= m; i++) { + const cur = [i]; + for (let j = 1; j <= n; j++) { + cur[j] = Math.min( + prev[j]! + 1, + cur[j - 1]! + 1, + prev[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + } + prev = cur; + } + return prev[n]!; +} + +/** + * For each field, pick the best-matching header. Exact normalized match on + * key / label / alias wins; otherwise a substring or close edit-distance + * match. A header is claimed by at most one field (first-come by field order). + * Returns `fieldKey → header` (only confident matches; unmatched fields absent). + */ +export function suggestMapping(headers: string[], fields: ImportField[]): Record { + const out: Record = {}; + const taken = new Set(); + const normHeaders = headers.map((h) => ({ raw: h, n: norm(h) })); + + for (const field of fields) { + const candidates = [field.key, field.label, ...(field.aliases ?? [])].map(norm); + let best: { header: string; score: number } | null = null; + + for (const h of normHeaders) { + if (taken.has(h.raw) || !h.n) continue; + let score = Infinity; + for (const c of candidates) { + if (!c) continue; + if (c === h.n) score = Math.min(score, 0); + else if (c.includes(h.n) || h.n.includes(c)) score = Math.min(score, 1); + else { + const d = lev(c, h.n); + // Accept only close matches (≤2 edits, and not longer than the token). + if (d <= 2 && d < Math.max(c.length, h.n.length)) score = Math.min(score, 1 + d); + } + } + if (score < Infinity && (!best || score < best.score)) best = { header: h.raw, score }; + } + + if (best) { + out[field.key] = best.header; + taken.add(best.header); + } + } + return out; +} + +/** + * Apply a `fieldKey → header` mapping to a raw row, producing `fieldKey → cell`. + * Empty / whitespace-only cells are dropped so downstream "required" checks and + * optional-field omission behave correctly. + */ +export function applyMapping(raw: RawRow, mapping: Record): MappedRow { + const out: MappedRow = {}; + for (const [fieldKey, header] of Object.entries(mapping)) { + const cell = (raw[header] ?? '').trim(); + if (cell !== '') out[fieldKey] = cell; + } + return out; +} diff --git a/src/lib/import/registry.ts b/src/lib/import/registry.ts new file mode 100644 index 00000000..38bccd39 --- /dev/null +++ b/src/lib/import/registry.ts @@ -0,0 +1,28 @@ +/** + * Import adapter registry. Adding an entity = add an adapter file + register + * it here; the engine and UI are driven entirely off this map. + * + * Implemented so far: companies, clients, berths (no-FK). The FK entities + * (yachts, interests, tenancies, expenses) layer in as their adapters land. + */ +import { berthsAdapter } from './adapters/berths'; +import { clientsAdapter } from './adapters/clients'; +import { companiesAdapter } from './adapters/companies'; +import type { ImportAdapter, ImportEntityKey } from './types'; + +export const IMPORT_REGISTRY: Partial> = { + companies: companiesAdapter, + clients: clientsAdapter, + berths: berthsAdapter, +}; + +export function getAdapter(key: string): ImportAdapter | null { + return IMPORT_REGISTRY[key as ImportEntityKey] ?? null; +} + +/** Implemented adapters in dependency order (companies → … → expenses). */ +export function listAdapters(): ImportAdapter[] { + return Object.values(IMPORT_REGISTRY) + .filter((a): a is ImportAdapter => Boolean(a)) + .sort((a, b) => a.order - b.order); +} diff --git a/src/lib/import/types.ts b/src/lib/import/types.ts new file mode 100644 index 00000000..f9ebde6a --- /dev/null +++ b/src/lib/import/types.ts @@ -0,0 +1,103 @@ +/** + * Bulk-importer engine types. + * + * One generic pipeline parameterised by a per-entity {@link ImportAdapter}, + * mirroring the custom-report registry pattern. The engine (parse → map → + * classify → commit) never knows about a specific entity; everything + * entity-specific lives in an adapter. + * + * See docs/superpowers/specs/2026-06-01-bulk-import-design.md. + */ +import type { z } from 'zod'; + +import type { AuditMeta } from '@/lib/audit'; + +export const IMPORT_ENTITY_KEYS = [ + 'companies', + 'clients', + 'yachts', + 'berths', + 'interests', + 'tenancies', + 'expenses', +] as const; + +export type ImportEntityKey = (typeof IMPORT_ENTITY_KEYS)[number]; + +export const CONFLICT_POLICIES = ['skip-matches', 'update-matches', 'error-on-match'] as const; +export type ConflictPolicy = (typeof CONFLICT_POLICIES)[number]; + +/** Per-target-field definition. Drives the column-mapping UI + row validation. */ +export interface ImportField { + key: string; + label: string; + required: boolean; + /** Extra header spellings for fuzzy auto-mapping (besides key + label). */ + aliases?: string[]; + /** Per-cell validation. Empty optional cells are dropped before this runs. */ + zod: z.ZodTypeAny; +} + +/** A raw parsed row: source header → cell value (strings). */ +export type RawRow = Record; +/** A mapped row: target field key → cell value (strings), empties removed. */ +export type MappedRow = Record; + +export interface ImportCtx { + portId: string; + meta: AuditMeta; +} + +export interface RowError { + field: string; + message: string; +} + +export type FkResult = + | { ok: true; resolved: Record } + | { ok: false; errors: RowError[] }; + +/** The four dry-run outcomes for a single source row. */ +export type RowOutcome = 'insert' | 'update' | 'skip' | 'error'; + +export interface ClassifiedRow { + rowNumber: number; + outcome: RowOutcome; + /** Set when outcome=update/skip (the matched existing entity). */ + existingId?: string; + errors?: RowError[]; + /** Validated + FK-resolved payload, carried to the commit step. */ + mapped?: MappedRow; + resolved?: Record; +} + +export interface ImportAdapter { + key: ImportEntityKey; + label: string; + /** Dependency order — companies(1) … expenses(7). */ + order: number; + dependsOn: ImportEntityKey[]; + targetFields: ImportField[]; + /** Optional cross-field validation (e.g. "email or phone required") run + * after per-field zod. Returns errors, never throws. */ + extraValidate?: (row: MappedRow) => RowError[]; + /** Natural dedup key derived from a validated mapped row (or null). */ + matchKey: (row: MappedRow) => string | null; + /** Find an existing entity by its natural key, port-scoped. */ + findExisting: (portId: string, matchKey: string) => Promise<{ id: string } | null>; + /** Resolve FK ids by natural key. Omit for entities with no FKs. */ + resolveForeignKeys?: (row: MappedRow, ctx: ImportCtx) => Promise; + /** Delegates to the entity's own create service (audit + validation free). */ + insert: ( + row: MappedRow, + resolved: Record, + ctx: ImportCtx, + ) => Promise<{ id: string }>; + /** Omit when the entity is insert-only under the importer. */ + update?: ( + existingId: string, + row: MappedRow, + resolved: Record, + ctx: ImportCtx, + ) => Promise; +} diff --git a/tests/integration/import-classify.test.ts b/tests/integration/import-classify.test.ts new file mode 100644 index 00000000..14962bff --- /dev/null +++ b/tests/integration/import-classify.test.ts @@ -0,0 +1,67 @@ +/** + * Integration test: dry-run classifier dedup + conflict-policy matrix. + * + * Seeds an existing company, then classifies a 3-row "file" (one matching the + * existing record, one new, one invalid) under each conflict policy and asserts + * the per-row outcomes + summary counts. Runs against the real test DB. + */ +import { beforeAll, describe, expect, it } from 'vitest'; + +import { classifyRows } from '@/lib/import/classify'; +import { companiesAdapter } from '@/lib/import/adapters/companies'; +import type { RawRow } from '@/lib/import/types'; + +let makePort: typeof import('../helpers/factories').makePort; +let makeCompany: typeof import('../helpers/factories').makeCompany; +let makeAuditMeta: typeof import('../helpers/factories').makeAuditMeta; + +beforeAll(async () => { + const f = await import('../helpers/factories'); + makePort = f.makePort; + makeCompany = f.makeCompany; + makeAuditMeta = f.makeAuditMeta; +}); + +const MAPPING = { name: 'Name' }; + +describe('classifyRows — dedup + conflict policy', () => { + async function setup() { + const port = await makePort(); + await makeCompany({ portId: port.id, overrides: { name: 'Acme Marine' } }); + const ctx = { portId: port.id, meta: makeAuditMeta({ portId: port.id }) }; + const rows: RawRow[] = [ + { Name: 'Acme Marine' }, // matches existing (case-insensitive) + { Name: 'Brand New Co' }, // insert + { Name: '' }, // error: required + ]; + return { ctx, rows }; + } + + it('skip-matches: existing→skip, new→insert, blank→error', async () => { + const { ctx, rows } = await setup(); + const r = await classifyRows(companiesAdapter, rows, MAPPING, 'skip-matches', ctx); + expect({ insert: r.insert, update: r.update, skip: r.skip, error: r.error }).toEqual({ + insert: 1, + update: 0, + skip: 1, + error: 1, + }); + expect(r.rows.map((x) => x.outcome)).toEqual(['skip', 'insert', 'error']); + expect(r.rows[0]!.existingId).toBeTruthy(); + }); + + it('update-matches: existing→update (adapter supports update)', async () => { + const { ctx, rows } = await setup(); + const r = await classifyRows(companiesAdapter, rows, MAPPING, 'update-matches', ctx); + expect(r.rows[0]!.outcome).toBe('update'); + expect(r.update).toBe(1); + }); + + it('error-on-match: existing→error', async () => { + const { ctx, rows } = await setup(); + const r = await classifyRows(companiesAdapter, rows, MAPPING, 'error-on-match', ctx); + expect(r.rows[0]!.outcome).toBe('error'); + // The blank row is still an error too → 2 errors total. + expect(r.error).toBe(2); + }); +}); diff --git a/tests/unit/import/engine.test.ts b/tests/unit/import/engine.test.ts new file mode 100644 index 00000000..4b8b2435 --- /dev/null +++ b/tests/unit/import/engine.test.ts @@ -0,0 +1,91 @@ +/** + * Unit tests for the importer engine internals (no DB): fuzzy column mapping, + * row validation (per-field + cross-field), natural-key derivation, and CSV + * parsing. + */ +import { describe, expect, it } from 'vitest'; + +import { applyMapping, suggestMapping } from '@/lib/import/mapping'; +import { validateRow } from '@/lib/import/classify'; +import { parseCsv } from '@/lib/import/engine'; +import { companiesAdapter } from '@/lib/import/adapters/companies'; +import { clientsAdapter } from '@/lib/import/adapters/clients'; +import { berthsAdapter } from '@/lib/import/adapters/berths'; + +describe('suggestMapping', () => { + it('maps headers to fields by exact, alias, and fuzzy match', () => { + const m = suggestMapping( + ['Company Name', 'VAT', 'Billing Email', 'Unrelated'], + companiesAdapter.targetFields, + ); + expect(m.name).toBe('Company Name'); // alias "companyname" + expect(m.taxId).toBe('VAT'); // alias "vat" + expect(m.billingEmail).toBe('Billing Email'); + // A header claimed by one field isn't reused by another. + expect(Object.values(m).filter((h) => h === 'Company Name')).toHaveLength(1); + }); + + it('leaves unmatched fields out of the mapping', () => { + const m = suggestMapping(['xyz', 'qqq'], companiesAdapter.targetFields); + expect(m.name).toBeUndefined(); + }); +}); + +describe('applyMapping', () => { + it('produces fieldKey→cell and drops empty cells', () => { + const mapped = applyMapping( + { 'Company Name': ' Acme ', VAT: '', Email: 'a@b.com' }, + { name: 'Company Name', taxId: 'VAT', billingEmail: 'Email' }, + ); + expect(mapped).toEqual({ name: 'Acme', billingEmail: 'a@b.com' }); + }); +}); + +describe('validateRow', () => { + it('flags a missing required field', () => { + const errs = validateRow(companiesAdapter, { legalName: 'X Ltd' }); + expect(errs).toEqual([{ field: 'name', message: 'Name is required' }]); + }); + + it('flags an invalid email', () => { + const errs = validateRow(companiesAdapter, { name: 'Acme', billingEmail: 'not-an-email' }); + expect(errs.some((e) => e.field === 'billingEmail')).toBe(true); + }); + + it('clients require an email or a phone (cross-field)', () => { + expect(validateRow(clientsAdapter, { fullName: 'Jane Doe' })).toEqual([ + { field: 'email', message: 'An email or phone is required' }, + ]); + expect(validateRow(clientsAdapter, { fullName: 'Jane Doe', email: 'j@d.com' })).toEqual([]); + }); + + it('berths reject a malformed mooring number', () => { + const errs = validateRow(berthsAdapter, { mooringNumber: 'not-a-mooring', area: 'A' }); + expect(errs.some((e) => e.field === 'mooringNumber')).toBe(true); + }); +}); + +describe('matchKey', () => { + it('companies: case-insensitive name', () => { + expect(companiesAdapter.matchKey({ name: ' AcMe ' })).toBe('acme'); + }); + it('berths: canonicalizes mooring (D-032 → D32)', () => { + expect(berthsAdapter.matchKey({ mooringNumber: 'd-032', area: 'D' })).toBe('D32'); + }); + it('clients: email key, phone fallback', () => { + expect(clientsAdapter.matchKey({ fullName: 'X', email: 'A@B.com' })).toBe('email:a@b.com'); + const k = clientsAdapter.matchKey({ fullName: 'X', phone: '+1 574 274 0548' }); + expect(k?.startsWith('phone:')).toBe(true); + }); +}); + +describe('parseCsv', () => { + it('parses headers + rows, trims, skips blank lines', () => { + const { headers, rows } = parseCsv('Name, Email\nAcme, a@b.com\n\nBeta, b@c.com\n'); + expect(headers).toEqual(['Name', 'Email']); + expect(rows).toEqual([ + { Name: 'Acme', Email: 'a@b.com' }, + { Name: 'Beta', Email: 'b@c.com' }, + ]); + }); +});