feat(import): engine core + companies/clients/berths adapters
Second importer increment: the generic engine + the three no-FK adapters,
fully unit + integration tested.
- types: ImportAdapter contract (targetFields, matchKey, findExisting,
resolveForeignKeys, insert/update) + engine types.
- mapping: fuzzy header → target-field auto-suggest (exact / alias / edit
distance, one header per field) + applyMapping (drops empty cells).
- classify: per-field zod + cross-field extraValidate, FK resolution hook,
natural-key dedup, and the conflict-policy matrix
(skip-matches / update-matches / error-on-match) → row outcomes + summary.
- engine: CSV (papaparse) + XLSX (ExcelJS) parse into a uniform
{headers, rows} of trimmed strings.
- adapters (delegating to existing create/update services for audit +
validation): companies (name dedup, update), clients (flat email/phone →
contacts[], email-or-phone dedup, insert-only), berths (canonical mooring
dedup, numeric coercion, update).
- registry: implemented adapters in dependency order.
Tests: 11 unit (mapping/validation/matchKey/parse) + 3 integration
(dedup + all three conflict policies on a seeded DB). 14 passing.
Next increments: FK adapters (yachts/interests/tenancies/expenses),
commit runner + worker, API routes + permission, wizard UI + undo.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
100
src/lib/import/adapters/berths.ts
Normal file
100
src/lib/import/adapters/berths.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { db } from '@/lib/db';
|
||||
import { berths } from '@/lib/db/schema/berths';
|
||||
import { createBerth, updateBerth } from '@/lib/services/berths.service';
|
||||
|
||||
import type { ImportAdapter, MappedRow } from '../types';
|
||||
|
||||
/** Canonicalize a mooring to the unified `^[A-Z]+\d+$` form ("A1", "D32"):
|
||||
* uppercase letters, drop a hyphen + leading zeros on the number. */
|
||||
function canonMoo(raw: string): string {
|
||||
const m = /^([A-Za-z]+)-?0*(\d+)$/.exec(raw.trim());
|
||||
return m ? `${m[1]!.toUpperCase()}${parseInt(m[2]!, 10)}` : raw.trim().toUpperCase();
|
||||
}
|
||||
|
||||
const num = (s: string | undefined): number | undefined =>
|
||||
s === undefined || s === '' ? undefined : Number(s);
|
||||
|
||||
export const berthsAdapter: ImportAdapter = {
|
||||
key: 'berths',
|
||||
label: 'Berths',
|
||||
order: 4,
|
||||
dependsOn: [],
|
||||
targetFields: [
|
||||
{
|
||||
key: 'mooringNumber',
|
||||
label: 'Mooring number',
|
||||
required: true,
|
||||
aliases: ['mooring', 'berth', 'berthnumber'],
|
||||
zod: z.string().regex(/^[A-Za-z]+-?0*\d+$/, 'Use a form like A1, B12, E18'),
|
||||
},
|
||||
{
|
||||
key: 'area',
|
||||
label: 'Area',
|
||||
required: true,
|
||||
aliases: ['dock', 'zone'],
|
||||
zod: z.string().min(1),
|
||||
},
|
||||
{ key: 'lengthFt', label: 'Length (ft)', required: false, zod: z.coerce.number() },
|
||||
{ key: 'widthFt', label: 'Width (ft)', required: false, zod: z.coerce.number() },
|
||||
{ key: 'draftFt', label: 'Draft (ft)', required: false, zod: z.coerce.number() },
|
||||
{ key: 'price', label: 'Price', required: false, zod: z.coerce.number() },
|
||||
{
|
||||
key: 'priceCurrency',
|
||||
label: 'Currency',
|
||||
required: false,
|
||||
zod: z.string().length(3),
|
||||
},
|
||||
{
|
||||
key: 'status',
|
||||
label: 'Status',
|
||||
required: false,
|
||||
zod: z.enum(['available', 'under_offer', 'sold']),
|
||||
},
|
||||
],
|
||||
matchKey: (row) => (row.mooringNumber ? canonMoo(row.mooringNumber) : null),
|
||||
findExisting: async (portId, key) => {
|
||||
const row = await db.query.berths.findFirst({
|
||||
where: and(eq(berths.portId, portId), eq(berths.mooringNumber, key)),
|
||||
columns: { id: true },
|
||||
});
|
||||
return row ?? null;
|
||||
},
|
||||
insert: async (row, _resolved, ctx) => {
|
||||
const b = await createBerth(ctx.portId, buildBerthInput(row), ctx.meta);
|
||||
return { id: b.id };
|
||||
},
|
||||
update: async (id, row, _resolved, ctx) => {
|
||||
const full = buildBerthInput(row);
|
||||
// Update spec fields only — mooring is the match key, and status has its
|
||||
// own dedicated endpoint (not part of UpdateBerthInput).
|
||||
await updateBerth(
|
||||
id,
|
||||
ctx.portId,
|
||||
{
|
||||
area: full.area,
|
||||
lengthFt: full.lengthFt,
|
||||
widthFt: full.widthFt,
|
||||
draftFt: full.draftFt,
|
||||
price: full.price,
|
||||
priceCurrency: full.priceCurrency,
|
||||
},
|
||||
ctx.meta,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
function buildBerthInput(row: MappedRow) {
|
||||
return {
|
||||
mooringNumber: canonMoo(row.mooringNumber!),
|
||||
area: row.area!, // required field — present after validation
|
||||
lengthFt: num(row.lengthFt),
|
||||
widthFt: num(row.widthFt),
|
||||
draftFt: num(row.draftFt),
|
||||
price: num(row.price),
|
||||
priceCurrency: row.priceCurrency,
|
||||
status: (row.status as 'available' | 'under_offer' | 'sold' | undefined) ?? 'available',
|
||||
};
|
||||
}
|
||||
125
src/lib/import/adapters/clients.ts
Normal file
125
src/lib/import/adapters/clients.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { db } from '@/lib/db';
|
||||
import { clients, clientContacts } from '@/lib/db/schema/clients';
|
||||
import { createClient } from '@/lib/services/clients.service';
|
||||
import { normalizeEmail, normalizePhone } from '@/lib/dedup/normalize';
|
||||
|
||||
import type { ImportAdapter, MappedRow } from '../types';
|
||||
|
||||
/** Build the contacts array createClient requires from the flat email/phone
|
||||
* columns. Phone carries its E.164 form (the contact schema requires it). */
|
||||
function buildContacts(row: MappedRow) {
|
||||
const contacts: Array<{
|
||||
channel: 'email' | 'phone';
|
||||
value: string;
|
||||
valueE164?: string;
|
||||
valueCountry?: string;
|
||||
isPrimary: boolean;
|
||||
}> = [];
|
||||
if (row.email) contacts.push({ channel: 'email', value: row.email, isPrimary: true });
|
||||
if (row.phone) {
|
||||
const ph = normalizePhone(row.phone);
|
||||
contacts.push({
|
||||
channel: 'phone',
|
||||
value: row.phone,
|
||||
valueE164: ph?.e164 ?? undefined,
|
||||
valueCountry: ph?.country ?? undefined,
|
||||
isPrimary: !row.email,
|
||||
});
|
||||
}
|
||||
return contacts;
|
||||
}
|
||||
|
||||
export const clientsAdapter: ImportAdapter = {
|
||||
key: 'clients',
|
||||
label: 'Clients',
|
||||
order: 2,
|
||||
dependsOn: [],
|
||||
targetFields: [
|
||||
{
|
||||
key: 'fullName',
|
||||
label: 'Full name',
|
||||
required: true,
|
||||
aliases: ['name', 'client', 'clientname'],
|
||||
zod: z.string().min(1),
|
||||
},
|
||||
{ key: 'email', label: 'Email', required: false, zod: z.string().email() },
|
||||
{ key: 'phone', label: 'Phone', required: false, aliases: ['mobile', 'tel'], zod: z.string() },
|
||||
{
|
||||
key: 'nationalityIso',
|
||||
label: 'Nationality (ISO-2)',
|
||||
required: false,
|
||||
aliases: ['nationality', 'country'],
|
||||
zod: z.string().length(2),
|
||||
},
|
||||
{
|
||||
key: 'preferredContactMethod',
|
||||
label: 'Preferred contact',
|
||||
required: false,
|
||||
zod: z.enum(['email', 'phone', 'whatsapp']),
|
||||
},
|
||||
{
|
||||
key: 'source',
|
||||
label: 'Source',
|
||||
required: false,
|
||||
zod: z.enum(['website', 'manual', 'referral', 'broker', 'other']),
|
||||
},
|
||||
{ key: 'sourceDetails', label: 'Source details', required: false, zod: z.string() },
|
||||
],
|
||||
extraValidate: (row) =>
|
||||
row.email || row.phone ? [] : [{ field: 'email', message: 'An email or phone is required' }],
|
||||
matchKey: (row) => {
|
||||
const email = normalizeEmail(row.email ?? null);
|
||||
if (email) return `email:${email}`;
|
||||
const ph = normalizePhone(row.phone ?? null);
|
||||
return ph?.e164 ? `phone:${ph.e164}` : null;
|
||||
},
|
||||
findExisting: async (portId, key) => {
|
||||
const idx = key.indexOf(':');
|
||||
const channel = key.slice(0, idx);
|
||||
const value = key.slice(idx + 1);
|
||||
const base = db
|
||||
.select({ id: clients.id })
|
||||
.from(clientContacts)
|
||||
.innerJoin(clients, eq(clients.id, clientContacts.clientId));
|
||||
const rows =
|
||||
channel === 'email'
|
||||
? await base
|
||||
.where(
|
||||
and(
|
||||
eq(clients.portId, portId),
|
||||
eq(clientContacts.channel, 'email'),
|
||||
sql`lower(${clientContacts.value}) = ${value}`,
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
: await base
|
||||
.where(and(eq(clients.portId, portId), eq(clientContacts.valueE164, value)))
|
||||
.limit(1);
|
||||
return rows[0] ?? null;
|
||||
},
|
||||
insert: async (row, _resolved, ctx) => {
|
||||
const c = await createClient(
|
||||
ctx.portId,
|
||||
{
|
||||
fullName: row.fullName!,
|
||||
contacts: buildContacts(row),
|
||||
nationalityIso: row.nationalityIso,
|
||||
preferredContactMethod: row.preferredContactMethod as
|
||||
| 'email'
|
||||
| 'phone'
|
||||
| 'whatsapp'
|
||||
| undefined,
|
||||
source: row.source as 'website' | 'manual' | 'referral' | 'broker' | 'other' | undefined,
|
||||
sourceDetails: row.sourceDetails,
|
||||
tagIds: [],
|
||||
},
|
||||
ctx.meta,
|
||||
);
|
||||
return { id: c.id };
|
||||
},
|
||||
// Insert-only under the importer (v1): a client's contacts/addresses are a
|
||||
// graph the flat update can't safely overwrite. update-matches → skip.
|
||||
};
|
||||
87
src/lib/import/adapters/companies.ts
Normal file
87
src/lib/import/adapters/companies.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { db } from '@/lib/db';
|
||||
import { companies } from '@/lib/db/schema/companies';
|
||||
import { createCompany, updateCompany } from '@/lib/services/companies.service';
|
||||
|
||||
import type { ImportAdapter } from '../types';
|
||||
|
||||
export const companiesAdapter: ImportAdapter = {
|
||||
key: 'companies',
|
||||
label: 'Companies',
|
||||
order: 1,
|
||||
dependsOn: [],
|
||||
targetFields: [
|
||||
{
|
||||
key: 'name',
|
||||
label: 'Name',
|
||||
required: true,
|
||||
aliases: ['company', 'companyname'],
|
||||
zod: z.string().min(1),
|
||||
},
|
||||
{ key: 'legalName', label: 'Legal name', required: false, zod: z.string() },
|
||||
{
|
||||
key: 'taxId',
|
||||
label: 'Tax ID',
|
||||
required: false,
|
||||
aliases: ['vat', 'vatnumber'],
|
||||
zod: z.string(),
|
||||
},
|
||||
{ key: 'registrationNumber', label: 'Registration number', required: false, zod: z.string() },
|
||||
{
|
||||
key: 'billingEmail',
|
||||
label: 'Billing email',
|
||||
required: false,
|
||||
aliases: ['email'],
|
||||
zod: z.string().email(),
|
||||
},
|
||||
{
|
||||
key: 'status',
|
||||
label: 'Status',
|
||||
required: false,
|
||||
zod: z.enum(['active', 'dissolved']),
|
||||
},
|
||||
{ key: 'notes', label: 'Notes', required: false, zod: z.string() },
|
||||
],
|
||||
matchKey: (row) => (row.name ? row.name.trim().toLowerCase() : null),
|
||||
findExisting: async (portId, key) => {
|
||||
const row = await db.query.companies.findFirst({
|
||||
where: and(eq(companies.portId, portId), sql`lower(${companies.name}) = ${key}`),
|
||||
columns: { id: true },
|
||||
});
|
||||
return row ?? null;
|
||||
},
|
||||
insert: async (row, _resolved, ctx) => {
|
||||
const c = await createCompany(
|
||||
ctx.portId,
|
||||
{
|
||||
name: row.name!,
|
||||
legalName: row.legalName,
|
||||
taxId: row.taxId,
|
||||
registrationNumber: row.registrationNumber,
|
||||
billingEmail: row.billingEmail,
|
||||
status: (row.status as 'active' | 'dissolved' | undefined) ?? 'active',
|
||||
notes: row.notes,
|
||||
tagIds: [],
|
||||
},
|
||||
ctx.meta,
|
||||
);
|
||||
return { id: c.id };
|
||||
},
|
||||
update: async (id, row, _resolved, ctx) => {
|
||||
await updateCompany(
|
||||
id,
|
||||
ctx.portId,
|
||||
{
|
||||
legalName: row.legalName,
|
||||
taxId: row.taxId,
|
||||
registrationNumber: row.registrationNumber,
|
||||
billingEmail: row.billingEmail,
|
||||
status: row.status as 'active' | 'dissolved' | undefined,
|
||||
notes: row.notes,
|
||||
},
|
||||
ctx.meta,
|
||||
);
|
||||
},
|
||||
};
|
||||
108
src/lib/import/classify.ts
Normal file
108
src/lib/import/classify.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Row classification — the heart of both the dry-run preview and the commit.
|
||||
*
|
||||
* For one mapped row: validate (required + per-field zod) → resolve FKs →
|
||||
* dedup by natural key → resolve to an outcome under the chosen conflict
|
||||
* policy. Pure with respect to writes; the commit step acts on the outcome.
|
||||
*/
|
||||
import { applyMapping } from './mapping';
|
||||
import type {
|
||||
ClassifiedRow,
|
||||
ConflictPolicy,
|
||||
ImportAdapter,
|
||||
ImportCtx,
|
||||
MappedRow,
|
||||
RawRow,
|
||||
RowError,
|
||||
} from './types';
|
||||
|
||||
/** Required-field + per-field zod validation. Never throws; returns all errors. */
|
||||
export function validateRow(adapter: ImportAdapter, mapped: MappedRow): RowError[] {
|
||||
const errors: RowError[] = [];
|
||||
for (const field of adapter.targetFields) {
|
||||
const value = mapped[field.key];
|
||||
if (value === undefined) {
|
||||
if (field.required) errors.push({ field: field.key, message: `${field.label} is required` });
|
||||
continue;
|
||||
}
|
||||
const res = field.zod.safeParse(value);
|
||||
if (!res.success) {
|
||||
const first = res.error.issues[0];
|
||||
errors.push({ field: field.key, message: first?.message ?? 'Invalid value' });
|
||||
}
|
||||
}
|
||||
if (errors.length === 0 && adapter.extraValidate) errors.push(...adapter.extraValidate(mapped));
|
||||
return errors;
|
||||
}
|
||||
|
||||
export async function classifyRow(
|
||||
adapter: ImportAdapter,
|
||||
raw: RawRow,
|
||||
mapping: Record<string, string>,
|
||||
rowNumber: number,
|
||||
policy: ConflictPolicy,
|
||||
ctx: ImportCtx,
|
||||
): Promise<ClassifiedRow> {
|
||||
const mapped = applyMapping(raw, mapping);
|
||||
|
||||
const errors = validateRow(adapter, mapped);
|
||||
if (errors.length > 0) return { rowNumber, outcome: 'error', errors };
|
||||
|
||||
// Foreign keys (by natural key) — resolution failures are row errors.
|
||||
let resolved: Record<string, string> = {};
|
||||
if (adapter.resolveForeignKeys) {
|
||||
const fk = await adapter.resolveForeignKeys(mapped, ctx);
|
||||
if (!fk.ok) return { rowNumber, outcome: 'error', errors: fk.errors };
|
||||
resolved = fk.resolved;
|
||||
}
|
||||
|
||||
// Dedup by natural key.
|
||||
const key = adapter.matchKey(mapped);
|
||||
const existing = key ? await adapter.findExisting(ctx.portId, key) : null;
|
||||
|
||||
if (existing) {
|
||||
if (policy === 'error-on-match') {
|
||||
return {
|
||||
rowNumber,
|
||||
outcome: 'error',
|
||||
existingId: existing.id,
|
||||
errors: [{ field: '*', message: 'Matches an existing record' }],
|
||||
};
|
||||
}
|
||||
if (policy === 'update-matches' && adapter.update) {
|
||||
return { rowNumber, outcome: 'update', existingId: existing.id, mapped, resolved };
|
||||
}
|
||||
// skip-matches, or update requested on an insert-only adapter.
|
||||
return { rowNumber, outcome: 'skip', existingId: existing.id, mapped, resolved };
|
||||
}
|
||||
|
||||
return { rowNumber, outcome: 'insert', mapped, resolved };
|
||||
}
|
||||
|
||||
export interface DryRunSummary {
|
||||
total: number;
|
||||
insert: number;
|
||||
update: number;
|
||||
skip: number;
|
||||
error: number;
|
||||
rows: ClassifiedRow[];
|
||||
}
|
||||
|
||||
/** Classify a whole file. Sequential to keep FK lookups + dedup ordering
|
||||
* deterministic (migration files are small). */
|
||||
export async function classifyRows(
|
||||
adapter: ImportAdapter,
|
||||
rawRows: RawRow[],
|
||||
mapping: Record<string, string>,
|
||||
policy: ConflictPolicy,
|
||||
ctx: ImportCtx,
|
||||
): Promise<DryRunSummary> {
|
||||
const rows: ClassifiedRow[] = [];
|
||||
const summary = { total: rawRows.length, insert: 0, update: 0, skip: 0, error: 0 };
|
||||
for (let i = 0; i < rawRows.length; i++) {
|
||||
const c = await classifyRow(adapter, rawRows[i]!, mapping, i + 1, policy, ctx);
|
||||
rows.push(c);
|
||||
summary[c.outcome] += 1;
|
||||
}
|
||||
return { ...summary, rows };
|
||||
}
|
||||
77
src/lib/import/engine.ts
Normal file
77
src/lib/import/engine.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* File parsing for the importer — CSV (papaparse) and XLSX (ExcelJS) into a
|
||||
* uniform `{ headers, rows }` shape where every cell is a trimmed string.
|
||||
*/
|
||||
import ExcelJS from 'exceljs';
|
||||
import Papa from 'papaparse';
|
||||
|
||||
import type { RawRow } from './types';
|
||||
|
||||
export interface ParsedFile {
|
||||
headers: string[];
|
||||
rows: RawRow[];
|
||||
}
|
||||
|
||||
function cellToString(v: unknown): string {
|
||||
if (v === null || v === undefined) return '';
|
||||
if (typeof v === 'object') {
|
||||
// ExcelJS rich-text / hyperlink / formula cells.
|
||||
const o = v as { text?: unknown; result?: unknown; hyperlink?: unknown };
|
||||
if (typeof o.text === 'string') return o.text;
|
||||
if (o.result !== undefined) return String(o.result);
|
||||
if (typeof o.hyperlink === 'string') return o.hyperlink;
|
||||
}
|
||||
return String(v);
|
||||
}
|
||||
|
||||
export function parseCsv(content: string): ParsedFile {
|
||||
const res = Papa.parse<Record<string, unknown>>(content, {
|
||||
header: true,
|
||||
skipEmptyLines: 'greedy',
|
||||
transformHeader: (h) => h.trim(),
|
||||
});
|
||||
const headers = (res.meta.fields ?? []).map((h) => h.trim()).filter(Boolean);
|
||||
const rows: RawRow[] = res.data.map((r) => {
|
||||
const out: RawRow = {};
|
||||
for (const h of headers) out[h] = cellToString(r[h]).trim();
|
||||
return out;
|
||||
});
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
export async function parseXlsx(buffer: Buffer): Promise<ParsedFile> {
|
||||
const wb = new ExcelJS.Workbook();
|
||||
await wb.xlsx.load(buffer as unknown as ArrayBuffer);
|
||||
const ws = wb.worksheets[0];
|
||||
if (!ws) return { headers: [], rows: [] };
|
||||
|
||||
const headerRow = ws.getRow(1);
|
||||
const headers: string[] = [];
|
||||
const colIndex: number[] = [];
|
||||
headerRow.eachCell((cell, col) => {
|
||||
const h = cellToString(cell.value).trim();
|
||||
if (h) {
|
||||
headers.push(h);
|
||||
colIndex.push(col);
|
||||
}
|
||||
});
|
||||
|
||||
const rows: RawRow[] = [];
|
||||
for (let r = 2; r <= ws.rowCount; r++) {
|
||||
const row = ws.getRow(r);
|
||||
const out: RawRow = {};
|
||||
let nonEmpty = false;
|
||||
headers.forEach((h, i) => {
|
||||
const v = cellToString(row.getCell(colIndex[i]!).value).trim();
|
||||
out[h] = v;
|
||||
if (v) nonEmpty = true;
|
||||
});
|
||||
if (nonEmpty) rows.push(out);
|
||||
}
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
export async function parseImportFile(filename: string, buffer: Buffer): Promise<ParsedFile> {
|
||||
if (/\.xlsx?$/i.test(filename)) return parseXlsx(buffer);
|
||||
return parseCsv(buffer.toString('utf8'));
|
||||
}
|
||||
83
src/lib/import/mapping.ts
Normal file
83
src/lib/import/mapping.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Column mapping: auto-suggest target-field → source-header pairings by fuzzy
|
||||
* header match, and apply a chosen mapping to a raw row.
|
||||
*/
|
||||
import type { ImportField, MappedRow, RawRow } from './types';
|
||||
|
||||
/** lowercase, strip everything but a-z0-9 → comparable token. */
|
||||
function norm(s: string): string {
|
||||
return s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
function lev(a: string, b: string): number {
|
||||
const m = a.length;
|
||||
const n = b.length;
|
||||
if (!m) return n;
|
||||
if (!n) return m;
|
||||
let prev = Array.from({ length: n + 1 }, (_, i) => i);
|
||||
for (let i = 1; i <= m; i++) {
|
||||
const cur = [i];
|
||||
for (let j = 1; j <= n; j++) {
|
||||
cur[j] = Math.min(
|
||||
prev[j]! + 1,
|
||||
cur[j - 1]! + 1,
|
||||
prev[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
|
||||
);
|
||||
}
|
||||
prev = cur;
|
||||
}
|
||||
return prev[n]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* For each field, pick the best-matching header. Exact normalized match on
|
||||
* key / label / alias wins; otherwise a substring or close edit-distance
|
||||
* match. A header is claimed by at most one field (first-come by field order).
|
||||
* Returns `fieldKey → header` (only confident matches; unmatched fields absent).
|
||||
*/
|
||||
export function suggestMapping(headers: string[], fields: ImportField[]): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const taken = new Set<string>();
|
||||
const normHeaders = headers.map((h) => ({ raw: h, n: norm(h) }));
|
||||
|
||||
for (const field of fields) {
|
||||
const candidates = [field.key, field.label, ...(field.aliases ?? [])].map(norm);
|
||||
let best: { header: string; score: number } | null = null;
|
||||
|
||||
for (const h of normHeaders) {
|
||||
if (taken.has(h.raw) || !h.n) continue;
|
||||
let score = Infinity;
|
||||
for (const c of candidates) {
|
||||
if (!c) continue;
|
||||
if (c === h.n) score = Math.min(score, 0);
|
||||
else if (c.includes(h.n) || h.n.includes(c)) score = Math.min(score, 1);
|
||||
else {
|
||||
const d = lev(c, h.n);
|
||||
// Accept only close matches (≤2 edits, and not longer than the token).
|
||||
if (d <= 2 && d < Math.max(c.length, h.n.length)) score = Math.min(score, 1 + d);
|
||||
}
|
||||
}
|
||||
if (score < Infinity && (!best || score < best.score)) best = { header: h.raw, score };
|
||||
}
|
||||
|
||||
if (best) {
|
||||
out[field.key] = best.header;
|
||||
taken.add(best.header);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a `fieldKey → header` mapping to a raw row, producing `fieldKey → cell`.
|
||||
* Empty / whitespace-only cells are dropped so downstream "required" checks and
|
||||
* optional-field omission behave correctly.
|
||||
*/
|
||||
export function applyMapping(raw: RawRow, mapping: Record<string, string>): MappedRow {
|
||||
const out: MappedRow = {};
|
||||
for (const [fieldKey, header] of Object.entries(mapping)) {
|
||||
const cell = (raw[header] ?? '').trim();
|
||||
if (cell !== '') out[fieldKey] = cell;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
28
src/lib/import/registry.ts
Normal file
28
src/lib/import/registry.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Import adapter registry. Adding an entity = add an adapter file + register
|
||||
* it here; the engine and UI are driven entirely off this map.
|
||||
*
|
||||
* Implemented so far: companies, clients, berths (no-FK). The FK entities
|
||||
* (yachts, interests, tenancies, expenses) layer in as their adapters land.
|
||||
*/
|
||||
import { berthsAdapter } from './adapters/berths';
|
||||
import { clientsAdapter } from './adapters/clients';
|
||||
import { companiesAdapter } from './adapters/companies';
|
||||
import type { ImportAdapter, ImportEntityKey } from './types';
|
||||
|
||||
export const IMPORT_REGISTRY: Partial<Record<ImportEntityKey, ImportAdapter>> = {
|
||||
companies: companiesAdapter,
|
||||
clients: clientsAdapter,
|
||||
berths: berthsAdapter,
|
||||
};
|
||||
|
||||
export function getAdapter(key: string): ImportAdapter | null {
|
||||
return IMPORT_REGISTRY[key as ImportEntityKey] ?? null;
|
||||
}
|
||||
|
||||
/** Implemented adapters in dependency order (companies → … → expenses). */
|
||||
export function listAdapters(): ImportAdapter[] {
|
||||
return Object.values(IMPORT_REGISTRY)
|
||||
.filter((a): a is ImportAdapter => Boolean(a))
|
||||
.sort((a, b) => a.order - b.order);
|
||||
}
|
||||
103
src/lib/import/types.ts
Normal file
103
src/lib/import/types.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Bulk-importer engine types.
|
||||
*
|
||||
* One generic pipeline parameterised by a per-entity {@link ImportAdapter},
|
||||
* mirroring the custom-report registry pattern. The engine (parse → map →
|
||||
* classify → commit) never knows about a specific entity; everything
|
||||
* entity-specific lives in an adapter.
|
||||
*
|
||||
* See docs/superpowers/specs/2026-06-01-bulk-import-design.md.
|
||||
*/
|
||||
import type { z } from 'zod';
|
||||
|
||||
import type { AuditMeta } from '@/lib/audit';
|
||||
|
||||
export const IMPORT_ENTITY_KEYS = [
|
||||
'companies',
|
||||
'clients',
|
||||
'yachts',
|
||||
'berths',
|
||||
'interests',
|
||||
'tenancies',
|
||||
'expenses',
|
||||
] as const;
|
||||
|
||||
export type ImportEntityKey = (typeof IMPORT_ENTITY_KEYS)[number];
|
||||
|
||||
export const CONFLICT_POLICIES = ['skip-matches', 'update-matches', 'error-on-match'] as const;
|
||||
export type ConflictPolicy = (typeof CONFLICT_POLICIES)[number];
|
||||
|
||||
/** Per-target-field definition. Drives the column-mapping UI + row validation. */
|
||||
export interface ImportField {
|
||||
key: string;
|
||||
label: string;
|
||||
required: boolean;
|
||||
/** Extra header spellings for fuzzy auto-mapping (besides key + label). */
|
||||
aliases?: string[];
|
||||
/** Per-cell validation. Empty optional cells are dropped before this runs. */
|
||||
zod: z.ZodTypeAny;
|
||||
}
|
||||
|
||||
/** A raw parsed row: source header → cell value (strings). */
|
||||
export type RawRow = Record<string, string>;
|
||||
/** A mapped row: target field key → cell value (strings), empties removed. */
|
||||
export type MappedRow = Record<string, string>;
|
||||
|
||||
export interface ImportCtx {
|
||||
portId: string;
|
||||
meta: AuditMeta;
|
||||
}
|
||||
|
||||
export interface RowError {
|
||||
field: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type FkResult =
|
||||
| { ok: true; resolved: Record<string, string> }
|
||||
| { ok: false; errors: RowError[] };
|
||||
|
||||
/** The four dry-run outcomes for a single source row. */
|
||||
export type RowOutcome = 'insert' | 'update' | 'skip' | 'error';
|
||||
|
||||
export interface ClassifiedRow {
|
||||
rowNumber: number;
|
||||
outcome: RowOutcome;
|
||||
/** Set when outcome=update/skip (the matched existing entity). */
|
||||
existingId?: string;
|
||||
errors?: RowError[];
|
||||
/** Validated + FK-resolved payload, carried to the commit step. */
|
||||
mapped?: MappedRow;
|
||||
resolved?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ImportAdapter {
|
||||
key: ImportEntityKey;
|
||||
label: string;
|
||||
/** Dependency order — companies(1) … expenses(7). */
|
||||
order: number;
|
||||
dependsOn: ImportEntityKey[];
|
||||
targetFields: ImportField[];
|
||||
/** Optional cross-field validation (e.g. "email or phone required") run
|
||||
* after per-field zod. Returns errors, never throws. */
|
||||
extraValidate?: (row: MappedRow) => RowError[];
|
||||
/** Natural dedup key derived from a validated mapped row (or null). */
|
||||
matchKey: (row: MappedRow) => string | null;
|
||||
/** Find an existing entity by its natural key, port-scoped. */
|
||||
findExisting: (portId: string, matchKey: string) => Promise<{ id: string } | null>;
|
||||
/** Resolve FK ids by natural key. Omit for entities with no FKs. */
|
||||
resolveForeignKeys?: (row: MappedRow, ctx: ImportCtx) => Promise<FkResult>;
|
||||
/** Delegates to the entity's own create service (audit + validation free). */
|
||||
insert: (
|
||||
row: MappedRow,
|
||||
resolved: Record<string, string>,
|
||||
ctx: ImportCtx,
|
||||
) => Promise<{ id: string }>;
|
||||
/** Omit when the entity is insert-only under the importer. */
|
||||
update?: (
|
||||
existingId: string,
|
||||
row: MappedRow,
|
||||
resolved: Record<string, string>,
|
||||
ctx: ImportCtx,
|
||||
) => Promise<void>;
|
||||
}
|
||||
67
tests/integration/import-classify.test.ts
Normal file
67
tests/integration/import-classify.test.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Integration test: dry-run classifier dedup + conflict-policy matrix.
|
||||
*
|
||||
* Seeds an existing company, then classifies a 3-row "file" (one matching the
|
||||
* existing record, one new, one invalid) under each conflict policy and asserts
|
||||
* the per-row outcomes + summary counts. Runs against the real test DB.
|
||||
*/
|
||||
import { beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { classifyRows } from '@/lib/import/classify';
|
||||
import { companiesAdapter } from '@/lib/import/adapters/companies';
|
||||
import type { RawRow } from '@/lib/import/types';
|
||||
|
||||
let makePort: typeof import('../helpers/factories').makePort;
|
||||
let makeCompany: typeof import('../helpers/factories').makeCompany;
|
||||
let makeAuditMeta: typeof import('../helpers/factories').makeAuditMeta;
|
||||
|
||||
beforeAll(async () => {
|
||||
const f = await import('../helpers/factories');
|
||||
makePort = f.makePort;
|
||||
makeCompany = f.makeCompany;
|
||||
makeAuditMeta = f.makeAuditMeta;
|
||||
});
|
||||
|
||||
const MAPPING = { name: 'Name' };
|
||||
|
||||
describe('classifyRows — dedup + conflict policy', () => {
|
||||
async function setup() {
|
||||
const port = await makePort();
|
||||
await makeCompany({ portId: port.id, overrides: { name: 'Acme Marine' } });
|
||||
const ctx = { portId: port.id, meta: makeAuditMeta({ portId: port.id }) };
|
||||
const rows: RawRow[] = [
|
||||
{ Name: 'Acme Marine' }, // matches existing (case-insensitive)
|
||||
{ Name: 'Brand New Co' }, // insert
|
||||
{ Name: '' }, // error: required
|
||||
];
|
||||
return { ctx, rows };
|
||||
}
|
||||
|
||||
it('skip-matches: existing→skip, new→insert, blank→error', async () => {
|
||||
const { ctx, rows } = await setup();
|
||||
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'skip-matches', ctx);
|
||||
expect({ insert: r.insert, update: r.update, skip: r.skip, error: r.error }).toEqual({
|
||||
insert: 1,
|
||||
update: 0,
|
||||
skip: 1,
|
||||
error: 1,
|
||||
});
|
||||
expect(r.rows.map((x) => x.outcome)).toEqual(['skip', 'insert', 'error']);
|
||||
expect(r.rows[0]!.existingId).toBeTruthy();
|
||||
});
|
||||
|
||||
it('update-matches: existing→update (adapter supports update)', async () => {
|
||||
const { ctx, rows } = await setup();
|
||||
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'update-matches', ctx);
|
||||
expect(r.rows[0]!.outcome).toBe('update');
|
||||
expect(r.update).toBe(1);
|
||||
});
|
||||
|
||||
it('error-on-match: existing→error', async () => {
|
||||
const { ctx, rows } = await setup();
|
||||
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'error-on-match', ctx);
|
||||
expect(r.rows[0]!.outcome).toBe('error');
|
||||
// The blank row is still an error too → 2 errors total.
|
||||
expect(r.error).toBe(2);
|
||||
});
|
||||
});
|
||||
91
tests/unit/import/engine.test.ts
Normal file
91
tests/unit/import/engine.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Unit tests for the importer engine internals (no DB): fuzzy column mapping,
|
||||
* row validation (per-field + cross-field), natural-key derivation, and CSV
|
||||
* parsing.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { applyMapping, suggestMapping } from '@/lib/import/mapping';
|
||||
import { validateRow } from '@/lib/import/classify';
|
||||
import { parseCsv } from '@/lib/import/engine';
|
||||
import { companiesAdapter } from '@/lib/import/adapters/companies';
|
||||
import { clientsAdapter } from '@/lib/import/adapters/clients';
|
||||
import { berthsAdapter } from '@/lib/import/adapters/berths';
|
||||
|
||||
describe('suggestMapping', () => {
|
||||
it('maps headers to fields by exact, alias, and fuzzy match', () => {
|
||||
const m = suggestMapping(
|
||||
['Company Name', 'VAT', 'Billing Email', 'Unrelated'],
|
||||
companiesAdapter.targetFields,
|
||||
);
|
||||
expect(m.name).toBe('Company Name'); // alias "companyname"
|
||||
expect(m.taxId).toBe('VAT'); // alias "vat"
|
||||
expect(m.billingEmail).toBe('Billing Email');
|
||||
// A header claimed by one field isn't reused by another.
|
||||
expect(Object.values(m).filter((h) => h === 'Company Name')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('leaves unmatched fields out of the mapping', () => {
|
||||
const m = suggestMapping(['xyz', 'qqq'], companiesAdapter.targetFields);
|
||||
expect(m.name).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyMapping', () => {
|
||||
it('produces fieldKey→cell and drops empty cells', () => {
|
||||
const mapped = applyMapping(
|
||||
{ 'Company Name': ' Acme ', VAT: '', Email: 'a@b.com' },
|
||||
{ name: 'Company Name', taxId: 'VAT', billingEmail: 'Email' },
|
||||
);
|
||||
expect(mapped).toEqual({ name: 'Acme', billingEmail: 'a@b.com' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateRow', () => {
|
||||
it('flags a missing required field', () => {
|
||||
const errs = validateRow(companiesAdapter, { legalName: 'X Ltd' });
|
||||
expect(errs).toEqual([{ field: 'name', message: 'Name is required' }]);
|
||||
});
|
||||
|
||||
it('flags an invalid email', () => {
|
||||
const errs = validateRow(companiesAdapter, { name: 'Acme', billingEmail: 'not-an-email' });
|
||||
expect(errs.some((e) => e.field === 'billingEmail')).toBe(true);
|
||||
});
|
||||
|
||||
it('clients require an email or a phone (cross-field)', () => {
|
||||
expect(validateRow(clientsAdapter, { fullName: 'Jane Doe' })).toEqual([
|
||||
{ field: 'email', message: 'An email or phone is required' },
|
||||
]);
|
||||
expect(validateRow(clientsAdapter, { fullName: 'Jane Doe', email: 'j@d.com' })).toEqual([]);
|
||||
});
|
||||
|
||||
it('berths reject a malformed mooring number', () => {
|
||||
const errs = validateRow(berthsAdapter, { mooringNumber: 'not-a-mooring', area: 'A' });
|
||||
expect(errs.some((e) => e.field === 'mooringNumber')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('matchKey', () => {
|
||||
it('companies: case-insensitive name', () => {
|
||||
expect(companiesAdapter.matchKey({ name: ' AcMe ' })).toBe('acme');
|
||||
});
|
||||
it('berths: canonicalizes mooring (D-032 → D32)', () => {
|
||||
expect(berthsAdapter.matchKey({ mooringNumber: 'd-032', area: 'D' })).toBe('D32');
|
||||
});
|
||||
it('clients: email key, phone fallback', () => {
|
||||
expect(clientsAdapter.matchKey({ fullName: 'X', email: 'A@B.com' })).toBe('email:a@b.com');
|
||||
const k = clientsAdapter.matchKey({ fullName: 'X', phone: '+1 574 274 0548' });
|
||||
expect(k?.startsWith('phone:')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseCsv', () => {
|
||||
it('parses headers + rows, trims, skips blank lines', () => {
|
||||
const { headers, rows } = parseCsv('Name, Email\nAcme, a@b.com\n\nBeta, b@c.com\n');
|
||||
expect(headers).toEqual(['Name', 'Email']);
|
||||
expect(rows).toEqual([
|
||||
{ Name: 'Acme', Email: 'a@b.com' },
|
||||
{ Name: 'Beta', Email: 'b@c.com' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user