feat(import): engine core + companies/clients/berths adapters
Second importer increment: the generic engine + the three no-FK adapters,
fully unit + integration tested.
- types: ImportAdapter contract (targetFields, matchKey, findExisting,
resolveForeignKeys, insert/update) + engine types.
- mapping: fuzzy header → target-field auto-suggest (exact / alias / edit
distance, one header per field) + applyMapping (drops empty cells).
- classify: per-field zod + cross-field extraValidate, FK resolution hook,
natural-key dedup, and the conflict-policy matrix
(skip-matches / update-matches / error-on-match) → row outcomes + summary.
- engine: CSV (papaparse) + XLSX (ExcelJS) parse into a uniform
{headers, rows} of trimmed strings.
- adapters (delegating to existing create/update services for audit +
validation): companies (name dedup, update), clients (flat email/phone →
contacts[], email-or-phone dedup, insert-only), berths (canonical mooring
dedup, numeric coercion, update).
- registry: implemented adapters in dependency order.
Tests: 11 unit (mapping/validation/matchKey/parse) + 3 integration
(dedup + all three conflict policies on a seeded DB). 14 passing.
Next increments: FK adapters (yachts/interests/tenancies/expenses),
commit runner + worker, API routes + permission, wizard UI + undo.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
100
src/lib/import/adapters/berths.ts
Normal file
100
src/lib/import/adapters/berths.ts
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
import { and, eq } from 'drizzle-orm';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
import { db } from '@/lib/db';
|
||||||
|
import { berths } from '@/lib/db/schema/berths';
|
||||||
|
import { createBerth, updateBerth } from '@/lib/services/berths.service';
|
||||||
|
|
||||||
|
import type { ImportAdapter, MappedRow } from '../types';
|
||||||
|
|
||||||
|
/** Canonicalize a mooring to the unified `^[A-Z]+\d+$` form ("A1", "D32"):
|
||||||
|
* uppercase letters, drop a hyphen + leading zeros on the number. */
|
||||||
|
function canonMoo(raw: string): string {
|
||||||
|
const m = /^([A-Za-z]+)-?0*(\d+)$/.exec(raw.trim());
|
||||||
|
return m ? `${m[1]!.toUpperCase()}${parseInt(m[2]!, 10)}` : raw.trim().toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
const num = (s: string | undefined): number | undefined =>
|
||||||
|
s === undefined || s === '' ? undefined : Number(s);
|
||||||
|
|
||||||
|
export const berthsAdapter: ImportAdapter = {
|
||||||
|
key: 'berths',
|
||||||
|
label: 'Berths',
|
||||||
|
order: 4,
|
||||||
|
dependsOn: [],
|
||||||
|
targetFields: [
|
||||||
|
{
|
||||||
|
key: 'mooringNumber',
|
||||||
|
label: 'Mooring number',
|
||||||
|
required: true,
|
||||||
|
aliases: ['mooring', 'berth', 'berthnumber'],
|
||||||
|
zod: z.string().regex(/^[A-Za-z]+-?0*\d+$/, 'Use a form like A1, B12, E18'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'area',
|
||||||
|
label: 'Area',
|
||||||
|
required: true,
|
||||||
|
aliases: ['dock', 'zone'],
|
||||||
|
zod: z.string().min(1),
|
||||||
|
},
|
||||||
|
{ key: 'lengthFt', label: 'Length (ft)', required: false, zod: z.coerce.number() },
|
||||||
|
{ key: 'widthFt', label: 'Width (ft)', required: false, zod: z.coerce.number() },
|
||||||
|
{ key: 'draftFt', label: 'Draft (ft)', required: false, zod: z.coerce.number() },
|
||||||
|
{ key: 'price', label: 'Price', required: false, zod: z.coerce.number() },
|
||||||
|
{
|
||||||
|
key: 'priceCurrency',
|
||||||
|
label: 'Currency',
|
||||||
|
required: false,
|
||||||
|
zod: z.string().length(3),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'status',
|
||||||
|
label: 'Status',
|
||||||
|
required: false,
|
||||||
|
zod: z.enum(['available', 'under_offer', 'sold']),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
matchKey: (row) => (row.mooringNumber ? canonMoo(row.mooringNumber) : null),
|
||||||
|
findExisting: async (portId, key) => {
|
||||||
|
const row = await db.query.berths.findFirst({
|
||||||
|
where: and(eq(berths.portId, portId), eq(berths.mooringNumber, key)),
|
||||||
|
columns: { id: true },
|
||||||
|
});
|
||||||
|
return row ?? null;
|
||||||
|
},
|
||||||
|
insert: async (row, _resolved, ctx) => {
|
||||||
|
const b = await createBerth(ctx.portId, buildBerthInput(row), ctx.meta);
|
||||||
|
return { id: b.id };
|
||||||
|
},
|
||||||
|
update: async (id, row, _resolved, ctx) => {
|
||||||
|
const full = buildBerthInput(row);
|
||||||
|
// Update spec fields only — mooring is the match key, and status has its
|
||||||
|
// own dedicated endpoint (not part of UpdateBerthInput).
|
||||||
|
await updateBerth(
|
||||||
|
id,
|
||||||
|
ctx.portId,
|
||||||
|
{
|
||||||
|
area: full.area,
|
||||||
|
lengthFt: full.lengthFt,
|
||||||
|
widthFt: full.widthFt,
|
||||||
|
draftFt: full.draftFt,
|
||||||
|
price: full.price,
|
||||||
|
priceCurrency: full.priceCurrency,
|
||||||
|
},
|
||||||
|
ctx.meta,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function buildBerthInput(row: MappedRow) {
|
||||||
|
return {
|
||||||
|
mooringNumber: canonMoo(row.mooringNumber!),
|
||||||
|
area: row.area!, // required field — present after validation
|
||||||
|
lengthFt: num(row.lengthFt),
|
||||||
|
widthFt: num(row.widthFt),
|
||||||
|
draftFt: num(row.draftFt),
|
||||||
|
price: num(row.price),
|
||||||
|
priceCurrency: row.priceCurrency,
|
||||||
|
status: (row.status as 'available' | 'under_offer' | 'sold' | undefined) ?? 'available',
|
||||||
|
};
|
||||||
|
}
|
||||||
125
src/lib/import/adapters/clients.ts
Normal file
125
src/lib/import/adapters/clients.ts
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
import { and, eq, sql } from 'drizzle-orm';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
import { db } from '@/lib/db';
|
||||||
|
import { clients, clientContacts } from '@/lib/db/schema/clients';
|
||||||
|
import { createClient } from '@/lib/services/clients.service';
|
||||||
|
import { normalizeEmail, normalizePhone } from '@/lib/dedup/normalize';
|
||||||
|
|
||||||
|
import type { ImportAdapter, MappedRow } from '../types';
|
||||||
|
|
||||||
|
/** Build the contacts array createClient requires from the flat email/phone
|
||||||
|
* columns. Phone carries its E.164 form (the contact schema requires it). */
|
||||||
|
function buildContacts(row: MappedRow) {
|
||||||
|
const contacts: Array<{
|
||||||
|
channel: 'email' | 'phone';
|
||||||
|
value: string;
|
||||||
|
valueE164?: string;
|
||||||
|
valueCountry?: string;
|
||||||
|
isPrimary: boolean;
|
||||||
|
}> = [];
|
||||||
|
if (row.email) contacts.push({ channel: 'email', value: row.email, isPrimary: true });
|
||||||
|
if (row.phone) {
|
||||||
|
const ph = normalizePhone(row.phone);
|
||||||
|
contacts.push({
|
||||||
|
channel: 'phone',
|
||||||
|
value: row.phone,
|
||||||
|
valueE164: ph?.e164 ?? undefined,
|
||||||
|
valueCountry: ph?.country ?? undefined,
|
||||||
|
isPrimary: !row.email,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return contacts;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const clientsAdapter: ImportAdapter = {
|
||||||
|
key: 'clients',
|
||||||
|
label: 'Clients',
|
||||||
|
order: 2,
|
||||||
|
dependsOn: [],
|
||||||
|
targetFields: [
|
||||||
|
{
|
||||||
|
key: 'fullName',
|
||||||
|
label: 'Full name',
|
||||||
|
required: true,
|
||||||
|
aliases: ['name', 'client', 'clientname'],
|
||||||
|
zod: z.string().min(1),
|
||||||
|
},
|
||||||
|
{ key: 'email', label: 'Email', required: false, zod: z.string().email() },
|
||||||
|
{ key: 'phone', label: 'Phone', required: false, aliases: ['mobile', 'tel'], zod: z.string() },
|
||||||
|
{
|
||||||
|
key: 'nationalityIso',
|
||||||
|
label: 'Nationality (ISO-2)',
|
||||||
|
required: false,
|
||||||
|
aliases: ['nationality', 'country'],
|
||||||
|
zod: z.string().length(2),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'preferredContactMethod',
|
||||||
|
label: 'Preferred contact',
|
||||||
|
required: false,
|
||||||
|
zod: z.enum(['email', 'phone', 'whatsapp']),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'source',
|
||||||
|
label: 'Source',
|
||||||
|
required: false,
|
||||||
|
zod: z.enum(['website', 'manual', 'referral', 'broker', 'other']),
|
||||||
|
},
|
||||||
|
{ key: 'sourceDetails', label: 'Source details', required: false, zod: z.string() },
|
||||||
|
],
|
||||||
|
extraValidate: (row) =>
|
||||||
|
row.email || row.phone ? [] : [{ field: 'email', message: 'An email or phone is required' }],
|
||||||
|
matchKey: (row) => {
|
||||||
|
const email = normalizeEmail(row.email ?? null);
|
||||||
|
if (email) return `email:${email}`;
|
||||||
|
const ph = normalizePhone(row.phone ?? null);
|
||||||
|
return ph?.e164 ? `phone:${ph.e164}` : null;
|
||||||
|
},
|
||||||
|
findExisting: async (portId, key) => {
|
||||||
|
const idx = key.indexOf(':');
|
||||||
|
const channel = key.slice(0, idx);
|
||||||
|
const value = key.slice(idx + 1);
|
||||||
|
const base = db
|
||||||
|
.select({ id: clients.id })
|
||||||
|
.from(clientContacts)
|
||||||
|
.innerJoin(clients, eq(clients.id, clientContacts.clientId));
|
||||||
|
const rows =
|
||||||
|
channel === 'email'
|
||||||
|
? await base
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(clients.portId, portId),
|
||||||
|
eq(clientContacts.channel, 'email'),
|
||||||
|
sql`lower(${clientContacts.value}) = ${value}`,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
: await base
|
||||||
|
.where(and(eq(clients.portId, portId), eq(clientContacts.valueE164, value)))
|
||||||
|
.limit(1);
|
||||||
|
return rows[0] ?? null;
|
||||||
|
},
|
||||||
|
insert: async (row, _resolved, ctx) => {
|
||||||
|
const c = await createClient(
|
||||||
|
ctx.portId,
|
||||||
|
{
|
||||||
|
fullName: row.fullName!,
|
||||||
|
contacts: buildContacts(row),
|
||||||
|
nationalityIso: row.nationalityIso,
|
||||||
|
preferredContactMethod: row.preferredContactMethod as
|
||||||
|
| 'email'
|
||||||
|
| 'phone'
|
||||||
|
| 'whatsapp'
|
||||||
|
| undefined,
|
||||||
|
source: row.source as 'website' | 'manual' | 'referral' | 'broker' | 'other' | undefined,
|
||||||
|
sourceDetails: row.sourceDetails,
|
||||||
|
tagIds: [],
|
||||||
|
},
|
||||||
|
ctx.meta,
|
||||||
|
);
|
||||||
|
return { id: c.id };
|
||||||
|
},
|
||||||
|
// Insert-only under the importer (v1): a client's contacts/addresses are a
|
||||||
|
// graph the flat update can't safely overwrite. update-matches → skip.
|
||||||
|
};
|
||||||
87
src/lib/import/adapters/companies.ts
Normal file
87
src/lib/import/adapters/companies.ts
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
import { and, eq, sql } from 'drizzle-orm';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
import { db } from '@/lib/db';
|
||||||
|
import { companies } from '@/lib/db/schema/companies';
|
||||||
|
import { createCompany, updateCompany } from '@/lib/services/companies.service';
|
||||||
|
|
||||||
|
import type { ImportAdapter } from '../types';
|
||||||
|
|
||||||
|
export const companiesAdapter: ImportAdapter = {
|
||||||
|
key: 'companies',
|
||||||
|
label: 'Companies',
|
||||||
|
order: 1,
|
||||||
|
dependsOn: [],
|
||||||
|
targetFields: [
|
||||||
|
{
|
||||||
|
key: 'name',
|
||||||
|
label: 'Name',
|
||||||
|
required: true,
|
||||||
|
aliases: ['company', 'companyname'],
|
||||||
|
zod: z.string().min(1),
|
||||||
|
},
|
||||||
|
{ key: 'legalName', label: 'Legal name', required: false, zod: z.string() },
|
||||||
|
{
|
||||||
|
key: 'taxId',
|
||||||
|
label: 'Tax ID',
|
||||||
|
required: false,
|
||||||
|
aliases: ['vat', 'vatnumber'],
|
||||||
|
zod: z.string(),
|
||||||
|
},
|
||||||
|
{ key: 'registrationNumber', label: 'Registration number', required: false, zod: z.string() },
|
||||||
|
{
|
||||||
|
key: 'billingEmail',
|
||||||
|
label: 'Billing email',
|
||||||
|
required: false,
|
||||||
|
aliases: ['email'],
|
||||||
|
zod: z.string().email(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'status',
|
||||||
|
label: 'Status',
|
||||||
|
required: false,
|
||||||
|
zod: z.enum(['active', 'dissolved']),
|
||||||
|
},
|
||||||
|
{ key: 'notes', label: 'Notes', required: false, zod: z.string() },
|
||||||
|
],
|
||||||
|
matchKey: (row) => (row.name ? row.name.trim().toLowerCase() : null),
|
||||||
|
findExisting: async (portId, key) => {
|
||||||
|
const row = await db.query.companies.findFirst({
|
||||||
|
where: and(eq(companies.portId, portId), sql`lower(${companies.name}) = ${key}`),
|
||||||
|
columns: { id: true },
|
||||||
|
});
|
||||||
|
return row ?? null;
|
||||||
|
},
|
||||||
|
insert: async (row, _resolved, ctx) => {
|
||||||
|
const c = await createCompany(
|
||||||
|
ctx.portId,
|
||||||
|
{
|
||||||
|
name: row.name!,
|
||||||
|
legalName: row.legalName,
|
||||||
|
taxId: row.taxId,
|
||||||
|
registrationNumber: row.registrationNumber,
|
||||||
|
billingEmail: row.billingEmail,
|
||||||
|
status: (row.status as 'active' | 'dissolved' | undefined) ?? 'active',
|
||||||
|
notes: row.notes,
|
||||||
|
tagIds: [],
|
||||||
|
},
|
||||||
|
ctx.meta,
|
||||||
|
);
|
||||||
|
return { id: c.id };
|
||||||
|
},
|
||||||
|
update: async (id, row, _resolved, ctx) => {
|
||||||
|
await updateCompany(
|
||||||
|
id,
|
||||||
|
ctx.portId,
|
||||||
|
{
|
||||||
|
legalName: row.legalName,
|
||||||
|
taxId: row.taxId,
|
||||||
|
registrationNumber: row.registrationNumber,
|
||||||
|
billingEmail: row.billingEmail,
|
||||||
|
status: row.status as 'active' | 'dissolved' | undefined,
|
||||||
|
notes: row.notes,
|
||||||
|
},
|
||||||
|
ctx.meta,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
108
src/lib/import/classify.ts
Normal file
108
src/lib/import/classify.ts
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* Row classification — the heart of both the dry-run preview and the commit.
|
||||||
|
*
|
||||||
|
* For one mapped row: validate (required + per-field zod) → resolve FKs →
|
||||||
|
* dedup by natural key → resolve to an outcome under the chosen conflict
|
||||||
|
* policy. Pure with respect to writes; the commit step acts on the outcome.
|
||||||
|
*/
|
||||||
|
import { applyMapping } from './mapping';
|
||||||
|
import type {
|
||||||
|
ClassifiedRow,
|
||||||
|
ConflictPolicy,
|
||||||
|
ImportAdapter,
|
||||||
|
ImportCtx,
|
||||||
|
MappedRow,
|
||||||
|
RawRow,
|
||||||
|
RowError,
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
/** Required-field + per-field zod validation. Never throws; returns all errors. */
|
||||||
|
export function validateRow(adapter: ImportAdapter, mapped: MappedRow): RowError[] {
|
||||||
|
const errors: RowError[] = [];
|
||||||
|
for (const field of adapter.targetFields) {
|
||||||
|
const value = mapped[field.key];
|
||||||
|
if (value === undefined) {
|
||||||
|
if (field.required) errors.push({ field: field.key, message: `${field.label} is required` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const res = field.zod.safeParse(value);
|
||||||
|
if (!res.success) {
|
||||||
|
const first = res.error.issues[0];
|
||||||
|
errors.push({ field: field.key, message: first?.message ?? 'Invalid value' });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (errors.length === 0 && adapter.extraValidate) errors.push(...adapter.extraValidate(mapped));
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function classifyRow(
|
||||||
|
adapter: ImportAdapter,
|
||||||
|
raw: RawRow,
|
||||||
|
mapping: Record<string, string>,
|
||||||
|
rowNumber: number,
|
||||||
|
policy: ConflictPolicy,
|
||||||
|
ctx: ImportCtx,
|
||||||
|
): Promise<ClassifiedRow> {
|
||||||
|
const mapped = applyMapping(raw, mapping);
|
||||||
|
|
||||||
|
const errors = validateRow(adapter, mapped);
|
||||||
|
if (errors.length > 0) return { rowNumber, outcome: 'error', errors };
|
||||||
|
|
||||||
|
// Foreign keys (by natural key) — resolution failures are row errors.
|
||||||
|
let resolved: Record<string, string> = {};
|
||||||
|
if (adapter.resolveForeignKeys) {
|
||||||
|
const fk = await adapter.resolveForeignKeys(mapped, ctx);
|
||||||
|
if (!fk.ok) return { rowNumber, outcome: 'error', errors: fk.errors };
|
||||||
|
resolved = fk.resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dedup by natural key.
|
||||||
|
const key = adapter.matchKey(mapped);
|
||||||
|
const existing = key ? await adapter.findExisting(ctx.portId, key) : null;
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
if (policy === 'error-on-match') {
|
||||||
|
return {
|
||||||
|
rowNumber,
|
||||||
|
outcome: 'error',
|
||||||
|
existingId: existing.id,
|
||||||
|
errors: [{ field: '*', message: 'Matches an existing record' }],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (policy === 'update-matches' && adapter.update) {
|
||||||
|
return { rowNumber, outcome: 'update', existingId: existing.id, mapped, resolved };
|
||||||
|
}
|
||||||
|
// skip-matches, or update requested on an insert-only adapter.
|
||||||
|
return { rowNumber, outcome: 'skip', existingId: existing.id, mapped, resolved };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { rowNumber, outcome: 'insert', mapped, resolved };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DryRunSummary {
|
||||||
|
total: number;
|
||||||
|
insert: number;
|
||||||
|
update: number;
|
||||||
|
skip: number;
|
||||||
|
error: number;
|
||||||
|
rows: ClassifiedRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Classify a whole file. Sequential to keep FK lookups + dedup ordering
|
||||||
|
* deterministic (migration files are small). */
|
||||||
|
export async function classifyRows(
|
||||||
|
adapter: ImportAdapter,
|
||||||
|
rawRows: RawRow[],
|
||||||
|
mapping: Record<string, string>,
|
||||||
|
policy: ConflictPolicy,
|
||||||
|
ctx: ImportCtx,
|
||||||
|
): Promise<DryRunSummary> {
|
||||||
|
const rows: ClassifiedRow[] = [];
|
||||||
|
const summary = { total: rawRows.length, insert: 0, update: 0, skip: 0, error: 0 };
|
||||||
|
for (let i = 0; i < rawRows.length; i++) {
|
||||||
|
const c = await classifyRow(adapter, rawRows[i]!, mapping, i + 1, policy, ctx);
|
||||||
|
rows.push(c);
|
||||||
|
summary[c.outcome] += 1;
|
||||||
|
}
|
||||||
|
return { ...summary, rows };
|
||||||
|
}
|
||||||
77
src/lib/import/engine.ts
Normal file
77
src/lib/import/engine.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
/**
|
||||||
|
* File parsing for the importer — CSV (papaparse) and XLSX (ExcelJS) into a
|
||||||
|
* uniform `{ headers, rows }` shape where every cell is a trimmed string.
|
||||||
|
*/
|
||||||
|
import ExcelJS from 'exceljs';
|
||||||
|
import Papa from 'papaparse';
|
||||||
|
|
||||||
|
import type { RawRow } from './types';
|
||||||
|
|
||||||
|
export interface ParsedFile {
|
||||||
|
headers: string[];
|
||||||
|
rows: RawRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function cellToString(v: unknown): string {
|
||||||
|
if (v === null || v === undefined) return '';
|
||||||
|
if (typeof v === 'object') {
|
||||||
|
// ExcelJS rich-text / hyperlink / formula cells.
|
||||||
|
const o = v as { text?: unknown; result?: unknown; hyperlink?: unknown };
|
||||||
|
if (typeof o.text === 'string') return o.text;
|
||||||
|
if (o.result !== undefined) return String(o.result);
|
||||||
|
if (typeof o.hyperlink === 'string') return o.hyperlink;
|
||||||
|
}
|
||||||
|
return String(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseCsv(content: string): ParsedFile {
|
||||||
|
const res = Papa.parse<Record<string, unknown>>(content, {
|
||||||
|
header: true,
|
||||||
|
skipEmptyLines: 'greedy',
|
||||||
|
transformHeader: (h) => h.trim(),
|
||||||
|
});
|
||||||
|
const headers = (res.meta.fields ?? []).map((h) => h.trim()).filter(Boolean);
|
||||||
|
const rows: RawRow[] = res.data.map((r) => {
|
||||||
|
const out: RawRow = {};
|
||||||
|
for (const h of headers) out[h] = cellToString(r[h]).trim();
|
||||||
|
return out;
|
||||||
|
});
|
||||||
|
return { headers, rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function parseXlsx(buffer: Buffer): Promise<ParsedFile> {
|
||||||
|
const wb = new ExcelJS.Workbook();
|
||||||
|
await wb.xlsx.load(buffer as unknown as ArrayBuffer);
|
||||||
|
const ws = wb.worksheets[0];
|
||||||
|
if (!ws) return { headers: [], rows: [] };
|
||||||
|
|
||||||
|
const headerRow = ws.getRow(1);
|
||||||
|
const headers: string[] = [];
|
||||||
|
const colIndex: number[] = [];
|
||||||
|
headerRow.eachCell((cell, col) => {
|
||||||
|
const h = cellToString(cell.value).trim();
|
||||||
|
if (h) {
|
||||||
|
headers.push(h);
|
||||||
|
colIndex.push(col);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows: RawRow[] = [];
|
||||||
|
for (let r = 2; r <= ws.rowCount; r++) {
|
||||||
|
const row = ws.getRow(r);
|
||||||
|
const out: RawRow = {};
|
||||||
|
let nonEmpty = false;
|
||||||
|
headers.forEach((h, i) => {
|
||||||
|
const v = cellToString(row.getCell(colIndex[i]!).value).trim();
|
||||||
|
out[h] = v;
|
||||||
|
if (v) nonEmpty = true;
|
||||||
|
});
|
||||||
|
if (nonEmpty) rows.push(out);
|
||||||
|
}
|
||||||
|
return { headers, rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function parseImportFile(filename: string, buffer: Buffer): Promise<ParsedFile> {
|
||||||
|
if (/\.xlsx?$/i.test(filename)) return parseXlsx(buffer);
|
||||||
|
return parseCsv(buffer.toString('utf8'));
|
||||||
|
}
|
||||||
83
src/lib/import/mapping.ts
Normal file
83
src/lib/import/mapping.ts
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
/**
|
||||||
|
* Column mapping: auto-suggest target-field → source-header pairings by fuzzy
|
||||||
|
* header match, and apply a chosen mapping to a raw row.
|
||||||
|
*/
|
||||||
|
import type { ImportField, MappedRow, RawRow } from './types';
|
||||||
|
|
||||||
|
/** lowercase, strip everything but a-z0-9 → comparable token. */
|
||||||
|
function norm(s: string): string {
|
||||||
|
return s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function lev(a: string, b: string): number {
|
||||||
|
const m = a.length;
|
||||||
|
const n = b.length;
|
||||||
|
if (!m) return n;
|
||||||
|
if (!n) return m;
|
||||||
|
let prev = Array.from({ length: n + 1 }, (_, i) => i);
|
||||||
|
for (let i = 1; i <= m; i++) {
|
||||||
|
const cur = [i];
|
||||||
|
for (let j = 1; j <= n; j++) {
|
||||||
|
cur[j] = Math.min(
|
||||||
|
prev[j]! + 1,
|
||||||
|
cur[j - 1]! + 1,
|
||||||
|
prev[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
prev = cur;
|
||||||
|
}
|
||||||
|
return prev[n]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For each field, pick the best-matching header. Exact normalized match on
|
||||||
|
* key / label / alias wins; otherwise a substring or close edit-distance
|
||||||
|
* match. A header is claimed by at most one field (first-come by field order).
|
||||||
|
* Returns `fieldKey → header` (only confident matches; unmatched fields absent).
|
||||||
|
*/
|
||||||
|
export function suggestMapping(headers: string[], fields: ImportField[]): Record<string, string> {
|
||||||
|
const out: Record<string, string> = {};
|
||||||
|
const taken = new Set<string>();
|
||||||
|
const normHeaders = headers.map((h) => ({ raw: h, n: norm(h) }));
|
||||||
|
|
||||||
|
for (const field of fields) {
|
||||||
|
const candidates = [field.key, field.label, ...(field.aliases ?? [])].map(norm);
|
||||||
|
let best: { header: string; score: number } | null = null;
|
||||||
|
|
||||||
|
for (const h of normHeaders) {
|
||||||
|
if (taken.has(h.raw) || !h.n) continue;
|
||||||
|
let score = Infinity;
|
||||||
|
for (const c of candidates) {
|
||||||
|
if (!c) continue;
|
||||||
|
if (c === h.n) score = Math.min(score, 0);
|
||||||
|
else if (c.includes(h.n) || h.n.includes(c)) score = Math.min(score, 1);
|
||||||
|
else {
|
||||||
|
const d = lev(c, h.n);
|
||||||
|
// Accept only close matches (≤2 edits, and not longer than the token).
|
||||||
|
if (d <= 2 && d < Math.max(c.length, h.n.length)) score = Math.min(score, 1 + d);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (score < Infinity && (!best || score < best.score)) best = { header: h.raw, score };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (best) {
|
||||||
|
out[field.key] = best.header;
|
||||||
|
taken.add(best.header);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply a `fieldKey → header` mapping to a raw row, producing `fieldKey → cell`.
|
||||||
|
* Empty / whitespace-only cells are dropped so downstream "required" checks and
|
||||||
|
* optional-field omission behave correctly.
|
||||||
|
*/
|
||||||
|
export function applyMapping(raw: RawRow, mapping: Record<string, string>): MappedRow {
|
||||||
|
const out: MappedRow = {};
|
||||||
|
for (const [fieldKey, header] of Object.entries(mapping)) {
|
||||||
|
const cell = (raw[header] ?? '').trim();
|
||||||
|
if (cell !== '') out[fieldKey] = cell;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
28
src/lib/import/registry.ts
Normal file
28
src/lib/import/registry.ts
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
/**
|
||||||
|
* Import adapter registry. Adding an entity = add an adapter file + register
|
||||||
|
* it here; the engine and UI are driven entirely off this map.
|
||||||
|
*
|
||||||
|
* Implemented so far: companies, clients, berths (no-FK). The FK entities
|
||||||
|
* (yachts, interests, tenancies, expenses) layer in as their adapters land.
|
||||||
|
*/
|
||||||
|
import { berthsAdapter } from './adapters/berths';
|
||||||
|
import { clientsAdapter } from './adapters/clients';
|
||||||
|
import { companiesAdapter } from './adapters/companies';
|
||||||
|
import type { ImportAdapter, ImportEntityKey } from './types';
|
||||||
|
|
||||||
|
export const IMPORT_REGISTRY: Partial<Record<ImportEntityKey, ImportAdapter>> = {
|
||||||
|
companies: companiesAdapter,
|
||||||
|
clients: clientsAdapter,
|
||||||
|
berths: berthsAdapter,
|
||||||
|
};
|
||||||
|
|
||||||
|
export function getAdapter(key: string): ImportAdapter | null {
|
||||||
|
return IMPORT_REGISTRY[key as ImportEntityKey] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Implemented adapters in dependency order (companies → … → expenses). */
|
||||||
|
export function listAdapters(): ImportAdapter[] {
|
||||||
|
return Object.values(IMPORT_REGISTRY)
|
||||||
|
.filter((a): a is ImportAdapter => Boolean(a))
|
||||||
|
.sort((a, b) => a.order - b.order);
|
||||||
|
}
|
||||||
103
src/lib/import/types.ts
Normal file
103
src/lib/import/types.ts
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
/**
|
||||||
|
* Bulk-importer engine types.
|
||||||
|
*
|
||||||
|
* One generic pipeline parameterised by a per-entity {@link ImportAdapter},
|
||||||
|
* mirroring the custom-report registry pattern. The engine (parse → map →
|
||||||
|
* classify → commit) never knows about a specific entity; everything
|
||||||
|
* entity-specific lives in an adapter.
|
||||||
|
*
|
||||||
|
* See docs/superpowers/specs/2026-06-01-bulk-import-design.md.
|
||||||
|
*/
|
||||||
|
import type { z } from 'zod';
|
||||||
|
|
||||||
|
import type { AuditMeta } from '@/lib/audit';
|
||||||
|
|
||||||
|
export const IMPORT_ENTITY_KEYS = [
|
||||||
|
'companies',
|
||||||
|
'clients',
|
||||||
|
'yachts',
|
||||||
|
'berths',
|
||||||
|
'interests',
|
||||||
|
'tenancies',
|
||||||
|
'expenses',
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type ImportEntityKey = (typeof IMPORT_ENTITY_KEYS)[number];
|
||||||
|
|
||||||
|
export const CONFLICT_POLICIES = ['skip-matches', 'update-matches', 'error-on-match'] as const;
|
||||||
|
export type ConflictPolicy = (typeof CONFLICT_POLICIES)[number];
|
||||||
|
|
||||||
|
/** Per-target-field definition. Drives the column-mapping UI + row validation. */
|
||||||
|
export interface ImportField {
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
required: boolean;
|
||||||
|
/** Extra header spellings for fuzzy auto-mapping (besides key + label). */
|
||||||
|
aliases?: string[];
|
||||||
|
/** Per-cell validation. Empty optional cells are dropped before this runs. */
|
||||||
|
zod: z.ZodTypeAny;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A raw parsed row: source header → cell value (strings). */
|
||||||
|
export type RawRow = Record<string, string>;
|
||||||
|
/** A mapped row: target field key → cell value (strings), empties removed. */
|
||||||
|
export type MappedRow = Record<string, string>;
|
||||||
|
|
||||||
|
export interface ImportCtx {
|
||||||
|
portId: string;
|
||||||
|
meta: AuditMeta;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RowError {
|
||||||
|
field: string;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type FkResult =
|
||||||
|
| { ok: true; resolved: Record<string, string> }
|
||||||
|
| { ok: false; errors: RowError[] };
|
||||||
|
|
||||||
|
/** The four dry-run outcomes for a single source row. */
|
||||||
|
export type RowOutcome = 'insert' | 'update' | 'skip' | 'error';
|
||||||
|
|
||||||
|
export interface ClassifiedRow {
|
||||||
|
rowNumber: number;
|
||||||
|
outcome: RowOutcome;
|
||||||
|
/** Set when outcome=update/skip (the matched existing entity). */
|
||||||
|
existingId?: string;
|
||||||
|
errors?: RowError[];
|
||||||
|
/** Validated + FK-resolved payload, carried to the commit step. */
|
||||||
|
mapped?: MappedRow;
|
||||||
|
resolved?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportAdapter {
|
||||||
|
key: ImportEntityKey;
|
||||||
|
label: string;
|
||||||
|
/** Dependency order — companies(1) … expenses(7). */
|
||||||
|
order: number;
|
||||||
|
dependsOn: ImportEntityKey[];
|
||||||
|
targetFields: ImportField[];
|
||||||
|
/** Optional cross-field validation (e.g. "email or phone required") run
|
||||||
|
* after per-field zod. Returns errors, never throws. */
|
||||||
|
extraValidate?: (row: MappedRow) => RowError[];
|
||||||
|
/** Natural dedup key derived from a validated mapped row (or null). */
|
||||||
|
matchKey: (row: MappedRow) => string | null;
|
||||||
|
/** Find an existing entity by its natural key, port-scoped. */
|
||||||
|
findExisting: (portId: string, matchKey: string) => Promise<{ id: string } | null>;
|
||||||
|
/** Resolve FK ids by natural key. Omit for entities with no FKs. */
|
||||||
|
resolveForeignKeys?: (row: MappedRow, ctx: ImportCtx) => Promise<FkResult>;
|
||||||
|
/** Delegates to the entity's own create service (audit + validation free). */
|
||||||
|
insert: (
|
||||||
|
row: MappedRow,
|
||||||
|
resolved: Record<string, string>,
|
||||||
|
ctx: ImportCtx,
|
||||||
|
) => Promise<{ id: string }>;
|
||||||
|
/** Omit when the entity is insert-only under the importer. */
|
||||||
|
update?: (
|
||||||
|
existingId: string,
|
||||||
|
row: MappedRow,
|
||||||
|
resolved: Record<string, string>,
|
||||||
|
ctx: ImportCtx,
|
||||||
|
) => Promise<void>;
|
||||||
|
}
|
||||||
67
tests/integration/import-classify.test.ts
Normal file
67
tests/integration/import-classify.test.ts
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
/**
|
||||||
|
* Integration test: dry-run classifier dedup + conflict-policy matrix.
|
||||||
|
*
|
||||||
|
* Seeds an existing company, then classifies a 3-row "file" (one matching the
|
||||||
|
* existing record, one new, one invalid) under each conflict policy and asserts
|
||||||
|
* the per-row outcomes + summary counts. Runs against the real test DB.
|
||||||
|
*/
|
||||||
|
import { beforeAll, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { classifyRows } from '@/lib/import/classify';
|
||||||
|
import { companiesAdapter } from '@/lib/import/adapters/companies';
|
||||||
|
import type { RawRow } from '@/lib/import/types';
|
||||||
|
|
||||||
|
let makePort: typeof import('../helpers/factories').makePort;
|
||||||
|
let makeCompany: typeof import('../helpers/factories').makeCompany;
|
||||||
|
let makeAuditMeta: typeof import('../helpers/factories').makeAuditMeta;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const f = await import('../helpers/factories');
|
||||||
|
makePort = f.makePort;
|
||||||
|
makeCompany = f.makeCompany;
|
||||||
|
makeAuditMeta = f.makeAuditMeta;
|
||||||
|
});
|
||||||
|
|
||||||
|
const MAPPING = { name: 'Name' };
|
||||||
|
|
||||||
|
describe('classifyRows — dedup + conflict policy', () => {
|
||||||
|
async function setup() {
|
||||||
|
const port = await makePort();
|
||||||
|
await makeCompany({ portId: port.id, overrides: { name: 'Acme Marine' } });
|
||||||
|
const ctx = { portId: port.id, meta: makeAuditMeta({ portId: port.id }) };
|
||||||
|
const rows: RawRow[] = [
|
||||||
|
{ Name: 'Acme Marine' }, // matches existing (case-insensitive)
|
||||||
|
{ Name: 'Brand New Co' }, // insert
|
||||||
|
{ Name: '' }, // error: required
|
||||||
|
];
|
||||||
|
return { ctx, rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
it('skip-matches: existing→skip, new→insert, blank→error', async () => {
|
||||||
|
const { ctx, rows } = await setup();
|
||||||
|
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'skip-matches', ctx);
|
||||||
|
expect({ insert: r.insert, update: r.update, skip: r.skip, error: r.error }).toEqual({
|
||||||
|
insert: 1,
|
||||||
|
update: 0,
|
||||||
|
skip: 1,
|
||||||
|
error: 1,
|
||||||
|
});
|
||||||
|
expect(r.rows.map((x) => x.outcome)).toEqual(['skip', 'insert', 'error']);
|
||||||
|
expect(r.rows[0]!.existingId).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('update-matches: existing→update (adapter supports update)', async () => {
|
||||||
|
const { ctx, rows } = await setup();
|
||||||
|
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'update-matches', ctx);
|
||||||
|
expect(r.rows[0]!.outcome).toBe('update');
|
||||||
|
expect(r.update).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('error-on-match: existing→error', async () => {
|
||||||
|
const { ctx, rows } = await setup();
|
||||||
|
const r = await classifyRows(companiesAdapter, rows, MAPPING, 'error-on-match', ctx);
|
||||||
|
expect(r.rows[0]!.outcome).toBe('error');
|
||||||
|
// The blank row is still an error too → 2 errors total.
|
||||||
|
expect(r.error).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
91
tests/unit/import/engine.test.ts
Normal file
91
tests/unit/import/engine.test.ts
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
/**
|
||||||
|
* Unit tests for the importer engine internals (no DB): fuzzy column mapping,
|
||||||
|
* row validation (per-field + cross-field), natural-key derivation, and CSV
|
||||||
|
* parsing.
|
||||||
|
*/
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { applyMapping, suggestMapping } from '@/lib/import/mapping';
|
||||||
|
import { validateRow } from '@/lib/import/classify';
|
||||||
|
import { parseCsv } from '@/lib/import/engine';
|
||||||
|
import { companiesAdapter } from '@/lib/import/adapters/companies';
|
||||||
|
import { clientsAdapter } from '@/lib/import/adapters/clients';
|
||||||
|
import { berthsAdapter } from '@/lib/import/adapters/berths';
|
||||||
|
|
||||||
|
describe('suggestMapping', () => {
|
||||||
|
it('maps headers to fields by exact, alias, and fuzzy match', () => {
|
||||||
|
const m = suggestMapping(
|
||||||
|
['Company Name', 'VAT', 'Billing Email', 'Unrelated'],
|
||||||
|
companiesAdapter.targetFields,
|
||||||
|
);
|
||||||
|
expect(m.name).toBe('Company Name'); // alias "companyname"
|
||||||
|
expect(m.taxId).toBe('VAT'); // alias "vat"
|
||||||
|
expect(m.billingEmail).toBe('Billing Email');
|
||||||
|
// A header claimed by one field isn't reused by another.
|
||||||
|
expect(Object.values(m).filter((h) => h === 'Company Name')).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leaves unmatched fields out of the mapping', () => {
|
||||||
|
const m = suggestMapping(['xyz', 'qqq'], companiesAdapter.targetFields);
|
||||||
|
expect(m.name).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('applyMapping', () => {
|
||||||
|
it('produces fieldKey→cell and drops empty cells', () => {
|
||||||
|
const mapped = applyMapping(
|
||||||
|
{ 'Company Name': ' Acme ', VAT: '', Email: 'a@b.com' },
|
||||||
|
{ name: 'Company Name', taxId: 'VAT', billingEmail: 'Email' },
|
||||||
|
);
|
||||||
|
expect(mapped).toEqual({ name: 'Acme', billingEmail: 'a@b.com' });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('validateRow', () => {
|
||||||
|
it('flags a missing required field', () => {
|
||||||
|
const errs = validateRow(companiesAdapter, { legalName: 'X Ltd' });
|
||||||
|
expect(errs).toEqual([{ field: 'name', message: 'Name is required' }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('flags an invalid email', () => {
|
||||||
|
const errs = validateRow(companiesAdapter, { name: 'Acme', billingEmail: 'not-an-email' });
|
||||||
|
expect(errs.some((e) => e.field === 'billingEmail')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clients require an email or a phone (cross-field)', () => {
|
||||||
|
expect(validateRow(clientsAdapter, { fullName: 'Jane Doe' })).toEqual([
|
||||||
|
{ field: 'email', message: 'An email or phone is required' },
|
||||||
|
]);
|
||||||
|
expect(validateRow(clientsAdapter, { fullName: 'Jane Doe', email: 'j@d.com' })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('berths reject a malformed mooring number', () => {
|
||||||
|
const errs = validateRow(berthsAdapter, { mooringNumber: 'not-a-mooring', area: 'A' });
|
||||||
|
expect(errs.some((e) => e.field === 'mooringNumber')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('matchKey', () => {
|
||||||
|
it('companies: case-insensitive name', () => {
|
||||||
|
expect(companiesAdapter.matchKey({ name: ' AcMe ' })).toBe('acme');
|
||||||
|
});
|
||||||
|
it('berths: canonicalizes mooring (D-032 → D32)', () => {
|
||||||
|
expect(berthsAdapter.matchKey({ mooringNumber: 'd-032', area: 'D' })).toBe('D32');
|
||||||
|
});
|
||||||
|
it('clients: email key, phone fallback', () => {
|
||||||
|
expect(clientsAdapter.matchKey({ fullName: 'X', email: 'A@B.com' })).toBe('email:a@b.com');
|
||||||
|
const k = clientsAdapter.matchKey({ fullName: 'X', phone: '+1 574 274 0548' });
|
||||||
|
expect(k?.startsWith('phone:')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseCsv', () => {
|
||||||
|
it('parses headers + rows, trims, skips blank lines', () => {
|
||||||
|
const { headers, rows } = parseCsv('Name, Email\nAcme, a@b.com\n\nBeta, b@c.com\n');
|
||||||
|
expect(headers).toEqual(['Name', 'Email']);
|
||||||
|
expect(rows).toEqual([
|
||||||
|
{ Name: 'Acme', Email: 'a@b.com' },
|
||||||
|
{ Name: 'Beta', Email: 'b@c.com' },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user