feat(berths): nocodb berth import script + helpers + unit tests
Idempotent NocoDB Berths -> CRM `berths` import script with full re-run safety. Re-running picks up NocoDB additions/edits without clobbering CRM-side overrides (compares updated_at vs last_imported_at, 1-second tolerance for sub-second clock drift). --force overrides the edit guard. Mitigates the §14.1 critical/high cases: - Mooring collisions: unique (port_id, mooring_number) on the table. - Concurrent runs: pg_advisory_xact_lock on a stable BIGINT key. - Numeric-with-units inputs: parseDecimalWithUnit() strips trailing ft/m/kw/v/usd/$ markers before parsing. - Metric drift: NocoDB's metric formula columns are ignored; metric values recomputed from imperial via 0.3048 + round-to-2-decimals to match NocoDB's `precision: 2` columns and avoid spurious diffs. - Map Data shape: zod-validated; failures are skipped rather than aborting the import. - Status enum mapping: NocoDB display strings -> CRM snake_case. - NocoDB row deleted: reported as "orphaned in CRM"; never auto- deleted (rep decides via admin UI in a future phase). Pure helpers (parseDecimalWithUnit, mapStatus, parseMapData, extractNumerics, mapRow, buildPlan) live in src/lib/services/berth-import.ts so vitest can exercise the mapping logic without triggering the script's top-level db connection. 40 new unit tests (956 -> 996 passing). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Ciaccio
245
src/lib/services/berth-import.ts
Normal file
245
src/lib/services/berth-import.ts
Normal file
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Pure helpers + plan-builder for the NocoDB → CRM berth import.
|
||||
*
|
||||
* Lives outside the CLI script (`scripts/import-berths-from-nocodb.ts`)
|
||||
* so vitest can exercise the mapping/normalization/plan logic without
|
||||
* triggering the script's top-level db connection.
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { NocoDbRow } from '@/lib/dedup/nocodb-source';
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Strip trailing units ("63ft", "12 m") and return a JS number, or null
|
||||
* if the input doesn't parse cleanly. NocoDB stores plain numerics for
|
||||
* the Berth fields we care about, but defensive against future drift or
|
||||
* legacy import data.
|
||||
*/
|
||||
export function parseDecimalWithUnit(raw: unknown): number | null {
|
||||
if (raw == null) return null;
|
||||
if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null;
|
||||
if (typeof raw !== 'string') return null;
|
||||
const m = /^\s*(-?\d+(?:\.\d+)?)\s*(?:ft|feet|m|metres|meters|kw|v|usd|\$)?\s*$/i.exec(raw);
|
||||
if (!m) return null;
|
||||
const n = parseFloat(m[1]!);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
/** Round to 2 decimals to match NocoDB's `precision: 2` decimal columns. */
|
||||
export function round2(n: number | null): number | null {
|
||||
if (n == null) return null;
|
||||
return Math.round(n * 100) / 100;
|
||||
}
|
||||
|
||||
const FT_TO_M = 0.3048;
|
||||
|
||||
/** Imperial → metric. Returns null when the input is null. */
|
||||
export function ftToM(ft: number | null): number | null {
|
||||
if (ft == null) return null;
|
||||
return round2(ft * FT_TO_M);
|
||||
}
|
||||
|
||||
/** NocoDB display Status → CRM internal status. Defaults to 'available'. */
|
||||
export function mapStatus(raw: unknown): 'available' | 'under_offer' | 'sold' {
|
||||
switch (typeof raw === 'string' ? raw.trim() : raw) {
|
||||
case 'Available':
|
||||
return 'available';
|
||||
case 'Under Offer':
|
||||
return 'under_offer';
|
||||
case 'Sold':
|
||||
return 'sold';
|
||||
default:
|
||||
return 'available';
|
||||
}
|
||||
}
|
||||
|
||||
const MapDataSchema = z.object({
|
||||
path: z.string().optional(),
|
||||
x: z.union([z.string(), z.number()]).optional(),
|
||||
y: z.union([z.string(), z.number()]).optional(),
|
||||
transform: z.string().optional(),
|
||||
fontSize: z.union([z.string(), z.number()]).optional(),
|
||||
});
|
||||
export type MapData = z.infer<typeof MapDataSchema>;
|
||||
|
||||
export function parseMapData(raw: unknown): MapData | null {
|
||||
if (raw == null) return null;
|
||||
const candidate = typeof raw === 'string' ? safeJsonParse(raw) : raw;
|
||||
if (candidate == null) return null;
|
||||
const parsed = MapDataSchema.safeParse(candidate);
|
||||
return parsed.success ? parsed.data : null;
|
||||
}
|
||||
|
||||
function safeJsonParse(s: string): unknown {
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function toNumberish(raw: unknown): number | null {
|
||||
if (raw == null || raw === '') return null;
|
||||
if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null;
|
||||
if (typeof raw === 'string') {
|
||||
const n = parseFloat(raw);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Numerics extractor ─────────────────────────────────────────────────────
|
||||
|
||||
export interface NumericFields {
|
||||
lengthFt: number | null;
|
||||
widthFt: number | null;
|
||||
draftFt: number | null;
|
||||
lengthM: number | null;
|
||||
widthM: number | null;
|
||||
draftM: number | null;
|
||||
nominalBoatSize: number | null;
|
||||
nominalBoatSizeM: number | null;
|
||||
waterDepth: number | null;
|
||||
waterDepthM: number | null;
|
||||
powerCapacity: number | null;
|
||||
voltage: number | null;
|
||||
price: number | null;
|
||||
}
|
||||
|
||||
export function extractNumerics(row: NocoDbRow): NumericFields {
|
||||
const lengthFt = round2(parseDecimalWithUnit(row['Length']));
|
||||
const widthFt = round2(parseDecimalWithUnit(row['Width']));
|
||||
const draftFt = round2(parseDecimalWithUnit(row['Draft']));
|
||||
const waterDepth = round2(parseDecimalWithUnit(row['Water Depth']));
|
||||
const nominalBoatSize = toNumberish(row['Nominal Boat Size']);
|
||||
return {
|
||||
lengthFt,
|
||||
widthFt,
|
||||
draftFt,
|
||||
lengthM: ftToM(lengthFt),
|
||||
widthM: ftToM(widthFt),
|
||||
draftM: ftToM(draftFt),
|
||||
nominalBoatSize,
|
||||
nominalBoatSizeM: ftToM(nominalBoatSize),
|
||||
waterDepth,
|
||||
waterDepthM: ftToM(waterDepth),
|
||||
powerCapacity: toNumberish(row['Power Capacity']),
|
||||
voltage: toNumberish(row['Voltage']),
|
||||
price: toNumberish(row['Price']),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Row mapper ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImportedBerth {
|
||||
legacyId: number;
|
||||
mooringNumber: string;
|
||||
area: string | null;
|
||||
status: 'available' | 'under_offer' | 'sold';
|
||||
numerics: NumericFields;
|
||||
widthIsMinimum: boolean;
|
||||
waterDepthIsMinimum: boolean;
|
||||
sidePontoon: string | null;
|
||||
mooringType: string | null;
|
||||
cleatType: string | null;
|
||||
cleatCapacity: string | null;
|
||||
bollardType: string | null;
|
||||
bollardCapacity: string | null;
|
||||
access: string | null;
|
||||
bowFacing: string | null;
|
||||
berthApproved: boolean;
|
||||
statusOverrideMode: string | null;
|
||||
mapData: MapData | null;
|
||||
}
|
||||
|
||||
export function mapRow(row: NocoDbRow): ImportedBerth | null {
|
||||
const mooringNumber =
|
||||
typeof row['Mooring Number'] === 'string' ? row['Mooring Number'].trim() : '';
|
||||
if (!mooringNumber) return null;
|
||||
return {
|
||||
legacyId: row.Id,
|
||||
mooringNumber,
|
||||
area: typeof row['Area'] === 'string' ? row['Area'] : null,
|
||||
status: mapStatus(row['Status']),
|
||||
numerics: extractNumerics(row),
|
||||
widthIsMinimum: row['Width Is Minimum'] === true,
|
||||
waterDepthIsMinimum: row['Water Depth Is Minimum'] === true,
|
||||
sidePontoon: typeof row['Side Pontoon'] === 'string' ? row['Side Pontoon'] : null,
|
||||
mooringType: typeof row['Mooring Type'] === 'string' ? row['Mooring Type'] : null,
|
||||
cleatType: typeof row['Cleat Type'] === 'string' ? row['Cleat Type'] : null,
|
||||
cleatCapacity: typeof row['Cleat Capacity'] === 'string' ? row['Cleat Capacity'] : null,
|
||||
bollardType: typeof row['Bollard Type'] === 'string' ? row['Bollard Type'] : null,
|
||||
bollardCapacity: typeof row['Bollard Capacity'] === 'string' ? row['Bollard Capacity'] : null,
|
||||
access: typeof row['Access'] === 'string' ? row['Access'] : null,
|
||||
bowFacing: typeof row['Bow Facing'] === 'string' ? row['Bow Facing'] : null,
|
||||
berthApproved: row['Berth Approved'] === true,
|
||||
statusOverrideMode:
|
||||
typeof row['status_override_mode'] === 'string' ? row['status_override_mode'] : null,
|
||||
mapData: parseMapData(row['Map Data']),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Plan builder ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface ExistingBerthRow {
|
||||
id: string;
|
||||
mooringNumber: string;
|
||||
updatedAt: Date;
|
||||
lastImportedAt: Date | null;
|
||||
}
|
||||
|
||||
export type Action = 'insert' | 'update' | 'skip-edited' | 'noop';
|
||||
|
||||
export interface PlanEntry {
|
||||
action: Action;
|
||||
imported: ImportedBerth;
|
||||
existing: ExistingBerthRow | null;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface BuildPlanResult {
|
||||
plan: PlanEntry[];
|
||||
orphans: ExistingBerthRow[];
|
||||
}
|
||||
|
||||
export function buildPlan(
|
||||
imported: ImportedBerth[],
|
||||
existingByMooring: Map<string, ExistingBerthRow>,
|
||||
force: boolean,
|
||||
): BuildPlanResult {
|
||||
const plan: PlanEntry[] = [];
|
||||
const seenMoorings = new Set<string>();
|
||||
for (const row of imported) {
|
||||
seenMoorings.add(row.mooringNumber);
|
||||
const existing = existingByMooring.get(row.mooringNumber) ?? null;
|
||||
if (!existing) {
|
||||
plan.push({ action: 'insert', imported: row, existing: null });
|
||||
continue;
|
||||
}
|
||||
const lastImported = existing.lastImportedAt;
|
||||
// 1-second tolerance: an UPDATE inside the apply transaction sets
|
||||
// updated_at and last_imported_at to the same `importedAt` timestamp,
|
||||
// but Postgres can race them by sub-second on busy boxes.
|
||||
const isHumanEdited =
|
||||
lastImported == null ? false : existing.updatedAt.getTime() > lastImported.getTime() + 1000;
|
||||
if (isHumanEdited && !force) {
|
||||
plan.push({
|
||||
action: 'skip-edited',
|
||||
imported: row,
|
||||
existing,
|
||||
reason: `CRM updated_at=${existing.updatedAt.toISOString()} > last_imported_at=${
|
||||
lastImported?.toISOString() ?? 'null'
|
||||
}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
plan.push({ action: 'update', imported: row, existing });
|
||||
}
|
||||
const orphans = Array.from(existingByMooring.values()).filter(
|
||||
(e) => !seenMoorings.has(e.mooringNumber),
|
||||
);
|
||||
return { plan, orphans };
|
||||
}
|
||||
Reference in New Issue
Block a user