278 lines
14 KiB
TypeScript
278 lines
14 KiB
TypeScript
|
/**
|
||
|
|
* Exhaustive migration reconciliation (read-only): cross-checks EVERY migrated
|
||
|
|
* record against its legacy NocoDB source row (via the migration ledger) and
|
||
|
|
* verifies every relationship is connected. Independently re-derives the
|
||
|
|
* expected mapped values (stage, eoiStatus, berth, …) so it validates the
|
||
|
|
* migration logic, not just echoes it.
|
||
|
|
*
|
||
|
|
* Connects to BOTH local DBs:
|
||
|
|
* - CRM : DATABASE_URL (the migrated data)
|
||
|
|
* - legacy : LEGACY_DB_URL (the nocodb_legacy snapshot); defaults to the
|
||
|
|
* CRM url with the db name swapped to `nocodb_legacy`.
|
||
|
|
*
|
||
|
|
* pnpm tsx scripts/migration/reconcile-migration.ts [--port-slug port-nimara]
|
||
|
|
*/
|
||
|
|
import 'dotenv/config';
|
||
|
|
import postgres from 'postgres';
|
||
|
|
|
||
|
|
const slugArg = (() => {
|
||
|
|
const i = process.argv.indexOf('--port-slug');
|
||
|
|
return i >= 0 ? (process.argv[i + 1] ?? 'port-nimara') : 'port-nimara';
|
||
|
|
})();
|
||
|
|
|
||
|
|
const CRM_URL = process.env.DATABASE_URL!;
|
||
|
|
const LEGACY_URL = process.env.LEGACY_DB_URL ?? CRM_URL.replace(/\/[^/]+$/, '/nocodb_legacy');
|
||
|
|
const crm = postgres(CRM_URL, { max: 4 });
|
||
|
|
const legacy = postgres(LEGACY_URL, { max: 4 });
|
||
|
|
|
||
|
|
// ── transforms, re-implemented independently (cross-validation) ──────────────
|
||
|
|
const STAGE_MAP: Record<string, string> = {
|
||
|
|
'General Qualified Interest': 'qualified',
|
||
|
|
'Specific Qualified Interest': 'nurturing',
|
||
|
|
'EOI and NDA Sent': 'eoi',
|
||
|
|
'Signed EOI and NDA': 'eoi',
|
||
|
|
'Made Reservation': 'reservation',
|
||
|
|
'Contract Negotiation': 'contract',
|
||
|
|
'Contract Negotiations Finalized': 'contract',
|
||
|
|
'Contract Signed': 'contract',
|
||
|
|
};
|
||
|
|
const expectStage = (level: string | undefined, deposit: string | undefined): string => {
|
||
|
|
let s = STAGE_MAP[(level ?? '').trim()] ?? 'enquiry';
|
||
|
|
if ((deposit ?? '').trim() === 'Received' && s !== 'contract') s = 'deposit_paid';
|
||
|
|
return s;
|
||
|
|
};
|
||
|
|
const expectEoi = (
|
||
|
|
eoiStatus: string | undefined,
|
||
|
|
loi: string | undefined,
|
||
|
|
documensoId: string | undefined,
|
||
|
|
): string | null => {
|
||
|
|
const e = (eoiStatus ?? '').trim();
|
||
|
|
const l = (loi ?? '').trim();
|
||
|
|
if (e === 'Signed' || ['Signing Complete', 'Signed by Client', 'Signed by Developer'].includes(l))
|
||
|
|
return 'signed';
|
||
|
|
if (e === 'Waiting for Signatures' || (documensoId ?? '').trim()) return 'waiting_for_signatures';
|
||
|
|
return null;
|
||
|
|
};
|
||
|
|
const canonMoo = (raw: string): string => {
|
||
|
|
const m = /^([A-Za-z]+)-?0*(\d+)$/.exec((raw ?? '').trim());
|
||
|
|
return m ? `${m[1]!.toUpperCase()}${parseInt(m[2]!, 10)}` : (raw ?? '').trim();
|
||
|
|
};
|
||
|
|
const normEmail = (e: string) => (e ?? '').trim().toLowerCase();
|
||
|
|
|
||
|
|
const issues: string[] = [];
|
||
|
|
const add = (cat: string, msg: string) => issues.push(`[${cat}] ${msg}`);
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
const [port] = await crm`select id, slug from ports where slug=${slugArg} limit 1`;
|
||
|
|
if (!port) throw new Error(`no port ${slugArg}`);
|
||
|
|
const portId = port.id as string;
|
||
|
|
|
||
|
|
// ── load legacy source (by id) ───────────────────────────────────────────
|
||
|
|
const legacyInterests = new Map<number, Record<string, unknown>>();
|
||
|
|
for (const r of await legacy`select * from plplouets5zw1um."Interests"`)
|
||
|
|
legacyInterests.set(r.id as number, r);
|
||
|
|
const legacyExpenses = new Map<number, Record<string, unknown>>();
|
||
|
|
for (const r of await legacy`select * from p3hq2fxdevqcaq8."Expenses"`)
|
||
|
|
legacyExpenses.set(r.id as number, r);
|
||
|
|
const legacyRes = new Map<number, Record<string, unknown>>();
|
||
|
|
for (const r of await legacy`select * from plplouets5zw1um."Interests (Residences)"`)
|
||
|
|
legacyRes.set(r.id as number, r);
|
||
|
|
// legacy berth links per interest (Interests_id -> [mooring])
|
||
|
|
const berthMooById = new Map<number, string>();
|
||
|
|
for (const b of await legacy`select id, "Mooring_Number" m from plplouets5zw1um."Berths"`)
|
||
|
|
berthMooById.set(b.id as number, b.m as string);
|
||
|
|
const legacyBerthsByInterest = new Map<number, string[]>();
|
||
|
|
for (const j of await legacy`select "Interests_id" i, "Berths_id" b from plplouets5zw1um."_nc_m2m_Berths_Interests"`) {
|
||
|
|
const arr = legacyBerthsByInterest.get(j.i as number) ?? [];
|
||
|
|
const moo = berthMooById.get(j.b as number);
|
||
|
|
if (moo) arr.push(canonMoo(moo));
|
||
|
|
legacyBerthsByInterest.set(j.i as number, arr);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── ledger ────────────────────────────────────────────────────────────────
|
||
|
|
const ledger =
|
||
|
|
await crm`select source_system, source_id, target_entity_type, target_entity_id from migration_source_links`;
|
||
|
|
const interestLinks = ledger.filter((l) => l.target_entity_type === 'interest'); // sourceId(legacy interest) -> new interest
|
||
|
|
const expenseLinks = ledger.filter((l) => l.target_entity_type === 'expense');
|
||
|
|
const resLinks = ledger.filter((l) => l.target_entity_type === 'residential_client');
|
||
|
|
const clientLinks = ledger.filter((l) => l.target_entity_type === 'client');
|
||
|
|
|
||
|
|
// ── 1. COVERAGE — every legacy row migrated; nothing dropped ──────────────
|
||
|
|
const migratedInterestSrc = new Set(interestLinks.map((l) => Number(l.source_id)));
|
||
|
|
const droppedInterests = [...legacyInterests.keys()].filter((id) => !migratedInterestSrc.has(id));
|
||
|
|
const migratedExpSrc = new Set(expenseLinks.map((l) => Number(l.source_id)));
|
||
|
|
const droppedExp = [...legacyExpenses.keys()].filter((id) => !migratedExpSrc.has(id));
|
||
|
|
const migratedResSrc = new Set(resLinks.map((l) => Number(l.source_id)));
|
||
|
|
const droppedRes = [...legacyRes.keys()].filter((id) => !migratedResSrc.has(id));
|
||
|
|
for (const id of droppedInterests)
|
||
|
|
add(
|
||
|
|
'COVERAGE',
|
||
|
|
`legacy interest #${id} NOT migrated (${(legacyInterests.get(id) as { Full_Name?: string }).Full_Name ?? '?'})`,
|
||
|
|
);
|
||
|
|
for (const id of droppedExp) add('COVERAGE', `legacy expense #${id} NOT migrated`);
|
||
|
|
for (const id of droppedRes) add('COVERAGE', `legacy residential #${id} NOT migrated`);
|
||
|
|
|
||
|
|
// ── 2. INTEREST field fidelity (every migrated deal vs legacy) ────────────
|
||
|
|
const newInterests = await crm`
|
||
|
|
select i.id, i.pipeline_stage, i.lead_category, i.source, i.eoi_status, i.documenso_id, i.client_id, i.yacht_id
|
||
|
|
from interests i where i.port_id=${portId}`;
|
||
|
|
const newInterestById = new Map(newInterests.map((i) => [i.id as string, i]));
|
||
|
|
// berths per new interest
|
||
|
|
const ibRows = await crm`
|
||
|
|
select ib.interest_id, b.mooring_number from interest_berths ib join berths b on b.id=ib.berth_id where b.port_id=${portId}`;
|
||
|
|
const newBerthsByInterest = new Map<string, string[]>();
|
||
|
|
for (const r of ibRows) {
|
||
|
|
const a = newBerthsByInterest.get(r.interest_id as string) ?? [];
|
||
|
|
a.push(r.mooring_number as string);
|
||
|
|
newBerthsByInterest.set(r.interest_id as string, a);
|
||
|
|
}
|
||
|
|
let stageMiss = 0,
|
||
|
|
eoiMiss = 0,
|
||
|
|
docMiss = 0,
|
||
|
|
berthMiss = 0;
|
||
|
|
for (const l of interestLinks) {
|
||
|
|
const legacyRow = legacyInterests.get(Number(l.source_id));
|
||
|
|
const ni = newInterestById.get(l.target_entity_id as string);
|
||
|
|
if (!legacyRow || !ni) {
|
||
|
|
add(
|
||
|
|
'INTEGRITY',
|
||
|
|
`interest link sourceId=${l.source_id} → ${l.target_entity_id}: ${!legacyRow ? 'legacy row missing' : 'new interest missing'}`,
|
||
|
|
);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
const lr = legacyRow as Record<string, string>;
|
||
|
|
const exp = expectStage(lr.Sales_Process_Level, lr.Deposit_10__Status);
|
||
|
|
if (ni.pipeline_stage !== exp) {
|
||
|
|
stageMiss++;
|
||
|
|
add(
|
||
|
|
'STAGE',
|
||
|
|
`interest src#${l.source_id} (${lr.Full_Name}): legacy "${lr.Sales_Process_Level}" → expected ${exp}, got ${ni.pipeline_stage}`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
const expEoi = expectEoi(lr.EOI_Status, lr.LOI_NDA_Document, lr.documensoID);
|
||
|
|
if ((ni.eoi_status ?? null) !== expEoi) {
|
||
|
|
eoiMiss++;
|
||
|
|
add(
|
||
|
|
'EOI',
|
||
|
|
`interest src#${l.source_id} (${lr.Full_Name}): expected eoiStatus ${expEoi}, got ${ni.eoi_status}`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
if ((ni.documenso_id ?? null) !== ((lr.documensoID ?? '').trim() || null)) {
|
||
|
|
docMiss++;
|
||
|
|
add(
|
||
|
|
'DOCID',
|
||
|
|
`interest src#${l.source_id} (${lr.Full_Name}): documensoId legacy="${lr.documensoID}" vs new="${ni.documenso_id}"`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
// berth: every legacy-linked mooring should be present on the new interest
|
||
|
|
const legacyMoo = new Set([...(legacyBerthsByInterest.get(Number(l.source_id)) ?? [])]);
|
||
|
|
if (lr.Berth_Number && /^[A-Za-z]+-?0*\d+$/.test(lr.Berth_Number.trim()))
|
||
|
|
legacyMoo.add(canonMoo(lr.Berth_Number));
|
||
|
|
const newMoo = new Set(newBerthsByInterest.get(ni.id as string) ?? []);
|
||
|
|
const missingBerths = [...legacyMoo].filter((m) => !newMoo.has(m));
|
||
|
|
if (missingBerths.length > 0) {
|
||
|
|
berthMiss++;
|
||
|
|
add(
|
||
|
|
'BERTH',
|
||
|
|
`interest src#${l.source_id} (${lr.Full_Name}): legacy berths [${[...legacyMoo].join(',')}] but new has [${[...newMoo].join(',') || '-'}] (missing ${missingBerths.join(',')})`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── 3. CLIENT contact fidelity (migrated email is from a legacy source row)
|
||
|
|
const clientContacts = await crm`
|
||
|
|
select c.id, c.full_name, string_agg(cc.value, '|') filter (where cc.channel='email') emails
|
||
|
|
from clients c left join client_contacts cc on cc.client_id=c.id
|
||
|
|
where c.port_id=${portId} group by c.id, c.full_name`;
|
||
|
|
const emailsByClient = new Map(
|
||
|
|
clientContacts.map((c) => [
|
||
|
|
c.id as string,
|
||
|
|
(c.emails as string | null)?.split('|').map(normEmail) ?? [],
|
||
|
|
]),
|
||
|
|
);
|
||
|
|
// group ledger client links: client -> its legacy source emails
|
||
|
|
const legacyEmailsByClient = new Map<string, Set<string>>();
|
||
|
|
for (const l of clientLinks) {
|
||
|
|
const lr = legacyInterests.get(Number(l.source_id)) as Record<string, string> | undefined;
|
||
|
|
const e = normEmail(lr?.Email_Address ?? '');
|
||
|
|
if (!e) continue;
|
||
|
|
const set = legacyEmailsByClient.get(l.target_entity_id as string) ?? new Set();
|
||
|
|
set.add(e);
|
||
|
|
legacyEmailsByClient.set(l.target_entity_id as string, set);
|
||
|
|
}
|
||
|
|
let emailMiss = 0;
|
||
|
|
for (const [cid, legacyEmails] of legacyEmailsByClient) {
|
||
|
|
const newEmails = new Set(emailsByClient.get(cid) ?? []);
|
||
|
|
const missing = [...legacyEmails].filter((e) => !newEmails.has(e));
|
||
|
|
if (missing.length > 0) {
|
||
|
|
emailMiss++;
|
||
|
|
const nm = clientContacts.find((c) => c.id === cid)?.full_name;
|
||
|
|
add(
|
||
|
|
'EMAIL',
|
||
|
|
`client ${nm}: legacy email(s) [${[...legacyEmails].join(',')}] not all on client (have [${[...newEmails].join(',') || '-'}])`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── 4. RELATIONSHIP integrity (orphans / dangling FKs) ────────────────────
|
||
|
|
const orphanInterests =
|
||
|
|
await crm`select count(*) n from interests i where i.port_id=${portId} and not exists (select 1 from clients c where c.id=i.client_id)`;
|
||
|
|
const orphanIB =
|
||
|
|
await crm`select count(*) n from interest_berths ib where not exists (select 1 from interests i where i.id=ib.interest_id) or not exists (select 1 from berths b where b.id=ib.berth_id)`;
|
||
|
|
const orphanDocs =
|
||
|
|
await crm`select count(*) n from documents d where d.port_id=${portId} and d.interest_id is not null and not exists (select 1 from interests i where i.id=d.interest_id)`;
|
||
|
|
const orphanYachts =
|
||
|
|
await crm`select count(*) n from yachts y where y.port_id=${portId} and y.current_owner_type='client' and not exists (select 1 from clients c where c.id=y.current_owner_id)`;
|
||
|
|
const danglingSignedFile =
|
||
|
|
await crm`select count(*) n from documents d where d.signed_file_id is not null and not exists (select 1 from files f where f.id=d.signed_file_id)`;
|
||
|
|
if (Number(orphanInterests[0]!.n) > 0)
|
||
|
|
add('INTEGRITY', `${orphanInterests[0]!.n} interests with no client`);
|
||
|
|
if (Number(orphanIB[0]!.n) > 0)
|
||
|
|
add('INTEGRITY', `${orphanIB[0]!.n} interest_berths with dangling FK`);
|
||
|
|
if (Number(orphanDocs[0]!.n) > 0)
|
||
|
|
add('INTEGRITY', `${orphanDocs[0]!.n} documents with dangling interest`);
|
||
|
|
if (Number(orphanYachts[0]!.n) > 0)
|
||
|
|
add('INTEGRITY', `${orphanYachts[0]!.n} yachts with missing owner`);
|
||
|
|
if (Number(danglingSignedFile[0]!.n) > 0)
|
||
|
|
add('INTEGRITY', `${danglingSignedFile[0]!.n} documents with dangling signed_file_id`);
|
||
|
|
|
||
|
|
// ── report ────────────────────────────────────────────────────────────────
|
||
|
|
console.log('═══════════ MIGRATION RECONCILIATION ═══════════\n');
|
||
|
|
console.log(
|
||
|
|
`Coverage: legacy interests ${legacyInterests.size} → migrated ${migratedInterestSrc.size} (dropped ${droppedInterests.length})`,
|
||
|
|
);
|
||
|
|
console.log(
|
||
|
|
` legacy expenses ${legacyExpenses.size} → migrated ${migratedExpSrc.size} (dropped ${droppedExp.length})`,
|
||
|
|
);
|
||
|
|
console.log(
|
||
|
|
` legacy residential ${legacyRes.size} → migrated ${migratedResSrc.size} (dropped ${droppedRes.length})`,
|
||
|
|
);
|
||
|
|
console.log(
|
||
|
|
`Fidelity: stage mismatches ${stageMiss} · eoiStatus ${eoiMiss} · documensoId ${docMiss} · berth-link ${berthMiss} · client-email ${emailMiss}`,
|
||
|
|
);
|
||
|
|
console.log(
|
||
|
|
`Integrity: orphan interests ${orphanInterests[0]!.n} · interest_berths ${orphanIB[0]!.n} · docs ${orphanDocs[0]!.n} · yachts ${orphanYachts[0]!.n} · signed-file ${danglingSignedFile[0]!.n}`,
|
||
|
|
);
|
||
|
|
console.log(`\nTotal discrepancies: ${issues.length}`);
|
||
|
|
const byCat = issues.reduce<Record<string, number>>((a, s) => {
|
||
|
|
const c = s.slice(1, s.indexOf(']'));
|
||
|
|
a[c] = (a[c] || 0) + 1;
|
||
|
|
return a;
|
||
|
|
}, {});
|
||
|
|
console.log('By category:', JSON.stringify(byCat));
|
||
|
|
console.log('\n── discrepancy detail (first 60) ──');
|
||
|
|
for (const i of issues.slice(0, 60)) console.log(' ' + i);
|
||
|
|
if (issues.length > 60) console.log(` … +${issues.length - 60} more`);
|
||
|
|
|
||
|
|
await crm.end();
|
||
|
|
await legacy.end();
|
||
|
|
process.exit(0);
|
||
|
|
}
|
||
|
|
|
||
|
|
main().catch(async (e) => {
|
||
|
|
console.error('reconcile failed:', e);
|
||
|
|
await crm.end().catch(() => {});
|
||
|
|
await legacy.end().catch(() => {});
|
||
|
|
process.exit(1);
|
||
|
|
});
|