feat(dedup): wire --apply path for NocoDB migration
Completes the migration script's apply phase, which was stubbed at
the P3 ship to defer until after the runtime surfaces (P2) and the
comms safety net were in place. Both prerequisites just landed on
main, so this unblocks the actual data import.
src/lib/dedup/migration-apply.ts (new):
Idempotent apply driver. Walks the MigrationPlan, inserting clients,
contacts, addresses, yacht stubs, and interests, threading every
insert through the migration_source_links ledger so re-runs against
the same data are safe. Per-entity transactions (not one giant
transaction) so partial-failure resumption is just "run again."
Per-entity behavior:
- clients: idempotent on (source_system, source_id, target_type=client)
across the entire dedup cluster — if any source row already maps
to a client, reuse that record.
- contacts: bulk insert, primary email + primary phone independent.
- addresses: bulk insert, port_id required (schema enforces it),
first address marked primary when multiple.
- yachts: minimal stub when the legacy interest had a yachtName,
currentOwnerType=client + currentOwnerId=migrated client. Linked
via migration_source_links target_type=yacht.
- interests: looks up berthId via mooring number, yachtId via the
stub above. Carries Documenso ID forward when present.
surnameToken from PlannedClient is dropped on insert (it's a dedup
blocking-index artifact; runtime dedup re-derives from fullName).
scripts/migrate-from-nocodb.ts:
- Removes the "not yet implemented" guard for --apply.
- Adds EMAIL_REDIRECT_TO precondition gate: --apply errors out unless
the env var is set, OR --unsafe-skip-redirect-check is also passed
(production cutover only). Refers to docs/operations/outbound-comms-safety.md.
- Re-fetches NocoDB at apply time (rather than reading a saved report
dir) so the data is always fresh. Re-running is safe via the
idempotency ledger.
- Resolves target port via --port-slug (or first port if omitted).
- Generates a UUID applyId tagged on every link, which pairs with a
future --rollback flag.
- Apply summary prints inserted/skipped counts per entity type plus
the first 20 warnings.
Verification: 0 tsc errors, 926/926 vitest passing, lint clean.
The actual end-to-end run requires NOCODB_URL + NOCODB_TOKEN in .env
which aren't configured in this checkout; that's the operator's next
step.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,21 +7,30 @@
|
||||
* Pulls the live NocoDB base, runs the transform + dedup pipeline,
|
||||
* writes a report to .migration/<timestamp>/. NO database writes.
|
||||
*
|
||||
* pnpm tsx scripts/migrate-from-nocodb.ts --dry-run --port-slug harbor-royale
|
||||
* pnpm tsx scripts/migrate-from-nocodb.ts --dry-run --port-slug port-nimara
|
||||
* Same, but tags the planned writes with the named port (matters for
|
||||
* the apply phase — every client/interest belongs to one port).
|
||||
*
|
||||
* pnpm tsx scripts/migrate-from-nocodb.ts --apply --report .migration/<dir>/
|
||||
* [Not yet implemented — apply phase comes in a follow-up PR.]
|
||||
* pnpm tsx scripts/migrate-from-nocodb.ts --apply --port-slug port-nimara
|
||||
* Re-fetches NocoDB, re-transforms, then writes the planned rows
|
||||
* into the target port via the idempotent `migration_source_links`
|
||||
* ledger. Re-runs are safe — already-imported source IDs are skipped.
|
||||
* REQUIRES `EMAIL_REDIRECT_TO` to be set in env (safety net) unless
|
||||
* `--unsafe-skip-redirect-check` is also passed.
|
||||
*
|
||||
* Design reference: docs/superpowers/specs/2026-05-03-dedup-and-migration-design.md §9.
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
import { db } from '@/lib/db';
|
||||
import { ports } from '@/lib/db/schema/ports';
|
||||
import { applyPlan } from '@/lib/dedup/migration-apply';
|
||||
import { fetchSnapshot, loadNocoDbConfig } from '@/lib/dedup/nocodb-source';
|
||||
import { transformSnapshot } from '@/lib/dedup/migration-transform';
|
||||
import { resolveReportPaths, writeReport } from '@/lib/dedup/migration-report';
|
||||
@@ -31,6 +40,7 @@ interface CliArgs {
|
||||
apply: boolean;
|
||||
portSlug: string | null;
|
||||
reportDir: string | null;
|
||||
unsafeSkipRedirectCheck: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliArgs {
|
||||
@@ -39,6 +49,7 @@ function parseArgs(argv: string[]): CliArgs {
|
||||
apply: false,
|
||||
portSlug: null,
|
||||
reportDir: null,
|
||||
unsafeSkipRedirectCheck: false,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const a = argv[i]!;
|
||||
@@ -46,6 +57,7 @@ function parseArgs(argv: string[]): CliArgs {
|
||||
else if (a === '--apply') args.apply = true;
|
||||
else if (a === '--port-slug') args.portSlug = argv[++i] ?? null;
|
||||
else if (a === '--report') args.reportDir = argv[++i] ?? null;
|
||||
else if (a === '--unsafe-skip-redirect-check') args.unsafeSkipRedirectCheck = true;
|
||||
else if (a === '-h' || a === '--help') {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
@@ -64,20 +76,50 @@ function printHelp(): void {
|
||||
Pulls NocoDB → transforms → writes report to .migration/<timestamp>/.
|
||||
No database writes.
|
||||
|
||||
pnpm tsx scripts/migrate-from-nocodb.ts --apply --report .migration/<dir>/
|
||||
Apply phase. (Not yet implemented.)
|
||||
pnpm tsx scripts/migrate-from-nocodb.ts --apply --port-slug <slug>
|
||||
Re-fetches NocoDB, re-transforms, writes via migration_source_links
|
||||
ledger. Idempotent — safe to re-run. Requires EMAIL_REDIRECT_TO set
|
||||
(unless --unsafe-skip-redirect-check is also passed).
|
||||
|
||||
Flags:
|
||||
--dry-run Read NocoDB, write report only.
|
||||
--apply Actually write to the new DB. (Not yet supported.)
|
||||
--port-slug <slug> Port slug to attach to all imported entities.
|
||||
Defaults to the first available port if omitted.
|
||||
--report <dir> Path to a previously-generated report dir
|
||||
(only used by --apply).
|
||||
-h, --help Show this help.
|
||||
--dry-run Read NocoDB, write report only.
|
||||
--apply Actually write rows to the DB.
|
||||
--port-slug <slug> Port slug to attach to all imported
|
||||
entities. Defaults to the first
|
||||
available port if omitted.
|
||||
--report <dir> Path to a previously-generated report
|
||||
dir (only used by --apply).
|
||||
--unsafe-skip-redirect-check Skip the EMAIL_REDIRECT_TO precondition
|
||||
check. Only use in production cutover.
|
||||
-h, --help Show this help.
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the target port: use the slug if provided, otherwise the first
|
||||
* port found. Errors out cleanly if the slug doesn't match any port.
|
||||
*/
|
||||
async function resolvePort(slug: string | null): Promise<{ id: string; slug: string }> {
|
||||
if (slug) {
|
||||
const [p] = await db
|
||||
.select({ id: ports.id, slug: ports.slug })
|
||||
.from(ports)
|
||||
.where(eq(ports.slug, slug))
|
||||
.limit(1);
|
||||
if (!p) {
|
||||
console.error(`No port found with slug "${slug}".`);
|
||||
process.exit(1);
|
||||
}
|
||||
return { id: p.id, slug: p.slug };
|
||||
}
|
||||
const [first] = await db.select({ id: ports.id, slug: ports.slug }).from(ports).limit(1);
|
||||
if (!first) {
|
||||
console.error('No ports exist in the target DB. Seed at least one port before applying.');
|
||||
process.exit(1);
|
||||
}
|
||||
return { id: first.id, slug: first.slug };
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
@@ -87,13 +129,21 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (args.apply) {
|
||||
console.error('--apply is not yet implemented in this version. P3 ships dry-run first.');
|
||||
console.error('See docs/superpowers/specs/2026-05-03-dedup-and-migration-design.md §9.2.');
|
||||
// Safety gate: --apply must run with EMAIL_REDIRECT_TO set, unless the
|
||||
// operator explicitly opts out (production cutover).
|
||||
if (args.apply && !process.env.EMAIL_REDIRECT_TO && !args.unsafeSkipRedirectCheck) {
|
||||
console.error(
|
||||
'--apply requires EMAIL_REDIRECT_TO to be set in the environment as a safety net.',
|
||||
);
|
||||
console.error('See docs/operations/outbound-comms-safety.md for the rationale.');
|
||||
console.error(
|
||||
'If you are running the production cutover and have read that doc, add ' +
|
||||
'--unsafe-skip-redirect-check to override.',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// ── Dry-run path ───────────────────────────────────────────────────────────
|
||||
// ── Fetch + transform (shared by dry-run and apply) ──────────────────────
|
||||
|
||||
console.log('[migrate] Loading NocoDB config…');
|
||||
const config = loadNocoDbConfig();
|
||||
@@ -110,8 +160,7 @@ async function main(): Promise<void> {
|
||||
console.log('[migrate] Running transform + dedup pipeline…');
|
||||
const plan = transformSnapshot(snapshot);
|
||||
|
||||
// Resolve output paths relative to the worktree root (the script itself
|
||||
// lives in scripts/; we want the .migration dir at the repo root).
|
||||
// Resolve output paths relative to the worktree root.
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(scriptDir, '..');
|
||||
const generatedAt = new Date().toISOString();
|
||||
@@ -120,7 +169,7 @@ async function main(): Promise<void> {
|
||||
console.log(`[migrate] Writing report to ${paths.rootDir}…`);
|
||||
await writeReport(paths, plan, generatedAt);
|
||||
|
||||
// ── Console summary ──────────────────────────────────────────────────────
|
||||
// ── Plan summary ─────────────────────────────────────────────────────────
|
||||
const s = plan.stats;
|
||||
console.log('');
|
||||
console.log('=== Migration Plan Summary ===');
|
||||
@@ -135,6 +184,50 @@ async function main(): Promise<void> {
|
||||
console.log(` Quality: ${s.flaggedRows} rows flagged (see report.csv)`);
|
||||
console.log('');
|
||||
console.log(` Full report: ${paths.summaryPath}`);
|
||||
|
||||
if (args.dryRun) {
|
||||
console.log('');
|
||||
console.log('Dry-run complete. Re-run with --apply to write rows.');
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Apply path ───────────────────────────────────────────────────────────
|
||||
|
||||
const port = await resolvePort(args.portSlug);
|
||||
const applyId = randomUUID();
|
||||
|
||||
console.log('');
|
||||
console.log(`[migrate] Applying to port "${port.slug}" (id=${port.id})`);
|
||||
console.log(`[migrate] Apply id: ${applyId}`);
|
||||
console.log('[migrate] Inserting…');
|
||||
|
||||
const applyStart = Date.now();
|
||||
const result = await applyPlan(plan, { port, applyId });
|
||||
const applyElapsed = ((Date.now() - applyStart) / 1000).toFixed(1);
|
||||
|
||||
console.log('');
|
||||
console.log('=== Apply Result ===');
|
||||
console.log(` Time: ${applyElapsed}s`);
|
||||
console.log(
|
||||
` Clients: ${result.clientsInserted} inserted, ${result.clientsSkipped} already linked`,
|
||||
);
|
||||
console.log(` Contacts: ${result.contactsInserted} inserted`);
|
||||
console.log(` Addresses: ${result.addressesInserted} inserted`);
|
||||
console.log(` Yachts: ${result.yachtsInserted} inserted`);
|
||||
console.log(
|
||||
` Interests: ${result.interestsInserted} inserted, ${result.interestsSkipped} already linked`,
|
||||
);
|
||||
|
||||
if (result.warnings.length > 0) {
|
||||
console.log('');
|
||||
console.log('Warnings:');
|
||||
for (const w of result.warnings.slice(0, 20)) {
|
||||
console.log(` - ${w}`);
|
||||
}
|
||||
if (result.warnings.length > 20) {
|
||||
console.log(` … ${result.warnings.length - 20} more`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user