feat(migration): document backfill — legacy MinIO → CRM storage (Phase 2)
backfill-documents.ts pulls signed EOI PDFs + berth spec PDFs from the legacy MinIO (client-portal bucket; read-only via dedicated LEGACY_MINIO_* creds) and deposits them into the CRM (getStorageBackend), linking: - berth PDFs → berth_pdf_versions + berths.current_pdf_version_id (mooring from filename; 113/113 matched) - signed EOIs → documents.signed_file_id + status=completed + a files row filed into the client folder (exact name + conservative lev<=2 fuzzy; 33 linked) Idempotent (skips when signedFileId / current_pdf_version_id already set). Strictly prod-READ-only; all writes local (dev storage_backend=filesystem). Unmatched EOIs reported (mostly in-flight deals w/ no signed PDF yet + old-LOI docs in the NocoDB attachment bucket). Adds probe-minio.ts (read-only bucket inventory). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
102
scripts/migration/probe-minio.ts
Normal file
102
scripts/migration/probe-minio.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Read-only MinIO inventory for the legacy → new-CRM migration (Phase 2 sizing).
|
||||
*
|
||||
* Lists every bucket the creds can see, then for the document buckets
|
||||
* (`client-portal`, `signatures`) groups objects by top-level prefix with
|
||||
* counts + sizes + samples — so we can see exactly where the EOIs, berth
|
||||
* PDFs, receipts and business-card images live before backfilling them.
|
||||
*
|
||||
* Secret-free: reads creds from env. Run with:
|
||||
* MINIO_ACCESS_KEY=... MINIO_SECRET_KEY=... \
|
||||
* pnpm tsx scripts/migration/probe-minio.ts
|
||||
*
|
||||
* Strictly read-only (listBuckets + listObjectsV2). No writes.
|
||||
*/
|
||||
import { Client } from 'minio';
|
||||
|
||||
const endPoint = process.env.MINIO_ENDPOINT || 's3.portnimara.com';
|
||||
const accessKey = process.env.MINIO_ACCESS_KEY;
|
||||
const secretKey = process.env.MINIO_SECRET_KEY;
|
||||
|
||||
if (!accessKey || !secretKey) {
|
||||
console.error('Set MINIO_ACCESS_KEY and MINIO_SECRET_KEY');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const client = new Client({ endPoint, port: 443, useSSL: true, accessKey, secretKey });
|
||||
|
||||
interface PrefixStat {
|
||||
count: number;
|
||||
bytes: number;
|
||||
samples: string[];
|
||||
}
|
||||
|
||||
async function inventory(bucket: string) {
|
||||
const byPrefix = new Map<string, PrefixStat>();
|
||||
let total = 0;
|
||||
let totalBytes = 0;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const stream = client.listObjectsV2(bucket, '', true);
|
||||
stream.on('data', (o) => {
|
||||
if (!o.name) return;
|
||||
total++;
|
||||
totalBytes += o.size || 0;
|
||||
const top = o.name.includes('/') ? o.name.split('/')[0] + '/' : '(root)';
|
||||
const e = byPrefix.get(top) || { count: 0, bytes: 0, samples: [] };
|
||||
e.count++;
|
||||
e.bytes += o.size || 0;
|
||||
if (e.samples.length < 4) e.samples.push(`${o.name} (${o.size}b)`);
|
||||
byPrefix.set(top, e);
|
||||
});
|
||||
stream.on('end', () => resolve());
|
||||
stream.on('error', reject);
|
||||
});
|
||||
return { bucket, total, totalBytes, byPrefix };
|
||||
}
|
||||
|
||||
const mb = (b: number) => (b / 1e6).toFixed(1);
|
||||
|
||||
async function main() {
|
||||
console.log(`MinIO @ ${endPoint}\n`);
|
||||
|
||||
let buckets: string[] = [];
|
||||
try {
|
||||
const list = await client.listBuckets();
|
||||
buckets = list.map((b) => b.name);
|
||||
console.log('=== all buckets visible to these creds ===');
|
||||
for (const b of list) console.log(` ${b.name}`);
|
||||
} catch (err) {
|
||||
console.log(`listBuckets failed: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const targets = (process.env.MINIO_BUCKETS || 'client-portal,signatures')
|
||||
.split(',')
|
||||
.map((s) => s.trim());
|
||||
|
||||
for (const bucket of targets) {
|
||||
if (buckets.length && !buckets.includes(bucket)) {
|
||||
console.log(`\n=== bucket: ${bucket} — NOT VISIBLE to these creds ===`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const inv = await inventory(bucket);
|
||||
console.log(
|
||||
`\n=== bucket: ${inv.bucket} — ${inv.total} objects, ${mb(inv.totalBytes)} MB ===`,
|
||||
);
|
||||
const rows = [...inv.byPrefix.entries()].sort((a, z) => z[1].count - a[1].count);
|
||||
for (const [prefix, e] of rows) {
|
||||
console.log(
|
||||
` ${prefix.padEnd(30)} ${String(e.count).padStart(5)} obj ${mb(e.bytes).padStart(8)} MB`,
|
||||
);
|
||||
for (const s of e.samples) console.log(` e.g. ${s}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`\n=== bucket: ${bucket} — ERROR: ${(err as Error).message} ===`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('probe-minio failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user