feat(import): data model for the bulk CSV/XLSX importer

First increment of the importer (docs/superpowers/specs/2026-06-01-bulk-import-design.md):
the three port-scoped tables, no changes to entity tables.

- import_batches — one row per run: entity_type, filename, storage_key,
  status, conflict_policy, mapping_json, live counts, created_by, timestamps.
- import_batch_rows — per-row action ledger (inserted/updated/skipped/errored)
  with entity_id + error; partial index on inserted rows powers Undo.
- import_mappings — saved column mappings, unique per (port, entity, name).

Migration 0090 applied via psql; schema re-exported from the index.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-01 22:23:50 +02:00
parent a343eaa257
commit 372b585bf9
3 changed files with 143 additions and 0 deletions

View File

@@ -0,0 +1,90 @@
/**
* Bulk CSV/XLSX importer tables.
*
* See docs/superpowers/specs/2026-06-01-bulk-import-design.md. Three tables,
* no changes to entity tables:
* - `import_batches` — one row per import run (header + live counts).
* - `import_batch_rows` — per-row action ledger; powers the error report
* and inserts-only Undo.
* - `import_mappings` — saved column mappings, reusable across runs.
*/
import { index, integer, jsonb, pgTable, text, timestamp, uniqueIndex } from 'drizzle-orm/pg-core';
import { ports } from './ports';
/** A saved or in-flight column mapping: target field key → source header. */
export type ImportMappingJson = Record<string, string>;
export const importBatches = pgTable(
'import_batches',
{
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
portId: text('port_id')
.notNull()
.references(() => ports.id, { onDelete: 'cascade' }),
/** Which adapter this batch targets (ImportEntityKey). */
entityType: text('entity_type').notNull(),
filename: text('filename').notNull(),
/** getStorageBackend() key of the uploaded file (re-read by the worker). */
storageKey: text('storage_key'),
/** uploaded | dry_run | committing | completed | failed | undone */
status: text('status').notNull().default('uploaded'),
/** skip-matches | update-matches | error-on-match */
conflictPolicy: text('conflict_policy').notNull().default('skip-matches'),
mappingJson: jsonb('mapping_json').$type<ImportMappingJson>(),
totalRows: integer('total_rows').notNull().default(0),
inserted: integer('inserted').notNull().default(0),
updated: integer('updated').notNull().default(0),
skipped: integer('skipped').notNull().default(0),
errored: integer('errored').notNull().default(0),
createdBy: text('created_by').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
completedAt: timestamp('completed_at', { withTimezone: true }),
},
(t) => [index('idx_import_batches_port').on(t.portId, t.createdAt)],
);
export const importBatchRows = pgTable(
'import_batch_rows',
{
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
batchId: text('batch_id')
.notNull()
.references(() => importBatches.id, { onDelete: 'cascade' }),
/** 1-based source row number (after the header). */
rowNumber: integer('row_number').notNull(),
/** inserted | updated | skipped | errored */
action: text('action').notNull(),
/** Created/updated entity id (null for skipped/errored). */
entityId: text('entity_id'),
error: text('error'),
},
(t) => [index('idx_import_batch_rows_batch').on(t.batchId, t.rowNumber)],
);
export const importMappings = pgTable(
'import_mappings',
{
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
portId: text('port_id')
.notNull()
.references(() => ports.id, { onDelete: 'cascade' }),
entityType: text('entity_type').notNull(),
name: text('name').notNull(),
mappingJson: jsonb('mapping_json').$type<ImportMappingJson>().notNull(),
createdBy: text('created_by').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [uniqueIndex('uniq_import_mappings_name').on(t.portId, t.entityType, t.name)],
);
export type ImportBatch = typeof importBatches.$inferSelect;
export type NewImportBatch = typeof importBatches.$inferInsert;
export type ImportBatchRow = typeof importBatchRows.$inferSelect;
export type ImportMapping = typeof importMappings.$inferSelect;

View File

@@ -65,6 +65,9 @@ export * from './gdpr';
// Migration ledger (one-shot scripts - NocoDB import etc.)
export * from './migration';
// Bulk CSV/XLSX importer (batches, per-row ledger, saved mappings)
export * from './imports';
// Website submissions (dual-write capture from the marketing site)
export * from './website-submissions';