From 372b585bf97f77ce6e76dc29a58e797987a3a159 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 1 Jun 2026 22:23:50 +0200 Subject: [PATCH] feat(import): data model for the bulk CSV/XLSX importer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First increment of the importer (docs/superpowers/specs/2026-06-01-bulk-import-design.md): the three port-scoped tables, no changes to entity tables. - import_batches — one row per run: entity_type, filename, storage_key, status, conflict_policy, mapping_json, live counts, created_by, timestamps. - import_batch_rows — per-row action ledger (inserted/updated/skipped/errored) with entity_id + error; partial index on inserted rows powers Undo. - import_mappings — saved column mappings, unique per (port, entity, name). Migration 0090 applied via psql; schema re-exported from the index. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/db/migrations/0090_bulk_import.sql | 50 ++++++++++++ src/lib/db/schema/imports.ts | 90 ++++++++++++++++++++++ src/lib/db/schema/index.ts | 3 + 3 files changed, 143 insertions(+) create mode 100644 src/lib/db/migrations/0090_bulk_import.sql create mode 100644 src/lib/db/schema/imports.ts diff --git a/src/lib/db/migrations/0090_bulk_import.sql b/src/lib/db/migrations/0090_bulk_import.sql new file mode 100644 index 00000000..022eaab7 --- /dev/null +++ b/src/lib/db/migrations/0090_bulk_import.sql @@ -0,0 +1,50 @@ +-- 0090_bulk_import.sql +-- +-- Bulk CSV/XLSX importer (docs/superpowers/specs/2026-06-01-bulk-import-design.md). +-- Three new tables, no changes to entity tables: a per-import batch header, +-- a per-row action/undo ledger, and saved column mappings. All port-scoped. +-- IDs are app-generated (crypto.randomUUID via Drizzle $defaultFn), so no DB +-- default on the PK columns — matches the rest of the schema. + +CREATE TABLE import_batches ( + id text PRIMARY KEY, + port_id text NOT NULL REFERENCES ports(id) ON DELETE CASCADE, + entity_type text NOT NULL, + filename text NOT NULL, + storage_key text, + status text NOT NULL DEFAULT 'uploaded', + conflict_policy text NOT NULL DEFAULT 'skip-matches', + mapping_json jsonb, + total_rows integer NOT NULL DEFAULT 0, + inserted integer NOT NULL DEFAULT 0, + updated integer NOT NULL DEFAULT 0, + skipped integer NOT NULL DEFAULT 0, + errored integer NOT NULL DEFAULT 0, + created_by text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + completed_at timestamptz +); +CREATE INDEX idx_import_batches_port ON import_batches (port_id, created_at DESC); + +CREATE TABLE import_batch_rows ( + id text PRIMARY KEY, + batch_id text NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE, + row_number integer NOT NULL, + action text NOT NULL, + entity_id text, + error text +); +CREATE INDEX idx_import_batch_rows_batch ON import_batch_rows (batch_id, row_number); +-- Undo walks only the rows a batch INSERTED; partial index keeps that scan tight. +CREATE INDEX idx_import_batch_rows_inserted ON import_batch_rows (batch_id) WHERE action = 'inserted'; + +CREATE TABLE import_mappings ( + id text PRIMARY KEY, + port_id text NOT NULL REFERENCES ports(id) ON DELETE CASCADE, + entity_type text NOT NULL, + name text NOT NULL, + mapping_json jsonb NOT NULL, + created_by text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX uniq_import_mappings_name ON import_mappings (port_id, entity_type, name); diff --git a/src/lib/db/schema/imports.ts b/src/lib/db/schema/imports.ts new file mode 100644 index 00000000..a762dbbd --- /dev/null +++ b/src/lib/db/schema/imports.ts @@ -0,0 +1,90 @@ +/** + * Bulk CSV/XLSX importer tables. + * + * See docs/superpowers/specs/2026-06-01-bulk-import-design.md. Three tables, + * no changes to entity tables: + * - `import_batches` — one row per import run (header + live counts). + * - `import_batch_rows` — per-row action ledger; powers the error report + * and inserts-only Undo. + * - `import_mappings` — saved column mappings, reusable across runs. + */ +import { index, integer, jsonb, pgTable, text, timestamp, uniqueIndex } from 'drizzle-orm/pg-core'; + +import { ports } from './ports'; + +/** A saved or in-flight column mapping: target field key → source header. */ +export type ImportMappingJson = Record; + +export const importBatches = pgTable( + 'import_batches', + { + id: text('id') + .primaryKey() + .$defaultFn(() => crypto.randomUUID()), + portId: text('port_id') + .notNull() + .references(() => ports.id, { onDelete: 'cascade' }), + /** Which adapter this batch targets (ImportEntityKey). */ + entityType: text('entity_type').notNull(), + filename: text('filename').notNull(), + /** getStorageBackend() key of the uploaded file (re-read by the worker). */ + storageKey: text('storage_key'), + /** uploaded | dry_run | committing | completed | failed | undone */ + status: text('status').notNull().default('uploaded'), + /** skip-matches | update-matches | error-on-match */ + conflictPolicy: text('conflict_policy').notNull().default('skip-matches'), + mappingJson: jsonb('mapping_json').$type(), + totalRows: integer('total_rows').notNull().default(0), + inserted: integer('inserted').notNull().default(0), + updated: integer('updated').notNull().default(0), + skipped: integer('skipped').notNull().default(0), + errored: integer('errored').notNull().default(0), + createdBy: text('created_by').notNull(), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + completedAt: timestamp('completed_at', { withTimezone: true }), + }, + (t) => [index('idx_import_batches_port').on(t.portId, t.createdAt)], +); + +export const importBatchRows = pgTable( + 'import_batch_rows', + { + id: text('id') + .primaryKey() + .$defaultFn(() => crypto.randomUUID()), + batchId: text('batch_id') + .notNull() + .references(() => importBatches.id, { onDelete: 'cascade' }), + /** 1-based source row number (after the header). */ + rowNumber: integer('row_number').notNull(), + /** inserted | updated | skipped | errored */ + action: text('action').notNull(), + /** Created/updated entity id (null for skipped/errored). */ + entityId: text('entity_id'), + error: text('error'), + }, + (t) => [index('idx_import_batch_rows_batch').on(t.batchId, t.rowNumber)], +); + +export const importMappings = pgTable( + 'import_mappings', + { + id: text('id') + .primaryKey() + .$defaultFn(() => crypto.randomUUID()), + portId: text('port_id') + .notNull() + .references(() => ports.id, { onDelete: 'cascade' }), + entityType: text('entity_type').notNull(), + name: text('name').notNull(), + mappingJson: jsonb('mapping_json').$type().notNull(), + createdBy: text('created_by').notNull(), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [uniqueIndex('uniq_import_mappings_name').on(t.portId, t.entityType, t.name)], +); + +export type ImportBatch = typeof importBatches.$inferSelect; +export type NewImportBatch = typeof importBatches.$inferInsert; +export type ImportBatchRow = typeof importBatchRows.$inferSelect; +export type ImportMapping = typeof importMappings.$inferSelect; diff --git a/src/lib/db/schema/index.ts b/src/lib/db/schema/index.ts index 852bc084..945ffbbf 100644 --- a/src/lib/db/schema/index.ts +++ b/src/lib/db/schema/index.ts @@ -65,6 +65,9 @@ export * from './gdpr'; // Migration ledger (one-shot scripts - NocoDB import etc.) export * from './migration'; +// Bulk CSV/XLSX importer (batches, per-row ledger, saved mappings) +export * from './imports'; + // Website submissions (dual-write capture from the marketing site) export * from './website-submissions';