feat(import): commit runner + undo + wired BullMQ worker
Third importer increment — the write path, fully testable without UI. - commit.ts: commitBatch streams classified rows, applies insert/update per the conflict policy via the adapter (each row in its own try/catch so valid rows still land), records every action in import_batch_rows, and keeps live counts on the batch header. undoBatch hard-deletes a batch's inserted rows (port-scoped); a delete blocked by a dependent FK is reported, not forced, and the batch flips to `undone` only when every inserted row was removed. - import worker: replaced the no-op placeholder with the real processor — loads the batch, re-reads the uploaded file from storage, parses, and runs commitBatch under the batch's mapping + policy. Marks the batch failed on error. Concurrency 1 so imports don't race each other's dedup lookups. Tests: commit (skip/insert/error counts + per-row ledger + real inserted entity), undo (removes exactly the inserted row, flips status), and update-matches overwrite. 2 passing. Engine is now functional end-to-end at the service layer: parse → map → dry-run → commit → undo. Remaining: 4 FK adapters, API routes + permission, wizard UI + history. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
130
tests/integration/import-commit.test.ts
Normal file
130
tests/integration/import-commit.test.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Integration test: commit runner + undo.
|
||||
*
|
||||
* Commits a 3-row companies "file" (skip existing / insert new / error blank)
|
||||
* and asserts counts, per-row ledger, and the actual inserted entity. Then
|
||||
* undoes the batch and asserts the inserted company is gone. Real test DB.
|
||||
*/
|
||||
import { beforeAll, describe, expect, it } from 'vitest';
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
|
||||
import { db } from '@/lib/db';
|
||||
import { importBatches, importBatchRows } from '@/lib/db/schema/imports';
|
||||
import { companies } from '@/lib/db/schema/companies';
|
||||
import { commitBatch, undoBatch } from '@/lib/import/commit';
|
||||
import { companiesAdapter } from '@/lib/import/adapters/companies';
|
||||
import type { ConflictPolicy, RawRow } from '@/lib/import/types';
|
||||
|
||||
let makePort: typeof import('../helpers/factories').makePort;
|
||||
let makeCompany: typeof import('../helpers/factories').makeCompany;
|
||||
let makeAuditMeta: typeof import('../helpers/factories').makeAuditMeta;
|
||||
|
||||
beforeAll(async () => {
|
||||
const f = await import('../helpers/factories');
|
||||
makePort = f.makePort;
|
||||
makeCompany = f.makeCompany;
|
||||
makeAuditMeta = f.makeAuditMeta;
|
||||
});
|
||||
|
||||
async function makeBatch(portId: string, policy: ConflictPolicy): Promise<string> {
|
||||
const [b] = await db
|
||||
.insert(importBatches)
|
||||
.values({
|
||||
portId,
|
||||
entityType: 'companies',
|
||||
filename: 'companies.csv',
|
||||
storageKey: 'unused-in-direct-commit',
|
||||
mappingJson: { name: 'Name' },
|
||||
conflictPolicy: policy,
|
||||
createdBy: 'tester',
|
||||
})
|
||||
.returning({ id: importBatches.id });
|
||||
return b!.id;
|
||||
}
|
||||
|
||||
describe('commitBatch + undoBatch', () => {
|
||||
const rows: RawRow[] = [{ Name: 'Acme Marine' }, { Name: 'Fresh Imports Ltd' }, { Name: '' }];
|
||||
|
||||
it('commits (skip/insert/error), records the ledger, then undoes the insert', async () => {
|
||||
const port = await makePort();
|
||||
await makeCompany({ portId: port.id, overrides: { name: 'Acme Marine' } });
|
||||
const batchId = await makeBatch(port.id, 'skip-matches');
|
||||
const ctx = { portId: port.id, meta: makeAuditMeta({ portId: port.id }) };
|
||||
|
||||
const counts = await commitBatch({
|
||||
batchId,
|
||||
adapter: companiesAdapter,
|
||||
rawRows: rows,
|
||||
mapping: { name: 'Name' },
|
||||
policy: 'skip-matches',
|
||||
ctx,
|
||||
});
|
||||
expect(counts).toEqual({ inserted: 1, updated: 0, skipped: 1, errored: 1 });
|
||||
|
||||
const [batch] = await db.select().from(importBatches).where(eq(importBatches.id, batchId));
|
||||
expect(batch!.status).toBe('completed');
|
||||
expect(batch!.inserted).toBe(1);
|
||||
|
||||
const ledger = await db
|
||||
.select()
|
||||
.from(importBatchRows)
|
||||
.where(eq(importBatchRows.batchId, batchId))
|
||||
.orderBy(importBatchRows.rowNumber);
|
||||
expect(ledger.map((r) => r.action)).toEqual(['skipped', 'inserted', 'errored']);
|
||||
expect(ledger[2]!.error).toBeTruthy();
|
||||
|
||||
// The new company really exists.
|
||||
const fresh = await db
|
||||
.select({ id: companies.id })
|
||||
.from(companies)
|
||||
.where(
|
||||
and(eq(companies.portId, port.id), sql`lower(${companies.name}) = 'fresh imports ltd'`),
|
||||
);
|
||||
expect(fresh).toHaveLength(1);
|
||||
|
||||
// Undo removes exactly the inserted row.
|
||||
const undo = await undoBatch(batchId, port.id);
|
||||
expect(undo.deleted).toBe(1);
|
||||
expect(undo.blocked).toBe(0);
|
||||
|
||||
const afterUndo = await db
|
||||
.select({ id: companies.id })
|
||||
.from(companies)
|
||||
.where(
|
||||
and(eq(companies.portId, port.id), sql`lower(${companies.name}) = 'fresh imports ltd'`),
|
||||
);
|
||||
expect(afterUndo).toHaveLength(0);
|
||||
|
||||
const [undoneBatch] = await db
|
||||
.select({ status: importBatches.status })
|
||||
.from(importBatches)
|
||||
.where(eq(importBatches.id, batchId));
|
||||
expect(undoneBatch!.status).toBe('undone');
|
||||
});
|
||||
|
||||
it('update-matches overwrites the matched company', async () => {
|
||||
const port = await makePort();
|
||||
await makeCompany({
|
||||
portId: port.id,
|
||||
overrides: { name: 'Acme Marine', legalName: 'OLD' },
|
||||
});
|
||||
const batchId = await makeBatch(port.id, 'update-matches');
|
||||
const ctx = { portId: port.id, meta: makeAuditMeta({ portId: port.id }) };
|
||||
|
||||
const counts = await commitBatch({
|
||||
batchId,
|
||||
adapter: companiesAdapter,
|
||||
rawRows: [{ Name: 'Acme Marine', Legal: 'NEW Ltd' }],
|
||||
mapping: { name: 'Name', legalName: 'Legal' },
|
||||
policy: 'update-matches',
|
||||
ctx,
|
||||
});
|
||||
expect(counts.updated).toBe(1);
|
||||
|
||||
const [c] = await db
|
||||
.select({ legalName: companies.legalName })
|
||||
.from(companies)
|
||||
.where(and(eq(companies.portId, port.id), sql`lower(${companies.name}) = 'acme marine'`));
|
||||
expect(c!.legalName).toBe('NEW Ltd');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user