letsbe/pn-new-crm
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(website-intake): dual-write endpoint + migration chain repair

Browse Source 
Adds website_submissions table + shared-secret POST endpoint so the
marketing site can dual-write inquiries alongside its NocoDB write.
Race-safe via INSERT ... ON CONFLICT, idempotent on submission_id,
refuses every request when WEBSITE_INTAKE_SECRET is unset. Also
repairs pre-existing 0020/0021/0022 prevId collision (renumbered +
journal re-sorted) so db:generate works again. 11 unit tests.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Ciaccio
2026-05-04 22:52:33 +02:00
parent c612bbdfd9
commit 49d34e00c8
16 changed files with 11556 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/lib/db/migrations/0021_unusual_azazel.sql → src/lib/db/migrations/0020_unusual_azazel.sql
View File

0
src/lib/db/migrations/0022_magenta_madame_hydra.sql → src/lib/db/migrations/0021_magenta_madame_hydra.sql
View File

0
src/lib/db/migrations/0020_medical_betty_brant.sql → src/lib/db/migrations/0022_medical_betty_brant.sql
View File

16
src/lib/db/migrations/0023_omniscient_reaper.sql Normal file
View File

@@ -0,0 +1,16 @@
CREATE TABLE "website_submissions" (
"id" text PRIMARY KEY NOT NULL,
"port_id" text NOT NULL,
"submission_id" text NOT NULL,
"kind" text NOT NULL,
"payload" jsonb NOT NULL,
"legacy_nocodb_id" text,
"source_ip" text,
"user_agent" text,
"received_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "website_submissions" ADD CONSTRAINT "website_submissions_port_id_ports_id_fk" FOREIGN KEY ("port_id") REFERENCES "public"."ports"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
CREATE UNIQUE INDEX "idx_ws_submission_id" ON "website_submissions" USING btree ("submission_id");--> statement-breakpoint
CREATE INDEX "idx_ws_port_received" ON "website_submissions" USING btree ("port_id","received_at");--> statement-breakpoint
CREATE INDEX "idx_ws_kind" ON "website_submissions" USING btree ("kind");

258
src/lib/db/migrations/meta/0020_snapshot.json
View File

@@ -1,5 +1,5 @@
{
"id": "e0e6a819-cf9f-45d3-b65b-19da27890f0b",
"id": "e9d830fc-ec81-42ab-bea6-232dd99d20d1",
"prevId": "6326a9a7-0b30-4647-bf86-b3d79e6a08bf",
"version": "7",
"dialect": "postgresql",
@@ -762,13 +762,13 @@
},
"nominal_boat_size": {
"name": "nominal_boat_size",
"type": "numeric",
"type": "text",
"primaryKey": false,
"notNull": false
},
"nominal_boat_size_m": {
"name": "nominal_boat_size_m",
"type": "numeric",
"type": "text",
"primaryKey": false,
"notNull": false
},
@@ -799,13 +799,13 @@
},
"power_capacity": {
"name": "power_capacity",
"type": "numeric",
"type": "text",
"primaryKey": false,
"notNull": false
},
"voltage": {
"name": "voltage",
"type": "numeric",
"type": "text",
"primaryKey": false,
"notNull": false
},
@@ -914,12 +914,6 @@
"primaryKey": false,
"notNull": false
},
"status_override_mode": {
"name": "status_override_mode",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
@@ -1339,6 +1333,158 @@
"checkConstraints": {},
"isRLSEnabled": false
},
"public.client_merge_candidates": {
"name": "client_merge_candidates",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"port_id": {
"name": "port_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"client_a_id": {
"name": "client_a_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"client_b_id": {
"name": "client_b_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"score": {
"name": "score",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"reasons": {
"name": "reasons",
"type": "jsonb",
"primaryKey": false,
"notNull": true
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'pending'"
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"resolved_at": {
"name": "resolved_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false
},
"resolved_by": {
"name": "resolved_by",
"type": "text",
"primaryKey": false,
"notNull": false
}
},
"indexes": {
"idx_cmc_port_status": {
"name": "idx_cmc_port_status",
"columns": [
{
"expression": "port_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "status",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_cmc_pair": {
"name": "idx_cmc_pair",
"columns": [
{
"expression": "port_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "client_a_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "client_b_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"client_merge_candidates_port_id_ports_id_fk": {
"name": "client_merge_candidates_port_id_ports_id_fk",
"tableFrom": "client_merge_candidates",
"tableTo": "ports",
"columnsFrom": ["port_id"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
},
"client_merge_candidates_client_a_id_clients_id_fk": {
"name": "client_merge_candidates_client_a_id_clients_id_fk",
"tableFrom": "client_merge_candidates",
"tableTo": "clients",
"columnsFrom": ["client_a_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
"client_merge_candidates_client_b_id_clients_id_fk": {
"name": "client_merge_candidates_client_b_id_clients_id_fk",
"tableFrom": "client_merge_candidates",
"tableTo": "clients",
"columnsFrom": ["client_b_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.client_merge_log": {
"name": "client_merge_log",
"schema": "",
@@ -10230,6 +10376,96 @@
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.migration_source_links": {
"name": "migration_source_links",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"source_system": {
"name": "source_system",
"type": "text",
"primaryKey": false,
"notNull": true
},
"source_id": {
"name": "source_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"target_entity_type": {
"name": "target_entity_type",
"type": "text",
"primaryKey": false,
"notNull": true
},
"target_entity_id": {
"name": "target_entity_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"applied_id": {
"name": "applied_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"applied_by": {
"name": "applied_by",
"type": "text",
"primaryKey": false,
"notNull": false
},
"applied_at": {
"name": "applied_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"idx_msl_source_target": {
"name": "idx_msl_source_target",
"columns": [
{
"expression": "source_system",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "source_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "target_entity_type",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
}
},
"enums": {},

25
src/lib/db/migrations/meta/0021_snapshot.json
View File

@@ -1,6 +1,6 @@
{
"id": "e9d830fc-ec81-42ab-bea6-232dd99d20d1",
"prevId": "6326a9a7-0b30-4647-bf86-b3d79e6a08bf",
"id": "9f6ae433-f075-4348-8109-3cd368344fa8",
"prevId": "e9d830fc-ec81-42ab-bea6-232dd99d20d1",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -1870,6 +1870,12 @@
"primaryKey": false,
"notNull": false
},
"merged_into_client_id": {
"name": "merged_into_client_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
@@ -1957,6 +1963,21 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_clients_merged_into": {
"name": "idx_clients_merged_into",
"columns": [
{
"expression": "merged_into_client_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {

18
src/lib/db/migrations/meta/0022_snapshot.json
View File

@@ -1,6 +1,6 @@
{
"id": "9f6ae433-f075-4348-8109-3cd368344fa8",
"prevId": "e9d830fc-ec81-42ab-bea6-232dd99d20d1",
"id": "e0e6a819-cf9f-45d3-b65b-19da27890f0b",
"prevId": "9f6ae433-f075-4348-8109-3cd368344fa8",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -762,13 +762,13 @@
},
"nominal_boat_size": {
"name": "nominal_boat_size",
"type": "text",
"type": "numeric",
"primaryKey": false,
"notNull": false
},
"nominal_boat_size_m": {
"name": "nominal_boat_size_m",
"type": "text",
"type": "numeric",
"primaryKey": false,
"notNull": false
},
@@ -799,13 +799,13 @@
},
"power_capacity": {
"name": "power_capacity",
"type": "text",
"type": "numeric",
"primaryKey": false,
"notNull": false
},
"voltage": {
"name": "voltage",
"type": "text",
"type": "numeric",
"primaryKey": false,
"notNull": false
},
@@ -914,6 +914,12 @@
"primaryKey": false,
"notNull": false
},
"status_override_mode": {
"name": "status_override_mode",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",

10639
src/lib/db/migrations/meta/0023_snapshot.json Normal file
View File

File diff suppressed because it is too large Load Diff

19
src/lib/db/migrations/meta/_journal.json
View File

@@ -145,22 +145,29 @@
{
"idx": 20,
"version": "7",
"when": 1777814682110,
"tag": "0020_medical_betty_brant",
"when": 1777811835982,
"tag": "0020_unusual_azazel",
"breakpoints": true
},
{
"idx": 21,
"version": "7",
"when": 1777811835982,
"tag": "0021_unusual_azazel",
"when": 1777812671833,
"tag": "0021_magenta_madame_hydra",
"breakpoints": true
},
{
"idx": 22,
"version": "7",
"when": 1777812671833,
"tag": "0022_magenta_madame_hydra",
"when": 1777814682110,
"tag": "0022_medical_betty_brant",
"breakpoints": true
},
{
"idx": 23,
"version": "7",
"when": 1777927586934,
"tag": "0023_omniscient_reaper",
"breakpoints": true
}
]

9
src/lib/db/schema/index.ts
View File

@@ -37,7 +37,7 @@ export * from './portal';
// CRM admin invites (better-auth realm)
export * from './crm-invites';
// Residential (parallel domain — separate clients & interests for the
// Residential (parallel domain - separate clients & interests for the
// external residential team)
export * from './residential';
@@ -56,8 +56,11 @@ export * from './ai-usage';
// GDPR export tracking (Phase 3d)
export * from './gdpr';
// Migration ledger (one-shot scripts — NocoDB import etc.)
// Migration ledger (one-shot scripts - NocoDB import etc.)
export * from './migration';
// Relations (must come last — references all tables)
// Website submissions (dual-write capture from the marketing site)
export * from './website-submissions';
// Relations (must come last - references all tables)
export * from './relations';

67
src/lib/db/schema/website-submissions.ts Normal file
View File

@@ -0,0 +1,67 @@
import { pgTable, text, jsonb, timestamp, index, uniqueIndex } from 'drizzle-orm/pg-core';
import { ports } from './ports';
/**
* Raw capture of every website inquiry submission, dual-written from the
* marketing site alongside its existing NocoDB write. Acts as a passive
* collector while the website still uses NocoDB as its primary system of
* record — the new CRM observes incoming traffic without altering it,
* letting us validate the data flow before any cutover.
*
* v1 deliberately stores the raw payload as JSON without promoting to
* `clients` / `interests` rows. Once we trust the pipeline, a separate
* "promote" job can transform these submissions into proper entities
* with full dedup / merge logic.
*
* Idempotency: each submission carries a `submission_id` UUID minted by
* the website's server. Re-delivery (network retry, double-click) hits
* the unique index and is treated as a no-op.
*/
export const websiteSubmissions = pgTable(
'website_submissions',
{
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
/** Multi-tenant: every submission belongs to a port. Resolved from
* `port_slug` in the request payload (defaults to port-nimara if
* unspecified, since it's currently the only port with a public
* marketing site). */
portId: text('port_id')
.notNull()
.references(() => ports.id),
/** UUID v4 minted by the website's server when the form is processed.
* The unique index on this column enforces idempotency: retries from
* the website resolve to the existing row instead of creating a
* duplicate. */
submissionId: text('submission_id').notNull(),
/** Discriminator for the form type. Mirrors the website's existing
* branches (`berth_inquiry` from /api/register?interest=berths,
* `residence_inquiry` from /api/register?interest=residences,
* `contact_form` from /api/contact). Add new kinds as the website
* grows new form types. */
kind: text('kind').notNull(),
/** Verbatim form payload, including any reCAPTCHA / IP / user-agent
* metadata the website chose to forward. Stored as JSONB so the
* capture stays schema-flexible while we figure out which fields
* matter for the eventual promote step. */
payload: jsonb('payload').notNull(),
/** Cross-reference back to the legacy NocoDB row id created by the
* same form submission. Useful for reconciling: pick any submission
* here, look up the matching NocoDB row, confirm both halves agree. */
legacyNocodbId: text('legacy_nocodb_id'),
/** Capture-time metadata for debugging. */
sourceIp: text('source_ip'),
userAgent: text('user_agent'),
receivedAt: timestamp('received_at', { withTimezone: true }).notNull().defaultNow(),
},
(table) => [
uniqueIndex('idx_ws_submission_id').on(table.submissionId),
index('idx_ws_port_received').on(table.portId, table.receivedAt),
index('idx_ws_kind').on(table.kind),
],
);
export type WebsiteSubmission = typeof websiteSubmissions.$inferSelect;
export type NewWebsiteSubmission = typeof websiteSubmissions.$inferInsert;

7
src/lib/env.ts
View File

@@ -50,6 +50,13 @@ const envSchema = z.object({
GOOGLE_CLIENT_ID: z.string().optional(),
GOOGLE_CLIENT_SECRET: z.string().optional(),
// Shared secret used by the marketing website's server-side dual-write
// helper (POST to /api/public/website-inquiries). Set the SAME value on
// the website's CRM_INTAKE_SECRET env. Leave unset in dev/staging until
// the website's CRM_INTAKE_URL is also set — without this, the public
// intake endpoint refuses every request.
WEBSITE_INTAKE_SECRET: z.string().min(16).optional(),
// OpenAI (optional)
OPENAI_API_KEY: z.string().optional(),

6
src/lib/rate-limit.ts
View File

@@ -85,6 +85,12 @@ export const rateLimiters = {
exports: { windowMs: 60 * 60 * 1000, max: 30, keyPrefix: 'export' },
/** Public unauthenticated form posts (interest, residential inquiry): 5 per hour per IP. */
publicForm: { windowMs: 60 * 60 * 1000, max: 5, keyPrefix: 'publicform' },
/** Server-to-server intake from the marketing website's dual-write helper.
* All traffic shares the website's egress IP, so the bucket has to
* accommodate every legitimate inquiry the site can produce in an hour
* without dropping data. The shared-secret header gates abuse; this
* limiter is just a defensive backstop in case the secret leaks. */
websiteIntake: { windowMs: 60 * 60 * 1000, max: 500, keyPrefix: 'websiteintake' },
} as const satisfies Record<string, RateLimitConfig>;
export type RateLimiterName = keyof typeof rateLimiters;