diff --git a/server/api/admin/duplicates/find.ts b/server/api/admin/duplicates/find.ts
index de96c8c..c1d51cc 100644
--- a/server/api/admin/duplicates/find.ts
+++ b/server/api/admin/duplicates/find.ts
@@ -1,12 +1,12 @@
-import { requireAuth } from '~/server/utils/auth';
+import { requireAuth, requireSalesOrAdmin } from '~/server/utils/auth';
import { getNocoDbConfiguration } from '~/server/utils/nocodb';
export default defineEventHandler(async (event) => {
console.log('[DUPLICATES] Find duplicates request');
try {
- // Require authentication (any authenticated user with interest access)
- await requireAuth(event);
+ // Require sales or admin access for duplicate detection
+ await requireSalesOrAdmin(event);
const query = getQuery(event);
const threshold = query.threshold ? parseFloat(query.threshold as string) : 0.8;
@@ -121,22 +121,22 @@ function calculateSimilarity(interest1: any, interest2: any) {
const scores: Array<{ type: string; score: number; weight: number }> = [];
// Email similarity (highest weight)
- if (interest1.Email && interest2.Email) {
- const emailScore = interest1.Email.toLowerCase() === interest2.Email.toLowerCase() ? 1.0 : 0.0;
+ if (interest1['Email Address'] && interest2['Email Address']) {
+ const emailScore = interest1['Email Address'].toLowerCase() === interest2['Email Address'].toLowerCase() ? 1.0 : 0.0;
scores.push({ type: 'email', score: emailScore, weight: 0.4 });
}
// Phone similarity
- if (interest1.Phone && interest2.Phone) {
- const phone1 = normalizePhone(interest1.Phone);
- const phone2 = normalizePhone(interest2.Phone);
+ if (interest1['Phone Number'] && interest2['Phone Number']) {
+ const phone1 = normalizePhone(interest1['Phone Number']);
+ const phone2 = normalizePhone(interest2['Phone Number']);
const phoneScore = phone1 === phone2 ? 1.0 : 0.0;
scores.push({ type: 'phone', score: phoneScore, weight: 0.3 });
}
// Name similarity
- if (interest1.Name && interest2.Name) {
- const nameScore = calculateNameSimilarity(interest1.Name, interest2.Name);
+ if (interest1['Full Name'] && interest2['Full Name']) {
+ const nameScore = calculateNameSimilarity(interest1['Full Name'], interest2['Full Name']);
scores.push({ type: 'name', score: nameScore, weight: 0.2 });
}
@@ -237,7 +237,7 @@ function selectMasterCandidate(interests: any[]) {
* Calculate completeness score for an interest record
*/
function calculateCompletenessScore(interest: any): number {
- const fields = ['Name', 'Email', 'Phone', 'Address', 'Comments', 'BerthRequirements'];
+ const fields = ['Full Name', 'Email Address', 'Phone Number', 'Address', 'Extra Comments', 'Berth Size Desired'];
const filledFields = fields.filter(field =>
interest[field] && interest[field].toString().trim().length > 0
);
@@ -245,8 +245,8 @@ function calculateCompletenessScore(interest: any): number {
let score = filledFields.length / fields.length;
// Bonus for recent creation
- if (interest.CreatedAt) {
- const created = new Date(interest.CreatedAt);
+ if (interest['Created At']) {
+ const created = new Date(interest['Created At']);
const now = new Date();
const daysOld = (now.getTime() - created.getTime()) / (1000 * 60 * 60 * 24);
diff --git a/server/api/create-expense.ts b/server/api/create-expense.ts
index 94c504e..3d99bd6 100644
--- a/server/api/create-expense.ts
+++ b/server/api/create-expense.ts
@@ -1,5 +1,5 @@
import { requireSalesOrAdmin } from '@/server/utils/auth';
-import { getNocoDbConfiguration } from '@/server/utils/nocodb';
+import { getNocoDbConfiguration, normalizePersonName } from '@/server/utils/nocodb';
import type { Expense } from '@/utils/types';
export default defineEventHandler(async (event) => {
@@ -30,7 +30,7 @@ export default defineEventHandler(async (event) => {
"Establishment Name": body["Establishment Name"],
Price: body.Price,
Category: body.Category,
- Payer: body.Payer,
+ Payer: normalizePersonName(body.Payer), // Normalize the payer name
Time: body.Time,
Contents: body.Contents || null,
"Payment Method": body["Payment Method"] || "Card",
diff --git a/server/api/expenses/duplicates/find.ts b/server/api/expenses/duplicates/find.ts
new file mode 100644
index 0000000..d640643
--- /dev/null
+++ b/server/api/expenses/duplicates/find.ts
@@ -0,0 +1,326 @@
+import { requireSalesOrAdmin } from '~/server/utils/auth';
+import { getNocoDbConfiguration, normalizePersonName } from '~/server/utils/nocodb';
+import type { Expense } from '~/utils/types';
+
+export default defineEventHandler(async (event) => {
+ console.log('[EXPENSES] Find duplicate expenses request');
+
+ try {
+ // Require sales or admin access
+ await requireSalesOrAdmin(event);
+
+ const query = getQuery(event);
+ const dateRange = query.dateRange as string || '30'; // Default to last 30 days
+
+ // Calculate date range
+ const endDate = new Date();
+ const startDate = new Date();
+ startDate.setDate(startDate.getDate() - parseInt(dateRange));
+
+ // Get expenses from NocoDB
+ const config = getNocoDbConfiguration();
+ const expenseTableId = "mxfcefkk4dqs6uq";
+
+ const response = await $fetch(`${config.url}/api/v2/tables/${expenseTableId}/records`, {
+ headers: {
+ 'xc-token': config.token
+ },
+ params: {
+ limit: 5000,
+ where: `(Time,gte,${startDate.toISOString().split('T')[0]})~and(Time,lte,${endDate.toISOString().split('T')[0]})`,
+ sort: '-Time'
+ }
+ }) as any;
+
+ const expenses = response.list || [];
+ console.log('[EXPENSES] Analyzing', expenses.length, 'expenses for duplicates');
+
+ // Find duplicate groups
+ const duplicateGroups = findDuplicateExpenses(expenses);
+
+ // Also find payer name variations
+ const payerVariations = findPayerNameVariations(expenses);
+
+ console.log('[EXPENSES] Found', duplicateGroups.length, 'duplicate groups and', payerVariations.length, 'payer variations');
+
+ return {
+ success: true,
+ data: {
+ duplicateGroups,
+ payerVariations,
+ totalExpenses: expenses.length,
+ duplicateCount: duplicateGroups.reduce((sum, group) => sum + group.expenses.length, 0),
+ dateRange: {
+ start: startDate.toISOString().split('T')[0],
+ end: endDate.toISOString().split('T')[0]
+ }
+ }
+ };
+
+ } catch (error: any) {
+ console.error('[EXPENSES] Failed to find duplicates:', error);
+
+ if (error.statusCode === 403) {
+ throw createError({
+ statusCode: 403,
+ statusMessage: 'Access denied. Sales or admin role required.'
+ });
+ }
+
+ throw createError({
+ statusCode: 500,
+ statusMessage: 'Failed to find duplicate expenses'
+ });
+ }
+});
+
+/**
+ * Find duplicate expenses based on multiple criteria
+ */
+function findDuplicateExpenses(expenses: any[]) {
+ const duplicateGroups: Array<{
+ id: string;
+ expenses: any[];
+ matchReason: string;
+ confidence: number;
+ masterCandidate: any;
+ }> = [];
+
+ const processedIds = new Set();
+
+ for (let i = 0; i < expenses.length; i++) {
+ const expense1 = expenses[i];
+
+ if (processedIds.has(expense1.Id)) continue;
+
+ const matches = [expense1];
+ let matchReasons = new Set();
+
+ for (let j = i + 1; j < expenses.length; j++) {
+ const expense2 = expenses[j];
+
+ if (processedIds.has(expense2.Id)) continue;
+
+ const similarity = calculateExpenseSimilarity(expense1, expense2);
+
+ if (similarity.score >= 0.8) {
+ matches.push(expense2);
+ processedIds.add(expense2.Id);
+ similarity.reasons.forEach(r => matchReasons.add(r));
+ }
+ }
+
+ if (matches.length > 1) {
+ // Mark all as processed
+ matches.forEach(match => processedIds.add(match.Id));
+
+ // Determine the best master candidate
+ const masterCandidate = selectMasterExpense(matches);
+
+ duplicateGroups.push({
+ id: `group_${duplicateGroups.length + 1}`,
+ expenses: matches,
+ matchReason: Array.from(matchReasons).join(', '),
+ confidence: Math.max(...matches.slice(1).map(match =>
+ calculateExpenseSimilarity(masterCandidate, match).score
+ )),
+ masterCandidate
+ });
+ }
+ }
+
+ return duplicateGroups;
+}
+
+/**
+ * Find payer name variations (like "Abbie" vs "abbie")
+ */
+function findPayerNameVariations(expenses: any[]) {
+ const payerMap = new Map>();
+
+ // Group payers by normalized name
+ expenses.forEach(expense => {
+ if (expense.Payer) {
+ const normalized = normalizePersonName(expense.Payer);
+ if (!payerMap.has(normalized)) {
+ payerMap.set(normalized, new Set());
+ }
+ payerMap.get(normalized)!.add(expense.Payer);
+ }
+ });
+
+ // Find variations
+ const variations: Array<{
+ normalizedName: string;
+ variations: string[];
+ expenseCount: number;
+ }> = [];
+
+ payerMap.forEach((variationSet, normalized) => {
+ if (variationSet.size > 1) {
+ const variationArray = Array.from(variationSet);
+ const expenseCount = expenses.filter(e =>
+ e.Payer && normalizePersonName(e.Payer) === normalized
+ ).length;
+
+ variations.push({
+ normalizedName: normalized,
+ variations: variationArray,
+ expenseCount
+ });
+ }
+ });
+
+ return variations.sort((a, b) => b.expenseCount - a.expenseCount);
+}
+
+/**
+ * Calculate similarity between two expenses
+ */
+function calculateExpenseSimilarity(expense1: any, expense2: any) {
+ const scores: Array<{ type: string; score: number; weight: number }> = [];
+ const reasons: string[] = [];
+
+ // Exact match on establishment, price, and date (highest weight for true duplicates)
+ if (expense1['Establishment Name'] === expense2['Establishment Name'] &&
+ expense1.Price === expense2.Price &&
+ expense1.Time === expense2.Time) {
+ scores.push({ type: 'exact', score: 1.0, weight: 0.5 });
+ reasons.push('Exact match');
+ }
+
+ // Same payer, establishment, and price on same day (likely duplicate)
+ const date1 = expense1.Time?.split('T')[0];
+ const date2 = expense2.Time?.split('T')[0];
+
+ if (normalizePersonName(expense1.Payer) === normalizePersonName(expense2.Payer) &&
+ expense1['Establishment Name'] === expense2['Establishment Name'] &&
+ expense1.Price === expense2.Price &&
+ date1 === date2) {
+ scores.push({ type: 'same-day', score: 0.95, weight: 0.4 });
+ reasons.push('Same person, place, amount on same day');
+ }
+
+ // Similar establishment names with same price and payer
+ if (expense1['Establishment Name'] && expense2['Establishment Name']) {
+ const nameSimilarity = calculateStringSimilarity(
+ expense1['Establishment Name'],
+ expense2['Establishment Name']
+ );
+
+ if (nameSimilarity > 0.8 &&
+ expense1.Price === expense2.Price &&
+ normalizePersonName(expense1.Payer) === normalizePersonName(expense2.Payer)) {
+ scores.push({ type: 'similar', score: nameSimilarity, weight: 0.3 });
+ reasons.push('Similar establishment name');
+ }
+ }
+
+ // Time proximity check (within 5 minutes)
+ if (expense1.Time && expense2.Time) {
+ const time1 = new Date(expense1.Time).getTime();
+ const time2 = new Date(expense2.Time).getTime();
+ const timeDiff = Math.abs(time1 - time2);
+
+ if (timeDiff < 5 * 60 * 1000 && // 5 minutes
+ expense1['Establishment Name'] === expense2['Establishment Name']) {
+ scores.push({ type: 'time-proximity', score: 0.9, weight: 0.2 });
+ reasons.push('Within 5 minutes at same establishment');
+ }
+ }
+
+ // Calculate weighted average
+ const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
+ const weightedScore = totalWeight > 0
+ ? scores.reduce((sum, s) => sum + (s.score * s.weight), 0) / totalWeight
+ : 0;
+
+ return {
+ score: weightedScore,
+ reasons,
+ details: scores
+ };
+}
+
+/**
+ * Calculate string similarity using Levenshtein distance
+ */
+function calculateStringSimilarity(str1: string, str2: string): number {
+ const s1 = str1.toLowerCase().trim();
+ const s2 = str2.toLowerCase().trim();
+
+ if (s1 === s2) return 1.0;
+
+ const distance = levenshteinDistance(s1, s2);
+ const maxLength = Math.max(s1.length, s2.length);
+
+ return maxLength > 0 ? 1 - (distance / maxLength) : 0;
+}
+
+/**
+ * Calculate Levenshtein distance between two strings
+ */
+function levenshteinDistance(str1: string, str2: string): number {
+ const matrix = Array(str2.length + 1).fill(null).map(() => Array(str1.length + 1).fill(null));
+
+ for (let i = 0; i <= str1.length; i += 1) {
+ matrix[0][i] = i;
+ }
+
+ for (let j = 0; j <= str2.length; j += 1) {
+ matrix[j][0] = j;
+ }
+
+ for (let j = 1; j <= str2.length; j += 1) {
+ for (let i = 1; i <= str1.length; i += 1) {
+ const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
+ matrix[j][i] = Math.min(
+ matrix[j][i - 1] + 1, // deletion
+ matrix[j - 1][i] + 1, // insertion
+ matrix[j - 1][i - 1] + indicator // substitution
+ );
+ }
+ }
+
+ return matrix[str2.length][str1.length];
+}
+
+/**
+ * Select the best master expense from a group
+ */
+function selectMasterExpense(expenses: any[]) {
+ return expenses.reduce((best, current) => {
+ const bestScore = calculateExpenseCompletenessScore(best);
+ const currentScore = calculateExpenseCompletenessScore(current);
+
+ return currentScore > bestScore ? current : best;
+ });
+}
+
+/**
+ * Calculate completeness score for an expense
+ */
+function calculateExpenseCompletenessScore(expense: any): number {
+ const fields = ['Establishment Name', 'Price', 'Payer', 'Category', 'Contents', 'Time'];
+ const filledFields = fields.filter(field =>
+ expense[field] && expense[field].toString().trim().length > 0
+ );
+
+ let score = filledFields.length / fields.length;
+
+ // Bonus for having contents description
+ if (expense.Contents && expense.Contents.length > 10) {
+ score += 0.2;
+ }
+
+ // Bonus for recent creation (more likely to be accurate)
+ if (expense.CreatedAt) {
+ const created = new Date(expense.CreatedAt);
+ const now = new Date();
+ const hoursOld = (now.getTime() - created.getTime()) / (1000 * 60 * 60);
+
+ if (hoursOld < 24) score += 0.1;
+ }
+
+ return Math.min(score, 1.0);
+}
diff --git a/server/api/expenses/duplicates/merge.ts b/server/api/expenses/duplicates/merge.ts
new file mode 100644
index 0000000..2ee1cc0
--- /dev/null
+++ b/server/api/expenses/duplicates/merge.ts
@@ -0,0 +1,190 @@
+import { requireSalesOrAdmin } from '~/server/utils/auth';
+import { getNocoDbConfiguration, normalizePersonName } from '~/server/utils/nocodb';
+import { logAuditEvent } from '~/server/utils/audit-logger';
+
+export default defineEventHandler(async (event) => {
+ console.log('[EXPENSES] Merge duplicates request');
+
+ let body: any;
+
+ try {
+ // Require sales or admin access
+ const session = await requireSalesOrAdmin(event);
+ const userId = session.user?.id || 'unknown';
+ const userEmail = session.user?.email || 'unknown';
+
+ body = await readBody(event);
+ const { masterId, duplicateIds, mergeData } = body;
+
+ if (!masterId || !duplicateIds || !Array.isArray(duplicateIds) || duplicateIds.length === 0) {
+ throw createError({
+ statusCode: 400,
+ statusMessage: 'Invalid merge request. Master ID and duplicate IDs required.'
+ });
+ }
+
+ console.log('[EXPENSES] Merging', duplicateIds.length, 'duplicates into master expense', masterId);
+
+ // Get NocoDB configuration
+ const config = getNocoDbConfiguration();
+ const expenseTableId = "mxfcefkk4dqs6uq";
+
+ // First, get all expenses involved
+ const allIds = [masterId, ...duplicateIds];
+ const expensesToMerge: any[] = [];
+
+ for (const id of allIds) {
+ try {
+ const expense = await $fetch(`${config.url}/api/v2/tables/${expenseTableId}/records/${id}`, {
+ headers: {
+ 'xc-token': config.token
+ }
+ });
+ expensesToMerge.push(expense);
+ } catch (error) {
+ console.error(`[EXPENSES] Failed to fetch expense ${id}:`, error);
+ }
+ }
+
+ if (expensesToMerge.length < 2) {
+ throw createError({
+ statusCode: 404,
+ statusMessage: 'Could not find enough expenses to merge'
+ });
+ }
+
+ const masterExpense = expensesToMerge.find(e => e.Id === parseInt(masterId));
+ if (!masterExpense) {
+ throw createError({
+ statusCode: 404,
+ statusMessage: 'Master expense not found'
+ });
+ }
+
+ // Log the action before making changes
+ await logAuditEvent(event, 'MERGE_EXPENSE_DUPLICATES', 'expense', {
+ resourceId: masterId,
+ changes: {
+ duplicateIds,
+ mergeData,
+ originalExpenses: expensesToMerge
+ }
+ });
+
+ // Update master expense with merged data
+ if (mergeData) {
+ const updateData: any = { Id: parseInt(masterId) };
+
+ // Normalize payer name if provided
+ if (mergeData.Payer) {
+ updateData.Payer = normalizePersonName(mergeData.Payer);
+ }
+
+ // Copy other fields
+ const allowedFields = [
+ 'Establishment Name',
+ 'Price',
+ 'Category',
+ 'Contents',
+ 'Time',
+ 'Payment Method',
+ 'currency',
+ 'Paid'
+ ];
+
+ allowedFields.forEach(field => {
+ if (mergeData[field] !== undefined) {
+ updateData[field] = mergeData[field];
+ }
+ });
+
+ console.log('[EXPENSES] Updating master expense with:', updateData);
+
+ await $fetch(`${config.url}/api/v2/tables/${expenseTableId}/records`, {
+ method: 'PATCH',
+ headers: {
+ 'xc-token': config.token,
+ 'Content-Type': 'application/json'
+ },
+ body: updateData
+ });
+ }
+
+ // Delete duplicate expenses
+ const deleteResults = [];
+ for (const duplicateId of duplicateIds) {
+ try {
+ console.log('[EXPENSES] Deleting duplicate expense:', duplicateId);
+
+ await $fetch(`${config.url}/api/v2/tables/${expenseTableId}/records`, {
+ method: 'DELETE',
+ headers: {
+ 'xc-token': config.token,
+ 'Content-Type': 'application/json'
+ },
+ body: { Id: parseInt(duplicateId) }
+ });
+
+ deleteResults.push({ id: duplicateId, success: true });
+ } catch (error: any) {
+ console.error('[EXPENSES] Failed to delete duplicate:', duplicateId, error);
+ deleteResults.push({
+ id: duplicateId,
+ success: false,
+ error: error.message || 'Delete failed'
+ });
+ }
+ }
+
+ // Check if all deletes were successful
+ const failedDeletes = deleteResults.filter(r => !r.success);
+ if (failedDeletes.length > 0) {
+ console.warn('[EXPENSES] Some duplicates could not be deleted:', failedDeletes);
+ }
+
+ // Log successful completion
+ await logAuditEvent(event, 'MERGE_EXPENSE_DUPLICATES_COMPLETE', 'expense', {
+ resourceId: masterId,
+ changes: {
+ deletedCount: deleteResults.filter(r => r.success).length,
+ failedDeletes,
+ mergeData
+ },
+ status: 'success'
+ });
+
+ return {
+ success: true,
+ data: {
+ masterId,
+ mergedData: mergeData,
+ deletedCount: deleteResults.filter(r => r.success).length,
+ deleteResults,
+ message: `Successfully merged ${deleteResults.filter(r => r.success).length} duplicates into expense ${masterId}`
+ }
+ };
+
+ } catch (error: any) {
+ console.error('[EXPENSES] Failed to merge duplicates:', error);
+
+ // Log the failure
+ await logAuditEvent(event, 'MERGE_EXPENSE_DUPLICATES_FAILED', 'expense', {
+ resourceId: body?.masterId || 'unknown',
+ changes: {
+ error: error.message || 'Unknown error',
+ requestBody: body
+ },
+ status: 'failure',
+ errorMessage: error.message || 'Unknown error'
+ });
+
+ if (error.statusCode) {
+ throw error;
+ }
+
+ throw createError({
+ statusCode: 500,
+ statusMessage: 'Failed to merge duplicate expenses'
+ });
+ }
+});
diff --git a/server/api/get-expenses.ts b/server/api/get-expenses.ts
index 83451ec..885eeb1 100644
--- a/server/api/get-expenses.ts
+++ b/server/api/get-expenses.ts
@@ -4,8 +4,26 @@ import { processExpenseWithCurrency } from '@/server/utils/currency';
import type { ExpenseFilters } from '@/utils/types';
export default defineEventHandler(async (event) => {
+ console.log('[get-expenses] API called with query:', getQuery(event));
+
try {
- await requireSalesOrAdmin(event);
+ // Check authentication
+ try {
+ await requireSalesOrAdmin(event);
+ } catch (authError: any) {
+ console.error('[get-expenses] Authentication failed:', authError);
+
+ // Return proper error status
+ if (authError.statusCode === 403) {
+ throw createError({
+ statusCode: 403,
+ statusMessage: 'Access denied. You need sales or admin role to view expenses.'
+ });
+ }
+
+ // Re-throw other auth errors
+ throw authError;
+ }
const query = getQuery(event);
diff --git a/server/utils/nocodb.ts b/server/utils/nocodb.ts
index ef74487..d8b6b0c 100644
--- a/server/utils/nocodb.ts
+++ b/server/utils/nocodb.ts
@@ -1,5 +1,16 @@
import type { Interest, Berth, Expense, ExpenseFilters } from "@/utils/types";
+// Data normalization functions
+export const normalizePersonName = (name: string): string => {
+ if (!name) return 'Unknown';
+
+ // Trim whitespace and normalize case
+ return name.trim()
+ .split(' ')
+ .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
+ .join(' ');
+};
+
export interface PageInfo {
pageSize: number;
totalRows: number;