matt/port-nimara-client-portal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Implement centralized duplicate detection utility for expenses and interests

Browse Source
This commit is contained in:
Matt
2025-07-12 13:42:53 -04:00
parent b8a6a52417
commit 080cb60d71
3 changed files with 441 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
server/api/expenses/duplicates/find.ts
View File

@@ -1,5 +1,6 @@
import { requireSalesOrAdmin } from '~/server/utils/auth';
import { getNocoDbConfiguration, normalizePersonName } from '~/server/utils/nocodb';
import { findDuplicates, createExpenseConfig } from '~/server/utils/duplicate-detection';
import type { Expense } from '~/utils/types';
export default defineEventHandler(async (event) => {
@@ -35,21 +36,31 @@ export default defineEventHandler(async (event) => {
const expenses = response.list || [];
console.log('[EXPENSES] Analyzing', expenses.length, 'expenses for duplicates');
// Find duplicate groups
const duplicateGroups = findDuplicateExpenses(expenses);
// Find duplicate groups using the new centralized utility
const duplicateConfig = createExpenseConfig();
const duplicateGroups = findDuplicates(expenses, duplicateConfig);
// Convert to the expected format
const formattedGroups = duplicateGroups.map(group => ({
id: group.id,
expenses: group.items,
matchReason: group.matchReason,
confidence: group.confidence,
masterCandidate: group.masterCandidate
}));
// Also find payer name variations
const payerVariations = findPayerNameVariations(expenses);
console.log('[EXPENSES] Found', duplicateGroups.length, 'duplicate groups and', payerVariations.length, 'payer variations');
console.log('[EXPENSES] Found', formattedGroups.length, 'duplicate groups and', payerVariations.length, 'payer variations');
return {
success: true,
data: {
duplicateGroups,
duplicateGroups: formattedGroups,
payerVariations,
totalExpenses: expenses.length,
duplicateCount: duplicateGroups.reduce((sum, group) => sum + group.expenses.length, 0),
duplicateCount: formattedGroups.reduce((sum, group) => sum + group.expenses.length, 0),
dateRange: {
start: startDate.toISOString().split('T')[0],
end: endDate.toISOString().split('T')[0]
@@ -74,71 +85,6 @@ export default defineEventHandler(async (event) => {
}
});
/**
* Find duplicate expenses based on multiple criteria
*/
function findDuplicateExpenses(expenses: any[]) {
console.log('[EXPENSES] Starting duplicate detection for', expenses.length, 'expenses');
const duplicateGroups: Array<{
id: string;
expenses: any[];
matchReason: string;
confidence: number;
masterCandidate: any;
}> = [];
const processedIds = new Set<number>();
let comparisons = 0;
for (let i = 0; i < expenses.length; i++) {
const expense1 = expenses[i];
if (processedIds.has(expense1.Id)) continue;
const matches = [expense1];
let matchReasons = new Set<string>();
for (let j = i + 1; j < expenses.length; j++) {
const expense2 = expenses[j];
if (processedIds.has(expense2.Id)) continue;
const similarity = calculateExpenseSimilarity(expense1, expense2);
comparisons++;
console.log(`[EXPENSES] Comparing ${expense1.Id} vs ${expense2.Id}: score=${similarity.score.toFixed(3)}, threshold=0.7`);
if (similarity.score >= 0.7) { // Lower threshold for expenses
console.log(`[EXPENSES] MATCH FOUND! ${expense1.Id} vs ${expense2.Id} (score: ${similarity.score.toFixed(3)})`);
console.log('[EXPENSES] Match reasons:', similarity.reasons);
matches.push(expense2);
processedIds.add(expense2.Id);
similarity.reasons.forEach(r => matchReasons.add(r));
}
}
if (matches.length > 1) {
// Mark all as processed
matches.forEach(match => processedIds.add(match.Id));
// Determine the best master candidate
const masterCandidate = selectMasterExpense(matches);
duplicateGroups.push({
id: `group_${duplicateGroups.length + 1}`,
expenses: matches,
matchReason: Array.from(matchReasons).join(', '),
confidence: Math.max(...matches.slice(1).map(match =>
calculateExpenseSimilarity(masterCandidate, match).score
)),
masterCandidate
});
}
}
return duplicateGroups;
}
/**
* Find payer name variations (like "Abbie" vs "abbie")
@@ -181,154 +127,3 @@ function findPayerNameVariations(expenses: any[]) {
return variations.sort((a, b) => b.expenseCount - a.expenseCount);
}
/**
* Calculate similarity between two expenses
*/
function calculateExpenseSimilarity(expense1: any, expense2: any) {
const scores: Array<{ type: string; score: number; weight: number }> = [];
const reasons: string[] = [];
// Exact match on establishment, price, and date (highest weight for true duplicates)
if (expense1['Establishment Name'] === expense2['Establishment Name'] &&
expense1.Price === expense2.Price &&
expense1.Time === expense2.Time) {
scores.push({ type: 'exact', score: 1.0, weight: 0.5 });
reasons.push('Exact match');
}
// Same payer, establishment, and price on same day (likely duplicate)
const date1 = expense1.Time?.split('T')[0];
const date2 = expense2.Time?.split('T')[0];
if (normalizePersonName(expense1.Payer) === normalizePersonName(expense2.Payer) &&
expense1['Establishment Name'] === expense2['Establishment Name'] &&
expense1.Price === expense2.Price &&
date1 === date2) {
scores.push({ type: 'same-day', score: 0.95, weight: 0.4 });
reasons.push('Same person, place, amount on same day');
}
// Similar establishment names with same price and payer
if (expense1['Establishment Name'] && expense2['Establishment Name']) {
const nameSimilarity = calculateStringSimilarity(
expense1['Establishment Name'],
expense2['Establishment Name']
);
if (nameSimilarity > 0.8 &&
expense1.Price === expense2.Price &&
normalizePersonName(expense1.Payer) === normalizePersonName(expense2.Payer)) {
scores.push({ type: 'similar', score: nameSimilarity, weight: 0.3 });
reasons.push('Similar establishment name');
}
}
// Time proximity check (within 5 minutes)
if (expense1.Time && expense2.Time) {
const time1 = new Date(expense1.Time).getTime();
const time2 = new Date(expense2.Time).getTime();
const timeDiff = Math.abs(time1 - time2);
if (timeDiff < 5 * 60 * 1000 && // 5 minutes
expense1['Establishment Name'] === expense2['Establishment Name']) {
scores.push({ type: 'time-proximity', score: 0.9, weight: 0.2 });
reasons.push('Within 5 minutes at same establishment');
}
}
// Calculate weighted average
const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
const weightedScore = totalWeight > 0
? scores.reduce((sum, s) => sum + (s.score * s.weight), 0) / totalWeight
: 0;
return {
score: weightedScore,
reasons,
details: scores
};
}
/**
* Calculate string similarity using Levenshtein distance
*/
function calculateStringSimilarity(str1: string, str2: string): number {
const s1 = str1.toLowerCase().trim();
const s2 = str2.toLowerCase().trim();
if (s1 === s2) return 1.0;
const distance = levenshteinDistance(s1, s2);
const maxLength = Math.max(s1.length, s2.length);
return maxLength > 0 ? 1 - (distance / maxLength) : 0;
}
/**
* Calculate Levenshtein distance between two strings
*/
function levenshteinDistance(str1: string, str2: string): number {
const matrix = Array(str2.length + 1).fill(null).map(() => Array(str1.length + 1).fill(null));
for (let i = 0; i <= str1.length; i += 1) {
matrix[0][i] = i;
}
for (let j = 0; j <= str2.length; j += 1) {
matrix[j][0] = j;
}
for (let j = 1; j <= str2.length; j += 1) {
for (let i = 1; i <= str1.length; i += 1) {
const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
matrix[j][i] = Math.min(
matrix[j][i - 1] + 1, // deletion
matrix[j - 1][i] + 1, // insertion
matrix[j - 1][i - 1] + indicator // substitution
);
}
}
return matrix[str2.length][str1.length];
}
/**
* Select the best master expense from a group
*/
function selectMasterExpense(expenses: any[]) {
return expenses.reduce((best, current) => {
const bestScore = calculateExpenseCompletenessScore(best);
const currentScore = calculateExpenseCompletenessScore(current);
return currentScore > bestScore ? current : best;
});
}
/**
* Calculate completeness score for an expense
*/
function calculateExpenseCompletenessScore(expense: any): number {
const fields = ['Establishment Name', 'Price', 'Payer', 'Category', 'Contents', 'Time'];
const filledFields = fields.filter(field =>
expense[field] && expense[field].toString().trim().length > 0
);
let score = filledFields.length / fields.length;
// Bonus for having contents description
if (expense.Contents && expense.Contents.length > 10) {
score += 0.2;
}
// Bonus for recent creation (more likely to be accurate)
if (expense.CreatedAt) {
const created = new Date(expense.CreatedAt);
const now = new Date();
const hoursOld = (now.getTime() - created.getTime()) / (1000 * 60 * 60);
if (hoursOld < 24) score += 0.1;
}
return Math.min(score, 1.0);
}