feat: Enhance error handling and logging in expense and interest duplicate detection, add retry logic for document deletion, and improve PDF generation with detailed receipt processing
This commit is contained in:
parent
06500a614d
commit
a00b3918be
|
|
@ -79,6 +79,7 @@ export default defineNuxtConfig({
|
|||
workbox: {
|
||||
navigateFallback: '/',
|
||||
globPatterns: ['**/*.{js,css,html,png,jpg,jpeg,svg,ico}'],
|
||||
navigateFallbackDenylist: [/^\/api\//],
|
||||
runtimeCaching: [
|
||||
{
|
||||
urlPattern: /^https:\/\/cms\.portnimara\.dev\/.*/i,
|
||||
|
|
@ -94,7 +95,9 @@ export default defineNuxtConfig({
|
|||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
],
|
||||
skipWaiting: true,
|
||||
clientsClaim: true
|
||||
},
|
||||
client: {
|
||||
installPrompt: true,
|
||||
|
|
|
|||
|
|
@ -413,7 +413,17 @@ const fetchExpenses = async () => {
|
|||
|
||||
} catch (err: any) {
|
||||
console.error('[expenses] Error fetching expenses:', err);
|
||||
error.value = err.message || 'Failed to fetch expenses';
|
||||
|
||||
// Better error messages based on status codes
|
||||
if (err.statusCode === 401) {
|
||||
error.value = 'Authentication required. Please refresh the page and log in again.';
|
||||
} else if (err.statusCode === 403) {
|
||||
error.value = 'Access denied. You need proper permissions to view expenses.';
|
||||
} else if (err.statusCode === 503) {
|
||||
error.value = 'Service temporarily unavailable. Please try again in a few moments.';
|
||||
} else {
|
||||
error.value = err.data?.message || err.message || 'Failed to fetch expenses. Please check your connection and try again.';
|
||||
}
|
||||
} finally {
|
||||
loading.value = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export default defineEventHandler(async (event) => {
|
|||
try {
|
||||
const body = await readBody(event);
|
||||
const { interestId } = body;
|
||||
const query = getQuery(event);
|
||||
|
||||
console.log('[Delete Generated EOI] Interest ID:', interestId);
|
||||
|
||||
|
|
@ -77,7 +78,11 @@ export default defineEventHandler(async (event) => {
|
|||
|
||||
console.log('[Delete Generated EOI] Deleting document from Documenso');
|
||||
let documensoDeleteSuccessful = false;
|
||||
let retryCount = 0;
|
||||
const maxRetries = 3;
|
||||
|
||||
// Retry logic for temporary failures
|
||||
while (!documensoDeleteSuccessful && retryCount < maxRetries) {
|
||||
try {
|
||||
const deleteResponse = await fetch(`${documensoBaseUrl}/api/v1/documents/${documensoID}`, {
|
||||
method: 'DELETE',
|
||||
|
|
@ -87,42 +92,119 @@ export default defineEventHandler(async (event) => {
|
|||
}
|
||||
});
|
||||
|
||||
if (!deleteResponse.ok) {
|
||||
const errorText = await deleteResponse.text();
|
||||
console.error('[Delete Generated EOI] Documenso deletion failed:', errorText);
|
||||
const responseStatus = deleteResponse.status;
|
||||
let errorDetails = '';
|
||||
|
||||
// If it's a 404, the document is already gone, which is what we want
|
||||
if (deleteResponse.status === 404) {
|
||||
try {
|
||||
errorDetails = await deleteResponse.text();
|
||||
} catch {
|
||||
errorDetails = 'No error details available';
|
||||
}
|
||||
|
||||
if (!deleteResponse.ok) {
|
||||
console.error(`[Delete Generated EOI] Documenso deletion failed (attempt ${retryCount + 1}/${maxRetries}):`, {
|
||||
status: responseStatus,
|
||||
statusText: deleteResponse.statusText,
|
||||
details: errorDetails
|
||||
});
|
||||
|
||||
// Handle specific status codes
|
||||
switch (responseStatus) {
|
||||
case 404:
|
||||
// Document already deleted - this is fine
|
||||
console.log('[Delete Generated EOI] Document already deleted from Documenso (404) - proceeding with database cleanup');
|
||||
documensoDeleteSuccessful = true;
|
||||
break;
|
||||
|
||||
case 403:
|
||||
// Permission denied - document might be in a protected state
|
||||
console.warn('[Delete Generated EOI] Permission denied (403) - document may be in a protected state');
|
||||
throw createError({
|
||||
statusCode: 403,
|
||||
statusMessage: 'Cannot delete document - it may be fully signed or in a protected state',
|
||||
});
|
||||
|
||||
case 500:
|
||||
case 502:
|
||||
case 503:
|
||||
case 504:
|
||||
// Server errors - retry if we haven't exceeded retries
|
||||
if (retryCount < maxRetries - 1) {
|
||||
console.log(`[Delete Generated EOI] Server error (${responseStatus}) - retrying in ${(retryCount + 1) * 2} seconds...`);
|
||||
await new Promise(resolve => setTimeout(resolve, (retryCount + 1) * 2000)); // Exponential backoff
|
||||
retryCount++;
|
||||
continue;
|
||||
} else {
|
||||
throw new Error(`Failed to delete document from Documenso: ${deleteResponse.statusText}`);
|
||||
console.error('[Delete Generated EOI] Max retries exceeded for server error');
|
||||
// Allow proceeding with cleanup for server errors after retries
|
||||
if (query.forceCleanup === 'true') {
|
||||
console.warn('[Delete Generated EOI] Force cleanup enabled - proceeding despite Documenso error');
|
||||
documensoDeleteSuccessful = true;
|
||||
break;
|
||||
}
|
||||
throw new Error(`Documenso server error after ${maxRetries} attempts (${responseStatus}): ${errorDetails}`);
|
||||
}
|
||||
|
||||
default:
|
||||
// Other errors - don't retry
|
||||
throw new Error(`Documenso API error (${responseStatus}): ${errorDetails || deleteResponse.statusText}`);
|
||||
}
|
||||
} else {
|
||||
console.log('[Delete Generated EOI] Successfully deleted document from Documenso');
|
||||
documensoDeleteSuccessful = true;
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error('[Delete Generated EOI] Documenso deletion error:', error);
|
||||
console.error(`[Delete Generated EOI] Documenso deletion error (attempt ${retryCount + 1}/${maxRetries}):`, error);
|
||||
|
||||
// Check if it's a network error or 404 - in those cases, proceed with cleanup
|
||||
if (error.message?.includes('404') || error.status === 404) {
|
||||
// Network errors - retry if we haven't exceeded retries
|
||||
if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT' || error.code === 'ENOTFOUND') {
|
||||
if (retryCount < maxRetries - 1) {
|
||||
console.log(`[Delete Generated EOI] Network error - retrying in ${(retryCount + 1) * 2} seconds...`);
|
||||
await new Promise(resolve => setTimeout(resolve, (retryCount + 1) * 2000));
|
||||
retryCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it's a 404 error wrapped in another error
|
||||
if (error.message?.includes('404') || error.status === 404 || error.statusCode === 404) {
|
||||
console.log('[Delete Generated EOI] Document not found in Documenso - proceeding with database cleanup');
|
||||
documensoDeleteSuccessful = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if force cleanup is enabled
|
||||
const query = getQuery(event);
|
||||
if (query.forceCleanup === 'true') {
|
||||
console.warn('[Delete Generated EOI] Force cleanup enabled - proceeding despite Documenso error:', error.message);
|
||||
documensoDeleteSuccessful = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Don't wrap error messages multiple times
|
||||
if (error.statusCode) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
statusMessage: `Failed to delete document from Documenso: ${error.message}`,
|
||||
statusMessage: error.message || 'Failed to communicate with Documenso API',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!documensoDeleteSuccessful) {
|
||||
const query = getQuery(event);
|
||||
if (query.forceCleanup === 'true') {
|
||||
console.warn('[Delete Generated EOI] Force cleanup enabled - proceeding with database cleanup despite Documenso failure');
|
||||
documensoDeleteSuccessful = true;
|
||||
} else {
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
statusMessage: 'Failed to delete document from Documenso',
|
||||
statusMessage: 'Failed to delete document from Documenso after multiple attempts. You can add ?forceCleanup=true to force database cleanup.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Reset interest fields
|
||||
const updateData = {
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ export default defineEventHandler(async (event) => {
|
|||
* Find duplicate expenses based on multiple criteria
|
||||
*/
|
||||
function findDuplicateExpenses(expenses: any[]) {
|
||||
console.log('[EXPENSES] Starting duplicate detection for', expenses.length, 'expenses');
|
||||
|
||||
const duplicateGroups: Array<{
|
||||
id: string;
|
||||
expenses: any[];
|
||||
|
|
@ -87,6 +89,7 @@ function findDuplicateExpenses(expenses: any[]) {
|
|||
}> = [];
|
||||
|
||||
const processedIds = new Set<number>();
|
||||
let comparisons = 0;
|
||||
|
||||
for (let i = 0; i < expenses.length; i++) {
|
||||
const expense1 = expenses[i];
|
||||
|
|
@ -102,8 +105,13 @@ function findDuplicateExpenses(expenses: any[]) {
|
|||
if (processedIds.has(expense2.Id)) continue;
|
||||
|
||||
const similarity = calculateExpenseSimilarity(expense1, expense2);
|
||||
comparisons++;
|
||||
|
||||
if (similarity.score >= 0.8) {
|
||||
console.log(`[EXPENSES] Comparing ${expense1.Id} vs ${expense2.Id}: score=${similarity.score.toFixed(3)}, threshold=0.7`);
|
||||
|
||||
if (similarity.score >= 0.7) { // Lower threshold for expenses
|
||||
console.log(`[EXPENSES] MATCH FOUND! ${expense1.Id} vs ${expense2.Id} (score: ${similarity.score.toFixed(3)})`);
|
||||
console.log('[EXPENSES] Match reasons:', similarity.reasons);
|
||||
matches.push(expense2);
|
||||
processedIds.add(expense2.Id);
|
||||
similarity.reasons.forEach(r => matchReasons.add(r));
|
||||
|
|
|
|||
|
|
@ -452,81 +452,202 @@ function groupExpenses(expenses: Expense[], groupBy: string): Record<string, Exp
|
|||
|
||||
async function addReceiptImages(doc: PDFKit.PDFDocument, expenses: Expense[]) {
|
||||
console.log('[expenses/generate-pdf] Adding receipt images...');
|
||||
console.log('[expenses/generate-pdf] Total expenses to check:', expenses.length);
|
||||
|
||||
// Log receipt data structure for debugging
|
||||
expenses.forEach((expense, index) => {
|
||||
console.log(`[expenses/generate-pdf] Expense ${index + 1} (ID: ${expense.Id}):`, {
|
||||
establishment: expense['Establishment Name'],
|
||||
hasReceipt: !!expense.Receipt,
|
||||
receiptType: typeof expense.Receipt,
|
||||
receiptLength: Array.isArray(expense.Receipt) ? expense.Receipt.length : 'N/A',
|
||||
receiptData: expense.Receipt
|
||||
});
|
||||
});
|
||||
|
||||
const expensesWithReceipts = expenses.filter(expense =>
|
||||
expense.Receipt && Array.isArray(expense.Receipt) && expense.Receipt.length > 0
|
||||
);
|
||||
|
||||
console.log('[expenses/generate-pdf] Expenses with receipts:', expensesWithReceipts.length);
|
||||
|
||||
if (expensesWithReceipts.length === 0) {
|
||||
console.log('[expenses/generate-pdf] No receipts found to include');
|
||||
return;
|
||||
}
|
||||
|
||||
// Add new page for receipts
|
||||
doc.addPage();
|
||||
let totalReceiptImages = 0;
|
||||
let processedImages = 0;
|
||||
|
||||
doc.fontSize(18)
|
||||
.font('Helvetica-Bold')
|
||||
.text('Receipt Images', { align: 'center' });
|
||||
// Count total receipt images for progress tracking
|
||||
expensesWithReceipts.forEach(expense => {
|
||||
if (expense.Receipt && Array.isArray(expense.Receipt)) {
|
||||
totalReceiptImages += expense.Receipt.length;
|
||||
}
|
||||
});
|
||||
|
||||
doc.y += 20;
|
||||
console.log('[expenses/generate-pdf] Total receipt images to process:', totalReceiptImages);
|
||||
|
||||
for (const expense of expensesWithReceipts) {
|
||||
try {
|
||||
// Add expense header
|
||||
doc.fontSize(14)
|
||||
.font('Helvetica-Bold')
|
||||
.text(`Receipt for: ${expense['Establishment Name']} - €${expense.PriceNumber?.toFixed(2)}`,
|
||||
{ align: 'left' });
|
||||
console.log('[expenses/generate-pdf] Processing receipts for expense:', expense.Id, expense['Establishment Name']);
|
||||
|
||||
doc.fontSize(12)
|
||||
.font('Helvetica')
|
||||
.text(`Date: ${expense.Time ? formatDate(expense.Time) : 'N/A'}`, { align: 'left' });
|
||||
|
||||
doc.y += 10;
|
||||
|
||||
// Process receipt images
|
||||
if (expense.Receipt) {
|
||||
for (const receipt of expense.Receipt) {
|
||||
if (receipt.url || receipt.directus_files_id?.filename_download) {
|
||||
// Process receipt images - each gets its own page
|
||||
if (expense.Receipt && Array.isArray(expense.Receipt)) {
|
||||
for (const [receiptIndex, receipt] of expense.Receipt.entries()) {
|
||||
if (receipt.url || receipt.directus_files_id?.filename_download || receipt.filename_download) {
|
||||
try {
|
||||
console.log(`[expenses/generate-pdf] Fetching receipt ${receiptIndex + 1}/${expense.Receipt.length} for expense ${expense.Id}`);
|
||||
const imageBuffer = await fetchReceiptImage(receipt);
|
||||
|
||||
if (imageBuffer) {
|
||||
// Check if we need a new page
|
||||
if (doc.y > doc.page.height - 400) {
|
||||
// Add new page for each receipt image
|
||||
doc.addPage();
|
||||
doc.y = 60;
|
||||
}
|
||||
|
||||
// Add image
|
||||
const maxWidth = 400;
|
||||
const maxHeight = 300;
|
||||
// Add header section for this receipt
|
||||
const headerHeight = 100;
|
||||
|
||||
doc.image(imageBuffer, {
|
||||
fit: [maxWidth, maxHeight],
|
||||
align: 'center'
|
||||
});
|
||||
// Header background
|
||||
doc.rect(60, 60, doc.page.width - 120, headerHeight)
|
||||
.fillColor('#f8f9fa')
|
||||
.fill()
|
||||
.strokeColor('#dee2e6')
|
||||
.lineWidth(1)
|
||||
.stroke();
|
||||
|
||||
doc.fillColor('#000000');
|
||||
|
||||
// Receipt header content
|
||||
doc.fontSize(16)
|
||||
.font('Helvetica-Bold')
|
||||
.text(`Receipt Image ${receiptIndex + 1}${expense.Receipt.length > 1 ? ` of ${expense.Receipt.length}` : ''}`,
|
||||
70, 80, { align: 'left' });
|
||||
|
||||
doc.fontSize(14)
|
||||
.font('Helvetica-Bold')
|
||||
.text(`${expense['Establishment Name']} - €${expense.PriceNumber?.toFixed(2)}`,
|
||||
70, 105, { align: 'left' });
|
||||
|
||||
doc.fontSize(12)
|
||||
.font('Helvetica')
|
||||
.text(`Date: ${expense.Time ? formatDate(expense.Time) : 'N/A'}`,
|
||||
70, 125, { align: 'left' });
|
||||
|
||||
doc.y += 20;
|
||||
}
|
||||
} catch (imageError) {
|
||||
console.error('[expenses/generate-pdf] Error adding receipt image:', imageError);
|
||||
doc.fontSize(10)
|
||||
.fillColor('#666666')
|
||||
.text('Receipt image could not be loaded', { align: 'center' });
|
||||
.text(`Payer: ${expense.Payer || 'N/A'} | Category: ${expense.Category || 'N/A'}`,
|
||||
70, 140, { align: 'left' });
|
||||
|
||||
doc.fillColor('#000000');
|
||||
|
||||
// Calculate available space for image (full page minus header and margins)
|
||||
const pageWidth = doc.page.width;
|
||||
const pageHeight = doc.page.height;
|
||||
const margin = 60;
|
||||
const imageStartY = 60 + headerHeight + 20; // Header + spacing
|
||||
|
||||
const maxImageWidth = pageWidth - (margin * 2);
|
||||
const maxImageHeight = pageHeight - imageStartY - margin;
|
||||
|
||||
console.log(`[expenses/generate-pdf] Adding large image - Max size: ${maxImageWidth}x${maxImageHeight}, Buffer size: ${imageBuffer.length} bytes`);
|
||||
|
||||
// Add the receipt image with maximum size
|
||||
try {
|
||||
doc.image(imageBuffer, margin, imageStartY, {
|
||||
fit: [maxImageWidth, maxImageHeight],
|
||||
align: 'center',
|
||||
valign: 'center'
|
||||
});
|
||||
|
||||
processedImages++;
|
||||
console.log(`[expenses/generate-pdf] Successfully added receipt image ${processedImages}/${totalReceiptImages}`);
|
||||
|
||||
} catch (imageEmbedError: any) {
|
||||
console.error('[expenses/generate-pdf] Error embedding image in PDF:', imageEmbedError);
|
||||
|
||||
// Add error message on the page
|
||||
doc.fontSize(14)
|
||||
.fillColor('#dc3545')
|
||||
.text('Receipt image could not be embedded', margin, imageStartY + 50, {
|
||||
align: 'center',
|
||||
width: maxImageWidth
|
||||
});
|
||||
|
||||
doc.fontSize(12)
|
||||
.fillColor('#6c757d')
|
||||
.text(`Error: ${imageEmbedError.message || 'Unknown error'}`, margin, imageStartY + 80, {
|
||||
align: 'center',
|
||||
width: maxImageWidth
|
||||
});
|
||||
|
||||
doc.fillColor('#000000');
|
||||
doc.y += 10;
|
||||
}
|
||||
|
||||
} else {
|
||||
console.warn(`[expenses/generate-pdf] No image buffer received for receipt ${receiptIndex + 1} of expense ${expense.Id}`);
|
||||
|
||||
// Add page with error message
|
||||
doc.addPage();
|
||||
|
||||
doc.fontSize(16)
|
||||
.font('Helvetica-Bold')
|
||||
.text(`Receipt Image ${receiptIndex + 1}${expense.Receipt.length > 1 ? ` of ${expense.Receipt.length}` : ''}`,
|
||||
{ align: 'center' });
|
||||
|
||||
doc.fontSize(14)
|
||||
.font('Helvetica')
|
||||
.text(`${expense['Establishment Name']} - €${expense.PriceNumber?.toFixed(2)}`,
|
||||
{ align: 'center' });
|
||||
|
||||
doc.y += 50;
|
||||
|
||||
doc.fontSize(12)
|
||||
.fillColor('#dc3545')
|
||||
.text('Receipt image could not be loaded from storage', { align: 'center' });
|
||||
|
||||
doc.fillColor('#000000');
|
||||
}
|
||||
|
||||
} catch (imageError: any) {
|
||||
console.error(`[expenses/generate-pdf] Error processing receipt ${receiptIndex + 1} for expense ${expense.Id}:`, imageError);
|
||||
|
||||
// Add page with error information
|
||||
doc.addPage();
|
||||
|
||||
doc.fontSize(16)
|
||||
.font('Helvetica-Bold')
|
||||
.text(`Receipt Image ${receiptIndex + 1}${expense.Receipt.length > 1 ? ` of ${expense.Receipt.length}` : ''}`,
|
||||
{ align: 'center' });
|
||||
|
||||
doc.fontSize(14)
|
||||
.font('Helvetica')
|
||||
.text(`${expense['Establishment Name']} - €${expense.PriceNumber?.toFixed(2)}`,
|
||||
{ align: 'center' });
|
||||
|
||||
doc.y += 50;
|
||||
|
||||
doc.fontSize(12)
|
||||
.fillColor('#dc3545')
|
||||
.text('Error loading receipt image', { align: 'center' });
|
||||
|
||||
doc.fontSize(10)
|
||||
.fillColor('#6c757d')
|
||||
.text(`${imageError.message || 'Unknown error'}`, { align: 'center' });
|
||||
|
||||
doc.fillColor('#000000');
|
||||
}
|
||||
} else {
|
||||
console.log(`[expenses/generate-pdf] Skipping receipt ${receiptIndex + 1} for expense ${expense.Id} - no valid file path`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doc.y += 20;
|
||||
} catch (error) {
|
||||
console.error('[expenses/generate-pdf] Error processing receipt for expense:', expense.Id, error);
|
||||
console.error('[expenses/generate-pdf] Error processing receipts for expense:', expense.Id, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[expenses/generate-pdf] Completed processing ${processedImages}/${totalReceiptImages} receipt images`);
|
||||
}
|
||||
|
||||
async function fetchReceiptImage(receipt: any): Promise<Buffer | null> {
|
||||
|
|
|
|||
|
|
@ -7,9 +7,13 @@ export default defineEventHandler(async (event) => {
|
|||
console.log('[get-expenses] API called with query:', getQuery(event));
|
||||
|
||||
try {
|
||||
// Check authentication
|
||||
// Set proper headers
|
||||
setHeader(event, 'Cache-Control', 'no-cache');
|
||||
setHeader(event, 'Content-Type', 'application/json');
|
||||
// Check authentication first
|
||||
try {
|
||||
await requireSalesOrAdmin(event);
|
||||
console.log('[get-expenses] Authentication successful');
|
||||
} catch (authError: any) {
|
||||
console.error('[get-expenses] Authentication failed:', authError);
|
||||
|
||||
|
|
@ -127,14 +131,34 @@ export default defineEventHandler(async (event) => {
|
|||
statusMessage: 'Unable to fetch expense data. Please try again later.'
|
||||
});
|
||||
}
|
||||
} catch (authError: any) {
|
||||
if (authError.statusCode === 403) {
|
||||
} catch (error: any) {
|
||||
console.error('[get-expenses] Top-level error:', error);
|
||||
|
||||
// If it's already a proper H3 error, re-throw it
|
||||
if (error.statusCode) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Handle authentication errors specifically
|
||||
if (error.message?.includes('authentication') || error.message?.includes('auth')) {
|
||||
throw createError({
|
||||
statusCode: 403,
|
||||
statusMessage: 'Access denied. This feature requires sales team or administrator privileges.'
|
||||
statusCode: 401,
|
||||
statusMessage: 'Authentication required. Please log in again.'
|
||||
});
|
||||
}
|
||||
|
||||
throw authError;
|
||||
// Handle database connection errors
|
||||
if (error.message?.includes('database') || error.message?.includes('connection')) {
|
||||
throw createError({
|
||||
statusCode: 503,
|
||||
statusMessage: 'Database temporarily unavailable. Please try again later.'
|
||||
});
|
||||
}
|
||||
|
||||
// Generic server error for anything else
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
statusMessage: 'An unexpected error occurred. Please try again later.'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -86,6 +86,9 @@ export default defineEventHandler(async (event) => {
|
|||
* Find duplicate interests based on multiple criteria
|
||||
*/
|
||||
function findDuplicateInterests(interests: any[], threshold: number = 0.8) {
|
||||
console.log('[INTERESTS] Starting duplicate detection with threshold:', threshold);
|
||||
console.log('[INTERESTS] Total interests to analyze:', interests.length);
|
||||
|
||||
const duplicateGroups: Array<{
|
||||
id: string;
|
||||
interests: any[];
|
||||
|
|
@ -95,6 +98,7 @@ function findDuplicateInterests(interests: any[], threshold: number = 0.8) {
|
|||
}> = [];
|
||||
|
||||
const processedIds = new Set<number>();
|
||||
let comparisons = 0;
|
||||
|
||||
for (let i = 0; i < interests.length; i++) {
|
||||
const interest1 = interests[i];
|
||||
|
|
@ -109,14 +113,21 @@ function findDuplicateInterests(interests: any[], threshold: number = 0.8) {
|
|||
if (processedIds.has(interest2.Id)) continue;
|
||||
|
||||
const similarity = calculateSimilarity(interest1, interest2);
|
||||
comparisons++;
|
||||
|
||||
console.log(`[INTERESTS] Comparing ${interest1.Id} vs ${interest2.Id}: score=${similarity.score.toFixed(3)}, threshold=${threshold}`);
|
||||
|
||||
if (similarity.score >= threshold) {
|
||||
console.log(`[INTERESTS] MATCH FOUND! ${interest1.Id} vs ${interest2.Id} (score: ${similarity.score.toFixed(3)})`);
|
||||
console.log('[INTERESTS] Match details:', similarity.details);
|
||||
matches.push(interest2);
|
||||
processedIds.add(interest2.Id);
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.length > 1) {
|
||||
console.log(`[INTERESTS] Creating duplicate group with ${matches.length} matches`);
|
||||
|
||||
// Mark all as processed
|
||||
matches.forEach(match => processedIds.add(match.Id));
|
||||
|
||||
|
|
@ -138,6 +149,7 @@ function findDuplicateInterests(interests: any[], threshold: number = 0.8) {
|
|||
}
|
||||
}
|
||||
|
||||
console.log(`[INTERESTS] Completed ${comparisons} comparisons, found ${duplicateGroups.length} duplicate groups`);
|
||||
return duplicateGroups;
|
||||
}
|
||||
|
||||
|
|
@ -147,36 +159,67 @@ function findDuplicateInterests(interests: any[], threshold: number = 0.8) {
|
|||
function calculateSimilarity(interest1: any, interest2: any) {
|
||||
const scores: Array<{ type: string; score: number; weight: number }> = [];
|
||||
|
||||
// Email similarity (highest weight)
|
||||
console.log(`[INTERESTS] Calculating similarity between:`, {
|
||||
id1: interest1.Id,
|
||||
name1: interest1['Full Name'],
|
||||
email1: interest1['Email Address'],
|
||||
phone1: interest1['Phone Number'],
|
||||
id2: interest2.Id,
|
||||
name2: interest2['Full Name'],
|
||||
email2: interest2['Email Address'],
|
||||
phone2: interest2['Phone Number']
|
||||
});
|
||||
|
||||
// Email similarity (highest weight) - exact match required
|
||||
if (interest1['Email Address'] && interest2['Email Address']) {
|
||||
const emailScore = normalizeEmail(interest1['Email Address']) === normalizeEmail(interest2['Email Address']) ? 1.0 : 0.0;
|
||||
scores.push({ type: 'email', score: emailScore, weight: 0.4 });
|
||||
const email1 = normalizeEmail(interest1['Email Address']);
|
||||
const email2 = normalizeEmail(interest2['Email Address']);
|
||||
const emailScore = email1 === email2 ? 1.0 : 0.0;
|
||||
scores.push({ type: 'email', score: emailScore, weight: 0.5 });
|
||||
console.log(`[INTERESTS] Email comparison: "${email1}" vs "${email2}" = ${emailScore}`);
|
||||
}
|
||||
|
||||
// Phone similarity
|
||||
// Phone similarity - exact match on normalized numbers
|
||||
if (interest1['Phone Number'] && interest2['Phone Number']) {
|
||||
const phone1 = normalizePhone(interest1['Phone Number']);
|
||||
const phone2 = normalizePhone(interest2['Phone Number']);
|
||||
const phoneScore = phone1 === phone2 ? 1.0 : 0.0;
|
||||
scores.push({ type: 'phone', score: phoneScore, weight: 0.3 });
|
||||
const phoneScore = phone1 === phone2 && phone1.length >= 8 ? 1.0 : 0.0; // Require at least 8 digits
|
||||
scores.push({ type: 'phone', score: phoneScore, weight: 0.4 });
|
||||
console.log(`[INTERESTS] Phone comparison: "${phone1}" vs "${phone2}" = ${phoneScore}`);
|
||||
}
|
||||
|
||||
// Name similarity
|
||||
// Name similarity - fuzzy matching
|
||||
if (interest1['Full Name'] && interest2['Full Name']) {
|
||||
const nameScore = calculateNameSimilarity(interest1['Full Name'], interest2['Full Name']);
|
||||
scores.push({ type: 'name', score: nameScore, weight: 0.2 });
|
||||
scores.push({ type: 'name', score: nameScore, weight: 0.3 });
|
||||
console.log(`[INTERESTS] Name comparison: "${interest1['Full Name']}" vs "${interest2['Full Name']}" = ${nameScore.toFixed(3)}`);
|
||||
}
|
||||
|
||||
// Address similarity
|
||||
if (interest1.Address && interest2.Address) {
|
||||
const addressScore = calculateStringSimilarity(interest1.Address, interest2.Address);
|
||||
scores.push({ type: 'address', score: addressScore, weight: 0.1 });
|
||||
scores.push({ type: 'address', score: addressScore, weight: 0.2 });
|
||||
console.log(`[INTERESTS] Address comparison: ${addressScore.toFixed(3)}`);
|
||||
}
|
||||
|
||||
// Calculate weighted average
|
||||
// Special case: if we have exact email OR phone match, give high score regardless of other fields
|
||||
const hasExactEmailMatch = scores.find(s => s.type === 'email' && s.score === 1.0);
|
||||
const hasExactPhoneMatch = scores.find(s => s.type === 'phone' && s.score === 1.0);
|
||||
|
||||
if (hasExactEmailMatch || hasExactPhoneMatch) {
|
||||
console.log('[INTERESTS] Exact email or phone match found - high confidence');
|
||||
return {
|
||||
score: 0.95, // High confidence for exact email/phone match
|
||||
details: scores
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate weighted average for other cases
|
||||
const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
|
||||
const weightedScore = scores.reduce((sum, s) => sum + (s.score * s.weight), 0) / (totalWeight || 1);
|
||||
|
||||
console.log(`[INTERESTS] Weighted score: ${weightedScore.toFixed(3)} (weights: ${totalWeight})`);
|
||||
|
||||
return {
|
||||
score: weightedScore,
|
||||
details: scores
|
||||
|
|
|
|||
Loading…
Reference in New Issue