Add EOI automation system with email processing and document management
- Implement automated EOI processing from sales emails - Add EOI document upload and management capabilities - Enhance email thread handling with better parsing and grouping - Add retry logic and error handling for file operations - Introduce Documeso integration for document processing - Create server tasks and plugins infrastructure - Update email composer with improved attachment handling
This commit is contained in:
@@ -388,40 +388,81 @@ async function fetchImapEmails(
|
||||
// Group emails into threads based on subject and references
|
||||
function groupIntoThreads(emails: EmailMessage[]): any[] {
|
||||
const threads = new Map<string, EmailMessage[]>();
|
||||
const emailById = new Map<string, EmailMessage>();
|
||||
|
||||
// First pass: index all emails by ID
|
||||
emails.forEach(email => {
|
||||
// Normalize subject by removing Re:, Fwd:, etc.
|
||||
emailById.set(email.id, email);
|
||||
});
|
||||
|
||||
// Second pass: group emails into threads
|
||||
emails.forEach(email => {
|
||||
// Normalize subject by removing Re:, Fwd:, etc. and extra whitespace
|
||||
const normalizedSubject = email.subject
|
||||
.replace(/^(Re:|Fwd:|Fw:)\s*/gi, '')
|
||||
.trim();
|
||||
.replace(/^(Re:|Fwd:|Fw:|RE:|FW:|FWD:)\s*/gi, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
// Find existing thread or create new one
|
||||
// Check if this email belongs to an existing thread
|
||||
let threadFound = false;
|
||||
for (const [threadId, threadEmails] of threads.entries()) {
|
||||
const threadSubject = threadEmails[0].subject
|
||||
.replace(/^(Re:|Fwd:|Fw:)\s*/gi, '')
|
||||
.trim();
|
||||
|
||||
if (threadSubject === normalizedSubject) {
|
||||
threadEmails.push(email);
|
||||
threadFound = true;
|
||||
break;
|
||||
|
||||
// First, check if it has a threadId (in-reply-to header)
|
||||
if (email.threadId) {
|
||||
// Look for the parent email
|
||||
const parentEmail = emailById.get(email.threadId);
|
||||
if (parentEmail) {
|
||||
// Find which thread the parent belongs to
|
||||
for (const [threadId, threadEmails] of threads.entries()) {
|
||||
if (threadEmails.some(e => e.id === parentEmail.id)) {
|
||||
threadEmails.push(email);
|
||||
threadFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not found by threadId, try to match by subject
|
||||
if (!threadFound) {
|
||||
for (const [threadId, threadEmails] of threads.entries()) {
|
||||
const threadSubject = threadEmails[0].subject
|
||||
.replace(/^(Re:|Fwd:|Fw:|RE:|FW:|FWD:)\s*/gi, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
// Check if subjects match (case-insensitive)
|
||||
if (threadSubject === normalizedSubject) {
|
||||
threadEmails.push(email);
|
||||
threadFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If still not found, create a new thread
|
||||
if (!threadFound) {
|
||||
threads.set(email.id, [email]);
|
||||
}
|
||||
});
|
||||
|
||||
// Convert to array format and sort threads by latest timestamp (newest first)
|
||||
// Convert to array format and sort emails within each thread
|
||||
return Array.from(threads.entries())
|
||||
.map(([threadId, emails]) => ({
|
||||
id: threadId,
|
||||
subject: emails[0].subject,
|
||||
emailCount: emails.length,
|
||||
latestTimestamp: emails[0].timestamp, // First email is newest since we sorted desc
|
||||
emails: emails
|
||||
}))
|
||||
.map(([threadId, threadEmails]) => {
|
||||
// Sort emails within thread by timestamp (oldest first for chronological order)
|
||||
threadEmails.sort((a, b) =>
|
||||
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
||||
);
|
||||
|
||||
return {
|
||||
id: threadId,
|
||||
subject: threadEmails[0].subject,
|
||||
emailCount: threadEmails.length,
|
||||
latestTimestamp: threadEmails[threadEmails.length - 1].timestamp, // Latest email
|
||||
emails: threadEmails
|
||||
};
|
||||
})
|
||||
// Sort threads by latest activity (newest first)
|
||||
.sort((a, b) => new Date(b.latestTimestamp).getTime() - new Date(a.latestTimestamp).getTime());
|
||||
}
|
||||
|
||||
254
server/api/email/process-sales-eois.ts
Normal file
254
server/api/email/process-sales-eois.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
import { parseEmail, getIMAPConnection } from '~/server/utils/email-utils';
|
||||
import { uploadFile } from '~/server/utils/minio';
|
||||
import { getInterestByFieldAsync, updateInterest } from '~/server/utils/nocodb';
|
||||
import type { ParsedMail } from 'mailparser';
|
||||
|
||||
interface ProcessedEOI {
|
||||
clientName: string;
|
||||
interestId?: string;
|
||||
fileName: string;
|
||||
processed: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const xTagHeader = getRequestHeader(event, "x-tag");
|
||||
|
||||
if (!xTagHeader || (xTagHeader !== "094ut234" && xTagHeader !== "pjnvü1230")) {
|
||||
throw createError({ statusCode: 401, statusMessage: "unauthenticated" });
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[Process Sales EOIs] Starting email processing...');
|
||||
|
||||
// Sales email credentials
|
||||
const credentials = {
|
||||
user: 'sales@portnimara.com',
|
||||
password: 'MDze7cSClQok8qWOf23X8Mb6lArdk0i42YnwJ1FskdtO2NCc9',
|
||||
host: 'mail.portnimara.com',
|
||||
port: 993,
|
||||
tls: true
|
||||
};
|
||||
|
||||
const connection = await getIMAPConnection(credentials);
|
||||
const results: ProcessedEOI[] = [];
|
||||
|
||||
try {
|
||||
// Open inbox
|
||||
await new Promise((resolve, reject) => {
|
||||
connection.openBox('INBOX', false, (err: any, box: any) => {
|
||||
if (err) reject(err);
|
||||
else resolve(box);
|
||||
});
|
||||
});
|
||||
|
||||
// Search for unread emails with attachments
|
||||
const searchCriteria = ['UNSEEN'];
|
||||
|
||||
const messages = await new Promise<number[]>((resolve, reject) => {
|
||||
connection.search(searchCriteria, (err: any, results: any) => {
|
||||
if (err) reject(err);
|
||||
else resolve(results || []);
|
||||
});
|
||||
});
|
||||
|
||||
console.log(`[Process Sales EOIs] Found ${messages.length} unread messages`);
|
||||
|
||||
for (const msgNum of messages) {
|
||||
try {
|
||||
const parsedEmail = await fetchAndParseEmail(connection, msgNum);
|
||||
|
||||
if (parsedEmail.attachments && parsedEmail.attachments.length > 0) {
|
||||
// Process PDF attachments
|
||||
for (const attachment of parsedEmail.attachments) {
|
||||
if (attachment.contentType === 'application/pdf') {
|
||||
const result = await processEOIAttachment(
|
||||
attachment,
|
||||
parsedEmail.subject || '',
|
||||
parsedEmail.from?.text || ''
|
||||
);
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as read
|
||||
connection.addFlags(msgNum, '\\Seen', (err: any) => {
|
||||
if (err) console.error('Failed to mark message as read:', err);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`[Process Sales EOIs] Error processing message ${msgNum}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
connection.end();
|
||||
} catch (error) {
|
||||
connection.end();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
processed: results.length,
|
||||
results
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[Process Sales EOIs] Failed to process emails:', error);
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
statusMessage: error.message || 'Failed to process sales emails',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
async function fetchAndParseEmail(connection: any, msgNum: number): Promise<ParsedMail> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const fetch = connection.fetch(msgNum, {
|
||||
bodies: '',
|
||||
struct: true
|
||||
});
|
||||
|
||||
fetch.on('message', (msg: any) => {
|
||||
let buffer = '';
|
||||
|
||||
msg.on('body', (stream: any) => {
|
||||
stream.on('data', (chunk: any) => {
|
||||
buffer += chunk.toString('utf8');
|
||||
});
|
||||
|
||||
stream.once('end', async () => {
|
||||
try {
|
||||
const parsed = await parseEmail(buffer);
|
||||
resolve(parsed);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
fetch.once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function processEOIAttachment(
|
||||
attachment: any,
|
||||
subject: string,
|
||||
from: string
|
||||
): Promise<ProcessedEOI> {
|
||||
const fileName = attachment.filename || 'unknown.pdf';
|
||||
|
||||
try {
|
||||
console.log(`[Process Sales EOIs] Processing attachment: ${fileName}`);
|
||||
|
||||
// Try to extract client name from filename or subject
|
||||
const clientName = extractClientName(fileName, subject);
|
||||
|
||||
if (!clientName) {
|
||||
return {
|
||||
clientName: 'Unknown',
|
||||
fileName,
|
||||
processed: false,
|
||||
error: 'Could not identify client from filename or subject'
|
||||
};
|
||||
}
|
||||
|
||||
// Find interest by client name
|
||||
const interest = await getInterestByFieldAsync('Full Name', clientName);
|
||||
|
||||
if (!interest) {
|
||||
return {
|
||||
clientName,
|
||||
fileName,
|
||||
processed: false,
|
||||
error: `No interest found for client: ${clientName}`
|
||||
};
|
||||
}
|
||||
|
||||
// Generate unique filename
|
||||
const timestamp = Date.now();
|
||||
const uploadFileName = `EOIs/${interest.Id}-${timestamp}-${fileName}`;
|
||||
|
||||
// Upload to MinIO
|
||||
await uploadFile(uploadFileName, attachment.content, 'application/pdf');
|
||||
|
||||
// Update interest with EOI document
|
||||
const documentData = {
|
||||
title: fileName,
|
||||
filename: uploadFileName,
|
||||
url: `/api/files/proxy-download?fileName=${encodeURIComponent(uploadFileName)}`,
|
||||
size: attachment.size,
|
||||
mimetype: 'application/pdf',
|
||||
icon: 'mdi-file-pdf-box',
|
||||
uploadedAt: new Date().toISOString(),
|
||||
source: 'email',
|
||||
from: from
|
||||
};
|
||||
|
||||
// Get existing documents and add new one
|
||||
const existingDocs = interest['EOI Document'] || [];
|
||||
const updatedDocs = [...existingDocs, documentData];
|
||||
|
||||
// Update interest
|
||||
await updateInterest(interest.Id.toString(), {
|
||||
'EOI Document': updatedDocs,
|
||||
'EOI Status': 'Signed',
|
||||
'Sales Process Level': 'Signed LOI and NDA'
|
||||
});
|
||||
|
||||
console.log(`[Process Sales EOIs] Successfully processed EOI for ${clientName}`);
|
||||
|
||||
return {
|
||||
clientName,
|
||||
interestId: interest.Id.toString(),
|
||||
fileName,
|
||||
processed: true
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error(`[Process Sales EOIs] Error processing attachment:`, error);
|
||||
return {
|
||||
clientName: 'Unknown',
|
||||
fileName,
|
||||
processed: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function extractClientName(fileName: string, subject: string): string | null {
|
||||
// Try to extract from filename patterns like:
|
||||
// "John_Doe_EOI_signed.pdf"
|
||||
// "EOI_John_Doe.pdf"
|
||||
// "John Doe - EOI.pdf"
|
||||
|
||||
// First try filename
|
||||
const filePatterns = [
|
||||
/^(.+?)[-_]EOI/i,
|
||||
/EOI[-_](.+?)\.pdf/i,
|
||||
/^(.+?)_signed/i,
|
||||
/^(.+?)\s*-\s*EOI/i
|
||||
];
|
||||
|
||||
for (const pattern of filePatterns) {
|
||||
const match = fileName.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return match[1].replace(/[_-]/g, ' ').trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Then try subject
|
||||
const subjectPatterns = [
|
||||
/EOI\s+(?:for\s+)?(.+?)(?:\s+signed)?$/i,
|
||||
/Signed\s+EOI\s*[-:]?\s*(.+)$/i,
|
||||
/(.+?)\s*EOI\s*(?:signed|completed)/i
|
||||
];
|
||||
|
||||
for (const pattern of subjectPatterns) {
|
||||
const match = subject.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -66,17 +66,28 @@ export default defineEventHandler(async (event) => {
|
||||
}
|
||||
};
|
||||
|
||||
const testImapConnection = () => {
|
||||
const testImapConnection = (retryCount = 0): Promise<boolean> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log('[test-connection] Testing IMAP connection...');
|
||||
console.log(`[test-connection] Testing IMAP connection... (Attempt ${retryCount + 1}/3)`);
|
||||
const imap = new Imap(imapConfig);
|
||||
|
||||
// Add a timeout to prevent hanging
|
||||
const timeout = setTimeout(() => {
|
||||
console.error('[test-connection] IMAP connection timeout');
|
||||
imap.end();
|
||||
reject(new Error('IMAP connection timeout after 10 seconds'));
|
||||
}, 10000); // 10 second timeout
|
||||
|
||||
// Retry on timeout if we haven't exceeded max retries
|
||||
if (retryCount < 2) {
|
||||
console.log('[test-connection] Retrying IMAP connection after timeout...');
|
||||
setTimeout(() => {
|
||||
testImapConnection(retryCount + 1)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}, (retryCount + 1) * 1000); // Exponential backoff
|
||||
} else {
|
||||
reject(new Error('IMAP connection timeout after 15 seconds and 3 attempts'));
|
||||
}
|
||||
}, 15000); // 15 second timeout per attempt
|
||||
|
||||
imap.once('ready', () => {
|
||||
console.log('[test-connection] IMAP connection successful');
|
||||
@@ -88,7 +99,25 @@ export default defineEventHandler(async (event) => {
|
||||
imap.once('error', (err: Error) => {
|
||||
console.error('[test-connection] IMAP connection error:', err);
|
||||
clearTimeout(timeout);
|
||||
reject(err);
|
||||
|
||||
// Retry on certain errors if we haven't exceeded max retries
|
||||
const shouldRetry = retryCount < 2 && (
|
||||
err.message.includes('ECONNRESET') ||
|
||||
err.message.includes('ETIMEDOUT') ||
|
||||
err.message.includes('ENOTFOUND') ||
|
||||
err.message.includes('socket hang up')
|
||||
);
|
||||
|
||||
if (shouldRetry) {
|
||||
console.log(`[test-connection] Retrying IMAP connection after error: ${err.message}`);
|
||||
setTimeout(() => {
|
||||
testImapConnection(retryCount + 1)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}, (retryCount + 1) * 1000); // Exponential backoff
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
@@ -98,7 +127,7 @@ export default defineEventHandler(async (event) => {
|
||||
try {
|
||||
await testImapConnection();
|
||||
} catch (imapError: any) {
|
||||
console.error('[test-connection] IMAP connection failed:', imapError);
|
||||
console.error('[test-connection] IMAP connection failed after all retries:', imapError);
|
||||
throw new Error(`IMAP connection failed: ${imapError.message || imapError}`);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user