Major Updates to Network and Load Balancing

This commit is contained in:
2025-06-12 21:50:01 +02:00
parent f111f76a3b
commit f6508aa435
17 changed files with 2098 additions and 163 deletions

259
server/api/health.ts Normal file
View File

@@ -0,0 +1,259 @@
import { getNocoDbConfiguration } from '../utils/nocodb';
import { getMinioClient } from '../utils/minio';
import { useDirectus } from '#imports';
interface HealthCheckResult {
status: 'healthy' | 'unhealthy' | 'degraded';
timestamp: string;
uptime: number;
services: {
[key: string]: {
status: 'up' | 'down' | 'slow';
responseTime?: number;
error?: string;
};
};
memory: {
used: number;
total: number;
percentage: number;
};
environment: {
nodeVersion: string;
configStatus: 'complete' | 'partial' | 'missing';
missingVars?: string[];
};
}
export default defineEventHandler(async (event): Promise<HealthCheckResult> => {
const startTime = Date.now();
const result: HealthCheckResult = {
status: 'healthy',
timestamp: new Date().toISOString(),
uptime: process.uptime(),
services: {},
memory: {
used: 0,
total: 0,
percentage: 0
},
environment: {
nodeVersion: process.version,
configStatus: 'complete'
}
};
// Memory check
const memUsage = process.memoryUsage();
result.memory = {
used: Math.round(memUsage.heapUsed / 1024 / 1024), // MB
total: Math.round(memUsage.heapTotal / 1024 / 1024), // MB
percentage: Math.round((memUsage.heapUsed / memUsage.heapTotal) * 100)
};
// Check if memory usage is too high
if (result.memory.percentage > 90) {
result.status = 'unhealthy';
} else if (result.memory.percentage > 80) {
result.status = 'degraded';
}
// Environment variables check
const requiredEnvVars = [
'NUXT_NOCODB_URL',
'NUXT_NOCODB_TOKEN',
'NUXT_MINIO_ACCESS_KEY',
'NUXT_MINIO_SECRET_KEY',
'NUXT_DOCUMENSO_API_KEY',
'NUXT_DOCUMENSO_BASE_URL'
];
const missingVars = requiredEnvVars.filter(varName => !process.env[varName]);
if (missingVars.length > 0) {
result.environment.configStatus = missingVars.length > 3 ? 'missing' : 'partial';
result.environment.missingVars = missingVars;
result.status = 'unhealthy';
}
// Service health checks
// 1. NocoDB Check
try {
const nocoStart = Date.now();
const nocoConfig = getNocoDbConfiguration();
if (!nocoConfig.url || !nocoConfig.token) {
result.services.nocodb = {
status: 'down',
error: 'Missing configuration'
};
result.status = 'unhealthy';
} else {
const response = await fetch(`${nocoConfig.url}/api/v1/health`, {
headers: {
'xc-token': nocoConfig.token
},
signal: AbortSignal.timeout(5000) // 5 second timeout
});
const responseTime = Date.now() - nocoStart;
if (response.ok) {
result.services.nocodb = {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
if (responseTime > 3000) result.status = 'degraded';
} else {
result.services.nocodb = {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
result.status = 'unhealthy';
}
}
} catch (error) {
result.services.nocodb = {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
result.status = 'unhealthy';
}
// 2. Directus Check
try {
const directusStart = Date.now();
const { $directus } = useNuxtApp();
// Check if Directus URL is configured
const directusUrl = useRuntimeConfig().public.directus?.url;
if (!directusUrl) {
result.services.directus = {
status: 'down',
error: 'Missing configuration'
};
result.status = 'unhealthy';
} else {
const response = await fetch(`${directusUrl}/server/health`, {
signal: AbortSignal.timeout(5000)
});
const responseTime = Date.now() - directusStart;
if (response.ok) {
result.services.directus = {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
if (responseTime > 3000) result.status = 'degraded';
} else {
result.services.directus = {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
result.status = 'unhealthy';
}
}
} catch (error) {
result.services.directus = {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
result.status = 'unhealthy';
}
// 3. MinIO Check
try {
const minioStart = Date.now();
const minioClient = getMinioClient();
const bucketName = useRuntimeConfig().minio.bucketName;
// Try to check if bucket exists
const bucketExists = await minioClient.bucketExists(bucketName);
const responseTime = Date.now() - minioStart;
if (bucketExists) {
result.services.minio = {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
if (responseTime > 3000) result.status = 'degraded';
} else {
result.services.minio = {
status: 'down',
responseTime,
error: 'Bucket not found'
};
result.status = 'unhealthy';
}
} catch (error) {
result.services.minio = {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
result.status = 'unhealthy';
}
// 4. Documenso Check
try {
const documensoStart = Date.now();
const documensoUrl = process.env.NUXT_DOCUMENSO_BASE_URL;
const documensoKey = process.env.NUXT_DOCUMENSO_API_KEY;
if (!documensoUrl || !documensoKey) {
result.services.documenso = {
status: 'down',
error: 'Missing configuration'
};
result.status = 'unhealthy';
} else {
const response = await fetch(`${documensoUrl}/api/health`, {
headers: {
'Authorization': `Bearer ${documensoKey}`
},
signal: AbortSignal.timeout(5000)
});
const responseTime = Date.now() - documensoStart;
if (response.ok || response.status === 401) { // 401 means API is up but key might be wrong
result.services.documenso = {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
if (responseTime > 3000) result.status = 'degraded';
} else {
result.services.documenso = {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
result.status = 'unhealthy';
}
}
} catch (error) {
result.services.documenso = {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
// Documenso being down shouldn't make the whole app unhealthy
if (result.status === 'healthy') result.status = 'degraded';
}
// Overall response time
const totalTime = Date.now() - startTime;
if (totalTime > 10000 && result.status === 'healthy') {
result.status = 'degraded';
}
// Set appropriate HTTP status code
if (result.status === 'unhealthy') {
setResponseStatus(event, 503);
} else if (result.status === 'degraded') {
setResponseStatus(event, 200); // Still return 200 for degraded to not trigger Nginx failures
}
return result;
});

View File

@@ -0,0 +1,67 @@
export default defineNitroPlugin((nitroApp) => {
let lastHeapSnapshot: number = 0;
const heapSnapshotThreshold = 1024 * 1024 * 1024 * 6; // 6GB
// Monitor memory usage every 30 seconds
setInterval(() => {
const memUsage = process.memoryUsage();
const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
const heapTotalMB = Math.round(memUsage.heapTotal / 1024 / 1024);
const rssMB = Math.round(memUsage.rss / 1024 / 1024);
const externalMB = Math.round(memUsage.external / 1024 / 1024);
console.log(`[Memory Monitor] Heap: ${heapUsedMB}MB / ${heapTotalMB}MB | RSS: ${rssMB}MB | External: ${externalMB}MB`);
// Warning if memory usage is high
if (memUsage.heapUsed > heapSnapshotThreshold) {
console.warn(`[Memory Monitor] High memory usage detected: ${heapUsedMB}MB`);
// Take heap snapshot if we haven't taken one in the last hour
const now = Date.now();
if (now - lastHeapSnapshot > 3600000) { // 1 hour
lastHeapSnapshot = now;
console.warn('[Memory Monitor] Memory usage critical - consider taking heap snapshot');
// Force garbage collection if available
if (global.gc) {
console.log('[Memory Monitor] Running garbage collection...');
global.gc();
// Log memory after GC
setTimeout(() => {
const afterGC = process.memoryUsage();
const heapFreed = memUsage.heapUsed - afterGC.heapUsed;
console.log(`[Memory Monitor] GC freed ${Math.round(heapFreed / 1024 / 1024)}MB`);
}, 1000);
}
}
}
}, 30000); // Every 30 seconds
// Monitor event loop lag
let lastCheck = Date.now();
setInterval(() => {
const now = Date.now();
const lag = now - lastCheck - 1000;
if (lag > 100) {
console.warn(`[Memory Monitor] Event loop lag detected: ${lag}ms`);
}
lastCheck = now;
}, 1000);
// Add memory info to health endpoint
nitroApp.hooks.hook('request', async (event) => {
if (event.node.req.url === '/api/health') {
const memUsage = process.memoryUsage();
event.context.memoryInfo = {
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
rss: Math.round(memUsage.rss / 1024 / 1024),
external: Math.round(memUsage.external / 1024 / 1024),
arrayBuffers: Math.round(memUsage.arrayBuffers / 1024 / 1024)
};
}
});
});

View File

@@ -0,0 +1,30 @@
import { healthChecker } from '../utils/health-checker';
export default defineNitroPlugin(async (nitroApp) => {
console.log('[Startup Health] Initializing application health checks...');
// Perform initial health checks on startup
const startTime = Date.now();
try {
const isReady = await healthChecker.performStartupHealthChecks();
if (isReady) {
console.log(`[Startup Health] Application ready in ${Date.now() - startTime}ms`);
} else {
console.warn('[Startup Health] Application started but some services are unavailable');
console.warn('[Startup Health] Status:', healthChecker.getStartupStatus());
}
} catch (error) {
console.error('[Startup Health] Failed to perform startup checks:', error);
}
// Monitor service health periodically
setInterval(async () => {
try {
await healthChecker.performStartupHealthChecks();
} catch (error) {
console.error('[Startup Health] Periodic health check failed:', error);
}
}, 60000); // Check every minute
});

View File

@@ -0,0 +1,160 @@
interface CleanupTask {
name: string;
cleanup: () => void | Promise<void>;
interval?: number;
}
class CleanupManager {
private static instance: CleanupManager;
private tasks: Map<string, CleanupTask> = new Map();
private intervals: Map<string, NodeJS.Timeout> = new Map();
private constructor() {
// Register default cleanup tasks
this.registerDefaultTasks();
}
static getInstance(): CleanupManager {
if (!CleanupManager.instance) {
CleanupManager.instance = new CleanupManager();
}
return CleanupManager.instance;
}
private registerDefaultTasks() {
// Clean up old temporary files
this.registerTask({
name: 'temp-files-cleanup',
cleanup: async () => {
console.log('[Cleanup] Running temporary files cleanup...');
// Implementation would go here
},
interval: 3600000 // Every hour
});
// Clean up expired cache entries
this.registerTask({
name: 'cache-cleanup',
cleanup: async () => {
console.log('[Cleanup] Running cache cleanup...');
// Implementation would go here
},
interval: 1800000 // Every 30 minutes
});
}
registerTask(task: CleanupTask) {
this.tasks.set(task.name, task);
if (task.interval) {
// Clear existing interval if any
const existingInterval = this.intervals.get(task.name);
if (existingInterval) {
clearInterval(existingInterval);
}
// Set up new interval
const interval = setInterval(async () => {
try {
await task.cleanup();
} catch (error) {
console.error(`[Cleanup] Task '${task.name}' failed:`, error);
}
}, task.interval);
this.intervals.set(task.name, interval);
}
}
async runTask(taskName: string) {
const task = this.tasks.get(taskName);
if (!task) {
throw new Error(`Cleanup task '${taskName}' not found`);
}
try {
await task.cleanup();
console.log(`[Cleanup] Task '${taskName}' completed successfully`);
} catch (error) {
console.error(`[Cleanup] Task '${taskName}' failed:`, error);
throw error;
}
}
async runAllTasks() {
console.log('[Cleanup] Running all cleanup tasks...');
const results: { task: string; success: boolean; error?: any }[] = [];
for (const [name, task] of this.tasks) {
try {
await task.cleanup();
results.push({ task: name, success: true });
} catch (error) {
results.push({ task: name, success: false, error });
}
}
return results;
}
stopAllIntervals() {
for (const [name, interval] of this.intervals) {
clearInterval(interval);
console.log(`[Cleanup] Stopped interval for task '${name}'`);
}
this.intervals.clear();
}
}
export const cleanupManager = CleanupManager.getInstance();
// Utility functions for common cleanup operations
export const cleanupUtils = {
// Clean up old files in a directory
async cleanupOldFiles(directory: string, maxAgeMs: number) {
// Implementation would check file ages and remove old ones
console.log(`[Cleanup] Would clean files older than ${maxAgeMs}ms in ${directory}`);
},
// Clean up unused stream handles
cleanupStreams(streams: any[]) {
streams.forEach(stream => {
if (stream && typeof stream.destroy === 'function') {
stream.destroy();
}
});
},
// Clean up IMAP connections
cleanupIMAPConnections(connections: any[]) {
connections.forEach(conn => {
try {
if (conn && typeof conn.end === 'function') {
conn.end();
}
} catch (error) {
console.error('[Cleanup] Failed to close IMAP connection:', error);
}
});
},
// Memory-efficient array processing
async processInBatches<T, R>(
items: T[],
batchSize: number,
processor: (batch: T[]) => Promise<R[]>
): Promise<R[]> {
const results: R[] = [];
for (let i = 0; i < items.length; i += batchSize) {
const batch = items.slice(i, i + batchSize);
const batchResults = await processor(batch);
results.push(...batchResults);
// Allow garbage collection between batches
await new Promise(resolve => setImmediate(resolve));
}
return results;
}
};

View File

@@ -1,153 +1,139 @@
// Documeso API client utilities
interface DocumesoConfig {
apiUrl: string;
import { resilientHttp } from './resilient-http';
interface DocumensoConfig {
apiKey: string;
baseUrl: string;
}
interface DocumesoRecipient {
id: number;
documentId: number;
email: string;
name: string;
role: 'SIGNER' | 'APPROVER' | 'VIEWER';
signingOrder: number;
token: string;
signedAt: string | null;
readStatus: 'NOT_OPENED' | 'OPENED';
signingStatus: 'NOT_SIGNED' | 'SIGNED';
sendStatus: 'NOT_SENT' | 'SENT';
signingUrl: string;
}
interface DocumesoDocument {
id: number;
externalId: string;
userId: number;
teamId: number;
title: string;
status: 'DRAFT' | 'PENDING' | 'COMPLETED' | 'CANCELLED';
documentDataId: string;
createdAt: string;
updatedAt: string;
completedAt: string | null;
recipients: DocumesoRecipient[];
}
interface DocumesoListResponse {
documents: DocumesoDocument[];
total: number;
page: number;
perPage: number;
}
// Get Documeso configuration
const getDocumesoConfig = (): DocumesoConfig => {
return {
apiUrl: 'https://signatures.portnimara.dev/api/v1',
apiKey: 'Bearer api_malptg62zqyb0wrp'
};
};
// Fetch a single document by ID
export const getDocumesoDocument = async (documentId: number): Promise<DocumesoDocument> => {
const config = getDocumesoConfig();
export const getDocumensoConfiguration = (): DocumensoConfig => {
const apiKey = process.env.NUXT_DOCUMENSO_API_KEY;
const baseUrl = process.env.NUXT_DOCUMENSO_BASE_URL;
try {
const response = await $fetch<DocumesoDocument>(`${config.apiUrl}/documents/${documentId}`, {
headers: {
'Authorization': config.apiKey,
'Content-Type': 'application/json'
}
});
return response;
} catch (error) {
console.error('Failed to fetch Documeso document:', error);
throw error;
if (!apiKey || !baseUrl) {
throw new Error('Documenso configuration missing');
}
return { apiKey, baseUrl };
};
// Search documents by external ID (e.g., 'loi-94')
export const searchDocumesoDocuments = async (externalId?: string): Promise<DocumesoDocument[]> => {
const config = getDocumesoConfig();
// Helper function for resilient Documenso requests
async function documensoFetch<T>(endpoint: string, options: RequestInit = {}): Promise<T> {
const config = getDocumensoConfiguration();
const url = `${config.baseUrl}${endpoint}`;
try {
const response = await $fetch<DocumesoListResponse>(`${config.apiUrl}/documents`, {
const response = await resilientHttp.fetchWithRetry(
url,
{
...options,
headers: {
'Authorization': config.apiKey,
'Content-Type': 'application/json'
'Authorization': `Bearer ${config.apiKey}`,
'Content-Type': 'application/json',
...options.headers
},
params: {
perPage: 100
}
serviceName: 'documenso'
},
{
maxRetries: options.method === 'POST' || options.method === 'PUT' ? 2 : 3,
timeout: 20000, // 20 seconds for document operations
retryableStatuses: [408, 429, 500, 502, 503, 504]
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Documenso request failed: ${response.status} - ${errorText}`);
}
return response.json();
}
export const checkDocumentStatus = async (documentId: string): Promise<any> => {
console.log('[Documenso] Checking document status:', documentId);
try {
const result = await documensoFetch<any>(`/api/v1/documents/${documentId}`, {
method: 'GET'
});
// If externalId is provided, filter by it
if (externalId) {
return response.documents.filter(doc => doc.externalId === externalId);
}
return response.documents;
console.log('[Documenso] Document status retrieved:', result.status);
return result;
} catch (error) {
console.error('Failed to search Documeso documents:', error);
console.error('[Documenso] Failed to check document status:', error);
throw error;
}
};
// Get document by external ID (e.g., 'loi-94')
export const getDocumesoDocumentByExternalId = async (externalId: string): Promise<DocumesoDocument | null> => {
const documents = await searchDocumesoDocuments(externalId);
return documents.length > 0 ? documents[0] : null;
};
// Check signature status for a document
export const checkDocumentSignatureStatus = async (documentId: number): Promise<{
documentStatus: string;
unsignedRecipients: DocumesoRecipient[];
signedRecipients: DocumesoRecipient[];
clientSigned: boolean;
allSigned: boolean;
}> => {
const document = await getDocumesoDocument(documentId);
export const createDocument = async (templateId: string, data: any): Promise<any> => {
console.log('[Documenso] Creating document from template:', templateId);
const unsignedRecipients = document.recipients.filter(r => r.signingStatus === 'NOT_SIGNED');
const signedRecipients = document.recipients.filter(r => r.signingStatus === 'SIGNED');
// Check if client (signingOrder = 1) has signed
const clientRecipient = document.recipients.find(r => r.signingOrder === 1);
const clientSigned = clientRecipient ? clientRecipient.signingStatus === 'SIGNED' : false;
const allSigned = unsignedRecipients.length === 0;
return {
documentStatus: document.status,
unsignedRecipients,
signedRecipients,
clientSigned,
allSigned
};
};
// Get recipients who need to sign (excluding client)
export const getRecipientsToRemind = async (documentId: number): Promise<DocumesoRecipient[]> => {
const status = await checkDocumentSignatureStatus(documentId);
// Only remind if client has signed
if (!status.clientSigned) {
return [];
try {
const result = await documensoFetch<any>('/api/v1/documents', {
method: 'POST',
body: JSON.stringify({
templateId,
...data
})
});
console.log('[Documenso] Document created with ID:', result.id);
return result;
} catch (error) {
console.error('[Documenso] Failed to create document:', error);
throw error;
}
};
export const sendDocument = async (documentId: string, signers: any[]): Promise<any> => {
console.log('[Documenso] Sending document:', documentId);
// Return unsigned recipients with signingOrder > 1
return status.unsignedRecipients.filter(r => r.signingOrder > 1);
try {
const result = await documensoFetch<any>(`/api/v1/documents/${documentId}/send`, {
method: 'POST',
body: JSON.stringify({ signers })
});
console.log('[Documenso] Document sent successfully');
return result;
} catch (error) {
console.error('[Documenso] Failed to send document:', error);
throw error;
}
};
// Format recipient name for emails
export const formatRecipientName = (recipient: DocumesoRecipient): string => {
const firstName = recipient.name.split(' ')[0];
return firstName;
export const deleteDocument = async (documentId: string): Promise<any> => {
console.log('[Documenso] Deleting document:', documentId);
try {
const result = await documensoFetch<any>(`/api/v1/documents/${documentId}`, {
method: 'DELETE'
});
console.log('[Documenso] Document deleted successfully');
return result;
} catch (error) {
console.error('[Documenso] Failed to delete document:', error);
throw error;
}
};
// Get signing URL for a recipient
export const getSigningUrl = (recipient: DocumesoRecipient): string => {
return recipient.signingUrl;
export const verifyTemplate = async (templateId: string): Promise<boolean> => {
console.log('[Documenso] Verifying template:', templateId);
try {
await documensoFetch<any>(`/api/v1/templates/${templateId}`, {
method: 'GET'
});
console.log('[Documenso] Template verified successfully');
return true;
} catch (error) {
console.error('[Documenso] Template verification failed:', error);
return false;
}
};
// Get circuit breaker status for monitoring
export const getDocumensoHealthStatus = () => {
const status = resilientHttp.getCircuitBreakerStatus();
return status.documenso || { state: 'UNKNOWN', failures: 0 };
};

View File

@@ -0,0 +1,144 @@
import type { H3Event } from 'h3';
interface ServiceCheckResult {
status: 'up' | 'down' | 'slow';
responseTime?: number;
error?: string;
}
class HealthChecker {
private static instance: HealthChecker;
private isReady: boolean = false;
private startupChecks: Map<string, boolean> = new Map();
private requiredServices = ['nocodb', 'directus', 'minio'];
private constructor() {
this.initializeStartupChecks();
}
static getInstance(): HealthChecker {
if (!HealthChecker.instance) {
HealthChecker.instance = new HealthChecker();
}
return HealthChecker.instance;
}
private initializeStartupChecks() {
this.requiredServices.forEach(service => {
this.startupChecks.set(service, false);
});
}
async checkServiceWithRetry(
serviceName: string,
checkFunction: () => Promise<ServiceCheckResult>,
maxRetries: number = 3,
retryDelay: number = 1000
): Promise<ServiceCheckResult> {
let lastError: string | undefined;
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
const result = await checkFunction();
if (result.status === 'up' || result.status === 'slow') {
this.startupChecks.set(serviceName, true);
return result;
}
lastError = result.error;
} catch (error) {
lastError = error instanceof Error ? error.message : 'Unknown error';
}
if (attempt < maxRetries - 1) {
await new Promise(resolve => setTimeout(resolve, retryDelay * Math.pow(2, attempt)));
}
}
return {
status: 'down',
error: lastError || 'Max retries exceeded'
};
}
markServiceReady(serviceName: string) {
this.startupChecks.set(serviceName, true);
this.checkOverallReadiness();
}
markServiceDown(serviceName: string) {
this.startupChecks.set(serviceName, false);
this.isReady = false;
}
private checkOverallReadiness() {
const allRequired = this.requiredServices.every(service =>
this.startupChecks.get(service) === true
);
if (allRequired) {
this.isReady = true;
console.log('[HealthChecker] All required services are ready');
}
}
isApplicationReady(): boolean {
return this.isReady;
}
getStartupStatus(): { [key: string]: boolean } {
const status: { [key: string]: boolean } = {};
this.startupChecks.forEach((value, key) => {
status[key] = value;
});
return status;
}
async performStartupHealthChecks(): Promise<boolean> {
console.log('[HealthChecker] Performing startup health checks...');
// Import check functions dynamically to avoid circular dependencies
const { checkNocoDB, checkDirectus, checkMinIO } = await import('./service-checks');
// Check all services in parallel
const checks = await Promise.all([
this.checkServiceWithRetry('nocodb', checkNocoDB, 5, 2000),
this.checkServiceWithRetry('directus', checkDirectus, 5, 2000),
this.checkServiceWithRetry('minio', checkMinIO, 5, 2000)
]);
// Log results
checks.forEach((result, index) => {
const serviceName = ['nocodb', 'directus', 'minio'][index];
console.log(`[HealthChecker] ${serviceName}: ${result.status}${result.error ? ` - ${result.error}` : ''}`);
});
this.checkOverallReadiness();
return this.isReady;
}
}
export const healthChecker = HealthChecker.getInstance();
// Middleware to check if app is ready
export function createReadinessMiddleware() {
return defineEventHandler(async (event: H3Event) => {
// Skip health check endpoint itself
if (event.node.req.url === '/api/health') {
return;
}
// During startup, return 503 for all requests except health
if (!healthChecker.isApplicationReady()) {
const startupStatus = healthChecker.getStartupStatus();
setResponseStatus(event, 503);
return {
error: 'Service Unavailable',
message: 'Application is starting up. Please wait...',
startupStatus
};
}
});
}

View File

@@ -1,3 +1,5 @@
import { resilientHttp } from './resilient-http';
export interface PageInfo {
pageSize: number;
totalRows: number;
@@ -27,24 +29,43 @@ export const createTableUrl = (table: Table) => {
return url;
};
export const getInterests = async () =>
$fetch<InterestsResponse>(createTableUrl(Table.Interest), {
headers: {
"xc-token": getNocoDbConfiguration().token,
// Helper function for resilient NocoDB requests
async function nocoDbFetch<T>(url: string, options: RequestInit = {}): Promise<T> {
const response = await resilientHttp.fetchWithRetry(
url,
{
...options,
headers: {
'xc-token': getNocoDbConfiguration().token,
'Content-Type': 'application/json',
...options.headers
},
serviceName: 'nocodb'
},
params: {
limit: 1000,
},
});
{
maxRetries: options.method === 'POST' || options.method === 'PATCH' || options.method === 'DELETE' ? 2 : 3,
timeout: 15000
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(`NocoDB request failed: ${response.status} - ${errorText}`);
}
return response.json();
}
export const getInterests = async () => {
const url = createTableUrl(Table.Interest);
return nocoDbFetch<InterestsResponse>(url + '?limit=1000');
};
export const getInterestById = async (id: string) => {
console.log('[nocodb.getInterestById] Fetching interest ID:', id);
const result = await $fetch<Interest>(`${createTableUrl(Table.Interest)}/${id}`, {
headers: {
"xc-token": getNocoDbConfiguration().token,
},
});
const url = `${createTableUrl(Table.Interest)}/${id}`;
const result = await nocoDbFetch<any>(url);
console.log('[nocodb.getInterestById] Raw result from NocoDB:', {
id: result.Id,
@@ -166,13 +187,9 @@ export const updateInterest = async (id: string, data: Partial<Interest>, retryC
console.log('[nocodb.updateInterest] Request body:', JSON.stringify(cleanData, null, 2));
// Try sending as a single object first (as shown in the API docs)
const result = await $fetch<Interest>(url, {
const result = await nocoDbFetch<any>(url, {
method: "PATCH",
headers: {
"xc-token": getNocoDbConfiguration().token,
"Content-Type": "application/json"
},
body: cleanData
body: JSON.stringify(cleanData)
});
console.log('[nocodb.updateInterest] Update successful for ID:', id);
return result;
@@ -205,7 +222,7 @@ export const updateInterest = async (id: string, data: Partial<Interest>, retryC
}
};
export const createInterest = async (data: Partial<Interest>) => {
export const createInterest = async (data: Partial<any>) => {
console.log('[nocodb.createInterest] Creating interest with fields:', Object.keys(data));
// Create a clean data object that matches the InterestsRequest schema
@@ -253,12 +270,9 @@ export const createInterest = async (data: Partial<Interest>) => {
console.log('[nocodb.createInterest] URL:', url);
try {
const result = await $fetch<Interest>(url, {
const result = await nocoDbFetch<any>(url, {
method: "POST",
headers: {
"xc-token": getNocoDbConfiguration().token,
},
body: cleanData,
body: JSON.stringify(cleanData)
});
console.log('[nocodb.createInterest] Created interest with ID:', result.Id);
return result;
@@ -293,13 +307,9 @@ export const deleteInterest = async (id: string) => {
try {
// According to NocoDB API docs, DELETE requires ID in the body
const result = await $fetch(url, {
const result = await nocoDbFetch<any>(url, {
method: "DELETE",
headers: {
"xc-token": getNocoDbConfiguration().token,
"Content-Type": "application/json"
},
body: requestBody
body: JSON.stringify(requestBody)
});
console.log('[nocodb.deleteInterest] DELETE successful');
@@ -323,11 +333,22 @@ export const deleteInterest = async (id: string) => {
}
};
export const triggerWebhook = async (url: string, payload: any) =>
$fetch(url, {
export const triggerWebhook = async (url: string, payload: any) => {
// Webhooks might not need the same resilience as data operations
const response = await fetch(url, {
method: "POST",
body: payload,
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(payload)
});
if (!response.ok) {
throw new Error(`Webhook failed: ${response.status}`);
}
return response.json();
};
export const updateInterestEOIDocument = async (id: string, documentData: any) => {
console.log('[nocodb.updateInterestEOIDocument] Updating EOI document for interest:', id);

View File

@@ -0,0 +1,190 @@
interface QueuedRequest<T> {
id: string;
priority: 'high' | 'normal' | 'low';
execute: () => Promise<T>;
resolve: (value: T) => void;
reject: (error: any) => void;
timestamp: number;
retries: number;
}
interface QueueOptions {
maxConcurrent: number;
maxQueueSize: number;
requestTimeout: number;
maxRetries: number;
}
class RequestQueue {
private static instances: Map<string, RequestQueue> = new Map();
private queue: QueuedRequest<any>[] = [];
private activeRequests: number = 0;
private requestId: number = 0;
constructor(private name: string, private options: QueueOptions) {
this.options = {
maxConcurrent: 5,
maxQueueSize: 100,
requestTimeout: 30000,
maxRetries: 2,
...options
};
}
static getInstance(name: string, options?: Partial<QueueOptions>): RequestQueue {
if (!RequestQueue.instances.has(name)) {
RequestQueue.instances.set(name, new RequestQueue(name, options as QueueOptions));
}
return RequestQueue.instances.get(name)!;
}
async add<T>(
execute: () => Promise<T>,
priority: 'high' | 'normal' | 'low' = 'normal'
): Promise<T> {
if (this.queue.length >= this.options.maxQueueSize) {
throw new Error(`Queue ${this.name} is full (${this.queue.length} items)`);
}
return new Promise<T>((resolve, reject) => {
const request: QueuedRequest<T> = {
id: `${this.name}-${++this.requestId}`,
priority,
execute,
resolve,
reject,
timestamp: Date.now(),
retries: 0
};
// Insert based on priority
if (priority === 'high') {
// Find the first non-high priority item
const index = this.queue.findIndex(item => item.priority !== 'high');
if (index === -1) {
this.queue.push(request);
} else {
this.queue.splice(index, 0, request);
}
} else if (priority === 'low') {
this.queue.push(request);
} else {
// Normal priority - insert before low priority items
const index = this.queue.findIndex(item => item.priority === 'low');
if (index === -1) {
this.queue.push(request);
} else {
this.queue.splice(index, 0, request);
}
}
console.log(`[Queue ${this.name}] Added request ${request.id} (priority: ${priority}, queue size: ${this.queue.length})`);
// Process queue
this.processQueue();
});
}
private async processQueue() {
while (this.queue.length > 0 && this.activeRequests < this.options.maxConcurrent) {
const request = this.queue.shift();
if (!request) continue;
this.activeRequests++;
// Process request with timeout
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error('Request timeout')), this.options.requestTimeout);
});
try {
console.log(`[Queue ${this.name}] Processing request ${request.id} (active: ${this.activeRequests}/${this.options.maxConcurrent})`);
const result = await Promise.race([
request.execute(),
timeoutPromise
]);
request.resolve(result);
console.log(`[Queue ${this.name}] Request ${request.id} completed successfully`);
} catch (error) {
console.error(`[Queue ${this.name}] Request ${request.id} failed:`, error);
// Retry logic
if (request.retries < this.options.maxRetries) {
request.retries++;
console.log(`[Queue ${this.name}] Retrying request ${request.id} (attempt ${request.retries + 1}/${this.options.maxRetries + 1})`);
// Re-queue with same priority
this.queue.unshift(request);
} else {
request.reject(error);
}
} finally {
this.activeRequests--;
// Continue processing
if (this.queue.length > 0) {
setImmediate(() => this.processQueue());
}
}
}
}
getStatus() {
return {
name: this.name,
queueLength: this.queue.length,
activeRequests: this.activeRequests,
maxConcurrent: this.options.maxConcurrent,
queueByPriority: {
high: this.queue.filter(r => r.priority === 'high').length,
normal: this.queue.filter(r => r.priority === 'normal').length,
low: this.queue.filter(r => r.priority === 'low').length
}
};
}
clear() {
const cleared = this.queue.length;
this.queue.forEach(request => {
request.reject(new Error('Queue cleared'));
});
this.queue = [];
console.log(`[Queue ${this.name}] Cleared ${cleared} requests`);
return cleared;
}
}
// Pre-configured queues for different operations
export const queues = {
documenso: RequestQueue.getInstance('documenso', {
maxConcurrent: 3,
maxQueueSize: 50,
requestTimeout: 45000,
maxRetries: 2
}),
email: RequestQueue.getInstance('email', {
maxConcurrent: 2,
maxQueueSize: 30,
requestTimeout: 60000,
maxRetries: 1
}),
nocodb: RequestQueue.getInstance('nocodb', {
maxConcurrent: 5,
maxQueueSize: 100,
requestTimeout: 20000,
maxRetries: 3
})
};
// Utility function to get all queue statuses
export function getAllQueueStatuses() {
return {
documenso: queues.documenso.getStatus(),
email: queues.email.getStatus(),
nocodb: queues.nocodb.getStatus()
};
}

View File

@@ -0,0 +1,229 @@
interface RetryOptions {
maxRetries?: number;
initialDelay?: number;
maxDelay?: number;
backoffMultiplier?: number;
timeout?: number;
retryableStatuses?: number[];
retryableErrors?: string[];
}
interface CircuitBreakerOptions {
failureThreshold?: number;
resetTimeout?: number;
halfOpenRequests?: number;
}
enum CircuitState {
CLOSED = 'CLOSED',
OPEN = 'OPEN',
HALF_OPEN = 'HALF_OPEN'
}
class CircuitBreaker {
private state: CircuitState = CircuitState.CLOSED;
private failures: number = 0;
private lastFailureTime: number = 0;
private halfOpenRequests: number = 0;
constructor(private options: CircuitBreakerOptions = {}) {
this.options = {
failureThreshold: 10,
resetTimeout: 30000, // 30 seconds
halfOpenRequests: 3,
...options
};
}
async execute<T>(fn: () => Promise<T>): Promise<T> {
// Check circuit state
if (this.state === CircuitState.OPEN) {
const timeSinceLastFailure = Date.now() - this.lastFailureTime;
if (timeSinceLastFailure > this.options.resetTimeout!) {
this.state = CircuitState.HALF_OPEN;
this.halfOpenRequests = 0;
} else {
throw new Error('Circuit breaker is OPEN');
}
}
if (this.state === CircuitState.HALF_OPEN) {
if (this.halfOpenRequests >= this.options.halfOpenRequests!) {
throw new Error('Circuit breaker is HALF_OPEN - max requests reached');
}
this.halfOpenRequests++;
}
try {
const result = await fn();
// Success - reset failures
if (this.state === CircuitState.HALF_OPEN) {
this.state = CircuitState.CLOSED;
this.failures = 0;
}
return result;
} catch (error) {
this.failures++;
this.lastFailureTime = Date.now();
if (this.failures >= this.options.failureThreshold!) {
this.state = CircuitState.OPEN;
console.error(`[CircuitBreaker] Opening circuit after ${this.failures} failures`);
}
throw error;
}
}
getState(): { state: CircuitState; failures: number } {
return {
state: this.state,
failures: this.failures
};
}
reset(): void {
this.state = CircuitState.CLOSED;
this.failures = 0;
this.halfOpenRequests = 0;
}
}
export class ResilientHttpClient {
private circuitBreakers: Map<string, CircuitBreaker> = new Map();
constructor(
private defaultRetryOptions: RetryOptions = {},
private defaultCircuitBreakerOptions: CircuitBreakerOptions = {}
) {
this.defaultRetryOptions = {
maxRetries: 3,
initialDelay: 1000,
maxDelay: 16000,
backoffMultiplier: 2,
timeout: 30000,
retryableStatuses: [408, 429, 500, 502, 503, 504],
retryableErrors: ['ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'ECONNRESET'],
...defaultRetryOptions
};
}
private getCircuitBreaker(serviceName: string): CircuitBreaker {
if (!this.circuitBreakers.has(serviceName)) {
this.circuitBreakers.set(
serviceName,
new CircuitBreaker(this.defaultCircuitBreakerOptions)
);
}
return this.circuitBreakers.get(serviceName)!;
}
async fetchWithRetry(
url: string,
options: RequestInit & { serviceName?: string } = {},
retryOptions: RetryOptions = {}
): Promise<Response> {
const mergedRetryOptions = { ...this.defaultRetryOptions, ...retryOptions };
const serviceName = options.serviceName || new URL(url).hostname;
const circuitBreaker = this.getCircuitBreaker(serviceName);
return circuitBreaker.execute(async () => {
return this.executeWithRetry(url, options, mergedRetryOptions);
});
}
private async executeWithRetry(
url: string,
options: RequestInit,
retryOptions: RetryOptions
): Promise<Response> {
let lastError: Error | null = null;
let delay = retryOptions.initialDelay!;
for (let attempt = 0; attempt <= retryOptions.maxRetries!; attempt++) {
try {
// Add timeout to request
const controller = new AbortController();
const timeoutId = setTimeout(
() => controller.abort(),
retryOptions.timeout!
);
const response = await fetch(url, {
...options,
signal: controller.signal
});
clearTimeout(timeoutId);
// Check if response is retryable
if (
!response.ok &&
retryOptions.retryableStatuses!.includes(response.status) &&
attempt < retryOptions.maxRetries!
) {
console.warn(
`[ResilientHttp] Retryable status ${response.status} for ${url}, attempt ${attempt + 1}/${retryOptions.maxRetries}`
);
await this.delay(delay);
delay = Math.min(delay * retryOptions.backoffMultiplier!, retryOptions.maxDelay!);
continue;
}
return response;
} catch (error: any) {
lastError = error;
// Check if error is retryable
const isRetryable = retryOptions.retryableErrors!.some(
errType => error.message?.includes(errType) || error.code === errType
);
if (isRetryable && attempt < retryOptions.maxRetries!) {
console.warn(
`[ResilientHttp] Retryable error for ${url}: ${error.message}, attempt ${attempt + 1}/${retryOptions.maxRetries}`
);
await this.delay(delay);
delay = Math.min(delay * retryOptions.backoffMultiplier!, retryOptions.maxDelay!);
continue;
}
throw error;
}
}
throw lastError || new Error(`Failed after ${retryOptions.maxRetries} retries`);
}
private delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
getCircuitBreakerStatus(): { [serviceName: string]: { state: string; failures: number } } {
const status: { [serviceName: string]: { state: string; failures: number } } = {};
this.circuitBreakers.forEach((breaker, serviceName) => {
status[serviceName] = breaker.getState();
});
return status;
}
}
// Create a singleton instance
export const resilientHttp = new ResilientHttpClient(
{
maxRetries: 3,
initialDelay: 1000,
maxDelay: 8000,
backoffMultiplier: 2,
timeout: 30000
},
{
failureThreshold: 10,
resetTimeout: 30000,
halfOpenRequests: 3
}
);

View File

@@ -0,0 +1,162 @@
import { getNocoDbConfiguration } from './nocodb';
import { getMinioClient } from './minio';
interface ServiceCheckResult {
status: 'up' | 'down' | 'slow';
responseTime?: number;
error?: string;
}
export async function checkNocoDB(): Promise<ServiceCheckResult> {
const startTime = Date.now();
try {
const config = getNocoDbConfiguration();
if (!config.url || !config.token) {
return {
status: 'down',
error: 'Missing NocoDB configuration'
};
}
const response = await fetch(`${config.url}/api/v1/health`, {
headers: {
'xc-token': config.token
},
signal: AbortSignal.timeout(5000)
});
const responseTime = Date.now() - startTime;
if (response.ok) {
return {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
}
return {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
} catch (error) {
return {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
export async function checkDirectus(): Promise<ServiceCheckResult> {
const startTime = Date.now();
try {
const directusUrl = useRuntimeConfig().public.directus?.url;
if (!directusUrl) {
return {
status: 'down',
error: 'Missing Directus configuration'
};
}
const response = await fetch(`${directusUrl}/server/health`, {
signal: AbortSignal.timeout(5000)
});
const responseTime = Date.now() - startTime;
if (response.ok) {
return {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
}
return {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
} catch (error) {
return {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
export async function checkMinIO(): Promise<ServiceCheckResult> {
const startTime = Date.now();
try {
const minioClient = getMinioClient();
const bucketName = useRuntimeConfig().minio.bucketName;
const bucketExists = await minioClient.bucketExists(bucketName);
const responseTime = Date.now() - startTime;
if (bucketExists) {
return {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
}
return {
status: 'down',
responseTime,
error: 'Bucket not found'
};
} catch (error) {
return {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
export async function checkDocumenso(): Promise<ServiceCheckResult> {
const startTime = Date.now();
try {
const documensoUrl = process.env.NUXT_DOCUMENSO_BASE_URL;
const documensoKey = process.env.NUXT_DOCUMENSO_API_KEY;
if (!documensoUrl || !documensoKey) {
return {
status: 'down',
error: 'Missing Documenso configuration'
};
}
const response = await fetch(`${documensoUrl}/api/health`, {
headers: {
'Authorization': `Bearer ${documensoKey}`
},
signal: AbortSignal.timeout(5000)
});
const responseTime = Date.now() - startTime;
if (response.ok || response.status === 401) {
return {
status: responseTime > 3000 ? 'slow' : 'up',
responseTime
};
}
return {
status: 'down',
responseTime,
error: `HTTP ${response.status}`
};
} catch (error) {
return {
status: 'down',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}