From a921731c52e7104469af673f89a995c93e57dbf4 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 17 Feb 2026 09:29:46 +0100 Subject: [PATCH] Pass tag confidence scores to AI assignment for weighted matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AI assignment path was receiving project tags as flat strings, losing the confidence scores from AI tagging. Now both the GPT path and the fallback algorithm weight tag matches by confidence — a 0.9 tag matters more than a 0.5 one. Co-Authored-By: Claude Opus 4.6 --- src/server/routers/assignment.ts | 14 +++++++++++- src/server/services/ai-assignment.ts | 33 +++++++++++++++++++++++----- src/server/services/anonymization.ts | 9 +++++--- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/server/routers/assignment.ts b/src/server/routers/assignment.ts index 94b66b5..d8bfd71 100644 --- a/src/server/routers/assignment.ts +++ b/src/server/routers/assignment.ts @@ -74,10 +74,22 @@ async function runAIAssignmentJob(jobId: string, roundId: string, userId: string description: true, tags: true, teamName: true, + projectTags: { + select: { tag: { select: { name: true } }, confidence: true }, + }, _count: { select: { assignments: { where: { roundId } } } }, }, }) + // Enrich projects with tag confidence data for AI matching + const projectsWithConfidence = projects.map((p) => ({ + ...p, + tagConfidences: p.projectTags.map((pt) => ({ + name: pt.tag.name, + confidence: pt.confidence, + })), + })) + const existingAssignments = await prisma.assignment.findMany({ where: { roundId }, select: { userId: true, projectId: true }, @@ -124,7 +136,7 @@ async function runAIAssignmentJob(jobId: string, roundId: string, userId: string const result = await generateAIAssignments( jurors, - projects, + projectsWithConfidence, constraints, userId, roundId, diff --git a/src/server/services/ai-assignment.ts b/src/server/services/ai-assignment.ts index 60bde8c..17e162b 100644 --- a/src/server/services/ai-assignment.ts +++ b/src/server/services/ai-assignment.ts @@ -38,7 +38,7 @@ const ASSIGNMENT_SYSTEM_PROMPT = `You are an expert jury assignment optimizer fo Match jurors to projects based on expertise alignment, workload balance, and coverage requirements. ## Matching Criteria (Weighted) -- Expertise Match (50%): How well juror tags/expertise align with project topics +- Expertise Match (50%): How well juror tags/expertise align with project topics. Project tags include a confidence score (0-1) — weight higher-confidence tags more heavily as they are more reliably assigned. A tag with confidence 0.9 is a strong signal; one with 0.5 is uncertain. - Workload Balance (30%): Distribute assignments evenly; prefer jurors below capacity - Minimum Target (20%): Prioritize jurors who haven't reached their minimum assignment count @@ -99,6 +99,7 @@ interface ProjectForAssignment { title: string description?: string | null tags: string[] + tagConfidences?: Array<{ name: string; confidence: number }> teamName?: string | null _count?: { assignments: number @@ -539,7 +540,7 @@ export function generateFallbackAssignments( return { juror, - score: calculateExpertiseScore(juror.expertiseTags, project.tags), + score: calculateExpertiseScore(juror.expertiseTags, project.tags, project.tagConfidences), loadScore: calculateLoadScore(currentLoad, maxLoad), underMinBonus: calculateUnderMinBonus(currentLoad, minTarget), } @@ -586,24 +587,44 @@ export function generateFallbackAssignments( /** * Calculate expertise match score based on tag overlap + * When tagConfidences are available, weights matches by confidence */ function calculateExpertiseScore( jurorTags: string[], - projectTags: string[] + projectTags: string[], + tagConfidences?: Array<{ name: string; confidence: number }> ): number { if (jurorTags.length === 0 || projectTags.length === 0) { return 0.5 // Neutral score if no tags } const jurorTagsLower = new Set(jurorTags.map((t) => t.toLowerCase())) + + // If we have confidence data, use weighted scoring + if (tagConfidences && tagConfidences.length > 0) { + let weightedMatches = 0 + let totalWeight = 0 + + for (const tc of tagConfidences) { + totalWeight += tc.confidence + if (jurorTagsLower.has(tc.name.toLowerCase())) { + weightedMatches += tc.confidence + } + } + + if (totalWeight === 0) return 0.5 + + const weightedRatio = weightedMatches / totalWeight + const hasExpertise = weightedMatches > 0 ? 0.2 : 0 + return Math.min(1, weightedRatio * 0.8 + hasExpertise) + } + + // Fallback: unweighted matching using flat tags const matchingTags = projectTags.filter((t) => jurorTagsLower.has(t.toLowerCase()) ) - // Score based on percentage of project tags matched const matchRatio = matchingTags.length / projectTags.length - - // Boost for having expertise, even if not all match const hasExpertise = matchingTags.length > 0 ? 0.2 : 0 return Math.min(1, matchRatio * 0.8 + hasExpertise) diff --git a/src/server/services/anonymization.ts b/src/server/services/anonymization.ts index 9b25656..0032cb0 100644 --- a/src/server/services/anonymization.ts +++ b/src/server/services/anonymization.ts @@ -52,7 +52,7 @@ export interface AnonymizedProject { anonymousId: string title: string description: string | null - tags: string[] + tags: Array<{ name: string; confidence: number }> teamName: string | null } @@ -209,6 +209,7 @@ interface ProjectInput { title: string description?: string | null tags: string[] + tagConfidences?: Array<{ name: string; confidence: number }> teamName?: string | null } @@ -253,7 +254,9 @@ export function anonymizeForAI( description: project.description ? truncateAndSanitize(project.description, DESCRIPTION_LIMITS.ASSIGNMENT) : null, - tags: project.tags, + tags: project.tagConfidences && project.tagConfidences.length > 0 + ? project.tagConfidences + : project.tags.map((t) => ({ name: t, confidence: 1.0 })), teamName: project.teamName ? `Team ${index + 1}` : null, } } @@ -524,7 +527,7 @@ export function validateAnonymization(data: AnonymizationResult): boolean { if (!checkText(project.title)) return false if (!checkText(project.description)) return false for (const tag of project.tags) { - if (!checkText(tag)) return false + if (!checkText(typeof tag === 'string' ? tag : tag.name)) return false } }