From cdb89553e00bde87f1637b4e42223b07c91da932 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 6 Apr 2026 14:43:27 -0400 Subject: [PATCH] feat: add contact card, deferred tool responses, and reconnection logic - request_contact tool shows on-screen card for name/email verification - Deferred tool responses let the UI wait for user confirmation - WebSocket close preserves transcript and enables reconnection - Reconnect seeds new Gemini session with prior conversation context Co-Authored-By: Claude Opus 4.6 (1M context) --- .../configurator/VoiceAgentProvider.tsx | 150 ++++++++++++++++-- 1 file changed, 135 insertions(+), 15 deletions(-) diff --git a/src/components/configurator/VoiceAgentProvider.tsx b/src/components/configurator/VoiceAgentProvider.tsx index 1b8af8f..79b0ce8 100644 --- a/src/components/configurator/VoiceAgentProvider.tsx +++ b/src/components/configurator/VoiceAgentProvider.tsx @@ -1,6 +1,6 @@ 'use client'; -import { createContext, useContext, useState, useRef, useCallback, type ReactNode } from 'react'; +import { createContext, useContext, useState, useRef, useCallback, useEffect, type ReactNode } from 'react'; import type { WizardFormData } from './WizardContainer'; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -13,6 +13,11 @@ export interface TranscriptEntry { type ConnectionStatus = 'idle' | 'connecting' | 'active' | 'ending' | 'error'; +export interface PendingContact { + name: string; + email: string; +} + interface VoiceAgentContextValue { status: ConnectionStatus; errorMessage: string | null; @@ -28,6 +33,11 @@ interface VoiceAgentContextValue { endConversation: () => void; completedBrief: string | null; completedFormData: WizardFormData | null; + pendingContact: PendingContact | null; + confirmContact: () => void; + updatePendingContact: (field: 'name' | 'email', value: string) => void; + canReconnect: boolean; + reconnect: () => Promise; } // ─── Context ───────────────────────────────────────────────────────────────── @@ -133,9 +143,15 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi const [agentAmplitude, setAgentAmplitude] = useState(0); const [completedBrief, setCompletedBrief] = useState(null); const [completedFormData, setCompletedFormData] = useState(null); + const [pendingContact, setPendingContact] = useState(null); + const [canReconnect, setCanReconnect] = useState(false); const turnCompleteRef = useRef(true); const briefSubmittedRef = useRef(false); + const pendingContactRef = useRef(null); + const pendingContactCallIdRef = useRef(''); + const reconnectTranscriptRef = useRef([]); + const statusRef = useRef('idle'); const wsRef = useRef(null); const mediaStreamRef = useRef(null); const audioContextRef = useRef(null); @@ -144,6 +160,9 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi const analyserRef = useRef(null); const animFrameRef = useRef(0); + // Keep statusRef in sync for use in closures + useEffect(() => { statusRef.current = status; }, [status]); + const addTranscript = useCallback((role: 'user' | 'agent', text: string) => { setTranscript((prev) => { const last = prev[prev.length - 1]; @@ -193,6 +212,16 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi } } + if (name === 'request_contact') { + const { name: contactName, email: contactEmail } = args as { name: string; email: string }; + const contact = { name: contactName, email: contactEmail }; + setPendingContact(contact); + pendingContactRef.current = contact; + pendingContactCallIdRef.current = callId; + // Don't return a tool response yet — wait for user confirmation via confirmContact() + return '__DEFERRED__'; + } + if (name === 'complete_brief') { // Prevent duplicate submissions if (briefSubmittedRef.current) return JSON.stringify({ success: true, message: 'Brief already submitted' }); @@ -204,7 +233,10 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi const summary = toolArgs.conversationSummary ?? ''; const existingScope = toolArgs.scope ?? ''; const combinedScope = [existingScope, summary].filter(Boolean).join('\n\n'); - const formData = { ...DEFAULT_FORM_DATA, ...toolArgs, scope: combinedScope, locale }; + // Use confirmed contact details from the on-screen card if available + const contactName = pendingContactRef.current?.name ?? toolArgs.name ?? ''; + const contactEmail = pendingContactRef.current?.email ?? toolArgs.email ?? ''; + const formData = { ...DEFAULT_FORM_DATA, ...toolArgs, name: contactName, email: contactEmail, scope: combinedScope, locale }; delete (formData as Record).conversationSummary; const res = await fetch('/api/configure', { method: 'POST', @@ -254,8 +286,15 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi const startConversation = useCallback(async () => { setStatus('connecting'); setErrorMessage(null); - setTranscript([]); - setSelections({}); + setCanReconnect(false); + // Only reset transcript/selections on fresh start (not reconnect) + if (reconnectTranscriptRef.current.length === 0) { + setTranscript([]); + setSelections({}); + setPendingContact(null); + pendingContactRef.current = null; + pendingContactCallIdRef.current = ''; + } setCompletedBrief(null); setCompletedFormData(null); briefSubmittedRef.current = false; @@ -364,12 +403,27 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi clearTimeout(setupTimeout); setStatus('active'); trackAmplitude(); - // Prompt the agent to introduce itself - ws.send(JSON.stringify({ - realtimeInput: { - text: 'Hello, please introduce yourself.', - }, - })); + + // If reconnecting, seed with prior conversation context + const priorTranscript = reconnectTranscriptRef.current; + if (priorTranscript.length > 0) { + const summary = priorTranscript + .map((e) => `${e.role === 'user' ? 'User' : 'Agent'}: ${e.text}`) + .join('\n'); + ws.send(JSON.stringify({ + realtimeInput: { + text: `We were having a conversation but got disconnected. Here is what was discussed so far:\n\n${summary}\n\nPlease acknowledge the reconnection briefly and continue where we left off.`, + }, + })); + reconnectTranscriptRef.current = []; + } else { + // Prompt the agent to introduce itself + ws.send(JSON.stringify({ + realtimeInput: { + text: 'Hello, please introduce yourself.', + }, + })); + } return; } @@ -409,9 +463,13 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi const responses = []; for (const call of calls) { const result = await handleToolCall(call.name, call.args ?? {}, call.id); - responses.push({ id: call.id, name: call.name, response: { result } }); + if (result !== '__DEFERRED__') { + responses.push({ id: call.id, name: call.name, response: { result } }); + } + } + if (responses.length > 0) { + ws.send(JSON.stringify({ toolResponse: { functionResponses: responses } })); } - ws.send(JSON.stringify({ toolResponse: { functionResponses: responses } })); } } }; @@ -424,8 +482,28 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi ws.onclose = (e) => { console.log('[VoiceAgent] WebSocket closed:', e.code, e.reason); - if (status === 'active') { - setStatus('idle'); + // Clean up audio but preserve transcript and selections + cancelAnimationFrame(animFrameRef.current); + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach((track) => track.stop()); + mediaStreamRef.current = null; + } + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + if (playbackContextRef.current) { + void playbackContextRef.current.close(); + playbackContextRef.current = null; + } + wsRef.current = null; + setUserAmplitude(0); + setAgentAmplitude(0); + // If we weren't intentionally ending, allow reconnect + if (statusRef.current !== 'ending' && !briefSubmittedRef.current) { + setStatus('error'); + setErrorMessage(null); + setCanReconnect(true); } }; } catch (error) { @@ -438,7 +516,7 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi setErrorMessage(`Failed to start: ${msg}`); } } - }, [locale, trackAmplitude, handleToolCall, playAudioChunk, addTranscript, status]); + }, [locale, trackAmplitude, handleToolCall, playAudioChunk, addTranscript]); const endConversation = useCallback(() => { setStatus('ending'); @@ -463,9 +541,46 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi setUserAmplitude(0); setAgentAmplitude(0); + setCanReconnect(false); + reconnectTranscriptRef.current = []; + pendingContactCallIdRef.current = ''; setStatus('idle'); }, []); + const updatePendingContact = useCallback((field: 'name' | 'email', value: string) => { + setPendingContact((prev) => { + if (!prev) return null; + const updated = { ...prev, [field]: value }; + pendingContactRef.current = updated; + return updated; + }); + }, []); + + const confirmContact = useCallback(() => { + if (!pendingContactRef.current) return; + // Send confirmation back through WebSocket so the agent knows + if (wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ + toolResponse: { + functionResponses: [{ + id: pendingContactCallIdRef.current, + name: 'request_contact', + response: { result: JSON.stringify({ confirmed: true, name: pendingContactRef.current.name, email: pendingContactRef.current.email }) }, + }], + }, + })); + } + pendingContactCallIdRef.current = ''; + }, []); + + const reconnect = useCallback(async () => { + setCanReconnect(false); + setErrorMessage(null); + // Preserve transcript for the new session to pick up context + reconnectTranscriptRef.current = transcript; + await startConversation(); + }, [startConversation, transcript]); + const toggleMic = useCallback(() => { if (!mediaStreamRef.current) return; const track = mediaStreamRef.current.getAudioTracks()[0]; @@ -490,6 +605,11 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi endConversation, completedBrief, completedFormData, + pendingContact, + confirmContact, + updatePendingContact, + canReconnect, + reconnect, }; return (