fix: respond to request_contact immediately, send confirmation as text
All checks were successful
Build & Push / build-and-push (push) Successful in 1m43s

The deferred tool response approach caused Gemini to timeout waiting.
Now request_contact responds immediately (telling the agent to wait),
and the confirm button sends a text message through the live WebSocket
to trigger complete_brief.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-06 15:52:53 -04:00
parent a174518496
commit bcc09542b7

View File

@@ -149,7 +149,6 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
const turnCompleteRef = useRef(true); const turnCompleteRef = useRef(true);
const briefSubmittedRef = useRef(false); const briefSubmittedRef = useRef(false);
const pendingContactRef = useRef<PendingContact | null>(null); const pendingContactRef = useRef<PendingContact | null>(null);
const pendingContactCallIdRef = useRef('');
const reconnectTranscriptRef = useRef<TranscriptEntry[]>([]); const reconnectTranscriptRef = useRef<TranscriptEntry[]>([]);
const statusRef = useRef<ConnectionStatus>('idle'); const statusRef = useRef<ConnectionStatus>('idle');
const wsRef = useRef<WebSocket | null>(null); const wsRef = useRef<WebSocket | null>(null);
@@ -217,9 +216,9 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
const contact = { name: contactName, email: contactEmail }; const contact = { name: contactName, email: contactEmail };
setPendingContact(contact); setPendingContact(contact);
pendingContactRef.current = contact; pendingContactRef.current = contact;
pendingContactCallIdRef.current = callId; // Respond immediately so Gemini doesn't timeout waiting for a tool response.
// Don't return a tool response yet — wait for user confirmation via confirmContact() // The agent is told to wait user confirmation comes as a text message via confirmContact().
return '__DEFERRED__'; return JSON.stringify({ success: true, message: 'Contact card is now shown on screen. Wait for the user to review and confirm before calling complete_brief. Do not proceed until you hear confirmation.' });
} }
if (name === 'complete_brief') { if (name === 'complete_brief') {
@@ -293,7 +292,6 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
setSelections({}); setSelections({});
setPendingContact(null); setPendingContact(null);
pendingContactRef.current = null; pendingContactRef.current = null;
pendingContactCallIdRef.current = '';
} }
setCompletedBrief(null); setCompletedBrief(null);
setCompletedFormData(null); setCompletedFormData(null);
@@ -463,15 +461,11 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
const responses = []; const responses = [];
for (const call of calls) { for (const call of calls) {
const result = await handleToolCall(call.name, call.args ?? {}, call.id); const result = await handleToolCall(call.name, call.args ?? {}, call.id);
if (result !== '__DEFERRED__') {
responses.push({ id: call.id, name: call.name, response: { result } }); responses.push({ id: call.id, name: call.name, response: { result } });
} }
}
if (responses.length > 0) {
ws.send(JSON.stringify({ toolResponse: { functionResponses: responses } })); ws.send(JSON.stringify({ toolResponse: { functionResponses: responses } }));
} }
} }
}
}; };
ws.onerror = (e) => { ws.onerror = (e) => {
@@ -543,7 +537,6 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
setAgentAmplitude(0); setAgentAmplitude(0);
setCanReconnect(false); setCanReconnect(false);
reconnectTranscriptRef.current = []; reconnectTranscriptRef.current = [];
pendingContactCallIdRef.current = '';
setStatus('idle'); setStatus('idle');
}, []); }, []);
@@ -558,19 +551,18 @@ export default function VoiceAgentProvider({ locale, children }: VoiceAgentProvi
const confirmContact = useCallback(() => { const confirmContact = useCallback(() => {
if (!pendingContactRef.current) return; if (!pendingContactRef.current) return;
// Send confirmation back through WebSocket so the agent knows // Send a text message to let the agent know the user confirmed their details
if (wsRef.current?.readyState === WebSocket.OPEN) { if (wsRef.current?.readyState === WebSocket.OPEN) {
const { name, email } = pendingContactRef.current;
wsRef.current.send(JSON.stringify({ wsRef.current.send(JSON.stringify({
toolResponse: { realtimeInput: {
functionResponses: [{ text: `The user has confirmed their contact details on screen. Name: ${name}, Email: ${email}. You may now call complete_brief.`,
id: pendingContactCallIdRef.current,
name: 'request_contact',
response: { result: JSON.stringify({ confirmed: true, name: pendingContactRef.current.name, email: pendingContactRef.current.email }) },
}],
}, },
})); }));
console.log('[VoiceAgent] Contact confirmed, notified agent');
} else {
console.warn('[VoiceAgent] Cannot confirm contact — WebSocket not open');
} }
pendingContactCallIdRef.current = '';
}, []); }, []);
const reconnect = useCallback(async () => { const reconnect = useCallback(async () => {