/** * Copyright 2026 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * app.js: JS code for the ADK Gemini Live API Toolkit demo app. */ /** * WebSocket handling */ // Connect the server with a WebSocket connection const userId = "demo-user"; const sessionId = "demo-session-" + Math.random().toString(36).substring(7); let websocket = null; let is_audio = false; // Get checkbox elements for RunConfig options const enableProactivityCheckbox = document.getElementById("enableProactivity"); const enableAffectiveDialogCheckbox = document.getElementById("enableAffectiveDialog"); // Reconnect WebSocket when RunConfig options change function handleRunConfigChange() { if (websocket && websocket.readyState === WebSocket.OPEN) { addSystemMessage("Reconnecting with updated settings..."); addConsoleEntry('outgoing', 'Reconnecting due to settings change', { proactivity: enableProactivityCheckbox.checked, affective_dialog: enableAffectiveDialogCheckbox.checked }, 'πŸ”„', 'system'); websocket.close(); // connectWebsocket() will be called by onclose handler after delay } } // Add change listeners to RunConfig checkboxes enableProactivityCheckbox.addEventListener("change", handleRunConfigChange); enableAffectiveDialogCheckbox.addEventListener("change", handleRunConfigChange); // Build WebSocket URL with RunConfig options as query parameters function getWebSocketUrl() { // Use wss:// for HTTPS pages, ws:// for HTTP (localhost development) const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; const baseUrl = wsProtocol + "//" + window.location.host + "/ws/" + userId + "/" + sessionId; const params = new URLSearchParams(); // Add proactivity option if checked if (enableProactivityCheckbox && enableProactivityCheckbox.checked) { params.append("proactivity", "true"); } // Add affective dialog option if checked if (enableAffectiveDialogCheckbox && enableAffectiveDialogCheckbox.checked) { params.append("affective_dialog", "true"); } const queryString = params.toString(); return queryString ? baseUrl + "?" + queryString : baseUrl; } // Get DOM elements const messageForm = document.getElementById("messageForm"); const messageInput = document.getElementById("message"); const messagesDiv = document.getElementById("messages"); const statusIndicator = document.getElementById("statusIndicator"); const statusText = document.getElementById("statusText"); const consoleContent = document.getElementById("consoleContent"); const clearConsoleBtn = document.getElementById("clearConsole"); const showAudioEventsCheckbox = document.getElementById("showAudioEvents"); let currentMessageId = null; let currentBubbleElement = null; let currentInputTranscriptionId = null; let currentInputTranscriptionElement = null; let currentOutputTranscriptionId = null; let currentOutputTranscriptionElement = null; let inputTranscriptionFinished = false; // Track if input transcription is complete for this turn let hasOutputTranscriptionInTurn = false; // Track if output transcription delivered the response // Helper function to clean spaces between CJK characters // Removes spaces between Japanese/Chinese/Korean characters while preserving spaces around Latin text function cleanCJKSpaces(text) { // CJK Unicode ranges: Hiragana, Katakana, Kanji, CJK Unified Ideographs, Full width forms const cjkPattern = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9faf\uff00-\uffef]/; // Remove spaces between two CJK characters return text.replace(/(\S)\s+(?=\S)/g, (match, char1) => { // Get the character after the space(s) const nextCharMatch = text.match(new RegExp(char1 + '\\s+(.)', 'g')); if (nextCharMatch && nextCharMatch.length > 0) { const char2 = nextCharMatch[0].slice(-1); // If both characters are CJK, remove the space if (cjkPattern.test(char1) && cjkPattern.test(char2)) { return char1; } } return match; }); } // Console logging functionality function formatTimestamp() { const now = new Date(); return now.toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit', fractionalSecondDigits: 3 }); } function addConsoleEntry(type, content, data = null, emoji = null, author = null, isAudio = false) { // Skip audio events if checkbox is unchecked if (isAudio && !showAudioEventsCheckbox.checked) { return; } const entry = document.createElement("div"); entry.className = `console-entry ${type}`; const header = document.createElement("div"); header.className = "console-entry-header"; const leftSection = document.createElement("div"); leftSection.className = "console-entry-left"; // Add emoji icon if provided if (emoji) { const emojiIcon = document.createElement("span"); emojiIcon.className = "console-entry-emoji"; emojiIcon.textContent = emoji; leftSection.appendChild(emojiIcon); } // Add expand/collapse icon const expandIcon = document.createElement("span"); expandIcon.className = "console-expand-icon"; expandIcon.textContent = data ? "β–Ά" : ""; const typeLabel = document.createElement("span"); typeLabel.className = "console-entry-type"; typeLabel.textContent = type === 'outgoing' ? '↑ Upstream' : type === 'incoming' ? '↓ Downstream' : '⚠ Error'; leftSection.appendChild(expandIcon); leftSection.appendChild(typeLabel); // Add author badge if provided if (author) { const authorBadge = document.createElement("span"); authorBadge.className = "console-entry-author"; authorBadge.textContent = author; authorBadge.setAttribute('data-author', author); leftSection.appendChild(authorBadge); } const timestamp = document.createElement("span"); timestamp.className = "console-entry-timestamp"; timestamp.textContent = formatTimestamp(); header.appendChild(leftSection); header.appendChild(timestamp); const contentDiv = document.createElement("div"); contentDiv.className = "console-entry-content"; contentDiv.textContent = content; entry.appendChild(header); entry.appendChild(contentDiv); // JSON details (hidden by default) let jsonDiv = null; if (data) { jsonDiv = document.createElement("div"); jsonDiv.className = "console-entry-json collapsed"; const pre = document.createElement("pre"); pre.textContent = JSON.stringify(data, null, 2); jsonDiv.appendChild(pre); entry.appendChild(jsonDiv); // Make entry clickable if it has data entry.classList.add("expandable"); // Toggle expand/collapse on click entry.addEventListener("click", () => { const isExpanded = !jsonDiv.classList.contains("collapsed"); if (isExpanded) { // Collapse jsonDiv.classList.add("collapsed"); expandIcon.textContent = "β–Ά"; entry.classList.remove("expanded"); } else { // Expand jsonDiv.classList.remove("collapsed"); expandIcon.textContent = "β–Ό"; entry.classList.add("expanded"); } }); } consoleContent.appendChild(entry); consoleContent.scrollTop = consoleContent.scrollHeight; } function clearConsole() { consoleContent.innerHTML = ''; } // Clear console button handler clearConsoleBtn.addEventListener('click', clearConsole); // Update connection status UI function updateConnectionStatus(connected) { if (connected) { statusIndicator.classList.remove("disconnected"); statusText.textContent = "Connected"; } else { statusIndicator.classList.add("disconnected"); statusText.textContent = "Disconnected"; } } // Create a message bubble element function createMessageBubble(text, isUser, isPartial = false) { const messageDiv = document.createElement("div"); messageDiv.className = `message ${isUser ? "user" : "agent"}`; const bubbleDiv = document.createElement("div"); bubbleDiv.className = "bubble"; const textP = document.createElement("p"); textP.className = "bubble-text"; textP.textContent = text; // Add typing indicator for partial messages if (isPartial && !isUser) { const typingSpan = document.createElement("span"); typingSpan.className = "typing-indicator"; textP.appendChild(typingSpan); } bubbleDiv.appendChild(textP); messageDiv.appendChild(bubbleDiv); return messageDiv; } // Create an image message bubble element function createImageBubble(imageDataUrl, isUser) { const messageDiv = document.createElement("div"); messageDiv.className = `message ${isUser ? "user" : "agent"}`; const bubbleDiv = document.createElement("div"); bubbleDiv.className = "bubble image-bubble"; const img = document.createElement("img"); img.src = imageDataUrl; img.className = "bubble-image"; img.alt = "Captured image"; bubbleDiv.appendChild(img); messageDiv.appendChild(bubbleDiv); return messageDiv; } // Update existing message bubble text function updateMessageBubble(element, text, isPartial = false) { const textElement = element.querySelector(".bubble-text"); // Remove existing typing indicator const existingIndicator = textElement.querySelector(".typing-indicator"); if (existingIndicator) { existingIndicator.remove(); } textElement.textContent = text; // Add typing indicator for partial messages if (isPartial) { const typingSpan = document.createElement("span"); typingSpan.className = "typing-indicator"; textElement.appendChild(typingSpan); } } // Add a system message function addSystemMessage(text) { const messageDiv = document.createElement("div"); messageDiv.className = "system-message"; messageDiv.textContent = text; messagesDiv.appendChild(messageDiv); scrollToBottom(); } // Scroll to bottom of messages function scrollToBottom() { messagesDiv.scrollTop = messagesDiv.scrollHeight; } // Sanitize event data for console display (replace large audio data with summary) function sanitizeEventForDisplay(event) { // Deep clone the event object const sanitized = JSON.parse(JSON.stringify(event)); // Check for audio data in content.parts if (sanitized.content && sanitized.content.parts) { sanitized.content.parts = sanitized.content.parts.map(part => { if (part.inlineData && part.inlineData.data) { // Calculate byte size (base64 string length / 4 * 3, roughly) const byteSize = Math.floor(part.inlineData.data.length * 0.75); return { ...part, inlineData: { ...part.inlineData, data: `(${byteSize.toLocaleString()} bytes)` } }; } return part; }); } return sanitized; } // WebSocket handlers function connectWebsocket() { // Connect websocket const ws_url = getWebSocketUrl(); websocket = new WebSocket(ws_url); // Handle connection open websocket.onopen = function () { console.log("WebSocket connection opened."); updateConnectionStatus(true); addSystemMessage("Connected to ADK streaming server"); // Log to console addConsoleEntry('incoming', 'WebSocket Connected', { userId: userId, sessionId: sessionId, url: ws_url }, 'πŸ”Œ', 'system'); // Enable the Send button document.getElementById("sendButton").disabled = false; addSubmitHandler(); }; // Handle incoming messages websocket.onmessage = function (event) { // Parse the incoming ADK Event const adkEvent = JSON.parse(event.data); console.log("[AGENT TO CLIENT] ", adkEvent); // Log to console panel let eventSummary = 'Event'; let eventEmoji = 'πŸ“¨'; // Default emoji const author = adkEvent.author || 'system'; if (adkEvent.turnComplete) { eventSummary = 'Turn Complete'; eventEmoji = 'βœ…'; } else if (adkEvent.interrupted) { eventSummary = 'Interrupted'; eventEmoji = '⏸️'; } else if (adkEvent.inputTranscription) { // Show transcription text in summary const transcriptionText = adkEvent.inputTranscription.text || ''; const truncated = transcriptionText.length > 60 ? transcriptionText.substring(0, 60) + '...' : transcriptionText; eventSummary = `Input Transcription: "${truncated}"`; eventEmoji = 'πŸ“'; } else if (adkEvent.outputTranscription) { // Show transcription text in summary const transcriptionText = adkEvent.outputTranscription.text || ''; const truncated = transcriptionText.length > 60 ? transcriptionText.substring(0, 60) + '...' : transcriptionText; eventSummary = `Output Transcription: "${truncated}"`; eventEmoji = 'πŸ“'; } else if (adkEvent.usageMetadata) { // Show token usage information const usage = adkEvent.usageMetadata; const promptTokens = usage.promptTokenCount || 0; const responseTokens = usage.candidatesTokenCount || 0; const totalTokens = usage.totalTokenCount || 0; eventSummary = `Token Usage: ${totalTokens.toLocaleString()} total (${promptTokens.toLocaleString()} prompt + ${responseTokens.toLocaleString()} response)`; eventEmoji = 'πŸ“Š'; } else if (adkEvent.content && adkEvent.content.parts) { const hasText = adkEvent.content.parts.some(p => p.text); const hasAudio = adkEvent.content.parts.some(p => p.inlineData); const hasExecutableCode = adkEvent.content.parts.some(p => p.executableCode); const hasCodeExecutionResult = adkEvent.content.parts.some(p => p.codeExecutionResult); if (hasExecutableCode) { // Show executable code const codePart = adkEvent.content.parts.find(p => p.executableCode); if (codePart && codePart.executableCode) { const code = codePart.executableCode.code || ''; const language = codePart.executableCode.language || 'unknown'; const truncated = code.length > 60 ? code.substring(0, 60).replace(/\n/g, ' ') + '...' : code.replace(/\n/g, ' '); eventSummary = `Executable Code (${language}): ${truncated}`; eventEmoji = 'πŸ’»'; } } if (hasCodeExecutionResult) { // Show code execution result const resultPart = adkEvent.content.parts.find(p => p.codeExecutionResult); if (resultPart && resultPart.codeExecutionResult) { const outcome = resultPart.codeExecutionResult.outcome || 'UNKNOWN'; const output = resultPart.codeExecutionResult.output || ''; const truncatedOutput = output.length > 60 ? output.substring(0, 60).replace(/\n/g, ' ') + '...' : output.replace(/\n/g, ' '); eventSummary = `Code Execution Result (${outcome}): ${truncatedOutput}`; eventEmoji = outcome === 'OUTCOME_OK' ? 'βœ…' : '❌'; } } if (hasText) { // Show text preview in summary const textPart = adkEvent.content.parts.find(p => p.text); if (textPart && textPart.text) { const text = textPart.text; const truncated = text.length > 80 ? text.substring(0, 80) + '...' : text; eventSummary = `Text: "${truncated}"`; eventEmoji = 'πŸ’­'; } else { eventSummary = 'Text Response'; eventEmoji = 'πŸ’­'; } } if (hasAudio) { // Extract audio info for summary const audioPart = adkEvent.content.parts.find(p => p.inlineData); if (audioPart && audioPart.inlineData) { const mimeType = audioPart.inlineData.mimeType || 'unknown'; const dataLength = audioPart.inlineData.data ? audioPart.inlineData.data.length : 0; // Base64 string length / 4 * 3 gives approximate bytes const byteSize = Math.floor(dataLength * 0.75); eventSummary = `Audio Response: ${mimeType} (${byteSize.toLocaleString()} bytes)`; eventEmoji = 'πŸ”Š'; } else { eventSummary = 'Audio Response'; eventEmoji = 'πŸ”Š'; } // Log audio event with isAudio flag (filtered by checkbox) const sanitizedEvent = sanitizeEventForDisplay(adkEvent); addConsoleEntry('incoming', eventSummary, sanitizedEvent, eventEmoji, author, true); } } // Create a sanitized version for console display (replace large audio data with summary) // Skip if already logged as audio event above const isAudioOnlyEvent = adkEvent.content && adkEvent.content.parts && adkEvent.content.parts.some(p => p.inlineData) && !adkEvent.content.parts.some(p => p.text); if (!isAudioOnlyEvent) { const sanitizedEvent = sanitizeEventForDisplay(adkEvent); addConsoleEntry('incoming', eventSummary, sanitizedEvent, eventEmoji, author); } // Handle turn complete event if (adkEvent.turnComplete === true) { // Remove typing indicator from current message if (currentBubbleElement) { const textElement = currentBubbleElement.querySelector(".bubble-text"); const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } } // Remove typing indicator from current output transcription if (currentOutputTranscriptionElement) { const textElement = currentOutputTranscriptionElement.querySelector(".bubble-text"); const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } } currentMessageId = null; currentBubbleElement = null; currentOutputTranscriptionId = null; currentOutputTranscriptionElement = null; inputTranscriptionFinished = false; // Reset for next turn hasOutputTranscriptionInTurn = false; // Reset for next turn return; } // Handle interrupted event if (adkEvent.interrupted === true) { // Stop audio playback if it's playing if (audioPlayerNode) { audioPlayerNode.port.postMessage({ command: "endOfAudio" }); } // Keep the partial message but mark it as interrupted if (currentBubbleElement) { const textElement = currentBubbleElement.querySelector(".bubble-text"); // Remove typing indicator const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } // Add interrupted marker currentBubbleElement.classList.add("interrupted"); } // Keep the partial output transcription but mark it as interrupted if (currentOutputTranscriptionElement) { const textElement = currentOutputTranscriptionElement.querySelector(".bubble-text"); // Remove typing indicator const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } // Add interrupted marker currentOutputTranscriptionElement.classList.add("interrupted"); } // Reset state so new content creates a new bubble currentMessageId = null; currentBubbleElement = null; currentOutputTranscriptionId = null; currentOutputTranscriptionElement = null; inputTranscriptionFinished = false; // Reset for next turn hasOutputTranscriptionInTurn = false; // Reset for next turn return; } // Handle input transcription (user's spoken words) if (adkEvent.inputTranscription && adkEvent.inputTranscription.text) { const transcriptionText = adkEvent.inputTranscription.text; const isFinished = adkEvent.inputTranscription.finished; if (transcriptionText) { // Ignore late-arriving transcriptions after we've finished for this turn if (inputTranscriptionFinished) { return; } if (currentInputTranscriptionId == null) { // Create new transcription bubble currentInputTranscriptionId = Math.random().toString(36).substring(7); // Clean spaces between CJK characters const cleanedText = cleanCJKSpaces(transcriptionText); currentInputTranscriptionElement = createMessageBubble(cleanedText, true, !isFinished); currentInputTranscriptionElement.id = currentInputTranscriptionId; // Add a special class to indicate it's a transcription currentInputTranscriptionElement.classList.add("transcription"); messagesDiv.appendChild(currentInputTranscriptionElement); } else { // Update existing transcription bubble only if model hasn't started responding // This prevents late partial transcriptions from overwriting complete ones if (currentOutputTranscriptionId == null && currentMessageId == null) { if (isFinished) { // Final transcription contains the complete text, replace entirely const cleanedText = cleanCJKSpaces(transcriptionText); updateMessageBubble(currentInputTranscriptionElement, cleanedText, false); } else { // Partial transcription - append to existing text const existingText = currentInputTranscriptionElement.querySelector(".bubble-text").textContent; // Remove typing indicator if present const cleanText = existingText.replace(/\.\.\.$/, ''); // Clean spaces between CJK characters before updating const accumulatedText = cleanCJKSpaces(cleanText + transcriptionText); updateMessageBubble(currentInputTranscriptionElement, accumulatedText, true); } } } // If transcription is finished, reset the state and mark as complete if (isFinished) { currentInputTranscriptionId = null; currentInputTranscriptionElement = null; inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events } scrollToBottom(); } } // Handle output transcription (model's spoken words) if (adkEvent.outputTranscription && adkEvent.outputTranscription.text) { const transcriptionText = adkEvent.outputTranscription.text; const isFinished = adkEvent.outputTranscription.finished; hasOutputTranscriptionInTurn = true; if (transcriptionText) { // Finalize any active input transcription when server starts responding if (currentInputTranscriptionId != null && currentOutputTranscriptionId == null) { // This is the first output transcription - finalize input transcription const textElement = currentInputTranscriptionElement.querySelector(".bubble-text"); const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } // Reset input transcription state so next user input creates new balloon currentInputTranscriptionId = null; currentInputTranscriptionElement = null; inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events } if (currentOutputTranscriptionId == null) { // Create new transcription bubble for agent currentOutputTranscriptionId = Math.random().toString(36).substring(7); currentOutputTranscriptionElement = createMessageBubble(transcriptionText, false, !isFinished); currentOutputTranscriptionElement.id = currentOutputTranscriptionId; // Add a special class to indicate it's a transcription currentOutputTranscriptionElement.classList.add("transcription"); messagesDiv.appendChild(currentOutputTranscriptionElement); } else { // Update existing transcription bubble if (isFinished) { // Final transcription contains the complete text, replace entirely updateMessageBubble(currentOutputTranscriptionElement, transcriptionText, false); } else { // Partial transcription - append to existing text const existingText = currentOutputTranscriptionElement.querySelector(".bubble-text").textContent; // Remove typing indicator if present const cleanText = existingText.replace(/\.\.\.$/, ''); updateMessageBubble(currentOutputTranscriptionElement, cleanText + transcriptionText, true); } } // If transcription is finished, reset the state if (isFinished) { currentOutputTranscriptionId = null; currentOutputTranscriptionElement = null; } scrollToBottom(); } } // Handle content events (text or audio) if (adkEvent.content && adkEvent.content.parts) { const parts = adkEvent.content.parts; // Finalize any active input transcription when server starts responding with content if (currentInputTranscriptionId != null && currentMessageId == null && currentOutputTranscriptionId == null) { // This is the first content event - finalize input transcription const textElement = currentInputTranscriptionElement.querySelector(".bubble-text"); const typingIndicator = textElement.querySelector(".typing-indicator"); if (typingIndicator) { typingIndicator.remove(); } // Reset input transcription state so next user input creates new balloon currentInputTranscriptionId = null; currentInputTranscriptionElement = null; inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events } for (const part of parts) { // Handle inline data (audio) if (part.inlineData) { const mimeType = part.inlineData.mimeType; const data = part.inlineData.data; if (mimeType && mimeType.startsWith("audio/pcm") && audioPlayerNode) { audioPlayerNode.port.postMessage(base64ToArray(data)); } } // Handle text if (part.text) { // Skip thinking/reasoning text from chat bubbles (shown in event console) if (part.thought) { continue; } // Skip final aggregated content when output transcription already // delivered the response (prevents duplicate thinking text replay) if (!adkEvent.partial && hasOutputTranscriptionInTurn) { continue; } // Add a new message bubble for a new turn if (currentMessageId == null) { currentMessageId = Math.random().toString(36).substring(7); currentBubbleElement = createMessageBubble(part.text, false, true); currentBubbleElement.id = currentMessageId; messagesDiv.appendChild(currentBubbleElement); } else { // Update the existing message bubble with accumulated text const existingText = currentBubbleElement.querySelector(".bubble-text").textContent; // Remove the "..." if present const cleanText = existingText.replace(/\.\.\.$/, ''); updateMessageBubble(currentBubbleElement, cleanText + part.text, true); } // Scroll down to the bottom of the messagesDiv scrollToBottom(); } } } }; // Handle connection close websocket.onclose = function () { console.log("WebSocket connection closed."); updateConnectionStatus(false); document.getElementById("sendButton").disabled = true; addSystemMessage("Connection closed. Reconnecting in 5 seconds..."); // Log to console addConsoleEntry('error', 'WebSocket Disconnected', { status: 'Connection closed', reconnecting: true, reconnectDelay: '5 seconds' }, 'πŸ”Œ', 'system'); setTimeout(function () { console.log("Reconnecting..."); // Log reconnection attempt to console addConsoleEntry('outgoing', 'Reconnecting to ADK server...', { userId: userId, sessionId: sessionId }, 'πŸ”„', 'system'); connectWebsocket(); }, 5000); }; websocket.onerror = function (e) { console.log("WebSocket error: ", e); updateConnectionStatus(false); // Log to console addConsoleEntry('error', 'WebSocket Error', { error: e.type, message: 'Connection error occurred' }, '⚠️', 'system'); }; } connectWebsocket(); // Add submit handler to the form function addSubmitHandler() { messageForm.onsubmit = function (e) { e.preventDefault(); const message = messageInput.value.trim(); if (message) { // Add user message bubble const userBubble = createMessageBubble(message, true, false); messagesDiv.appendChild(userBubble); scrollToBottom(); // Clear input messageInput.value = ""; // Send message to server sendMessage(message); console.log("[CLIENT TO AGENT] " + message); } return false; }; } // Send a message to the server as JSON function sendMessage(message) { if (websocket && websocket.readyState == WebSocket.OPEN) { const jsonMessage = JSON.stringify({ type: "text", text: message }); websocket.send(jsonMessage); // Log to console panel addConsoleEntry('outgoing', 'User Message: ' + message, null, 'πŸ’¬', 'user'); } } // Decode Base64 data to Array // Handles both standard base64 and base64url encoding function base64ToArray(base64) { // Convert base64url to standard base64 // Replace URL-safe characters: - with +, _ with / let standardBase64 = base64.replace(/-/g, '+').replace(/_/g, '/'); // Add padding if needed while (standardBase64.length % 4) { standardBase64 += '='; } const binaryString = window.atob(standardBase64); const len = binaryString.length; const bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes.buffer; } /** * Camera handling */ const cameraButton = document.getElementById("cameraButton"); const cameraModal = document.getElementById("cameraModal"); const cameraPreview = document.getElementById("cameraPreview"); const closeCameraModal = document.getElementById("closeCameraModal"); const cancelCamera = document.getElementById("cancelCamera"); const captureImageBtn = document.getElementById("captureImage"); let cameraStream = null; // Open camera modal and start preview async function openCameraPreview() { try { // Request access to the user's webcam cameraStream = await navigator.mediaDevices.getUserMedia({ video: { width: { ideal: 768 }, height: { ideal: 768 }, facingMode: 'user' } }); // Set the stream to the video element cameraPreview.srcObject = cameraStream; // Show the modal cameraModal.classList.add('show'); } catch (error) { console.error('Error accessing camera:', error); addSystemMessage(`Failed to access camera: ${error.message}`); // Log to console addConsoleEntry('error', 'Camera access failed', { error: error.message, name: error.name }, '⚠️', 'system'); } } // Close camera modal and stop preview function closeCameraPreview() { // Stop the camera stream if (cameraStream) { cameraStream.getTracks().forEach(track => track.stop()); cameraStream = null; } // Clear the video source cameraPreview.srcObject = null; // Hide the modal cameraModal.classList.remove('show'); } // Capture image from the live preview function captureImageFromPreview() { if (!cameraStream) { addSystemMessage('No camera stream available'); return; } try { // Create canvas to capture the frame const canvas = document.createElement('canvas'); canvas.width = cameraPreview.videoWidth; canvas.height = cameraPreview.videoHeight; const context = canvas.getContext('2d'); // Draw current video frame to canvas context.drawImage(cameraPreview, 0, 0, canvas.width, canvas.height); // Convert canvas to data URL for display const imageDataUrl = canvas.toDataURL('image/jpeg', 0.85); // Display the captured image in the chat const imageBubble = createImageBubble(imageDataUrl, true); messagesDiv.appendChild(imageBubble); scrollToBottom(); // Convert canvas to blob for sending to server canvas.toBlob((blob) => { // Convert blob to base64 for sending to server const reader = new FileReader(); reader.onloadend = () => { const base64data = reader.result.split(',')[1]; // Remove data:image/jpeg;base64, prefix sendImage(base64data); }; reader.readAsDataURL(blob); // Log to console addConsoleEntry('outgoing', `Image captured: ${blob.size} bytes (JPEG)`, { size: blob.size, type: 'image/jpeg', dimensions: `${canvas.width}x${canvas.height}` }, 'πŸ“·', 'user'); }, 'image/jpeg', 0.85); // Close the camera modal closeCameraPreview(); } catch (error) { console.error('Error capturing image:', error); addSystemMessage(`Failed to capture image: ${error.message}`); // Log to console addConsoleEntry('error', 'Image capture failed', { error: error.message, name: error.name }, '⚠️', 'system'); } } // Send image to server function sendImage(base64Image) { if (websocket && websocket.readyState === WebSocket.OPEN) { const jsonMessage = JSON.stringify({ type: "image", data: base64Image, mimeType: "image/jpeg" }); websocket.send(jsonMessage); console.log("[CLIENT TO AGENT] Sent image"); } } // Event listeners cameraButton.addEventListener("click", openCameraPreview); closeCameraModal.addEventListener("click", closeCameraPreview); cancelCamera.addEventListener("click", closeCameraPreview); captureImageBtn.addEventListener("click", captureImageFromPreview); // Close modal when clicking outside of it cameraModal.addEventListener("click", (event) => { if (event.target === cameraModal) { closeCameraPreview(); } }); /** * Audio handling */ let audioPlayerNode; let audioPlayerContext; let audioRecorderNode; let audioRecorderContext; let micStream; // Import the audio worklets import { startAudioPlayerWorklet } from "./audio-player.js"; import { startAudioRecorderWorklet } from "./audio-recorder.js"; // Start audio function startAudio() { // Start audio output startAudioPlayerWorklet().then(([node, ctx]) => { audioPlayerNode = node; audioPlayerContext = ctx; }); // Start audio input startAudioRecorderWorklet(audioRecorderHandler).then( ([node, ctx, stream]) => { audioRecorderNode = node; audioRecorderContext = ctx; micStream = stream; } ); } // Start the audio only when the user clicked the button // (due to the gesture requirement for the Web Audio API) const startAudioButton = document.getElementById("startAudioButton"); startAudioButton.addEventListener("click", () => { startAudioButton.disabled = true; startAudio(); is_audio = true; addSystemMessage("Audio mode enabled - you can now speak to the agent"); // Log to console addConsoleEntry('outgoing', 'Audio Mode Enabled', { status: 'Audio worklets started', message: 'Microphone active - audio input will be sent to agent' }, '🎀', 'system'); }); // Audio recorder handler function audioRecorderHandler(pcmData) { if (websocket && websocket.readyState === WebSocket.OPEN && is_audio) { // Send audio as binary WebSocket frame (more efficient than base64 JSON) websocket.send(pcmData); console.log("[CLIENT TO AGENT] Sent audio chunk: %s bytes", pcmData.byteLength); // Log to console panel (optional, can be noisy with frequent audio chunks) // addConsoleEntry('outgoing', `Audio chunk: ${pcmData.byteLength} bytes`); } }