/** * `IMAGE_DESCRIPTION` handler: fetches an image by URL, encodes it inline, and * asks the multimodal Gemini model for a `{ title, description }`. Prefers the * model's JSON output and falls back to regex title extraction from prose. The * call is wrapped in `recordLlmCall` for trajectory capture. * * Provider failures (image fetch error, bad key, model-not-found, rate-limit, * timeout, safety block, empty completion) surface as typed errors so the caller * and model see a real failure — they are never fabricated into a * `{ title, description }` result the runtime would read as a real description. */ import type { IAgentRuntime, ImageDescriptionParams, RecordLlmCallDetails, } from "@elizaos/core"; import { logger, recordLlmCall } from "@elizaos/core"; import type { ImageDescriptionResponse } from "../types"; import { createGoogleGenAI, getImageModel, getSafetySettings, } from "../utils/config"; import { countTokens } from "../utils/tokenization"; export async function handleImageDescription( runtime: IAgentRuntime, params: ImageDescriptionParams | string, ): Promise { const genAI = createGoogleGenAI(runtime); if (!genAI) { throw new Error("Google Generative AI client not initialized"); } let imageUrl: string; let promptText: string; const modelName = getImageModel(runtime); logger.log(`[IMAGE_DESCRIPTION] Using model: ${modelName}`); if (typeof params === "string") { imageUrl = params; promptText = "Please analyze this image and provide a title and detailed description."; } else { imageUrl = params.imageUrl; promptText = params.prompt || "Please analyze this image and provide a title and detailed description."; } try { const imageResponse = await fetch(imageUrl); if (!imageResponse.ok) { throw new Error(`Failed to fetch image: ${imageResponse.statusText}`); } const imageData = await imageResponse.arrayBuffer(); const base64Image = Buffer.from(imageData).toString("base64"); const contentType = imageResponse.headers.get("content-type") || "image/jpeg"; const details: RecordLlmCallDetails = { model: modelName, systemPrompt: "", userPrompt: promptText, temperature: 0.7, maxTokens: 8192, purpose: "external_llm", actionType: "google-genai.IMAGE_DESCRIPTION.generateContent", }; const response = await recordLlmCall(runtime, details, async () => { const result = await genAI.models.generateContent({ model: modelName, contents: [ { role: "user", parts: [ { text: promptText }, { inlineData: { mimeType: contentType, data: base64Image, }, }, ], }, ], config: { temperature: 0.7, topK: 40, topP: 0.95, maxOutputTokens: 8192, safetySettings: getSafetySettings(), }, }); const responseText = result.text || ""; details.response = responseText; details.promptTokens = await countTokens(promptText); details.completionTokens = await countTokens(responseText); return result; }); const responseText = (response.text || "").trim(); if (!responseText) { // An empty completion is a provider failure (safety block, truncation, // model error), not a describable image. Surface it instead of returning // a fabricated "Image Analysis" / empty-description result. throw new Error("Google GenAI API returned an empty image description"); } try { const jsonResponse = JSON.parse(responseText) as { title?: string; description?: string; }; if ( typeof jsonResponse.title === "string" && typeof jsonResponse.description === "string" ) { return { title: jsonResponse.title, description: jsonResponse.description, }; } } catch { // error-policy:J3 untrusted-input sanitizing — a non-JSON completion is // an expected model-output shape; fall through to prose title parsing. } const titleMatch = responseText.match(/title[:\s]+(.+?)(?:\n|$)/i); const title = titleMatch?.[1]?.trim() || "Image Analysis"; const description = titleMatch ? responseText.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim() : responseText; return { title, description }; } catch (error) { // error-policy:J2 context-adding rethrow — do not fabricate a // `{ title, description }` on failure; the caller/model must see the real // error, not an "Error: ..." string dressed up as a successful description. const message = error instanceof Error ? error.message : String(error); logger.error(`Error analyzing image: ${message}`); throw error instanceof Error ? error : new Error(message); } }