/** * `IMAGE_DESCRIPTION` handler: sends an image URL to the configured small model * and parses the `Title:` / `Description:` response into an * `ImageDescriptionResult`. Uses the shared client factory, retry wrapper, and * usage-event emitter; the small-model default comes from `getSmallModel`. */ import type { IAgentRuntime, ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core"; import { logger, ModelType } from "@elizaos/core"; import { generateText } from "ai"; import { createAnthropicClientWithTopPSupport } from "../providers/anthropic"; import { getSmallModel } from "../utils/config"; import { emitModelUsageEvent } from "../utils/events"; import { executeWithRetry, formatModelError, sanitizeUrlForLogs } from "../utils/retry"; const DEFAULT_IMAGE_DESCRIPTION_PROMPT = "Analyze this image and respond with:\nTitle: \nDescription: "; function parseTitle(content: string): string { const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i); if (titleMatch?.[1]) { return titleMatch[1].trim(); } const firstLine = content .split("\n") .map((line) => line.trim()) .find((line) => line.length > 0); return firstLine ? firstLine.slice(0, 100) : "Image Analysis"; } function parseDescription(content: string): string { const withoutTitle = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim(); const withoutDescriptionLabel = withoutTitle.replace(/^description[:\s]+/i, "").trim(); return withoutDescriptionLabel.length > 0 ? withoutDescriptionLabel : content.trim(); } export async function handleImageDescription( runtime: IAgentRuntime, params: ImageDescriptionParams | string ): Promise { const anthropic = createAnthropicClientWithTopPSupport(runtime); const modelName = getSmallModel(runtime); const imageUrl = typeof params === "string" ? params : params.imageUrl; const promptText = typeof params === "string" ? DEFAULT_IMAGE_DESCRIPTION_PROMPT : (params.prompt ?? DEFAULT_IMAGE_DESCRIPTION_PROMPT); const operationName = `${ModelType.IMAGE_DESCRIPTION} request using ${modelName}`; if (!imageUrl || imageUrl.trim().length === 0) { throw new Error("[Anthropic] IMAGE_DESCRIPTION requires a valid image URL."); } logger.log(`[Anthropic] Using ${ModelType.IMAGE_DESCRIPTION} model: ${modelName}`); try { const response = await executeWithRetry(operationName, () => generateText({ model: anthropic(modelName), messages: [ { role: "user" as const, content: [ { type: "text" as const, text: promptText }, { type: "image" as const, image: imageUrl }, ], }, ], maxOutputTokens: 1_024, }) ); if (response.usage) { emitModelUsageEvent( runtime, ModelType.IMAGE_DESCRIPTION, promptText, response.usage, modelName ); } return { title: parseTitle(response.text), description: parseDescription(response.text), }; } catch (error) { // error-policy:J2 context-adding rethrow — formatModelError wraps the // provider error with `cause`; never fabricate a { title, description }. logger.error( `[Anthropic] IMAGE_DESCRIPTION failed for ${modelName} ` + `(${sanitizeUrlForLogs(imageUrl)}): ${error instanceof Error ? error.message : String(error)}` ); throw formatModelError(operationName, error); } }