/** * Text-generation handlers backing every text `ModelType` tier * (nano/small/medium/large/mega, response-handler, action-planner). Each * handler resolves its concrete Gemini model name via `../utils/config`, builds * a `generateContent` request, and runs it through `recordLlmCall` so the call * lands in the trajectory log before returning. * * This module owns the translation between elizaOS's generic call shape and * Google's native `@google/genai` protocol: `normalizeToolsForGoogle` / * `normalizeToolConfigForGoogle` convert generic or native tool definitions to * `functionDeclarations` + `functionCallingConfig`, `resolveResponseJsonSchema` * routes structured-output schemas into `responseJsonSchema`, and * `buildPromptParts` inlines attachments (data URLs, remote URIs, raw bytes). * On the way back, `buildGoogleNativeTextResult` folds text, tool calls, finish * reason, and token usage into a single object that is returned as a * string-with-attached-fields (`GoogleTextModelResult`) whenever the caller * passed messages/tools/toolChoice/responseSchema, else as plain text. */ import type { GenerateTextParams, IAgentRuntime, JsonValue, RecordLlmCallDetails, TokenUsage, ToolCall, } from "@elizaos/core"; import { buildCanonicalSystemPrompt, ElizaError, logger, ModelType, recordLlmCall, renderChatMessagesForPrompt, resolveEffectiveSystemPrompt, } from "@elizaos/core"; import { createGoogleGenAI, getActionPlannerModel, getLargeModel, getMediumModel, getMegaModel, getNanoModel, getResponseHandlerModel, getSafetySettings, getSmallModel, } from "../utils/config"; import { emitModelUsageEvent } from "../utils/events"; import { countTokens } from "../utils/tokenization"; const TEXT_NANO_MODEL_TYPE = ModelType.TEXT_NANO as string; const TEXT_MEDIUM_MODEL_TYPE = ModelType.TEXT_MEDIUM as string; const TEXT_SMALL_MODEL_TYPE = ModelType.TEXT_SMALL as string; const TEXT_LARGE_MODEL_TYPE = ModelType.TEXT_LARGE as string; const TEXT_MEGA_MODEL_TYPE = ModelType.TEXT_MEGA as string; const RESPONSE_HANDLER_MODEL_TYPE = ModelType.RESPONSE_HANDLER as string; const ACTION_PLANNER_MODEL_TYPE = ModelType.ACTION_PLANNER as string; type ChatAttachment = { data: string | Uint8Array | URL; mediaType: string; filename?: string; }; /** * Native Google GenAI tool input. Each function declaration carries a name, * description, and JSON Schema parameters object that the model can choose to * invoke. */ type GoogleFunctionDeclaration = { name: string; description?: string; parameters?: Record; }; type GoogleToolDeclaration = { functionDeclarations?: GoogleFunctionDeclaration[]; }; type GoogleFunctionCall = { id?: string; name?: string; args?: Record; }; type GoogleContentPart = { text?: string; thought?: boolean; functionCall?: GoogleFunctionCall; }; type GoogleGenerateContentResponse = { text?: string; functionCalls?: GoogleFunctionCall[]; candidates?: Array<{ content?: { parts?: GoogleContentPart[]; }; finishReason?: string; }>; usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number; totalTokenCount?: number; cachedContentTokenCount?: number; }; modelVersion?: string; responseId?: string; createTime?: string; }; type GoogleNativeTextResult = { text: string; toolCalls: ToolCall[]; finishReason?: string; usage: TokenUsage; providerMetadata: Record; }; type GoogleTextModelResult = string & GoogleNativeTextResult; type GenericToolDescriptor = { name?: string; description?: string; parameters?: unknown; inputSchema?: unknown; function?: { name?: string; description?: string; parameters?: unknown }; }; type LocalToolChoice = | "auto" | "required" | "none" | { type: "tool"; toolName?: string; name?: string } | { type: "function"; function: { name: string } } | { name: string }; type GenerateTextParamsWithAttachments = Omit< GenerateTextParams, "tools" | "toolChoice" | "responseSchema" > & { attachments?: ChatAttachment[]; /** Native or generic tool definitions; converted to Google functionDeclarations. */ tools?: | GenericToolDescriptor[] | GoogleToolDeclaration[] | Record; /** Tool selection hint: "auto" | "required" | "none" | { type: "tool"; toolName } | { type: "function"; function }. */ toolChoice?: LocalToolChoice; /** JSON Schema for structured output; routes through responseJsonSchema. */ responseSchema?: | Record | { schema: Record }; }; type GoogleGenAIClient = NonNullable>; type GenerateContentParams = Parameters< GoogleGenAIClient["models"]["generateContent"] >[0]; function normalizeToolsForGoogle( tools: GenerateTextParamsWithAttachments["tools"], ): GoogleToolDeclaration[] | undefined { if (!tools) return undefined; // Already-shaped Google tools: array of { functionDeclarations: [...] }. if ( Array.isArray(tools) && tools.length > 0 && typeof tools[0] === "object" && tools[0] !== null && "functionDeclarations" in (tools[0] as object) ) { return tools as GoogleToolDeclaration[]; } const flat: GenericToolDescriptor[] = Array.isArray(tools) ? (tools as GenericToolDescriptor[]) : Object.entries(tools).map(([name, value]) => ({ name, ...value })); const declarations: GoogleFunctionDeclaration[] = []; for (const tool of flat) { const name = tool.name ?? tool.function?.name; if (!name) { throw new Error("[GoogleGenAI] Tool definition is missing a name."); } const description = tool.description ?? tool.function?.description; const parameters = (tool.parameters ?? tool.inputSchema ?? tool.function?.parameters ?? { type: "object", properties: {}, }) as Record; declarations.push({ name, ...(description ? { description } : {}), parameters, }); } return declarations.length > 0 ? [{ functionDeclarations: declarations }] : undefined; } function normalizeToolConfigForGoogle( toolChoice: GenerateTextParamsWithAttachments["toolChoice"], ): | { functionCallingConfig: { mode: "AUTO" | "ANY" | "NONE"; allowedFunctionNames?: string[]; }; } | undefined { if (!toolChoice) return undefined; if (toolChoice === "auto") { return { functionCallingConfig: { mode: "AUTO" } }; } if (toolChoice === "required") { return { functionCallingConfig: { mode: "ANY" } }; } if (toolChoice === "none") { return { functionCallingConfig: { mode: "NONE" } }; } let toolName: string | undefined; if ("type" in toolChoice) { toolName = toolChoice.type === "function" ? toolChoice.function.name : (toolChoice.toolName ?? toolChoice.name); } else { toolName = toolChoice.name; } if (toolName) { return { functionCallingConfig: { mode: "ANY", allowedFunctionNames: [toolName], }, }; } return undefined; } function resolveResponseJsonSchema( responseSchema: GenerateTextParamsWithAttachments["responseSchema"], ): Record | undefined { if (!responseSchema) return undefined; if ("schema" in responseSchema && responseSchema.schema) { return responseSchema.schema as Record; } return responseSchema as Record; } function buildPromptParts(prompt: string, attachments?: ChatAttachment[]) { const parts: Array< | { text: string } | { fileData: { mimeType: string; fileUri: string } } | { inlineData: { mimeType: string; data: string } } > = [{ text: prompt }]; for (const attachment of attachments ?? []) { if (attachment.data instanceof URL) { parts.push({ fileData: { mimeType: attachment.mediaType, fileUri: attachment.data.toString(), }, }); continue; } if ( typeof attachment.data === "string" && /^https?:\/\//i.test(attachment.data) ) { parts.push({ fileData: { mimeType: attachment.mediaType, fileUri: attachment.data, }, }); continue; } if (typeof attachment.data === "string") { const dataUrlMatch = attachment.data.match( /^data:([^;,]+);base64,(.+)$/i, ); parts.push({ inlineData: { mimeType: dataUrlMatch?.[1] ?? attachment.mediaType, data: dataUrlMatch?.[2] ?? attachment.data, }, }); continue; } parts.push({ inlineData: { mimeType: attachment.mediaType, data: Buffer.from(attachment.data).toString("base64"), }, }); } return parts; } function resolveGoogleSystemInstruction( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): string | undefined { return resolveEffectiveSystemPrompt({ params, fallback: buildCanonicalSystemPrompt({ character: runtime.character }), }); } function resolveGooglePrompt( params: GenerateTextParamsWithAttachments, systemInstruction: string | undefined, ): string { return ( renderChatMessagesForPrompt(params.messages, { omitDuplicateSystem: systemInstruction, }) ?? params.prompt ?? "" ); } function getModelNameForType( runtime: IAgentRuntime, modelType: string, ): string { switch (modelType) { case TEXT_NANO_MODEL_TYPE: return getNanoModel(runtime); case TEXT_MEDIUM_MODEL_TYPE: return getMediumModel(runtime); case TEXT_SMALL_MODEL_TYPE: return getSmallModel(runtime); case TEXT_LARGE_MODEL_TYPE: return getLargeModel(runtime); case TEXT_MEGA_MODEL_TYPE: return getMegaModel(runtime); case RESPONSE_HANDLER_MODEL_TYPE: return getResponseHandlerModel(runtime); case ACTION_PLANNER_MODEL_TYPE: return getActionPlannerModel(runtime); default: return getLargeModel(runtime); } } function isRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); } function toJsonValue(value: unknown): JsonValue { if ( value === null || typeof value === "string" || typeof value === "boolean" ) { return value; } if (typeof value === "number") { return Number.isFinite(value) ? value : null; } if (Array.isArray(value)) { return value.map((entry) => toJsonValue(entry)); } if (isRecord(value)) { const record: Record = {}; for (const [key, entry] of Object.entries(value)) { if (entry !== undefined) { record[key] = toJsonValue(entry); } } return record; } return String(value); } function toToolArguments(value: unknown): Record { if (!isRecord(value)) { return {}; } const jsonValue = toJsonValue(value); return isRecord(jsonValue) ? (jsonValue as Record) : {}; } function readGoogleText(response: GoogleGenerateContentResponse): string { if (typeof response.text === "string") { return response.text; } return ( response.candidates?.[0]?.content?.parts ?.filter((part) => !part.thought && typeof part.text === "string") .map((part) => part.text) .join("") ?? "" ); } function readGoogleFunctionCalls( response: GoogleGenerateContentResponse, ): GoogleFunctionCall[] { if ( Array.isArray(response.functionCalls) && response.functionCalls.length > 0 ) { return response.functionCalls; } const calls: GoogleFunctionCall[] = []; for (const candidate of response.candidates ?? []) { for (const part of candidate.content?.parts ?? []) { if (part.functionCall) { calls.push(part.functionCall); } } } return calls; } function normalizeGoogleToolCalls( response: GoogleGenerateContentResponse, ): ToolCall[] { return readGoogleFunctionCalls(response) .map((call, index): ToolCall | undefined => { const name = typeof call.name === "string" ? call.name : ""; if (!name) { return undefined; } const id = typeof call.id === "string" && call.id.length > 0 ? call.id : `google-genai-tool-call-${index + 1}`; const args = toToolArguments(call.args); return { id, name, arguments: args, toolName: name, toolCallId: id, type: "function", args, input: args, }; }) .filter((call): call is ToolCall => Boolean(call)); } function normalizeGoogleFinishReason( response: GoogleGenerateContentResponse, toolCalls: ToolCall[], ): string | undefined { if (toolCalls.length > 0) { return "tool-calls"; } return response.candidates?.find((candidate) => candidate.finishReason) ?.finishReason; } function firstNumber(...values: unknown[]): number | undefined { for (const value of values) { if (typeof value === "number" && Number.isFinite(value)) { return value; } } return undefined; } async function normalizeGoogleUsage( response: GoogleGenerateContentResponse, prompt: string, text: string, ): Promise { const metadata = response.usageMetadata; const promptTokens = firstNumber(metadata?.promptTokenCount) ?? (await countTokens(prompt)); const completionTokens = firstNumber(metadata?.candidatesTokenCount) ?? (await countTokens(text)); const totalTokens = firstNumber(metadata?.totalTokenCount) ?? promptTokens + completionTokens; return { promptTokens, completionTokens, totalTokens, cacheReadInputTokens: firstNumber(metadata?.cachedContentTokenCount), }; } async function buildGoogleNativeTextResult( response: GoogleGenerateContentResponse, prompt: string, modelName: string, ): Promise { const text = readGoogleText(response); const toolCalls = normalizeGoogleToolCalls(response); const usage = await normalizeGoogleUsage(response, prompt, text); return { text, toolCalls, finishReason: normalizeGoogleFinishReason(response, toolCalls), usage, providerMetadata: { provider: "google-genai", modelName, modelVersion: response.modelVersion, responseId: response.responseId, createTime: response.createTime, usageMetadata: response.usageMetadata, }, }; } function usesNativeTextResult( params: GenerateTextParamsWithAttachments, ): boolean { return Boolean( params.messages || params.tools || params.toolChoice || params.responseSchema, ); } function buildGoogleGenerationConfig( params: GenerateTextParamsWithAttachments, systemInstruction: string | undefined, temperature: number, maxTokens: number | undefined, stopSequences: string[], ): NonNullable { const tools = normalizeToolsForGoogle(params.tools); const toolConfig = normalizeToolConfigForGoogle(params.toolChoice); const responseJsonSchema = resolveResponseJsonSchema(params.responseSchema); const baseConfig: Record = { temperature, topK: 40, topP: 0.95, stopSequences, safetySettings: getSafetySettings(), ...(typeof maxTokens === "number" ? { maxOutputTokens: maxTokens } : {}), ...(systemInstruction && { systemInstruction }), ...(tools ? { tools } : {}), ...(toolConfig ? { toolConfig } : {}), ...(responseJsonSchema ? { responseMimeType: "application/json", responseJsonSchema, } : {}), }; return baseConfig as NonNullable; } function createLlmCallDetails( modelName: string, modelType: string, prompt: string, systemInstruction: string | undefined, temperature: number, maxTokens: number | undefined, maxTokensOmitted?: boolean, ): RecordLlmCallDetails { return { model: modelName, systemPrompt: systemInstruction ?? "", userPrompt: prompt, temperature, maxTokens: maxTokens ?? 0, maxTokensOmitted: maxTokensOmitted ? true : undefined, purpose: "external_llm", actionType: `google-genai.${modelType}.generateContent`, }; } async function generateContentWithTrajectory( runtime: IAgentRuntime, genAI: GoogleGenAIClient, modelName: string, modelType: string, prompt: string, systemInstruction: string | undefined, temperature: number, maxTokens: number | undefined, maxTokensOmitted: boolean | undefined, request: GenerateContentParams, shouldReturnNativeResult: boolean, ): Promise { const details = createLlmCallDetails( modelName, modelType, prompt, systemInstruction, temperature, maxTokens, maxTokensOmitted, ); const response = await recordLlmCall(runtime, details, async () => { const result = (await genAI.models.generateContent( request, )) as GoogleGenerateContentResponse; const normalized = await buildGoogleNativeTextResult( result, prompt, modelName, ); details.response = normalized.text; details.toolCalls = normalized.toolCalls; details.finishReason = normalized.finishReason; details.providerMetadata = normalized.providerMetadata; details.promptTokens = normalized.usage.promptTokens; details.completionTokens = normalized.usage.completionTokens; details.cacheReadInputTokens = normalized.usage.cacheReadInputTokens; return normalized; }); // A completion with no text and no tool calls is a provider failure (safety // block, empty candidates, truncation) — never a legitimate result. Returning // "" here would fabricate a healthy-empty completion the planner cannot // distinguish from a real answer (#9324: throw, never fabricate). if (response.text.length === 0 && response.toolCalls.length === 0) { throw new ElizaError( `[Google GenAI] ${modelType} returned an empty completion${ response.finishReason ? ` (finishReason: ${response.finishReason})` : "" }`, { code: "MODEL_EMPTY_COMPLETION", context: { modelType, finishReason: response.finishReason }, }, ); } emitModelUsageEvent(runtime, modelType, prompt, response.usage); if (shouldReturnNativeResult) { return response as GoogleTextModelResult; } return response.text; } export async function handleTextSmall( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { const { stopSequences = [], temperature = 0.7, attachments } = params; const maxTokens = params.omitMaxTokens ? undefined : (params.maxTokens ?? 8192); const genAI = createGoogleGenAI(runtime); if (!genAI) { throw new Error("Google Generative AI client not initialized"); } const modelName = getModelNameForType(runtime, TEXT_SMALL_MODEL_TYPE); logger.log(`[TEXT_SMALL] Using model: ${modelName}`); try { const systemInstruction = resolveGoogleSystemInstruction(runtime, params); const promptText = resolveGooglePrompt(params, systemInstruction); return await generateContentWithTrajectory( runtime, genAI, modelName, TEXT_SMALL_MODEL_TYPE, promptText, systemInstruction, temperature, maxTokens, params.omitMaxTokens, { model: modelName, contents: (attachments?.length ?? 0) > 0 ? [ { role: "user", parts: buildPromptParts(promptText, attachments), }, ] : promptText, config: buildGoogleGenerationConfig( params, systemInstruction, temperature, maxTokens, stopSequences, ), }, usesNativeTextResult(params), ); } catch (error) { // error-policy:J2 context-adding rethrow — log the model label for // operators, then rethrow the original provider error unchanged so the // planner/runtime sees the real failure. logger.error( `[TEXT_SMALL] Error: ${error instanceof Error ? error.message : String(error)}`, ); throw error; } } export async function handleTextLarge( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { const { stopSequences = [], temperature = 0.7, attachments } = params; const maxTokens = params.omitMaxTokens ? undefined : (params.maxTokens ?? 8192); const genAI = createGoogleGenAI(runtime); if (!genAI) { throw new Error("Google Generative AI client not initialized"); } const modelName = getModelNameForType(runtime, TEXT_LARGE_MODEL_TYPE); logger.log(`[TEXT_LARGE] Using model: ${modelName}`); try { const systemInstruction = resolveGoogleSystemInstruction(runtime, params); const promptText = resolveGooglePrompt(params, systemInstruction); return await generateContentWithTrajectory( runtime, genAI, modelName, TEXT_LARGE_MODEL_TYPE, promptText, systemInstruction, temperature, maxTokens, params.omitMaxTokens, { model: modelName, contents: (attachments?.length ?? 0) > 0 ? [ { role: "user", parts: buildPromptParts(promptText, attachments), }, ] : promptText, config: buildGoogleGenerationConfig( params, systemInstruction, temperature, maxTokens, stopSequences, ), }, usesNativeTextResult(params), ); } catch (error) { // error-policy:J2 context-adding rethrow — log the model label for // operators, then rethrow the original provider error unchanged. logger.error( `[TEXT_LARGE] Error: ${error instanceof Error ? error.message : String(error)}`, ); throw error; } } export async function handleTextNano( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { return handleTextWithType(runtime, TEXT_NANO_MODEL_TYPE, params); } export async function handleTextMedium( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { return handleTextWithType(runtime, TEXT_MEDIUM_MODEL_TYPE, params); } export async function handleTextMega( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { return handleTextWithType(runtime, TEXT_MEGA_MODEL_TYPE, params); } export async function handleResponseHandler( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { return handleTextWithType(runtime, RESPONSE_HANDLER_MODEL_TYPE, params); } export async function handleActionPlanner( runtime: IAgentRuntime, params: GenerateTextParamsWithAttachments, ): Promise { return handleTextWithType(runtime, ACTION_PLANNER_MODEL_TYPE, params); } async function handleTextWithType( runtime: IAgentRuntime, modelType: string, params: GenerateTextParamsWithAttachments, ): Promise { const { stopSequences = [], temperature = 0.7, attachments } = params; const maxTokens = params.omitMaxTokens ? undefined : (params.maxTokens ?? 8192); const genAI = createGoogleGenAI(runtime); if (!genAI) { throw new Error("Google Generative AI client not initialized"); } const modelName = getModelNameForType(runtime, modelType); logger.log(`[${modelType}] Using model: ${modelName}`); try { const systemInstruction = resolveGoogleSystemInstruction(runtime, params); const promptText = resolveGooglePrompt(params, systemInstruction); return await generateContentWithTrajectory( runtime, genAI, modelName, modelType, promptText, systemInstruction, temperature, maxTokens, params.omitMaxTokens, { model: modelName, contents: (attachments?.length ?? 0) > 0 ? [ { role: "user", parts: buildPromptParts(promptText, attachments), }, ] : promptText, config: buildGoogleGenerationConfig( params, systemInstruction, temperature, maxTokens, stopSequences, ), }, usesNativeTextResult(params), ); } catch (error) { // error-policy:J2 context-adding rethrow — log the model label for // operators, then rethrow the original provider error unchanged. logger.error( `[${modelType}] Error: ${error instanceof Error ? error.message : String(error)}`, ); throw error; } }