/** * Groq plugin: registers the text-generation ModelType handlers (nano through * mega, plus RESPONSE_HANDLER and ACTION_PLANNER) as well as TRANSCRIPTION and * TEXT_TO_SPEECH, all via the Vercel AI SDK's @ai-sdk/groq provider. Init * requires GROQ_API_KEY. Calls go through a shared retry loop that classifies * failures (classifyRetryError) into rate-limit / transient / fatal and backs * off on the first two. */ import { createGroq } from "@ai-sdk/groq"; import type { EventPayload, IAgentRuntime, ModelTypeName, Plugin, RecordLlmCallDetails, } from "@elizaos/core"; import { buildCanonicalSystemPrompt, ElizaError, EventType, type GenerateTextParams, logger, ModelType, recordLlmCall, renderChatMessagesForPrompt, resolveEffectiveSystemPrompt, } from "@elizaos/core"; import { APICallError, generateText, type JSONSchema7, jsonSchema, type ModelMessage, Output, type ToolChoice, type ToolSet, } from "ai"; type RuntimeProcess = { env?: Record; }; type RuntimeBufferConstructor = { from(input: string, encoding?: string): Uint8Array; from(input: ArrayBufferLike | ArrayLike): Uint8Array; alloc(size: number): Uint8Array; isBuffer(value: unknown): boolean; }; const _globalThis = globalThis as { AI_SDK_LOG_WARNINGS?: boolean; process?: RuntimeProcess; Buffer?: RuntimeBufferConstructor; }; _globalThis.AI_SDK_LOG_WARNINGS ??= false; const DEFAULT_SMALL_MODEL = "openai/gpt-oss-120b"; const DEFAULT_LARGE_MODEL = "openai/gpt-oss-120b"; const DEFAULT_TTS_MODEL = "canopylabs/orpheus-v1-english"; const DEFAULT_TTS_VOICE = "troy"; const DEFAULT_TTS_RESPONSE_FORMAT = "wav"; const DEFAULT_TRANSCRIPTION_MODEL = "whisper-large-v3-turbo"; const DEFAULT_BASE_URL = "https://api.groq.com/openai/v1"; function resolveGroqSystemPrompt( runtime: IAgentRuntime, params: GenerateTextParams ): string | undefined { return resolveEffectiveSystemPrompt({ params, fallback: buildCanonicalSystemPrompt({ character: runtime.character }), }); } function resolveGroqPrompt(params: GenerateTextParams, systemPrompt: string | undefined): string { return ( renderChatMessagesForPrompt(params.messages, { omitDuplicateSystem: systemPrompt, }) ?? params.prompt ?? "" ); } type ProviderUsage = { inputTokens?: number; outputTokens?: number; promptTokens?: number; completionTokens?: number; totalTokens?: number; }; type NormalizedUsage = { promptTokens: number; completionTokens: number; totalTokens: number; estimated?: boolean; }; function toFiniteNumber(value: unknown): number | undefined { if (typeof value !== "number" || !Number.isFinite(value)) { return undefined; } return Math.max(0, Math.round(value)); } function normalizeTokenUsage(usage: unknown): NormalizedUsage | null { if (!usage || typeof usage !== "object") { return null; } const record = usage as ProviderUsage; const promptTokens = toFiniteNumber(record.inputTokens ?? record.promptTokens); const completionTokens = toFiniteNumber(record.outputTokens ?? record.completionTokens); const totalTokens = toFiniteNumber(record.totalTokens); if (promptTokens === undefined && completionTokens === undefined && totalTokens === undefined) { return null; } const normalizedPromptTokens = promptTokens ?? (completionTokens === undefined && totalTokens !== undefined ? totalTokens : Math.max(0, (totalTokens ?? 0) - (completionTokens ?? 0))); const normalizedCompletionTokens = completionTokens ?? Math.max(0, (totalTokens ?? normalizedPromptTokens) - normalizedPromptTokens); return { promptTokens: normalizedPromptTokens, completionTokens: normalizedCompletionTokens, totalTokens: totalTokens ?? normalizedPromptTokens + normalizedCompletionTokens, }; } function applyUsageToDetails(details: RecordLlmCallDetails, usage: unknown): void { const normalized = normalizeTokenUsage(usage); if (!normalized) { return; } details.promptTokens = normalized.promptTokens; details.completionTokens = normalized.completionTokens; } function estimateTokenCount(text: string): number { return text.length === 0 ? 0 : Math.ceil(text.length / 4); } function stringifyForUsage(value: unknown): string { if (typeof value === "string") { return value; } try { return JSON.stringify(value); } catch { // error-policy:J3 untrusted-input sanitizing — a circular/unstringifiable // response degrades to String() for token *estimation* only (the usage // event is flagged `estimated`); no completion data is fabricated. return String(value); } } function estimateUsage(prompt: string, response: unknown): NormalizedUsage { const promptTokens = estimateTokenCount(prompt); const completionTokens = estimateTokenCount(stringifyForUsage(response)); return { promptTokens, completionTokens, totalTokens: promptTokens + completionTokens, estimated: true, }; } function boundedNumber(value: unknown, fallback: number, min: number, max: number): number { if (typeof value !== "number" || !Number.isFinite(value)) { return fallback; } return Math.min(max, Math.max(min, value)); } function positiveInteger(value: unknown, fallback: number): number { if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { return fallback; } return Math.floor(value); } function stringArray(value: unknown): string[] { if (!Array.isArray(value)) { return []; } return value.filter((item): item is string => typeof item === "string"); } function emitModelUsed( runtime: IAgentRuntime, type: ModelTypeName, model: string, usage: NormalizedUsage ): void { void runtime.emitEvent( EventType.MODEL_USED as string, { runtime, source: "groq", provider: "groq", type, model, modelName: model, tokens: { prompt: usage.promptTokens, completion: usage.completionTokens, total: usage.totalTokens, ...(usage.estimated ? { estimated: true } : {}), }, ...(usage.estimated ? { usageEstimated: true } : {}), } as EventPayload ); } function isBrowser(): boolean { return ( typeof globalThis !== "undefined" && typeof (globalThis as { document?: Document }).document !== "undefined" ); } function env(name: string): string | null { return _globalThis.process?.env?.[name] ?? null; } function nonEmptyString(value: unknown): string | undefined { if (typeof value !== "string") { return undefined; } const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : undefined; } function getRuntimeBuffer(): RuntimeBufferConstructor | null { return _globalThis.Buffer ?? null; } function getBaseURL(runtime: IAgentRuntime): string { const configured = nonEmptyString(runtime.getSetting("GROQ_BASE_URL")); if (!configured) { return DEFAULT_BASE_URL; } let parsed: URL; try { parsed = new URL(configured); } catch (error) { // error-policy:J2 context-adding rethrow — names the misconfigured setting; // the original URL parse failure travels as `cause`. throw new Error("GROQ_BASE_URL must be a valid http(s) URL", { cause: error }); } if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { throw new Error("GROQ_BASE_URL must be a valid http(s) URL"); } return configured.replace(/\/+$/, ""); } function getSmallModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_SMALL_MODEL") || runtime.getSetting("SMALL_MODEL"); return typeof setting === "string" ? setting : DEFAULT_SMALL_MODEL; } function getNanoModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_NANO_MODEL") || runtime.getSetting("NANO_MODEL"); return typeof setting === "string" ? setting : getSmallModel(runtime); } function getMediumModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_MEDIUM_MODEL") || runtime.getSetting("MEDIUM_MODEL"); return typeof setting === "string" ? setting : getSmallModel(runtime); } function getLargeModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_LARGE_MODEL") || runtime.getSetting("LARGE_MODEL"); return typeof setting === "string" ? setting : DEFAULT_LARGE_MODEL; } function getMegaModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_MEGA_MODEL") || runtime.getSetting("MEGA_MODEL"); return typeof setting === "string" ? setting : getLargeModel(runtime); } function getResponseHandlerModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_RESPONSE_HANDLER_MODEL") || runtime.getSetting("GROQ_SHOULD_RESPOND_MODEL") || runtime.getSetting("RESPONSE_HANDLER_MODEL") || runtime.getSetting("SHOULD_RESPOND_MODEL"); return typeof setting === "string" ? setting : getNanoModel(runtime); } function getTranscriptionModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_TRANSCRIPTION_MODEL") || runtime.getSetting("TRANSCRIPTION_MODEL"); return typeof setting === "string" ? setting : DEFAULT_TRANSCRIPTION_MODEL; } function getActionPlannerModel(runtime: IAgentRuntime): string { const setting = runtime.getSetting("GROQ_ACTION_PLANNER_MODEL") || runtime.getSetting("GROQ_PLANNER_MODEL") || runtime.getSetting("ACTION_PLANNER_MODEL") || runtime.getSetting("PLANNER_MODEL"); // Action planning is a reasoning-heavy task — route to the LARGE tier by // default (gpt-oss-120b) rather than the SMALL/MEDIUM tier. Small models // mis-classify semantically adjacent actions too often to be the default. return typeof setting === "string" ? setting : getLargeModel(runtime); } function createGroqClient(runtime: IAgentRuntime) { // In browsers, default to *not* sending secrets. // Use a server-side proxy and configure GROQ_BASE_URL (or explicitly opt-in). const allowBrowserKey = !isBrowser() || String(runtime.getSetting("GROQ_ALLOW_BROWSER_API_KEY") ?? "").toLowerCase() === "true"; const apiKey = allowBrowserKey ? nonEmptyString(runtime.getSetting("GROQ_API_KEY")) : undefined; return createGroq({ apiKey, fetch: runtime.fetch ?? undefined, baseURL: getBaseURL(runtime), }); } // Groq 429s phrase the cooldown as a Go-style duration: "7m30s", "2m59.56s", // "859ms", or plain "30s". Sum every component; a seconds-only regex reads // "7m30s" as no match (10s default) and re-collides with the same window. export function extractRetryDelay(message: string): number { const match = message.match(/try again in ((?:\d+(?:\.\d+)?(?:ms|[smhd]))+)/i); if (!match?.[1]) { return 10000; } const unitMs: Record = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000, }; let totalMs = 0; for (const part of match[1].matchAll(/(\d+(?:\.\d+)?)(ms|[smhd])/gi)) { totalMs += Number.parseFloat(part[1] ?? "0") * (unitMs[(part[2] ?? "s").toLowerCase()] ?? 1_000); } return Math.ceil(totalMs) + 1000; } /** * Classify an error thrown by `generateText`/`generateObject`. The AI SDK * already retries transient 5xx and network failures up to `maxRetries` * times with exponential backoff (~2s, 4s, 8s). This outer layer only kicks * in when the AI SDK gives up — typically for 429 rate limits whose * server-suggested cooldown (often 30–60s) exceeds the AI SDK's budget. * * Returns `"rate-limit"` for 429s (where we honor `try again in Ns`), * `"transient"` for 5xx / network failures worth one more shot, and * `"fatal"` for auth / validation / unknown errors that should propagate * immediately. */ export function classifyRetryError(error: unknown): "rate-limit" | "transient" | "fatal" { if (APICallError.isInstance(error)) { if (error.statusCode === 429) return "rate-limit"; if (typeof error.statusCode === "number" && error.statusCode >= 500 && error.statusCode < 600) { return "transient"; } if (error.isRetryable) return "transient"; return "fatal"; } if (!(error instanceof Error)) return "fatal"; const message = error.message.toLowerCase(); if ( message.includes("rate limit") || message.includes("rate_limit") || message.includes("too many requests") || /try again in \d/i.test(error.message) ) { return "rate-limit"; } // Node fetch / undici transient network failures. if ( message.includes("econnreset") || message.includes("etimedout") || message.includes("enotfound") || message.includes("econnrefused") || message.includes("socket hang up") || message.includes("network error") || message.includes("fetch failed") ) { return "transient"; } return "fatal"; } type NativeOutput = NonNullable>[0]["output"]>; function buildGroqStructuredOutput(responseSchema: unknown): NativeOutput { if ( responseSchema && typeof responseSchema === "object" && "responseFormat" in responseSchema && "parseCompleteOutput" in responseSchema ) { return responseSchema as NativeOutput; } const schemaOptions = responseSchema && typeof responseSchema === "object" && "schema" in responseSchema ? (responseSchema as { schema: unknown; name?: string; description?: string }) : { schema: responseSchema }; return Output.object({ schema: jsonSchema(schemaOptions.schema as JSONSchema7), ...(schemaOptions.name ? { name: schemaOptions.name } : {}), ...(schemaOptions.description ? { description: schemaOptions.description } : {}), }) as NativeOutput; } function isRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); } // The runtime exposes tools as ordered core `ToolDefinition[]` ({ name, // description, parameters }). The AI SDK expects a `ToolSet` keyed by // provider-visible tool names with `inputSchema`; passing the array through // gives Groq function names like "0" with an empty schema. Pre-built ToolSet // objects (no `name` field on values) pass through unchanged. function readGroqToolSet(value: unknown): ToolSet | undefined { if (!value) { return undefined; } const isArr = Array.isArray(value); if (!isArr && !isRecord(value)) { return undefined; } const entries: Array<[string, unknown]> = isArr ? (value as unknown[]).map((v, i) => [String(i), v] as [string, unknown]) : Object.entries(value as Record); const tools: Record = {}; let sawNamedTool = false; for (const [origKey, rawTool] of entries) { if (!isRecord(rawTool)) { continue; } const functionTool = isRecord(rawTool.function) ? rawTool.function : undefined; const name = typeof rawTool.name === "string" && rawTool.name ? rawTool.name : typeof functionTool?.name === "string" && functionTool.name ? functionTool.name : undefined; if (name) { sawNamedTool = true; const schema = isRecord(rawTool.parameters) ? (rawTool.parameters as JSONSchema7) : isRecord(functionTool?.parameters) ? (functionTool.parameters as JSONSchema7) : isRecord(rawTool.input_schema) ? (rawTool.input_schema as JSONSchema7) : ({ type: "object" } satisfies JSONSchema7); const description = typeof rawTool.description === "string" ? rawTool.description : typeof functionTool?.description === "string" ? functionTool.description : undefined; tools[name] = { ...(description ? { description } : {}), inputSchema: jsonSchema(schema), }; } else if (!isArr) { tools[origKey] = rawTool; } } if (sawNamedTool) { return Object.keys(tools).length > 0 ? (tools as ToolSet) : undefined; } return !isArr && isRecord(value) ? (value as ToolSet) : undefined; } // Core `ToolChoice` uses `{ type: "tool", name }` / `{ type: "function", // function: { name } }`; the AI SDK only understands `{ type: "tool", // toolName }` and the string enums. function readGroqToolChoice(value: unknown): ToolChoice | undefined { if (!value) { return undefined; } if (typeof value === "string" && (value === "auto" || value === "none" || value === "required")) { return value; } if (!isRecord(value)) { return undefined; } if (value.type === "tool" && typeof value.toolName === "string") { return value as ToolChoice; } if (value.type === "tool" && typeof value.name === "string") { return { type: "tool", toolName: value.name }; } if (value.type === "function" && isRecord(value.function)) { const name = value.function.name; return typeof name === "string" ? { type: "tool", toolName: name } : undefined; } return typeof value.name === "string" ? { type: "tool", toolName: value.name } : undefined; } type GroqUsage = { inputTokens?: number; outputTokens?: number; promptTokens?: number; completionTokens?: number; totalTokens?: number; }; interface GroqNativeTextResult { text: string; toolCalls: unknown[]; finishReason?: string; usage?: { promptTokens: number; completionTokens: number; totalTokens: number }; } function buildGroqNativeTextResult(result: { text: string; toolCalls?: unknown[]; finishReason?: string; usage?: GroqUsage; }): GroqNativeTextResult { const inputTokens = result.usage?.inputTokens ?? result.usage?.promptTokens ?? 0; const outputTokens = result.usage?.outputTokens ?? result.usage?.completionTokens ?? 0; const usage = result.usage ? { promptTokens: inputTokens, completionTokens: outputTokens, totalTokens: result.usage.totalTokens ?? inputTokens + outputTokens, } : undefined; return { text: result.text, toolCalls: result.toolCalls ?? [], finishReason: result.finishReason, ...(usage ? { usage } : {}), }; } async function generateWithRetry( runtime: IAgentRuntime, groq: ReturnType, modelType: ModelTypeName, model: string, params: { prompt: string; system?: string; temperature: number; maxTokens?: number; omitMaxTokens?: boolean; frequencyPenalty: number; presencePenalty: number; stopSequences: string[]; messages?: ModelMessage[]; tools?: ToolSet; toolChoice?: ToolChoice; responseSchema?: unknown; returnNative?: boolean; } ): Promise { const generate = () => { const details: RecordLlmCallDetails = { model, systemPrompt: params.system ?? "", userPrompt: params.prompt, temperature: params.temperature, maxTokens: params.maxTokens ?? 0, maxTokensOmitted: params.omitMaxTokens ? true : undefined, purpose: "external_llm", actionType: "ai.generateText", }; return recordLlmCall(runtime, details, async () => { // Native tool calling + structured output: when callers pass `tools`, // `toolChoice`, `responseSchema`, or `messages`, route through the AI // SDK's native shape (Groq's OpenAI-compatible chat.completions API // accepts `tools`, `tool_choice`, and `response_format` for JSON mode). // When only `prompt` is supplied, fall back to the simple generate-text // shape — this keeps caching/cost flow untouched for the common path. const sharedSettings = { model: groq.languageModel(model), system: params.system, temperature: params.temperature, // Omit the cap on opt-out (direct-channel Stage-1) so the model's own // max applies; otherwise send the resolved value. ...(params.omitMaxTokens ? {} : { maxOutputTokens: params.maxTokens }), maxRetries: 3, frequencyPenalty: params.frequencyPenalty, presencePenalty: params.presencePenalty, stopSequences: params.stopSequences, ...(params.tools ? { tools: params.tools } : {}), ...(params.toolChoice ? { toolChoice: params.toolChoice } : {}), ...(params.responseSchema ? { output: buildGroqStructuredOutput(params.responseSchema) } : {}), }; const result = params.messages && params.messages.length > 0 ? await generateText({ ...sharedSettings, messages: params.messages }) : await generateText({ ...sharedSettings, prompt: params.prompt }); details.response = result.text; applyUsageToDetails(details, result.usage); return result; }); }; const MAX_RATE_LIMIT_RETRIES = 5; const MAX_TRANSIENT_RETRIES = 2; let rateLimitAttempts = 0; let transientAttempts = 0; while (true) { try { const result = await generate(); // A completion with no text and no tool calls is a provider failure // (moderation block, truncation, upstream bug) — never a legitimate // result. Returning "" here would fabricate a healthy-empty completion // the planner cannot distinguish from a real answer (#9324: throw, // never fabricate) — and would emit success usage telemetry for it. const resultToolCalls = Array.isArray(result.toolCalls) ? result.toolCalls : []; if (result.text.length === 0 && resultToolCalls.length === 0) { throw new ElizaError( `[Groq] ${modelType} returned an empty completion${ result.finishReason ? ` (finishReason: ${result.finishReason})` : "" }`, { code: "MODEL_EMPTY_COMPLETION", context: { modelType, model, finishReason: result.finishReason }, } ); } const usage = normalizeTokenUsage(result.usage) ?? estimateUsage(params.prompt, result.text); emitModelUsed(runtime, modelType, model, usage); if (params.returnNative) { return buildGroqNativeTextResult(result); } const { text } = result; return text; } catch (error) { // error-policy:J2 context-adding rethrow — rate-limit/transient errors // are retried with backoff; fatal errors and exhausted attempts rethrow // the original provider error unchanged. No failure becomes a result. const kind = classifyRetryError(error); if (kind === "rate-limit" && rateLimitAttempts < MAX_RATE_LIMIT_RETRIES) { const message = error instanceof Error ? error.message : String(error); // Respect the server-suggested wait, then add exponential jitter on // top so multiple parallel callers don't re-collide on the same // window boundary. const hinted = extractRetryDelay(message); const backoff = Math.min(30_000, 500 * 2 ** rateLimitAttempts); const delay = hinted + backoff; rateLimitAttempts += 1; logger.warn( `Groq rate limit hit (attempt ${rateLimitAttempts}/${MAX_RATE_LIMIT_RETRIES}), retrying in ${delay}ms` ); await new Promise((resolve) => setTimeout(resolve, delay)); continue; } if (kind === "transient" && transientAttempts < MAX_TRANSIENT_RETRIES) { // AI SDK already retried with exponential backoff; use a small fixed // backoff with jitter here to smooth over post-exhaustion flakiness. const delay = 1_000 + Math.floor(Math.random() * 1_500); transientAttempts += 1; logger.warn( `Groq transient failure (attempt ${transientAttempts}/${MAX_TRANSIENT_RETRIES}), retrying in ${delay}ms: ${error instanceof Error ? error.message : String(error)}` ); await new Promise((resolve) => setTimeout(resolve, delay)); continue; } throw error; } } } function buildGroqGenerateParams( params: GenerateTextParams, systemPrompt: string | undefined, promptText: string ): { prompt: string; system?: string; temperature: number; maxTokens?: number; omitMaxTokens?: boolean; frequencyPenalty: number; presencePenalty: number; stopSequences: string[]; messages?: ModelMessage[]; tools?: ToolSet; toolChoice?: ToolChoice; responseSchema?: unknown; returnNative?: boolean; } { const paramsWithNative = params as GenerateTextParams & { messages?: ModelMessage[]; tools?: unknown; toolChoice?: unknown; responseSchema?: unknown; }; const returnNative = Boolean( paramsWithNative.messages || paramsWithNative.tools || paramsWithNative.toolChoice || paramsWithNative.responseSchema ); const normalizedTools = readGroqToolSet(paramsWithNative.tools); const normalizedToolChoice = readGroqToolChoice(paramsWithNative.toolChoice); return { prompt: promptText, system: systemPrompt, temperature: boundedNumber(params.temperature, 0.7, 0, 2), // Stage-1 direct reply opts out of any cap; everyone else keeps the 8192 // default so they stay bounded. maxTokens: params.omitMaxTokens ? undefined : positiveInteger(params.maxTokens, 8192), omitMaxTokens: params.omitMaxTokens, frequencyPenalty: boundedNumber(params.frequencyPenalty, 0.7, -2, 2), presencePenalty: boundedNumber(params.presencePenalty, 0.7, -2, 2), stopSequences: stringArray(params.stopSequences), ...(paramsWithNative.messages ? { messages: paramsWithNative.messages } : {}), ...(normalizedTools ? { tools: normalizedTools } : {}), ...(normalizedToolChoice ? { toolChoice: normalizedToolChoice } : {}), ...(paramsWithNative.responseSchema ? { responseSchema: paramsWithNative.responseSchema } : {}), ...(returnNative ? { returnNative } : {}), }; } async function handleTextModel( runtime: IAgentRuntime, params: GenerateTextParams, modelType: ModelTypeName ): Promise { const groq = createGroqClient(runtime); const model = getTextModelForType(runtime, modelType); const system = resolveGroqSystemPrompt(runtime, params); const result = await generateWithRetry( runtime, groq, modelType, model, buildGroqGenerateParams(params, system, resolveGroqPrompt(params, system)) ); // Native result (with toolCalls / usage / finishReason) is cast through the // string return type because elizaOS's plugin Model handler signature is // `(runtime, params) => Promise`. The runtime // unwraps the native shape via `useModel` consumers that pass `tools` / // `messages` / `responseSchema` / `toolChoice`. return result as string; } function getTextModelForType(runtime: IAgentRuntime, modelType: string): string { switch (modelType) { case ModelType.TEXT_NANO: return getNanoModel(runtime); case ModelType.TEXT_MEDIUM: return getMediumModel(runtime); case ModelType.TEXT_SMALL: return getSmallModel(runtime); case ModelType.TEXT_LARGE: return getLargeModel(runtime); case ModelType.TEXT_MEGA: return getMegaModel(runtime); case ModelType.RESPONSE_HANDLER: return getResponseHandlerModel(runtime); case ModelType.ACTION_PLANNER: return getActionPlannerModel(runtime); default: return getLargeModel(runtime); } } export const groqPlugin: Plugin = { name: "groq", description: "Groq LLM provider - fast inference with GPT-OSS models", autoEnable: { envKeys: ["GROQ_API_KEY"], }, config: { GROQ_API_KEY: env("GROQ_API_KEY"), GROQ_BASE_URL: env("GROQ_BASE_URL"), GROQ_NANO_MODEL: env("GROQ_NANO_MODEL"), GROQ_MEDIUM_MODEL: env("GROQ_MEDIUM_MODEL"), GROQ_SMALL_MODEL: env("GROQ_SMALL_MODEL"), GROQ_LARGE_MODEL: env("GROQ_LARGE_MODEL"), GROQ_MEGA_MODEL: env("GROQ_MEGA_MODEL"), GROQ_RESPONSE_HANDLER_MODEL: env("GROQ_RESPONSE_HANDLER_MODEL"), GROQ_SHOULD_RESPOND_MODEL: env("GROQ_SHOULD_RESPOND_MODEL"), GROQ_ACTION_PLANNER_MODEL: env("GROQ_ACTION_PLANNER_MODEL"), GROQ_PLANNER_MODEL: env("GROQ_PLANNER_MODEL"), GROQ_TRANSCRIPTION_MODEL: env("GROQ_TRANSCRIPTION_MODEL"), TRANSCRIPTION_MODEL: env("TRANSCRIPTION_MODEL"), NANO_MODEL: env("NANO_MODEL"), MEDIUM_MODEL: env("MEDIUM_MODEL"), SMALL_MODEL: env("SMALL_MODEL"), LARGE_MODEL: env("LARGE_MODEL"), MEGA_MODEL: env("MEGA_MODEL"), RESPONSE_HANDLER_MODEL: env("RESPONSE_HANDLER_MODEL"), SHOULD_RESPOND_MODEL: env("SHOULD_RESPOND_MODEL"), ACTION_PLANNER_MODEL: env("ACTION_PLANNER_MODEL"), PLANNER_MODEL: env("PLANNER_MODEL"), }, async init(_config: Record, runtime: IAgentRuntime): Promise { const apiKey = nonEmptyString(runtime.getSetting("GROQ_API_KEY")); if (!apiKey && !isBrowser()) { throw new Error("GROQ_API_KEY is required"); } }, models: { [ModelType.TEXT_NANO]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.TEXT_NANO), [ModelType.TEXT_SMALL]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.TEXT_SMALL), [ModelType.TEXT_MEDIUM]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.TEXT_MEDIUM), [ModelType.TEXT_LARGE]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.TEXT_LARGE), [ModelType.TEXT_MEGA]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.TEXT_MEGA), [ModelType.RESPONSE_HANDLER]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.RESPONSE_HANDLER), [ModelType.ACTION_PLANNER]: (runtime, params: GenerateTextParams) => handleTextModel(runtime, params, ModelType.ACTION_PLANNER), [ModelType.TRANSCRIPTION]: async (runtime, params) => { type AudioDataShape = { audioData: Uint8Array }; function hasAudioData(obj: object): obj is AudioDataShape { return "audioData" in obj && (obj as AudioDataShape).audioData instanceof Uint8Array; } if (isBrowser()) { throw new Error( "Groq TRANSCRIPTION is not supported directly in browsers. Use a server proxy or submit a Blob/ArrayBuffer to a server." ); } const buffer = getRuntimeBuffer(); if (!buffer) { throw new Error("Groq TRANSCRIPTION requires Buffer support outside browsers."); } const audioBuffer: Uint8Array = typeof params === "string" ? buffer.from(params, "base64") : buffer.isBuffer(params) ? (params as Uint8Array) : typeof params === "object" && params !== null && hasAudioData(params) ? buffer.from((params as AudioDataShape).audioData) : buffer.alloc(0); if (audioBuffer.byteLength === 0) { throw new Error("Groq TRANSCRIPTION requires non-empty audio data."); } const baseURL = getBaseURL(runtime); const transcriptionModel = getTranscriptionModel(runtime); const formData = new FormData(); formData.append( "file", new File([audioBuffer as BlobPart], "audio.mp3", { type: "audio/mp3" }) ); formData.append("model", transcriptionModel); const apiKey = nonEmptyString(runtime.getSetting("GROQ_API_KEY")); // A missing credential must surface as a typed failure before the // request goes out — an empty bearer token would masquerade as a // provider-side 401 and hide the real misconfiguration. if (!apiKey) { throw new ElizaError("[Groq] TRANSCRIPTION requires GROQ_API_KEY", { code: "MODEL_MISSING_CREDENTIAL", context: { modelType: ModelType.TRANSCRIPTION }, }); } const details: RecordLlmCallDetails = { model: transcriptionModel, systemPrompt: "", userPrompt: `audio transcription request: ${audioBuffer.byteLength} bytes`, temperature: 0, maxTokens: 0, purpose: "external_llm", actionType: "groq.audio.transcriptions.create", }; const data = await recordLlmCall(runtime, details, async () => { const response = await fetch(`${baseURL}/audio/transcriptions`, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, }, body: formData, }); if (!response.ok) { throw new Error(`Transcription failed: ${response.status} ${await response.text()}`); } const result = (await response.json()) as { text: string }; details.response = result.text; return result; }); return data.text; }, [ModelType.TEXT_TO_SPEECH]: async (runtime: IAgentRuntime, params) => { if (isBrowser()) { throw new Error( "Groq TEXT_TO_SPEECH is not supported directly in browsers. Use a server proxy." ); } const payload = typeof params === "string" ? { text: params } : params && typeof params === "object" ? (params as { text?: string; voice?: string; model?: string; responseFormat?: string; response_format?: string; }) : {}; const text = nonEmptyString(payload.text); if (!text) { throw new Error("Groq TEXT_TO_SPEECH requires non-empty text."); } const baseURL = getBaseURL(runtime); const modelSetting = runtime.getSetting("GROQ_TTS_MODEL"); const voiceSetting = runtime.getSetting("GROQ_TTS_VOICE"); const responseFormatSetting = runtime.getSetting("GROQ_TTS_RESPONSE_FORMAT"); const model = typeof payload.model === "string" && payload.model ? payload.model : typeof modelSetting === "string" ? modelSetting : DEFAULT_TTS_MODEL; const voice = typeof payload.voice === "string" && payload.voice ? payload.voice : typeof voiceSetting === "string" ? voiceSetting : DEFAULT_TTS_VOICE; const responseFormat = typeof payload.responseFormat === "string" && payload.responseFormat ? payload.responseFormat : typeof payload.response_format === "string" && payload.response_format ? payload.response_format : typeof responseFormatSetting === "string" ? responseFormatSetting : DEFAULT_TTS_RESPONSE_FORMAT; const apiKey = nonEmptyString(runtime.getSetting("GROQ_API_KEY")); // Same rule as TRANSCRIPTION: a missing credential is a typed local // failure, never an empty bearer token sent upstream. if (!apiKey) { throw new ElizaError("[Groq] TEXT_TO_SPEECH requires GROQ_API_KEY", { code: "MODEL_MISSING_CREDENTIAL", context: { modelType: ModelType.TEXT_TO_SPEECH }, }); } const details: RecordLlmCallDetails = { model, systemPrompt: "", userPrompt: text, temperature: 0, maxTokens: 0, purpose: "external_llm", actionType: "groq.audio.speech.create", }; const arrayBuffer = await recordLlmCall(runtime, details, async () => { const response = await fetch(`${baseURL}/audio/speech`, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model, voice, input: text, response_format: responseFormat, }), }); if (!response.ok) { throw new Error(`TTS failed: ${response.status} ${await response.text()}`); } const result = await response.arrayBuffer(); details.response = `[audio bytes=${result.byteLength} format=${responseFormat}]`; return result; }); return new Uint8Array(arrayBuffer); }, }, tests: [ { name: "groq_plugin_tests", tests: [ { name: "validate_api_key", fn: async (runtime) => { const baseURL = getBaseURL(runtime); const response = await fetch(`${baseURL}/models`, { headers: { Authorization: `Bearer ${runtime.getSetting("GROQ_API_KEY")}`, }, }); if (!response.ok) { throw new Error(`API key validation failed: ${response.statusText}`); } const data = (await response.json()) as { data: Array<{ id: string; owned_by: string }>; }; logger.info(`Groq API validated, ${data.data.length} models available`); }, }, { name: "text_small", fn: async (runtime) => { const text = await runtime.useModel(ModelType.TEXT_SMALL, { prompt: "Say hello in exactly 3 words.", }); if (!text || text.length === 0) { throw new Error("Empty response from TEXT_SMALL"); } logger.info("TEXT_SMALL:", text); }, }, { name: "text_large", fn: async (runtime) => { const text = await runtime.useModel(ModelType.TEXT_LARGE, { prompt: "What is 2+2? Answer with just the number.", }); if (!text || text.length === 0) { throw new Error("Empty response from TEXT_LARGE"); } logger.info("TEXT_LARGE:", text); }, }, ], }, ], }; export default groqPlugin;