/** * Assembles `openaiPlugin` — the elizaOS `Plugin` object that registers every * OpenAI-backed model handler on the `AgentRuntime`: text tiers (small→mega, * response-handler, action-planner), embeddings, tokenizer, image * generation/description, transcription, TTS, and deep research. * * Text/embedding/tokenizer/research handlers register statically via `models`. * The media handlers (IMAGE, IMAGE_DESCRIPTION, TRANSCRIPTION, TEXT_TO_SPEECH) * register in `init()` through `registerMediaModels`, which skips them in * Cerebras mode unless a per-capability endpoint override points at a server * that serves them. `tests` carries the live connectivity/round-trip suite the * plugin loader runs against a real endpoint. */ import type { TextToSpeechParams as CoreTextToSpeechParams, TranscriptionParams as CoreTranscriptionParams, DetokenizeTextParams, GenerateTextParams, IAgentRuntime, ImageDescriptionParams, ImageGenerationParams, Plugin, ProcessEnvLike, ResearchParams, ResearchResult, TextEmbeddingParams, TokenizeTextParams, } from "@elizaos/core"; import { logger, ModelType } from "@elizaos/core"; import { initializeOpenAI } from "./init"; import { handleActionPlanner, handleImageDescription, handleImageGeneration, handleResearch, handleResponseHandler, handleTextEmbedding, handleTextLarge, handleTextMedium, handleTextMega, handleTextNano, handleTextSmall, handleTextToSpeech, handleTokenizerDecode, handleTokenizerEncode, handleTranscription, } from "./models"; import type { ImageGenerationResult, OpenAIPluginConfig, TextStreamResult } from "./types"; import { getAuthHeader, getBaseURL, getSetting, isCerebrasMode } from "./utils/config"; function getProcessEnv(): ProcessEnvLike { if (typeof process === "undefined") { return {}; } return process.env as ProcessEnvLike; } const env = getProcessEnv(); const TEXT_NANO_MODEL_TYPE = ModelType.TEXT_NANO as string; const TEXT_MEDIUM_MODEL_TYPE = ModelType.TEXT_MEDIUM as string; const TEXT_MEGA_MODEL_TYPE = ModelType.TEXT_MEGA as string; const RESPONSE_HANDLER_MODEL_TYPE = ModelType.RESPONSE_HANDLER as string; const ACTION_PLANNER_MODEL_TYPE = ModelType.ACTION_PLANNER as string; function hasExplicitCapabilityOverride( runtime: IAgentRuntime, overrideKeys: readonly string[] ): boolean { return overrideKeys.some((key) => { const value = getSetting(runtime, key); return typeof value === "string" && value.trim().length > 0; }); } // Per-capability endpoint overrides: when set, the capability does not POST to // getBaseURL, so it stays registered even in Cerebras mode. Only the base URL // counts: OPENAI_IMAGE_DESCRIPTION_API_KEY alone still posts to getBaseURL // (getImageDescriptionBaseURL falls back to it), i.e. straight at Cerebras. // TRANSCRIPTION, TEXT_TO_SPEECH, and IMAGE have no such override today. const mediaModelOverrideKeys: Record = { [ModelType.IMAGE_DESCRIPTION]: ["OPENAI_IMAGE_DESCRIPTION_BASE_URL"], }; const mediaModels: NonNullable = { [ModelType.IMAGE]: async ( runtime: IAgentRuntime, params: ImageGenerationParams ): Promise => { return handleImageGeneration(runtime, params); }, [ModelType.IMAGE_DESCRIPTION]: async ( runtime: IAgentRuntime, params: ImageDescriptionParams | string ): Promise<{ title: string; description: string }> => { return handleImageDescription(runtime, params); }, [ModelType.TRANSCRIPTION]: async ( runtime: IAgentRuntime, input: CoreTranscriptionParams | Buffer | string ): Promise => { return handleTranscription(runtime, input); }, [ModelType.TEXT_TO_SPEECH]: async ( runtime: IAgentRuntime, input: CoreTextToSpeechParams | string ): Promise => { return handleTextToSpeech(runtime, input); }, }; // Cerebras serves text models only: vision chat completions, /audio/transcriptions, // /audio/speech, and /images/generations all fail against its endpoint. Mirror the // embedding shouldUseLocalEmbeddingFallback gate (models/embedding.ts): in Cerebras // mode these capabilities stay unregistered unless an explicit per-capability // override points them at an endpoint that serves them, so consumers (e.g. // plugin-discord's isImageDescriptionEnabled) skip gracefully instead of failing // on every attachment. function registerMediaModels(runtime: IAgentRuntime): void { const cerebras = isCerebrasMode(runtime); for (const [modelType, handler] of Object.entries(mediaModels)) { if ( cerebras && !hasExplicitCapabilityOverride(runtime, mediaModelOverrideKeys[modelType] ?? []) ) { logger.info(`[OpenAI] Not registering ${modelType}: the Cerebras endpoint does not serve it`); continue; } runtime.registerModel( modelType, handler as Parameters[1], openaiPlugin.name, openaiPlugin.priority ); } } export const openaiPlugin: Plugin = { name: "openai", description: "OpenAI API integration for text, image, audio, and embedding models", autoEnable: { envKeys: ["OPENAI_API_KEY", "CEREBRAS_API_KEY", "EVOLINK_API_KEY"], }, config: { OPENAI_API_KEY: env.OPENAI_API_KEY ?? null, OPENAI_BASE_URL: env.OPENAI_BASE_URL ?? null, EVOLINK_API_KEY: env.EVOLINK_API_KEY ?? null, EVOLINK_BASE_URL: env.EVOLINK_BASE_URL ?? null, EVOLINK_MODEL: env.EVOLINK_MODEL ?? null, OPENAI_NANO_MODEL: env.OPENAI_NANO_MODEL ?? null, OPENAI_MEDIUM_MODEL: env.OPENAI_MEDIUM_MODEL ?? null, OPENAI_SMALL_MODEL: env.OPENAI_SMALL_MODEL ?? null, OPENAI_LARGE_MODEL: env.OPENAI_LARGE_MODEL ?? null, OPENAI_MEGA_MODEL: env.OPENAI_MEGA_MODEL ?? null, OPENAI_RESPONSE_HANDLER_MODEL: env.OPENAI_RESPONSE_HANDLER_MODEL ?? null, OPENAI_SHOULD_RESPOND_MODEL: env.OPENAI_SHOULD_RESPOND_MODEL ?? null, OPENAI_ACTION_PLANNER_MODEL: env.OPENAI_ACTION_PLANNER_MODEL ?? null, OPENAI_PLANNER_MODEL: env.OPENAI_PLANNER_MODEL ?? null, NANO_MODEL: env.NANO_MODEL ?? null, MEDIUM_MODEL: env.MEDIUM_MODEL ?? null, SMALL_MODEL: env.SMALL_MODEL ?? null, LARGE_MODEL: env.LARGE_MODEL ?? null, MEGA_MODEL: env.MEGA_MODEL ?? null, RESPONSE_HANDLER_MODEL: env.RESPONSE_HANDLER_MODEL ?? null, SHOULD_RESPOND_MODEL: env.SHOULD_RESPOND_MODEL ?? null, ACTION_PLANNER_MODEL: env.ACTION_PLANNER_MODEL ?? null, PLANNER_MODEL: env.PLANNER_MODEL ?? null, OPENAI_EMBEDDING_MODEL: env.OPENAI_EMBEDDING_MODEL ?? null, OPENAI_EMBEDDING_API_KEY: env.OPENAI_EMBEDDING_API_KEY ?? null, OPENAI_EMBEDDING_URL: env.OPENAI_EMBEDDING_URL ?? null, OPENAI_EMBEDDING_DIMENSIONS: env.OPENAI_EMBEDDING_DIMENSIONS ?? null, OPENAI_IMAGE_DESCRIPTION_API_KEY: env.OPENAI_IMAGE_DESCRIPTION_API_KEY ?? null, OPENAI_IMAGE_DESCRIPTION_BASE_URL: env.OPENAI_IMAGE_DESCRIPTION_BASE_URL ?? null, OPENAI_IMAGE_DESCRIPTION_MODEL: env.OPENAI_IMAGE_DESCRIPTION_MODEL ?? null, OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS ?? null, OPENAI_EXPERIMENTAL_TELEMETRY: env.OPENAI_EXPERIMENTAL_TELEMETRY ?? null, OPENAI_RESEARCH_MODEL: env.OPENAI_RESEARCH_MODEL ?? null, OPENAI_RESEARCH_TIMEOUT: env.OPENAI_RESEARCH_TIMEOUT ?? null, }, async init(config: Record, runtime: IAgentRuntime): Promise { initializeOpenAI(config as OpenAIPluginConfig | undefined, runtime); registerMediaModels(runtime); }, models: { [ModelType.TEXT_EMBEDDING]: async ( runtime: IAgentRuntime, params: TextEmbeddingParams | string | null ): Promise => { return handleTextEmbedding(runtime, params); }, [ModelType.TEXT_TOKENIZER_ENCODE]: async ( runtime: IAgentRuntime, params: TokenizeTextParams ): Promise => { return handleTokenizerEncode(runtime, params); }, [ModelType.TEXT_TOKENIZER_DECODE]: async ( runtime: IAgentRuntime, params: DetokenizeTextParams ): Promise => { return handleTokenizerDecode(runtime, params); }, [ModelType.TEXT_SMALL]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleTextSmall(runtime, params); }, [TEXT_NANO_MODEL_TYPE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleTextNano(runtime, params); }, [TEXT_MEDIUM_MODEL_TYPE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleTextMedium(runtime, params); }, [ModelType.TEXT_LARGE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleTextLarge(runtime, params); }, [TEXT_MEGA_MODEL_TYPE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleTextMega(runtime, params); }, [RESPONSE_HANDLER_MODEL_TYPE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleResponseHandler(runtime, params); }, [ACTION_PLANNER_MODEL_TYPE]: async ( runtime: IAgentRuntime, params: GenerateTextParams ): Promise => { return handleActionPlanner(runtime, params); }, // IMAGE / IMAGE_DESCRIPTION / TRANSCRIPTION / TEXT_TO_SPEECH are registered // in init() via registerMediaModels so registration can be gated on the // resolved endpoint actually serving them (see Cerebras gate above). [ModelType.RESEARCH]: async ( runtime: IAgentRuntime, params: ResearchParams ): Promise => { return handleResearch(runtime, params); }, }, tests: [ { name: "openai_plugin_tests", tests: [ { name: "openai_test_api_connectivity", fn: async (runtime: IAgentRuntime): Promise => { const baseURL = getBaseURL(runtime); const response = await fetch(`${baseURL}/models`, { headers: getAuthHeader(runtime), }); if (!response.ok) { throw new Error( `API connectivity test failed: ${response.status} ${response.statusText}` ); } const data = (await response.json()) as { data?: unknown[] }; logger.info(`[OpenAI Test] API connected. ${data.data?.length ?? 0} models available.`); }, }, { name: "openai_test_text_embedding", fn: async (runtime: IAgentRuntime): Promise => { const embedding = await runtime.useModel(ModelType.TEXT_EMBEDDING, { text: "Hello, world!", }); if (!Array.isArray(embedding) || embedding.length === 0) { throw new Error("Embedding should return a non-empty array"); } logger.info(`[OpenAI Test] Generated embedding with ${embedding.length} dimensions`); }, }, { name: "openai_test_text_small", fn: async (runtime: IAgentRuntime): Promise => { const text = await runtime.useModel(ModelType.TEXT_SMALL, { prompt: "Say hello in exactly 5 words.", }); if (typeof text !== "string" || text.length === 0) { throw new Error("TEXT_SMALL should return non-empty string"); } logger.info(`[OpenAI Test] TEXT_SMALL generated: "${text.substring(0, 50)}..."`); }, }, { name: "openai_test_text_large", fn: async (runtime: IAgentRuntime): Promise => { const text = await runtime.useModel(ModelType.TEXT_LARGE, { prompt: "Explain quantum computing in 2 sentences.", }); if (typeof text !== "string" || text.length === 0) { throw new Error("TEXT_LARGE should return non-empty string"); } logger.info(`[OpenAI Test] TEXT_LARGE generated: "${text.substring(0, 50)}..."`); }, }, { name: "openai_test_tokenizer_roundtrip", fn: async (runtime: IAgentRuntime): Promise => { const originalText = "Hello, tokenizer test!"; const tokens = await runtime.useModel(ModelType.TEXT_TOKENIZER_ENCODE, { prompt: originalText, modelType: ModelType.TEXT_SMALL, }); if (!Array.isArray(tokens) || tokens.length === 0) { throw new Error("Tokenization should return non-empty token array"); } const decodedText = await runtime.useModel(ModelType.TEXT_TOKENIZER_DECODE, { tokens, modelType: ModelType.TEXT_SMALL, }); if (decodedText !== originalText) { throw new Error( `Tokenizer roundtrip failed: expected "${originalText}", got "${decodedText}"` ); } logger.info(`[OpenAI Test] Tokenizer roundtrip successful (${tokens.length} tokens)`); }, }, { name: "openai_test_streaming", fn: async (runtime: IAgentRuntime): Promise => { const chunks: string[] = []; const result = await runtime.useModel(ModelType.TEXT_LARGE, { prompt: "Count from 1 to 5, one number per line.", stream: true, onStreamChunk: (chunk: string) => { chunks.push(chunk); }, }); if (typeof result !== "string" || result.length === 0) { throw new Error("Streaming should return non-empty result"); } if (chunks.length === 0) { throw new Error("No streaming chunks received"); } logger.info(`[OpenAI Test] Streaming test: ${chunks.length} chunks received`); }, }, { name: "openai_test_image_description", fn: async (runtime: IAgentRuntime): Promise => { const testImageUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a7/Camponotus_flavomarginatus_ant.jpg/440px-Camponotus_flavomarginatus_ant.jpg"; const result = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, testImageUrl); if ( !result || typeof result !== "object" || !("title" in result) || !("description" in result) ) { throw new Error("Image description should return { title, description }"); } logger.info(`[OpenAI Test] Image described: "${result.title}"`); }, }, { name: "openai_test_transcription", fn: async (runtime: IAgentRuntime): Promise => { // Fetch a short audio sample const audioUrl = "https://upload.wikimedia.org/wikipedia/commons/2/25/En-Open_Source.ogg"; const response = await fetch(audioUrl); const arrayBuffer = await response.arrayBuffer(); const audioBuffer = Buffer.from(new Uint8Array(arrayBuffer)); const transcription = await runtime.useModel(ModelType.TRANSCRIPTION, audioBuffer); if (typeof transcription !== "string") { throw new Error("Transcription should return a string"); } logger.info(`[OpenAI Test] Transcription: "${transcription.substring(0, 50)}..."`); }, }, { name: "openai_test_text_to_speech", fn: async (runtime: IAgentRuntime): Promise => { const audioData = await runtime.useModel(ModelType.TEXT_TO_SPEECH, { text: "Hello, this is a text-to-speech test.", }); if (!(audioData instanceof ArrayBuffer) || audioData.byteLength === 0) { throw new Error("TTS should return non-empty ArrayBuffer"); } logger.info(`[OpenAI Test] TTS generated ${audioData.byteLength} bytes of audio`); }, }, { name: "openai_test_structured_output_via_text_large", fn: async (runtime: IAgentRuntime): Promise => { const result = await runtime.useModel(ModelType.TEXT_LARGE, { prompt: "Return a JSON object with exactly these fields: name (string), age (number), active (boolean)", responseSchema: { type: "object", properties: { name: { type: "string" }, age: { type: "number" }, active: { type: "boolean" }, }, required: ["name", "age", "active"], }, } as GenerateTextParams); if (!result || (typeof result !== "object" && typeof result !== "string")) { throw new Error("Structured output should return an object or text"); } logger.info( `[OpenAI Test] Structured output: ${JSON.stringify(result).substring(0, 100)}` ); }, }, { name: "openai_test_research", fn: async (runtime: IAgentRuntime): Promise => { // Note: Deep research can take a long time (minutes to hours) // This test uses a simple query with maxToolCalls to limit execution time const result = await runtime.useModel(ModelType.RESEARCH, { input: "What is the current date and time?", tools: [{ type: "web_search_preview" }], maxToolCalls: 3, // Limit tool calls for faster test execution }); if (!result || typeof result !== "object" || !("text" in result)) { throw new Error("Research should return an object with text property"); } if (typeof result.text !== "string" || result.text.length === 0) { throw new Error("Research result text should be a non-empty string"); } logger.info( `[OpenAI Test] Research completed. Text length: ${result.text.length}, Annotations: ${result.annotations.length}` ); }, }, ], }, ], }; export default openaiPlugin;