/** * Shape tests exercising the text handler's plumbing — message normalization, * model-usage events, and trajectory recording — against a mocked `ai` SDK * (`generateText`/`streamText`), no network. */ import type { IAgentRuntime } from "@elizaos/core"; import { EventType, ModelType, runWithTrajectoryContext } from "@elizaos/core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const aiMocks = vi.hoisted(() => ({ generateText: vi.fn(), streamText: vi.fn(), })); // `getSetting` in utils/config falls back to `process.env` when the test // runtime returns undefined. The repo-root `.env` is auto-loaded by bun (and // re-injected on dynamic import), so a developer or CI environment with // `OPENAI_BASE_URL=https://api.cerebras.ai/v1` or `OPENAI_SMALL_MODEL=...` // flips the Cerebras codepath / overrides the model default. We use // `vi.stubEnv` to pin env vars deterministically — vitest restores them // in `vi.unstubAllEnvs`, and the pinned values survive bun's dotenv re-injection. // // `OPENAI_BASE_URL` is pinned to a non-Cerebras URL (rather than empty) // because empty strings short-circuit `getSetting` to `""`, which is not // the same as "unset" for downstream callers. const ENV_KEYS_TO_CLEAR = [ "ELIZA_PROVIDER", "CEREBRAS_API_KEY", "OPENAI_SMALL_MODEL", "SMALL_MODEL", "OPENAI_LARGE_MODEL", "LARGE_MODEL", "OPENAI_RESPONSE_HANDLER_MODEL", "OPENAI_SHOULD_RESPOND_MODEL", "RESPONSE_HANDLER_MODEL", "SHOULD_RESPOND_MODEL", ] as const; beforeEach(() => { vi.stubEnv("OPENAI_BASE_URL", "https://api.openai.com/v1"); vi.stubEnv("OPENAI_API_KEY", "test-key"); for (const key of ENV_KEYS_TO_CLEAR) { vi.stubEnv(key, undefined); } }); vi.mock("ai", () => ({ generateText: aiMocks.generateText, streamText: aiMocks.streamText, jsonSchema: (schema: unknown) => ({ jsonSchema: schema }), Output: { object: ({ schema, name, description, }: { schema: unknown; name?: string; description?: string; }) => ({ name: "object", responseFormat: Promise.resolve({ type: "json", schema: (schema as { jsonSchema?: unknown }).jsonSchema ?? schema, ...(name ? { name } : {}), ...(description ? { description } : {}), }), parseCompleteOutput: async ({ text }: { text: string }) => JSON.parse(text), parsePartialOutput: async () => undefined, createElementStreamTransform: () => undefined, }), }, })); vi.mock("../providers", () => ({ createOpenAIClient: () => ({ chat: (modelName: string) => ({ modelName }), // Genuine-OpenAI text now routes through the Responses API so the // agent-level injector can attach `web_search`; both surfaces share the // same param plumbing these tests assert. responses: (modelName: string) => ({ modelName }), }), })); interface CapturedLlmCall { stepId: string; actionType: string; response?: string; promptTokens?: number; completionTokens?: number; finishReason?: string; toolCalls?: unknown; } function createRuntime(options?: { trajectoryCalls?: CapturedLlmCall[] }) { const trajectoryLogger = options?.trajectoryCalls ? { isEnabled: () => true, logLlmCall: (params: CapturedLlmCall) => { options.trajectoryCalls?.push(params); }, } : null; const runtime = { character: { name: "Ada", system: "system prompt" }, emitEvent: vi.fn(), getService: vi.fn((name: string) => (name === "trajectories" ? trajectoryLogger : null)), getServicesByType: vi.fn((type: string) => type === "trajectories" && trajectoryLogger ? [trajectoryLogger] : [] ), getSetting: vi.fn((key: string) => { const settings: Record = { OPENAI_API_KEY: "test-key", OPENAI_SMALL_MODEL: "gpt-test-small", }; return settings[key]; }), }; return runtime as IAgentRuntime; } function expectNativeTextResult(value: unknown): asserts value is Record { expect(value).toEqual(expect.objectContaining({ text: expect.any(String) })); } afterEach(() => { vi.unstubAllEnvs(); vi.clearAllMocks(); }); describe("OpenAI native text plumbing", () => { it("passes messages, tools, toolChoice, schema, and provider options through", async () => { aiMocks.generateText.mockResolvedValue({ text: "ok", toolCalls: [{ toolName: "lookup", input: { q: "x" } }], finishReason: "tool-calls", usage: { inputTokens: 7, outputTokens: 3, cachedInputTokens: 5 }, }); const { handleTextSmall } = await import("../models/text"); const messages = [{ role: "user", content: "use the tool" }]; const tools = { lookup: { description: "Lookup", inputSchema: { type: "object" } } }; const toolChoice = { type: "tool", toolName: "lookup" }; const responseSchema = { type: "object", properties: { answer: { type: "string" } } }; const result = await handleTextSmall(createRuntime(), { prompt: "legacy prompt", messages, tools, toolChoice, responseSchema, providerOptions: { agentName: "Ada", openai: { promptCacheKey: "cache-key", promptCacheRetention: "24h" }, custom: { enabled: true }, }, } as never); expectNativeTextResult(result); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.messages).toEqual(messages); expect(call).not.toHaveProperty("prompt"); expect(call.tools).toBe(tools); expect(call.toolChoice).toBe(toolChoice); expect(call.providerOptions).toEqual({ custom: { enabled: true }, openai: { promptCacheKey: "cache-key", promptCacheRetention: "24h" }, }); expect(call.experimental_telemetry).toMatchObject({ functionId: "agent:Ada", metadata: { agentName: "Ada" }, }); await expect( (call.output as { responseFormat: Promise }).responseFormat ).resolves.toEqual({ type: "json", schema: { type: "object", properties: { answer: { type: "string" } }, required: ["answer"], additionalProperties: false, }, }); expect(result).toMatchObject({ text: "ok", toolCalls: [{ toolName: "lookup", input: { q: "x" } }], finishReason: "tool-calls", usage: { promptTokens: 7, completionTokens: 3, totalTokens: 10, cachedPromptTokens: 5, cacheReadInputTokens: 5, }, }); }, 180_000); it("honors a per-call model override before slot defaults", async () => { aiMocks.generateText.mockResolvedValue({ text: "ok", usage: { inputTokens: 1, outputTokens: 1 }, }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(createRuntime(), { prompt: "use the workflow model", model: " gpt-oss-120b ", }); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.model).toEqual({ modelName: "gpt-oss-120b" }); }); it("omits maxOutputTokens only when omitMaxTokens is set", async () => { aiMocks.generateText.mockResolvedValue({ text: "ok", usage: { inputTokens: 1, outputTokens: 1 }, }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(createRuntime(), { prompt: "use provider max", omitMaxTokens: true, } as never); await handleTextSmall(createRuntime(), { prompt: "use default cap", } as never); const omittedCall = aiMocks.generateText.mock.calls[0][0] as Record; const defaultCall = aiMocks.generateText.mock.calls[1][0] as Record; expect(omittedCall).not.toHaveProperty("maxOutputTokens"); expect(defaultCall.maxOutputTokens).toBe(8192); }); it("keeps streaming native tool-call plumbing in parity with non-streaming", async () => { const toolCalls = [{ toolName: "lookup", input: { q: "x" } }]; const usage = { inputTokens: 7, outputTokens: 3, cachedInputTokens: 5 }; aiMocks.generateText.mockResolvedValue({ text: "ok", toolCalls, finishReason: "tool-calls", usage, }); aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { yield "ok"; })(), text: Promise.resolve("ok"), toolCalls: Promise.resolve(toolCalls), finishReason: Promise.resolve("tool-calls"), usage: Promise.resolve(usage), }); const { handleTextSmall } = await import("../models/text"); const baseParams = { prompt: "legacy prompt", messages: [{ role: "user", content: "use the tool" }], tools: { lookup: { description: "Lookup", inputSchema: { type: "object" } } }, toolChoice: { type: "tool", toolName: "lookup" }, responseSchema: { type: "object", properties: { answer: { type: "string" } } }, providerOptions: { openai: { promptCacheKey: "cache-key", promptCacheRetention: "24h" }, custom: { enabled: true }, }, }; const nonStream = await handleTextSmall(createRuntime(), baseParams as never); const stream = await handleTextSmall(createRuntime(), { ...baseParams, stream: true } as never); const nonStreamCall = aiMocks.generateText.mock.calls[0][0] as Record; const streamCall = aiMocks.streamText.mock.calls[0][0] as Record; expect(streamCall.messages).toEqual(nonStreamCall.messages); expect(streamCall).not.toHaveProperty("prompt"); expect(streamCall.tools).toBe(nonStreamCall.tools); expect(streamCall.toolChoice).toBe(nonStreamCall.toolChoice); expect(streamCall.providerOptions).toEqual(nonStreamCall.providerOptions); await expect( (streamCall.output as { responseFormat: Promise }).responseFormat ).resolves.toEqual( await (nonStreamCall.output as { responseFormat: Promise }).responseFormat ); expectNativeTextResult(nonStream); expect(nonStream).toMatchObject({ toolCalls, finishReason: "tool-calls" }); await expect((stream as { toolCalls: Promise }).toolCalls).resolves.toEqual(toolCalls); await expect((stream as { finishReason: Promise }).finishReason).resolves.toBe( "tool-calls" ); await expect((stream as { usage: Promise }).usage).resolves.toMatchObject({ promptTokens: 7, completionTokens: 3, totalTokens: 10, cachedPromptTokens: 5, }); }, 180_000); it("forwards streaming text chunks to the core onStreamChunk callback", async () => { aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { yield "hel"; yield "lo"; })(), text: Promise.resolve("hello"), toolCalls: Promise.resolve([]), finishReason: Promise.resolve("stop"), usage: Promise.resolve({ inputTokens: 2, outputTokens: 1 }), }); const onStreamChunk = vi.fn(); const { handleTextSmall } = await import("../models/text"); const stream = (await handleTextSmall(createRuntime(), { prompt: "stream", stream: true, onStreamChunk, } as never)) as { textStream: AsyncIterable }; const chunks: string[] = []; for await (const chunk of stream.textStream) { chunks.push(chunk); } expect(chunks).toEqual(["hel", "lo"]); expect(onStreamChunk).toHaveBeenNthCalledWith(1, "hel"); expect(onStreamChunk).toHaveBeenNthCalledWith(2, "lo"); }); it("emits usage and records the completed live-stream response after consumption", async () => { const trajectoryCalls: CapturedLlmCall[] = []; const toolCalls = [{ toolName: "lookup", input: { q: "x" } }]; aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { yield "hel"; yield "lo"; })(), text: Promise.resolve("hello"), toolCalls: Promise.resolve(toolCalls), finishReason: Promise.resolve("stop"), usage: Promise.resolve({ inputTokens: 2, outputTokens: 1, cachedInputTokens: 1 }), }); const runtime = createRuntime({ trajectoryCalls }); const { handleTextSmall } = await import("../models/text"); await runWithTrajectoryContext({ trajectoryStepId: "step-openai-stream" }, async () => { const stream = (await handleTextSmall(runtime, { prompt: "stream", stream: true, } as never)) as { textStream: AsyncIterable }; const chunks: string[] = []; for await (const chunk of stream.textStream) { chunks.push(chunk); } expect(chunks.join("")).toBe("hello"); }); expect(runtime.emitEvent).toHaveBeenCalledWith( EventType.MODEL_USED, expect.objectContaining({ source: "openai", provider: "openai", type: ModelType.TEXT_SMALL, prompt: "stream", tokens: { prompt: 2, completion: 1, total: 3, cached: 1 }, }) ); expect(trajectoryCalls).toHaveLength(1); expect(trajectoryCalls[0]).toMatchObject({ stepId: "step-openai-stream", actionType: "ai.streamText", response: "hello", promptTokens: 2, completionTokens: 1, finishReason: "stop", toolCalls, }); }); it("finalizes live-stream telemetry when the runtime breaks the stream loop early", async () => { const trajectoryCalls: CapturedLlmCall[] = []; aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { yield "first"; yield "second"; })(), text: Promise.resolve("firstsecond"), toolCalls: Promise.resolve([]), finishReason: Promise.resolve("stop"), usage: Promise.resolve({ inputTokens: 5, outputTokens: 2 }), }); const runtime = createRuntime({ trajectoryCalls }); const { handleTextSmall } = await import("../models/text"); await runWithTrajectoryContext({ trajectoryStepId: "step-openai-break" }, async () => { const stream = (await handleTextSmall(runtime, { prompt: "break stream", stream: true, } as never)) as { textStream: AsyncIterable }; for await (const chunk of stream.textStream) { expect(chunk).toBe("first"); break; } }); expect(runtime.emitEvent).toHaveBeenCalledWith( EventType.MODEL_USED, expect.objectContaining({ type: ModelType.TEXT_SMALL, prompt: "break stream", tokens: { prompt: 5, completion: 2, total: 7 }, }) ); expect(trajectoryCalls).toHaveLength(1); expect(trajectoryCalls[0]).toMatchObject({ stepId: "step-openai-break", actionType: "ai.streamText", response: "first", promptTokens: 5, completionTokens: 2, finishReason: "stop", }); }); it("surfaces live-stream provider errors reported through the AI SDK onError hook", async () => { const providerError = new Error("stream provider failed"); aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { yield "partial"; })(), text: Promise.resolve("partial"), toolCalls: Promise.resolve([]), finishReason: Promise.resolve("stop"), usage: Promise.resolve({ inputTokens: 1, outputTokens: 1 }), }); const { handleTextSmall } = await import("../models/text"); const stream = (await handleTextSmall(createRuntime(), { prompt: "stream error", stream: true, } as never)) as { textStream: AsyncIterable }; const call = aiMocks.streamText.mock.calls[0][0] as { onError?: (event: { error: unknown }) => void; }; call.onError?.({ error: providerError }); await expect(async () => { for await (const _chunk of stream.textStream) { // consume the stream so the deferred onError hook is checked } }).rejects.toThrow("stream provider failed"); }); it("maps string responseFormat json_object into the AI SDK responseFormat", async () => { aiMocks.generateText.mockResolvedValue({ text: "{}", finishReason: "stop", usage: { inputTokens: 3, outputTokens: 1 }, }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(createRuntime(), { prompt: "json", responseFormat: "json_object", } as never); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.responseFormat).toEqual({ type: "json" }); }); it("marks unconsumed streaming companion promises as handled", async () => { const noOutputError = Object.assign( new Error("No output generated. Check the stream for errors."), { name: "AI_NoOutputGeneratedError" } ); aiMocks.streamText.mockResolvedValue({ textStream: (async function* textStream() { // Empty stream: the runtime consumes this path and records an empty // response, while the AI SDK `text` promise rejects during flush. })(), text: Promise.reject(noOutputError), toolCalls: Promise.resolve([]), finishReason: Promise.resolve("stop"), usage: Promise.resolve({ inputTokens: 1, outputTokens: 0 }), }); const { handleTextSmall } = await import("../models/text"); const stream = (await handleTextSmall(createRuntime(), { prompt: "empty stream", stream: true, } as never)) as { textStream: AsyncIterable; text: Promise }; for await (const _chunk of stream.textStream) { // consume the primary stream path } await new Promise((resolve) => setTimeout(resolve, 0)); await expect(stream.text).rejects.toThrow("No output generated"); }); it("preserves Cerebras cache keys while stripping OpenAI-only cache retention", async () => { aiMocks.generateText.mockResolvedValue({ text: "ok", finishReason: "stop", usage: { inputTokens: 4, outputTokens: 1 }, }); const runtime = createRuntime(); vi.mocked(runtime.getSetting).mockImplementation((key: string) => { const settings: Record = { OPENAI_API_KEY: "test-key", OPENAI_BASE_URL: "https://api.cerebras.ai/v1", OPENAI_SMALL_MODEL: "gpt-oss-120b", }; return settings[key]; }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(runtime, { prompt: "cache", providerOptions: { openai: { promptCacheKey: "v5:abc", promptCacheRetention: "24h" }, cerebras: { promptCacheKey: "v5:abc", prompt_cache_key: "v5:abc" }, gateway: { caching: "auto" }, }, } as never); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.providerOptions).toEqual({ cerebras: { promptCacheKey: "v5:abc", prompt_cache_key: "v5:abc" }, gateway: { caching: "auto" }, // Cerebras mode defaults reasoningEffort to "low" (gpt-oss-120b returns // empty content when reasoning runs unbounded); see resolveReasoningEffort. openai: { promptCacheKey: "v5:abc", reasoningEffort: "low" }, }); }); it("defaults small and response handler models to gpt-5.4-mini while preserving explicit overrides", async () => { const { getResponseHandlerModel, getSmallModel } = await import("../utils/config"); const runtime = { getSetting: vi.fn(() => undefined), } as IAgentRuntime; expect(getSmallModel(runtime)).toBe("gpt-5.4-mini"); expect(getResponseHandlerModel(runtime)).toBe("gpt-5.4-mini"); const overrideRuntime = { getSetting: vi.fn((key: string) => { const settings: Record = { OPENAI_SMALL_MODEL: "custom-small", OPENAI_RESPONSE_HANDLER_MODEL: "custom-response", }; return settings[key]; }), } as IAgentRuntime; expect(getSmallModel(overrideRuntime)).toBe("custom-small"); expect(getResponseHandlerModel(overrideRuntime)).toBe("custom-response"); }); it("passes the effective system separately without duplicating the leading system message", async () => { aiMocks.generateText.mockResolvedValue({ text: "ok", finishReason: "stop", usage: { inputTokens: 4, outputTokens: 1 }, }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(createRuntime(), { prompt: "legacy prompt", messages: [ { role: "system", content: "system prompt" }, { role: "user", content: "hello" }, ], } as never); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.system).toBe("system prompt"); expect(call.messages).toEqual([{ role: "user", content: "hello" }]); }); it("normalizes core tool arrays and tool choice into AI SDK tool sets", async () => { aiMocks.generateText.mockResolvedValue({ text: "", toolCalls: [{ toolName: "WEB_SEARCH", input: { q: "eliza" } }], finishReason: "tool-calls", usage: { inputTokens: 11, outputTokens: 2 }, }); const { handleTextSmall } = await import("../models/text"); const coreTools = [ { name: "WEB_SEARCH", description: "Search the web", type: "function", strict: true, parameters: { properties: { q: { description: "Query", type: "string" }, }, required: ["q"], additionalProperties: false, }, }, ]; await handleTextSmall(createRuntime(), { prompt: "use native tool", messages: [{ role: "user", content: "search eliza" }], tools: coreTools, toolChoice: { type: "tool", name: "WEB_SEARCH" }, } as never); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.tools).not.toBe(coreTools); expect(Object.keys(call.tools as Record)).toEqual(["WEB_SEARCH"]); expect(call.toolChoice).toEqual({ type: "tool", toolName: "WEB_SEARCH" }); const webSearch = (call.tools as Record) .WEB_SEARCH; expect(webSearch.inputSchema.jsonSchema).toEqual({ type: "object", properties: { q: { description: "Query", type: "string" }, }, required: ["q"], additionalProperties: false, }); }, 60_000); it("restores strict-safe record/map tool-call args before returning native results", async () => { aiMocks.generateText.mockResolvedValue({ text: "", toolCalls: [ { toolName: "SAVE_CONTACT", input: { customFields: { __eliza_record_entries: [ { key: "favoriteColor", value: "blue" }, { key: "score", value: "7" }, ], }, }, }, ], finishReason: "tool-calls", usage: { inputTokens: 13, outputTokens: 4 }, }); const { handleTextSmall } = await import("../models/text"); const result = (await handleTextSmall(createRuntime(), { prompt: "save contact", messages: [{ role: "user", content: "save this" }], tools: [ { name: "SAVE_CONTACT", description: "Save contact", parameters: { type: "object", properties: { customFields: { type: "object", additionalProperties: true, }, }, required: ["customFields"], }, }, ], toolChoice: { type: "tool", name: "SAVE_CONTACT" }, } as never)) as { toolCalls: unknown[] }; expect(result.toolCalls).toEqual([ { toolName: "SAVE_CONTACT", input: { customFields: { favoriteColor: "blue", score: 7, }, }, }, ]); const call = aiMocks.generateText.mock.calls[0][0] as Record; const saveContact = (call.tools as Record) .SAVE_CONTACT; const schema = saveContact.inputSchema.jsonSchema as { properties: Record }>; }; expect(schema.properties.customFields.properties.__eliza_record_entries).toBeDefined(); }, 60_000); it("normalizes core assistant/tool history into AI SDK model messages", async () => { aiMocks.generateText.mockResolvedValue({ text: JSON.stringify({ decision: "FINISH", success: true }), finishReason: "stop", usage: { inputTokens: 17, outputTokens: 4 }, }); const { handleTextSmall } = await import("../models/text"); await handleTextSmall(createRuntime(), { prompt: "evaluate", messages: [ { role: "user", content: "search eliza" }, { role: "assistant", content: null, toolCalls: [ { id: "tool-1", type: "function", name: "WEB_SEARCH", arguments: JSON.stringify({ q: "eliza" }), }, ], }, { role: "tool", toolCallId: "tool-1", name: "WEB_SEARCH", content: JSON.stringify({ success: true, text: "found results" }), }, ], } as never); const call = aiMocks.generateText.mock.calls[0][0] as Record; expect(call.messages).toEqual([ { role: "user", content: "search eliza" }, { role: "assistant", content: [ { type: "tool-call", toolCallId: "tool-1", toolName: "WEB_SEARCH", input: { q: "eliza" }, }, ], }, { role: "tool", content: [ { type: "tool-result", toolCallId: "tool-1", toolName: "WEB_SEARCH", output: { type: "json", value: { success: true, text: "found results" } }, }, ], }, ]); }, 60_000); });