import json from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from mlflow.assistant.config import PermissionsConfig from mlflow.assistant.providers.base import clear_config_cache from mlflow.assistant.providers.openai_compatible import ( _MAX_SESSION_BYTES, OpenAICompatibleProvider, _build_usage_event, _merge_tool_call_chunk, _strip_think_blocks, _trim_session, ) from mlflow.assistant.providers.tool_executor import static_permission_error from mlflow.assistant.types import EventType from mlflow.tracing.constant import CostKey, TokenUsageKey # --------------------------------------------------------------------------- # aiohttp mock helpers # --------------------------------------------------------------------------- class _AsyncLineIter: def __init__(self, lines: list[bytes]): self._iter = iter(lines) def __aiter__(self): return self async def __anext__(self) -> bytes: try: return next(self._iter) except StopIteration: raise StopAsyncIteration def _make_aiohttp_session(response_lines_per_call: list[list[bytes]], status: int = 200): responses = [] captured_calls: list[dict[str, Any]] = [] for lines in response_lines_per_call: resp = MagicMock() resp.status = status resp.content = _AsyncLineIter(lines) resp.text = AsyncMock(return_value="") resp.__aenter__ = AsyncMock(return_value=resp) resp.__aexit__ = AsyncMock(return_value=False) responses.append(resp) call_count = 0 def _post(url, **kwargs): nonlocal call_count captured_calls.append({"url": url, **kwargs}) r = responses[call_count] call_count += 1 return r session = MagicMock() session.post = _post session.__aenter__ = AsyncMock(return_value=session) session.__aexit__ = AsyncMock(return_value=False) return session, captured_calls def _sse(payload: dict[str, Any]) -> bytes: return f"data: {json.dumps(payload)}\n".encode() def _delta( content: str = "", tool_calls: list[dict[str, Any]] | None = None, role: str = "assistant", ): delta: dict[str, Any] = {"role": role} if content: delta["content"] = content if tool_calls is not None: delta["tool_calls"] = tool_calls return {"choices": [{"delta": delta, "index": 0}]} # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- def _list_models_stub(*_args, **_kwargs): return ["model-a"] @pytest.fixture def provider(): return OpenAICompatibleProvider( name="oai_test", display_name="OAI Test", description="Test provider", list_models_fn=_list_models_stub, connection_hint="hint", default_base_url="http://localhost:9999", ) @pytest.fixture(autouse=True) def config_file(tmp_path): cfg = tmp_path / "config.json" cfg.write_text(json.dumps({"providers": {"oai_test": {"model": "model-a"}}})) clear_config_cache() with patch("mlflow.assistant.config.CONFIG_PATH", cfg): yield cfg clear_config_cache() # --------------------------------------------------------------------------- # Pure helpers # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("buf", "in_think", "expected_emit", "expected_remaining", "expected_in_think"), [ ("hello world", False, "hello world", "", False), ("foosecretbar", False, "foobar", "", False), ("partial", False, "", "", True), ("rest of thoughtafter", True, "after", "", False), ("plain", True, "", "", True), # Partial-tag-at-tail: a chunk ending with a prefix of "" # must not leak that prefix to the user — it must be held back as # the remainder so the next chunk can complete the tag. ("foo" start. That's the safe # default: hold one char, emit it next round if it doesn't grow. ], ) def test_strip_think_blocks(buf, in_think, expected_emit, expected_remaining, expected_in_think): emit, remaining, new_in_think = _strip_think_blocks(buf, in_think) assert emit == expected_emit assert remaining == expected_remaining assert new_in_think is expected_in_think def test_strip_think_blocks_completes_partial_tag_across_chunks(): """Reproduces the SSE-frame split that previously leaked `` to the user. Frame 1 ends mid-opening-tag; frame 2 supplies the rest of the tag plus the secret content and the closing tag. The combined behavior must emit nothing user-visible (only "foo"). """ emit1, remaining1, in_think1 = _strip_think_blocks("foosecret", in_think1) assert emit2 == "" assert remaining2 == "" assert in_think2 is False def test_merge_tool_call_chunk_accumulates_arguments(): acc: list[dict[str, Any]] = [] _merge_tool_call_chunk( acc, {"index": 0, "id": "call_1", "function": {"name": "Bash", "arguments": '{"comm'}}, ) _merge_tool_call_chunk(acc, {"index": 0, "function": {"arguments": 'and": "ls"}'}}) assert acc == [{"id": "call_1", "function": {"name": "Bash", "arguments": '{"command": "ls"}'}}] def test_merge_tool_call_chunk_supports_multiple_calls(): acc: list[dict[str, Any]] = [] _merge_tool_call_chunk( acc, {"index": 0, "id": "a", "function": {"name": "X", "arguments": "{}"}} ) _merge_tool_call_chunk( acc, {"index": 1, "id": "b", "function": {"name": "Y", "arguments": "{}"}} ) assert len(acc) == 2 assert acc[0]["id"] == "a" assert acc[1]["id"] == "b" def test_trim_session_drops_oldest_keeping_system(): big = "x" * (_MAX_SESSION_BYTES // 3) messages = [ {"role": "system", "content": "sys"}, {"role": "user", "content": f"old-{big}"}, {"role": "assistant", "content": f"middle-{big}"}, {"role": "user", "content": f"new-{big}"}, ] trimmed = _trim_session(messages) assert trimmed[0]["role"] == "system" assert trimmed[-1]["content"].startswith("new-") assert not any(m["content"].startswith("old-") for m in trimmed[1:]) def test_build_usage_event_remaps_cache_tokens_and_prices(): usage = { "prompt_tokens": 35257, "completion_tokens": 5, "total_tokens": 35262, "prompt_tokens_details": {"cached_tokens": 100}, "cache_creation_input_tokens": 35155, } with patch( "mlflow.assistant.providers.openai_compatible.calculate_cost_by_model_and_token_usage", return_value={CostKey.TOTAL_COST: 0.1319}, ) as mock_cost: event = _build_usage_event(usage, "claude-3-5-sonnet") mock_cost.assert_called_once_with( "claude-3-5-sonnet", { TokenUsageKey.INPUT_TOKENS: 35257, TokenUsageKey.OUTPUT_TOKENS: 5, TokenUsageKey.CACHE_READ_INPUT_TOKENS: 100, TokenUsageKey.CACHE_CREATION_INPUT_TOKENS: 35155, }, ) assert event.type == EventType.STREAM_EVENT assert event.data["event"]["usage"] == { "prompt_tokens": 35257, "completion_tokens": 5, "total_tokens": 35262, "total_cost_usd": 0.1319, } def test_build_usage_event_cost_none_when_model_not_priced(): usage = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} with patch( "mlflow.assistant.providers.openai_compatible.calculate_cost_by_model_and_token_usage", return_value=None, ) as mock_cost: event = _build_usage_event(usage, "local-ollama-model") mock_cost.assert_called_once_with( "local-ollama-model", {TokenUsageKey.INPUT_TOKENS: 10, TokenUsageKey.OUTPUT_TOKENS: 5}, ) assert event.data["event"]["usage"]["total_cost_usd"] is None # --------------------------------------------------------------------------- # astream — basic streaming # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_astream_emits_content_deltas(provider): lines = [ _sse(_delta(content="Hello")), _sse(_delta(content=" world")), b"data: [DONE]\n", ] session, calls = _make_aiohttp_session([lines]) with patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ): events = [e async for e in provider.astream("hi", "http://localhost:5000")] stream_events = [e for e in events if e.type == EventType.STREAM_EVENT] assert [e.data["event"]["delta"]["text"] for e in stream_events] == ["Hello", " world"] assert any(e.type == EventType.DONE for e in events) assert calls[0]["url"] == "http://localhost:9999/v1/chat/completions" assert calls[0]["headers"] == {} @pytest.mark.asyncio async def test_astream_requests_usage_via_stream_options(provider): lines = [_sse(_delta(content="hi")), b"data: [DONE]\n"] session, calls = _make_aiohttp_session([lines]) with patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ): _ = [e async for e in provider.astream("hi", "http://localhost:5000")] assert calls[0]["json"]["stream_options"] == {"include_usage": True} @pytest.mark.asyncio async def test_astream_tolerates_done_terminator_and_blank_lines(provider): lines = [ b"\n", _sse(_delta(content="A")), b":heartbeat\n", _sse(_delta(content="B")), b"data: [DONE]\n", ] session, _calls = _make_aiohttp_session([lines]) with patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ): events = [e async for e in provider.astream("hi", "http://localhost:5000")] deltas = [e.data["event"]["delta"]["text"] for e in events if e.type == EventType.STREAM_EVENT] assert deltas == ["A", "B"] @pytest.mark.asyncio async def test_astream_strips_think_blocks_from_stream(provider): lines = [ _sse(_delta(content="ans:")), _sse(_delta(content="internal")), _sse(_delta(content=" reasoningreal")), _sse(_delta(content=" answer")), b"data: [DONE]\n", ] session, _calls = _make_aiohttp_session([lines]) with patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ): events = [e async for e in provider.astream("hi", "http://localhost:5000")] visible = "".join( e.data["event"]["delta"]["text"] for e in events if e.type == EventType.STREAM_EVENT ) assert "internal" not in visible assert "reasoning" not in visible assert "ans:real answer" == visible @pytest.mark.asyncio async def test_astream_uses_api_key_header(tmp_path): cfg = tmp_path / "config.json" cfg.write_text( json.dumps({ "providers": { "oai_test": { "model": "model-a", "base_url": "http://gateway.example", "api_key": "sk-abc", } } }) ) clear_config_cache() provider = OpenAICompatibleProvider( name="oai_test", display_name="OAI", description="d", list_models_fn=_list_models_stub, connection_hint="h", ) lines = [_sse(_delta(content="ok")), b"data: [DONE]\n"] session, calls = _make_aiohttp_session([lines]) with ( patch("mlflow.assistant.config.CONFIG_PATH", cfg), patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), ): _ = [e async for e in provider.astream("hi", "http://localhost:5000")] assert calls[0]["url"] == "http://gateway.example/v1/chat/completions" assert calls[0]["headers"] == {"Authorization": "Bearer sk-abc"} clear_config_cache() @pytest.mark.asyncio async def test_astream_uses_tracking_uri_via_custom_chat_url_builder(tmp_path): cfg = tmp_path / "config.json" cfg.write_text(json.dumps({"providers": {"gw_test": {"model": "ep-1"}}})) clear_config_cache() def chat_url_builder(_base_url, tracking_uri): return f"{tracking_uri.rstrip('/')}/gateway/mlflow/v1/chat/completions" provider = OpenAICompatibleProvider( name="gw_test", display_name="Gateway", description="d", connection_hint="h", chat_url_builder=chat_url_builder, ) lines = [_sse(_delta(content="ok")), b"data: [DONE]\n"] session, calls = _make_aiohttp_session([lines]) with ( patch("mlflow.assistant.config.CONFIG_PATH", cfg), patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), ): _ = [e async for e in provider.astream("hi", "http://mlflow.server:5000")] assert calls[0]["url"] == "http://mlflow.server:5000/gateway/mlflow/v1/chat/completions" clear_config_cache() def test_list_models_raises_not_implemented_when_no_fn(): provider = OpenAICompatibleProvider( name="gw_test2", display_name="Gateway", description="d", connection_hint="h", ) with pytest.raises(NotImplementedError, match="Model listing is not supported"): provider.list_models() @pytest.mark.asyncio async def test_astream_yields_error_on_http_error(provider): session, _calls = _make_aiohttp_session([[b""]], status=500) # Wrap the failing response so .text() returns the error body. bad_resp = MagicMock() bad_resp.status = 500 bad_resp.text = AsyncMock(return_value="boom") bad_resp.content = _AsyncLineIter([]) bad_resp.__aenter__ = AsyncMock(return_value=bad_resp) bad_resp.__aexit__ = AsyncMock(return_value=False) session.post = lambda url, **kw: bad_resp with patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ): events = [e async for e in provider.astream("hi", "http://localhost:5000")] errors = [e for e in events if e.type == EventType.ERROR] assert len(errors) == 1 assert "boom" in errors[0].data["error"] # --------------------------------------------------------------------------- # astream — tool call round trip # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_astream_tool_call_round_trip(provider): # Turn 1: streamed tool call with chunked arguments. lines_turn1 = [ _sse( _delta( tool_calls=[ { "index": 0, "id": "call_1", "function": {"name": "Bash", "arguments": '{"comm'}, } ] ) ), _sse(_delta(tool_calls=[{"index": 0, "function": {"arguments": 'and": "ls"}'}}])), b"data: [DONE]\n", ] lines_turn2 = [_sse(_delta(content="Done")), b"data: [DONE]\n"] session, calls = _make_aiohttp_session([lines_turn1, lines_turn2]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mock_tool, ): events = [e async for e in provider.astream("ls", "http://localhost:5000")] mock_tool.assert_awaited_once() args, kwargs = mock_tool.await_args assert args[0] == "Bash" assert args[1] == {"command": "ls"} tool_use_events = [ e for e in events if e.type == EventType.MESSAGE and isinstance(e.data["message"]["content"], list) and e.data["message"]["content"][0].get("name") == "Bash" ] assert len(tool_use_events) == 1 stream_events = [e for e in events if e.type == EventType.STREAM_EVENT] assert any(ev.data["event"]["delta"]["text"] == "Done" for ev in stream_events) # Second request should include the tool message in history. second_payload = calls[1]["json"] assert any(m["role"] == "tool" for m in second_payload["messages"]) # --------------------------------------------------------------------------- # astream — session-scoped permission gating # --------------------------------------------------------------------------- _SESSION_ID = "11111111-1111-1111-1111-111111111111" def _tool_call_turns(): turn1 = [ _sse( _delta( tool_calls=[ { "index": 0, "id": "call_1", "function": {"name": "Bash", "arguments": '{"command": "ls"}'}, } ] ) ), b"data: [DONE]\n", ] turn2 = [_sse(_delta(content="Done")), b"data: [DONE]\n"] return [turn1, turn2] def _done_session_id(events) -> str: for e in reversed(events): if e.type == EventType.DONE: return e.data["session_id"] raise AssertionError("no DONE event found") @pytest.mark.asyncio async def test_astream_pauses_at_permission_without_executing(provider): # Full access off + a session: the turn must END at the prompt (no in-process # await), emitting PERMISSION_REQUEST then DONE, with the tool unexecuted and # the pending tool_call persisted in the returned history. session, _calls = _make_aiohttp_session([_tool_call_turns()[0]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mock_tool, ): events = [ e async for e in provider.astream( "ls", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] mock_tool.assert_not_awaited() prompts = [e for e in events if e.type == EventType.PERMISSION_REQUEST] assert len(prompts) == 1 assert prompts[0].data["request_id"] == "call_1" assert prompts[0].data["tool_name"] == "Bash" assert prompts[0].data["tool_input"] == {"command": "ls"} assert events[-1].type == EventType.DONE history = json.loads(_done_session_id(events)) assert history[-1]["role"] == "assistant" assert history[-1].get("tool_calls") assert not any(m.get("role") == "tool" for m in history) @pytest.mark.asyncio async def test_astream_resume_allow_executes_and_continues(provider): # Pause to capture the persisted history. s1, _ = _make_aiohttp_session([_tool_call_turns()[0]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s1, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("x", False)), ) as mt1, ): ev1 = [ e async for e in provider.astream( "ls", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] mt1.assert_not_awaited() history = _done_session_id(ev1) # Resume with allow: the decision is delivered via context, no new user turn. s2, _ = _make_aiohttp_session([_tool_call_turns()[1]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s2, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mt2, ): ev2 = [ e async for e in provider.astream( "", "http://localhost:5000", mlflow_session_id=_SESSION_ID, session_id=history, context={"tool_decisions": {"call_1": "allow"}}, ) ] mt2.assert_awaited_once() # An explicit allow overrides the static allowlist for this call. assert mt2.await_args.kwargs["permissions"].full_access is True assert not any(e.type == EventType.PERMISSION_REQUEST for e in ev2) assert any( e.type == EventType.STREAM_EVENT and e.data["event"]["delta"]["text"] == "Done" for e in ev2 ) @pytest.mark.asyncio async def test_astream_resume_deny_skips_execution(provider): s1, _ = _make_aiohttp_session([_tool_call_turns()[0]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s1, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("x", False)), ), ): ev1 = [ e async for e in provider.astream( "ls", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] history = _done_session_id(ev1) s2, _ = _make_aiohttp_session([_tool_call_turns()[1]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s2, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mt2, ): ev2 = [ e async for e in provider.astream( "", "http://localhost:5000", mlflow_session_id=_SESSION_ID, session_id=history, context={"tool_decisions": {"call_1": "deny"}}, ) ] mt2.assert_not_awaited() denied = [ e for e in ev2 if e.type == EventType.MESSAGE and isinstance(e.data["message"]["content"], list) and e.data["message"]["content"][0].get("content") == "Permission denied by user." ] assert len(denied) == 1 assert any( e.type == EventType.STREAM_EVENT and e.data["event"]["delta"]["text"] == "Done" for e in ev2 ) @pytest.mark.asyncio async def test_astream_fresh_message_after_abandoned_tool_call(provider): # A turn paused at a prompt, then cancelled (a no-op for this provider, so the # unresolved tool_call stays in history). A NEW user message must start a fresh # turn — NOT silently re-resume the abandoned call and drop the message. s1, _ = _make_aiohttp_session([_tool_call_turns()[0]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s1, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("x", False)), ), ): ev1 = [ e async for e in provider.astream( "ls", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] history = _done_session_id(ev1) # New message, NO tool_decisions: the abandoned call must be closed out and the # new message must reach the model (turn 2 returns plain text, no tool calls). s2, _ = _make_aiohttp_session([_tool_call_turns()[1]]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=s2, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mt2, ): ev2 = [ e async for e in provider.astream( "what is 2+2", "http://localhost:5000", mlflow_session_id=_SESSION_ID, session_id=history, ) ] # No duplicate prompt for the old call, and the old call is never executed. assert not any(e.type == EventType.PERMISSION_REQUEST for e in ev2) mt2.assert_not_awaited() # The stream completes with the model's reply to the NEW message. assert any( e.type == EventType.STREAM_EVENT and e.data["event"]["delta"]["text"] == "Done" for e in ev2 ) # History: the orphaned call is closed with a cancellation result, and the new # user message is present. final = json.loads(_done_session_id(ev2)) assert any( m.get("role") == "tool" and m.get("tool_call_id") == "call_1" and m.get("content") == "Tool call cancelled by user." for m in final ) assert any(m.get("role") == "user" and m.get("content") == "what is 2+2" for m in final) @pytest.mark.asyncio async def test_astream_global_full_access_skips_prompt(tmp_path): # When full access is enabled in the global config, no per-call prompt fires # even for a session — this preserves the pre-existing "run freely" setting. cfg = tmp_path / "config.json" cfg.write_text( json.dumps({ "providers": {"oai_test": {"model": "model-a", "permissions": {"full_access": True}}} }) ) clear_config_cache() provider = OpenAICompatibleProvider( name="oai_test", display_name="OAI Test", description="d", list_models_fn=_list_models_stub, connection_hint="h", default_base_url="http://localhost:9999", ) session, _calls = _make_aiohttp_session(_tool_call_turns()) with ( patch("mlflow.assistant.config.CONFIG_PATH", cfg), patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("file1.py\n", False)), ) as mock_tool, ): events = [ e async for e in provider.astream( "ls", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] clear_config_cache() assert not any(e.type == EventType.PERMISSION_REQUEST for e in events) mock_tool.assert_awaited_once() @pytest.mark.parametrize( ("tool_name", "tool_input", "allowed"), [ ("Bash", {"command": "mlflow experiments search"}, True), ("Bash", {"command": "python script.py"}, True), ("Bash", {"command": "rm -rf /"}, False), ("Bash", {"command": "ls"}, False), ], ) def test_static_permission_error_bash_allowlist(tool_name, tool_input, allowed): err = static_permission_error(tool_name, tool_input, PermissionsConfig(full_access=False), None) assert (err is None) == allowed def test_static_permission_error_full_access_allows_everything(): assert ( static_permission_error( "Bash", {"command": "rm -rf /"}, PermissionsConfig(full_access=True), None ) is None ) @pytest.mark.asyncio async def test_astream_allowlisted_command_runs_without_prompt(provider): # Regression guard for #24084: an `mlflow` CLI command is on the static allowlist, so even with # full access off and a session present it must run WITHOUT a per-call permission prompt. turn1 = [ _sse( _delta( tool_calls=[ { "index": 0, "id": "call_1", "function": { "name": "Bash", "arguments": '{"command": "mlflow experiments search"}', }, } ] ) ), b"data: [DONE]\n", ] turn2 = [_sse(_delta(content="Done")), b"data: [DONE]\n"] session, _ = _make_aiohttp_session([turn1, turn2]) with ( patch( "mlflow.assistant.providers.openai_compatible.aiohttp.ClientSession", return_value=session, ), patch( "mlflow.assistant.providers.openai_compatible.execute_tool", AsyncMock(return_value=("ok", False)), ) as mock_tool, ): events = [ e async for e in provider.astream( "go", "http://localhost:5000", mlflow_session_id=_SESSION_ID ) ] assert not any(e.type == EventType.PERMISSION_REQUEST for e in events) mock_tool.assert_awaited_once() assert events[-1].type == EventType.DONE