/tests/contracts/test_smoke_config.py

from pathlib import Path
from types import SimpleNamespace

from smoke.conftest import (
    DISABLED_PROVIDER_MODEL,
    provider_model_params,
    provider_xdist_group,
)
from smoke.lib.config import (
    ALL_TARGETS,
    DEFAULT_TARGETS,
    MISTRAL_REASONING_SMOKE_DEFAULT_MODEL,
    NVIDIA_NIM_CLI_DEFAULT_MODELS,
    OPENROUTER_FREE_CLI_DEFAULT_MODELS,
    OPT_IN_TARGETS,
    PROVIDER_SMOKE_DEFAULT_MODELS,
    TARGET_REQUIRED_ENV,
    SmokeConfig,
    nvidia_nim_cli_model_refs,
    openrouter_free_cli_model_refs,
)


def _settings(**overrides):
    values = {
        "model": "ollama/llama3.1",
        "model_fable": None,
        "model_opus": None,
        "model_sonnet": None,
        "model_haiku": None,
        "nvidia_nim_api_key": "",
        "open_router_api_key": "",
        "mistral_api_key": "",
        "codestral_api_key": "",
        "deepseek_api_key": "",
        "kimi_api_key": "",
        "kimi_code_api_key": "",
        "wafer_api_key": "",
        "minimax_api_key": "",
        "opencode_api_key": "",
        "vercel_ai_gateway_api_key": "",
        "bedrock_api_key": "",
        "bedrock_base_url": "https://bedrock-mantle.us-east-1.api.aws/v1",
        "huggingface_api_key": "",
        "cohere_api_key": "",
        "github_models_token": "",
        "zai_api_key": "",
        "gemini_api_key": "",
        "vertex_project_id": "",
        "vertex_location": "global",
        "groq_api_key": "",
        "sambanova_api_key": "",
        "cerebras_api_key": "",
        "ollama_api_key": "",
        "fireworks_api_key": "",
        "cloudflare_api_token": "",
        "cloudflare_account_id": "",
        "lm_studio_base_url": "",
        "llamacpp_base_url": "",
        "ollama_base_url": "http://localhost:11434",
    }
    values.update(overrides)
    return SimpleNamespace(**values)


def _smoke_config(**overrides) -> SmokeConfig:
    values = {
        "root": Path("."),
        "results_dir": Path(".smoke-results"),
        "live": False,
        "interactive": False,
        "targets": DEFAULT_TARGETS,
        "provider_matrix": frozenset(),
        "timeout_s": 45.0,
        "prompt": "Reply with exactly: FCC_SMOKE_PONG",
        "claude_bin": "claude",
        "worker_id": "main",
        "settings": _settings(),
    }
    values.update(overrides)
    return SmokeConfig(**values)


def test_ollama_is_default_smoke_target() -> None:
    assert "ollama" in DEFAULT_TARGETS
    assert "ollama" in TARGET_REQUIRED_ENV


def test_nvidia_nim_cli_is_opt_in_smoke_target() -> None:
    assert "nvidia_nim_cli" not in DEFAULT_TARGETS
    assert "nvidia_nim_cli" in OPT_IN_TARGETS
    assert "nvidia_nim_cli" in ALL_TARGETS
    assert "nvidia_nim_cli" in TARGET_REQUIRED_ENV
    assert "openrouter_free_cli" not in DEFAULT_TARGETS
    assert "openrouter_free_cli" in OPT_IN_TARGETS
    assert "openrouter_free_cli" in ALL_TARGETS
    assert "openrouter_free_cli" in TARGET_REQUIRED_ENV


def test_ollama_provider_configuration_uses_base_url() -> None:
    config = _smoke_config()

    assert config.has_provider_configuration("ollama")
    assert config.provider_models()[0].full_model == "ollama/llama3.1"


def test_ollama_provider_matrix_filters_models() -> None:
    config = _smoke_config(provider_matrix=frozenset({"ollama"}))

    assert [model.provider for model in config.provider_models()] == ["ollama"]


def test_ollama_cloud_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_OLLAMA_CLOUD", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            ollama_api_key="ollama-cloud-key",
        )
    )

    assert config.has_provider_configuration("ollama_cloud")
    models = config.provider_smoke_models()
    assert [model.provider for model in models] == ["ollama_cloud"]
    assert models[0].full_model == "ollama_cloud/qwen3-coder:480b"
    assert models[0].source == "provider_default"


def test_provider_smoke_models_cover_configured_providers_independent_of_model_mapping(
    monkeypatch,
) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            deepseek_api_key="deepseek-key",
            ollama_base_url="",
        )
    )

    models = config.provider_smoke_models()

    assert [model.provider for model in models] == ["deepseek"]
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["deepseek"]
    assert models[0].source == "provider_default"


def test_openrouter_provider_smoke_uses_concrete_free_model(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_OPEN_ROUTER", raising=False)
    config = _smoke_config(
        settings=_settings(open_router_api_key="openrouter-key", ollama_base_url="")
    )

    models = config.provider_smoke_models()

    assert [model.provider for model in models] == ["open_router"]
    assert models[0].full_model == "open_router/moonshotai/kimi-k2.6:free"
    assert models[0].source == "provider_default"


def test_bedrock_provider_configuration_uses_official_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_BEDROCK", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            bedrock_api_key="bedrock-key",
        )
    )

    assert config.has_provider_configuration("bedrock")
    models = config.provider_smoke_models()
    assert [model.provider for model in models] == ["bedrock"]
    assert models[0].full_model == "bedrock/openai.gpt-oss-120b"
    assert models[0].source == "provider_default"


def test_vertex_provider_configuration_uses_project_id(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_VERTEX", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            vertex_project_id="vertex-project",
        )
    )

    assert config.has_provider_configuration("vertex")
    models = config.provider_smoke_models()
    assert [model.provider for model in models] == ["vertex"]
    assert models[0].full_model == "vertex/google/gemini-3.5-flash"
    assert models[0].source == "provider_default"


def test_wafer_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_WAFER", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            wafer_api_key="wafer-key",
        )
    )

    assert config.has_provider_configuration("wafer")
    models = config.provider_smoke_models()
    assert models[0].provider == "wafer"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["wafer"]


def test_kimi_code_provider_configuration_uses_subscription_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_KIMI_CODE", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            kimi_code_api_key="subscription-key",
        )
    )

    assert config.has_provider_configuration("kimi_code")
    models = config.provider_smoke_models()
    assert [model.provider for model in models] == ["kimi_code"]
    assert models[0].full_model == "kimi_code/k3"
    assert models[0].source == "provider_default"


def test_minimax_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_MINIMAX", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            minimax_api_key="minimax-key",
        )
    )

    assert config.has_provider_configuration("minimax")
    models = config.provider_smoke_models()
    assert models[0].provider == "minimax"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["minimax"]


def test_cloudflare_provider_configuration_requires_token_and_account(
    monkeypatch,
) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_CLOUDFLARE", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            cloudflare_api_token="cf-token",
            cloudflare_account_id="cf-account",
        )
    )

    assert config.has_provider_configuration("cloudflare")
    models = config.provider_smoke_models()
    assert models[0].provider == "cloudflare"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["cloudflare"]


def test_cloudflare_provider_configuration_missing_account_is_unconfigured() -> None:
    config = _smoke_config(
        settings=_settings(
            ollama_base_url="",
            cloudflare_api_token="cf-token",
            cloudflare_account_id="",
        )
    )

    assert not config.has_provider_configuration("cloudflare")


def test_vercel_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_VERCEL", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            vercel_ai_gateway_api_key="vercel-key",
        )
    )

    assert config.has_provider_configuration("vercel")
    models = config.provider_smoke_models()
    assert models[0].provider == "vercel"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["vercel"]


def test_huggingface_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_HUGGINGFACE", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            huggingface_api_key="hf-key",
        )
    )

    assert config.has_provider_configuration("huggingface")
    models = config.provider_smoke_models()
    assert models[0].provider == "huggingface"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["huggingface"]


def test_cohere_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_COHERE", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            cohere_api_key="cohere-key",
        )
    )

    assert config.has_provider_configuration("cohere")
    models = config.provider_smoke_models()
    assert models[0].provider == "cohere"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["cohere"]


def test_github_models_provider_configuration_uses_token(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_GITHUB_MODELS", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            github_models_token="github-token",
        )
    )

    assert config.has_provider_configuration("github_models")
    models = config.provider_smoke_models()
    assert models[0].provider == "github_models"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["github_models"]


def test_sambanova_provider_configuration_uses_api_key(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_SAMBANOVA", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="ollama/llama3.1",
            ollama_base_url="",
            sambanova_api_key="sambanova-key",
        )
    )

    assert config.has_provider_configuration("sambanova")
    models = config.provider_smoke_models()
    assert models[0].provider == "sambanova"
    assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["sambanova"]


def test_provider_smoke_model_override_accepts_model_name_without_prefix(
    monkeypatch,
) -> None:
    monkeypatch.setenv("FCC_SMOKE_MODEL_DEEPSEEK", "deepseek-reasoner")
    config = _smoke_config(
        settings=_settings(
            deepseek_api_key="deepseek-key",
            ollama_base_url="",
        ),
        provider_matrix=frozenset({"deepseek"}),
    )

    models = config.provider_smoke_models()

    assert models[0].full_model == "deepseek/deepseek-reasoner"
    assert models[0].source == "FCC_SMOKE_MODEL_DEEPSEEK"


def test_provider_smoke_model_override_accepts_owner_model_name(
    monkeypatch,
) -> None:
    monkeypatch.setenv(
        "FCC_SMOKE_MODEL_NVIDIA_NIM", "nvidia/nemotron-3-super-120b-a12b"
    )
    config = _smoke_config(
        settings=_settings(
            model="deepseek/deepseek-chat",
            deepseek_api_key="",
            nvidia_nim_api_key="nim-key",
            ollama_base_url="",
        ),
        provider_matrix=frozenset({"nvidia_nim"}),
    )

    models = config.provider_smoke_models()

    assert models[0].full_model == "nvidia_nim/nvidia/nemotron-3-super-120b-a12b"
    assert models[0].source == "FCC_SMOKE_MODEL_NVIDIA_NIM"


def test_provider_smoke_model_override_rejects_wrong_provider_prefix(
    monkeypatch,
) -> None:
    monkeypatch.setenv("FCC_SMOKE_MODEL_DEEPSEEK", "ollama/llama3.1")
    config = _smoke_config(
        settings=_settings(
            deepseek_api_key="deepseek-key",
            ollama_base_url="",
        ),
        provider_matrix=frozenset({"deepseek"}),
    )

    try:
        config.provider_smoke_models()
    except ValueError as exc:
        assert "FCC_SMOKE_MODEL_DEEPSEEK" in str(exc)
    else:
        raise AssertionError("expected wrong provider prefix to fail")


def test_mistral_reasoning_smoke_uses_reasoning_default(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_MISTRAL_REASONING", raising=False)
    config = _smoke_config(
        settings=_settings(mistral_api_key="mistral-key", ollama_base_url="")
    )

    model = config.mistral_reasoning_smoke_model()

    assert model is not None
    assert model.provider == "mistral"
    assert model.full_model == MISTRAL_REASONING_SMOKE_DEFAULT_MODEL
    assert model.source == "mistral_reasoning_default"


def test_mistral_reasoning_smoke_accepts_override(monkeypatch) -> None:
    monkeypatch.setenv("FCC_SMOKE_MODEL_MISTRAL_REASONING", "mistral-medium-3-5")
    config = _smoke_config(
        settings=_settings(mistral_api_key="mistral-key", ollama_base_url="")
    )

    model = config.mistral_reasoning_smoke_model()

    assert model is not None
    assert model.full_model == "mistral/mistral-medium-3-5"
    assert model.source == "FCC_SMOKE_MODEL_MISTRAL_REASONING"


def test_mistral_reasoning_smoke_respects_provider_matrix(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_MISTRAL_REASONING", raising=False)
    config = _smoke_config(
        settings=_settings(mistral_api_key="mistral-key", ollama_base_url=""),
        provider_matrix=frozenset({"deepseek"}),
    )

    assert config.mistral_reasoning_smoke_model() is None


def test_provider_smoke_matrix_filters_provider_catalog(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
    config = _smoke_config(
        settings=_settings(
            deepseek_api_key="deepseek-key",
            nvidia_nim_api_key="nim-key",
            ollama_base_url="",
        ),
        provider_matrix=frozenset({"nvidia_nim"}),
    )

    assert [model.provider for model in config.provider_smoke_models()] == [
        "nvidia_nim"
    ]


def test_provider_smoke_collection_params_are_grouped_by_provider(
    monkeypatch,
) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
    monkeypatch.delenv("FCC_SMOKE_MODEL_NVIDIA_NIM", raising=False)
    config = _smoke_config(
        live=True,
        settings=_settings(
            deepseek_api_key="deepseek-key",
            nvidia_nim_api_key="nim-key",
            ollama_base_url="",
        ),
    )

    params = provider_model_params(config)

    assert [param.id for param in params] == ["nvidia_nim", "deepseek"]
    groups = [
        mark.args[0]
        for param in params
        for mark in param.marks
        if mark.name == "xdist_group"
    ]
    assert groups == ["provider:nvidia_nim", "provider:deepseek"]


def test_provider_smoke_collection_uses_disabled_placeholder_when_not_live() -> None:
    config = _smoke_config(live=False, settings=_settings(ollama_base_url=""))

    params = provider_model_params(config)

    assert [param.values[0] for param in params] == [DISABLED_PROVIDER_MODEL]
    assert provider_xdist_group(DISABLED_PROVIDER_MODEL) == "provider:smoke_disabled"


def test_provider_smoke_includes_local_provider_when_model_mapping_uses_it(
    monkeypatch,
) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_OLLAMA", raising=False)
    config = _smoke_config()

    assert [model.provider for model in config.provider_smoke_models()] == ["ollama"]


def test_provider_smoke_does_not_include_default_local_urls_when_unmapped(
    monkeypatch,
) -> None:
    monkeypatch.delenv("FCC_SMOKE_MODEL_OLLAMA", raising=False)
    config = _smoke_config(settings=_settings(model="nvidia_nim/test"))

    assert config.provider_smoke_models() == []


def test_nvidia_nim_cli_default_models_are_normalized() -> None:
    refs = nvidia_nim_cli_model_refs({})

    assert tuple(refs) == tuple(
        f"nvidia_nim/{model}" for model in NVIDIA_NIM_CLI_DEFAULT_MODELS
    )
    assert "nvidia_nim/deepseek-ai/deepseek-v4-pro" in refs
    assert "nvidia_nim/deepseek-ai/deepseek-v4-flash" in refs
    assert set(refs.values()) == {"nvidia_nim_cli_default"}


def test_nvidia_nim_cli_models_override_and_append() -> None:
    refs = nvidia_nim_cli_model_refs(
        {
            "FCC_SMOKE_NIM_MODELS": "z-ai/glm-5.2,nvidia_nim/custom/model",
            "FCC_SMOKE_NIM_EXTRA_MODELS": "moonshotai/kimi-k2.6,z-ai/glm-5.2",
        }
    )

    assert tuple(refs) == (
        "nvidia_nim/z-ai/glm-5.2",
        "nvidia_nim/custom/model",
        "nvidia_nim/moonshotai/kimi-k2.6",
    )
    assert refs["nvidia_nim/z-ai/glm-5.2"] == "FCC_SMOKE_NIM_MODELS"
    assert refs["nvidia_nim/moonshotai/kimi-k2.6"] == ("FCC_SMOKE_NIM_EXTRA_MODELS")


def test_nvidia_nim_cli_models_reject_empty_override() -> None:
    try:
        nvidia_nim_cli_model_refs({"FCC_SMOKE_NIM_MODELS": " , "})
    except ValueError as exc:
        assert "FCC_SMOKE_NIM_MODELS" in str(exc)
    else:
        raise AssertionError("expected empty NVIDIA NIM CLI model override to fail")


def test_nvidia_nim_cli_models_reject_wrong_provider_prefix() -> None:
    try:
        nvidia_nim_cli_model_refs({"FCC_SMOKE_NIM_MODELS": "open_router/model"})
    except ValueError as exc:
        assert "nvidia_nim" in str(exc)
    else:
        raise AssertionError("expected wrong provider prefix to fail")


def test_smoke_config_returns_nvidia_nim_cli_provider_models(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_NIM_MODELS", raising=False)
    monkeypatch.delenv("FCC_SMOKE_NIM_EXTRA_MODELS", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="nvidia_nim/z-ai/glm-5.2",
            nvidia_nim_api_key="nim-key",
            ollama_base_url="",
        )
    )

    models = config.nvidia_nim_cli_models()

    assert models[0].provider == "nvidia_nim"
    assert models[0].full_model == "nvidia_nim/z-ai/glm-5.2"
    assert models[0].source == "nvidia_nim_cli_default"


def test_openrouter_free_cli_default_models_are_normalized() -> None:
    refs = openrouter_free_cli_model_refs({})

    assert tuple(refs) == tuple(
        f"open_router/{model}" for model in OPENROUTER_FREE_CLI_DEFAULT_MODELS
    )
    assert "open_router/nvidia/nemotron-3-super-120b-a12b:free" in refs
    assert "open_router/poolside/laguna-m.1:free" in refs
    assert set(refs.values()) == {"openrouter_free_cli_default"}


def test_openrouter_free_cli_models_override_and_append() -> None:
    refs = openrouter_free_cli_model_refs(
        {
            "FCC_SMOKE_OPENROUTER_FREE_MODELS": (
                "openai/gpt-oss-120b:free,open_router/custom/model:free"
            ),
            "FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS": (
                "poolside/laguna-m.1:free,openai/gpt-oss-120b:free"
            ),
        }
    )

    assert tuple(refs) == (
        "open_router/openai/gpt-oss-120b:free",
        "open_router/custom/model:free",
        "open_router/poolside/laguna-m.1:free",
    )
    assert refs["open_router/openai/gpt-oss-120b:free"] == (
        "FCC_SMOKE_OPENROUTER_FREE_MODELS"
    )
    assert refs["open_router/poolside/laguna-m.1:free"] == (
        "FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS"
    )


def test_openrouter_free_cli_models_reject_empty_override() -> None:
    try:
        openrouter_free_cli_model_refs({"FCC_SMOKE_OPENROUTER_FREE_MODELS": " , "})
    except ValueError as exc:
        assert "FCC_SMOKE_OPENROUTER_FREE_MODELS" in str(exc)
    else:
        raise AssertionError("expected empty OpenRouter free CLI override to fail")


def test_openrouter_free_cli_models_reject_wrong_provider_prefix() -> None:
    try:
        openrouter_free_cli_model_refs(
            {"FCC_SMOKE_OPENROUTER_FREE_MODELS": "nvidia_nim/model"}
        )
    except ValueError as exc:
        assert "open_router" in str(exc)
    else:
        raise AssertionError("expected wrong provider prefix to fail")


def test_smoke_config_returns_openrouter_free_cli_provider_models(monkeypatch) -> None:
    monkeypatch.delenv("FCC_SMOKE_OPENROUTER_FREE_MODELS", raising=False)
    monkeypatch.delenv("FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS", raising=False)
    config = _smoke_config(
        settings=_settings(
            model="open_router/openai/gpt-oss-120b:free",
            open_router_api_key="openrouter-key",
            ollama_base_url="",
        )
    )

    models = config.openrouter_free_cli_models()

    assert models[0].provider == "open_router"
    assert models[0].full_model == "open_router/nvidia/nemotron-3-super-120b-a12b:free"
    assert models[0].source == "openrouter_free_cli_default"