/libs/cua-bench/KiCad-task/main.py

"""KiCad circuit-creation tasks for cua-bench.

Tasks ask the agent to build a circuit in KiCad's schematic editor and export
a netlist in KiCad format (.net). Evaluation compares the agent's .net output
against a reference netlist (structural comparison) when one is provided, or
falls back to component-count checking when only ``expected_components`` is set.

Adding a new task variant:
    1. Add a dict to TASK_VARIANTS with at minimum a ``description`` key.
    2. Optionally include ``reference_netlist`` (path relative to this file's
       directory) for full structural comparison via session.apps.kicad.compare_netlist().
    3. Optionally include ``expected_components`` (dict of ref-prefix → count)
       as a lighter-weight fallback when no reference netlist is available.
    4. Optionally include ``initial_circuit_file`` (path relative to this
       file's directory) to seed the agent with a starting schematic.
"""

from pathlib import Path

import cua_bench as cb

NETLIST_PATH = "/home/cua/output.net"
_TASK_DIR = Path(__file__).resolve().parent

# ---------------------------------------------------------------------------
# Task variants — extend this list to add new tasks.
# Each dict becomes one Task; only ``description`` is required.
# ---------------------------------------------------------------------------
TASK_VARIANTS = [
    {
        "circuit_name": "LEDCircuit",
        "description": (
            "KiCad is already open. Using the Schematic Editor, "
            "draw a clean schematic in KiCad for a 9V battery driving "
            "two infrared LN271 LEDs. The battery positive terminal is first "
            "connected to a 100 ohm current limiting resistor followed by the "
            "two LEDs. Save the schematic, then export the netlist in KiCad "
            f"format to {NETLIST_PATH}."
        ),
        # Full structural comparison against the bundled reference netlist.
        "reference_netlist": "kicad_twoleds_netlist.net",
    },
    {
        "circuit_name": "EmptyNetlistExport",
        "description": (
            "KiCad is already open. Create an empty project (new schematic, "
            "no components required). Then export the netlist in KiCad format "
            f"to {NETLIST_PATH} using all default settings. "
            "The goal is to produce a valid KiCad .net file."
        ),
        # No reference netlist and no expected components: any valid .net passes.
        "netlist_export_only": True,
    },
]


@cb.tasks_config(split="train")
def load():
    """Build Task objects from TASK_VARIANTS."""
    return [
        cb.Task(
            description=variant["description"],
            metadata=variant,
            computer={
                "provider": "native",
                "setup_config": {
                    "os_type": "linux",
                    "width": 1920,
                    "height": 1080,
                },
            },
        )
        for variant in TASK_VARIANTS
    ]


@cb.setup_task(split="train")
async def start(task_cfg: cb.Task, session: cb.DesktopSession):
    """Install KiCad, optionally seed with an initial circuit file, then launch."""
    await session.apps.kicad.install(with_shortcut=True)

    initial_file = task_cfg.metadata.get("initial_circuit_file")
    if initial_file:
        await session.apps.kicad.launch(project_file=str(_TASK_DIR / initial_file))
    else:
        await session.apps.kicad.launch()


@cb.evaluate_task(split="train")
async def evaluate(task_cfg: cb.Task, session: cb.DesktopSession) -> list[float]:
    """Evaluate the agent's output netlist.

    Evaluation priority:
    1. ``reference_netlist`` in metadata → full structural comparison
       (component set + net topology) via compare_kicad_netlists().
    2. ``expected_components`` in metadata → lightweight component-count check.
    3. ``netlist_export_only`` or neither → any valid .net file scores 1.0.
    """
    netlist = await session.apps.kicad.read_netlist(netlist_path=NETLIST_PATH)
    if not netlist.strip():
        print(f"Evaluation: netlist missing or empty at {NETLIST_PATH}")
        return [0.0]

    meta = task_cfg.metadata

    # --- Path 1: full structural comparison ---
    ref_path = meta.get("reference_netlist")
    if ref_path:
        score = await session.apps.kicad.compare_netlist(
            candidate_path=NETLIST_PATH,
            reference_path=str(_TASK_DIR / ref_path),
        )
        print(f"Evaluation (structural): score={score:.3f}")
        return [score]

    # --- Path 2: component-count fallback ---
    expected = meta.get("expected_components", {})
    if expected:
        components = await session.apps.kicad.get_components(netlist_path=NETLIST_PATH)
        scores = [
            1.0 if sum(1 for c in components if c["ref"].upper().startswith(prefix)) == count else 0.0
            for prefix, count in expected.items()
        ]
        score = sum(scores) / len(scores)
        print(f"Evaluation (component counts): score={score:.3f}")
        return [score]

    # --- Path 3: valid .net file is sufficient ---
    print("Evaluation: valid netlist, no further checks → 1.0")
    return [1.0]


@cb.solve_task(split="train")
async def solve(task_cfg: cb.Task, session: cb.DesktopSession):
    """Oracle not implemented — KiCad requires GUI interaction."""
    pass


if __name__ == "__main__":
    cb.interact(__file__)