6krrt/tests/test_eval_classifier.py

"""Tests for the eval_classifier CLI backends (Todo 18).

Covers the ``--backend encoder|decision`` and ``--head-off`` flags:
flag parsing, the ``--head-off`` effect on the encoder's trainable-head
globals, and the decision backend dispatching through
``local_decision.classify_choice`` over ``_DECISION_DESCRIPTIONS``.

``local_decision`` does not exist on this branch yet (it is a sibling todo),
so the decision-backend tests stub the module in ``sys.modules``. Nothing here
touches the network, transformers or torch.
"""

import sys
from unittest import mock

import pytest

import local_encoder
from eval_classifier import (
    _build_parser,
    _disable_trainable_head,
    run_eval,
)


def _sample_tasks():
    return [
        {"id": "t1", "category": "coding_general", "prompt": "write a function"},
        {"id": "t2", "category": "debugging", "prompt": "fix this crash"},
    ]


CATS = ["coding_general", "debugging"]


# --- flag parsing ---------------------------------------------------------

def test_backend_defaults_to_encoder():
    args = _build_parser().parse_args([])
    assert args.backend == "encoder"
    assert args.head_off is False


def test_backend_accepts_decision():
    args = _build_parser().parse_args(["--backend", "decision"])
    assert args.backend == "decision"


def test_backend_rejects_unknown_choice():
    with pytest.raises(SystemExit):
        _build_parser().parse_args(["--backend", "bogus"])


def test_head_off_flag_is_store_true():
    args = _build_parser().parse_args(["--head-off"])
    assert args.head_off is True


def test_decision_override_flags_parse():
    args = _build_parser().parse_args([
        "--backend", "decision",
        "--decision-base-url", "http://x:11434/api/chat",
        "--decision-model", "qwen3.5:4b",
        "--decision-num-ctx", "8192",
        "--decision-timeout", "60",
        "--decision-coverage-min", "0.5",
    ])
    assert args.decision_base_url == "http://x:11434/api/chat"
    assert args.decision_model == "qwen3.5:4b"
    assert args.decision_num_ctx == 8192
    assert args.decision_timeout == 60
    assert args.decision_coverage_min == 0.5


# --- --head-off -----------------------------------------------------------

def test_disable_trainable_head_sets_globals():
    local_encoder._TRAINABLE_HEAD = object()
    local_encoder._TRAINABLE_HEAD_RESOLVED = False
    try:
        _disable_trainable_head()
        assert local_encoder._TRAINABLE_HEAD is None
        assert local_encoder._TRAINABLE_HEAD_RESOLVED is True
    finally:
        local_encoder._TRAINABLE_HEAD = None
        local_encoder._TRAINABLE_HEAD_RESOLVED = False


def test_run_eval_encoder_head_off_disables_head():
    # --head-off with the encoder backend must set the globals before scoring.
    local_encoder._TRAINABLE_HEAD = object()
    local_encoder._TRAINABLE_HEAD_RESOLVED = False
    try:
        with mock.patch(
            "local_encoder.classify_zero_shot",
            return_value=("coding_general", 0.9),
        ):
            run_eval(
                _sample_tasks(), CATS,
                model_id="x", device="cpu", noise_levels=("clean",),
                backend="encoder", head_off=True,
            )
        assert local_encoder._TRAINABLE_HEAD is None
        assert local_encoder._TRAINABLE_HEAD_RESOLVED is True
    finally:
        local_encoder._TRAINABLE_HEAD = None
        local_encoder._TRAINABLE_HEAD_RESOLVED = False


def test_run_eval_encoder_default_does_not_touch_head():
    # Without --head-off the encoder backend leaves the globals alone.
    local_encoder._TRAINABLE_HEAD = None
    local_encoder._TRAINABLE_HEAD_RESOLVED = False
    try:
        with mock.patch(
            "local_encoder.classify_zero_shot",
            return_value=("coding_general", 0.9),
        ):
            run_eval(
                _sample_tasks(), CATS,
                model_id="x", device="cpu", noise_levels=("clean",),
                backend="encoder", head_off=False,
            )
        assert local_encoder._TRAINABLE_HEAD_RESOLVED is False
    finally:
        local_encoder._TRAINABLE_HEAD = None
        local_encoder._TRAINABLE_HEAD_RESOLVED = False


# --- decision backend -----------------------------------------------------

def _fake_decision_module():
    mod = mock.Mock()
    mod._DECISION_DESCRIPTIONS = {
        "coding_general": "writing or editing code",
        "debugging": "finding and fixing bugs",
    }
    mod.classify_category.return_value = ("coding_general", 0.95, 1.0)
    return mod


def test_run_eval_decision_uses_classify_category():
    fake = _fake_decision_module()
    decision = {
        "base_url": "http://x:11434/api/chat",
        "model": "qwen3.5:4b",
        "num_ctx": 8192,
        "timeout_s": 60,
        "coverage_min": 0.0,
    }
    with mock.patch.dict(sys.modules, {"local_decision": fake}):
        results = run_eval(
            _sample_tasks(), CATS,
            model_id="x", device="cpu", noise_levels=("clean",),
            backend="decision", decision=decision,
        )

    assert fake.classify_category.called
    _, kwargs = fake.classify_category.call_args
    assert kwargs["base_url"] == decision["base_url"]
    assert kwargs["model"] == decision["model"]
    assert kwargs["num_ctx"] == decision["num_ctx"]
    assert kwargs["timeout_s"] == decision["timeout_s"]
    assert kwargs["coverage_min"] == decision["coverage_min"]

    # The returned category name is used directly as the predicted verdict.
    tv = results["task_verdicts"]
    assert tv["t1"][1] == "coding_general"
    assert tv["t2"][1] == "coding_general"


def test_run_eval_decision_defaults_coverage_min():
    fake = _fake_decision_module()
    with mock.patch.dict(sys.modules, {"local_decision": fake}):
        run_eval(
            _sample_tasks(), CATS,
            model_id="x", device="cpu", noise_levels=("clean",),
            backend="decision", decision={"base_url": "u", "model": "m"},
        )
    _, kwargs = fake.classify_category.call_args
    assert kwargs["coverage_min"] == 0.0