6krrt/tests/test_eval_scoring.py

"""Tests for eval_proficiency.py's scorers.

These decide what every proficiency number means, so the cases that matter
are the ones where a model did something *nearly* right — partial credit,
stray formatting, reasoning aloud before answering. Scoring those as zero
would make the whole table measure instruction-following rather than
competence.

No network: the code scorer really does execute a subprocess, which is the
point of testing it.
"""

import pytest

from eval_proficiency import (
    judge_for,
    parse_check_results,
    extract_judge_json,
    judge_for,
    normalize_answer,
    score_code,
    score_exact,
    score_tool,
    strip_fences,
)


# --- code -----------------------------------------------------------------

def test_all_checks_passing_scores_one():
    assert score_code("def f(x):\n    return x * 2", ["f(2)==4", "f(0)==0"])[0] == 1.0


def test_partial_credit_is_proportional():
    # A function right on one of two cases is genuinely better than one that
    # fails both, and a binary score would throw that distinction away
    score, detail = score_code(
        "def f(x):\n    return x * 2 if x else 99", ["f(2)==4", "f(0)==0"]
    )
    assert score == 0.5
    assert "1/2" in detail


def test_code_that_does_not_parse_scores_zero_and_says_so():
    score, detail = score_code("def f(x)\n  return", ["f(1)==1"])
    assert score == 0.0
    # Distinguishable from "ran but failed the cases", which matters when
    # reading results — one is a formatting problem, the other is competence
    assert "did not run" in detail


def test_infinite_loop_is_bounded_not_hung():
    score, detail = score_code("def f(x):\n    while True: pass", ["f(1)==1"])
    assert (score, detail) == (0.0, "timeout")


def test_markdown_fences_do_not_cost_the_model_the_task():
    # Every code prompt says "no fences" and models add them anyway. Scoring
    # that zero would measure instruction-following, not coding.
    fenced = "```python\ndef f(x):\n    return x * 2\n```"
    assert score_code(fenced, ["f(2)==4"])[0] == 1.0


def test_raises_helper_is_available_to_checks():
    # Checks express "should raise" through a helper the harness injects, so
    # the sandbox needs no test framework of its own
    code = "def g(n):\n    if n < 1: raise ValueError()\n    return n"
    assert score_code(code, ["raises(ValueError, g, 0)", "g(3)==3"])[0] == 1.0


def test_common_imports_are_available():
    # Ordinary solutions reach for re/math without importing them explicitly
    code = "def f(s):\n    return re.findall(r'\\d+', s)"
    assert score_code(code, ["f('a1b22') == ['1','22']"])[0] == 1.0


def test_a_task_with_no_checks_cannot_score_credit():
    assert score_code("def f(): pass", [])[0] == 0.0


# --- exact ----------------------------------------------------------------

def test_exact_ignores_surrounding_prose():
    assert score_exact("The answer is 36.", "36")[0] == 1.0


def test_exact_ignores_thousands_separators():
    assert score_exact("5,400", "5400")[0] == 1.0


def test_exact_takes_the_last_number_as_the_conclusion():
    # A model that reasons aloud states intermediate values first; the answer
    # is what it ends on
    assert score_exact("1/60 + 1/90 = 1/36, so 36 minutes", "36")[0] == 1.0


def test_exact_wrong_answer_scores_zero_and_reports_both():
    score, detail = score_exact("42", "36")
    assert score == 0.0
    assert "42" in detail and "36" in detail


def test_trailing_zeros_do_not_break_equality():
    assert normalize_answer("36.0") == normalize_answer("36")


# --- tool use -------------------------------------------------------------

def _call(name, arguments="{}"):
    return [{"function": {"name": name, "arguments": arguments}}]


def test_right_tool_and_arguments_scores_one():
    calls = _call("get_weather", '{"location": "Reykjavik, Iceland"}')
    task = {"expect_tool": "get_weather", "expect_args": {"location": "reykjavik"}}
    assert score_tool(calls, task)[0] == 1.0


def test_right_tool_wrong_arguments_keeps_half_credit():
    # Choosing correctly between tools is most of the skill; the arguments
    # are the rest
    calls = _call("get_weather", '{"location": "Paris"}')
    task = {"expect_tool": "get_weather", "expect_args": {"location": "reykjavik"}}
    assert score_tool(calls, task)[0] == 0.5


def test_wrong_tool_scores_zero():
    task = {"expect_tool": "convert_currency"}
    assert score_tool(_call("get_weather"), task)[0] == 0.0


def test_no_call_when_one_was_needed_scores_zero():
    assert score_tool([], {"expect_tool": "get_weather"})[0] == 0.0


def test_abstaining_when_no_tool_applies_is_the_correct_answer():
    # A model that reaches for a tool on every prompt is a real failure mode
    # in an agent loop, so not calling one is scored, not just tolerated
    assert score_tool([], {"expect_tool": None})[0] == 1.0


def test_calling_a_tool_when_none_applied_scores_zero():
    score, detail = score_tool(_call("get_weather"), {"expect_tool": None})
    assert score == 0.0
    assert "when none applied" in detail


def test_unparseable_arguments_keep_the_tool_choice_credit():
    calls = _call("get_weather", "{not json")
    task = {"expect_tool": "get_weather", "expect_args": {"location": "x"}}
    assert score_tool(calls, task)[0] == 0.5


# --- fence stripping ------------------------------------------------------

def test_strip_fences_leaves_bare_code_alone():
    assert strip_fences("def f(): pass") == "def f(): pass"


def test_strip_fences_handles_an_unlabelled_block():
    assert strip_fences("```\ndef f(): pass\n```").strip() == "def f(): pass"


# --- judge robustness -----------------------------------------------------

def test_judge_json_parses_a_bare_object():
    assert extract_judge_json('{"score": 0.8, "reason": "ok"}')["score"] == 0.8


def test_judge_json_survives_a_thinking_preamble():
    # Judges are reasoning models and leak their thinking into the content
    # despite response_format. This was 44% of judge calls on the first run.
    raw = 'Let me check the rubric:\n\n1. Distinguishes...\n\n{"score": 0.6, "reason": "partial"}'
    assert extract_judge_json(raw)["score"] == 0.6


def test_truncated_judge_output_is_a_failure_not_a_guess():
    # A cut-off object must not be salvaged into a number — better no sample
    assert extract_judge_json('{"score": 0.8, "reason": "Accurate with') is None


def test_judge_prose_without_json_is_a_failure():
    assert extract_judge_json("I think it is pretty good actually") is None
    assert extract_judge_json("") is None


def test_object_without_a_score_is_not_a_verdict():
    assert extract_judge_json('{"reason": "forgot the score"}') is None


def test_a_model_never_judges_its_own_family():
    # Self-scoring is a known bias and the default judge is itself in the
    # evaluated set, so its own family gets an alternate
    assert judge_for("kimi-k3", "kimi-k3") != "kimi-k3"
    assert judge_for("kimi-k3-fast", "kimi-k3") != "kimi-k3"
    assert judge_for("kimi-k3-flex", "kimi-k3") != "kimi-k3"
    # Everyone else is judged by the default
    assert judge_for("gemma-4-31b", "kimi-k3") == "kimi-k3"


def test_a_single_leading_space_does_not_destroy_a_correct_answer():
    # Observed: kimi-k2.7-code returns " def f(...)" with one leading space,
    # which is an IndentationError once the harness prepends its imports. It
    # scored 0.00 on every coding task for this and nothing else.
    code = ' def f(x):\n    return x * 2'
    assert score_code(code, ["f(2)==4"])[0] == 1.0


def test_indented_fenced_block_is_still_recovered():
    code = '  ```python\ndef f(x):\n    return x * 2\n```'
    assert score_code(code, ["f(2)==4"])[0] == 1.0


# --- harness integrity ----------------------------------------------------
#
# The harness runs the model's code as `__main__`, so anything the model prints
# lands in the same stdout the verdicts are read from. Counting the substring
# "PASS" therefore let a model vote on its own work.

def test_a_models_own_demo_output_cannot_award_it_marks():
    """Measured at 1.50 on a two-check task before the verdicts carried a nonce."""
    code = (
        "def add(a, b):\n"
        "    return a + b\n"
        "\n"
        'if __name__ == "__main__":\n'
        '    print("self-test:", add(1, 2) == 3 and "PASS" or "FAIL")\n'
    )

    score, detail = score_code(code, ["add(1,2)==3", "add(-1,1)==0"])

    assert score == 1.0
    assert detail == "2/2 checks"


def test_printing_PASS_cannot_rescue_a_wrong_answer():
    """The failure that matters: noise must not manufacture credit."""
    code = (
        "def add(a, b):\n"
        "    return 0\n"
        "\n"
        'if __name__ == "__main__":\n'
        '    print("PASS PASS PASS")\n'
    )

    # Both checks genuinely fail for a function that always returns 0.
    assert score_code(code, ["add(1,2)==3", "add(2,2)==4"])[0] == 0.0


def test_a_score_can_never_exceed_one():
    """Whatever a model prints, the fraction stays a fraction."""
    code = "def f(x):\n    return x\n" + 'print("CHECK 0 PASS\\n" * 50)\n'

    assert score_code(code, ["f(1)==1"])[0] <= 1.0


def test_check_results_are_read_by_index_not_counted():
    marker = "CHECK-deadbeef"
    stdout = (
        "PASS\n"                          # the model's noise
        f"{marker} 0 PASS\n"
        f"{marker} 1 FAIL\n"
        "CHECK 2 PASS\n"                  # right shape, wrong marker
        f"{marker} 0 PASS\n"              # a repeat cannot double-count
    )

    assert parse_check_results(stdout, marker) == {0: True, 1: False}


# --- judge selection ------------------------------------------------------

def test_the_configured_judge_is_used_when_it_is_a_different_model():
    assert judge_for("gemma-4-31b", "kimi-k3") == "kimi-k3"


def test_a_model_is_never_judged_by_its_own_family():
    """Including the case that broke it: the judge and the alternate agreeing."""
    assert judge_for("kimi-k3", "kimi-k3") != "kimi-k3"
    # `--judge-model qwen3.6-35b` used to hand qwen3.6-35b back to itself.
    assert judge_for("qwen3.6-35b", "qwen3.6-35b") != "qwen3.6-35b"


def test_a_fast_row_is_not_judged_by_its_reasoning_on_sibling():
    """Same weights, so the conflict is the same one."""
    assert judge_for("qwen3.6-35b-fast", "qwen3.6-35b") not in (
        "qwen3.6-35b", "qwen3.6-35b-fast"
    )


def test_no_judge_outside_the_family_means_no_sample():
    """Skipping beats self-grading, the same rule an unusable judge reply follows."""
    assert judge_for("kimi-k3", "kimi-k3", ) is not None  # alternates exist
    import eval_proficiency

    original = eval_proficiency.ALTERNATE_JUDGES
    try:
        eval_proficiency.ALTERNATE_JUDGES = ("kimi-k3-fast",)
        assert eval_proficiency.judge_for("kimi-k3", "kimi-k3") is None
    finally:
        eval_proficiency.ALTERNATE_JUDGES = original