6krrt/tests/test_verification.py

"""Tests for verification.py — structural checks on model responses.

The load-bearing distinction here is between "this is wrong" and "we could
not check this". Conflating them would penalize a model for the checker's
limits, which is the same mistake as scoring a judge malfunction against a
model — something this project has already done once.

Nothing here executes model output, and the tests assert that.
"""

import pytest

from verification import (
    Block,
    check_block,
    check_json,
    check_python,
    check_yaml,
    extract_blocks,
    verify_response,
)


# --- truncation, the failure that looks like success ----------------------

def test_finish_reason_length_is_decisive():
    # A truncated answer that happens to parse is the dangerous case: it looks
    # complete to the client, which then acts on a fragment.
    v = verify_response("```python\nx = 1\n```", finish_reason="length")
    assert v.verdict == "truncated"
    assert v.failed


def test_unterminated_fence_is_truncated_even_if_it_parses():
    # "x = 1" is valid Python; the missing closing fence is the signal
    v = verify_response("here you go:\n```python\nx = 1\n")
    assert v.verdict == "truncated"


def test_complete_response_is_ok():
    v = verify_response("```python\ndef f(x):\n    return x * 2\n```")
    assert v.verdict == "ok"
    assert not v.failed


# --- malformed content ----------------------------------------------------

def test_broken_python_is_malformed_with_a_line_number():
    v = verify_response("```python\ndef f(x)\n    return x\n```")
    assert v.verdict == "malformed"
    assert "line" in v.detail


def test_broken_json_is_malformed():
    assert check_json('{"a": 1,}').verdict == "malformed"
    assert check_json('{"a": 1}').verdict == "ok"


def test_broken_yaml_is_malformed():
    assert check_yaml("a: [1, 2\nb: 3").verdict == "malformed"
    assert check_yaml("a: 1\nb: two").verdict == "ok"


def test_prose_left_inside_a_code_fence_is_caught():
    # Models sometimes trail off into explanation without closing the fence
    v = verify_response("```python\ndef f():\n    return 1\nThis function returns one.\n```")
    assert v.verdict == "malformed"


# --- unverifiable is not failure -----------------------------------------

def test_plain_prose_is_unverifiable_not_failed():
    v = verify_response("A B-tree keeps itself balanced by splitting nodes.")
    assert v.verdict == "unverifiable"
    assert not v.failed


def test_unknown_language_is_unverifiable_not_failed():
    v = verify_response("```rust\nfn main() { let x = ; }\n```")
    assert v.verdict == "unverifiable"
    assert not v.failed


def test_shell_is_deferred_to_the_caller_not_guessed():
    # bash -n is a subprocess, so this module declines rather than pretending
    c = check_block(Block(lang="bash", code="echo hi", closed=True))
    assert c.verdict == "unverifiable"


def test_empty_block_is_unverifiable():
    assert check_block(Block(lang="python", code="   \n", closed=True)).verdict == "unverifiable"


def test_response_with_no_blocks_at_all_is_unverifiable():
    # Prose, not empty — an empty response is a failure, decided separately
    assert verify_response("A B-tree splits nodes to stay balanced.").verdict == "unverifiable"


# --- multiple blocks ------------------------------------------------------

def test_one_bad_block_fails_the_response():
    text = "```python\nx = 1\n```\nand\n```json\n{bad\n```"
    v = verify_response(text)
    assert v.verdict == "malformed"
    assert len(v.checks) == 2


def test_truncation_outranks_malformation_in_reporting():
    # If a response is both cut off and broken, the truncation is the cause
    text = "```json\n{bad\n```\n```python\nx = (\n"
    v = verify_response(text)
    assert v.verdict == "truncated"


def test_all_good_blocks_pass():
    text = "```python\nx = 1\n```\ntext\n```json\n{\"a\": 1}\n```"
    assert verify_response(text).verdict == "ok"


def test_checkable_block_alongside_an_uncheckable_one_still_passes():
    text = "```rust\nfn main() {}\n```\n```python\nx = 1\n```"
    assert verify_response(text).verdict == "ok"


# --- extraction -----------------------------------------------------------

def test_extract_records_language_and_closure():
    blocks = extract_blocks("```python\na\n```\n```\nb\n")
    assert [(b.lang, b.closed) for b in blocks] == [("python", True), ("", False)]


def test_language_tag_is_case_insensitive():
    assert verify_response("```PYTHON\nx = 1\n```").verdict == "ok"


# --- safety ---------------------------------------------------------------

def test_verification_never_executes_the_code_it_checks(tmp_path):
    # If verification ran this, the file would exist. It must not.
    canary = tmp_path / "canary.txt"
    payload = f"```python\nopen({str(canary)!r}, 'w').write('executed')\n```"
    v = verify_response(payload)
    assert v.verdict == "ok"          # it parses fine
    assert not canary.exists()        # ...and was never run


def test_a_syntactically_valid_destructive_snippet_is_only_parsed(tmp_path):
    victim = tmp_path / "important.txt"
    victim.write_text("still here")
    payload = f"```python\nimport os\nos.remove({str(victim)!r})\n```"
    verify_response(payload)
    assert victim.read_text() == "still here"


# --- local LLM verification -----------------------------------------------

def test_local_verdict_survives_a_thinking_preamble():
    from verification import parse_verdict_json
    raw = 'Let me consider the answer...\n\n{"ok": false, "reason": "cut off mid-sentence"}'
    assert parse_verdict_json(raw)["ok"] is False


def test_unusable_local_output_records_no_verdict():
    # A checker that malfunctions must not produce a failure verdict — that
    # charges the model for the checker's problem
    from verification import interpret_local_verdict, parse_verdict_json
    assert interpret_local_verdict(parse_verdict_json("I think it's fine")) is None
    assert interpret_local_verdict(None) is None


def test_non_boolean_ok_is_rejected():
    from verification import interpret_local_verdict
    assert interpret_local_verdict({"ok": "yes"}) is None
    assert interpret_local_verdict({"reason": "no verdict"}) is None


def test_local_verdict_maps_to_a_check():
    from verification import interpret_local_verdict
    assert interpret_local_verdict({"ok": True, "reason": "fine"}).verdict == "ok"
    assert interpret_local_verdict({"ok": False, "reason": "empty"}).verdict == "malformed"


# --- the size gate is economics, not taste --------------------------------

def test_small_answers_are_not_worth_checking():
    # A local check costs ~15% of a median 193-token answer, so it only pays
    # if such answers fail more than ~15% of the time
    from verification import worth_local_check
    assert worth_local_check("unverifiable", 193, 600) is False
    assert worth_local_check("unverifiable", 599, 600) is False


def test_large_answers_are_worth_checking():
    from verification import worth_local_check
    assert worth_local_check("unverifiable", 600, 600) is True
    assert worth_local_check("unverifiable", 4000, 600) is True


def test_structurally_decided_responses_skip_the_fuzzy_check():
    # If the code already parsed (or failed to), that verdict is exact and
    # free; a fuzzy second opinion adds nothing and costs real time
    from verification import worth_local_check
    for verdict in ("ok", "malformed", "truncated"):
        assert worth_local_check(verdict, 5000, 600) is False


def test_missing_token_count_does_not_trigger_a_check():
    from verification import worth_local_check
    assert worth_local_check("unverifiable", None, 600) is False


# --- excerpting must not manufacture truncation ---------------------------

def test_excerpt_leaves_short_text_alone():
    from verification import excerpt
    assert excerpt("short", 4000) == "short"


def test_excerpt_preserves_the_real_ending():
    # The bug this guards: head-truncating a complete 6,637-char answer to
    # 4,000 made it end mid-sentence, and the local checker correctly reported
    # "cut off mid-thought" — a false failure created by the harness.
    from verification import excerpt
    text = "START" + ("x" * 10_000) + "THE ACTUAL ENDING."
    out = excerpt(text, 4000)
    assert out.startswith("START")
    assert out.endswith("THE ACTUAL ENDING.")
    assert "elided" in out


def test_excerpt_labels_the_cut_so_the_checker_knows():
    from verification import excerpt
    out = excerpt("a" * 10_000, 1000)
    assert "characters elided" in out
    # Budget is respected apart from the marker itself
    assert len(out) < 1000 + 100


def test_prompt_carries_both_sides_with_the_real_ending():
    from verification import build_local_verify_prompt
    answer = "A" * 9000 + "FINAL SENTENCE."
    prompt = build_local_verify_prompt("do a thing", answer, limit=2000)
    assert "FINAL SENTENCE." in prompt
    assert "do a thing" in prompt


def test_empty_response_is_decided_in_code_not_by_a_model():
    # Asking the local 9.7B checker about an empty answer returned ok=true
    # with the reason "Answer is too short" — a self-contradiction. This is
    # settled by an if-statement instead.
    assert verify_response("").verdict == "malformed"
    assert verify_response("   \n\t ").verdict == "malformed"
    assert verify_response("").failed


def test_whitespace_only_beats_the_no_blocks_path():
    # It must not fall through to 'unverifiable' just for lacking code fences
    assert verify_response("\n\n").verdict != "unverifiable"


# --- agent turns are not prose answers ------------------------------------

def test_a_tool_call_turn_is_not_a_failure():
    # Real measurement: 29 of 63 completions in one agent session were
    # recorded as "malformed: empty response" purely for making tool calls.
    # Folding those in would have penalized the models that shipped a working
    # feature with 349 passing tests.
    v = verify_response("", has_tool_calls=True)
    assert v.verdict == "unverifiable"
    assert not v.failed


def test_a_half_sentence_before_a_tool_call_is_not_a_failure():
    v = verify_response("Let me check the config", has_tool_calls=True)
    assert not v.failed


def test_tool_call_outranks_even_truncation():
    # An agent turn ending in a tool call routinely reports finish_reason
    # length; that is the call boundary, not a cut-off answer
    assert verify_response("x", "length", has_tool_calls=True).verdict == "unverifiable"


def test_a_genuinely_empty_answer_is_still_a_failure():
    # The fix must not blind the checker to real emptiness
    assert verify_response("", has_tool_calls=False).verdict == "malformed"


def test_broken_code_is_still_caught_when_a_tool_was_not_called():
    assert verify_response("```python\ndef f(\n```").verdict == "malformed"


def test_the_local_checker_skips_tool_turns_too():
    # Mirror-image blind spot: "cuts off mid-sentence" is exactly what a turn
    # looks like when it ends by calling a tool. Observed producing 8 false
    # failures in 9 checks on real agent traffic.
    from verification import worth_local_check
    assert worth_local_check("unverifiable", 5000, 600, has_tool_calls=True) is False
    assert worth_local_check("unverifiable", 5000, 600, has_tool_calls=False) is True