6krrt/tests/test_feedback.py

"""Tests for feedback.py — folding observed failures into proficiency.

The design decision under test is the asymmetry: failures are recorded,
passes are not. A structural 'ok' only means the code parsed, so treating it
as a quality sample would flood self_eval_score with 1.0s and wash out the
benchmark's discrimination — the coding categories already sit at 1.00 for
every model, and this would spread that flatness everywhere.
"""

import sqlite3
from pathlib import Path

import pytest

from config import load_config
from feedback import FAILURE_VERDICTS, apply_failures, summarize, unapplied_failures

ROOT = Path(__file__).resolve().parent.parent
SCHEMA_SQL = (ROOT / "schema.sql").read_text()
CFG = load_config(ROOT / "config.yaml")


@pytest.fixture
def db(tmp_path):
    conn = sqlite3.connect(tmp_path / "t.db")
    conn.row_factory = sqlite3.Row
    conn.executescript(SCHEMA_SQL)
    conn.execute(
        """
        INSERT INTO models (model_id, provider, base_model_id, availability, last_updated)
        VALUES ('m', 'nw', 'm', 'active', '2026-08-17T00:00:00+00:00')
        """
    )
    conn.commit()
    yield conn
    conn.close()


def _verify(conn, verdict, category="coding_general", kind="structural"):
    conn.execute(
        """
        INSERT INTO verifications (model_id, provider, task_category, kind, verdict, observed_at)
        VALUES ('m', 'nw', ?, ?, ?, '2026-08-17T00:00:00+00:00')
        """,
        (category, kind, verdict),
    )
    conn.commit()


def _prof(conn):
    return conn.execute(
        "SELECT self_eval_score s, self_eval_samples n FROM proficiency WHERE model_id='m'"
    ).fetchone()


# --- only failures are folded in ------------------------------------------

def test_passes_are_not_recorded_as_samples(db):
    # A structural 'ok' means "it parsed", not "it was correct". Recording it
    # as a 1.0 would inflate every model toward the ceiling.
    for _ in range(5):
        _verify(db, "ok")
    assert unapplied_failures(db) == []


def test_unverifiable_is_not_evidence(db):
    # The checker had nothing to say; that says nothing about the model
    for _ in range(5):
        _verify(db, "unverifiable")
    assert unapplied_failures(db) == []


@pytest.mark.parametrize("verdict", FAILURE_VERDICTS)
def test_failures_are_collected(db, verdict):
    _verify(db, verdict)
    assert len(unapplied_failures(db)) == 1


def test_a_failure_drags_the_score_down_proportionally(db):
    from proficiency_store import add_self_eval

    # Given: a benchmark score of 0.9 over 9 samples
    add_self_eval(db, CFG, "m", "nw", "coding_general", [0.9] * 9)
    _verify(db, "malformed")

    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    row = _prof(db)
    # One 0.0 folded into the running mean: (0.9*9 + 0)/10
    assert row["n"] == 10
    assert row["s"] == pytest.approx(0.81)


def test_a_model_that_never_fails_keeps_its_benchmark_score(db):
    from proficiency_store import add_self_eval

    add_self_eval(db, CFG, "m", "nw", "coding_general", [0.9] * 9)
    for _ in range(20):
        _verify(db, "ok")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    row = _prof(db)
    assert (row["n"], row["s"]) == (9, pytest.approx(0.9))


# --- idempotence ----------------------------------------------------------

def test_a_failure_is_applied_only_once(db):
    _verify(db, "malformed")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    first = _prof(db)["n"]
    # Re-running must not punish the model again for the same bad response
    assert unapplied_failures(db) == []
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    assert _prof(db)["n"] == first


def test_dry_run_changes_nothing(db):
    _verify(db, "truncated")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=True)
    assert _prof(db) is None
    assert len(unapplied_failures(db)) == 1


# --- grouping -------------------------------------------------------------

def test_failures_group_by_model_and_category(db):
    _verify(db, "malformed", category="coding_general")
    _verify(db, "malformed", category="coding_general")
    _verify(db, "truncated", category="debugging")
    grouped = summarize(unapplied_failures(db))
    assert {k[2]: len(v) for k, v in grouped.items()} == {
        "coding_general": 2,
        "debugging": 1,
    }


def test_failures_without_a_category_are_skipped(db):
    # Nothing to attribute them to — proficiency is per-category
    db.execute(
        """
        INSERT INTO verifications (model_id, provider, task_category, kind, verdict, observed_at)
        VALUES ('m', 'nw', NULL, 'structural', 'malformed', '2026-08-17T00:00:00+00:00')
        """
    )
    db.commit()
    assert unapplied_failures(db) == []


def test_both_check_kinds_count(db):
    _verify(db, "malformed", kind="structural")
    _verify(db, "malformed", kind="local_llm")
    assert len(unapplied_failures(db)) == 2


# --- attribution: not every failure is the model's fault -------------------

def _verify_capped(conn, verdict="truncated", category="coding_general"):
    conn.execute(
        """
        INSERT INTO verifications (model_id, provider, task_category, kind, verdict,
                                   observed_at, model_attributable)
        VALUES ('m', 'nw', ?, 'structural', ?, '2026-08-17T00:00:00+00:00', 0)
        """,
        (category, verdict),
    )
    conn.commit()


def test_client_capped_truncation_is_not_the_models_fault(db):
    # Found by forcing it: a request with max_tokens=40 truncates, and without
    # this any agent using a tight cap would systematically drag down whatever
    # model it routed to.
    _verify_capped(db)
    assert unapplied_failures(db) == []


def test_capped_failures_are_still_recorded_for_visibility(db):
    # The response really was unusable — it just says nothing about the model
    _verify_capped(db)
    n = db.execute("SELECT COUNT(*) c FROM verifications WHERE verdict='truncated'").fetchone()["c"]
    assert n == 1


def test_attributable_and_capped_failures_are_separated(db):
    _verify(db, "malformed")      # model's fault
    _verify_capped(db)            # client's cap
    rows = unapplied_failures(db)
    assert len(rows) == 1
    assert rows[0]["verdict"] == "malformed"


# --- client outcomes: the only two-way evidence ----------------------------

def _outcome(conn, verdict, category="coding_general"):
    conn.execute(
        """
        INSERT INTO verifications (model_id, provider, task_category, kind, verdict, observed_at)
        VALUES ('m', 'nw', ?, 'client_outcome', ?, '2026-08-17T00:00:00+00:00')
        """,
        (category, verdict),
    )
    conn.commit()


def test_a_client_reported_success_counts_as_a_positive_sample(db):
    # Unlike a parser's 'ok' — which means the code parsed — this means the
    # client ran it and the work worked. That is the only ground truth here.
    from proficiency_store import add_self_eval

    add_self_eval(db, CFG, "m", "nw", "coding_general", [0.5] * 3)
    _outcome(db, "succeeded")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    row = _prof(db)
    assert row["n"] == 4
    assert row["s"] == pytest.approx((0.5 * 3 + 1.0) / 4)


def test_a_client_reported_failure_counts_against(db):
    from proficiency_store import add_self_eval

    add_self_eval(db, CFG, "m", "nw", "coding_general", [1.0] * 3)
    _outcome(db, "failed")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    assert _prof(db)["s"] == pytest.approx(0.75)


def test_a_structural_pass_still_contributes_nothing(db):
    # The asymmetry that matters: 'ok' from a parser is weak evidence and
    # would inflate every score toward the ceiling
    for _ in range(10):
        _verify(db, "ok")
    assert unapplied_failures(db) == []


def test_successes_and_failures_mix_into_one_rate(db):
    for _ in range(3):
        _outcome(db, "succeeded")
    _outcome(db, "failed")
    apply_failures(db, CFG, summarize(unapplied_failures(db)), dry_run=False)
    row = _prof(db)
    assert (row["n"], row["s"]) == (4, pytest.approx(0.75))