6krrt/tests/test_outcome_attribution.py

"""Tests for which completion an outcome report attaches to.

`POST /outcome` is the only ground truth the router gets, so attributing one
to the wrong conversation is worse than losing it: a model gets penalized for
work it never did. The guard is a short window plus a refusal -- if more than
one conversation was served inside it, the report is refused rather than
guessed at.

The window was not a window. `observed_at` is written by
`datetime.now(timezone.utc).isoformat()`, which separates date from time with
'T', while `datetime('now', ...)` returns a space. Compared as strings, 'T'
sorts after ' ', so the time of day never participated once the dates matched
and a 120-second window admitted everything served that day. Measured on the
live DB: 14 rows across 2 sessions where the correct comparison matched 0.

These tests write timestamps through the same call the dispatcher uses, so
they stay honest if that format ever changes.
"""

import sqlite3
from datetime import datetime, timedelta, timezone
from pathlib import Path

import pytest

import dispatcher
from dispatcher import AMBIGUOUS, SEED_CATEGORY, _most_recent_if_unambiguous
from metrics import quota_burn

ROOT = Path(__file__).resolve().parent.parent
SCHEMA_SQL = (ROOT / "schema.sql").read_text()

# Short, so "earlier today" is unambiguously outside it. The old row below sits
# at the first instant of the current UTC day -- the earliest time that still
# shares today's date, which is what the string comparison needed to go wrong.
WINDOW_SECONDS = 5


def _now() -> datetime:
    return datetime.now(timezone.utc)


def _start_of_today() -> datetime:
    return _now().replace(hour=0, minute=0, second=0, microsecond=1)


@pytest.fixture(autouse=True)
def short_window(monkeypatch):
    monkeypatch.setattr(
        dispatcher.cfg.verification,
        "outcome_attribution_window_seconds",
        WINDOW_SECONDS,
    )


@pytest.fixture
def db(tmp_path):
    conn = sqlite3.connect(tmp_path / "test.db")
    conn.row_factory = sqlite3.Row
    conn.executescript(SCHEMA_SQL)
    yield conn
    conn.close()


def _observe(conn, *, when: datetime, session_key: str, request_id: str,
             category: str = "coding_general"):
    conn.execute(
        """
        INSERT INTO energy_observations (
            model_id, provider, request_id, session_key, task_category, observed_at
        ) VALUES ('m', 'neuralwatt', ?, ?, ?, ?)
        """,
        # Written exactly the way log_observation writes it.
        (request_id, session_key, category, when.isoformat()),
    )
    conn.commit()


def test_a_session_from_earlier_today_is_outside_the_window(db):
    """The regression: same calendar date is not the same as inside 5 seconds."""
    _observe(db, when=_start_of_today(), session_key="morning", request_id="old")
    _observe(db, when=_now(), session_key="now", request_id="new")

    row = _most_recent_if_unambiguous(db)

    assert row is not AMBIGUOUS, "an 8-hours-stale session must not create ambiguity"
    assert row["request_id"] == "new"


def test_an_old_session_alone_is_not_recent_enough_to_attribute(db):
    """With nothing inside the window there is nothing to attach to -- 404, not a guess."""
    _observe(db, when=_start_of_today(), session_key="morning", request_id="old")

    assert _most_recent_if_unambiguous(db) is None


def test_two_live_conversations_are_refused(db):
    """The guard the window exists to serve, still working."""
    _observe(db, when=_now(), session_key="alice", request_id="a")
    _observe(db, when=_now(), session_key="bob", request_id="b")

    assert _most_recent_if_unambiguous(db) is AMBIGUOUS


def test_one_conversation_across_several_turns_is_not_ambiguous(db):
    """Several completions, one session -- the ordinary case must still attribute."""
    for i in range(3):
        _observe(db, when=_now(), session_key="alice", request_id=f"a{i}")

    row = _most_recent_if_unambiguous(db)

    assert row is not AMBIGUOUS
    assert row["request_id"] == "a2"


def test_the_reference_sweep_is_not_a_conversation(db):
    """seed_energy's traffic must never make a real report look ambiguous."""
    _observe(db, when=_now(), session_key=None, request_id="seed",
             category=SEED_CATEGORY)
    _observe(db, when=_now(), session_key="alice", request_id="a")

    row = _most_recent_if_unambiguous(db)

    assert row is not AMBIGUOUS
    assert row["request_id"] == "a"


def test_quota_burn_counts_only_the_last_thirty_days(db, tmp_path, monkeypatch):
    """Same comparison, same fix -- an old row must not inflate the burn figure.

    The row sits at the START of the day 30 days ago: the cutoff falls on that
    same date but later in it, which is exactly the case a string comparison
    gets wrong. A row from 45 days ago would be excluded either way and would
    pin nothing.
    """
    monkeypatch.setattr(dispatcher.cfg.objective, "plan_kwh_per_period", 6.25)
    just_outside = (_now() - timedelta(days=30)).replace(
        hour=0, minute=0, second=0, microsecond=1
    )
    db.execute(
        """
        INSERT INTO energy_observations (model_id, provider, energy_kwh, observed_at)
        VALUES ('m', 'neuralwatt', 1.0, ?)
        """,
        (just_outside.isoformat(),),
    )
    db.execute(
        """
        INSERT INTO energy_observations (model_id, provider, energy_kwh, observed_at)
        VALUES ('m', 'neuralwatt', 0.25, ?)
        """,
        (_now().isoformat(),),
    )
    db.commit()

    assert quota_burn(db, dispatcher.cfg)["metered_kwh_30d"] == pytest.approx(0.25)