"""Tests for which completion an outcome report attaches to. `POST /outcome` is the only ground truth the router gets, so attributing one to the wrong conversation is worse than losing it: a model gets penalized for work it never did. The guard is a short window plus a refusal -- if more than one conversation was served inside it, the report is refused rather than guessed at. The window was not a window. `observed_at` is written by `datetime.now(timezone.utc).isoformat()`, which separates date from time with 'T', while `datetime('now', ...)` returns a space. Compared as strings, 'T' sorts after ' ', so the time of day never participated once the dates matched and a 120-second window admitted everything served that day. Measured on the live DB: 14 rows across 2 sessions where the correct comparison matched 0. These tests write timestamps through the same call the dispatcher uses, so they stay honest if that format ever changes. """ import sqlite3 from datetime import datetime, timedelta, timezone from pathlib import Path import pytest import dispatcher from dispatcher import AMBIGUOUS, SEED_CATEGORY, _most_recent_if_unambiguous from metrics import quota_burn ROOT = Path(__file__).resolve().parent.parent SCHEMA_SQL = (ROOT / "schema.sql").read_text() # Short, so "earlier today" is unambiguously outside it. The old row below sits # at the first instant of the current UTC day -- the earliest time that still # shares today's date, which is what the string comparison needed to go wrong. WINDOW_SECONDS = 5 def _now() -> datetime: return datetime.now(timezone.utc) def _start_of_today() -> datetime: return _now().replace(hour=0, minute=0, second=0, microsecond=1) @pytest.fixture(autouse=True) def short_window(monkeypatch): monkeypatch.setattr( dispatcher.cfg.verification, "outcome_attribution_window_seconds", WINDOW_SECONDS, ) @pytest.fixture def db(tmp_path): conn = sqlite3.connect(tmp_path / "test.db") conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) yield conn conn.close() def _observe(conn, *, when: datetime, session_key: str, request_id: str, category: str = "coding_general"): conn.execute( """ INSERT INTO energy_observations ( model_id, provider, request_id, session_key, task_category, observed_at ) VALUES ('m', 'neuralwatt', ?, ?, ?, ?) """, # Written exactly the way log_observation writes it. (request_id, session_key, category, when.isoformat()), ) conn.commit() def test_a_session_from_earlier_today_is_outside_the_window(db): """The regression: same calendar date is not the same as inside 5 seconds.""" _observe(db, when=_start_of_today(), session_key="morning", request_id="old") _observe(db, when=_now(), session_key="now", request_id="new") row = _most_recent_if_unambiguous(db) assert row is not AMBIGUOUS, "an 8-hours-stale session must not create ambiguity" assert row["request_id"] == "new" def test_an_old_session_alone_is_not_recent_enough_to_attribute(db): """With nothing inside the window there is nothing to attach to -- 404, not a guess.""" _observe(db, when=_start_of_today(), session_key="morning", request_id="old") assert _most_recent_if_unambiguous(db) is None def test_two_live_conversations_are_refused(db): """The guard the window exists to serve, still working.""" _observe(db, when=_now(), session_key="alice", request_id="a") _observe(db, when=_now(), session_key="bob", request_id="b") assert _most_recent_if_unambiguous(db) is AMBIGUOUS def test_one_conversation_across_several_turns_is_not_ambiguous(db): """Several completions, one session -- the ordinary case must still attribute.""" for i in range(3): _observe(db, when=_now(), session_key="alice", request_id=f"a{i}") row = _most_recent_if_unambiguous(db) assert row is not AMBIGUOUS assert row["request_id"] == "a2" def test_the_reference_sweep_is_not_a_conversation(db): """seed_energy's traffic must never make a real report look ambiguous.""" _observe(db, when=_now(), session_key=None, request_id="seed", category=SEED_CATEGORY) _observe(db, when=_now(), session_key="alice", request_id="a") row = _most_recent_if_unambiguous(db) assert row is not AMBIGUOUS assert row["request_id"] == "a" def test_quota_burn_counts_only_the_last_thirty_days(db, tmp_path, monkeypatch): """Same comparison, same fix -- an old row must not inflate the burn figure. The row sits at the START of the day 30 days ago: the cutoff falls on that same date but later in it, which is exactly the case a string comparison gets wrong. A row from 45 days ago would be excluded either way and would pin nothing. """ monkeypatch.setattr(dispatcher.cfg.objective, "plan_kwh_per_period", 6.25) just_outside = (_now() - timedelta(days=30)).replace( hour=0, minute=0, second=0, microsecond=1 ) db.execute( """ INSERT INTO energy_observations (model_id, provider, energy_kwh, observed_at) VALUES ('m', 'neuralwatt', 1.0, ?) """, (just_outside.isoformat(),), ) db.execute( """ INSERT INTO energy_observations (model_id, provider, energy_kwh, observed_at) VALUES ('m', 'neuralwatt', 0.25, ?) """, (_now().isoformat(),), ) db.commit() assert quota_burn(db, dispatcher.cfg)["metered_kwh_30d"] == pytest.approx(0.25)