bfb0ff7c00
The router previously recorded only completions (energy_observations), not the routing decisions behind them, so 'how routing is performing' was not answerable from data. This adds: - route_decisions table + idempotent ensure_route_decisions (guarded CREATE TABLE IF NOT EXISTS, never regenerates a live DB) gated by logging.log_route_decisions; every decision kind (route/dispatch/chat/ passthrough/local-vision) is persisted best-effort via persist_route_decision (never fails a request; only session_key, never session_dir). The table is ensured on the write path (mirroring proficiency_store._write -> ensure_columns) so a live DB that predates the feature migrates safely. - metrics.py aggregator moved quota_burn/scoring_coverage in from the dispatcher (breaking a would-be circular import) and adds recent_decisions/per_model/verdict_mix/top_proficiency; /health now imports them and GET /metrics exposes the 7-key JSON (window-bounded, loopback-only, no auth). - observed_at indexes on energy_observations/verifications.
202 lines
6.9 KiB
Python
202 lines
6.9 KiB
Python
"""Tests for the GET /metrics endpoint on dispatcher.
|
|
|
|
Seeds a throwaway temp DB and asserts on the actual JSON returned by a real
|
|
TestClient GET (never a mock-call assertion), to defeat
|
|
``misleading_success_output``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from starlette.testclient import TestClient
|
|
|
|
import dispatcher
|
|
from config import load_config
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMA_SQL = (ROOT / "schema.sql").read_text()
|
|
CFG = load_config(str(ROOT / "config.yaml"))
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _make_db(tmp_path: Path) -> sqlite3.Connection:
|
|
conn = sqlite3.connect(str(tmp_path / "test.db"))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
return conn
|
|
|
|
|
|
def _seed_models(conn: sqlite3.Connection) -> None:
|
|
for model_id, tier, context, cost, vision in (
|
|
("cheap", 2, 262128, 0.30, 1),
|
|
("dear", 2, 262128, 9.00, 0),
|
|
("tiny", 1, 131072, 0.10, 1),
|
|
):
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO models (
|
|
model_id, provider, base_model_id, tier, context_window,
|
|
effective_context_window, max_output_tokens,
|
|
cost_per_1m_prompt, cost_per_1m_completion,
|
|
supports_vision, supports_json_mode,
|
|
latency_class, reasoning_mode, context_variant,
|
|
access_level, availability, last_updated
|
|
) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?,
|
|
?, 1, 'standard', 'default', 'full', 'public', 'active',
|
|
'2026-08-22T00:00:00+00:00')
|
|
""",
|
|
(model_id, model_id, tier, context, cost, cost / 3, vision),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_decision(conn: sqlite3.Connection) -> None:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO route_decisions (
|
|
observed_at, kind, task_category, task_tier, required_context_tokens,
|
|
confidence, classifier_ms, classification_source, latency_tolerance,
|
|
candidates_considered, selected_model, selected_provider,
|
|
runner_up_models, est_cost_usd, est_proficiency,
|
|
session_key, tools, images, json_mode, streamed
|
|
) VALUES (?, 'route', 'coding_general', 2, 100, 0.95, 200,
|
|
'classifier', 'interactive', 5, 'cheap', 'neuralwatt',
|
|
'[{"model_id":"dear","provider":"neuralwatt"}]',
|
|
0.001, 0.9, 'abc123', 0, 0, 0, 0)
|
|
""",
|
|
(_now().isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_energy(conn: sqlite3.Connection) -> None:
|
|
now = _now()
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, task_category, completion_tokens, energy_kwh, "
|
|
"cost_usd, carbon_g_co2eq, attribution_ratio, observed_at) "
|
|
"VALUES ('cheap', 'neuralwatt', 'coding_general', 100, 5.0e-05, 0.001, "
|
|
"2.4e-03, 0.25, ?)",
|
|
((now - timedelta(days=2)).isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_verification(conn: sqlite3.Connection) -> None:
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('cheap', 'neuralwatt', 'structural', 'ok', ?)",
|
|
(_now().isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_proficiency(conn: sqlite3.Connection) -> None:
|
|
conn.execute(
|
|
"INSERT INTO proficiency (model_id, provider, category, blended_score, "
|
|
"source, last_updated) "
|
|
"VALUES ('cheap', 'neuralwatt', 'coding_general', 0.9, "
|
|
"'self_eval_thin', '2026-01-01T00:00:00+00:00')",
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
@pytest.fixture
|
|
def seeded_client(tmp_path, monkeypatch):
|
|
"""A TestClient wired to a seeded temp DB, at /metrics."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
for i in range(60): # exceed the 50 cap
|
|
_seed_decision(conn)
|
|
_seed_energy(conn)
|
|
_seed_verification(conn)
|
|
_seed_proficiency(conn)
|
|
conn.close()
|
|
|
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db"))
|
|
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
|
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
|
|
|
with TestClient(dispatcher.app) as client:
|
|
yield client
|
|
|
|
|
|
def test_metrics_endpoint_has_all_top_level_keys(seeded_client):
|
|
"""GET /metrics returns 200 with every required top-level key."""
|
|
resp = seeded_client.get("/metrics")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
for key in (
|
|
"quota",
|
|
"coverage",
|
|
"recent_decisions",
|
|
"per_model",
|
|
"verdict_mix",
|
|
"top_proficiency",
|
|
"generated_at",
|
|
):
|
|
assert key in data, f"missing top-level key {key!r}"
|
|
|
|
|
|
def test_metrics_recent_decisions_capped_and_carries_provider(seeded_client):
|
|
"""recent_decisions is capped at 50 and each row has selected_provider."""
|
|
resp = seeded_client.get("/metrics")
|
|
data = resp.json()
|
|
decisions = data["recent_decisions"]
|
|
assert isinstance(decisions, list)
|
|
assert len(decisions) <= 50
|
|
assert len(decisions) > 0
|
|
for row in decisions:
|
|
assert "selected_provider" in row
|
|
# DESC order: the first decision has the largest id.
|
|
first_id = decisions[0]["id"]
|
|
for row in decisions[1:]:
|
|
assert row["id"] <= first_id
|
|
|
|
|
|
def test_metrics_aggregations_are_populated(seeded_client):
|
|
"""per_model / verdict_mix / top_proficiency reflect seeded data."""
|
|
resp = seeded_client.get("/metrics")
|
|
data = resp.json()
|
|
assert isinstance(data["per_model"], list)
|
|
assert any(r["model_id"] == "cheap" for r in data["per_model"])
|
|
assert data["per_model"][0]["calls"] == 1
|
|
assert data["verdict_mix"]["ok"] == 1
|
|
assert isinstance(data["top_proficiency"], list)
|
|
assert data["top_proficiency"][0]["model_id"] == "cheap"
|
|
assert data["quota"] is not None
|
|
|
|
|
|
def test_metrics_contains_no_session_dir(seeded_client):
|
|
"""The JSON must never name session_dir or expose conversation text."""
|
|
body = seeded_client.get("/metrics").text
|
|
assert "session_dir" not in body
|
|
|
|
|
|
def test_metrics_empty_db_returns_200(monkeypatch, tmp_path):
|
|
"""Fresh empty temp DB: 200 with empty arrays, no exception."""
|
|
conn = _make_db(tmp_path)
|
|
conn.close()
|
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db"))
|
|
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
|
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
|
|
|
with TestClient(dispatcher.app) as client:
|
|
resp = client.get("/metrics")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["recent_decisions"] == []
|
|
assert data["per_model"] == []
|
|
assert data["verdict_mix"] == {}
|
|
assert data["top_proficiency"] == []
|
|
assert "generated_at" in data
|