bfb0ff7c00
The router previously recorded only completions (energy_observations), not the routing decisions behind them, so 'how routing is performing' was not answerable from data. This adds: - route_decisions table + idempotent ensure_route_decisions (guarded CREATE TABLE IF NOT EXISTS, never regenerates a live DB) gated by logging.log_route_decisions; every decision kind (route/dispatch/chat/ passthrough/local-vision) is persisted best-effort via persist_route_decision (never fails a request; only session_key, never session_dir). The table is ensured on the write path (mirroring proficiency_store._write -> ensure_columns) so a live DB that predates the feature migrates safely. - metrics.py aggregator moved quota_burn/scoring_coverage in from the dispatcher (breaking a would-be circular import) and adds recent_decisions/per_model/verdict_mix/top_proficiency; /health now imports them and GET /metrics exposes the 7-key JSON (window-bounded, loopback-only, no auth). - observed_at indexes on energy_observations/verifications.
539 lines
19 KiB
Python
539 lines
19 KiB
Python
"""Tests for metrics.py — read-only aggregation helpers.
|
|
|
|
Every test seeds a throwaway SQLite DB directly from schema.sql, never writes
|
|
to the live ``router.db``, and asserts on *actual queried aggregates* rather
|
|
than mock-call assertions (to defeat ``misleading_success_output``).
|
|
|
|
Functions tested:
|
|
- quota_burn
|
|
- scoring_coverage
|
|
- recent_decisions
|
|
- per_model
|
|
- verdict_mix
|
|
- top_proficiency
|
|
|
|
Also verifies that ``import dispatcher`` and ``/health`` still work after the
|
|
move, and that ``import metrics`` alone succeeds (no circular import).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from starlette.testclient import TestClient
|
|
|
|
import dispatcher
|
|
from config import load_config
|
|
from metrics import (
|
|
quota_burn,
|
|
scoring_coverage,
|
|
recent_decisions,
|
|
per_model,
|
|
verdict_mix,
|
|
top_proficiency,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMA_SQL = (ROOT / "schema.sql").read_text()
|
|
CFG = load_config(str(ROOT / "config.yaml"))
|
|
|
|
|
|
# =============================================================================
|
|
# Helpers
|
|
# =============================================================================
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _make_db(tmp_path: Path, extra_sql: str = "") -> sqlite3.Connection:
|
|
"""Create a clean DB seeded from schema.sql, returning a Row-backed conn."""
|
|
conn = sqlite3.connect(str(tmp_path / "test.db"))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL + extra_sql)
|
|
return conn
|
|
|
|
|
|
def _seed_models(conn: sqlite3.Connection) -> None:
|
|
"""Insert routable model rows into an (empty) DB."""
|
|
for model_id, tier, context, cost, vision in (
|
|
("cheap", 2, 262128, 0.30, 1),
|
|
("dear", 2, 262128, 9.00, 0),
|
|
("tiny", 1, 131072, 0.10, 1),
|
|
):
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO models (
|
|
model_id, provider, base_model_id, tier, context_window,
|
|
effective_context_window, max_output_tokens,
|
|
cost_per_1m_prompt, cost_per_1m_completion,
|
|
supports_vision, supports_json_mode,
|
|
latency_class, reasoning_mode, context_variant,
|
|
access_level, availability, last_updated
|
|
) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?,
|
|
?, 1, 'standard', 'default', 'full', 'public', 'active',
|
|
'2026-08-22T00:00:00+00:00')
|
|
""",
|
|
(model_id, model_id, tier, context, cost, cost / 3, vision),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_proficiency(conn: sqlite3.Connection) -> None:
|
|
"""Insert proficiency rows for the seeded models."""
|
|
for model_id, score in (
|
|
("cheap", 0.90),
|
|
("dear", 0.95),
|
|
("tiny", 0.70),
|
|
):
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO proficiency (
|
|
model_id, provider, category, blended_score, source, last_updated
|
|
) VALUES (?, 'neuralwatt', 'coding_general', ?, 'self_eval_thin', '2026-01-01T00:00:00+00:00')
|
|
""",
|
|
(model_id, score),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def _seed_energy(conn: sqlite3.Connection) -> None:
|
|
now = _now()
|
|
recent_rows = [
|
|
("cheap", 5.0e-05, 100, 0.25), # 2 days ago
|
|
("cheap", 3.0e-05, 200, 0.50), # 2 days ago
|
|
("dear", 1.0e-04, 150, 0.75), # 5 days ago
|
|
("tiny", 1.0e-05, 50, 1.00), # 10 days ago
|
|
]
|
|
for (model_id, kwh, tokens, attr) in recent_rows:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO energy_observations (
|
|
model_id, provider, task_category, prompt_tokens,
|
|
completion_tokens, energy_kwh, attribution_ratio,
|
|
observed_at
|
|
) VALUES (?, 'neuralwatt', 'coding_general', 1000, ?, ?, ?, ?)
|
|
""",
|
|
(model_id, tokens, kwh, attr, (now - timedelta(days=2)).isoformat()),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# --- Import / no-circular-import smoke tests -----------------------------------
|
|
|
|
|
|
def test_metrics_can_be_imported_alone():
|
|
"""metrics.py must not require dispatcher — it *is* the cycle-breaker."""
|
|
# If this import raises ImportError (circular), we fail.
|
|
import metrics # noqa: F401
|
|
|
|
|
|
def test_dispatcher_imports_after_metrics():
|
|
"""importing metrics first, then dispatcher, must not raise."""
|
|
# This test runs *after* metrics has already been imported above.
|
|
# The import chain is: dispatcher → metrics (one-way).
|
|
assert hasattr(dispatcher, "app")
|
|
|
|
|
|
# --- quota_burn tests ---------------------------------------------------------
|
|
|
|
|
|
def test_quota_burn_returns_none_when_no_plan(tmp_path):
|
|
"""When plan_kwh_per_period is falsy, quota_burn returns None."""
|
|
conn = _make_db(tmp_path)
|
|
no_plan_cfg = SimpleNamespace(
|
|
objective=SimpleNamespace(plan_kwh_per_period=None)
|
|
)
|
|
assert quota_burn(conn, no_plan_cfg) is None
|
|
|
|
|
|
def test_quota_burn_returns_none_when_plan_is_zero(tmp_path):
|
|
"""plan_kwh_per_period == 0 is treated the same as None."""
|
|
conn = _make_db(tmp_path)
|
|
zero_plan_cfg = SimpleNamespace(
|
|
objective=SimpleNamespace(plan_kwh_per_period=0)
|
|
)
|
|
assert quota_burn(conn, zero_plan_cfg) is None
|
|
|
|
|
|
def test_quota_burn_aggregates_last_30_days(tmp_path):
|
|
"""Two recent rows and one old row — only the recent ones count."""
|
|
cfg = SimpleNamespace(
|
|
objective=SimpleNamespace(plan_kwh_per_period=6.25)
|
|
)
|
|
conn = _make_db(tmp_path)
|
|
now = _now()
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, energy_kwh, completion_tokens, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 0.10, 100, ?)",
|
|
((now - timedelta(days=1)).isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, energy_kwh, completion_tokens, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 0.15, 200, ?)",
|
|
((now - timedelta(days=5)).isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, energy_kwh, completion_tokens, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 0.90, 300, ?)",
|
|
((now - timedelta(days=60)).isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
result = quota_burn(conn, cfg)
|
|
assert result is not None
|
|
assert result["metered_kwh_30d"] == pytest.approx(0.25)
|
|
assert result["metered_calls_30d"] == 2
|
|
assert result["plan_kwh"] == 6.25
|
|
assert result["metered_fraction_of_plan"] == pytest.approx(0.25 / 6.25)
|
|
|
|
|
|
def test_quota_burn_empty_db(tmp_path):
|
|
"""Zero rows → kwh=0, calls=0, not an error."""
|
|
cfg = SimpleNamespace(
|
|
objective=SimpleNamespace(plan_kwh_per_period=1.0)
|
|
)
|
|
conn = _make_db(tmp_path)
|
|
result = quota_burn(conn, cfg)
|
|
assert result["metered_kwh_30d"] == 0.0
|
|
assert result["metered_calls_30d"] == 0
|
|
|
|
|
|
# --- scoring_coverage tests ---------------------------------------------------
|
|
|
|
|
|
def test_scoring_coverage_has_all_keys(tmp_path):
|
|
"""The return dict always has these keys, even when empty."""
|
|
conn = _make_db(tmp_path)
|
|
# Seed routable models
|
|
_seed_models(conn)
|
|
result = scoring_coverage(conn, CFG)
|
|
assert "routable_models" in result
|
|
assert "with_energy_data" in result
|
|
assert "with_proficiency_data" in result
|
|
assert "quota" in result
|
|
assert "warnings" in result
|
|
|
|
|
|
def test_scoring_coverage_warning_when_no_energy(tmp_path):
|
|
"""Models with no seed_reference observations produce a warning."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
# No energy_observations rows at all
|
|
result = scoring_coverage(conn, CFG)
|
|
warning_texts = result["warnings"]
|
|
assert any("no reference-workload observations" in w for w in warning_texts)
|
|
assert result["with_energy_data"] == 0
|
|
|
|
|
|
def test_scoring_coverage_no_warning_when_full_coverage(tmp_path):
|
|
"""When every routable model has energy + proficiency, no warnings."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
# Seed SEED_CATEGORY energy observations
|
|
now = _now()
|
|
for model in ["cheap", "dear"]:
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, task_category, prompt_tokens, "
|
|
"completion_tokens, energy_kwh, attribution_ratio, observed_at) "
|
|
"VALUES (?, 'neuralwatt', 'seed_reference', 1000, 100, 0.001, 0.25, ?)",
|
|
(model, now.isoformat()),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO proficiency "
|
|
"(model_id, provider, category, blended_score, source, last_updated) "
|
|
"VALUES (?, 'neuralwatt', 'coding_general', 0.9, 'self_eval', ?)",
|
|
(model, now.isoformat()),
|
|
)
|
|
conn.commit()
|
|
result = scoring_coverage(conn, CFG)
|
|
# 'tiny' has no data so we expect warnings. Let's also add 'tiny'.
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, task_category, prompt_tokens, "
|
|
"completion_tokens, energy_kwh, attribution_ratio, observed_at) "
|
|
"VALUES ('tiny', 'neuralwatt', 'seed_reference', 1000, 100, 0.001, 0.25, ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO proficiency "
|
|
"(model_id, provider, category, blended_score, source, last_updated) "
|
|
"VALUES ('tiny', 'neuralwatt', 'coding_general', 0.7, 'self_eval', ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
conn.commit()
|
|
result = scoring_coverage(conn, CFG)
|
|
assert result["with_energy_data"] == 3
|
|
assert result["with_proficiency_data"] == 3
|
|
assert len(result["warnings"]) == 0
|
|
|
|
|
|
def test_scoring_coverage_empty_db(tmp_path):
|
|
"""Empty DB: 0 routable, no warnings, quota=None."""
|
|
conn = _make_db(tmp_path)
|
|
result = scoring_coverage(conn, CFG)
|
|
assert result["routable_models"] == 0
|
|
assert result["with_energy_data"] == 0
|
|
assert result["with_proficiency_data"] == 0
|
|
# quota returns None because plan_kwh_per_period may be None in default cfg
|
|
# (it is 6.25 by default, but let's just check the structure)
|
|
assert result["warnings"] == []
|
|
|
|
|
|
# --- recent_decisions tests ---------------------------------------------------
|
|
|
|
|
|
def test_recent_decisions_returns_rows_in_desc_order(tmp_path, monkeypatch):
|
|
"""Rows come back ordered by id DESC, and selected_provider is included."""
|
|
conn = _make_db(tmp_path)
|
|
# Seed two models
|
|
_seed_models(conn)
|
|
|
|
# Create fake dispatcher state to use TestClient, but we'll insert
|
|
# route_decisions rows directly and call recent_decisions(conn).
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO route_decisions (
|
|
observed_at, kind, task_category, task_tier, required_context_tokens,
|
|
confidence, classifier_ms, classification_source, latency_tolerance,
|
|
candidates_considered, selected_model, selected_provider,
|
|
runner_up_models, est_cost_usd, est_proficiency,
|
|
session_key, tools, images, json_mode, streamed
|
|
) VALUES (?, 'route', 'coding_general', 2, 100, 0.95, 200,
|
|
'classifier', 'interactive', 5, 'cheap', 'neuralwatt',
|
|
'[{"model_id":"dear","provider":"neuralwatt"}]',
|
|
0.001, 0.9, 'abc123', 0, 0, 0, 0)
|
|
""",
|
|
(datetime.now(timezone.utc).isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO route_decisions (
|
|
observed_at, kind, task_category, task_tier, required_context_tokens,
|
|
confidence, classifier_ms, classification_source, latency_tolerance,
|
|
candidates_considered, selected_model, selected_provider,
|
|
runner_up_models, est_cost_usd, est_proficiency,
|
|
session_key, tools, images, json_mode, streamed
|
|
) VALUES (?, 'route', 'docs_writing', 1, 50, 0.88, 150,
|
|
'classifier', 'interactive', 3, 'dear', 'neuralwatt',
|
|
NULL, 0.005, 0.95, 'def456', 0, 0, 0, 0)
|
|
""",
|
|
(datetime.now(timezone.utc).isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
rows = recent_decisions(conn, limit=50)
|
|
assert len(rows) == 2
|
|
# DESC order: dear (id=2) first
|
|
assert rows[0]["selected_model"] == "dear"
|
|
assert rows[0]["kind"] == "route"
|
|
assert rows[0]["selected_provider"] == "neuralwatt"
|
|
assert rows[1]["selected_model"] == "cheap"
|
|
|
|
|
|
def test_recent_decisions_respects_limit(tmp_path):
|
|
"""limit=1 should return only one row regardless of DB content."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
now = _now().isoformat()
|
|
for i in range(5):
|
|
conn.execute(
|
|
"INSERT INTO route_decisions "
|
|
"(observed_at, kind, selected_model, selected_provider) "
|
|
"VALUES (?, 'route', 'm', 'neuralwatt')",
|
|
(now,),
|
|
)
|
|
conn.commit()
|
|
rows = recent_decisions(conn, limit=1)
|
|
assert len(rows) == 1
|
|
|
|
|
|
def test_recent_decisions_empty_db(tmp_path):
|
|
"""No rows: returns an empty list, not an error."""
|
|
conn = _make_db(tmp_path)
|
|
assert recent_decisions(conn) == []
|
|
|
|
|
|
# --- per_model tests ----------------------------------------------------------
|
|
|
|
|
|
def test_per_model_aggregates_correctly(tmp_path):
|
|
"""Sum/cost/energy/carbon/tokens match hand-computed values."""
|
|
conn = _make_db(tmp_path)
|
|
now = _now()
|
|
rows = [
|
|
("cheap", 0.001, 5.0e-05, 2.4e-03, 100, 0.25),
|
|
("cheap", 0.002, 3.0e-05, 1.2e-03, 200, 0.50),
|
|
("dear", 0.010, 1.0e-04, 5.0e-03, 150, 0.75),
|
|
]
|
|
for (model_id, cost, kwh, carbon, tokens, attr) in rows:
|
|
conn.execute(
|
|
"INSERT INTO energy_observations "
|
|
"(model_id, provider, cost_usd, energy_kwh, carbon_g_co2eq, "
|
|
"completion_tokens, attribution_ratio, observed_at) "
|
|
"VALUES (?, 'neuralwatt', ?, ?, ?, ?, ?, ?)",
|
|
(model_id, cost, kwh, carbon, tokens, attr, now.isoformat()),
|
|
)
|
|
conn.commit()
|
|
|
|
results = per_model(conn)
|
|
by_model = {r["model_id"]: r for r in results}
|
|
|
|
cheap = by_model["cheap"]
|
|
assert cheap["calls"] == 2
|
|
assert cheap["sum_cost_usd"] == pytest.approx(0.003)
|
|
assert cheap["sum_energy_kwh"] == pytest.approx(8.0e-05)
|
|
assert cheap["sum_carbon_g_co2eq"] == pytest.approx(3.6e-03)
|
|
assert cheap["avg_completion_tokens"] == pytest.approx(150.0)
|
|
assert cheap["avg_attribution_ratio"] == pytest.approx(0.375)
|
|
|
|
dear = by_model["dear"]
|
|
assert dear["calls"] == 1
|
|
assert dear["sum_cost_usd"] == pytest.approx(0.010)
|
|
|
|
|
|
def test_per_model_empty_db(tmp_path):
|
|
"""No energy rows: returns empty list."""
|
|
conn = _make_db(tmp_path)
|
|
assert per_model(conn) == []
|
|
|
|
|
|
# --- verdict_mix tests --------------------------------------------------------
|
|
|
|
|
|
def test_verdict_mix_counts_by_verdict(tmp_path):
|
|
"""Counts match the inserted rows."""
|
|
conn = _make_db(tmp_path)
|
|
now = _now()
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 'structural', 'ok', ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 'structural', 'ok', ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 'local_llm', 'malformed', ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 'structural', 'unverifiable', ?)",
|
|
(now.isoformat(),),
|
|
)
|
|
# Stale row outside the window
|
|
conn.execute(
|
|
"INSERT INTO verifications (model_id, provider, kind, verdict, observed_at) "
|
|
"VALUES ('m', 'neuralwatt', 'structural', 'truncated', ?)",
|
|
((now - timedelta(days=30)).isoformat(),),
|
|
)
|
|
conn.commit()
|
|
|
|
result = verdict_mix(conn, since_days=7)
|
|
assert result["ok"] == 2
|
|
assert result["malformed"] == 1
|
|
assert result["unverifiable"] == 1
|
|
# truncated is > 7 days ago, so excluded
|
|
assert "truncated" not in result or result["truncated"] == 0
|
|
|
|
|
|
def test_verdict_mix_empty_db(tmp_path):
|
|
"""No rows: returns empty dict."""
|
|
conn = _make_db(tmp_path)
|
|
assert verdict_mix(conn) == {}
|
|
|
|
|
|
# --- top_proficiency tests ----------------------------------------------------
|
|
|
|
|
|
def test_top_proficiency_ordered_correctly(tmp_path):
|
|
"""Models are returned ordered by blended_score DESC."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
_seed_proficiency(conn)
|
|
conn.commit()
|
|
|
|
results = top_proficiency(conn, "coding_general")
|
|
assert len(results) == 3
|
|
assert results[0]["model_id"] == "dear" # 0.95
|
|
assert results[1]["model_id"] == "cheap" # 0.90
|
|
assert results[2]["model_id"] == "tiny" # 0.70
|
|
|
|
|
|
def test_top_proficiency_filter_by_category(tmp_path):
|
|
"""Requests for one category exclude models that have scores only for another."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
# Only code categories
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO proficiency (
|
|
model_id, provider, category, blended_score, source, last_updated
|
|
) VALUES ('cheap', 'neuralwatt', 'coding_general', 0.90, 'self_eval_thin', '2026-01-01T00:00:00+00:00')
|
|
""",
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO proficiency (
|
|
model_id, provider, category, blended_score, source, last_updated
|
|
) VALUES ('cheap', 'neuralwatt', 'docs_writing', 0.85, 'self_eval_thin', '2026-01-01T00:00:00+00:00')
|
|
""",
|
|
)
|
|
conn.commit()
|
|
|
|
coding = top_proficiency(conn, "coding_general")
|
|
docs = top_proficiency(conn, "docs_writing")
|
|
assert len(coding) == 1
|
|
assert coding[0]["model_id"] == "cheap"
|
|
assert len(docs) == 1
|
|
assert docs[0]["model_id"] == "cheap"
|
|
|
|
|
|
def test_top_proficiency_empty_for_missing_category(tmp_path):
|
|
"""No proficiency rows for category → empty list."""
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
assert top_proficiency(conn, "nonexistent_category") == []
|
|
|
|
|
|
# --- /health endpoint compatibility -------------------------------------------
|
|
|
|
|
|
def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch):
|
|
"""/health still returns the same SHAPE after moving functions to metrics."""
|
|
db_path = tmp_path / "test.db"
|
|
conn = _make_db(tmp_path)
|
|
_seed_models(conn)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
|
|
# Disable local verification to avoid Ollama dependency
|
|
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
|
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
|
|
|
with TestClient(dispatcher.app) as client:
|
|
resp = client.get("/health")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert "scoring" in data
|
|
assert "routable_models" in data["scoring"]
|
|
assert "with_energy_data" in data["scoring"]
|
|
assert "with_proficiency_data" in data["scoring"]
|
|
assert "quota" in data["scoring"]
|
|
assert "warnings" in data["scoring"]
|