368 lines
14 KiB
Python
368 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from starlette.testclient import TestClient
|
|
|
|
import dispatcher
|
|
from admin import ensure_admin_tables
|
|
from watchdog_store import ensure_watchdog_tables
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text()
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path, monkeypatch):
|
|
db_path = tmp_path / "wd.db"
|
|
conn = sqlite3.connect(db_path)
|
|
conn.executescript(SCHEMA_SQL)
|
|
ensure_watchdog_tables(conn)
|
|
conn.close()
|
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
|
|
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
|
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
|
with TestClient(dispatcher.app) as c:
|
|
yield c
|
|
|
|
|
|
def test_watchdog_status_empty_db(client):
|
|
resp = client.get("/admin/api/watchdog/status")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["last_tick"] is None
|
|
assert data["open_alerts"] == 0
|
|
|
|
|
|
def test_watchdog_loops_empty_db(client):
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert "alerts" in data
|
|
assert "resolved" in data
|
|
assert data["alerts"] == []
|
|
assert data["resolved"] == []
|
|
|
|
|
|
def _seed_verdict(conn, tick_id, session_id, session_root, agent, model_id,
|
|
flagged, calls, cost, created_at):
|
|
conn.execute(
|
|
"""INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent,
|
|
model_id, provider, flagged, dup, top, top_what, landed, slow, coverage,
|
|
calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at)
|
|
VALUES (?, ?, ?, ?, ?, 'neuralwatt', ?, 0.9, 1, 'code', 1, 0, 0.8,
|
|
?, ?, 'no', ?)""",
|
|
(tick_id, session_id, session_root, agent, model_id,
|
|
flagged, calls, cost, created_at),
|
|
)
|
|
|
|
|
|
def test_watchdog_loops_with_verdict_evidence(client):
|
|
"""Alert joined with latest FLAGGED verdict evidence."""
|
|
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
conn.execute(
|
|
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
|
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
|
|
)
|
|
conn.execute(
|
|
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
|
opened_at, last_fired_at, resolved_at)
|
|
VALUES ('opencode-loop:root-abc', 'open', 'warning', 1,
|
|
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
|
|
)
|
|
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-1", "model-a",
|
|
0, 5, 0.5, "2026-09-26T10:00:00+00:00")
|
|
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-2", "model-b",
|
|
1, 10, 1.5, "2026-09-26T10:10:00+00:00")
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["alerts"]) == 1
|
|
|
|
alert = data["alerts"][0]
|
|
assert alert["title"] == "agent-2"
|
|
assert alert["model_id"] == "model-b"
|
|
assert alert["provider"] == "neuralwatt"
|
|
assert alert["session_root"] == "root-abc"
|
|
assert alert["calls_since_landed"] == 10
|
|
assert alert["cost_since_landed_usd"] == 1.5
|
|
assert alert["agent_slug"] is None
|
|
|
|
|
|
def test_watchdog_loops_alert_no_matching_verdict(client):
|
|
"""Alert with no matching verdict returns NULL evidence (no error)."""
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert "alerts" in data
|
|
assert "resolved" in data
|
|
|
|
|
|
def test_watchdog_loops_resolved_last_24h(client):
|
|
"""Resolved alert within 24h appears in resolved list."""
|
|
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
resolved_time = (datetime.now(timezone.utc) - timedelta(hours=6)).isoformat()
|
|
conn.execute(
|
|
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
|
opened_at, last_fired_at, resolved_at)
|
|
VALUES ('opencode-loop:root-xyz', 'resolved', 'critical', 3,
|
|
'2026-09-26T04:00:00+00:00', '2026-09-26T08:00:00+00:00', ?)""",
|
|
(resolved_time,),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["resolved"]) == 1
|
|
assert data["resolved"][0]["dedup_key"] == "opencode-loop:root-xyz"
|
|
|
|
|
|
def test_watchdog_loops_resolved_older_than_24h(client):
|
|
"""Resolved alert older than 24h does NOT appear in resolved list."""
|
|
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
resolved_time = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat()
|
|
conn.execute(
|
|
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
|
opened_at, last_fired_at, resolved_at)
|
|
VALUES ('opencode-loop:root-old', 'resolved', 'critical', 3,
|
|
'2026-09-23T04:00:00+00:00', '2026-09-23T08:00:00+00:00', ?)""",
|
|
(resolved_time,),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["resolved"]) == 0
|
|
|
|
|
|
def test_watchdog_loops_agent_slug_from_route_decisions(client):
|
|
"""Agent slug comes from route_decisions.agent, not watchdog_verdicts."""
|
|
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
conn.execute(
|
|
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
|
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
|
|
)
|
|
conn.execute(
|
|
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
|
opened_at, last_fired_at, resolved_at)
|
|
VALUES ('opencode-loop:root-ads', 'open', 'warning', 1,
|
|
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
|
|
)
|
|
_seed_verdict(conn, 1, "sess-ads", "root-ads", "test-agent", "model-c",
|
|
1, 3, 0.25, "2026-09-26T10:00:00+00:00")
|
|
conn.execute(
|
|
"""INSERT INTO route_decisions (observed_at, kind, task_category, task_tier,
|
|
selected_model, selected_provider, session_key, agent, streamed)
|
|
VALUES ('2026-09-26T10:01:00+00:00', 'dispatch', 'coding_refactor', 2,
|
|
'model-c', 'neuralwatt', 'c:sess-ads', 'opencode-agent', 0)"""
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
resp = client.get("/admin/api/watchdog/loops")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["alerts"]) == 1
|
|
assert data["alerts"][0]["agent_slug"] == "opencode-agent"
|
|
assert data["alerts"][0]["title"] == "test-agent"
|
|
|
|
|
|
def test_watchdog_channels_empty_db(client):
|
|
resp = client.get("/admin/api/watchdog/channels")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
def test_watchdog_channels_create(client):
|
|
resp = client.post(
|
|
"/admin/api/watchdog/channels",
|
|
json={"channel_name": "desktop", "enabled": True, "min_severity": "warning"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["ok"] is True
|
|
listing = client.get("/admin/api/watchdog/channels").json()
|
|
assert len(listing) == 1
|
|
assert listing[0]["channel_name"] == "desktop"
|
|
assert listing[0]["enabled"] == 1
|
|
|
|
|
|
def test_watchdog_channels_update_min_severity(client):
|
|
client.post(
|
|
"/admin/api/watchdog/channels",
|
|
json={"channel_name": "desktop", "enabled": True, "min_severity": "warning"},
|
|
)
|
|
client.post(
|
|
"/admin/api/watchdog/channels",
|
|
json={"channel_name": "desktop", "min_severity": "critical"},
|
|
)
|
|
listing = client.get("/admin/api/watchdog/channels").json()
|
|
assert listing[0]["min_severity"] == "critical"
|
|
assert listing[0]["enabled"] == 1
|
|
|
|
|
|
def test_watchdog_channels_requires_name(client):
|
|
resp = client.post("/admin/api/watchdog/channels", json={})
|
|
assert resp.status_code == 422
|
|
|
|
|
|
def test_watchdog_channels_invalid_severity(client):
|
|
resp = client.post(
|
|
"/admin/api/watchdog/channels",
|
|
json={"channel_name": "x", "min_severity": "bogus"},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
|
|
def test_watchdog_test_alert_no_channels(client):
|
|
resp = client.post(
|
|
"/admin/api/watchdog/test-alert",
|
|
json={"severity": "warning"},
|
|
)
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["sent"] == 0
|
|
|
|
|
|
def test_watchdog_test_alert_invalid_severity(client):
|
|
resp = client.post(
|
|
"/admin/api/watchdog/test-alert",
|
|
json={"severity": "bogus"},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
|
|
def test_watchdog_status_after_tick(tmp_path, monkeypatch):
|
|
import sqlite3 as _sqlite3
|
|
db_path = tmp_path / "wd_ticks.db"
|
|
conn = _sqlite3.connect(db_path)
|
|
conn.row_factory = _sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
ensure_watchdog_tables(conn)
|
|
conn.execute(
|
|
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) VALUES (?, ?, ?)",
|
|
("2026-09-25T10:00:00", 5, "ok"),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
|
|
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
|
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
|
with TestClient(dispatcher.app) as c:
|
|
resp = c.get("/admin/api/watchdog/status")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["last_tick"] is not None
|
|
assert data["last_tick"]["sessions_seen"] == 5
|
|
|
|
|
|
def test_watchdog_rollup_empty(client):
|
|
"""Empty watchdog_verdicts returns []."""
|
|
resp = client.get("/admin/api/watchdog/rollup?hours=24")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
def test_watchdog_rollup_invalid_hours(client):
|
|
"""hours=99 returns 422."""
|
|
resp = client.get("/admin/api/watchdog/rollup?hours=99")
|
|
assert resp.status_code == 422
|
|
|
|
|
|
def test_watchdog_rollup_with_data(client):
|
|
"""2 flagged verdicts for same session (costs 1.0 and 3.0), 1 flagged for
|
|
another session same model. Expect stalled_sessions=2, stalled_usd=4.0
|
|
(MAX 3.0 + MAX 1.0)."""
|
|
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
ensure_admin_tables(conn)
|
|
|
|
# Seed a model row
|
|
conn.execute(
|
|
"INSERT INTO models (model_id, provider, base_model_id, display_name, tier, "
|
|
"context_window, effective_context_window, max_output_tokens, "
|
|
"cost_per_1m_prompt, cost_per_1m_completion, access_level, latency_class, "
|
|
"availability, last_updated) "
|
|
"VALUES ('model-a', 'neuralwatt', 'model-a', 'Model A', 2, "
|
|
"128000, 128000, 8192, 0.5, 1.5, 'public', 'interactive', "
|
|
"'active', datetime('now'))"
|
|
)
|
|
|
|
# Need a tick for the FK
|
|
conn.execute(
|
|
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
|
"VALUES (datetime('now'), 5, 'flagged')"
|
|
)
|
|
|
|
# First session (root-1): two verdicts, costs 1.0 and 3.0
|
|
now = datetime.now(timezone.utc)
|
|
conn.execute(
|
|
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
|
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
|
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
|
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
|
|
"1, 0.9, 1, 'code', 1, 0, 0.8, 5, 1.0, 'no', ?)",
|
|
(now.isoformat(),)
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
|
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
|
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
|
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
|
|
"1, 0.9, 1, 'code', 1, 0, 0.8, 10, 3.0, 'no', ?)",
|
|
(now.isoformat(),)
|
|
)
|
|
|
|
# Second session (root-2): one verdict, cost 1.0
|
|
conn.execute(
|
|
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
|
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
|
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
|
"VALUES (1, 'sess-2', 'root-2', 'agent', 'model-a', 'neuralwatt', "
|
|
"1, 0.9, 1, 'code', 1, 0, 0.8, 3, 1.0, 'no', ?)",
|
|
(now.isoformat(),)
|
|
)
|
|
|
|
# Seed a route_decision for traffic_share
|
|
conn.execute(
|
|
"INSERT INTO route_decisions (observed_at, kind, task_category, task_tier, "
|
|
"selected_model, selected_provider, session_key, agent, streamed) "
|
|
"VALUES (?, 'dispatch', 'coding_refactor', 2, "
|
|
"'model-a', 'neuralwatt', 'c:sess-1a', 'opencode-agent', 0)",
|
|
(now.isoformat(),)
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
resp = client.get("/admin/api/watchdog/rollup?hours=24")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data) == 1
|
|
assert data[0]["model_id"] == "model-a"
|
|
assert data[0]["provider"] == "neuralwatt"
|
|
assert data[0]["stalled_sessions"] == 2 # root-1 and root-2
|
|
assert data[0]["stalled_usd"] == 4.0 # MAX(3.0, 1.0) + 1.0
|
|
assert data[0]["traffic_share"] == 1.0 # 1/1 route_decisions
|
|
assert data[0]["effective_availability"] == "active"
|