from __future__ import annotations import sqlite3 from datetime import datetime, timedelta, timezone from pathlib import Path import pytest from starlette.testclient import TestClient import dispatcher from admin import ensure_admin_tables from watchdog_store import ensure_watchdog_tables ROOT = Path(__file__).resolve().parent.parent SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() @pytest.fixture def client(tmp_path, monkeypatch): db_path = tmp_path / "wd.db" conn = sqlite3.connect(db_path) conn.executescript(SCHEMA_SQL) ensure_watchdog_tables(conn) conn.close() monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path)) monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False) monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False) monkeypatch.setenv("NEURALWATT_API_KEY", "test-key") with TestClient(dispatcher.app) as c: yield c def test_watchdog_status_empty_db(client): resp = client.get("/admin/api/watchdog/status") assert resp.status_code == 200 data = resp.json() assert data["last_tick"] is None assert data["open_alerts"] == 0 def test_watchdog_loops_empty_db(client): resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert "alerts" in data assert "resolved" in data assert data["alerts"] == [] assert data["resolved"] == [] def _seed_verdict(conn, tick_id, session_id, session_root, agent, model_id, flagged, calls, cost, created_at): conn.execute( """INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) VALUES (?, ?, ?, ?, ?, 'neuralwatt', ?, 0.9, 1, 'code', 1, 0, 0.8, ?, ?, 'no', ?)""", (tick_id, session_id, session_root, agent, model_id, flagged, calls, cost, created_at), ) def test_watchdog_loops_with_verdict_evidence(client): """Alert joined with latest FLAGGED verdict evidence.""" conn = sqlite3.connect(str(dispatcher.cfg.database.path)) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) conn.execute( "INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) " "VALUES ('2026-09-26T10:00:00', 5, 'ok')" ) conn.execute( """INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks, opened_at, last_fired_at, resolved_at) VALUES ('opencode-loop:root-abc', 'open', 'warning', 1, '2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)""" ) _seed_verdict(conn, 1, "sess-1", "root-abc", "agent-1", "model-a", 0, 5, 0.5, "2026-09-26T10:00:00+00:00") _seed_verdict(conn, 1, "sess-1", "root-abc", "agent-2", "model-b", 1, 10, 1.5, "2026-09-26T10:10:00+00:00") conn.commit() conn.close() resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert len(data["alerts"]) == 1 alert = data["alerts"][0] assert alert["title"] == "agent-2" assert alert["model_id"] == "model-b" assert alert["provider"] == "neuralwatt" assert alert["session_root"] == "root-abc" assert alert["calls_since_landed"] == 10 assert alert["cost_since_landed_usd"] == 1.5 assert alert["agent_slug"] is None def test_watchdog_loops_alert_no_matching_verdict(client): """Alert with no matching verdict returns NULL evidence (no error).""" resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert "alerts" in data assert "resolved" in data def test_watchdog_loops_resolved_last_24h(client): """Resolved alert within 24h appears in resolved list.""" conn = sqlite3.connect(str(dispatcher.cfg.database.path)) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) resolved_time = (datetime.now(timezone.utc) - timedelta(hours=6)).isoformat() conn.execute( """INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks, opened_at, last_fired_at, resolved_at) VALUES ('opencode-loop:root-xyz', 'resolved', 'critical', 3, '2026-09-26T04:00:00+00:00', '2026-09-26T08:00:00+00:00', ?)""", (resolved_time,), ) conn.commit() conn.close() resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert len(data["resolved"]) == 1 assert data["resolved"][0]["dedup_key"] == "opencode-loop:root-xyz" def test_watchdog_loops_resolved_older_than_24h(client): """Resolved alert older than 24h does NOT appear in resolved list.""" conn = sqlite3.connect(str(dispatcher.cfg.database.path)) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) resolved_time = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat() conn.execute( """INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks, opened_at, last_fired_at, resolved_at) VALUES ('opencode-loop:root-old', 'resolved', 'critical', 3, '2026-09-23T04:00:00+00:00', '2026-09-23T08:00:00+00:00', ?)""", (resolved_time,), ) conn.commit() conn.close() resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert len(data["resolved"]) == 0 def test_watchdog_loops_agent_slug_from_route_decisions(client): """Agent slug comes from route_decisions.agent, not watchdog_verdicts.""" conn = sqlite3.connect(str(dispatcher.cfg.database.path)) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) conn.execute( "INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) " "VALUES ('2026-09-26T10:00:00', 5, 'ok')" ) conn.execute( """INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks, opened_at, last_fired_at, resolved_at) VALUES ('opencode-loop:root-ads', 'open', 'warning', 1, '2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)""" ) _seed_verdict(conn, 1, "sess-ads", "root-ads", "test-agent", "model-c", 1, 3, 0.25, "2026-09-26T10:00:00+00:00") conn.execute( """INSERT INTO route_decisions (observed_at, kind, task_category, task_tier, selected_model, selected_provider, session_key, agent, streamed) VALUES ('2026-09-26T10:01:00+00:00', 'dispatch', 'coding_refactor', 2, 'model-c', 'neuralwatt', 'c:sess-ads', 'opencode-agent', 0)""" ) conn.commit() conn.close() resp = client.get("/admin/api/watchdog/loops") assert resp.status_code == 200 data = resp.json() assert len(data["alerts"]) == 1 assert data["alerts"][0]["agent_slug"] == "opencode-agent" assert data["alerts"][0]["title"] == "test-agent" def test_watchdog_channels_empty_db(client): resp = client.get("/admin/api/watchdog/channels") assert resp.status_code == 200 assert resp.json() == [] def test_watchdog_channels_create(client): resp = client.post( "/admin/api/watchdog/channels", json={"channel_name": "desktop", "enabled": True, "min_severity": "warning"}, ) assert resp.status_code == 200 assert resp.json()["ok"] is True listing = client.get("/admin/api/watchdog/channels").json() assert len(listing) == 1 assert listing[0]["channel_name"] == "desktop" assert listing[0]["enabled"] == 1 def test_watchdog_channels_update_min_severity(client): client.post( "/admin/api/watchdog/channels", json={"channel_name": "desktop", "enabled": True, "min_severity": "warning"}, ) client.post( "/admin/api/watchdog/channels", json={"channel_name": "desktop", "min_severity": "critical"}, ) listing = client.get("/admin/api/watchdog/channels").json() assert listing[0]["min_severity"] == "critical" assert listing[0]["enabled"] == 1 def test_watchdog_channels_requires_name(client): resp = client.post("/admin/api/watchdog/channels", json={}) assert resp.status_code == 422 def test_watchdog_channels_invalid_severity(client): resp = client.post( "/admin/api/watchdog/channels", json={"channel_name": "x", "min_severity": "bogus"}, ) assert resp.status_code == 422 def test_watchdog_test_alert_no_channels(client): resp = client.post( "/admin/api/watchdog/test-alert", json={"severity": "warning"}, ) assert resp.status_code == 200 data = resp.json() assert data["sent"] == 0 def test_watchdog_test_alert_invalid_severity(client): resp = client.post( "/admin/api/watchdog/test-alert", json={"severity": "bogus"}, ) assert resp.status_code == 422 def test_watchdog_status_after_tick(tmp_path, monkeypatch): import sqlite3 as _sqlite3 db_path = tmp_path / "wd_ticks.db" conn = _sqlite3.connect(db_path) conn.row_factory = _sqlite3.Row conn.executescript(SCHEMA_SQL) ensure_watchdog_tables(conn) conn.execute( "INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) VALUES (?, ?, ?)", ("2026-09-25T10:00:00", 5, "ok"), ) conn.commit() conn.close() monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path)) monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False) monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False) monkeypatch.setenv("NEURALWATT_API_KEY", "test-key") with TestClient(dispatcher.app) as c: resp = c.get("/admin/api/watchdog/status") assert resp.status_code == 200 data = resp.json() assert data["last_tick"] is not None assert data["last_tick"]["sessions_seen"] == 5 def test_watchdog_rollup_empty(client): """Empty watchdog_verdicts returns [].""" resp = client.get("/admin/api/watchdog/rollup?hours=24") assert resp.status_code == 200 assert resp.json() == [] def test_watchdog_rollup_invalid_hours(client): """hours=99 returns 422.""" resp = client.get("/admin/api/watchdog/rollup?hours=99") assert resp.status_code == 422 def test_watchdog_rollup_with_data(client): """2 flagged verdicts for same session (costs 1.0 and 3.0), 1 flagged for another session same model. Expect stalled_sessions=2, stalled_usd=4.0 (MAX 3.0 + MAX 1.0).""" conn = sqlite3.connect(str(dispatcher.cfg.database.path)) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA_SQL) ensure_admin_tables(conn) # Seed a model row conn.execute( "INSERT INTO models (model_id, provider, base_model_id, display_name, tier, " "context_window, effective_context_window, max_output_tokens, " "cost_per_1m_prompt, cost_per_1m_completion, access_level, latency_class, " "availability, last_updated) " "VALUES ('model-a', 'neuralwatt', 'model-a', 'Model A', 2, " "128000, 128000, 8192, 0.5, 1.5, 'public', 'interactive', " "'active', datetime('now'))" ) # Need a tick for the FK conn.execute( "INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) " "VALUES (datetime('now'), 5, 'flagged')" ) # First session (root-1): two verdicts, costs 1.0 and 3.0 now = datetime.now(timezone.utc) conn.execute( "INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, " "model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, " "calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) " "VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', " "1, 0.9, 1, 'code', 1, 0, 0.8, 5, 1.0, 'no', ?)", (now.isoformat(),) ) conn.execute( "INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, " "model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, " "calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) " "VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', " "1, 0.9, 1, 'code', 1, 0, 0.8, 10, 3.0, 'no', ?)", (now.isoformat(),) ) # Second session (root-2): one verdict, cost 1.0 conn.execute( "INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, " "model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, " "calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) " "VALUES (1, 'sess-2', 'root-2', 'agent', 'model-a', 'neuralwatt', " "1, 0.9, 1, 'code', 1, 0, 0.8, 3, 1.0, 'no', ?)", (now.isoformat(),) ) # Seed a route_decision for traffic_share conn.execute( "INSERT INTO route_decisions (observed_at, kind, task_category, task_tier, " "selected_model, selected_provider, session_key, agent, streamed) " "VALUES (?, 'dispatch', 'coding_refactor', 2, " "'model-a', 'neuralwatt', 'c:sess-1a', 'opencode-agent', 0)", (now.isoformat(),) ) conn.commit() conn.close() resp = client.get("/admin/api/watchdog/rollup?hours=24") assert resp.status_code == 200 data = resp.json() assert len(data) == 1 assert data[0]["model_id"] == "model-a" assert data[0]["provider"] == "neuralwatt" assert data[0]["stalled_sessions"] == 2 # root-1 and root-2 assert data[0]["stalled_usd"] == 4.0 # MAX(3.0, 1.0) + 1.0 assert data[0]["traffic_share"] == 1.0 # 1/1 route_decisions assert data[0]["effective_availability"] == "active"