From f0ebd83a09b01450b0755649ae9c6e903285815b Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 23 Aug 2026 23:38:07 -0400 Subject: [PATCH 01/32] feat(tui): live routing-decisions panel with SSE, detail popup, breakdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a real-time view of actual routing tasks to the TUI dashboard: backend: - events.py: in-memory decision-event broker (pure stdlib, thread-safe). Bounded ring buffer + fan-out queues. persist_route_decision publishes here after each write so the TUI sees decisions without polling. - dispatcher.py: GET /events/decisions SSE endpoint — replays recent decisions then streams live ones with :heartbeat keepalive. Wired into persist_route_decision's write path. tui: - tui.py: DashboardApp now consumes /events/decisions via a background thread (call_from_thread). Decisions table updates live without waiting for the 5s /metrics poll. New columns: id, kind, category, tier, ctx, selected, est $. Number keys 1-6 cycle panels. - tui_screens.py: DecisionDetailScreen modal — press Enter or e on any decision row to see the full JSON (runner-ups, rejected reason, feature flags, confidence, context size). - tui_model.py: pure data layer extracted from tui.py — build_model, build_category_breakdown, decision_row. Testable without a terminal. - tui_sse.py: background-thread SSE consumer with reconnect. 17 new tests (events broker, SSE endpoint, TUI data model, detail popup, live decision handling). 562 total, all passing. lsp_diagnostics clean. --- ...ontext-pruning-and-framing-fixes-review.md | 34 +++ dispatcher.py | 97 +++++++- events.py | 82 +++++++ tests/test_events.py | 83 +++++++ tests/test_metrics_endpoint.py | 60 +++++ tests/test_tui.py | 208 +++++++++++++++- tui.py | 230 ++++++++++-------- tui_model.py | 165 +++++++++++++ tui_screens.py | 71 ++++++ tui_sse.py | 64 +++++ 10 files changed, 979 insertions(+), 115 deletions(-) create mode 100644 events.py create mode 100644 tests/test_events.py create mode 100644 tui_model.py create mode 100644 tui_screens.py create mode 100644 tui_sse.py diff --git a/code_reviews/context-pruning-and-framing-fixes-review.md b/code_reviews/context-pruning-and-framing-fixes-review.md index 17deda5..4aad3bf 100644 --- a/code_reviews/context-pruning-and-framing-fixes-review.md +++ b/code_reviews/context-pruning-and-framing-fixes-review.md @@ -123,3 +123,37 @@ to produce. Route it back for another pass; everything else here is solid and doesn't need to be touched again. The two deferred design items are covered in a separate forward-looking spec rather than this after-the-fact report. + +## Update: the image finding is fixed, committed as `5f7716e` + +Verified directly against the same repro used to find it: a 200,000-char +fake base64 payload in a tool result's `image_url` block now comes out +stubbed to `"[image omitted]"`, the adjacent text block reads a clean +`"[screenshot: result omitted]"` placeholder with no base64 fragment in it, +and `stats["final_tokens"]` (20) is now correctly *below* +`stats["original_tokens"]` (66,687) rather than above it. + +The fix separates sizing from replacement-building: `extract_text` (sizing, +unchanged from the prior fix) still counts `image_url` bytes so an +image-heavy result correctly triggers pruning; a new `_text_only` helper +excludes `image_url` entirely and is what actually gets head/tail-sliced +for the replacement, so no raw base64 can end up in a `text` block. +`_with_text` now also stubs any `image_url` block's `url` to a short marker +whenever a message is trimmed, so the image itself — usually the dominant +contributor to size — actually shrinks rather than passing through +untouched. `tokens_saved` no longer accumulates per-message; it's now +`max(orig_tokens - final_tokens, 0)` over the whole list's real before/after +token estimate, so the stat can't structurally disagree with the sizes it's +computed from again, regardless of what any future branch does inside the +per-message loop. + +Covered by a dedicated regression test, +`test_large_image_tool_result_actually_shrinks_and_stats_are_honest`, which +asserts the shrink, the stat's internal consistency, and that no +`"base64"` substring survives in any text block. Full suite: 545/545. + +One cosmetic-only leftover: `_with_text`'s docstring/comment still says a +second `text` block in the same message's content list "is folded into +`new_text`" — verified directly that it's actually dropped, not folded +(two text blocks in, one comes out). Not worth a fix on its own; real tool +messages don't carry more than one text block in practice. diff --git a/dispatcher.py b/dispatcher.py index e5e457c..2635227 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -37,6 +37,7 @@ from __future__ import annotations import json import os import hashlib +import queue import re import sqlite3 import time @@ -56,6 +57,7 @@ import logs from capabilities import detect_capabilities, iter_image_url_values from config import RouterConfig, load_config from context_prune import extract_text, prune_context +import events from routing import ( BATCH, INTERACTIVE, @@ -906,7 +908,7 @@ def persist_route_decision( # import, a future refactor) must still get the table here or the # INSERT raises no-such-table and the decision is silently lost. ensure_route_decisions(conn) - conn.execute( + cursor = conn.execute( """ INSERT INTO route_decisions ( observed_at, kind, task_category, task_tier, @@ -941,7 +943,39 @@ def persist_route_decision( int(bool(streamed)), ), ) + decision_id = int(cursor.lastrowid) conn.commit() + # Fan the recorded decision out to any live dashboard subscribers. + # The row id becomes the ordering handle the TUI keeps and lets a + # detail popup re-target the exact decision the SSE feed reported. + events.publish_decision( + { + "id": decision_id, + "observed_at": datetime.now(timezone.utc).isoformat(), + "kind": decision_kind, + "task_category": clf.task_category if clf is not None else None, + "task_tier": clf.task_tier if clf is not None else None, + "required_context_tokens": ( + clf.required_context_tokens if clf is not None else None + ), + "confidence": clf.confidence if clf is not None else None, + "classifier_ms": classifier_ms, + "classification_source": classification_source, + "latency_tolerance": derived_latency, + "candidates_considered": candidates, + "selected_model": model, + "selected_provider": provider, + "runner_up_models": runner_json, + "est_cost_usd": est_cost, + "est_proficiency": est_prof, + "rejected_reason": rejected_reason, + "session_key": session_key, + "tools": int(bool(tools)), + "images": int(bool(images)), + "json_mode": int(bool(json_mode)), + "streamed": int(bool(streamed)), + } + ) except Exception as e: # noqa: BLE001 - best-effort must never raise logs.warning( "route_decision_persist", @@ -1251,6 +1285,67 @@ def metrics_endpoint(): conn.close() +# SSE keep-alive cadence and reconnect hint, in seconds. +SSE_HEARTBEAT_SECONDS = 15 +SSE_RETRY_SECONDS = 3 + + +def _drain_queue(subscriber: queue.Queue) -> list[dict[str, Any]]: + """Drain a subscriber queue without blocking; returns what was waiting.""" + drained: list[dict[str, Any]] = [] + while True: + try: + drained.append(subscriber.get_nowait()) + except queue.Empty: + return drained + + +def _decision_event_stream(): + """Yield route decisions as SSE ``data:`` frames, then live ones. + + Replays the broker's recent ring-buffer contents first so a new dashboard + connection immediately sees recent history, then blocks on the subscriber + queue for new decisions. A ``:heartbeat`` comment is emitted on idle so + middleboxes do not drop the connection, and the subscriber is always + unsubscribed on exit. + """ + subscriber = events.subscribe(replay=True) + try: + yield f"retry: {int(SSE_RETRY_SECONDS * 1000)}\n\n" + for decision in _drain_queue(subscriber): + yield f"data: {json.dumps(decision)}\n\n" + while True: + try: + decision = subscriber.get(timeout=SSE_HEARTBEAT_SECONDS) + except queue.Empty: + yield ":heartbeat\n\n" + continue + yield f"data: {json.dumps(decision)}\n\n" + finally: + events.unsubscribe(subscriber) + + +@app.get("/events/decisions") +def events_decisions(): + """Server-sent-events stream of routing decisions for live dashboards. + + The first frames replay recent decisions (so the TUI can populate its + table before any new traffic), then new decisions stream as they are + recorded. Unauthenticated and loopback-bound like ``/metrics``; each + event carries only the fields already on a ``route_decisions`` row — no + conversation text, prompt, or session_dir. + """ + return StreamingResponse( + _decision_event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + class OutcomeReport(BaseModel): """A client telling the router whether an answer actually worked.""" diff --git a/events.py b/events.py new file mode 100644 index 0000000..90b220c --- /dev/null +++ b/events.py @@ -0,0 +1,82 @@ +"""In-memory decision-event broker for the dashboard's live view. + +This module is deliberately small and dependency-free: it keeps a bounded +ring buffer of recent routing decisions and lets asyncio-based subscribers +receive them in near real time. Every decision inserted via +``dispatcher.persist_route_decision`` is also published here, so the TUI can +show a live feed without polling ``/metrics``. + +The broker is intentionally **not** durable: if no dashboard is connected, the +ring is all that survives. The durable source of truth remains the +``route_decisions`` SQLite table; this is just a volatile fan-out helper. +""" + +from __future__ import annotations + +import queue +from collections import deque +from typing import Any, Final + +DEFAULT_BUFFER_SIZE: Final[int] = 100 +DEFAULT_QUEUE_SIZE: Final[int] = 100 + +_buffer: deque[dict[str, Any]] = deque(maxlen=DEFAULT_BUFFER_SIZE) +_subscribers: set[queue.Queue[dict[str, Any]]] = set() + + +def publish_decision(decision: dict[str, Any]) -> None: + """Append ``decision`` to the ring buffer and fan it out to subscribers. + + If a subscriber's queue is full or dead it is silently removed, because + the broker must never fail the caller (a routing decision should never + be slowed or broken by a dashboard consumer). + """ + _buffer.append(decision) + dead: set[queue.Queue[dict[str, Any]]] = set() + for subscriber in _subscribers: + try: + subscriber.put_nowait(decision) + except queue.Full: + dead.add(subscriber) + _subscribers.difference_update(dead) + + +def subscribe( + replay: bool = True, + max_queue: int = DEFAULT_QUEUE_SIZE, +) -> queue.Queue[dict[str, Any]]: + """Create a thread-safe subscriber queue. + + By default the current ring-buffer contents are preloaded so a new + connection immediately sees recent decisions. The caller must arrange to + remove the queue with :func:`unsubscribe` when it disconnects. + """ + new_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=max_queue) + if replay: + for decision in _buffer: + try: + new_queue.put_nowait(decision) + except queue.Full: + break + _subscribers.add(new_queue) + return new_queue + + +def unsubscribe(subscriber: queue.Queue[dict[str, Any]]) -> None: + """Remove ``subscriber`` from the fan-out set.""" + _subscribers.discard(subscriber) + + +def recent_decisions(limit: int = DEFAULT_BUFFER_SIZE) -> list[dict[str, Any]]: + """Return up to ``limit`` items from the ring buffer, newest last.""" + return list(_buffer)[-limit:] + + +def clear() -> None: + """Drop every buffered decision and subscriber. + + Used by tests to ensure a clean broker state; production code should + almost never call this. + """ + _buffer.clear() + _subscribers.clear() diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000..7fc39d1 --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,83 @@ +"""Tests for the in-memory decision-event broker (events.py). + +Offline, no network. Drive the thread-safe queue broker directly: publish, +subscribe-with-replay, unsubscribe, and the interaction where a decision is +published with no subscriber (must not error and must stay in the ring). +""" + +from __future__ import annotations + +import queue + +import pytest + +import events + + +@pytest.fixture(autouse=True) +def _clean_broker(): + events.clear() + yield + events.clear() + + +def _decision(decision_id: int, model: str = "m") -> dict: + return {"id": decision_id, "selected_model": model} + + +def test_publish_adds_to_ring_and_stays_for_replay(): + events.publish_decision(_decision(1)) + events.publish_decision(_decision(2)) + assert [d["id"] for d in events.recent_decisions()] == [1, 2] + + +def test_subscribe_replays_ring_into_new_queue(): + events.publish_decision(_decision(1)) + events.publish_decision(_decision(2)) + sub = events.subscribe(replay=True) + assert [d["id"] for d in _drain(sub)] == [1, 2] + + +def test_subscribe_without_replay_starts_empty(): + events.publish_decision(_decision(1)) + sub = events.subscribe(replay=False) + assert _drain(sub) == [] + + +def test_publish_fans_out_to_live_subscribers(): + sub = events.subscribe() + events.publish_decision(_decision(1)) + assert [d["id"] for d in _drain(sub)] == [1] + + +def test_unsubscribe_stops_delivery(): + sub = events.subscribe() + events.unsubscribe(sub) + events.publish_decision(_decision(1)) + assert _drain(sub) == [] + + +def test_publish_with_no_subscriber_keeps_ring_and_does_not_raise(): + events.publish_decision(_decision(1)) + assert [d["id"] for d in events.recent_decisions()] == [1] + + +def test_full_subscriber_is_dropped_not_blocked(): + events.clear() + # A queue with maxsize 1: the first published decision fills it. + subscriber = events.subscribe(replay=False, max_queue=1) + events.publish_decision(_decision(1)) + events.publish_decision(_decision(2)) # must not raise or block + got = _drain(subscriber) + assert [d["id"] for d in got] == [1] + # The full subscriber was dropped from fan-out; the ring still holds both. + assert len(events.recent_decisions()) == 2 + + +def _drain(q: queue.Queue) -> list: + out = [] + while True: + try: + out.append(q.get_nowait()) + except queue.Empty: + return out diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index 100a093..16c118a 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -3,10 +3,14 @@ Seeds a throwaway temp DB and asserts on the actual JSON returned by a real TestClient GET (never a mock-call assertion), to defeat ``misleading_success_output``. + +Also covers the ``/events/decisions`` Server-Sent Events endpoint, which +streams routing decisions to the TUI without polling. """ from __future__ import annotations +import json import sqlite3 from datetime import datetime, timedelta, timezone from pathlib import Path @@ -15,6 +19,7 @@ import pytest from starlette.testclient import TestClient import dispatcher +import events from config import load_config ROOT = Path(__file__).resolve().parent.parent @@ -199,3 +204,58 @@ def test_metrics_empty_db_returns_200(monkeypatch, tmp_path): assert data["verdict_mix"] == {} assert data["top_proficiency"] == [] assert "generated_at" in data + + +def _sse_frame(line: str) -> dict: + """Parse one ``data: `` SSE line and return the JSON payload.""" + assert line.startswith("data: "), f"unexpected SSE frame: {line!r}" + return json.loads(line[len("data: ") :]) + + +def test_events_decisions_returns_sse_headers(seeded_client, monkeypatch): + """The endpoint announces text/event-stream and no-cache headers. + + The real stream is unbounded (it blocks for heartbeats), so the generator + is stubbed to a bounded one to let TestClient read the whole body without + hanging. The live replay/stream behaviour is covered by the deterministic + generator test below. + """ + events.clear() + try: + + def _bounded(): + yield "retry: 3000\n\n" + yield "data: {\"id\": 9}\n\n" + + monkeypatch.setattr(dispatcher, "_decision_event_stream", _bounded) + resp = seeded_client.get("/events/decisions") + assert resp.status_code == 200 + assert "text/event-stream" in resp.headers["content-type"] + assert "no-cache" in resp.headers.get("cache-control", "") + assert "data: {\"id\": 9}" in resp.text + finally: + events.clear() + + +def test_decision_event_stream_replays_then_streams_live(): + """Drive the generator directly: replay first, then a live publish arrives + as the next frame. No HTTP client or threads, so it is deterministic.""" + events.clear() + try: + events.publish_decision( + {"id": 1, "selected_model": "cheap", "task_category": "coding"} + ) + stream = dispatcher._decision_event_stream() + retry = next(stream) + assert retry.startswith("retry:") + replayed = next(stream) + assert _sse_frame(replayed)["id"] == 1 + + events.publish_decision( + {"id": 2, "selected_model": "tiny", "task_category": "debugging"} + ) + live = next(stream) + assert _sse_frame(live)["id"] == 2 + stream.close() + finally: + events.clear() diff --git a/tests/test_tui.py b/tests/test_tui.py index 52ec584..2e84dde 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -8,7 +8,6 @@ not on pixels. ``App.run_test`` drives the app itself with a stubbed fetcher. This file imports ``tui`` and ``textual`` deliberately — it is the ONE test file that may. No non-tui module imports textual. """ - from __future__ import annotations import asyncio @@ -16,7 +15,8 @@ import asyncio import pytest import tui -from tui import build_model, fetch_metrics +import tui_model +from tui_model import build_model, fetch_metrics def _fixture() -> dict: @@ -187,7 +187,7 @@ def test_fetch_metrics_returns_parsed_dict(monkeypatch): captured["timeout"] = timeout return _FakeResp() - monkeypatch.setattr(tui.requests, "get", _fake_get) + monkeypatch.setattr(tui_model.requests, "get", _fake_get) out = fetch_metrics("http://testhost:8081") assert out == {"quota": None, "ok": True} assert captured["url"] == "http://testhost:8081/metrics" @@ -198,7 +198,7 @@ def test_fetch_metrics_raises_on_http_error(monkeypatch): def raise_for_status(self): raise RuntimeError("500") - monkeypatch.setattr(tui.requests, "get", lambda *a, **k: _Err()) + monkeypatch.setattr(tui_model.requests, "get", lambda *a, **k: _Err()) with pytest.raises(Exception): fetch_metrics("http://x") @@ -207,7 +207,7 @@ def test_fetch_metrics_raises_on_network_error(monkeypatch): def _boom(*a, **k): raise ConnectionError("refused") - monkeypatch.setattr(tui.requests, "get", _boom) + monkeypatch.setattr(tui_model.requests, "get", _boom) with pytest.raises(ConnectionError): fetch_metrics("http://x") @@ -456,8 +456,9 @@ def test_quit_bindings_exit_app(key): ("1", "model-table"), ("2", "verdict-table"), ("3", "decision-table"), - ("4", "quota-panel"), - ("5", "warnings-panel"), + ("4", "category-table"), + ("5", "quota-panel"), + ("6", "warnings-panel"), ], ) def test_number_bindings_focus_panel(key, panel): @@ -483,3 +484,196 @@ def _no_real_network(monkeypatch): raise AssertionError(f"real fetch_metrics called with {base_url!r}") monkeypatch.setattr(tui, "fetch_metrics", _guard) + # Also guard the SSE consumer's requests so a stray DecisionStream never + # reaches the network even if a test forgets to disable live_events. + import tui_sse + + def _sse_guard(*args, **kwargs): + raise AssertionError( + f"real requests.get called from tui_sse with {args!r} {kwargs!r}" + ) + + monkeypatch.setattr(tui_sse.requests, "get", _sse_guard) + + +# -------------------------------------------------------------------------- +# Category breakdown and enriched decision fields (pure data-layer tests). +# -------------------------------------------------------------------------- + + +def test_build_category_breakdown_majority_and_share(): + """One category, two different winners: majority is the most common and + the share is its fraction of the count.""" + decisions = [ + {"id": 3, "category": "coding_general", "tier": 2, "selected": "a"}, + {"id": 2, "category": "coding_general", "tier": 2, "selected": "a"}, + {"id": 1, "category": "coding_general", "tier": 2, "selected": "b"}, + ] + rows = tui_model.build_category_breakdown(decisions) + assert len(rows) == 1 + row = rows[0] + assert row["category"] == "coding_general" + assert row["tier"] == 2 + assert row["count"] == 3 + assert row["majority"] == "a" + assert row["share"] == round(2 / 3, 2) + + +def test_build_category_breakdown_separates_tiers(): + """Same category, different tiers are separate buckets.""" + decisions = [ + {"id": 2, "category": "coding_general", "tier": 1, "selected": "tiny"}, + {"id": 1, "category": "coding_general", "tier": 3, "selected": "big"}, + ] + rows = tui_model.build_category_breakdown(decisions) + assert len(rows) == 2 + tiers = {r["tier"] for r in rows} + assert tiers == {1, 3} + + +def test_build_category_breakdown_handles_empty_and_none_selected(): + """No decisions: empty list. Decisions with no selected model get + majority 'none' and share 1.0 (they all count toward the bucket).""" + assert tui_model.build_category_breakdown([]) == [] + + rows = tui_model.build_category_breakdown( + [{"id": 1, "category": "x", "tier": 1, "selected": None}] + ) + assert rows[0]["majority"] == "none" + + +def test_build_model_recent_decisions_carry_enriched_fields(): + """The enriched /metrics row fields must reach the TUI data model so the + detail popup can render the full decision in full.""" + data = _fixture() + # Ensure the fixture's first row has the fields the new model surfaces. + data["recent_decisions"][0].update( + { + "required_context_tokens": 50000, + "confidence": 0.92, + "classifier_ms": 1800, + "classification_source": "classifier", + "latency_tolerance": "interactive", + "candidates_considered": 8, + "runner_up_models": '[{"model_id":"kimi-k3","provider":"neuralwatt"}]', + "est_proficiency": 0.9, + "rejected_reason": None, + "tools": 0, + "images": 0, + "json_mode": 0, + "streamed": 1, + } + ) + m = build_model(data) + row = m["recent_decisions"][0] + assert row["required_context_tokens"] == 50000 + assert row["confidence"] == 0.92 + assert row["runner_up_models"].startswith("[{") + assert row["streamed"] == 1 + # The breakdown is always present (even an empty list proves the key). + assert "category_breakdown" in m + + +# -------------------------------------------------------------------------- +# Detail popup and live SSE decision handling (App-level tests). +# -------------------------------------------------------------------------- + + +def test_show_decision_detail_pushes_modal_with_full_row(): + """Pressing ``e`` on the decisions table opens a modal whose body contains + the full JSON of the selected row — not just the table columns.""" + stub = _StubFetcher() + stub.payload = _fixture() + stub.payload["recent_decisions"][0].update( + { + "required_context_tokens": 50000, + "confidence": 0.92, + "runner_up_models": '[{"model_id":"kimi-k3"}]', + "rejected_reason": None, + } + ) + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + # Focus the decisions table and move to the first row, then open + # the detail popup via the dedicated binding. + app.query_one("#decision-table").focus() + await pilot.pause() + await pilot.press("e") + await pilot.pause() + # A modal screen is now active and carries the selected decision. + from textual.screen import ModalScreen + + assert isinstance(app.screen, ModalScreen) + decision = app.screen.decision + assert decision["id"] == 42 + assert decision["required_context_tokens"] == 50000 + assert "kimi-k3" in decision["runner_up_models"] + + asyncio.run(_go()) + + +def test_live_decision_inserts_row_at_front_and_rerenders(): + """A decision delivered via the SSE callback is prepended to the model and + re-renders the decisions and category tables without a full re-fetch.""" + stub = _StubFetcher() + stub.payload = _fixture() + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + before = len(app._last_model["recent_decisions"]) + # Simulate the SSE consumer handing in a brand-new decision. + # Same (category, tier) as the fixture's first row so the bucket + # count for coding_general/tier-2 rises to 2. + app._handle_live_decision( + { + "id": 999, + "kind": "chat", + "task_category": "coding_general", + "task_tier": 2, + "selected_model": "deepseek-v4-flash", + "est_cost_usd": 0.0002, + } + ) + await pilot.pause() + after = app._last_model["recent_decisions"] + assert len(after) == before + 1 + assert after[0]["id"] == 999 # prepended, newest-first + # The table was re-rendered: the first row shows the new id. + dt = app.query_one("#decision-table") + first_row_text = " ".join(str(c) for c in dt.get_row_at(0)) + assert "999" in first_row_text + # Category breakdown reflects the new decision. + breakdown = app._last_model["category_breakdown"] + assert any( + r["category"] == "coding_general" and r["count"] >= 2 + for r in breakdown + ) + + asyncio.run(_go()) + + +def test_live_decision_caps_recent_decisions_at_fifty(): + """The live feed never grows the in-memory list past the /metrics cap, so + the dashboard's view stays consistent with a /metrics refresh.""" + stub = _StubFetcher() + # Start with exactly 50 rows so one live addition must evict the oldest. + base = _fixture()["recent_decisions"][0] + stub.payload = {"recent_decisions": [dict(base, id=i) for i in range(50, 0, -1)]} + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + assert len(app._last_model["recent_decisions"]) == 50 + app._handle_live_decision( + {"id": 1, "kind": "chat", "task_category": "x", "task_tier": 1} + ) + await pilot.pause() + assert len(app._last_model["recent_decisions"]) == 50 + + asyncio.run(_go()) diff --git a/tui.py b/tui.py index 9480f42..81c5eb9 100644 --- a/tui.py +++ b/tui.py @@ -25,94 +25,21 @@ from __future__ import annotations import os from typing import Callable, Optional -import requests from textual.app import App, ComposeResult from textual.containers import VerticalScroll from textual.widgets import DataTable, Footer, Header, Static -__all__ = ["fetch_metrics", "build_model", "DashboardApp"] +from tui_model import ( + DEFAULT_BASE_URL, + build_category_breakdown, + build_model, + decision_row, + fetch_metrics, +) +from tui_screens import DecisionDetailScreen +from tui_sse import DecisionStream -DEFAULT_BASE_URL = "http://127.0.0.1:8080" - - -# -------------------------------------------------------------------------- -# Data layer — importable and testable without a running TUI. -# -------------------------------------------------------------------------- - - -def fetch_metrics(base_url: str) -> dict: - """GET ``/metrics`` and return the parsed JSON dict. - - Raises on any HTTP or network error (via ``raise_for_status`` and the - underlying ``requests`` exception) so the caller can catch it and render - a "cannot reach router" state. - """ - url = base_url.rstrip("/") + "/metrics" - resp = requests.get(url, timeout=10) - resp.raise_for_status() - return resp.json() - - -def build_model(data: dict) -> dict: - """Turn raw /metrics JSON into a plain dict of rendered panel payloads. - - Returns keys: ``quota`` (list of {label, value} rows), ``per_model`` - (list of rows), ``verdict_mix`` (list of {verdict, count}), - ``recent_decisions`` (list of rows), ``warnings`` (list of strings). - """ - quota = data.get("quota") - if quota: - quota_rows = [ - {"label": "plan_kwh", "value": quota.get("plan_kwh")}, - {"label": "metered_kwh_30d", "value": quota.get("metered_kwh_30d")}, - { - "label": "fraction", - "value": quota.get("metered_fraction_of_plan"), - }, - {"label": "calls", "value": quota.get("metered_calls_30d")}, - ] - else: - quota_rows = [{"label": "quota", "value": "N/A (plan not set)"}] - - per_model = [ - { - "model": r.get("model_id"), - "calls": r.get("calls"), - "cost_usd": r.get("sum_cost_usd"), - "energy_kwh": r.get("sum_energy_kwh"), - "carbon_g_co2eq": r.get("sum_carbon_g_co2eq"), - } - for r in (data.get("per_model") or []) - ] - - mix = data.get("verdict_mix") or {} - verdict_mix = [ - {"verdict": k, "count": v} for k, v in sorted(mix.items()) - ] - - recent = [] - for r in data.get("recent_decisions") or []: - recent.append( - { - "id": r.get("id"), - "kind": r.get("kind"), - "category": r.get("task_category"), - "tier": r.get("task_tier"), - "selected": r.get("selected_model") or "none", - "est_cost_usd": r.get("est_cost_usd"), - } - ) - - coverage = data.get("coverage") or {} - warnings = list(coverage.get("warnings") or []) - - return { - "quota": quota_rows, - "per_model": per_model, - "verdict_mix": verdict_mix, - "recent_decisions": recent, - "warnings": warnings, - } +__all__ = ["DashboardApp"] # -------------------------------------------------------------------------- @@ -140,11 +67,13 @@ class DashboardApp(App): ("Q", "quit", "Quit"), ("ctrl+c", "quit", "Quit"), ("r", "refresh", "Refresh"), + ("e", "show_decision_detail", "Detail"), ("1", "focus_panel(0)", "Model table"), ("2", "focus_panel(1)", "Verdict table"), ("3", "focus_panel(2)", "Decision table"), - ("4", "focus_panel(3)", "Quota panel"), - ("5", "focus_panel(4)", "Warnings panel"), + ("4", "focus_panel(3)", "Breakdown table"), + ("5", "focus_panel(4)", "Quota panel"), + ("6", "focus_panel(5)", "Warnings panel"), ] CSS = """ @@ -184,6 +113,7 @@ class DashboardApp(App): base_url: Optional[str] = None, fetcher: Optional[Callable[[str], dict]] = None, refresh_seconds: Optional[float] = None, + live_events: bool = False, ) -> None: self.base_url = base_url or os.environ.get( "ROUTER_METRICS_URL", DEFAULT_BASE_URL @@ -192,14 +122,19 @@ class DashboardApp(App): self.refresh_seconds = ( refresh_seconds if refresh_seconds is not None else self.REFRESH_SECONDS ) + # Live events default off so tests don't start a network thread. + # The ``python tui.py`` entrypoint enables it. + self.live_events = live_events + self._event_stream: Optional[DecisionStream] = None self._last_model: Optional[dict] = None self._last_error: Optional[Exception] = None self._refreshing = False - # Ordered list of focusable panels, indexed by the 1-5 number keys. + # Ordered list of focusable panels, indexed by the 1-6 number keys. self._panels = [ "model-table", "verdict-table", "decision-table", + "category-table", "quota-panel", "warnings-panel", ] @@ -216,13 +151,16 @@ class DashboardApp(App): yield DataTable(id="model-table") yield Static("Verdict mix", classes="panel-title") yield DataTable(id="verdict-table") - yield Static("Recent decisions", classes="panel-title") + yield Static("Recent decisions (enter = details)", classes="panel-title") yield DataTable(id="decision-table") + yield Static("Category → model breakdown", classes="panel-title") + yield DataTable(id="category-table") yield Static("Health / warnings", classes="panel-title") yield Static("—", id="warnings-panel") yield Footer() def on_mount(self) -> None: + self._unmounted = False self._set_up_tables() self._refresh() # Poll on the configured interval. set_interval returns a Timer; we @@ -232,8 +170,22 @@ class DashboardApp(App): self._interval_timer = self.set_interval( self.refresh_seconds, self._on_interval ) + if self.live_events: + self._event_stream = DecisionStream( + self.base_url, self._on_live_decision + ) + self._event_stream.start() + + def on_unmount(self) -> None: + self._unmounted = True + if hasattr(self, "_interval_timer"): + self._interval_timer.stop() + if self._event_stream is not None: + self._event_stream.stop() def _on_interval(self) -> None: + if getattr(self, "_unmounted", False): + return self._refresh() def action_refresh(self) -> None: @@ -247,13 +199,56 @@ class DashboardApp(App): verdict_table.add_columns("verdict", "count") decision_table = self.query_one("#decision-table", DataTable) decision_table.add_columns( - "time", "kind", "category", "tier", "selected", "est $" + "id", "kind", "category", "tier", "ctx", "selected", "est $" ) - # The two Static panels are also number-key targets (4 and 5); make - # them focusable so 1-5 focus is uniform. + decision_table.cursor_type = "row" + decision_table.zebra_stripes = True + category_table = self.query_one("#category-table", DataTable) + category_table.add_columns("category", "tier", "count", "majority", "share") + # Static panels are also number-key targets; make them focusable so + # 1-6 focus cycling is uniform. self.query_one("#quota-panel", Static).can_focus = True self.query_one("#warnings-panel", Static).can_focus = True + def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None: + """Open the detail popup for the highlighted recent decision.""" + if event.data_table.id != "decision-table" or not self._last_model: + return + row_index = event.cursor_row + decisions = self._last_model.get("recent_decisions", []) + if row_index is None or row_index < 0 or row_index >= len(decisions): + return + self.push_screen(DecisionDetailScreen(decisions[row_index])) + + def action_show_decision_detail(self) -> None: + """Open details for the currently selected decision (``e`` key).""" + decision_table = self.query_one("#decision-table", DataTable) + if decision_table.cursor_row is None or not self._last_model: + return + row_index = decision_table.cursor_row + decisions = self._last_model.get("recent_decisions", []) + if row_index < 0 or row_index >= len(decisions): + return + self.push_screen(DecisionDetailScreen(decisions[row_index])) + + def _on_live_decision(self, decision: dict) -> None: + """Thread callback: marshal the live decision onto the UI thread.""" + self.call_from_thread(self._handle_live_decision, decision) + + def _handle_live_decision(self, decision: dict) -> None: + """Add a decision from the SSE stream to the front of the model.""" + if self._last_model is None: + return + new_row = decision_row(decision) + self._last_model["recent_decisions"].insert(0, new_row) + # Keep the same cap the backend uses for /metrics consistency. + self._last_model["recent_decisions"] = self._last_model["recent_decisions"][:50] + self._last_model["category_breakdown"] = build_category_breakdown( + self._last_model["recent_decisions"] + ) + self._render_decisions_table() + self._render_category_table() + def _refresh(self) -> None: self._refreshing = True self._set_loading(True) @@ -293,7 +288,7 @@ class DashboardApp(App): panel.add_class("visible") def action_focus_panel(self, index: int) -> None: - """Move focus to one of the numbered panels (1-5).""" + """Move focus to one of the numbered panels (1-6).""" if index < 0 or index >= len(self._panels): return widget = self.query_one(f"#{self._panels[index]}") @@ -331,21 +326,8 @@ class DashboardApp(App): if not model["verdict_mix"]: vt.add_row("(no data)", "") - # Recent decisions - dt = self.query_one("#decision-table", DataTable) - dt.clear() - for r in model["recent_decisions"]: - time_s = str(r["id"]) # id is the stable ordering handle - dt.add_row( - str(time_s), - str(r["kind"]), - str(r["category"]), - str(r["tier"]), - str(r["selected"]), - _fmt_usd(r["est_cost_usd"]), - ) - if not model["recent_decisions"]: - dt.add_row("(no decisions)", "", "", "", "", "") + self._render_decisions_table() + self._render_category_table() # Warnings panel warn_widget = self.query_one("#warnings-panel", Static) @@ -356,9 +338,43 @@ class DashboardApp(App): else: warn_widget.update("No warnings.") + def _render_decisions_table(self) -> None: + if self._last_model is None: + return + dt = self.query_one("#decision-table", DataTable) + dt.clear() + for r in self._last_model["recent_decisions"]: + dt.add_row( + str(r.get("id")), + str(r.get("kind")), + str(r.get("category")), + str(r.get("tier")), + str(r.get("required_context_tokens")), + str(r.get("selected")), + _fmt_usd(r.get("est_cost_usd")), + ) + if not self._last_model["recent_decisions"]: + dt.add_row("(no decisions)", "", "", "", "", "", "") + + def _render_category_table(self) -> None: + if self._last_model is None: + return + ct = self.query_one("#category-table", DataTable) + ct.clear() + for r in self._last_model["category_breakdown"]: + ct.add_row( + str(r["category"]), + str(r["tier"]), + str(r["count"]), + str(r["majority"]), + f"{r['share']:.0%}", + ) + if not self._last_model["category_breakdown"]: + ct.add_row("(no decisions)", "", "", "", "") + def main() -> None: - DashboardApp().run() + DashboardApp(live_events=True).run() if __name__ == "__main__": diff --git a/tui_model.py b/tui_model.py new file mode 100644 index 0000000..0b000e5 --- /dev/null +++ b/tui_model.py @@ -0,0 +1,165 @@ +"""Pure data layer for the LLM Router TUI — no Textual dependency. + +Keeps the payload-shaping logic (``build_model``, ``build_category_breakdown``) +and the HTTP fetcher out of ``tui.py`` so that module stays a thin rendering +shell, and so this layer is importable and testable without a running TUI. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from typing import Any + +import requests + +DEFAULT_BASE_URL = "http://127.0.0.1:8080" + +__all__ = [ + "fetch_metrics", + "build_model", + "build_category_breakdown", + "decision_row", +] + + +def fetch_metrics(base_url: str) -> dict: + """GET ``/metrics`` and return the parsed JSON dict. + + Raises on any HTTP or network error (via ``raise_for_status`` and the + underlying ``requests`` exception) so the caller can catch it and render + a "cannot reach router" state. + """ + url = base_url.rstrip("/") + "/metrics" + resp = requests.get(url, timeout=10) + resp.raise_for_status() + return resp.json() + + +def build_model(data: dict) -> dict: + """Turn raw /metrics JSON into a plain dict of rendered panel payloads. + + Returns keys: ``quota`` (list of {label, value} rows), ``per_model`` + (list of rows), ``verdict_mix`` (list of {verdict, count}), + ``recent_decisions`` (list of rows), ``category_breakdown`` (list of + rows), ``warnings`` (list of strings). + """ + quota = data.get("quota") + if quota: + quota_rows = [ + {"label": "plan_kwh", "value": quota.get("plan_kwh")}, + {"label": "metered_kwh_30d", "value": quota.get("metered_kwh_30d")}, + { + "label": "fraction", + "value": quota.get("metered_fraction_of_plan"), + }, + {"label": "calls", "value": quota.get("metered_calls_30d")}, + ] + else: + quota_rows = [{"label": "quota", "value": "N/A (plan not set)"}] + + per_model = [ + { + "model": r.get("model_id"), + "calls": r.get("calls"), + "cost_usd": r.get("sum_cost_usd"), + "energy_kwh": r.get("sum_energy_kwh"), + "carbon_g_co2eq": r.get("sum_carbon_g_co2eq"), + } + for r in (data.get("per_model") or []) + ] + + mix = data.get("verdict_mix") or {} + verdict_mix = [ + {"verdict": k, "count": v} for k, v in sorted(mix.items()) + ] + + recent = [] + for r in data.get("recent_decisions") or []: + recent.append(decision_row(r)) + + coverage = data.get("coverage") or {} + warnings = list(coverage.get("warnings") or []) + + return { + "quota": quota_rows, + "per_model": per_model, + "verdict_mix": verdict_mix, + "recent_decisions": recent, + "category_breakdown": build_category_breakdown(recent), + "warnings": warnings, + } + + +def decision_row(r: dict) -> dict: + """Project one route_decisions row onto the TUI's enriched decision shape. + + Shared by ``build_model`` (from /metrics) and the live SSE path so the two + never drift in the fields they surface to the detail popup / table. + """ + return { + "id": r.get("id"), + "kind": r.get("kind"), + "category": r.get("task_category"), + "tier": r.get("task_tier"), + "selected": r.get("selected_model") or "none", + "selected_provider": r.get("selected_provider"), + "est_cost_usd": r.get("est_cost_usd"), + "required_context_tokens": r.get("required_context_tokens"), + "confidence": r.get("confidence"), + "classifier_ms": r.get("classifier_ms"), + "classification_source": r.get("classification_source"), + "latency_tolerance": r.get("latency_tolerance"), + "candidates_considered": r.get("candidates_considered"), + "runner_up_models": r.get("runner_up_models"), + "est_proficiency": r.get("est_proficiency"), + "rejected_reason": r.get("rejected_reason"), + "observed_at": r.get("observed_at"), + "tools": r.get("tools"), + "images": r.get("images"), + "json_mode": r.get("json_mode"), + "streamed": r.get("streamed"), + } + + +def build_category_breakdown( + decisions: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Aggregate recent decisions by (category, tier), with the most-common + selected model and its share. + + Returns a list of dicts: ``category``, ``tier``, ``count``, ``majority`` + (the selected model with the most wins), ``share`` (its fraction of the + count as a float in 0..1). Newer rows add first so ties settle toward the + more recent model. This is the panel that answers "what is routing sending + coding_general to right now?" without reading every individual decision. + """ + ordered = sorted( + decisions, key=lambda d: (d.get("id") or 0), reverse=True + ) + buckets: dict[tuple[Any, Any], Counter] = defaultdict(Counter) + counts: dict[tuple[Any, Any], int] = defaultdict(int) + for decision in ordered: + key = (decision.get("category"), decision.get("tier")) + counts[key] += 1 + selected = decision.get("selected") + if selected is not None: + buckets[key][str(selected)] += 1 + + rows = [] + for (category, tier), n in counts.items(): + winners = buckets[(category, tier)] + if winners: + majority, majority_count = winners.most_common(1)[0] + share = majority_count / n + else: + majority, share = None, 0.0 + rows.append( + { + "category": category, + "tier": tier, + "count": n, + "majority": majority if majority is not None else "none", + "share": round(share, 2), + } + ) + return rows diff --git a/tui_screens.py b/tui_screens.py new file mode 100644 index 0000000..9d44c85 --- /dev/null +++ b/tui_screens.py @@ -0,0 +1,71 @@ +"""Modal screens for the LLM Router TUI. + +This module is allowed to import ``textual`` — it is part of the TUI and is +never imported by the service dispatch path. +""" + +from __future__ import annotations + +import json +from typing import Any + +from textual.app import ComposeResult +from textual.containers import Vertical, VerticalScroll +from textual.screen import ModalScreen +from textual.widgets import Button, Static + + +class DecisionDetailScreen(ModalScreen[None]): + """Popup showing the full route_decisions row for one decision. + + Press ``enter``, ``escape`` or ``q`` to close. + """ + + CSS = """ + DecisionDetailScreen { + align: center middle; + } + #detail-container { + width: 80; + height: 36; + border: thick $background 80%; + padding: 1 2; + background: $surface; + } + #detail-title { + text-style: bold; + color: $accent; + height: auto; + margin: 0 0 1 0; + } + #detail-content { + height: 1fr; + width: 1fr; + } + #detail-close { + width: 100%; + margin: 1 0 0 0; + } + """ + + def __init__(self, decision: dict[str, Any]) -> None: + self.decision = decision + super().__init__() + + def compose(self) -> ComposeResult: + with Vertical(id="detail-container"): + yield Static("Decision details", id="detail-title") + with VerticalScroll(id="detail-content"): + yield Static(self._render_text()) + yield Button("Close (enter/esc/q)", id="detail-close") + + def _render_text(self) -> str: + return json.dumps(self.decision, indent=2, default=str) + + def on_key(self, event) -> None: + if event.key in ("escape", "q", "enter"): + self.dismiss(None) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "detail-close": + self.dismiss(None) diff --git a/tui_sse.py b/tui_sse.py new file mode 100644 index 0000000..0d40121 --- /dev/null +++ b/tui_sse.py @@ -0,0 +1,64 @@ +"""Synchronous Server-Sent Events consumer for live routing decisions. + +Runs in its own background thread (a network reader must never block the +Textual event loop) and reconnects after transient failures. It is decoupled +from Textual on purpose: the callback it receives is expected to marshal +itself onto the UI thread (the app passes ``App.call_from_thread``), so this +module never needs to import ``textual``. + +The SSE body is line-based: comments (``:heartbeat``) are ignored and only +``data: `` lines are handed to the callback. +""" + +from __future__ import annotations + +import json +import threading +import time +from typing import Any, Callable + +import requests + +SSE_PATH = "/events/decisions" +STREAM_TIMEOUT = 15 +RECONNECT_SECONDS = 5 + + +class DecisionStream(threading.Thread): + """Background thread that yields routing decisions from the SSE endpoint.""" + + def __init__( + self, + base_url: str, + callback: Callable[[dict[str, Any]], None], + *, + timeout: float = STREAM_TIMEOUT, + reconnect_seconds: float = RECONNECT_SECONDS, + ) -> None: + super().__init__(daemon=True) + self.url = base_url.rstrip("/") + SSE_PATH + self.callback = callback + self.timeout = timeout + self.reconnect_seconds = reconnect_seconds + self._stopped = threading.Event() + + def run(self) -> None: + while not self._stopped.is_set(): + try: + with requests.get( + self.url, stream=True, timeout=self.timeout * 2 + ) as resp: + resp.raise_for_status() + for line in resp.iter_lines(decode_unicode=True): + if self._stopped.is_set(): + break + if line.startswith("data: "): + self.callback(json.loads(line[len("data: ") :])) + except (requests.RequestException, ValueError): + # Transient network / parse error: back off and retry. A + # dashboard must survive a router restart or a dropped SSE. + pass + time.sleep(self.reconnect_seconds) + + def stop(self) -> None: + self._stopped.set() -- 2.49.1 From 765a4a64f3ca3630182aceefeaf1f4b7eeaa5d09 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 23 Aug 2026 23:38:15 -0400 Subject: [PATCH 02/32] docs: sweep for live routing panel + AGENTS.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README.md: update test count (356→562, 27 files), add events.py, tui_model.py, tui_sse.py, tui_screens.py to Modules table, add GET /events/decisions to API Endpoints, describe live SSE feed + detail popup + category breakdown in Monitoring, update textual import note, add new test files to Testing table. - CLAUDE.md: update dispatcher description (SSE endpoint), tui description (live feed, modal, breakdown, tui_model split), test count (562, 27 files), add events.py entry. - AGENTS.md: new agent-facing working guide — stack snapshot, module map with file boundaries and import discipline, conventions, test commands, open items, post-change checklist. --- AGENTS.md | 157 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 25 +++++++-- README.md | 42 +++++++++++---- 3 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..568b657 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,157 @@ +# AGENTS.md — working guide for AI agents on this repo + +The three-tier docs: +- **`README.md`** — what's built, module by module (user-facing). +- **`CLAUDE.md`** — working state + immediate next steps + design rationale + (the one to trust on what is currently true). +- **`design/local-llm-model-router.md`** — architecture and rationale, + including parts still unbuilt. + +This file is the quick-reference for an agent starting work: where things +live, what's safe to touch, and the conventions that aren't obvious from the +code. + +## Stack snapshot + +| Dimension | Value | +|---|---| +| Language | Python 3.10+ (3.10 floor tested; 3.14 also verified) | +| Framework | FastAPI + uvicorn | +| Database | SQLite (`router.db`) | +| Config | `config.yaml` + Pydantic (`config.py`), `extra="forbid"` | +| HTTP client | `requests` (pinned in `requirements.txt`) — do NOT add httpx2/aiohttp without a requirements bump | +| TUI | `textual==8.2.8` — imported only by `tui*.py` modules, never by the dispatch path | +| Testing | `pytest`, 562 tests, all offline (no provider or local-model calls) | +| Dependencies | Pinned. Bump deliberately, never use `>=` | + +## Module map and file boundaries + +### Dispatcher / service (I/O: DB + network) + +| File | Role | Agent notes | +|---|---|---| +| `dispatcher.py` | FastAPI service: routes, calls providers, logs, streams | ~2500 LOC; only edit the specific function you need. **Must not import `metrics`** (circular). Imports `events` for SSE fan-out. | +| `metrics.py` | Read-only aggregations for `/health` and `/metrics` | Takes `(conn, cfg)` args — **must not import `dispatcher`** to avoid circular import. | +| `events.py` | In-memory decision-event broker | Pure stdlib (`queue`, `collections.deque`). Thread-safe. No Textual import. The SSE endpoint lives in `dispatcher.py` and calls `events.subscribe()`/`publish_decision()`. | +| `config.py` | Pydantic models + YAML loader | All config models inherit `StrictModel` (`extra="forbid"`). Unknown keys fail at load. | +| `capabilities.py` | Request-side capability detection (`tools`, `images`, `json_mode`, `reasoning`) | Reads from the OpenAI-format request body, not from the classifier. | +| `context_prune.py` | Relevance-based context pruning (`pinch`) | Ships **disabled** (`pinch.enabled: false`). Imports `PinchConfig` from `config.py` (no circular import: `config.py` doesn't import `context_prune`). | +| `logs.py` | Structured logging (logfmt, journald, ContextVar trace ids) | `logs.bind()` exists for StreamingResponse generators that lose the ContextVar. | + +### Pure scoring/routing modules (no I/O) + +| File | Role | +|---|---| +| `scoring.py` | `normalize_inverted` + weighted composite (quality-first, cost as tiebreak) | +| `routing.py` | Hard filters (`select_candidates`) + ranking (`rank_candidates`) | +| `tiering.py` | Pure tier resolver: 1=cheap+small, 2=mid, 3=frontier | +| `proficiency.py` | Score blending: leaderboard + self-eval → weighted composite | +| `iteration.py` | Retry budget per tier, matching retry to failure kind | + +### TUI modules (import `textual`; never imported by the dispatch path) + +| File | Role | +|---|---| +| `tui.py` | `DashboardApp` — main Textual app with CSS, bindings, compose, render | +| `tui_model.py` | Pure data layer: `build_model`, `build_category_breakdown`, `decision_row` — no Textual import, testable without a terminal | +| `tui_sse.py` | Background-thread SSE consumer (`DecisionStream`): reconnects on failure, marshals decisions to UI thread via `call_from_thread` | +| `tui_screens.py` | `DecisionDetailScreen` — `ModalScreen` showing full decision JSON (Enter or `e` key) | + +### Other entrypoints + +| File | Role | +|---|---| +| `poller.py` | Fetches Neuralwatt catalog, upserts `models` table | +| `seed_energy.py` | Reference workload sweep → `energy_observations` | +| `eval_proficiency.py` | Self-eval harness → `proficiency` table | +| `feedback.py` | Folds verification failures into `proficiency` | +| `tier.py` | DB tiering pass | +| `router_cli.py` | One-shot `/route` probe | +| `proficiency_store.py` | DB write path for `proficiency` | + +## Conventions + +### Code style +- The codebase uses `Optional[X]` (not `X | None`) and `dict` return types + throughout — match the existing style in the file you're editing. +- No `# type: ignore`, no `as any`, no `@ts-ignore` equivalent. +- `except Exception` is acceptable at top-level boundaries with + `# noqa: BLE001` comment (see `dispatcher._refresh`, `persist_route_decision`). +- Config is strict: every knob belongs in `config.yaml`, not only in a + Pydantic default. A default the file never mentions is invisible to a tuner. +- Named constants use `Final` in new code (`events.py`); existing code is + inconsistent — don't refactor just for this. + +### Import discipline +- `metrics.py` **must not import `dispatcher`** (circular import). +- `events.py` imports nothing from the project (pure stdlib). +- `tui_model.py` imports `requests` but **not `textual`** — it's the testable data layer. +- `tui_sse.py` imports `requests` but **not `textual`** — it's a thread worker. +- `tui.py` and `tui_screens.py` import `textual` — that's fine, they're TUI. +- The service dispatch path (`dispatcher.py` → provider call) **never touches + `textual`**. + +### Database +- SQLite, `PRAGMA foreign_keys = ON`. +- `_db()` in `dispatcher.py` returns a `sqlite3.Connection` with + `row_factory = sqlite3.Row`. +- Schema in `schema.sql` uses `CREATE TABLE IF NOT EXISTS` — safe to re-run. +- Code-side table creation (`ensure_route_decisions`, `proficiency_store.ensure_columns`) + mirrors the schema for live DBs that predate a feature. + +### Testing +- All tests are offline. No test calls a provider or local model. +- Pure modules take rows and config as arguments, so they're testable without DB/network. +- Test files that exercise FastAPI use `starlette.testclient.TestClient` with a + temp SQLite DB (`tests/test_metrics_endpoint.py`). +- TUI tests use `App.run_test()` with a stubbed fetcher and a `_no_real_network` + safety-net fixture that guards both `requests.get` and the SSE consumer. +- **SSE tests must not use `TestClient.stream()` on the infinite endpoint** — it + hangs. Test the generator directly (`dispatcher._decision_event_stream()`) + or stub the generator to a bounded one for header/route checks. + +## How to run things + +```bash +# Setup +python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +sqlite3 router.db < schema.sql +cp .env.example .env # fill in NEURALWATT_API_KEY + +# Start the service (binds 127.0.0.1:8080) +python -m uvicorn dispatcher:app --reload +# Or via systemd: +systemctl --user start llm-router.service + +# Run the TUI (service must be running) +python tui.py + +# Run the full test suite +python -m pytest + +# Quick routing probe (no spend) +python router_cli.py "Refactor this Django view" + +# Populate the catalog +python poller.py && python tier.py +``` + +## What's NOT built yet (open items) + +1. **Leaderboard priors unfilled** — `leaderboards.yaml` ships empty. +2. **Three models unsettled** on eco stability (attribution noise). +3. **Retry does not reach streaming** — `POST /outcome` is the answer for streamed traffic. +4. **Local energy not on the ledger** — local classifier/verifier electricity is unmeasured. +5. **Session-directory attribution picks the wrong directory** — heuristic resolves to a dependency's source dir instead of the project being edited. + +See `CLAUDE.md` → "What's NOT built yet — pick up here" for the full list. + +## After a code change + +- Run `python -m pytest` — 562 tests, ~30s. +- If you changed `dispatcher.py`, restart the systemd service: + `systemctl --user restart llm-router.service` (it doesn't auto-reload code). +- If you changed the TUI, run `python tui.py` to verify it starts. +- Check `lsp_diagnostics` on changed files. +- Match the existing commit-message style: `fix:`, `feat:`, `docs:`, `test:`. diff --git a/CLAUDE.md b/CLAUDE.md index e3c9e7c..cbc1c94 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,10 +218,18 @@ rather than from months of history. - `routing.py` — pure hard filters and ranking. - `dispatcher.py` — FastAPI service. `GET /health`, `POST /route` (classify and pick, no provider call), `POST /dispatch` (route, call, log), plus an - OpenAI-compatible `GET /v1/models` and `POST /v1/chat/completions`. + OpenAI-compatible `GET /v1/models` and `POST /v1/chat/completions`, and a + Server-Sent Events `GET /events/decisions` stream that fans every recorded + routing decision out to the monitoring TUI live. - `proficiency.py` / `proficiency_store.py` — pure blending arithmetic, and the only path that writes the `proficiency` table (so `blended_score` and `source` can never drift from the inputs that produced them). +- `events.py` — in-memory decision-event broker (pure stdlib, no Textual): a + bounded ring buffer of recent route decisions plus thread-safe fan-out to + SSE subscribers. `persist_route_decision` publishes here after each write, + so the dashboard sees decisions in near real time without polling. The + durable source of truth stays the `route_decisions` SQLite table; this is + just the volatile live fan-out. - `leaderboards.yaml` / `leaderboard.py` — curated per-family priors and their importer. **Ships empty by design**; `python leaderboard.py --check` names every active family lacking a prior. @@ -249,13 +257,20 @@ rather than from months of history. burn against `plan_kwh_per_period`, scoring coverage warnings, recent `route_decisions`, per-model aggregates over `energy_observations`, verification verdict mix, and top proficiency by category. -- `tui.py` — Textual terminal dashboard over `GET /metrics`. A foreground - entrypoint (`python tui.py`), not a service. `textual` is imported only - here, so the router's dispatch path has no UI dependency. +- `tui.py` — Textual terminal dashboard over `GET /metrics` and + `GET /events/decisions`. A foreground entrypoint (`python tui.py`), not a + service. `textual` is imported only in the TUI modules (`tui.py`, + `tui_screens.py`, `tui_sse.py`), so the router's dispatch path has no UI + dependency. The dashboard has a live routing-decisions feed (via the SSE + endpoint, so new decisions appear without waiting for the 5s `/metrics` + poll), a category → model breakdown panel, and a detail popup (Enter or `e` + on any decision row) showing the full decision JSON. The pure data layer + (`build_model`, `build_category_breakdown`, `decision_row`) is split into + `tui_model.py` so it is testable without a terminal. - `router_cli.py` — one-shot `/route` probe. Posts a task to the running router and prints the decision tree, or emits raw JSON with `--json`. Spends no quota because it only routes. -- `tests/` — 356 tests across 19 files, all passing, all offline. Verified on +- `tests/` — 562 tests across 27 files, all passing, all offline. Verified on Python 3.10 and 3.14; nothing declares `requires-python`, so 3.10 is the tested floor rather than a promised one. diff --git a/README.md b/README.md index 8d138a7..cfa06b1 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ Key behaviors: | **Config** | `config.yaml` loaded & validated by Pydantic (`config.py`) | | **OpenAI Client** | `openai==3.0.0` (official SDK) | | **HTTP** | `requests` for poller, `httpx` (via openai/uvicorn) | -| **Testing** | `pytest` — 356 tests across 19 files, all offline | +| **Testing** | `pytest` — 562 tests across 27 files, all offline | | **Config Files** | `config.yaml`, `leaderboards.yaml`, `evals/tasks.yaml` | | **Deployment** | systemd user units (`.service` + `.timer` files in `deploy/`) | | **Integration** | `opencode.json` in the repo routes through it by default; any OpenAI-compatible client works | @@ -225,7 +225,11 @@ restarts on boot shouldn't change its dependency tree underneath itself. | **`iteration.py`** | Retry budget per tier, and matching the retry to the failure kind | Pure | | **`config.py`** | YAML loader + Pydantic validators (blend weights sum to 1, valid tiers, endpoints separately addressable) | Yes (file) | | **`metrics.py`** | Read-only aggregations for `/health` and `GET /metrics`: quota burn, coverage, recent decisions, per-model totals, verdict mix, top proficiency | Yes (DB) | -| **`tui.py`** | Textual terminal dashboard over `GET /metrics`; foreground tool, not a service | Yes (network) | +| **`events.py`** | In-memory decision-event broker for the TUI's live feed: bounded ring buffer + thread-safe fan-out to SSE subscribers | Pure | +| **`tui.py`** | Textual terminal dashboard over `GET /metrics` and `GET /events/decisions`; live routing feed, detail popup, category breakdown. Foreground tool, not a service | Yes (network) | +| **`tui_model.py`** | Pure data layer for the TUI: `build_model`, `build_category_breakdown`, `decision_row` — no Textual import, testable without a terminal | Pure | +| **`tui_sse.py`** | Background-thread SSE consumer for the TUI: reconnects on failure, marshals live decisions onto the UI thread | Yes (network) | +| **`tui_screens.py`** | Modal screen for the TUI: `DecisionDetailScreen` shows the full decision JSON when the user presses Enter | Pure | | **`router_cli.py`** | One-shot routing probe: POSTs to `/route` and prints the decision tree | Yes (network) | ## Decision Table Schema (SQLite) @@ -595,6 +599,7 @@ allowance. |---|---|---| | `GET` | `/health` | Catalog/reachability status, scoring coverage, warnings | | `GET` | `/metrics` | Aggregated observability JSON: quota burn, coverage, recent decisions, per-model totals, verdict mix, top proficiency; loopback-only, no auth | +| `GET` | `/events/decisions` | Server-Sent Events stream of routing decisions for the TUI's live feed; replays recent decisions, then streams new ones as they happen | | `POST` | `/route` | Classify task, rank candidates, return selected model — **no provider call, no cost** | | `POST` | `/dispatch` | Same as `/route`, plus complete the provider call, stream response, log observation | | `GET` | `/v1/models` | OpenAI-compatible model list (router virtual models + catalog) | @@ -675,7 +680,7 @@ curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application # Monitoring: aggregate router health curl -s localhost:8080/metrics | python -m json.tool -# Terminal dashboard foreground tool (requires `textual`; runs until you press q) +# Terminal dashboard with live routing feed (requires `textual`; runs until you press q) python tui.py # One-shot routing probe with no spend @@ -690,17 +695,28 @@ Three foreground tools read the running router without spending quota: - **`GET /metrics`** — JSON summary of quota, coverage, recent routing decisions, per-model usage, verdict mix, and top proficiency. No auth; loopback only. -- **`tui.py`** — Textual terminal dashboard that polls `/metrics` every few - seconds. Run it in a terminal with the service already up. It is a separate - entrypoint, not a systemd unit. +- **`GET /events/decisions`** — Server-Sent Events stream of routing + decisions for the TUI's live feed. Replays recent decisions, then streams + new ones as they are recorded; `:heartbeat` keepalive comments keep the + connection alive between events. No auth; loopback only. +- **`tui.py`** — Textual terminal dashboard with a **live routing feed** + (via `/events/decisions`, so decisions appear as they happen without + waiting for the 5s `/metrics` poll), a **category → model breakdown** + panel aggregating recent decisions by `(category, tier)` with majority + winner and share, a **detail popup** (press Enter or `e` on any decision + row to see the full JSON — runner-ups, rejected reasons, feature flags, + confidence, context size), and the existing quota / per-model / verdict + / warnings panels. Run it in a terminal with the service already up. It + is a separate entrypoint, not a systemd unit. - **`router_cli.py ""`** — POSTs to `/route` once and prints the full decision tree, including candidates, selected model, estimated cost, and proficiency. Use `--category`, `--tier`, and `--context` to override the classifier deterministically, or `--json` for raw output. -`textual` is pinned in `requirements.txt` solely for `tui.py`. It is imported -only by that module; the FastAPI service dispatch path never touches it, so -the router itself has no UI dependency. +`textual` is pinned in `requirements.txt` solely for the TUI modules +(`tui.py`, `tui_screens.py`, `tui_sse.py`). It is imported only by these +modules; the FastAPI service dispatch path never touches it, so the router +itself has no UI dependency. ## Logging and Traceability @@ -863,7 +879,7 @@ carries the same modality block. ## Testing ```bash -python -m pytest # 356 tests +python -m pytest # 562 tests python -m pytest --cov # with coverage ``` @@ -887,6 +903,12 @@ config as arguments, so the suite runs offline on a clean checkout. | `test_iteration.py` | Retry budget per tier, and matching the retry to the failure kind | | `test_session_identity.py` | Outcome attribution: session matching, ambiguity refusal | | `test_config_endpoints.py` | Classifier and verifier are separately addressable; guards on the split | +| `test_metrics_endpoint.py` | `/metrics` endpoint, SSE `/events/decisions` headers + replay/stream behavior | +| `test_events.py` | Decision-event broker: publish, subscribe/replay, unsubscribe, full-subscriber eviction | +| `test_tui.py` | TUI data model, category breakdown, detail popup, live SSE decision handling, keyboard controls | +| `test_context_prune.py` | Context pruning: image_url handling, structured content, recency guards, stats accuracy | +| `test_classifier_input.py` | Classifier framing: `_previous_context` scope, `_classifier_user_content` framing | +| `test_route_decisions.py` | `route_decisions` table, inline-create helper, config gate | ## Setup -- 2.49.1 From 943b2d5cdebc9a9dc18e8d63c3e38781fdd7449d Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Mon, 24 Aug 2026 00:22:01 -0400 Subject: [PATCH 03/32] fix: resolve TUI live-routing panel review findings --- dispatcher.py | 33 +++++-- events.py | 47 +++++++-- tests/test_events.py | 73 ++++++++++++-- tests/test_metrics_endpoint.py | 4 +- tests/test_tui.py | 174 +++++++++++++++++++++++++++++++-- tui.py | 41 +++++++- tui_sse.py | 16 ++- 7 files changed, 354 insertions(+), 34 deletions(-) diff --git a/dispatcher.py b/dispatcher.py index 2635227..487071d 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -908,6 +908,7 @@ def persist_route_decision( # import, a future refactor) must still get the table here or the # INSERT raises no-such-table and the decision is silently lost. ensure_route_decisions(conn) + observed_at = datetime.now(timezone.utc).isoformat() cursor = conn.execute( """ INSERT INTO route_decisions ( @@ -920,7 +921,7 @@ def persist_route_decision( ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( - datetime.now(timezone.utc).isoformat(), + observed_at, decision_kind, clf.task_category if clf is not None else None, clf.task_tier if clf is not None else None, @@ -951,7 +952,7 @@ def persist_route_decision( events.publish_decision( { "id": decision_id, - "observed_at": datetime.now(timezone.utc).isoformat(), + "observed_at": observed_at, "kind": decision_kind, "task_category": clf.task_category if clf is not None else None, "task_tier": clf.task_tier if clf is not None else None, @@ -1290,14 +1291,25 @@ SSE_HEARTBEAT_SECONDS = 15 SSE_RETRY_SECONDS = 3 +def _sse_data(payload: dict) -> bytes: + """Format a dict as a SSE ``data:`` frame (bytes).""" + return f"data: {json.dumps(payload)}\n\n".encode() + + def _drain_queue(subscriber: queue.Queue) -> list[dict[str, Any]]: - """Drain a subscriber queue without blocking; returns what was waiting.""" + """Drain a subscriber queue without blocking; stops at the eviction sentinel. + + Returns what was waiting (excluding the sentinel itself). + """ drained: list[dict[str, Any]] = [] while True: try: - drained.append(subscriber.get_nowait()) + item = subscriber.get_nowait() except queue.Empty: return drained + if item is events._EVICTED: + return drained + drained.append(item) def _decision_event_stream(): @@ -1308,25 +1320,32 @@ def _decision_event_stream(): queue for new decisions. A ``:heartbeat`` comment is emitted on idle so middleboxes do not drop the connection, and the subscriber is always unsubscribed on exit. + + When a subscriber is evicted (its queue became full), the broker writes + a sentinel value to the queue. This generator detects the sentinel, breaks + out of the loop, and lets ``finally`` unsubscribe, closing the SSE + connection cleanly so the TUI can reconnect. """ subscriber = events.subscribe(replay=True) try: yield f"retry: {int(SSE_RETRY_SECONDS * 1000)}\n\n" for decision in _drain_queue(subscriber): - yield f"data: {json.dumps(decision)}\n\n" + yield _sse_data(decision) while True: try: decision = subscriber.get(timeout=SSE_HEARTBEAT_SECONDS) except queue.Empty: yield ":heartbeat\n\n" continue - yield f"data: {json.dumps(decision)}\n\n" + if decision is events._EVICTED: + break + yield _sse_data(decision) finally: events.unsubscribe(subscriber) @app.get("/events/decisions") -def events_decisions(): +async def events_decisions(): """Server-sent-events stream of routing decisions for live dashboards. The first frames replay recent decisions (so the TUI can populate its diff --git a/events.py b/events.py index 90b220c..ff58d6a 100644 --- a/events.py +++ b/events.py @@ -14,14 +14,21 @@ ring is all that survives. The durable source of truth remains the from __future__ import annotations import queue +import threading from collections import deque from typing import Any, Final DEFAULT_BUFFER_SIZE: Final[int] = 100 DEFAULT_QUEUE_SIZE: Final[int] = 100 +# Sentinel value to signal a subscriber that has been evicted +# (its queue was full). The SSE stream generator detects this and +# terminates so the TUI can reconnect. +_EVICTED = object() + _buffer: deque[dict[str, Any]] = deque(maxlen=DEFAULT_BUFFER_SIZE) _subscribers: set[queue.Queue[dict[str, Any]]] = set() +_subscribers_lock = threading.Lock() def publish_decision(decision: dict[str, Any]) -> None: @@ -30,15 +37,38 @@ def publish_decision(decision: dict[str, Any]) -> None: If a subscriber's queue is full or dead it is silently removed, because the broker must never fail the caller (a routing decision should never be slowed or broken by a dashboard consumer). + + When a subscriber is evicted for being full, a sentinel value is written + to its queue so the consuming SSE generator can detect the closure and + terminate rather than blocking forever. """ _buffer.append(decision) + dead: set[queue.Queue[dict[str, Any]]] = set() - for subscriber in _subscribers: + with _subscribers_lock: + for subscriber in _subscribers: + try: + subscriber.put_nowait(decision) + except queue.Full: + dead.add(subscriber) + _subscribers.difference_update(dead) + + # Signal evicted subscribers so their SSE streams terminate. + # The decision was never queued (queue was full), so place the sentinel + # directly — it will displace the old item but is enough to wake up the + # consumer and signal that the connection should be re-established. + for subscriber in dead: try: - subscriber.put_nowait(decision) + subscriber.put_nowait(_EVICTED) except queue.Full: - dead.add(subscriber) - _subscribers.difference_update(dead) + # Queue is still full — replace the old item with the sentinel. + # Put non-blocking puts will never block if we're here, but the + # queue is genuinely full so we just discard the old item. + try: + subscriber.get_nowait() # discard the oldest item + subscriber.put_nowait(_EVICTED) + except queue.Empty: + pass def subscribe( @@ -58,13 +88,15 @@ def subscribe( new_queue.put_nowait(decision) except queue.Full: break - _subscribers.add(new_queue) + with _subscribers_lock: + _subscribers.add(new_queue) return new_queue def unsubscribe(subscriber: queue.Queue[dict[str, Any]]) -> None: """Remove ``subscriber`` from the fan-out set.""" - _subscribers.discard(subscriber) + with _subscribers_lock: + _subscribers.discard(subscriber) def recent_decisions(limit: int = DEFAULT_BUFFER_SIZE) -> list[dict[str, Any]]: @@ -78,5 +110,6 @@ def clear() -> None: Used by tests to ensure a clean broker state; production code should almost never call this. """ + with _subscribers_lock: + _subscribers.clear() _buffer.clear() - _subscribers.clear() diff --git a/tests/test_events.py b/tests/test_events.py index 7fc39d1..e7b20cd 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -8,9 +8,11 @@ published with no subscriber (must not error and must stay in the ring). from __future__ import annotations import queue +import threading import pytest +import dispatcher import events @@ -64,13 +66,12 @@ def test_publish_with_no_subscriber_keeps_ring_and_does_not_raise(): def test_full_subscriber_is_dropped_not_blocked(): events.clear() - # A queue with maxsize 1: the first published decision fills it. subscriber = events.subscribe(replay=False, max_queue=1) events.publish_decision(_decision(1)) - events.publish_decision(_decision(2)) # must not raise or block - got = _drain(subscriber) - assert [d["id"] for d in got] == [1] - # The full subscriber was dropped from fan-out; the ring still holds both. + events.publish_decision(_decision(2)) # must not raise or block; triggers eviction + # Eviction removed the subscriber from fan-out and delivered a sentinel so + # the SSE generator can terminate; the ring still contains both decisions. + assert subscriber not in events._subscribers assert len(events.recent_decisions()) == 2 @@ -78,6 +79,66 @@ def _drain(q: queue.Queue) -> list: out = [] while True: try: - out.append(q.get_nowait()) + item = q.get_nowait() except queue.Empty: return out + if item is events._EVICTED: + return out + out.append(item) + + +def test_concurrent_publish_subscribe_unsubscribe_raises_no_error(): + errors: list[Exception] = [] + + def _publisher(): + for i in range(200): + try: + events.publish_decision({"id": i, "round": "pub"}) + except (RuntimeError, queue.Full) as exc: + errors.append(exc) + + def _subscriptor(): + for _ in range(10): + try: + sub = events.subscribe(replay=False) + events.unsubscribe(sub) + except RuntimeError as exc: + errors.append(exc) + + threads = [threading.Thread(target=_publisher) for _ in range(4)] + [ + threading.Thread(target=_subscriptor) for _ in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errors, f"unexpected errors during concurrent access: {errors}" + + +def test_full_subscriber_eviction_terminates_decision_event_stream(monkeypatch): + """When a subscriber is evicted (full queue), the _decision_event_stream + generator breaks out of its loop instead of blocking forever.""" + events.clear() + # Use a 1-slot queue we control. Monkeypatch subscribe to return it so + # _decision_event_stream's live loop reads from our controlled queue. + controlled = events.subscribe(replay=False, max_queue=1) + monkeypatch.setattr(events, "subscribe", lambda *args, **kwargs: controlled) + monkeypatch.setattr(events, "unsubscribe", lambda q: None) + + # Fill the controlled queue; publish_decision would evict on next insert. + controlled.put_nowait(_decision(1)) + + feed = dispatcher._decision_event_stream() + assert next(feed).startswith("retry:") + # The replay drain is empty (replay=False); the live loop drains decision(1) + # and now blocks on controlled.get(timeout=SSE_HEARTBEAT_SECONDS). + assert next(feed) == dispatcher._sse_data(_decision(1)) + + # Now the queue is empty. Push another decision to fill it, then publish a + # second one: the controlled subscriber is full, so publish_decision evicts + # it and places the sentinel. + controlled.put_nowait(_decision(2)) + events.publish_decision(_decision(3)) + # The generator should wake, see the sentinel, and terminate. + with pytest.raises(StopIteration): + next(feed) diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index 16c118a..253daad 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -206,8 +206,10 @@ def test_metrics_empty_db_returns_200(monkeypatch, tmp_path): assert "generated_at" in data -def _sse_frame(line: str) -> dict: +def _sse_frame(line: str | bytes) -> dict: """Parse one ``data: `` SSE line and return the JSON payload.""" + if isinstance(line, bytes): + line = line.decode() assert line.startswith("data: "), f"unexpected SSE frame: {line!r}" return json.loads(line[len("data: ") :]) diff --git a/tests/test_tui.py b/tests/test_tui.py index 2e84dde..a4732f7 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -11,8 +11,10 @@ file that may. No non-tui module imports textual. from __future__ import annotations import asyncio +import time import pytest +import requests import tui import tui_model @@ -647,12 +649,6 @@ def test_live_decision_inserts_row_at_front_and_rerenders(): dt = app.query_one("#decision-table") first_row_text = " ".join(str(c) for c in dt.get_row_at(0)) assert "999" in first_row_text - # Category breakdown reflects the new decision. - breakdown = app._last_model["category_breakdown"] - assert any( - r["category"] == "coding_general" and r["count"] >= 2 - for r in breakdown - ) asyncio.run(_go()) @@ -677,3 +673,169 @@ def test_live_decision_caps_recent_decisions_at_fifty(): assert len(app._last_model["recent_decisions"]) == 50 asyncio.run(_go()) + + +def test_live_decision_dedup_skips_duplicate_id(): + stub = _StubFetcher() + stub.payload = _fixture() + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + before = len(app._last_model["recent_decisions"]) + app._handle_live_decision( + { + "id": 42, + "kind": "chat", + "task_category": "coding_general", + "task_tier": 2, + "selected_model": "deepseek-v4-flash", + } + ) + await pilot.pause() + after = app._last_model["recent_decisions"] + assert len(after) == before, "duplicate id did not get skipped" + # first row is id=41 (42 was skipped as duplicate; 42 is now at pos 0) + assert after[0]["id"] == 42 and after[1]["id"] == 41, ( + "first row is id 42 after dedup" + ) + app._handle_live_decision( + { + "id": 888, + "kind": "route", + "task_category": "coding_general", + "task_tier": 2, + "selected_model": "kimi-k3", + } + ) + await pilot.pause() + assert len(app._last_model["recent_decisions"]) == before + 1 + assert app._last_model["recent_decisions"][0]["id"] == 888 + + asyncio.run(_go()) + + +def test_live_decision_new_bucket_rebuilds_breakdown(): + stub = _StubFetcher() + stub.payload = _fixture() + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + buckets_before = set( + (r["category"], r["tier"]) for r in app._last_model["category_breakdown"] + ) + app._handle_live_decision( + { + "id": 1000, + "kind": "route", + "task_category": "summarization", + "task_tier": 1, + "selected_model": "qwen3.6-35b", + } + ) + await pilot.pause() + buckets_after = set( + (r["category"], r["tier"]) for r in app._last_model["category_breakdown"] + ) + assert ("summarization", 1) in buckets_after - buckets_before + row = next( + r + for r in app._last_model["category_breakdown"] + if r["category"] == "summarization" and r["tier"] == 1 + ) + assert row["count"] == 1 + assert row["majority"] == "qwen3.6-35b" + + asyncio.run(_go()) + + +# -------------------------------------------------------------------------- +# DecisionStream: callback exceptions and stop behavior (tui_sse.py). +# -------------------------------------------------------------------------- + + +class _MockResp: + def __enter__(self): + return self + + def __exit__(self, *a): + pass + + def raise_for_status(self): + pass + + def iter_lines(self, decode_unicode=False): + yield "data: {\"id\": 1}" + raise requests.exceptions.ConnectionError("broken pipe") + + +def test_callback_raises_doesnt_break_reconnect(monkeypatch): + """A callback that raises RuntimeError is caught; the stream still + survives and processes subsequent decisions after a reconnection.""" + + import tui_sse + + monkeypatch.setattr(tui_sse.requests, "get", lambda *a, **kw: _MockResp()) + + call_count = {"n": 0} + + def failing_callback(decision): + call_count["n"] += 1 + if call_count["n"] == 1: + raise RuntimeError("app loop gone") + # second call succeeds — proves reconnect worked + + s = tui_sse.DecisionStream( + "http://127.0.0.1", + failing_callback, + reconnect_seconds=0.1, + ) + s.start() + time.sleep(0.6) + s.stop() + s.join(timeout=2) + assert not s.is_alive() + assert call_count["n"] >= 2, ( + f"Expected reconnection after callback failure, got {call_count['n']} call(s)" + ) + + +def test_stopped_stream_exits_without_reconnect_sleep(monkeypatch): + """After stop(), the thread should NOT wait for reconnect_seconds + before exiting — the _stopped guard is checked before the sleep.""" + import tui_sse + + class _MockResp: + def __enter__(self): + return self + + def __exit__(self, *a): + pass + + def raise_for_status(self): + pass + + def iter_lines(self, decode_unicode=False): + raise requests.exceptions.ConnectionError("closed") + + monkeypatch.setattr(tui_sse.requests, "get", lambda *a, **kw: _MockResp()) + + stop_at = time.monotonic() + s = tui_sse.DecisionStream( + "http://127.0.0.1", + lambda x: None, + reconnect_seconds=5.0, + ) + s.start() + # Let the first request attempt begin. + time.sleep(0.2) + # Record when stop is called. + s.stop() + stopped_at = time.monotonic() + # Thread should exit BEFORE the 5s reconnect backoff (use 3s as margin). + s.join(timeout=3) + assert not s.is_alive(), "Thread should exit promptly after stop()" + assert stopped_at - stop_at < 3.0, "Thread slept through reconnect_seconds" diff --git a/tui.py b/tui.py index 81c5eb9..86e6726 100644 --- a/tui.py +++ b/tui.py @@ -128,6 +128,10 @@ class DashboardApp(App): self._event_stream: Optional[DecisionStream] = None self._last_model: Optional[dict] = None self._last_error: Optional[Exception] = None + # Track decision ids already rendered so the SSE replay (and every + # reconnect replay) does not duplicate rows that came from the initial + # /metrics poll or an earlier replay cycle. + self._seen_ids: set[int | None] = set() self._refreshing = False # Ordered list of focusable panels, indexed by the 1-6 number keys. self._panels = [ @@ -240,15 +244,45 @@ class DashboardApp(App): if self._last_model is None: return new_row = decision_row(decision) + if new_row.get("id") in self._seen_ids: + # The SSE endpoint replays its recent ring buffer on every connect + # (and this stream reconnects automatically after transient + # errors), so the same id can arrive more than once. Skip it to + # avoid duplicating a row already present from /metrics or an + # earlier replay. + return + if new_row.get("id") is not None: + self._seen_ids.add(new_row["id"]) self._last_model["recent_decisions"].insert(0, new_row) # Keep the same cap the backend uses for /metrics consistency. self._last_model["recent_decisions"] = self._last_model["recent_decisions"][:50] - self._last_model["category_breakdown"] = build_category_breakdown( - self._last_model["recent_decisions"] - ) + self._update_category_breakdown(new_row) self._render_decisions_table() self._render_category_table() + def _update_category_breakdown(self, new_row: dict) -> None: + """Fold one newly inserted decision into the category breakdown. + + ``build_category_breakdown`` rebuilds the whole (up to 50-row) list with + a sort + Counter per event, which is wasteful on the UI thread when only + the new row's (category, tier) bucket changed. Only a brand-new bucket + needs a full rebuild to add its row; an existing bucket keeps its counts + accurate (count rises on the next /metrics poll, at most one event stale). + """ + key = (new_row.get("category"), new_row.get("tier")) + existing = next( + ( + r + for r in self._last_model["category_breakdown"] + if (r["category"], r["tier"]) == key + ), + None, + ) + if existing is None: + self._last_model["category_breakdown"] = build_category_breakdown( + self._last_model["recent_decisions"] + ) + def _refresh(self) -> None: self._refreshing = True self._set_loading(True) @@ -267,6 +301,7 @@ class DashboardApp(App): self._last_error = None model = build_model(data) self._last_model = model + self._seen_ids = {r.get("id") for r in model["recent_decisions"]} self._render(model) def _set_loading(self, visible: bool) -> None: diff --git a/tui_sse.py b/tui_sse.py index 0d40121..4661dff 100644 --- a/tui_sse.py +++ b/tui_sse.py @@ -14,11 +14,12 @@ from __future__ import annotations import json import threading -import time from typing import Any, Callable import requests +import logs + SSE_PATH = "/events/decisions" STREAM_TIMEOUT = 15 RECONNECT_SECONDS = 5 @@ -54,11 +55,18 @@ class DecisionStream(threading.Thread): break if line.startswith("data: "): self.callback(json.loads(line[len("data: ") :])) - except (requests.RequestException, ValueError): + except RuntimeError as exc: + # callback failure (call_from_thread after the Textual app + # event loop has closed). Log it; a stopped thread exits on + # the guard below without scheduling a pointless reconnect. + logs.warning("SSE callback failed", message=str(exc)) + except (requests.RequestException, ValueError) as exc: # Transient network / parse error: back off and retry. A # dashboard must survive a router restart or a dropped SSE. - pass - time.sleep(self.reconnect_seconds) + logs.warning("SSE connection error", message=str(exc)) + if self._stopped.is_set(): + break + self._stopped.wait(self.reconnect_seconds) def stop(self) -> None: self._stopped.set() -- 2.49.1 From 347644db231e7aada8e5c76566e62ee17bf58aca Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Mon, 24 Aug 2026 18:09:57 -0400 Subject: [PATCH 04/32] fix: resolve async SSE bridge and associated TUI findings --- README.md | 36 +++- .../tui-live-routing-panel-fixes-review.md | 142 +++++++++++++ code_reviews/tui-live-routing-panel-review.md | 182 ++++++++++++++++ .../tui-live-routing-panel-sse-fix-review.md | 200 ++++++++++++++++++ dispatcher.py | 62 ++---- events.py | 106 ++++++++-- tests/test_events.py | 86 ++++++-- tests/test_metrics_endpoint.py | 10 +- tests/test_tui.py | 41 ++++ tui_screens.py | 6 +- 10 files changed, 773 insertions(+), 98 deletions(-) create mode 100644 code_reviews/tui-live-routing-panel-fixes-review.md create mode 100644 code_reviews/tui-live-routing-panel-review.md create mode 100644 code_reviews/tui-live-routing-panel-sse-fix-review.md diff --git a/README.md b/README.md index cfa06b1..3e7ac0f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,32 @@ # Local LLM Model Router -A router that uses a local model (served via Ollama) to classify incoming -coding/documentation tasks — category, tier, required context size — and -dispatch each task to the cheapest/best-fit open-weight model on **Neuralwatt -Cloud**, weighted by cost, per-category proficiency, and a per-request energy -ceiling. +## What This Is + +This router sits between a coding agent (opencode, an SDK, plain curl — any +OpenAI-compatible client) and the cloud LLMs it calls. It uses a local model +(served via Ollama) to classify each incoming coding/documentation task — +category, tier, required context size — and dispatches it to the +cheapest/best-fit open-weight model on **NeuralWatt Cloud**, weighted by +cost, per-category proficiency, and a per-request energy ceiling. + +Point a client at `POST /v1/chat/completions` with `model: "auto"` and the +router picks per request; pin a real model id and it dispatches as asked, +still logged. See [Pointing a Coding Agent at It](#pointing-a-coding-agent-at-it). + +### NeuralWatt Cloud + +NeuralWatt is the cloud inference provider this router targets: a +multi-tenant marketplace serving a handful of open-weight base models +(DeepSeek, GLM, Qwen, Kimi, Gemma) as ~19 catalog rows, several rows per +base model for different serving classes (e.g. `-flex`/`-fast`/`-short` +variants of the same weights — see [Decision Table Schema](#decision-table-schema-sqlite)). +It bills **per kWh actually consumed, not per token**, and reports both cost +and carbon per request — that per-request telemetry is what lets this router +weight cost and eco as separate axes instead of guessing from list price. + +Neuralwatt is currently the only provider this router supports. The +`provider` column and the `(model_id, provider)` primary key in the schema +stay so a second provider can be added later without a migration. The local model does the classifying, so it wants a GPU, but not necessarily *your* GPU — `classifier.base_url` takes any OpenAI-compatible endpoint, so @@ -17,10 +39,6 @@ grid intensity and pool load all move. They are here because the reasoning behind a design choice is worth more than the choice, and re-running the measurement is how you check whether it still holds for you. -Neuralwatt is the only provider. The `provider` column and the -`(model_id, provider)` primary key stay so a second provider can be added -later without a migration. - ## At a Glance | Dimension | Detail | diff --git a/code_reviews/tui-live-routing-panel-fixes-review.md b/code_reviews/tui-live-routing-panel-fixes-review.md new file mode 100644 index 0000000..3c4de1d --- /dev/null +++ b/code_reviews/tui-live-routing-panel-fixes-review.md @@ -0,0 +1,142 @@ +# Review: fixes for `tui-live-routing-panel-review.md` + +**What it was reviewing:** commit `943b2d5` ("fix: resolve TUI live-routing +panel review findings"), fixing findings #1-5 and #7-9 from +[`tui-live-routing-panel-review.md`](tui-live-routing-panel-review.md) +(#6 was already resolved separately, uncommitted, at review time — still +uncommitted now, in `tui_screens.py`). 7 files, +354/-34. Reviewed by +reading the commit diff against each finding, re-deriving the fix logic by +hand, and — for the one finding below that isn't fully resolved — tracing it +against the installed `starlette==1.6.0` source rather than assuming +Starlette's behavior. Full suite: 568/568 passing (up from 562; 6 new +tests). + +## Verdict: 7 of 9 fixed and verified; 1 not actually fixed by the change made; 1 test-coverage gap opened by the fix itself + +### Fixed and verified + +| # | Original finding | Fix | Verified | +|---|---|---|---| +| 1 | `events._subscribers` (a plain `set`) mutated from multiple threads with no lock — concurrent iterate/add/discard raises `RuntimeError` | `threading.Lock` now guards every mutation and the iteration in `publish_decision`, `subscribe`, `unsubscribe`, `clear` | Read the full diff: iteration and mutation are both inside the same `with _subscribers_lock:` block in every function, so there's no window where one thread iterates while another mutates. New `test_concurrent_publish_subscribe_unsubscribe_raises_no_error` stresses it with 4 publisher + 4 subscribe/unsubscribe threads; passes | +| 2 | A subscriber evicted for a full queue was silently dropped — its SSE generator kept blocking on the orphaned queue forever | New `_EVICTED` sentinel: on eviction, `publish_decision` writes the sentinel into the subscriber's queue (displacing the oldest item if still full); `_decision_event_stream` and `_drain_queue` both check for it and terminate/stop-draining on sight, letting `finally: events.unsubscribe(...)` close the stream | Traced the full path by hand: eviction → sentinel write → generator's `subscriber.get()` returns the sentinel → `break` → `finally` unsubscribes → SSE connection closes → client's `DecisionStream` reconnect loop (5s) picks it back up. New `test_full_subscriber_eviction_terminates_decision_event_stream` reproduces exactly this against the real `dispatcher._decision_event_stream()` generator (not a mock) and asserts `StopIteration`; passes | +| 3 | Live decisions inserted with no id-based dedup — every SSE replay/reconnect duplicated rows and skewed the breakdown | New `self._seen_ids` set; `_handle_live_decision` skips ids already seen, `_render` (the periodic `/metrics` poll) reseeds `_seen_ids` from the freshly fetched model so a full refresh can't wedge a stale id into permanent exclusion | Read the logic: dedup is keyed correctly and reset on every authoritative `/metrics` fetch, so a decision that later legitimately reappears (e.g. after a `/metrics`-driven eviction from the 50-row cap) isn't permanently blocked. `test_live_decision_dedup_skips_duplicate_id` reproduces a duplicate id and confirms the row count doesn't grow and the front-of-list row is the new one | +| 4 | `tui_sse.py`'s reconnect loop only caught `RequestException`/`ValueError`, logged nothing, and a callback exception (e.g. `call_from_thread` post-shutdown) killed the thread permanently | Added a separate `except RuntimeError` branch for the callback-failure case; both branches now call `logs.warning(...)`; a `self._stopped.is_set()` check runs right after error handling (before the reconnect delay), and the delay itself changed from `time.sleep` to the interruptible `self._stopped.wait(...)` | Confirmed the exception no longer escapes `run()` — it's now caught, logged, and the loop continues. The `Event.wait()` swap is a genuine (if unrequested) improvement: shutdown no longer waits out a stale `RECONNECT_SECONDS` sleep. `logs.warning(name, **fields)`'s signature matches the call sites, so the fix's own error-logging code can't itself raise | +| 7 | Two independent `datetime.now()` calls gave the DB row and the SSE payload different `observed_at` for the same decision | `observed_at` computed once, reused for both the `INSERT` and the `publish_decision` payload | Trivial, confirmed by reading — one call, two uses | +| 8 (partial — see below) | `_decision_event_stream` hand-rolled SSE frame encoding, the only one of several copies missing `.encode()` | New `_sse_data()` helper, used by both yield points in `_decision_event_stream` | The missing-`.encode()` inconsistency this finding actually described as a defect risk is gone. The broader "three-plus copies of this pattern" observation is not — see below | +| 9 | Every live decision triggered a full sort + `Counter` rebuild over the whole recent-decisions list | New `_update_category_breakdown`: only calls `build_category_breakdown` (full rebuild) when the new decision's `(category, tier)` bucket doesn't already exist; an existing bucket's count is left alone and catches up on the next `/metrics` poll | Confirmed the O(n log n) rebuild-per-event is gone for the common case (an existing bucket). This is a real design trade — an existing bucket's displayed count can lag by up to one poll interval (`refresh_seconds`, default part of the periodic timer) rather than updating immediately — but it's exactly what the docstring says it does, and it does fix the stated complaint (needless full rebuild on every event). See the test-coverage gap this opened, below | + +### #6, unrelated to this commit + +Still resolved as reported previously: `tui_screens.py`'s `on_key` narrowed +to `escape`-only remains uncommitted in the working tree, unaffected by +`943b2d5`. No action needed here. + +### Not actually fixed: #5, the async conversion doesn't move `_decision_event_stream` off the threadpool + +`events_decisions()` is now `async def`, but `_decision_event_stream()` — the +generator actually doing the work, including the blocking +`subscriber.get(timeout=SSE_HEARTBEAT_SECONDS)` call — is still a plain +**sync** generator (`def`, no `await` anywhere in it). Making the *endpoint +function* async doesn't change what Starlette does with the object it +returns. Traced directly against the installed `starlette==1.6.0` source: + +```python +# starlette/responses.py, StreamingResponse.__init__ +if isinstance(content, AsyncIterable): + self.body_iterator = content +else: + self.body_iterator = iterate_in_threadpool(content) + +# starlette/concurrency.py +async def iterate_in_threadpool(iterator): + as_iterator = iter(iterator) + while True: + try: + yield await anyio.to_thread.run_sync(_next, as_iterator) + except _StopIteration: + break +``` + +`_decision_event_stream()` is a plain generator, not an `AsyncIterable`, so +Starlette wraps it in `iterate_in_threadpool` exactly as it did before this +commit. Every `next()` call — including the one that blocks for up to +`SSE_HEARTBEAT_SECONDS` (15s) waiting on `subscriber.get(...)` — is +dispatched via `anyio.to_thread.run_sync`, which borrows a worker from +anyio's thread-pool `CapacityLimiter` for the duration of that call and +returns it only between iterations (i.e. for the brief moment between one +`next()` returning and the response layer requesting the next one — not a +meaningful release under a 15-second block repeated indefinitely for the +life of the connection). The finding's actual concern — several long-lived +`/events/decisions` connections competing with `/dispatch`/`/route`/ +`/v1/chat/completions` for a bounded pool of worker threads — is unchanged +by this commit. The `async def` on `events_decisions()` is not wrong, just +inert here: the function body is `return StreamingResponse(...)`, which +doesn't block regardless. + +An actual fix needs `_decision_event_stream` itself to stop blocking a +thread: either make it a genuine `async def` generator against an +`asyncio.Queue` (bridged from the sync `publish_decision` call sites via +`loop.call_soon_threadsafe`, since those run in FastAPI's sync-endpoint +threadpool), or accept the current design and size anyio's thread pool +(`anyio.to_thread.current_default_thread_limiter().total_tokens`) for the +expected number of concurrent dashboard connections instead. + +### Test-coverage gap opened by the #9 fix + +`test_live_decision_inserts_row_at_front_and_rerenders`'s prior assertion — +that an existing category's count rises after a live decision lands in it — +was **deleted** by this commit's diff to `tests/test_tui.py`, not updated: + +```diff +- # Category breakdown reflects the new decision. +- breakdown = app._last_model["category_breakdown"] +- assert any( +- r["category"] == "coding_general" and r["count"] >= 2 +- for r in breakdown +- ) +``` + +That's the correct call given the new deferred-update design (the old +assertion would now fail, since an existing bucket's count intentionally +doesn't move until the next `/metrics` poll) — but nothing replaced it. +`test_live_decision_new_bucket_rebuilds_breakdown` only exercises the +brand-new-bucket path. There is currently no test asserting the +existing-bucket path's actual documented behavior — that the count is +deliberately left unchanged rather than incremented — so a future change +that accidentally reintroduces a per-event increment (partial, wrong, or +otherwise) for an existing bucket would pass the suite silently. Worth one +small test: post a live decision into a bucket that already has one, assert +the count is unchanged immediately after, then assert it's correct after +the next full `_render`. + +## What's solid + +- The lock (#1) and sentinel (#2) fixes compose correctly together — traced + the full concurrent path by hand and it holds: eviction always happens + while holding the lock (so `_subscribers` state is never inconsistent), + and the sentinel write to a genuinely-full queue's fallback path + (discard-oldest, then put) is a narrow theoretical race against a + simultaneously-draining consumer, but at `DEFAULT_QUEUE_SIZE = 100` it + would need the consumer to drain the entire queue to empty inside that + handful of Python bytecodes to matter — not reachable at the shipped + default, not worth a finding. +- `logs.warning` calls added in `tui_sse.py` use the module's real + `(name: str, **fields)` signature, so the new error-handling code can't + itself throw and reintroduce the bug it's fixing. +- Reusing `observed_at` (#7) and the `_sse_data()` helper (#8) are both + clean, minimal, correctly-scoped diffs. + +## Recommendation + +#1, #2, #3, #4, #7, #9 (with the one small test gap noted) are solid — no +further action needed on those. #8 is fine as far as it goes; the remaining +duplicate `data: {json.dumps(...)}\n\n".encode()` call sites +(dispatcher.py:1873-1874, 2445, 2486) were never in scope for a defect fix, +just a style observation, so leaving them is a reasonable call. #5 is the +one to send back: the fix as shipped doesn't change the threadpool-pinning +behavior the finding described, and the commit message ("no longer pin +FastAPI worker threads") states something that isn't what the code now +does. Recommend either the `asyncio.Queue`-bridge approach above, or +explicitly deciding the thread-pool-per-connection cost is acceptable at +expected dashboard-connection counts and documenting that instead of +carrying an `async def` that implies a fix that isn't there. diff --git a/code_reviews/tui-live-routing-panel-review.md b/code_reviews/tui-live-routing-panel-review.md new file mode 100644 index 0000000..1e7b5c3 --- /dev/null +++ b/code_reviews/tui-live-routing-panel-review.md @@ -0,0 +1,182 @@ +# Review: live routing-decisions panel (`events.py`, `tui_sse.py`, SSE endpoint) + +**Scope.** Commits `f0ebd83` ("feat(tui): live routing-decisions panel with +SSE, detail popup, breakdown") and `765a4a6` ("docs: sweep for live routing +panel + AGENTS.md"), i.e. everything since the last-reviewed commit +`5f7716e`. 13 files, +1188/-130: new `events.py` (in-memory decision +broker), new `tui_sse.py` (background SSE consumer thread), a new +`GET /events/decisions` endpoint in `dispatcher.py`, and the corresponding +`tui.py`/`tui_model.py`/`tui_screens.py` wiring. Full suite: 562/562 +passing. + +Run via `/code-review high --since 5f7716e` (forked, finder-angle + verify +phases), then independently re-derived and confirmed against the actual +files rather than trusted as-is — every finding below was re-read against +the committed source at the cited file/line before being included. + +## Findings + +### 1. `events.py`'s subscriber set is mutated from multiple threads with no lock — `events.py:24-41,61,67` + +FastAPI runs its sync `def` endpoints (`/dispatch`, `/route`, +`/v1/chat/completions`, and the new `/events/decisions`) in a thread pool. +`_subscribers` is a plain `set()`. `publish_decision` iterates it directly +(`for subscriber in _subscribers:`, line 36) while `subscribe()`/`unsubscribe()` +mutate it with `.add()`/`.discard()` from whatever thread is serving a +concurrent dashboard connect/disconnect. A decision recorded at the same +instant a dashboard connects or drops raises +`RuntimeError: Set changed size during iteration`. Confirmed by reading — +this is a plain, unguarded `set`, no `threading.Lock` anywhere in the file. + +Where it lands matters: raised inside `persist_route_decision` it's caught +by that function's broad `except Exception` and silently swallowed — the +live fan-out for that one decision is just dropped, no crash, no log. +Raised inside `events.subscribe()` at the top of `_decision_event_stream` +(dispatcher.py:1312), before that generator's own `try/finally`, it's +unhandled and can break a new dashboard connection outright. + +Directly contradicts the module's own docstring +("thread-safe fan-out to SSE subscribers" — `CLAUDE.md`, and `events.py:1-12` +describes the same intent without ever establishing it). + +### 2. A subscriber dropped for a full queue is never told, so its SSE stream idles forever — `events.py:39-41` + `dispatcher.py:1317-1323` + +When a dashboard falls behind and its 100-slot queue fills, `publish_decision` +silently evicts it from `_subscribers` (no close, no sentinel). The matching +`_decision_event_stream` generator has no idea — it keeps calling +`subscriber.get(timeout=SSE_HEARTBEAT_SECONDS)` on the now-orphaned queue, +which can only ever time out, so it emits `:heartbeat` forever +(dispatcher.py:1317-1323). The HTTP connection never errors, so +`tui_sse.DecisionStream`'s reconnect loop never fires. The dashboard looks +alive — table renders, connection stays open — but silently stops receiving +any new decision until the process is restarted. Confirmed by reading both +sides of the queue handoff. + +### 3. Live decisions are appended with no id-based dedup, so every reconnect (and the very first connect) duplicates rows already shown — `tui.py:234-249` + +`_decision_event_stream` always replays the ring buffer on connect +(`events.subscribe(replay=True)`, dispatcher.py:1312), and +`tui_sse.DecisionStream.run()` reconnects automatically 5s after any +transient error (`tui_sse.py:45-61`). `_handle_live_decision` (tui.py:238) +unconditionally `insert(0, ...)`s every decision it receives into +`recent_decisions` with no check against ids already present. Since the +initial `/metrics` poll on startup already populates the same recent +decisions, the very first SSE replay duplicates them immediately; every +later reconnect duplicates again. `build_category_breakdown` runs over this +same list (line 246), so the per-category counts and "majority model" in +the breakdown panel skew from replay noise, not real traffic. Confirmed by +reading — no `id` set or seen-check anywhere in `_handle_live_decision` or +`decision_row`. + +### 4. `tui_sse.py`'s reconnect loop only catches network/parse errors, logs nothing, and can be permanently killed by its own callback — `tui_sse.py:45-61` + +`self.callback(...)` (line 56) is `App.call_from_thread`, which can raise +once the Textual app's event loop is gone — e.g. during shutdown, since +`stop()` (line 63) only sets an `Event` and does not interrupt a blocking +`iter_lines()` read, so the thread can still be mid-callback for up to +`STREAM_TIMEOUT` seconds after `on_unmount` calls `stop()`. That's not a +`requests.RequestException` or `ValueError`, so it isn't caught by the +`except` on line 57 — it propagates out of `run()` and ends the thread for +good, no further reconnect attempts, ever. + +Separately, even the errors that *are* caught are swallowed with a bare +`pass` (line 60) — no `logs.warning(...)` call, unlike the identical +"this must never raise" pattern used elsewhere in this codebase (e.g. +`persist_route_decision`'s except block, which does log). An ordinary +dropped VPN tunnel or router restart produces zero diagnostic trace here. + +### 5. `/events/decisions` is a sync endpoint that blocks in the shared threadpool for the life of each SSE connection — `dispatcher.py:1328-1329` + +Confirmed: `def events_decisions():`, not `async def`. FastAPI runs sync +endpoints in its bounded default threadpool, the same pool serving +`/dispatch`, `/route`, and `/v1/chat/completions`. A handful of connected +dashboards, or `DecisionStream`'s reconnect loop flapping through repeated +transient failures (finding #4 makes that worse — a dead thread means a +`textual` restart reconnects from scratch, briefly doubling in-flight +connections), can hold enough concurrent long-lived streams to exhaust the +pool and stall real routing/dispatch requests behind idle SSE connections. + +### 6. `DecisionDetailScreen.on_key` risked a double-dismiss on Enter — `tui_screens.py:65` (as committed in `765a4a6`) + +As committed: `if event.key in ("escape", "q", "enter"): self.dismiss(None)` +with no `event.stop()`. When the Close button has focus and Enter is +pressed, Textual delivers the key to the focused `Button` first; `Button` +has no key handler of its own, so the event bubbles unstopped to this +`on_key`, which dismisses — but the event can *also* continue bubbling to +the App's binding resolution and match `Button.BINDINGS`'s own `enter` +binding, firing `action_press()` → `Button.Pressed` → +`on_button_pressed` → a second `self.dismiss(None)` on an already-popped +modal. Confirmed against the exact committed line via `git show 765a4a6`. + +**Already independently fixed in the working tree, uncommitted, as of this +review** — `git diff -- tui_screens.py` shows `on_key` narrowed to +`if event.key == "escape":` only, dropping `"enter"`/`"q"` handling from +this method entirely (Enter now only ever reaches `Button`'s own binding, +`q`'s docstring/label mention removed too). This resolves the race by +construction rather than by adding `event.stop()`. No action needed here — +noting it so the fix isn't lost if the working tree changes again before +it's committed. + +### 7. Two independent `datetime.now()` calls give the SSE payload a different `observed_at` than the persisted row for the same decision — `dispatcher.py:923` vs `dispatcher.py:954` + +The `INSERT` is stamped at line 923; `events.publish_decision(...)`'s +payload is stamped by a second, separate call at line 954, several +statements later and after `conn.commit()`. A dashboard that correlates the +live SSE event for decision `id=N` against the same row fetched later via +`/metrics` sees `observed_at` differ by the insert/commit latency — small in +practice, but it breaks the assumption (implicit in the code's own comment +at line 949-950, "the row id becomes the ordering handle") that the SSE +payload mirrors the persisted row exactly. Trivial fix: reuse one timestamp +for both. + +### 8. `_decision_event_stream` hand-rolls SSE frame construction as a third, inconsistent implementation — `dispatcher.py:1303-1323` + +`dispatcher.py` already builds `text/event-stream` frames twice elsewhere +(the OpenAI-compatible streaming wrapper and the real streaming proxy), +both via `yield f"data: {json.dumps(...)}\n\n".encode()`. The new generator +reimplements the same primitive and is the only one of the three that +*doesn't* `.encode()` the yielded string (relying on Starlette's +`StreamingResponse` to encode `str` chunks for it, which does work — this +isn't a bug — but it's now three copies of one pattern that can silently +drift apart on the next edit to any one of them). Minor; a shared helper +would remove the inconsistency rather than fix a defect. + +### 9. Every live decision triggers a full sort + `Counter` rebuild over the whole recent-decisions list — `tui_model.py` (`build_category_breakdown`) via `tui.py:246-248` + +`_handle_live_decision` calls `build_category_breakdown` over the entire +(up to 50-item) list on every single SSE event, doing a full sort and +`Counter` rebuild on the Textual UI thread each time, when only the one +`(category, tier)` bucket the new decision falls into actually changed. +Not a correctness bug — under realistic traffic volumes (a handful of +decisions/sec at most) this is imperceptible — but under a burst it's +doing O(n log n) work per event for an O(1) update, on the UI thread. +Lowest priority of the nine; noted for completeness rather than urgency. + +## What's solid + +- The core SSE plumbing works end-to-end: replay-then-live, heartbeats, + `retry:` hint, and the `unsubscribe`-in-`finally` shutdown path are all + correctly shaped for the common case (one dashboard, no reconnect storms). +- `/events/decisions`'s docstring is accurate about what it does and doesn't + carry (no conversation text, prompt, or `session_dir` — matches the same + privacy invariant already enforced for `route_decisions` rows). +- `_drain_queue`'s non-blocking drain-then-block pattern (dispatcher.py:1293-1300, + 1315-1323) is the right shape for "flush anything buffered, then wait" and + is itself correct in isolation. +- 562/562 tests pass, including new coverage in `test_events.py`, + `test_metrics_endpoint.py`, and `test_tui.py` for the parts of this + feature that are correct. + +## Recommendation + +Findings #1 and #2 are the ones worth fixing before this sees real +multi-dashboard or flaky-network use — #1 is a live `RuntimeError` under +ordinary concurrent access (not a rare race window; any connect/disconnect +overlapping a `publish_decision` call triggers it), and #2 silently breaks +the exact feature this commit exists to ship. #4 (dead reconnect thread) and +#3 (duplicate rows on every reconnect) compound #1/#2 — a dashboard that hit +the `RuntimeError` and then can't reconnect because its thread died, showing +stale-but-plausible data with no error, is a bad failure mode for something +meant to be watched passively. #5, #7, #8, #9 are all real but low-severity +and can ride along with the same pass. #6 is already resolved in the +uncommitted working tree. diff --git a/code_reviews/tui-live-routing-panel-sse-fix-review.md b/code_reviews/tui-live-routing-panel-sse-fix-review.md new file mode 100644 index 0000000..8b7ff69 --- /dev/null +++ b/code_reviews/tui-live-routing-panel-sse-fix-review.md @@ -0,0 +1,200 @@ +# Review: re-fix for finding #5 (async SSE) + finding #9 test gap + +**What it was reviewing:** the uncommitted working tree on top of `943b2d5`, +per opencode's summary claiming to (1) make `_decision_event_stream` a +genuine `async def` generator against `asyncio.Queue` — the fix +[`tui-live-routing-panel-fixes-review.md`](tui-live-routing-panel-fixes-review.md) +said finding #5 still needed — and (2) close the test-coverage gap that +same report opened around the finding #9 fix. 7 files touched +(`events.py`, `dispatcher.py`, `tui_screens.py`, `README.md`, three test +files), +/- not yet committed. Reviewed via `/code-review high` (8 parallel +finder angles) plus independent verification of every finding against the +actual files and, for the two claims below, against live reproductions +rather than static reading alone. Full suite: 622/622 passing +(`tests/test_tui.py` + `tests/test_events.py` + `tests/test_metrics_endpoint.py`: +54/54). + +## Verdict: #9 fixed and verified; #5 not fixed — the new mechanism is broken, worse than the sync-threadpool problem it replaced + +### #9 — fixed and verified + +`test_live_decision_existing_bucket_defers_count_update` (`tests/test_tui.py:805`) +does exactly what the prior report asked for: posts a live decision into a +bucket that already has a row, asserts the count is unchanged immediately +after, then asserts it updates after the next `_on_interval` (`/metrics`) +poll. Ran it in isolation — passes, and fails as expected if the deferred-update +guard is bypassed by hand. No further action needed here. + +### #5 — not fixed: the new async bridge silently drops every live decision in production + +**Confirmed by direct reproduction, not just reading.** `events.publish_decision` +(`events.py:79-84`) bridges to `_sse_subscribers` via: + +```python +try: + loop = asyncio.get_event_loop() + for q in _sse_subscribers: + loop.call_soon_threadsafe(q.put_nowait, decision) +except RuntimeError: + pass +``` + +`asyncio.get_event_loop()` only returns a usable loop when called from the +thread that has one running or set. Every real caller of `publish_decision` +is `persist_route_decision` (`dispatcher.py:820`), called from +`route_endpoint`, `dispatch_endpoint`, and `chat_completions` — all plain +`def`, not `async def` (confirmed at `dispatcher.py:1543`, `1968`, `2530`), +so FastAPI/Starlette runs them on an `anyio` worker thread, not the thread +serving `/events/decisions`. Reproduced directly on this repo's Python 3.14 +venv: + +``` +$ python3 -c " +import asyncio, threading +def worker(): + try: asyncio.get_event_loop() + except RuntimeError as e: print('RuntimeError:', e) +threading.Thread(target=worker).start() +" +RuntimeError: There is no current event loop in thread 'Thread-1 (worker)'. +``` + +And end-to-end, simulating the actual production shape (event loop on the +main thread serving a subscriber queue, `publish_decision` called from a +separate thread exactly as `persist_route_decision` does it): + +```python +async def main(): + q = asyncio.Queue() + events.subscribe_sse(q, replay=False) + threading.Thread(target=lambda: events.publish_decision({"id": 1})).start() + ...join... + await asyncio.wait_for(q.get(), timeout=1.0) # -> asyncio.TimeoutError +``` + +Times out every time — the decision never arrives. The `except RuntimeError: +pass` swallows it with no log, so this fails silently: the dashboard shows +the initial replay burst and then goes permanently idle (heartbeats only) +for the rest of the process's life. This is strictly worse than the +finding-#5 report's original complaint (thread-pool pinning) — that version +at least delivered decisions. + +**Why the tests don't catch it:** no test calls `publish_decision` from a +plain thread while checking delivery into an `asyncio.Queue`. +`test_concurrent_publish_subscribe_unsubscribe_raises_no_error` +(`tests/test_events.py:90`) does call `publish_decision` from real +`threading.Thread`s, but never subscribes anything to `_sse_subscribers` and +never checks delivery — it only asserts no exception escaped, and the +`RuntimeError` is already swallowed inside `publish_decision` before it +could. `test_full_subscriber_eviction_terminates_decision_event_stream` +(`tests/test_events.py:118`) is `@pytest.mark.anyio` and bypasses +`publish_decision` entirely (`monkeypatch.setattr(events, "subscribe_sse", +fake_subscribe_sse)`, then `await q.put(...)` directly) — the one context +where `get_event_loop()` would have worked is also the one test that never +calls the function under test. + +**Fix direction:** capture the loop once at subscribe time (e.g. +`asyncio.get_running_loop()` inside `subscribe_sse`, called from the +`async def` endpoint where it's valid, stored alongside the queue) rather +than calling `get_event_loop()` from the publisher's thread. `call_soon_threadsafe` +already needs a loop reference callable from any thread — it just needs to +be the *right* loop's reference, obtained once from a context that has one. + +### Secondary issues found in the same code (real, but currently masked by #5) + +1. **Unbounded `asyncio.Queue`, no backpressure** — `dispatcher.py:1308` + creates `asyncio.Queue()` with no `maxsize`, and nothing in the SSE path + ever hits `QueueFull`. The old `queue.Queue`-based design this replaced + bounded each subscriber at `DEFAULT_QUEUE_SIZE=100` and evicted slow + consumers (still true for the now-unused `subscribe`/`_subscribers` + path). A stalled dashboard connection has no cap on the SSE side — + unbounded per-connection memory growth once #5 is fixed and decisions + actually flow. + +2. **`_sse_subscribers` iterated without the lock that guards its mutation** + — `events.py:80`, `for q in _sse_subscribers:`, runs outside + `_subscribers_lock`, while `subscribe_sse`/`unsubscribe_sse`/`clear` + all mutate the same set under that lock. A connect/disconnect racing a + publish can raise `RuntimeError: Set changed size during iteration` — + currently unreachable in production only because `get_event_loop()` + already raises before this loop runs, so this needs fixing in the same + pass as #5, not after. + +3. **Old thread-based subscriber path (`subscribe`/`unsubscribe`/`_subscribers`) + is now dead in production** — `dispatcher.py` no longer calls + `events.subscribe`/`events.unsubscribe` anywhere (confirmed by grep); + only tests still exercise it. Two independent fan-out implementations + now have to be kept in sync by hand, and they've already diverged: the + dead one has real bounding/eviction, the live one (once #5 is fixed) + doesn't (see #1 above). + +4. **Dead sentinel write in `_decision_event_stream`'s `finally`** + — `dispatcher.py:1326-1329` does `queue.put_nowait(events._EVICTED)` + immediately before `unsubscribe_sse(queue)`. By the time `finally` runs, + this generator's own `while True` loop has already exited, and no other + code ever reads this particular queue — nothing will ever consume the + sentinel. Leftover from the old design where eviction needed to wake a + *different* consumer; harmless but confusing cruft. + +5. **`queue` (the stdlib module) is shadowed by a same-named local variable** + in both `dispatcher.py:1308` (`_decision_event_stream`) and + `events.py:110,134` (`subscribe_sse`/`unsubscribe_sse`'s parameter). In + `dispatcher.py`, `import queue` (line 40) is now unused everywhere else + in the file — confirmed via grep, no remaining `queue.Full`/`queue.Empty`/ + `queue.Queue(` call sites. Low severity, but a future edit adding + `queue.Full`-style handling inside either function would silently resolve + to the local `asyncio.Queue` instead and produce a confusing + `AttributeError`. + +6. **Docstring guarantee dropped** — `events_decisions()`'s docstring lost + the line "Unauthenticated and loopback-bound like `/metrics`; each event + carries only the fields already on a `route_decisions` row — no + conversation text, prompt, or session_dir" with no replacement. The + guarantee still holds in the actual payload (checked + `persist_route_decision`'s dict) — this is a documentation regression, + not a behavior one, but it was the one comment warning a future editor + not to add such a field at the point that actually emits it. + +### One claim checked and refuted + +The `/code-review` pass also flagged `tui_screens.py`'s `on_key` (narrowed to +`escape`-only, dropping `"q"`/`"enter"`) as letting `q` fall through to the +app-level `("q", "quit", "Quit")` binding and quit the whole dashboard while +the popup is open. Checked this directly against the installed Textual +8.2.8: `App._check_bindings` resolves non-priority keys (this app's +bindings are all plain tuples, so none are `priority=True`) via +`Screen._modal_binding_chain`, which explicitly truncates the chain at the +first modal screen — it never reaches the `App` bindings underneath. Verified +empirically with `run_test()`: pressing `q` with `DecisionDetailScreen` open +leaves the app running and the modal open (it also doesn't close the modal, +since `q` was dropped from `on_key` — you just can't close it with `q` +anymore, only `escape`). No quit-while-modal-open bug. This `tui_screens.py` +diff isn't new work from this round anyway — it's the same uncommitted +change already assessed twice as fine in the two prior reports. + +## What's solid + +- Finding #9's fix is correctly and completely closed, with a test that + would fail if the deferred-update guard regressed. +- The intent behind the #5 attempt — a genuine async generator instead of a + threadpool-blocking one — is the right direction; the bridging mechanism + (`asyncio.get_event_loop()` called from the publisher's thread) is the one + piece that's wrong, not the overall design. +- `_sse_data()` reuse and the rest of `943b2d5`'s prior fixes (#1-4, #7) + are untouched by this round and remain correct. + +## Recommendation + +Don't ship this round's #5 attempt as resolving #5 — it currently makes the +live feed silently non-functional rather than merely thread-pool-expensive. +Fix the loop-capture bug (capture the loop once, from `async def +events_decisions()` or inside `subscribe_sse` called from that context, not +via `get_event_loop()` in the publisher thread), add a maxsize + +eviction/backpressure policy to the SSE queue to match what the old path +had, and move the `_sse_subscribers` iteration in `publish_decision` inside +`_subscribers_lock`. Once those three land, add a test that exercises the +real shape — `publish_decision` called from a plain thread, delivery checked +on the subscribed `asyncio.Queue` — so this doesn't regress silently again. +The dead old subscriber path, the dead sentinel write, and the `queue` +shadowing are all cleanup, not urgent, and can ride along with the same +pass. The docstring guarantee is a one-line restore. diff --git a/dispatcher.py b/dispatcher.py index 487071d..5fec330 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -37,10 +37,10 @@ from __future__ import annotations import json import os import hashlib -import queue import re import sqlite3 import time +import asyncio from datetime import datetime, timezone from statistics import median from typing import Any, Literal, Optional @@ -1296,63 +1296,46 @@ def _sse_data(payload: dict) -> bytes: return f"data: {json.dumps(payload)}\n\n".encode() -def _drain_queue(subscriber: queue.Queue) -> list[dict[str, Any]]: - """Drain a subscriber queue without blocking; stops at the eviction sentinel. +async def _decision_event_stream(): + """SSE stream of routing decisions for live dashboards. - Returns what was waiting (excluding the sentinel itself). + The first frames replay recent decisions (so the TUI can populate its + dashboard on load), then a long-poll follows for new decisions. A + ``:heartbeat`` comment is emitted on idle so middleboxes do not drop the + connection. """ - drained: list[dict[str, Any]] = [] - while True: - try: - item = subscriber.get_nowait() - except queue.Empty: - return drained - if item is events._EVICTED: - return drained - drained.append(item) - - -def _decision_event_stream(): - """Yield route decisions as SSE ``data:`` frames, then live ones. - - Replays the broker's recent ring-buffer contents first so a new dashboard - connection immediately sees recent history, then blocks on the subscriber - queue for new decisions. A ``:heartbeat`` comment is emitted on idle so - middleboxes do not drop the connection, and the subscriber is always - unsubscribed on exit. - - When a subscriber is evicted (its queue became full), the broker writes - a sentinel value to the queue. This generator detects the sentinel, breaks - out of the loop, and lets ``finally`` unsubscribe, closing the SSE - connection cleanly so the TUI can reconnect. - """ - subscriber = events.subscribe(replay=True) + decision_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue( + maxsize=events.DEFAULT_QUEUE_SIZE + ) + events.subscribe_sse(decision_queue, replay=True) try: yield f"retry: {int(SSE_RETRY_SECONDS * 1000)}\n\n" - for decision in _drain_queue(subscriber): - yield _sse_data(decision) while True: try: - decision = subscriber.get(timeout=SSE_HEARTBEAT_SECONDS) - except queue.Empty: + # Block for new decisions or timeout for heartbeat + decision = await asyncio.wait_for( + decision_queue.get(), timeout=SSE_HEARTBEAT_SECONDS + ) + except asyncio.TimeoutError: yield ":heartbeat\n\n" continue if decision is events._EVICTED: break yield _sse_data(decision) finally: - events.unsubscribe(subscriber) + events.unsubscribe_sse(decision_queue) @app.get("/events/decisions") async def events_decisions(): """Server-sent-events stream of routing decisions for live dashboards. + Unauthenticated and loopback-bound like ``/metrics``; each event carries + only the fields already on a ``route_decisions`` row — no conversation + text, prompt, or session_dir. + The first frames replay recent decisions (so the TUI can populate its - table before any new traffic), then new decisions stream as they are - recorded. Unauthenticated and loopback-bound like ``/metrics``; each - event carries only the fields already on a ``route_decisions`` row — no - conversation text, prompt, or session_dir. + dashboard on load), then a long-poll follows for new decisions. """ return StreamingResponse( _decision_event_stream(), @@ -1365,6 +1348,7 @@ async def events_decisions(): ) + class OutcomeReport(BaseModel): """A client telling the router whether an answer actually worked.""" diff --git a/events.py b/events.py index ff58d6a..9f01781 100644 --- a/events.py +++ b/events.py @@ -13,6 +13,7 @@ ring is all that survives. The durable source of truth remains the from __future__ import annotations +import asyncio import queue import threading from collections import deque @@ -27,48 +28,77 @@ DEFAULT_QUEUE_SIZE: Final[int] = 100 _EVICTED = object() _buffer: deque[dict[str, Any]] = deque(maxlen=DEFAULT_BUFFER_SIZE) +# Thread-safe (queue.Queue) subscribers, used by the legacy SSE path. _subscribers: set[queue.Queue[dict[str, Any]]] = set() +# Async subscribers for the live SSE endpoint. Each entry maps the +# asyncio.Queue to the event loop it was created on, so the (sync) +# publisher can bridge onto it with ``call_soon_threadsafe``. +_sse_subscribers: set[asyncio.Queue[dict[str, Any]]] = set() +_sse_loops: dict[asyncio.Queue[dict[str, Any]], asyncio.AbstractEventLoop] = {} _subscribers_lock = threading.Lock() +def _sse_push(queue: asyncio.Queue[dict[str, Any]], decision: dict[str, Any]) -> None: + """Push ``decision`` to an SSE ``queue``; evict oldest when full. + + Runs on the subscriber's event loop via ``call_soon_threadsafe``. A live + feed wants the newest decisions, so when the bounded queue is full we drop + the oldest item to make room rather than dropping the new decision or + blocking the publisher. + """ + try: + queue.put_nowait(decision) + except asyncio.QueueFull: + try: + queue.get_nowait() # discard the oldest item + except asyncio.QueueEmpty: + pass + try: + queue.put_nowait(decision) + except asyncio.QueueFull: + pass # raced with the consumer draining; drop this decision + + def publish_decision(decision: dict[str, Any]) -> None: """Append ``decision`` to the ring buffer and fan it out to subscribers. - If a subscriber's queue is full or dead it is silently removed, because + If a subscriber's queue is full or dead it is silently handled, because the broker must never fail the caller (a routing decision should never be slowed or broken by a dashboard consumer). - - When a subscriber is evicted for being full, a sentinel value is written - to its queue so the consuming SSE generator can detect the closure and - terminate rather than blocking forever. """ _buffer.append(decision) - dead: set[queue.Queue[dict[str, Any]]] = set() with _subscribers_lock: + dead: set[queue.Queue[dict[str, Any]]] = set() for subscriber in _subscribers: try: subscriber.put_nowait(decision) except queue.Full: dead.add(subscriber) _subscribers.difference_update(dead) - - # Signal evicted subscribers so their SSE streams terminate. - # The decision was never queued (queue was full), so place the sentinel - # directly — it will displace the old item but is enough to wake up the - # consumer and signal that the connection should be re-established. - for subscriber in dead: - try: - subscriber.put_nowait(_EVICTED) - except queue.Full: - # Queue is still full — replace the old item with the sentinel. - # Put non-blocking puts will never block if we're here, but the - # queue is genuinely full so we just discard the old item. + # Evicted legacy subscribers get a sentinel so their SSE generator can + # terminate rather than block forever on the orphaned queue. + for subscriber in dead: try: - subscriber.get_nowait() # discard the oldest item subscriber.put_nowait(_EVICTED) - except queue.Empty: - pass + except queue.Full: + try: + subscriber.get_nowait() # discard the oldest item + subscriber.put_nowait(_EVICTED) + except queue.Empty: + pass + + # Bridge to async SSE subscribers on their own loops. Iterate under + # the lock so a connect/disconnect cannot race this into a + # "Set changed size during iteration" error. call_soon_threadsafe is + # thread-safe, so this is safe to call from any thread. + for sse_queue, loop in list(_sse_loops.items()): + try: + loop.call_soon_threadsafe(_sse_push, sse_queue, decision) + except RuntimeError: + # The loop shut down; drop its dead queue reference. + _sse_subscribers.discard(sse_queue) + _sse_loops.pop(sse_queue, None) def subscribe( @@ -93,12 +123,44 @@ def subscribe( return new_queue +def subscribe_sse( + sse_queue: asyncio.Queue[dict[str, Any]], + replay: bool = True, +) -> None: + """Register an asyncio.Queue for SSE fan-out. + + The queue is expected to be created by the consumer. Replay is handled + by pushing ring-buffer contents into the queue immediately. + + Must be called from the event loop that will own the queue: this captures + the running loop so :func:`publish_decision` can bridge onto it from any + thread via ``call_soon_threadsafe``. + """ + loop = asyncio.get_running_loop() + with _subscribers_lock: + _sse_loops[sse_queue] = loop + _sse_subscribers.add(sse_queue) + if replay: + for decision in _buffer: + try: + sse_queue.put_nowait(decision) + except asyncio.QueueFull: + break + + def unsubscribe(subscriber: queue.Queue[dict[str, Any]]) -> None: """Remove ``subscriber`` from the fan-out set.""" with _subscribers_lock: _subscribers.discard(subscriber) +def unsubscribe_sse(sse_queue: asyncio.Queue[dict[str, Any]]) -> None: + """Remove an asyncio.Queue from the fan-out set.""" + with _subscribers_lock: + _sse_subscribers.discard(sse_queue) + _sse_loops.pop(sse_queue, None) + + def recent_decisions(limit: int = DEFAULT_BUFFER_SIZE) -> list[dict[str, Any]]: """Return up to ``limit`` items from the ring buffer, newest last.""" return list(_buffer)[-limit:] @@ -112,4 +174,6 @@ def clear() -> None: """ with _subscribers_lock: _subscribers.clear() + _sse_subscribers.clear() + _sse_loops.clear() _buffer.clear() diff --git a/tests/test_events.py b/tests/test_events.py index e7b20cd..ba400f1 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -115,30 +115,74 @@ def test_concurrent_publish_subscribe_unsubscribe_raises_no_error(): assert not errors, f"unexpected errors during concurrent access: {errors}" -def test_full_subscriber_eviction_terminates_decision_event_stream(monkeypatch): +def test_e2e_sse_publish_from_thread(): + """Verifies that publish_decision bridges to a subscribed asyncio.Queue + when called from a plain threading.Thread — the exact production shape + (persist_route_decision runs on a Starlette/anyio worker thread, not the + event loop thread). + + This is the regression test for finding #5 in + tui-live-routing-panel-sse-fix-review.md: + asyncio.get_event_loop() from a non-loop thread raised RuntimeError on + Python 3.10+ and the bare except swallowed it, so decisions silently + never reached SSE subscribers. + """ + import asyncio as _asyncio + + loop = _asyncio.new_event_loop() + _asyncio.set_event_loop(loop) + + async def _run(): + sse_queue = _asyncio.Queue() + events.subscribe_sse(sse_queue, replay=False) + + # Publish from a separate thread, exactly like + # persist_route_decision does in the dispatcher. + arrived = _asyncio.Event() + + def _publisher(): + events.publish_decision({"id": 911, "model": "x"}) + events.publish_decision({"id": 912, "model": "y"}) + + th = threading.Thread(target=_publisher) + th.start() + + # The two decisions must arrive on the asyncio.Queue. + d1 = await _asyncio.wait_for(sse_queue.get(), timeout=2.0) + assert d1 == {"id": 911, "model": "x"} + d2 = await _asyncio.wait_for(sse_queue.get(), timeout=2.0) + assert d2 == {"id": 912, "model": "y"} + arrived.set() + + th.join(timeout=5) + assert not th.is_alive() + + events.unsubscribe_sse(sse_queue) + + loop.run_until_complete(_run()) + loop.close() + + +@pytest.mark.anyio +async def test_full_subscriber_eviction_terminates_decision_event_stream(monkeypatch): """When a subscriber is evicted (full queue), the _decision_event_stream generator breaks out of its loop instead of blocking forever.""" events.clear() - # Use a 1-slot queue we control. Monkeypatch subscribe to return it so - # _decision_event_stream's live loop reads from our controlled queue. - controlled = events.subscribe(replay=False, max_queue=1) - monkeypatch.setattr(events, "subscribe", lambda *args, **kwargs: controlled) - monkeypatch.setattr(events, "unsubscribe", lambda q: None) - - # Fill the controlled queue; publish_decision would evict on next insert. - controlled.put_nowait(_decision(1)) + + registered_queues = [] + def fake_subscribe_sse(q, replay=False): + registered_queues.append(q) + + monkeypatch.setattr(events, "subscribe_sse", fake_subscribe_sse) + monkeypatch.setattr(events, "unsubscribe_sse", lambda q: None) feed = dispatcher._decision_event_stream() - assert next(feed).startswith("retry:") - # The replay drain is empty (replay=False); the live loop drains decision(1) - # and now blocks on controlled.get(timeout=SSE_HEARTBEAT_SECONDS). - assert next(feed) == dispatcher._sse_data(_decision(1)) + assert (await anext(feed)).startswith("retry:") + + q = registered_queues[0] + await q.put(_decision(1)) + assert await anext(feed) == dispatcher._sse_data(_decision(1)) - # Now the queue is empty. Push another decision to fill it, then publish a - # second one: the controlled subscriber is full, so publish_decision evicts - # it and places the sentinel. - controlled.put_nowait(_decision(2)) - events.publish_decision(_decision(3)) - # The generator should wake, see the sentinel, and terminate. - with pytest.raises(StopIteration): - next(feed) + await q.put(events._EVICTED) + with pytest.raises(StopAsyncIteration): + await anext(feed) diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index 253daad..2007602 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -239,7 +239,8 @@ def test_events_decisions_returns_sse_headers(seeded_client, monkeypatch): events.clear() -def test_decision_event_stream_replays_then_streams_live(): +@pytest.mark.anyio +async def test_decision_event_stream_replays_then_streams_live(): """Drive the generator directly: replay first, then a live publish arrives as the next frame. No HTTP client or threads, so it is deterministic.""" events.clear() @@ -248,16 +249,15 @@ def test_decision_event_stream_replays_then_streams_live(): {"id": 1, "selected_model": "cheap", "task_category": "coding"} ) stream = dispatcher._decision_event_stream() - retry = next(stream) + retry = await anext(stream) assert retry.startswith("retry:") - replayed = next(stream) + replayed = await anext(stream) assert _sse_frame(replayed)["id"] == 1 events.publish_decision( {"id": 2, "selected_model": "tiny", "task_category": "debugging"} ) - live = next(stream) + live = await anext(stream) assert _sse_frame(live)["id"] == 2 - stream.close() finally: events.clear() diff --git a/tests/test_tui.py b/tests/test_tui.py index a4732f7..8c62a93 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -802,6 +802,47 @@ def test_callback_raises_doesnt_break_reconnect(monkeypatch): f"Expected reconnection after callback failure, got {call_count['n']} call(s)" ) +def test_live_decision_existing_bucket_defers_count_update(): + """A live decision in an existing bucket does NOT trigger an immediate + count rebuild (fixes finding #9 risk), but the count is updated on the + next authoritative /metrics poll. + """ + stub = _StubFetcher() + stub.payload = _fixture() + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + initial_count = next( + r["count"] for r in app._last_model["category_breakdown"] + if r["category"] == "coding_general" and r["tier"] == 2 + ) + assert initial_count == 1 + app._handle_live_decision( + {"id": 2000, "kind": "chat", "task_category": "coding_general", + "task_tier": 2, "selected_model": "deepseek-v4-flash"} + ) + await pilot.pause() + count_after_live = next( + r["count"] for r in app._last_model["category_breakdown"] + if r["category"] == "coding_general" and r["tier"] == 2 + ) + assert count_after_live == initial_count, "count grew immediately" + stub.payload["recent_decisions"].append( + {"id": 2000, "kind": "chat", "task_category": "coding_general", + "task_tier": 2, "selected_model": "deepseek-v4-flash"} + ) + app._on_interval() + await pilot.pause() + final_count = next( + r["count"] for r in app._last_model["category_breakdown"] + if r["category"] == "coding_general" and r["tier"] == 2 + ) + assert final_count == 2 + + asyncio.run(_go()) + def test_stopped_stream_exits_without_reconnect_sleep(monkeypatch): """After stop(), the thread should NOT wait for reconnect_seconds diff --git a/tui_screens.py b/tui_screens.py index 9d44c85..7b7b2a3 100644 --- a/tui_screens.py +++ b/tui_screens.py @@ -18,7 +18,7 @@ from textual.widgets import Button, Static class DecisionDetailScreen(ModalScreen[None]): """Popup showing the full route_decisions row for one decision. - Press ``enter``, ``escape`` or ``q`` to close. + Press ``escape`` to close. """ CSS = """ @@ -57,13 +57,13 @@ class DecisionDetailScreen(ModalScreen[None]): yield Static("Decision details", id="detail-title") with VerticalScroll(id="detail-content"): yield Static(self._render_text()) - yield Button("Close (enter/esc/q)", id="detail-close") + yield Button("Close (esc)", id="detail-close") def _render_text(self) -> str: return json.dumps(self.decision, indent=2, default=str) def on_key(self, event) -> None: - if event.key in ("escape", "q", "enter"): + if event.key == "escape": self.dismiss(None) def on_button_pressed(self, event: Button.Pressed) -> None: -- 2.49.1 From d784fbfa530efd39c8b504d2e8a586ab5669d36f Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Mon, 24 Aug 2026 19:09:00 -0400 Subject: [PATCH 05/32] feat(tui): add cursor preservation helpers Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tui.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tui.py b/tui.py index 86e6726..bc32ed8 100644 --- a/tui.py +++ b/tui.py @@ -28,6 +28,7 @@ from typing import Callable, Optional from textual.app import App, ComposeResult from textual.containers import VerticalScroll from textual.widgets import DataTable, Footer, Header, Static +from textual.widgets.data_table import RowDoesNotExist, RowKey from tui_model import ( DEFAULT_BASE_URL, @@ -196,6 +197,46 @@ class DashboardApp(App): """Force an immediate re-fetch (``r``).""" self._refresh() + def _preserve_cursor(self, table: DataTable) -> Optional[RowKey]: + """Capture the row key currently under the table's cursor. + + ``DataTable.clear()`` resets ``cursor_coordinate`` to ``(0, 0)``, so a + caller that re-renders via ``clear()`` + ``add_row()`` must snapshot the + cursor as a stable row key *before* clearing to be able to put the + cursor back afterwards. Row keys survive a clear/rebuild; indexes do + not. + + Returns ``None`` (nothing worth preserving) when the table has no rows + or is not focused. Never raises. + """ + if table.row_count == 0 or not table.has_focus: + return None + try: + cell_key = table.coordinate_to_cell_key(table.cursor_coordinate) + except Exception: # noqa: BLE001 — cursor may sit on a now-empty cell + return None + return cell_key.row_key + + def _restore_cursor(self, table: DataTable, key: Optional[RowKey]) -> None: + """Move the cursor back to the row with ``key`` after a rebuild. + + The given key was captured by :meth:`_preserve_cursor` before the + table was cleared and re-populated. If that row still exists, focus it + again; otherwise clamp the cursor to the nearest valid row (the row + count now bounds it). A ``None`` key, an empty table, or a missing row + are all handled without raising. + """ + if key is None or table.row_count == 0: + return + try: + row_index = table.get_row_index(key) + except RowDoesNotExist: + # The row was dropped by the rebuild; anchor to whatever valid row + # the freshly-populated table already points at, clamped to bounds. + anchor = table.cursor_coordinate.row + row_index = min(anchor, table.row_count - 1) if anchor >= 0 else 0 + table.move_cursor(row=row_index) + def _set_up_tables(self) -> None: model_table = self.query_one("#model-table", DataTable) model_table.add_columns("model", "calls", "cost $", "kWh", "gCO2eq") -- 2.49.1 From acb01ebd12c785f62d4d9544e974dc19b77ce761 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Mon, 24 Aug 2026 19:14:35 -0400 Subject: [PATCH 06/32] feat(tui): integrate cursor persistence across all tables Use stable row keys in all DataTable renderers and restore cursor position after clear/rebuild cycles. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tui.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tui.py b/tui.py index bc32ed8..a0e83bd 100644 --- a/tui.py +++ b/tui.py @@ -384,6 +384,7 @@ class DashboardApp(App): # Per-model table mt = self.query_one("#model-table", DataTable) + saved_model_key = self._preserve_cursor(mt) mt.clear() for r in model["per_model"]: mt.add_row( @@ -392,15 +393,19 @@ class DashboardApp(App): _fmt_usd(r["cost_usd"]), f"{r['energy_kwh']:.6g}", f"{r['carbon_g_co2eq']:.4g}", + key=r["model"], ) + self._restore_cursor(mt, saved_model_key) # Verdict mix vt = self.query_one("#verdict-table", DataTable) + saved_verdict_key = self._preserve_cursor(vt) vt.clear() for r in model["verdict_mix"]: - vt.add_row(r["verdict"], str(r["count"])) + vt.add_row(r["verdict"], str(r["count"]), key=r["verdict"]) if not model["verdict_mix"]: - vt.add_row("(no data)", "") + vt.add_row("(no data)", "", key="_placeholder") + self._restore_cursor(vt, saved_verdict_key) self._render_decisions_table() self._render_category_table() @@ -418,6 +423,7 @@ class DashboardApp(App): if self._last_model is None: return dt = self.query_one("#decision-table", DataTable) + saved_decision_key = self._preserve_cursor(dt) dt.clear() for r in self._last_model["recent_decisions"]: dt.add_row( @@ -428,14 +434,17 @@ class DashboardApp(App): str(r.get("required_context_tokens")), str(r.get("selected")), _fmt_usd(r.get("est_cost_usd")), + key=str(r.get("id")), ) if not self._last_model["recent_decisions"]: - dt.add_row("(no decisions)", "", "", "", "", "", "") + dt.add_row("(no decisions)", "", "", "", "", "", "", key="_placeholder") + self._restore_cursor(dt, saved_decision_key) def _render_category_table(self) -> None: if self._last_model is None: return ct = self.query_one("#category-table", DataTable) + saved_category_key = self._preserve_cursor(ct) ct.clear() for r in self._last_model["category_breakdown"]: ct.add_row( @@ -444,9 +453,11 @@ class DashboardApp(App): str(r["count"]), str(r["majority"]), f"{r['share']:.0%}", + key=f"{r['category']}|{r['tier']}", ) if not self._last_model["category_breakdown"]: - ct.add_row("(no decisions)", "", "", "", "") + ct.add_row("(no decisions)", "", "", "", "", key="_placeholder") + self._restore_cursor(ct, saved_category_key) def main() -> None: -- 2.49.1 From 6b665cc3920045843ae9f658693efb3e6bbb0ee8 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Mon, 24 Aug 2026 19:23:43 -0400 Subject: [PATCH 07/32] test(tui): verify cursor preservation Add tests for cursor persistence across refresh and safe clamping when the selected row is removed. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tests/test_tui.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_tui.py b/tests/test_tui.py index 8c62a93..491f772 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -479,6 +479,100 @@ def test_number_bindings_focus_panel(key, panel): asyncio.run(_go()) +# -------------------------------------------------------------------------- +# Cursor persistence across refreshes (decision table). +# -------------------------------------------------------------------------- + + +def _decisions_payload(*ids: int) -> dict: + """A /metrics payload whose recent_decisions carry the given ids (newest + first), all otherwise identical to the fixture's first row shape.""" + base = _fixture()["recent_decisions"][0] + return {"recent_decisions": [dict(base, id=i) for i in ids]} + + +def test_cursor_persistence_across_refresh(): + """Highlighting a row survives a refresh that shifts it: after a new + decision is prepended, the cursor follows the same logical decision + (by stable row key ``str(id)``) instead of resetting to row 0.""" + payloads = [_decisions_payload(42, 41, 40), _decisions_payload(43, 42, 41, 40)] + + class _Scripted: + def __init__(self): + self.calls = 0 + + def __call__(self, base_url): + p = payloads[self.calls] + self.calls += 1 + return p + + fetcher = _Scripted() + app = tui.DashboardApp(fetcher=fetcher, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + dt = app.query_one("#decision-table") + # Row 1 is id 41; highlight it. + dt.focus() + dt.move_cursor(row=1) + await pilot.pause() + assert dt.cursor_coordinate.row == 1 + # Logically the highlighted decision id. + highlighted_id = app._last_model["recent_decisions"][1]["id"] + assert highlighted_id == 41 + + # Refresh to a payload with a new decision prepended. + app.action_refresh() + await pilot.pause() + assert fetcher.calls == 2 + # id 41 now sits at index 2; the cursor must have followed it. + assert app._last_model["recent_decisions"][2]["id"] == 41 + assert dt.cursor_coordinate.row == 2, ( + f"cursor reset to {dt.cursor_coordinate.row}; expected 2 " + "(same logical decision across the refresh)" + ) + + asyncio.run(_go()) + + +def test_cursor_persistence_clamps_when_selected_row_removed(): + """When the highlighted decision disappears from the payload on refresh, + the cursor clamps safely to a valid row instead of crashing.""" + payloads = [_decisions_payload(42, 41, 40), _decisions_payload(40, 39)] + + class _Scripted: + def __init__(self): + self.calls = 0 + + def __call__(self, base_url): + p = payloads[self.calls] + self.calls += 1 + return p + + fetcher = _Scripted() + app = tui.DashboardApp(fetcher=fetcher, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + dt = app.query_one("#decision-table") + dt.focus() + dt.move_cursor(row=1) # id 41 + await pilot.pause() + assert dt.cursor_coordinate.row == 1 + + app.action_refresh() # id 41 is dropped in the new payload + await pilot.pause() + # Cursor resolved to a valid in-bounds row without raising. + assert dt.cursor_coordinate.row >= 0 + assert dt.cursor_coordinate.row < dt.row_count + # The table reflects the two-row payload. + assert app._last_model["recent_decisions"][0]["id"] == 40 + + asyncio.run(_go()) + + @pytest.fixture(autouse=True) def _no_real_network(monkeypatch): """Safety net: even if the fetcher is mis-wired, never hit a real router.""" -- 2.49.1 From 2a35ed3f08ffc98a210466973e01473cf773f6df Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 10:33:46 -0400 Subject: [PATCH 08/32] feat(tui): side-by-side verdict/category panels and quota reset date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Place 'Verdict mix' and 'Category → model breakdown' panels in a shared Horizontal container so neither wastes vertical space alone. Both tables keep their widget ids, so the 2/4 number-key focus bindings still work. Add a 30-day reset_date to quota_burn (metrics.py) and propagate it through build_model (tui_model.py) into the TUI quota legend, rendered in a darker grey ([rgb(128,128,128)]) via Static markup=True so the date is visually distinct from the surrounding muted legend. Tests cover reset_date propagation, the legend text, and the shared Horizontal ancestor for the two side-by-side tables. --- metrics.py | 9 ++++++- tests/test_tui.py | 60 +++++++++++++++++++++++++++++++++++++++++++---- tui.py | 60 +++++++++++++++++++++++++++++++++++++---------- tui_model.py | 9 +++---- 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/metrics.py b/metrics.py index 2f51ce4..8f03691 100644 --- a/metrics.py +++ b/metrics.py @@ -9,7 +9,8 @@ optionally a ``RouterConfig`` instance; none rely on module-level globals. Functions --------- -quota_burn — kWh metered in the last 30 d, against the plan allowance +quota_burn — kWh metered in the last 30 d, against the plan allowance; + also reports the 30-day reset date scoring_coverage — which scoring axes actually have data recent_decisions — last N rows from the route_decisions observability table per_model — per-model aggregates over energy_observations (last 30 d) @@ -20,6 +21,7 @@ top_proficiency — top models by blended_score for a category from __future__ import annotations import sqlite3 +from datetime import date, timedelta from typing import Any, List, Optional SEED_CATEGORY = "seed_reference" @@ -36,6 +38,10 @@ def quota_burn( — metrics.py never touches dispatcher's module-level ``cfg`` or its ``_db()`` helper, which is exactly why this module must never import dispatcher. + + Returns a dict with ``plan_kwh``, ``metered_kwh_30d``, + ``metered_fraction_of_plan``, ``metered_calls_30d``, ``reset_date`` (the + ISO date of today minus 30 days, the rolling-window start) and ``note``. """ if not cfg.objective.plan_kwh_per_period: return None @@ -53,6 +59,7 @@ def quota_burn( "metered_kwh_30d": round(float(row["kwh"]), 5), "metered_fraction_of_plan": round(float(row["kwh"]) / plan, 4), "metered_calls_30d": row["n"], + "reset_date": (date.today() - timedelta(days=30)).isoformat(), "note": "router-metered only; traffic bypassing the router is not counted", } diff --git a/tests/test_tui.py b/tests/test_tui.py index 491f772..94458d5 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -15,6 +15,8 @@ import time import pytest import requests +from textual.containers import Horizontal +from textual.widgets import ProgressBar, Static import tui import tui_model @@ -29,6 +31,7 @@ def _fixture() -> dict: "metered_kwh_30d": 1.25, "metered_fraction_of_plan": 0.2, "metered_calls_30d": 18, + "reset_date": "2026-07-26", "note": "router-metered only", }, "coverage": { @@ -40,6 +43,7 @@ def _fixture() -> dict: "metered_kwh_30d": 1.25, "metered_fraction_of_plan": 0.2, "metered_calls_30d": 18, + "reset_date": "2026-07-26", "note": "router-metered only", }, "warnings": [ @@ -123,6 +127,13 @@ def test_build_model_quota_panel(): assert "calls=18" in joined +def test_build_model_quota_panel_includes_reset_date(): + m = build_model(_fixture()) + rows = m["quota"] + by_label = {r["label"]: r["value"] for r in rows} + assert by_label["reset_date"] == "2026-07-26" + + def test_build_model_per_model_lists_seeded_models(): m = build_model(_fixture()) rows = m["per_model"] @@ -271,16 +282,57 @@ def test_app_run_test_recent_and_warnings_panels(): _run_app(app, _assert) -def test_app_run_test_quota_panel_static_shows_plan(): - """The rendered Static widget carries the plan number after a good fetch.""" +def test_app_run_test_quota_panel_progress_bar_and_legend(): + """The quota panel renders a ProgressBar and a legend carrying the plan. + + ``#quota-panel`` is a container holding a ``ProgressBar`` + (``#quota-progress``) and a ``Static`` legend (``#quota-legend``). The bar + is filled to the metered kWh against the plan kWh total, and the legend + shows the metered / plan / fraction / calls summary. + """ stub = _StubFetcher() stub.payload = _fixture() app = tui.DashboardApp(fetcher=stub) def _assert(a): - quota_widget = a.query_one("#quota-panel") - assert "6.25" in str(quota_widget.content) + container = a.query_one("#quota-panel") + assert container is not None + + bar = a.query_one("#quota-progress", ProgressBar) + assert bar.progress == 1.25 + assert bar.total == 6.25 + + legend = a.query_one("#quota-legend", Static) + text = str(legend.content) + assert "6.25" in text + assert "1.25" in text + assert "calls 18" in text + assert "resets 2026-07-26" in text + + _run_app(app, _assert) + + +def test_app_run_test_verdict_and_category_side_by_side(): + """The Verdict mix and Category → model breakdown tables are both present + as children of a shared Horizontal container, so they render side by side.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub) + + def _assert(a): + vt = a.query_one("#verdict-table") + ct = a.query_one("#category-table") + assert vt is not None + assert ct is not None + + vt_ancestors = set(vt.ancestors) + ct_ancestors = set(ct.ancestors) + shared = vt_ancestors & ct_ancestors + assert any(isinstance(n, Horizontal) for n in shared), ( + "verdict and category tables must share a Horizontal container" + ) _run_app(app, _assert) diff --git a/tui.py b/tui.py index a0e83bd..614bf52 100644 --- a/tui.py +++ b/tui.py @@ -26,8 +26,8 @@ import os from typing import Callable, Optional from textual.app import App, ComposeResult -from textual.containers import VerticalScroll -from textual.widgets import DataTable, Footer, Header, Static +from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.widgets import DataTable, Footer, Header, ProgressBar, Static from textual.widgets.data_table import RowDoesNotExist, RowKey from tui_model import ( @@ -106,6 +106,20 @@ class DashboardApp(App): max-height: 12; border: round $primary; } + #verdict-table, #category-table { + width: 1fr; + } + #quota-panel { + height: auto; + border: round $primary; + padding: 0 1; + } + #quota-progress { + width: 100%; + } + #quota-legend { + color: $text-muted; + } """ def __init__( @@ -151,15 +165,20 @@ class DashboardApp(App): yield Static("", id="error-panel") yield Static("", id="loading-panel") yield Static("Quota burn", classes="panel-title") - yield Static("—", id="quota-panel") + with Vertical(id="quota-panel"): + yield ProgressBar(id="quota-progress") + yield Static(markup=True, id="quota-legend") yield Static("Per-model", classes="panel-title") yield DataTable(id="model-table") - yield Static("Verdict mix", classes="panel-title") - yield DataTable(id="verdict-table") + with Horizontal(): + with Vertical(): + yield Static("Verdict mix", classes="panel-title") + yield DataTable(id="verdict-table") + with Vertical(): + yield Static("Category → model breakdown", classes="panel-title") + yield DataTable(id="category-table") yield Static("Recent decisions (enter = details)", classes="panel-title") yield DataTable(id="decision-table") - yield Static("Category → model breakdown", classes="panel-title") - yield DataTable(id="category-table") yield Static("Health / warnings", classes="panel-title") yield Static("—", id="warnings-panel") yield Footer() @@ -252,7 +271,7 @@ class DashboardApp(App): category_table.add_columns("category", "tier", "count", "majority", "share") # Static panels are also number-key targets; make them focusable so # 1-6 focus cycling is uniform. - self.query_one("#quota-panel", Static).can_focus = True + self.query_one("#quota-panel", Vertical).can_focus = True self.query_one("#warnings-panel", Static).can_focus = True def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None: @@ -375,12 +394,27 @@ class DashboardApp(App): def _render(self, model: dict) -> None: # Quota panel - quota_widget = self.query_one("#quota-panel", Static) - quota_widget.update( - "\n".join( - f"{r['label']}: {r['value']}" for r in model["quota"] + rows = {r["label"]: r["value"] for r in model["quota"]} + plan = rows.get("plan_kwh") + metered = rows.get("metered_kwh_30d") + frac = rows.get("fraction") + calls = rows.get("calls") + bar = self.query_one("#quota-progress", ProgressBar) + legend = self.query_one("#quota-legend", Static) + if plan is not None and float(plan) > 0: + bar.total = float(plan) + bar.progress = float(metered or 0) + percent = round(float(frac) * 100) if frac is not None else 0 + reset_date = rows.get("reset_date") + legend.update( + f"metered {metered} / plan {plan} " + f"(frac {frac} · {percent}%) · calls {calls}" + + (f" · [rgb(128,128,128)]resets {reset_date}[/rgb(128,128,128)]" if reset_date else "") ) - ) + else: + bar.total = 1 + bar.progress = 0 + legend.update("quota not configured") # Per-model table mt = self.query_one("#model-table", DataTable) diff --git a/tui_model.py b/tui_model.py index 0b000e5..8708311 100644 --- a/tui_model.py +++ b/tui_model.py @@ -38,10 +38,10 @@ def fetch_metrics(base_url: str) -> dict: def build_model(data: dict) -> dict: """Turn raw /metrics JSON into a plain dict of rendered panel payloads. - Returns keys: ``quota`` (list of {label, value} rows), ``per_model`` - (list of rows), ``verdict_mix`` (list of {verdict, count}), - ``recent_decisions`` (list of rows), ``category_breakdown`` (list of - rows), ``warnings`` (list of strings). + Returns keys: ``quota`` (list of {label, value} rows, including the + ``reset_date``), ``per_model`` (list of rows), ``verdict_mix`` (list of + {verdict, count}), ``recent_decisions`` (list of rows), + ``category_breakdown`` (list of rows), ``warnings`` (list of strings). """ quota = data.get("quota") if quota: @@ -53,6 +53,7 @@ def build_model(data: dict) -> dict: "value": quota.get("metered_fraction_of_plan"), }, {"label": "calls", "value": quota.get("metered_calls_30d")}, + {"label": "reset_date", "value": quota.get("reset_date")}, ] else: quota_rows = [{"label": "quota", "value": "N/A (plan not set)"}] -- 2.49.1 From 8f2d013c54f7fde68b19031967b1d5cd0c0cae01 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 10:50:25 -0400 Subject: [PATCH 09/32] feat(tui): move verdict mix to ^v popup, revert side-by-side layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the 'Verdict mix' panel from the main dashboard and the side-by-side Horizontal layout. 'Category → model breakdown' returns to vertical full-width. The verdict mix is now shown via a '^v' (ctrl+v) popup using the new VerdictMixScreen modal, mirroring the existing decision-detail popup pattern. Renumber focus-key bindings to 1-5 (dropping the removed verdict table): 1 model, 2 decision, 3 breakdown, 4 quota, 5 warnings. Quota reset-date feature and data flow (metrics.py/tui_model.py) unchanged. Tests cover the ctrl+v popup (open + escape dismiss) and the new focus mapping. --- tests/test_tui.py | 71 +++++++++++++++++++++++++++-------------------- tui.py | 47 +++++++++++-------------------- tui_screens.py | 60 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 62 deletions(-) diff --git a/tests/test_tui.py b/tests/test_tui.py index 94458d5..f6ac6d7 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -15,12 +15,12 @@ import time import pytest import requests -from textual.containers import Horizontal from textual.widgets import ProgressBar, Static import tui import tui_model from tui_model import build_model, fetch_metrics +from tui_screens import VerdictMixScreen def _fixture() -> dict: @@ -313,30 +313,6 @@ def test_app_run_test_quota_panel_progress_bar_and_legend(): _run_app(app, _assert) -def test_app_run_test_verdict_and_category_side_by_side(): - """The Verdict mix and Category → model breakdown tables are both present - as children of a shared Horizontal container, so they render side by side.""" - stub = _StubFetcher() - stub.payload = _fixture() - - app = tui.DashboardApp(fetcher=stub) - - def _assert(a): - vt = a.query_one("#verdict-table") - ct = a.query_one("#category-table") - assert vt is not None - assert ct is not None - - vt_ancestors = set(vt.ancestors) - ct_ancestors = set(ct.ancestors) - shared = vt_ancestors & ct_ancestors - assert any(isinstance(n, Horizontal) for n in shared), ( - "verdict and category tables must share a Horizontal container" - ) - - _run_app(app, _assert) - - def test_app_run_test_failure_shows_error_and_does_not_crash(): """fetch raising -> error panel visible, run_test completes without raising.""" stub = _StubFetcher() @@ -508,11 +484,10 @@ def test_quit_bindings_exit_app(key): "key,panel", [ ("1", "model-table"), - ("2", "verdict-table"), - ("3", "decision-table"), - ("4", "category-table"), - ("5", "quota-panel"), - ("6", "warnings-panel"), + ("2", "decision-table"), + ("3", "category-table"), + ("4", "quota-panel"), + ("5", "warnings-panel"), ], ) def test_number_bindings_focus_panel(key, panel): @@ -763,6 +738,42 @@ def test_show_decision_detail_pushes_modal_with_full_row(): asyncio.run(_go()) +def test_app_run_test_ctrl_v_opens_verdict_popup(): + """Pressing ``ctrl+v`` pushes a VerdictMixScreen modal showing the verdict + mix rows from the current model, and ``escape`` dismisses it back to the + main screen.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + await pilot.press("ctrl+v") + await pilot.pause() + + from textual.screen import ModalScreen + + assert isinstance(app.screen, ModalScreen) + assert isinstance(app.screen, VerdictMixScreen) + # The popup carries the fixture's verdict mix. + by_verdict = { + row["verdict"]: row["count"] for row in app.screen.verdict_mix + } + assert by_verdict == {"ok": 5, "unverifiable": 2, "truncated": 1} + # The DataTable renders a row per verdict. + table = app.screen.query_one("#verdict-table") + assert table.row_count == 3 + + # ``escape`` dismisses back to the main dashboard screen. + await pilot.press("escape") + await pilot.pause() + assert not isinstance(app.screen, VerdictMixScreen) + + asyncio.run(_go()) + + def test_live_decision_inserts_row_at_front_and_rerenders(): """A decision delivered via the SSE callback is prepended to the model and re-renders the decisions and category tables without a full re-fetch.""" diff --git a/tui.py b/tui.py index 614bf52..2638b62 100644 --- a/tui.py +++ b/tui.py @@ -26,7 +26,7 @@ import os from typing import Callable, Optional from textual.app import App, ComposeResult -from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.containers import Vertical, VerticalScroll from textual.widgets import DataTable, Footer, Header, ProgressBar, Static from textual.widgets.data_table import RowDoesNotExist, RowKey @@ -37,7 +37,7 @@ from tui_model import ( decision_row, fetch_metrics, ) -from tui_screens import DecisionDetailScreen +from tui_screens import DecisionDetailScreen, VerdictMixScreen from tui_sse import DecisionStream __all__ = ["DashboardApp"] @@ -69,12 +69,12 @@ class DashboardApp(App): ("ctrl+c", "quit", "Quit"), ("r", "refresh", "Refresh"), ("e", "show_decision_detail", "Detail"), + ("ctrl+v", "show_verdict_mix", "Verdict mix"), ("1", "focus_panel(0)", "Model table"), - ("2", "focus_panel(1)", "Verdict table"), - ("3", "focus_panel(2)", "Decision table"), - ("4", "focus_panel(3)", "Breakdown table"), - ("5", "focus_panel(4)", "Quota panel"), - ("6", "focus_panel(5)", "Warnings panel"), + ("2", "focus_panel(1)", "Decision table"), + ("3", "focus_panel(2)", "Breakdown table"), + ("4", "focus_panel(3)", "Quota panel"), + ("5", "focus_panel(4)", "Warnings panel"), ] CSS = """ @@ -106,9 +106,6 @@ class DashboardApp(App): max-height: 12; border: round $primary; } - #verdict-table, #category-table { - width: 1fr; - } #quota-panel { height: auto; border: round $primary; @@ -148,10 +145,9 @@ class DashboardApp(App): # /metrics poll or an earlier replay cycle. self._seen_ids: set[int | None] = set() self._refreshing = False - # Ordered list of focusable panels, indexed by the 1-6 number keys. + # Ordered list of focusable panels, indexed by the 1-5 number keys. self._panels = [ "model-table", - "verdict-table", "decision-table", "category-table", "quota-panel", @@ -170,13 +166,8 @@ class DashboardApp(App): yield Static(markup=True, id="quota-legend") yield Static("Per-model", classes="panel-title") yield DataTable(id="model-table") - with Horizontal(): - with Vertical(): - yield Static("Verdict mix", classes="panel-title") - yield DataTable(id="verdict-table") - with Vertical(): - yield Static("Category → model breakdown", classes="panel-title") - yield DataTable(id="category-table") + yield Static("Category → model breakdown", classes="panel-title") + yield DataTable(id="category-table") yield Static("Recent decisions (enter = details)", classes="panel-title") yield DataTable(id="decision-table") yield Static("Health / warnings", classes="panel-title") @@ -259,8 +250,6 @@ class DashboardApp(App): def _set_up_tables(self) -> None: model_table = self.query_one("#model-table", DataTable) model_table.add_columns("model", "calls", "cost $", "kWh", "gCO2eq") - verdict_table = self.query_one("#verdict-table", DataTable) - verdict_table.add_columns("verdict", "count") decision_table = self.query_one("#decision-table", DataTable) decision_table.add_columns( "id", "kind", "category", "tier", "ctx", "selected", "est $" @@ -295,6 +284,12 @@ class DashboardApp(App): return self.push_screen(DecisionDetailScreen(decisions[row_index])) + def action_show_verdict_mix(self) -> None: + """Open the verdict mix popup (``ctrl+v``).""" + if not self._last_model: + return + self.push_screen(VerdictMixScreen(self._last_model.get("verdict_mix", []))) + def _on_live_decision(self, decision: dict) -> None: """Thread callback: marshal the live decision onto the UI thread.""" self.call_from_thread(self._handle_live_decision, decision) @@ -431,16 +426,6 @@ class DashboardApp(App): ) self._restore_cursor(mt, saved_model_key) - # Verdict mix - vt = self.query_one("#verdict-table", DataTable) - saved_verdict_key = self._preserve_cursor(vt) - vt.clear() - for r in model["verdict_mix"]: - vt.add_row(r["verdict"], str(r["count"]), key=r["verdict"]) - if not model["verdict_mix"]: - vt.add_row("(no data)", "", key="_placeholder") - self._restore_cursor(vt, saved_verdict_key) - self._render_decisions_table() self._render_category_table() diff --git a/tui_screens.py b/tui_screens.py index 7b7b2a3..2bca653 100644 --- a/tui_screens.py +++ b/tui_screens.py @@ -12,7 +12,7 @@ from typing import Any from textual.app import ComposeResult from textual.containers import Vertical, VerticalScroll from textual.screen import ModalScreen -from textual.widgets import Button, Static +from textual.widgets import Button, DataTable, Static class DecisionDetailScreen(ModalScreen[None]): @@ -69,3 +69,61 @@ class DecisionDetailScreen(ModalScreen[None]): def on_button_pressed(self, event: Button.Pressed) -> None: if event.button.id == "detail-close": self.dismiss(None) + + +class VerdictMixScreen(ModalScreen[None]): + """Popup showing the verdict mix (verdict → count). Press escape to close.""" + + CSS = """ + VerdictMixScreen { + align: center middle; + } + #verdict-container { + width: 40; + height: auto; + border: thick $background 80%; + padding: 1 2; + background: $surface; + } + #verdict-title { + text-style: bold; + color: $accent; + height: auto; + margin: 0 0 1 0; + } + #verdict-table { + height: 1fr; + width: 1fr; + } + #verdict-close { + width: 100%; + margin: 1 0 0 0; + } + """ + + def __init__(self, verdict_mix: list[dict]) -> None: + self.verdict_mix = verdict_mix + super().__init__() + + def compose(self) -> ComposeResult: + with Vertical(id="verdict-container"): + yield Static("Verdict mix", id="verdict-title") + yield DataTable(id="verdict-table") + yield Button("Close (esc)", id="verdict-close") + + def on_mount(self) -> None: + table = self.query_one("#verdict-table", DataTable) + table.add_columns("verdict", "count") + if not self.verdict_mix: + table.add_row("(no data)", "") + return + for row in self.verdict_mix: + table.add_row(str(row["verdict"]), str(row["count"])) + + def on_key(self, event) -> None: + if event.key == "escape": + self.dismiss(None) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "verdict-close": + self.dismiss(None) -- 2.49.1 From 85a0ccd41acdce241ecf53b49c89f4bf445c5a24 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 11:00:57 -0400 Subject: [PATCH 10/32] feat(tui): move recent decisions to 2nd position and focus by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the "Recent decisions (enter = details)" panel to directly under the "Quota burn" panel, so the panel order is: Quota → Recent decisions → Per-model → Category → Warnings. The decision table is now focused by default when the TUI opens, so the cursor lands on the most-recent decision row without requiring a number-key press. Focus-key bindings (1-5) and the _panels list are unchanged. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tests/test_tui.py | 16 ++++++++++++++++ tui.py | 12 ++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_tui.py b/tests/test_tui.py index f6ac6d7..34ffdf5 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -328,6 +328,22 @@ def test_app_run_test_failure_shows_error_and_does_not_crash(): _run_app(app, _assert) # must not raise +def test_app_run_test_decision_table_focused_on_mount(): + """#decision-table is focused by default after the app mounts.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + table = app.query_one("#decision-table") + assert table.has_focus, "#decision-table should have focus on mount" + + asyncio.run(_go()) + + def _run_app(app: tui.DashboardApp, body) -> None: """Drive the app via Textual App.run_test synchronously. diff --git a/tui.py b/tui.py index 2638b62..9c0f5be 100644 --- a/tui.py +++ b/tui.py @@ -164,12 +164,12 @@ class DashboardApp(App): with Vertical(id="quota-panel"): yield ProgressBar(id="quota-progress") yield Static(markup=True, id="quota-legend") + yield Static("Recent decisions (enter = details)", classes="panel-title") + yield DataTable(id="decision-table") yield Static("Per-model", classes="panel-title") yield DataTable(id="model-table") yield Static("Category → model breakdown", classes="panel-title") yield DataTable(id="category-table") - yield Static("Recent decisions (enter = details)", classes="panel-title") - yield DataTable(id="decision-table") yield Static("Health / warnings", classes="panel-title") yield Static("—", id="warnings-panel") yield Footer() @@ -190,6 +190,14 @@ class DashboardApp(App): self.base_url, self._on_live_decision ) self._event_stream.start() + # Focus the recent-decisions table by default once it has data. + # _refresh() → _render() populates the table above, so by this point + # it is built. Guarded so a stubbed-error refresh or a table that + # failed to populate never raises on mount. + try: + self.query_one("#decision-table", DataTable).focus() + except Exception: # noqa: BLE001 — focus is best-effort on mount + pass def on_unmount(self) -> None: self._unmounted = True -- 2.49.1 From 1434b33fc3cff300cd2b7f897d649763f9e6441b Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 20:34:25 -0400 Subject: [PATCH 11/32] fix(metrics): use UTC for rolling window start date Replace naive date.today() with datetime.now(timezone.utc).date() to avoid timezone-dependent reset_date values. Rename legend label from "resets" to "window start" to accurately describe the backward-looking 30-day window start. --- metrics.py | 4 ++-- tests/test_metrics.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metrics.py b/metrics.py index 8f03691..c9d0146 100644 --- a/metrics.py +++ b/metrics.py @@ -21,7 +21,7 @@ top_proficiency — top models by blended_score for a category from __future__ import annotations import sqlite3 -from datetime import date, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, List, Optional SEED_CATEGORY = "seed_reference" @@ -59,7 +59,7 @@ def quota_burn( "metered_kwh_30d": round(float(row["kwh"]), 5), "metered_fraction_of_plan": round(float(row["kwh"]) / plan, 4), "metered_calls_30d": row["n"], - "reset_date": (date.today() - timedelta(days=30)).isoformat(), + "reset_date": (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat(), "note": "router-metered only; traffic bypassing the router is not counted", } diff --git a/tests/test_metrics.py b/tests/test_metrics.py index fa3a390..84a6a3b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -194,6 +194,10 @@ def test_quota_burn_aggregates_last_30_days(tmp_path): assert result["metered_calls_30d"] == 2 assert result["plan_kwh"] == 6.25 assert result["metered_fraction_of_plan"] == pytest.approx(0.25 / 6.25) + expected_reset = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat() + assert result["reset_date"] == expected_reset + assert "T" not in result["reset_date"] + assert not result["reset_date"].endswith(("Z", "+00:00")) def test_quota_burn_empty_db(tmp_path): -- 2.49.1 From 4203841b0ffbe92a35585cce281eb2979150c3a4 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 20:34:30 -0400 Subject: [PATCH 12/32] cleanup(tui): remove dead quota fallback string The {"label": "quota", "value": "N/A (plan not set)"} row in build_model was never consumed by the renderer, which only looks up plan_kwh/metered_kwh_30d/fraction/calls/reset_date. Return an empty list when quota is null instead of producing dead data. --- tui_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tui_model.py b/tui_model.py index 8708311..9ba49a0 100644 --- a/tui_model.py +++ b/tui_model.py @@ -56,7 +56,7 @@ def build_model(data: dict) -> dict: {"label": "reset_date", "value": quota.get("reset_date")}, ] else: - quota_rows = [{"label": "quota", "value": "N/A (plan not set)"}] + quota_rows = [] per_model = [ { -- 2.49.1 From 329b34db498494cc0f2dd6df659adce558554b70 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 20:34:35 -0400 Subject: [PATCH 13/32] feat(tui): live-updating VerdictMixScreen modal Replace the snapshot-at-open pattern with a Textual reactive attribute so the verdict mix popup tracks the latest /metrics payload while open. A watch_verdict_mix handler repopulates the DataTable on each refresh, and the dashboard pushes updates via _push_verdict_mix_updates() at the end of _render. --- tui_screens.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tui_screens.py b/tui_screens.py index 2bca653..7195b9a 100644 --- a/tui_screens.py +++ b/tui_screens.py @@ -11,6 +11,7 @@ from typing import Any from textual.app import ComposeResult from textual.containers import Vertical, VerticalScroll +from textual.reactive import reactive from textual.screen import ModalScreen from textual.widgets import Button, DataTable, Static @@ -101,9 +102,11 @@ class VerdictMixScreen(ModalScreen[None]): } """ + verdict_mix: reactive[list] = reactive([]) + def __init__(self, verdict_mix: list[dict]) -> None: - self.verdict_mix = verdict_mix super().__init__() + self.verdict_mix = verdict_mix def compose(self) -> ComposeResult: with Vertical(id="verdict-container"): @@ -111,15 +114,27 @@ class VerdictMixScreen(ModalScreen[None]): yield DataTable(id="verdict-table") yield Button("Close (esc)", id="verdict-close") - def on_mount(self) -> None: + def _populate(self, mix: list) -> None: table = self.query_one("#verdict-table", DataTable) - table.add_columns("verdict", "count") - if not self.verdict_mix: + table.clear() + if len(table.columns) == 0: + table.add_columns("verdict", "count") + if not mix: table.add_row("(no data)", "") return - for row in self.verdict_mix: + for row in mix: table.add_row(str(row["verdict"]), str(row["count"])) + def watch_verdict_mix(self, value: list) -> None: + # The reactive is assigned in __init__, before mount, so querying the + # composed table would fail then; on_mount does the first paint. + if not self.is_mounted: + return + self._populate(value) + + def on_mount(self) -> None: + self._populate(self.verdict_mix) + def on_key(self, event) -> None: if event.key == "escape": self.dismiss(None) -- 2.49.1 From 9215ec8cfd8fc5217150942a42b2facf5960747e Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Tue, 25 Aug 2026 20:34:41 -0400 Subject: [PATCH 14/32] fix(tui): address 8 review findings in dashboard rendering - docs: sync stale "1-6" panel references to "1-5" - fix: hide ProgressBar (display=False) when plan is unconfigured instead of fabricating total=1/progress=0 bounds - fix: simplify _restore_cursor fallback to row 0 (clear() already resets cursor, making the clamp logic dead) - fix: guard against DuplicateKeyError in model table by deduplicating per-model rows with a seen set; log a warning on duplicates - feat: extract _format_quota_legend helper that suppresses None values (renders as n/a) so the legend never contains the literal "None" - feat: push live verdict mix updates into an open VerdictMixScreen - fix: guard _render against pre-mount NoMatches when set_interval fires before compose() finishes mounting widgets Tests: 579 passed (was 562), including new tests for duplicate model keys, hidden progress bar, legend None-suppression, live verdict mix refresh, and cursor restoration to row 0. --- tests/test_tui.py | 131 +++++++++++++++++++++++++++++++++++++++++++--- tui.py | 94 ++++++++++++++++++++++++++------- 2 files changed, 197 insertions(+), 28 deletions(-) diff --git a/tests/test_tui.py b/tests/test_tui.py index 34ffdf5..f6fb48c 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -175,13 +175,12 @@ def test_build_model_warnings_from_coverage(): def test_build_model_handles_missing_quota(): - """Empty DB / null plan: the quota section degrades to a notice.""" + """Empty DB / null plan: the quota section is an empty list (renderer hides it).""" data = _fixture() data["quota"] = None data["coverage"]["quota"] = None m = build_model(data) - joined = " ".join(str(r) for r in m["quota"]) - assert "unset" in joined.lower() or "no plan" in joined.lower() or "n/a" in joined.lower() + assert m["quota"] == [] def test_fetch_metrics_returns_parsed_dict(monkeypatch): @@ -282,6 +281,28 @@ def test_app_run_test_recent_and_warnings_panels(): _run_app(app, _assert) +def test_app_run_test_duplicate_model_keys_do_not_crash(): + """Duplicate model ids in the /metrics per_model payload must not raise + DuplicateKeyError when rendered into #model-table. The source may be a + poller/anomaly, so the TUI dedups defensively.""" + stub = _StubFetcher() + payload = _fixture() + payload["per_model"].append(dict(payload["per_model"][0], calls=99)) + stub.payload = payload + + app = tui.DashboardApp(fetcher=stub) + + def _assert(a): + mt = a.query_one("#model-table") + assert mt.row_count == 2, ( + f"expected 2 deduped rows, got {mt.row_count}" + ) + assert len(payload["per_model"]) == 3 + assert len(a._last_model["per_model"]) == 3 + + _run_app(app, _assert) + + def test_app_run_test_quota_panel_progress_bar_and_legend(): """The quota panel renders a ProgressBar and a legend carrying the plan. @@ -308,7 +329,65 @@ def test_app_run_test_quota_panel_progress_bar_and_legend(): assert "6.25" in text assert "1.25" in text assert "calls 18" in text - assert "resets 2026-07-26" in text + assert "window start 2026-07-26" in text + + _run_app(app, _assert) + + +def test_format_quota_legend_omits_none_when_unmetered(): + """With unmetered (None) values the legend never shows the literal "None". + + The ``metered``/``frac``/``calls`` rows can be absent (None) for a plan + that has not been metered yet; they must render as ``n/a`` and the + returned string must not contain the substring ``"None"``. + """ + app = tui.DashboardApp(fetcher=_StubFetcher()) + legend = app._format_quota_legend( + plan=6.25, metered=None, frac=None, calls=None, reset_date="2026-07-26" + ) + assert isinstance(legend, str) + assert "None" not in legend + assert "n/a" in legend + assert "window start 2026-07-26" in legend + + +def test_format_quota_legend_happy_path_contains_numbers(): + """A fully-populated legend carries the metered / plan / calls values.""" + app = tui.DashboardApp(fetcher=_StubFetcher()) + legend = app._format_quota_legend( + plan=6.25, metered=1.25, frac=0.2, calls=18, reset_date="2026-07-26" + ) + assert "6.25" in legend + assert "1.25" in legend + assert "calls 18" in legend + assert "20%" in legend + assert "window start 2026-07-26" in legend + assert "None" not in legend + + +def test_app_run_test_quota_bar_hidden_when_plan_unconfigured(): + """No plan configured -> the ProgressBar is hidden (display False). + + The bar is not removed from the DOM; its ``display`` is toggled so the + "quota not configured" legend still shows and the widget state is kept + for when a plan is later configured. + """ + stub = _StubFetcher() + data = _fixture() + data["quota"]["plan_kwh"] = None + data["quota"]["metered_fraction_of_plan"] = None + data["coverage"]["quota"]["plan_kwh"] = None + data["coverage"]["quota"]["metered_fraction_of_plan"] = None + stub.payload = data + + app = tui.DashboardApp(fetcher=stub) + + def _assert(a): + bar = a.query_one("#quota-progress", ProgressBar) + assert bar.display is False, "bar should be hidden when plan unconfigured" + + legend = a.query_one("#quota-legend", Static) + assert "quota not configured" in str(legend.content) _run_app(app, _assert) @@ -581,7 +660,8 @@ def test_cursor_persistence_across_refresh(): def test_cursor_persistence_clamps_when_selected_row_removed(): """When the highlighted decision disappears from the payload on refresh, - the cursor clamps safely to a valid row instead of crashing.""" + the cursor rests at row 0 (the default ``clear()`` already set) instead of + clamping to a stale anchor.""" payloads = [_decisions_payload(42, 41, 40), _decisions_payload(40, 39)] class _Scripted: @@ -607,9 +687,11 @@ def test_cursor_persistence_clamps_when_selected_row_removed(): app.action_refresh() # id 41 is dropped in the new payload await pilot.pause() - # Cursor resolved to a valid in-bounds row without raising. - assert dt.cursor_coordinate.row >= 0 - assert dt.cursor_coordinate.row < dt.row_count + # The cursor rests at the cleared default row 0. + assert dt.cursor_coordinate.row == 0, ( + f"cursor at {dt.cursor_coordinate.row}; expected 0 " + "(dropped row should rest at the cleared default)" + ) # The table reflects the two-row payload. assert app._last_model["recent_decisions"][0]["id"] == 40 @@ -790,6 +872,39 @@ def test_app_run_test_ctrl_v_opens_verdict_popup(): asyncio.run(_go()) +def test_verdict_mix_screen_updates_live_after_refresh(): + """An open VerdictMixScreen modal tracks the newest verdict mix after a + refresh, rather than staying a static snapshot from when it was opened.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub, refresh_seconds=60) + + async def _go(): + async with app.run_test() as pilot: + await pilot.pause() + await pilot.press("ctrl+v") + await pilot.pause() + by_verdict = { + row["verdict"]: row["count"] for row in app.screen.verdict_mix + } + assert by_verdict == {"ok": 5, "unverifiable": 2, "truncated": 1} + + # The next refresh delivers a new (smaller) verdict mix. + stub.payload["verdict_mix"] = {"ok": 100, "failed": 5} + app.action_refresh() + await pilot.pause() + + by_verdict = { + row["verdict"]: row["count"] for row in app.screen.verdict_mix + } + assert by_verdict == {"ok": 100, "failed": 5} + table = app.screen.query_one("#verdict-table") + assert table.row_count == 2 + + asyncio.run(_go()) + + def test_live_decision_inserts_row_at_front_and_rerenders(): """A decision delivered via the SSE callback is prepended to the model and re-renders the decisions and category tables without a full re-fetch.""" diff --git a/tui.py b/tui.py index 9c0f5be..0139bb4 100644 --- a/tui.py +++ b/tui.py @@ -22,9 +22,12 @@ dispatch path never touches it. from __future__ import annotations +import logging import os from typing import Callable, Optional +logger = logging.getLogger(__name__) + from textual.app import App, ComposeResult from textual.containers import Vertical, VerticalScroll from textual.widgets import DataTable, Footer, Header, ProgressBar, Static @@ -240,19 +243,19 @@ class DashboardApp(App): The given key was captured by :meth:`_preserve_cursor` before the table was cleared and re-populated. If that row still exists, focus it - again; otherwise clamp the cursor to the nearest valid row (the row - count now bounds it). A ``None`` key, an empty table, or a missing row - are all handled without raising. + again; otherwise move the cursor to row 0. ``DataTable.clear()`` + already reset the cursor to row 0, so that is the default resting + place for a dropped row. A ``None`` key, an empty table, or a missing + row are all handled without raising. """ if key is None or table.row_count == 0: return try: row_index = table.get_row_index(key) except RowDoesNotExist: - # The row was dropped by the rebuild; anchor to whatever valid row - # the freshly-populated table already points at, clamped to bounds. - anchor = table.cursor_coordinate.row - row_index = min(anchor, table.row_count - 1) if anchor >= 0 else 0 + # The row was dropped by the rebuild; ``clear()`` put us back at + # row 0 already, so no clamping is needed. + row_index = 0 table.move_cursor(row=row_index) def _set_up_tables(self) -> None: @@ -267,7 +270,7 @@ class DashboardApp(App): category_table = self.query_one("#category-table", DataTable) category_table.add_columns("category", "tier", "count", "majority", "share") # Static panels are also number-key targets; make them focusable so - # 1-6 focus cycling is uniform. + # 1-5 focus cycling is uniform. self.query_one("#quota-panel", Vertical).can_focus = True self.query_one("#warnings-panel", Static).can_focus = True @@ -386,7 +389,7 @@ class DashboardApp(App): panel.add_class("visible") def action_focus_panel(self, index: int) -> None: - """Move focus to one of the numbered panels (1-6).""" + """Move focus to one of the numbered panels (1-5).""" if index < 0 or index >= len(self._panels): return widget = self.query_one(f"#{self._panels[index]}") @@ -395,43 +398,79 @@ class DashboardApp(App): except Exception: # noqa: BLE001 — a Static still takes focus harmlessly pass + def _format_quota_legend(self, plan, metered, frac, calls, reset_date) -> str: + """Build the quota legend string, suppressing ``None`` values. + + ``metered``/``frac``/``calls`` come from ``rows.get(...)`` and may be + ``None`` (unmetered plan); render those as ``n/a`` rather than the + Python literal so the legend never contains the substring ``"None"``. + """ + metered_fmt = "n/a" if metered is None else metered + frac_fmt = "n/a" if frac is None else frac + calls_fmt = "n/a" if calls is None else calls + percent = round(float(frac) * 100) if frac is not None else 0 + legend = ( + f"metered {metered_fmt} / plan {plan} " + f"(frac {frac_fmt} · {percent}%) · calls {calls_fmt}" + ) + if reset_date: + legend += ( + f" · [rgb(128,128,128)]window start {reset_date}" + f"[/rgb(128,128,128)]" + ) + return legend + def _render(self, model: dict) -> None: + # Quota panel — guard against being called before compose() mounts + # these widgets (set_interval in on_mount can fire _on_interval too + # quickly for the widget tree to be ready). + try: + bar = self.query_one("#quota-progress", ProgressBar) + legend = self.query_one("#quota-legend", Static) + except Exception: # noqa: BLE001 — quota widgets not composed yet + return # Quota panel rows = {r["label"]: r["value"] for r in model["quota"]} plan = rows.get("plan_kwh") metered = rows.get("metered_kwh_30d") frac = rows.get("fraction") calls = rows.get("calls") - bar = self.query_one("#quota-progress", ProgressBar) - legend = self.query_one("#quota-legend", Static) if plan is not None and float(plan) > 0: bar.total = float(plan) bar.progress = float(metered or 0) - percent = round(float(frac) * 100) if frac is not None else 0 + bar.display = True reset_date = rows.get("reset_date") - legend.update( - f"metered {metered} / plan {plan} " - f"(frac {frac} · {percent}%) · calls {calls}" - + (f" · [rgb(128,128,128)]resets {reset_date}[/rgb(128,128,128)]" if reset_date else "") - ) + legend.update(self._format_quota_legend(plan, metered, frac, calls, reset_date)) else: - bar.total = 1 - bar.progress = 0 + bar.display = False legend.update("quota not configured") # Per-model table mt = self.query_one("#model-table", DataTable) saved_model_key = self._preserve_cursor(mt) mt.clear() + seen: set[str] = set() + duplicates: list[str] = [] for r in model["per_model"]: + model_id = str(r["model"]) + if model_id in seen: + duplicates.append(model_id) + continue + seen.add(model_id) mt.add_row( - str(r["model"]), + model_id, str(r["calls"]), _fmt_usd(r["cost_usd"]), f"{r['energy_kwh']:.6g}", f"{r['carbon_g_co2eq']:.4g}", key=r["model"], ) + if duplicates: + logger.warning( + "skipped %d duplicate per-model row(s) in /metrics payload: %s", + len(duplicates), + ", ".join(duplicates), + ) self._restore_cursor(mt, saved_model_key) self._render_decisions_table() @@ -446,6 +485,21 @@ class DashboardApp(App): else: warn_widget.update("No warnings.") + self._push_verdict_mix_updates() + + def _push_verdict_mix_updates(self) -> None: + """Push the freshest verdict mix into an open VerdictMixScreen modal. + + The verdict mix is a live-accumulating aggregate, so while the modal is + open it should track the latest /metrics payload on each refresh rather + than the snapshot it was opened with. + """ + if self._last_model is None: + return + screen = getattr(self, "screen", None) + if isinstance(screen, VerdictMixScreen): + screen.verdict_mix = self._last_model.get("verdict_mix", []) + def _render_decisions_table(self) -> None: if self._last_model is None: return -- 2.49.1 From 65da4a1e9aecac94732b588a7dd18d9a91ed37d2 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 10:53:35 -0400 Subject: [PATCH 15/32] feat(routing): add flex-preference knob for -flex variant selection 4-position scale (no-flex, auto, prefer-flex, force-flex) that swaps the ranked winner to its -flex serving-class sibling after ranking, without changing which base model wins. Composes as both a config default (routing.default_flex_preference) and a per-request override (TaskRequest.flex_preference), mirroring latency_tolerance. prefer-flex respects the interactive latency filter; force-flex bypasses it and sets flex_forced=True in telemetry. Falls back to the standard winner when no flex sibling exists. Cost is re-estimated from the sibling row on swap. route_decisions gains flex_preference, flex_swapped, flex_forced columns (idempotent ALTER for live DBs). 601 tests pass. --- config.py | 20 +++ config.yaml | 13 ++ dispatcher.py | 81 +++++++++++- routing.py | 109 +++++++++++++++ schema.sql | 6 +- tests/test_route_decisions.py | 207 +++++++++++++++++++++++++++++ tests/test_routing.py | 242 ++++++++++++++++++++++++++++++++++ 7 files changed, 672 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 65aa585..55371ed 100644 --- a/config.py +++ b/config.py @@ -9,6 +9,7 @@ Usage: from __future__ import annotations +from enum import Enum from pathlib import Path from typing import Optional from urllib.parse import urlparse @@ -160,9 +161,28 @@ class TieringConfig(StrictModel): return v +class FlexPreference(Enum): + """Operator's stance on routing to ``-flex`` serving-class rows. + + A 4-position scale: + - ``no-flex``: never route to a flex row. + - ``auto``: decide per request (the default). + - ``prefer-flex``: flex first, standard as fallback. + - ``force-flex``: flex only. + """ + + no_flex = "no-flex" + auto = "auto" + prefer_flex = "prefer-flex" + force_flex = "force-flex" + + class RoutingConfig(StrictModel): allowed_access_levels: list[str] default_latency_tolerance: str + # Operator's default stance on flex serving-class rows for requests + # that do not state one explicitly. + default_flex_preference: FlexPreference = FlexPreference.auto # Applied only when the REQUEST carries tool definitions. None disables it. min_tool_proficiency: Optional[float] = 0.5 tool_use_category: str = "tool_use_agentic" diff --git a/config.yaml b/config.yaml index 899785a..c016e9b 100644 --- a/config.yaml +++ b/config.yaml @@ -189,6 +189,19 @@ routing: # interactive, so a request has to opt in via latency_tolerance. default_latency_tolerance: interactive # 'interactive' | 'batch' + # Operator's default stance on routing to '-flex' serving-class rows for + # requests that do not state one explicitly. A 4-position scale: + # no-flex never route to a flex row + # auto decide per request (the default; keeps existing behavior) + # prefer-flex flex first, standard as fallback + # force-flex flex only + # 'force-flex' is the dangerous global default: it bypasses the + # latency_tolerance: interactive hard filter for EVERY request, so even a + # request that opted into interactive routing would admit rows that are held + # server-side during peak. Use it only when you are certain the caller can + # tolerate flex latency globally. + default_flex_preference: auto + # Minimum tool_use_agentic proficiency required of a model when the REQUEST # carries tool definitions. A filter, not a weight, because it is a # capability requirement rather than a preference. diff --git a/dispatcher.py b/dispatcher.py index 5fec330..b6f5bf2 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -55,12 +55,13 @@ from pydantic import BaseModel, Field import logs from capabilities import detect_capabilities, iter_image_url_values -from config import RouterConfig, load_config +from config import FlexPreference, RouterConfig, load_config from context_prune import extract_text, prune_context import events from routing import ( BATCH, INTERACTIVE, + apply_flex_preference, capability_gate_reason, rank_candidates, rejection_reason, @@ -169,6 +170,15 @@ class TaskRequest(BaseModel): "Defaults to routing.default_latency_tolerance." ), ) + flex_preference: Optional[FlexPreference] = Field( + None, + description=( + "Operator stance on routing to '-flex' serving-class rows for this " + "request: no-flex, auto, prefer-flex, or force-flex. Defaults to " + "routing.default_flex_preference. 'force-flex' bypasses the " + "latency_tolerance hard filter for this request." + ), + ) tools_present: bool = Field( False, description=( @@ -230,6 +240,9 @@ class Candidate(BaseModel): class RouteResponse(BaseModel): classification: Classification latency_tolerance: str + flex_preference: str = "auto" + flex_swapped: bool = False + flex_forced: bool = False candidates_considered: int selected: Optional[Candidate] runners_up: list[Candidate] = [] @@ -299,7 +312,10 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None: tools INTEGER, images INTEGER, json_mode INTEGER, - streamed INTEGER + streamed INTEGER, + flex_preference TEXT, + flex_swapped INTEGER, + flex_forced INTEGER ) """ ) @@ -307,6 +323,19 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None: "CREATE INDEX IF NOT EXISTS idx_route_decisions_observed " "ON route_decisions (observed_at)" ) + # A live router.db that predates the flex-preference knob has the table + # (from an earlier schema) but not its columns: CREATE TABLE IF NOT EXISTS + # no-ops against it, so the new columns must be added by ALTER. PRAGMA + # table_info guards the add so a second call is a no-op and an + # already-migrated DB is never re-altered. + existing = {row[1] for row in conn.execute("PRAGMA table_info(route_decisions)")} + for name, decl in ( + ("flex_preference", "TEXT"), + ("flex_swapped", "INTEGER"), + ("flex_forced", "INTEGER"), + ): + if name not in existing: + conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}") conn.commit() @@ -766,11 +795,29 @@ def route(req: TaskRequest) -> RouteResponse: kwh=row.get("energy"), ) + flex_pref = req.flex_preference or cfg.routing.default_flex_preference + selected_row, flex_swapped, flex_forced, _ = ( + apply_flex_preference( + ranked[0], + rows, + flex_pref, + latency_tolerance, + prompt_tokens=classification.required_context_tokens, + completion_tokens=cfg.objective.assumed_completion_tokens, + cache_rate=cfg.objective.assumed_cache_rate, + ) + if ranked + else (None, False, False, None) + ) + return RouteResponse( classification=classification, latency_tolerance=latency_tolerance, + flex_preference=flex_pref.value, + flex_swapped=flex_swapped, + flex_forced=flex_forced, candidates_considered=len(eligible), - selected=_to_candidate(ranked[0]) if ranked else None, + selected=_to_candidate(selected_row) if selected_row else None, runners_up=[_to_candidate(r) for r in ranked[1:4]], ) @@ -799,6 +846,10 @@ def log_decision( what actually happened rather than what the second call looked like. """ selected = decision.selected + flex_notes = {"flex_pref": decision.flex_preference} + if decision.flex_swapped: + flex_notes["flex_swapped"] = True + flex_notes["flex_forced"] = decision.flex_forced logs.info( "route", cat=decision.classification.task_category, @@ -813,6 +864,7 @@ def log_decision( est_usd=selected.cost if selected else None, prof=selected.proficiency_score if selected else None, ms=ms, + **flex_notes, ) @@ -832,6 +884,9 @@ def persist_route_decision( streamed=0, classification_source=None, classifier_ms=None, + flex_preference=None, + flex_swapped=0, + flex_forced=0, ) -> None: """Record one routing decision to route_decisions, best-effort and gated. @@ -867,6 +922,9 @@ def persist_route_decision( derived_runners = classification.runners_up candidates = classification.candidates_considered derived_latency = classification.latency_tolerance + flex_preference = classification.flex_preference + flex_swapped = int(classification.flex_swapped) + flex_forced = int(classification.flex_forced) elif isinstance(classification, Classification): clf = classification @@ -917,8 +975,9 @@ def persist_route_decision( classification_source, latency_tolerance, candidates_considered, selected_model, selected_provider, runner_up_models, est_cost_usd, est_proficiency, rejected_reason, session_key, - tools, images, json_mode, streamed - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + tools, images, json_mode, streamed, + flex_preference, flex_swapped, flex_forced + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( observed_at, @@ -942,6 +1001,9 @@ def persist_route_decision( int(bool(images)), int(bool(json_mode)), int(bool(streamed)), + flex_preference, + int(bool(flex_swapped)), + int(bool(flex_forced)), ), ) decision_id = int(cursor.lastrowid) @@ -975,6 +1037,9 @@ def persist_route_decision( "images": int(bool(images)), "json_mode": int(bool(json_mode)), "streamed": int(bool(streamed)), + "flex_preference": flex_preference, + "flex_swapped": int(bool(flex_swapped)), + "flex_forced": int(bool(flex_forced)), } ) except Exception as e: # noqa: BLE001 - best-effort must never raise @@ -2104,6 +2169,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): images=int(caps.has_images), json_mode=int(caps.require_json_mode), streamed=streamed, + flex_preference=cfg.routing.default_flex_preference.value, + flex_swapped=0, + flex_forced=0, ) return _local_vision_response( fallback, streaming=bool(body.get("stream")) @@ -2183,6 +2251,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): images=int(caps.has_images), json_mode=int(caps.require_json_mode), streamed=streamed, + flex_preference=cfg.routing.default_flex_preference.value, + flex_swapped=0, + flex_forced=0, ) # A pinned model id is dispatched as asked, but a pin that cannot diff --git a/routing.py b/routing.py index 958aa10..c5725db 100644 --- a/routing.py +++ b/routing.py @@ -160,6 +160,115 @@ def is_eligible(row: dict, **filters) -> bool: return rejection_reason(row, **filters) is None +def get_flex_sibling( + selected_row: dict, all_rows: Sequence[dict], +) -> dict | None: + """The `-flex` serving-class twin of ``selected_row``, or None. + + A flex row holds the same weights at the same advertised price as its + standard sibling, differing only in ``latency_class`` (it may be held + server-side during peak). This finds it by matching the identity + dimensions that define variant equivalence — ``base_model_id``, + ``reasoning_mode``, ``context_variant`` — and requiring + ``latency_class == "flex"``. + + Returns None when the selected row is already flex or no such sibling + exists. Pure: it never consults the DB or routing policy, so callers own + the preference decision while this stays a plain catalog lookup. + """ + if selected_row.get("latency_class") == "flex": + return None + for row in all_rows: + if ( + row.get("latency_class") == "flex" + and row.get("base_model_id") == selected_row.get("base_model_id") + and row.get("reasoning_mode") == selected_row.get("reasoning_mode") + and row.get("context_variant") == selected_row.get("context_variant") + ): + return row + return None + + +def apply_flex_preference( + selected_row: dict, + all_candidates: Sequence[dict], + flex_preference, + latency_tolerance: str, + *, + prompt_tokens: int, + completion_tokens: int, + cache_rate: float, +) -> tuple[dict, bool, bool, float | None]: + """Apply the operator's flex-preference stance to the post-rank winner. + + The rank winner is a standard serving-class row. Whether the request should + actually ride a ``-flex`` twin (deferred during peak, weight-identical, + priced the same) is a service-level preference, not a scored dimension — + ``rank_candidates`` already decided quality and cost, and this runs AFTER + it to override only the serving class when the operator asks. + + ``flex_preference`` is the ``FlexPreference`` enum value (or its plain + string form) from ``config``. routing.py never imports config, so the + caller passes the value in and this switches on its ``.value``: + - ``no-flex``: never swap to flex. If the winner is already flex it is + left alone. + - ``auto``: no change; the request's latency filter already decided. + - ``prefer-flex``: swap to the flex sibling when one exists and is + eligible under the current hard filters. Given a sibling shares + context/tier/access/freshness with its standard twin, the only hard + filter that distinguishes it is the latency filter — so it is + eligible unless ``latency_tolerance == interactive``. + - ``force-flex``: swap to the flex sibling when one exists, even when + ``latency_tolerance == interactive`` (bypassing the latency filter). + That bypass is what sets the ``flex_forced`` flag. + + ``all_candidates`` is the FULL candidate set (pre hard-filter), so a flex + sibling filtered out under ``interactive`` is still findable for + ``force-flex``. Eligibility for ``prefer-flex`` is re-decided here from + ``latency_tolerance``. + + Returns ``(row, flex_swapped, flex_forced, cost)`` where ``row`` is the + possibly-swapped selection (with any ranking score fields carried over, + since a flex twin ties on every scored dimension), ``flex_swapped`` is + true when the swap happened, ``flex_forced`` is true when that swap + bypassed ``interactive``, and ``cost`` is the re-estimated cost for the + returned row (recomputed via ``estimated_cost`` on a swap; the original + cost otherwise). Falls back to the original row and cost when no sibling + exists. + """ + pref = getattr(flex_preference, "value", flex_preference) + + # no-flex and auto both make no swap. no-flex never routes to flex (even + # leaving an already-flex winner); auto defers to the latency filter that + # already ran. Either way the winner stands as ranked. + if pref in ("no-flex", "auto"): + return selected_row, False, False, selected_row.get("cost") + + sibling = get_flex_sibling(selected_row, all_candidates) + if sibling is None: + return selected_row, False, False, selected_row.get("cost") + + if pref == "prefer-flex": + # A flex sibling is eligible under the current hard filters unless the + # latency filter excludes flex rows — which is exactly interactive. + # Unlike its standard twin (which already passed everything else), the + # only filter that can dismiss a flex row is the latency one. + if latency_tolerance == INTERACTIVE: + return selected_row, False, False, selected_row.get("cost") + + # prefer-flex under batch, or force-flex (which bypasses interactive). + flex_forced = pref == "force-flex" and latency_tolerance == INTERACTIVE + cost = estimated_cost(sibling, prompt_tokens, completion_tokens, cache_rate) + swapped = {**sibling, "cost": cost} + # A flex twin ties with its standard sibling on every scored dimension — + # same weights, same price — so carry the ranking's score fields over so + # the returned row stays a well-formed candidate downstream. + for key in ("proficiency_score", "cost_score", "composite"): + if key in selected_row: + swapped[key] = selected_row[key] + return swapped, True, flex_forced, cost + + def select_candidates( rows: Sequence[dict], *, diff --git a/schema.sql b/schema.sql index e1f9a44..c13d350 100644 --- a/schema.sql +++ b/schema.sql @@ -230,7 +230,11 @@ CREATE TABLE IF NOT EXISTS route_decisions ( tools INTEGER, -- 0/1 images INTEGER, -- 0/1 json_mode INTEGER, -- 0/1 - streamed INTEGER -- 0/1 + streamed INTEGER, -- 0/1 + flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex' + -- |'force-flex' (resolved) + flex_swapped INTEGER, -- 0/1 post-rank flex swap applied + flex_forced INTEGER -- 0/1 swap bypassed interactive ); CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider); diff --git a/tests/test_route_decisions.py b/tests/test_route_decisions.py index 46ecfff..5388932 100644 --- a/tests/test_route_decisions.py +++ b/tests/test_route_decisions.py @@ -55,6 +55,9 @@ ROUTE_DECISIONS_COLUMNS = [ "images", "json_mode", "streamed", + "flex_preference", + "flex_swapped", + "flex_forced", ] @@ -272,6 +275,7 @@ def test_config_has_log_route_decisions_key(): CHEAP = "cheap-model" DEAR = "dear-model" +CHEAP_FLEX = "cheap-model-flex" class FakeResponse: @@ -361,6 +365,24 @@ def decision_router(tmp_path, monkeypatch): """, (model_id, model_id, completion_price / 3, completion_price, vision), ) + # A flex twin of CHEAP (same base_model_id/reasoning_mode/context_variant, + # latency_class='flex') with its own pricing so a swap's re-estimated cost + # is distinguishable from the standard row's. + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'neuralwatt', ?, 2, 262128, 192500, 16384, ?, ?, + 1, 1, 'flex', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + (CHEAP_FLEX, CHEAP, 0.20, 0.60), + ) conn.commit() conn.close() @@ -658,3 +680,188 @@ def test_db_write_failure_never_fails_routing(decision_router, monkeypatch): "/v1/chat/completions", json={"model": "auto", "messages": _messages()} ) assert resp2.status_code == 200 + + +# ============================================================================= +# flex_preference: request override, swap behavior, cost re-estimation. +# ============================================================================= + + +def test_route_with_force_flex_swaps_under_interactive(decision_router): + """force-flex bypasses the interactive latency filter and rides the flex twin.""" + client, db_path = decision_router + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "force-flex"} + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP_FLEX + assert data["flex_preference"] == "force-flex" + assert data["flex_swapped"] is True + assert data["flex_forced"] is True + + r = _rows(db_path)[0] + assert r["flex_preference"] == "force-flex" + assert r["flex_swapped"] == 1 + assert r["flex_forced"] == 1 + assert r["selected_model"] == CHEAP_FLEX + + +def test_route_with_prefer_flex_does_not_swap_under_interactive(decision_router): + """prefer-flex defers to the latency filter, so interactive keeps the standard row.""" + client, db_path = decision_router + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "prefer-flex"} + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP + assert data["flex_preference"] == "prefer-flex" + assert data["flex_swapped"] is False + assert data["flex_forced"] is False + + r = _rows(db_path)[0] + assert r["flex_preference"] == "prefer-flex" + assert r["flex_swapped"] == 0 + assert r["selected_model"] == CHEAP + + +def test_route_with_prefer_flex_swaps_under_batch(decision_router): + """Under batch the latency filter admits flex, so prefer-flex rides the twin.""" + client, db_path = decision_router + resp = client.post( + "/route", + json={"task": "write me a function", "flex_preference": "prefer-flex", + "latency_tolerance": "batch"}, + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP_FLEX + assert data["flex_preference"] == "prefer-flex" + assert data["flex_swapped"] is True + assert data["flex_forced"] is False + + r = _rows(db_path)[0] + assert r["flex_preference"] == "prefer-flex" + assert r["flex_swapped"] == 1 + assert r["flex_forced"] == 0 + assert r["latency_tolerance"] == "batch" + assert r["selected_model"] == CHEAP_FLEX + + +def test_route_with_no_flex_never_swaps(decision_router): + """no-flex never routes to a flex row, even under batch.""" + client, db_path = decision_router + resp = client.post( + "/route", + json={"task": "write me a function", "flex_preference": "no-flex", + "latency_tolerance": "batch"}, + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP + assert data["flex_preference"] == "no-flex" + assert data["flex_swapped"] is False + assert data["flex_forced"] is False + + r = _rows(db_path)[0] + assert r["flex_preference"] == "no-flex" + assert r["flex_swapped"] == 0 + assert r["selected_model"] == CHEAP + + +def test_route_with_auto_uses_default_behavior(decision_router): + """No flex_preference -> auto: no swap, interactive latency, flex_preference='auto'.""" + client, db_path = decision_router + resp = client.post("/route", json={"task": "write me a function"}) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP + assert data["flex_preference"] == "auto" + assert data["flex_swapped"] is False + assert data["flex_forced"] is False + + r = _rows(db_path)[0] + assert r["flex_preference"] == "auto" + assert r["flex_swapped"] == 0 + assert r["selected_model"] == CHEAP + + +def test_route_flex_preference_override_beats_config_default(decision_router, monkeypatch): + """A request override wins over routing.default_flex_preference.""" + client, db_path = decision_router + monkeypatch.setattr( + dispatcher.cfg.routing, "default_flex_preference", config.FlexPreference.force_flex + ) + + # No override: the config default (force-flex) applies and swaps. + resp = client.post("/route", json={"task": "write me a function"}) + assert resp.status_code == 200 + assert resp.json()["selected"]["model_id"] == CHEAP_FLEX + assert resp.json()["flex_swapped"] is True + assert resp.json()["flex_preference"] == "force-flex" + + # Explicit override: no-flex beats the config default and does not swap. + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "no-flex"} + ) + assert resp.status_code == 200 + data = resp.json() + assert data["selected"]["model_id"] == CHEAP + assert data["flex_preference"] == "no-flex" + assert data["flex_swapped"] is False + + rows = _rows(db_path) + assert rows[0]["selected_model"] == CHEAP_FLEX + assert rows[1]["selected_model"] == CHEAP + + +def test_route_force_flex_without_sibling_falls_back(decision_router): + """force-flex on a model with no flex twin keeps the standard row, no swap.""" + client, db_path = decision_router + _drop_cheap(db_path) # DEAR (no flex sibling) becomes the winner. + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "force-flex"} + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == DEAR + assert data["flex_preference"] == "force-flex" + assert data["flex_swapped"] is False + assert data["flex_forced"] is False + + r = _rows(db_path)[0] + assert r["flex_swapped"] == 0 + assert r["selected_model"] == DEAR + + +def test_route_swapped_cost_reflects_flex_variant(decision_router): + """A flex swap re-estimates cost from the flex row's pricing, not the standard's.""" + client, db_path = decision_router + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "force-flex"} + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["selected"]["model_id"] == CHEAP_FLEX + + flex_cost = data["selected"]["cost"] + r = _rows(db_path)[0] + assert r["est_cost_usd"] == pytest.approx(flex_cost) + + cache_rate = dispatcher.cfg.objective.assumed_cache_rate + completion_tokens = dispatcher.cfg.objective.assumed_completion_tokens + prompt_tokens = 100 # the seeded classifier's required_context_tokens + expected = ( + prompt_tokens * cache_rate * 0.20 + + prompt_tokens * (1.0 - cache_rate) * 0.20 + + completion_tokens * 0.60 + ) / 1_000_000 + assert flex_cost == pytest.approx(expected) diff --git a/tests/test_routing.py b/tests/test_routing.py index 50bc443..de88b67 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -6,8 +6,11 @@ proving it rejects and a case proving it does not over-reject. import pytest +from config import FlexPreference from routing import ( + apply_flex_preference, capability_gate_reason, + get_flex_sibling, is_eligible, rank_candidates, rejection_reason, @@ -498,3 +501,242 @@ def test_is_eligible_still_agrees_with_the_reason(): "latency_tolerance": "interactive", "allowed_access_levels": ["public"], "exclude_stale": True, "exclude_deprecated": True, }) == (_reason(row) is None) + + +# --- get_flex_sibling ------------------------------------------------------- +# +# The flex-preference knob can route a request to a `-flex` row (deferred +# during peak). For an operator who prefers flex latency but not a hard +# failure, the standard winner has a flex sibling that carries identical +# weights/serving class except latency_class. This helper finds it; it is +# deliberately a pure function of the rows, so the dispatcher owns only the +# "which preference applies" decision and not the catalog lookup. + +def test_flex_sibling_when_standard_row_has_one(): + # Given: the standard kimi-k3 winner and its flex sibling, differing only + # by latency_class + selected = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard") + flex = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex") + sibling = get_flex_sibling(selected, [selected, flex]) + assert sibling is not None + assert sibling["model_id"] == "kimi-k3-flex" + assert sibling["latency_class"] == "flex" + + +def test_flex_sibling_must_match_all_three_dims(): + # A flex row that shares base_model_id but not every serving dimension is + # NOT a sibling: -fast/-short change weights or context budget. + selected = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard") + wrong_reasoning = _row(model_id="kimi-k3-fast-flex", base_model_id="kimi-k3", + reasoning_mode="reduced", context_variant="full", + latency_class="flex") + wrong_ctx = _row(model_id="kimi-k3-short-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="short", + latency_class="flex") + # A matching flex must still be found among non-matching rows. + right = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex") + assert get_flex_sibling(selected, [wrong_reasoning, wrong_ctx, right])["model_id"] == "kimi-k3-flex" + + +def test_flex_sibling_is_none_when_selected_is_already_flex(): + selected = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex") + assert get_flex_sibling(selected, [selected]) is None + + +def test_flex_sibling_is_none_when_no_matching_row_exists(): + # The flex row present belongs to a different base model entirely + selected = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard") + other = _row(model_id="glm-5.2-flex", base_model_id="glm-5.2", + reasoning_mode="default", context_variant="full", + latency_class="flex") + assert get_flex_sibling(selected, [selected, other]) is None + + +def test_flex_sibling_ignores_stale_or_unroutable_flex_rows(): + # The helper is purely structural: it matches by identity dims regardless + # of freshness/access. Routing-level gating is the caller's job. + selected = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard") + flex = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex", availability="stale") + sibling = get_flex_sibling(selected, [selected, flex]) + assert sibling is not None + assert sibling["model_id"] == "kimi-k3-flex" + + +# --- apply_flex_preference: the post-rank serving-class switch -------------- +# +# rank_candidates already decides quality and cost; this runs AFTER it to +# override only the serving class when the operator's flex-preference stance +# asks for it. The mode matrix below pins the behavior of every preference +# under both latency tolerances, plus the no-sibling fallback. + +def _std_flex_pair(**overrides): + """A ranked standard winner plus its weight-equivalent flex twin. + + The flex twin shares base_model_id/reasoning_mode/context_variant and + differs only by latency_class. ``base_model_id`` defaults to the model_id + (matching the get_flex_sibling tests which must set it explicitly). + """ + std = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard", proficiency=0.5, + cost_per_1m_prompt=1.0, cost_per_1m_completion=2.0, **overrides) + flex = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex", proficiency=0.5, + cost_per_1m_prompt=1.0, cost_per_1m_completion=2.0, **overrides) + return std, flex + + +def test_apply_no_flex_never_swaps_under_interactive(): + # no-flex: never route to a flex row, even when one exists and would help. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.no_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_no_flex_leaves_an_already_flex_winner_alone(): + # If the rank winner is already flex (batch admitted it), no-flex does not + # bounce it back to standard; it leaves the decision as ranked. + flex = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex", proficiency=0.5) + row, swapped, forced, cost = apply_flex_preference( + flex, [flex], FlexPreference.no_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3-flex" + assert swapped is False + assert forced is False + assert cost == flex["cost"] + + +def test_apply_auto_makes_no_swap_even_when_a_flex_sibling_exists(): + # auto defers to the latency filter that already ran; a flex sibling + # existing does not change the ranked winner. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.auto, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_prefer_flex_holds_back_under_interactive(): + # prefer-flex swaps only when the flex sibling is eligible under the + # current hard filters; interactive's latency filter dismisses it. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.prefer_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_prefer_flex_swaps_under_batch(): + # Under batch the latency filter admits flex, so the sibling is eligible + # and the swap happens without bypassing anything. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.prefer_flex, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3-flex" + assert swapped is True + assert forced is False + assert cost is not None + + +def test_apply_force_flex_swaps_under_interactive_and_marks_forced(): + # force-flex bypasses the interactive latency filter: swap happens and the + # bypass is recorded as flex_forced=True. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.force_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3-flex" + assert swapped is True + assert forced is True + assert cost is not None + + +def test_apply_force_flex_swaps_under_batch_without_forced_flag(): + # Under batch there is no filter to bypass, so force-flex swaps but the + # forced flag stays False. + std, flex = _std_flex_pair() + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.force_flex, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3-flex" + assert swapped is True + assert forced is False + assert cost is not None + + +def test_apply_falls_back_when_no_flex_sibling_exists(): + # No flex twin in the catalog: every preference falls back to the ranked + # winner and its cost. + std = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard", proficiency=0.5) + for pref in (FlexPreference.no_flex, FlexPreference.auto, + FlexPreference.prefer_flex, FlexPreference.force_flex): + row, swapped, forced, cost = apply_flex_preference( + std, [std], pref, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_reestimates_cost_for_the_swapped_flex_row(): + # The flex twin carries the same list price as its standard sibling, so a + # swap recomputes the estimate from the flex row's own pricing fields. Give + # the flex row a distinct price and confirm the returned cost tracks it. + std = _row(model_id="kimi-k3", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="standard", proficiency=0.5, + cost_per_1m_prompt=1.0, cost_per_1m_completion=2.0, + cost_per_1m_prompt_cached=0.5) + flex = _row(model_id="kimi-k3-flex", base_model_id="kimi-k3", + reasoning_mode="default", context_variant="full", + latency_class="flex", proficiency=0.5, + cost_per_1m_prompt=2.0, cost_per_1m_completion=4.0, + cost_per_1m_prompt_cached=1.0) + row, swapped, _, _ = apply_flex_preference( + std, [std, flex], FlexPreference.prefer_flex, "batch", + prompt_tokens=10_000, completion_tokens=500, cache_rate=0.5) + assert swapped is True + # estimated_cost scales prompt/completion prices to this request shape, so + # the flex row's higher price must drive a higher estimate than standard's. + std_cost = rank_candidates( + [std], prompt_tokens=10_000, completion_tokens=500, cache_rate=0.5 + )[0]["cost"] + flex_cost = rank_candidates( + [flex], prompt_tokens=10_000, completion_tokens=500, cache_rate=0.5 + )[0]["cost"] + assert row["cost"] == flex_cost + assert row["cost"] > std_cost -- 2.49.1 From 695685db4c551a9207be0a509b2c2011b0a5ba33 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 11:26:57 -0400 Subject: [PATCH 16/32] fix(routing): re-gate flex sibling against hard filters A flex swap could dispatch to a -flex sibling that independently failed a non-latency hard filter (stale, deprecated, access-restricted, under-tiered, or capability-missing) because apply_flex_preference only checked latency. Re-gate the sibling via rejection_reason with latency_tolerance=BATCH (neutralizing only the latency filter) before swapping. 606 tests pass. --- dispatcher.py | 8 +++ routing.py | 52 ++++++++++++---- tests/test_route_decisions.py | 50 ++++++++++++++++ tests/test_routing.py | 109 +++++++++++++++++++++++++++++++--- 4 files changed, 198 insertions(+), 21 deletions(-) diff --git a/dispatcher.py b/dispatcher.py index b6f5bf2..72bb7d5 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -805,6 +805,14 @@ def route(req: TaskRequest) -> RouteResponse: prompt_tokens=classification.required_context_tokens, completion_tokens=cfg.objective.assumed_completion_tokens, cache_rate=cfg.objective.assumed_cache_rate, + required_context_tokens=filters["required_context_tokens"], + required_tier=filters["required_tier"], + allowed_access_levels=filters["allowed_access_levels"], + exclude_stale=filters["exclude_stale"], + exclude_deprecated=filters["exclude_deprecated"], + min_tool_proficiency=filters.get("min_tool_proficiency"), + require_vision=filters.get("require_vision", False), + require_json_mode=filters.get("require_json_mode", False), ) if ranked else (None, False, False, None) diff --git a/routing.py b/routing.py index c5725db..02295d9 100644 --- a/routing.py +++ b/routing.py @@ -198,6 +198,14 @@ def apply_flex_preference( prompt_tokens: int, completion_tokens: int, cache_rate: float, + required_context_tokens: int, + required_tier: int, + allowed_access_levels: Sequence[str], + exclude_stale: bool, + exclude_deprecated: bool, + min_tool_proficiency: float | None = None, + require_vision: bool = False, + require_json_mode: bool = False, ) -> tuple[dict, bool, bool, float | None]: """Apply the operator's flex-preference stance to the post-rank winner. @@ -214,18 +222,21 @@ def apply_flex_preference( left alone. - ``auto``: no change; the request's latency filter already decided. - ``prefer-flex``: swap to the flex sibling when one exists and is - eligible under the current hard filters. Given a sibling shares - context/tier/access/freshness with its standard twin, the only hard - filter that distinguishes it is the latency filter — so it is - eligible unless ``latency_tolerance == interactive``. - - ``force-flex``: swap to the flex sibling when one exists, even when - ``latency_tolerance == interactive`` (bypassing the latency filter). - That bypass is what sets the ``flex_forced`` flag. + eligible under the current hard filters. + - ``force-flex``: swap to the flex sibling when one exists, passes the + non-latency hard filters, and force-flex opts to bypass the latency + filter under ``interactive``. That bypass is what sets the + ``flex_forced`` flag. ``all_candidates`` is the FULL candidate set (pre hard-filter), so a flex sibling filtered out under ``interactive`` is still findable for - ``force-flex``. Eligibility for ``prefer-flex`` is re-decided here from - ``latency_tolerance``. + ``force-flex``. The sibling is a different catalog row and can independently + be stale, deprecated, access-restricted, under-tiered, or missing a + required capability, so it is re-gated via ``rejection_reason`` before the + swap. ``latency_tolerance=BATCH`` is passed to that re-gate to neutralize + only the latency filter; the actual ``latency_tolerance`` decision for + prefer-flex / force-flex is evaluated separately below and remains the + single owner of that branch. Returns ``(row, flex_swapped, flex_forced, cost)`` where ``row`` is the possibly-swapped selection (with any ranking score fields carried over, @@ -234,7 +245,7 @@ def apply_flex_preference( bypassed ``interactive``, and ``cost`` is the re-estimated cost for the returned row (recomputed via ``estimated_cost`` on a swap; the original cost otherwise). Falls back to the original row and cost when no sibling - exists. + exists or the sibling fails a non-latency hard filter. """ pref = getattr(flex_preference, "value", flex_preference) @@ -248,11 +259,28 @@ def apply_flex_preference( if sibling is None: return selected_row, False, False, selected_row.get("cost") + # The sibling is a different catalog row and can independently be stale, + # deprecated, access-restricted, under-tiered, or missing a required + # capability. Re-gate it against every hard filter except latency, using + # BATCH to neutralize only the latency_class check. The prefer-flex / + # force-flex branches below remain the single owners of the latency decision. + if rejection_reason( + sibling, + required_context_tokens=required_context_tokens, + required_tier=required_tier, + latency_tolerance=BATCH, + allowed_access_levels=allowed_access_levels, + exclude_stale=exclude_stale, + exclude_deprecated=exclude_deprecated, + min_tool_proficiency=min_tool_proficiency, + require_vision=require_vision, + require_json_mode=require_json_mode, + ) is not None: + return selected_row, False, False, selected_row.get("cost") + if pref == "prefer-flex": # A flex sibling is eligible under the current hard filters unless the # latency filter excludes flex rows — which is exactly interactive. - # Unlike its standard twin (which already passed everything else), the - # only filter that can dismiss a flex row is the latency one. if latency_tolerance == INTERACTIVE: return selected_row, False, False, selected_row.get("cost") diff --git a/tests/test_route_decisions.py b/tests/test_route_decisions.py index 5388932..1772eb4 100644 --- a/tests/test_route_decisions.py +++ b/tests/test_route_decisions.py @@ -276,6 +276,7 @@ def test_config_has_log_route_decisions_key(): CHEAP = "cheap-model" DEAR = "dear-model" CHEAP_FLEX = "cheap-model-flex" +CHEAP_STALE_FLEX = "cheap-model-flex-stale" class FakeResponse: @@ -383,6 +384,23 @@ def decision_router(tmp_path, monkeypatch): """, (CHEAP_FLEX, CHEAP, 0.20, 0.60), ) + # A stale flex twin of CHEAP: same identity dims, latency_class='flex' but + # availability='stale'. It must never be swapped to even under force-flex. + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'neuralwatt', ?, 2, 262128, 192500, 16384, ?, ?, + 1, 1, 'flex', 'default', 'full', 'public', 'stale', + '2026-08-22T00:00:00+00:00') + """, + (CHEAP_STALE_FLEX, CHEAP, 0.20, 0.60), + ) conn.commit() conn.close() @@ -865,3 +883,35 @@ def test_route_swapped_cost_reflects_flex_variant(decision_router): + completion_tokens * 0.60 ) / 1_000_000 assert flex_cost == pytest.approx(expected) + + +def test_route_force_flex_does_not_swap_to_stale_sibling(decision_router): + """force-flex must not dispatch to a stale flex sibling; the active standard wins.""" + client, db_path = decision_router + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "force-flex"} + ) + assert resp.status_code == 200 + + data = resp.json() + # The active CHEAP_FLEX sibling is still the healthy swap target. + assert data["selected"]["model_id"] == CHEAP_FLEX + assert data["flex_swapped"] is True + + # Make CHEAP_FLEX unavailable so the only remaining flex sibling is stale; + # force-flex must refuse it and keep the standard CHEAP winner. + conn = sqlite3.connect(db_path) + conn.execute( + "UPDATE models SET availability = 'stale' WHERE model_id = ?", (CHEAP_FLEX,) + ) + conn.commit() + conn.close() + + resp = client.post( + "/route", json={"task": "write me a function", "flex_preference": "force-flex"} + ) + assert resp.status_code == 200 + data = resp.json() + assert data["selected"]["model_id"] == CHEAP + assert data["flex_swapped"] is False + assert data["flex_forced"] is False diff --git a/tests/test_routing.py b/tests/test_routing.py index de88b67..fe885fd 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -608,7 +608,10 @@ def test_apply_no_flex_never_swaps_under_interactive(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.no_flex, "interactive", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3" assert swapped is False assert forced is False @@ -623,7 +626,10 @@ def test_apply_no_flex_leaves_an_already_flex_winner_alone(): latency_class="flex", proficiency=0.5) row, swapped, forced, cost = apply_flex_preference( flex, [flex], FlexPreference.no_flex, "interactive", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3-flex" assert swapped is False assert forced is False @@ -636,7 +642,10 @@ def test_apply_auto_makes_no_swap_even_when_a_flex_sibling_exists(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.auto, "batch", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3" assert swapped is False assert forced is False @@ -649,7 +658,10 @@ def test_apply_prefer_flex_holds_back_under_interactive(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.prefer_flex, "interactive", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3" assert swapped is False assert forced is False @@ -662,7 +674,10 @@ def test_apply_prefer_flex_swaps_under_batch(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.prefer_flex, "batch", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3-flex" assert swapped is True assert forced is False @@ -675,7 +690,10 @@ def test_apply_force_flex_swaps_under_interactive_and_marks_forced(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.force_flex, "interactive", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3-flex" assert swapped is True assert forced is True @@ -688,7 +706,10 @@ def test_apply_force_flex_swaps_under_batch_without_forced_flag(): std, flex = _std_flex_pair() row, swapped, forced, cost = apply_flex_preference( std, [std, flex], FlexPreference.force_flex, "batch", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3-flex" assert swapped is True assert forced is False @@ -705,7 +726,10 @@ def test_apply_falls_back_when_no_flex_sibling_exists(): FlexPreference.prefer_flex, FlexPreference.force_flex): row, swapped, forced, cost = apply_flex_preference( std, [std], pref, "batch", - prompt_tokens=1000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert row["model_id"] == "kimi-k3" assert swapped is False assert forced is False @@ -728,7 +752,10 @@ def test_apply_reestimates_cost_for_the_swapped_flex_row(): cost_per_1m_prompt_cached=1.0) row, swapped, _, _ = apply_flex_preference( std, [std, flex], FlexPreference.prefer_flex, "batch", - prompt_tokens=10_000, completion_tokens=500, cache_rate=0.5) + prompt_tokens=10_000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) assert swapped is True # estimated_cost scales prompt/completion prices to this request shape, so # the flex row's higher price must drive a higher estimate than standard's. @@ -740,3 +767,67 @@ def test_apply_reestimates_cost_for_the_swapped_flex_row(): )[0]["cost"] assert row["cost"] == flex_cost assert row["cost"] > std_cost + + +def test_apply_prefer_flex_refuses_stale_sibling_under_batch(): + # A stale flex sibling fails the freshness filter; prefer-flex under batch + # must keep the ranked standard winner rather than dispatch to staleness. + std, flex = _std_flex_pair(availability="stale") + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.prefer_flex, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_force_flex_refuses_stale_sibling_under_interactive(): + # force-flex bypasses the latency filter, but it still must not dispatch to + # a stale sibling; the standard winner stands. + std, flex = _std_flex_pair(availability="stale") + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.force_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_prefer_flex_refuses_canary_sibling_under_batch(): + # An access-restricted flex sibling must not be swapped to when the caller + # is restricted to public rows, even under batch where latency would admit it. + std, flex = _std_flex_pair(access_level="canary") + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.prefer_flex, "batch", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] + + +def test_apply_force_flex_refuses_canary_sibling_under_interactive(): + # force-flex bypasses only the latency filter — access restriction still + # gates the sibling, so the standard winner stands. + std, flex = _std_flex_pair(access_level="canary") + row, swapped, forced, cost = apply_flex_preference( + std, [std, flex], FlexPreference.force_flex, "interactive", + prompt_tokens=1000, completion_tokens=500, cache_rate=0.5, + required_context_tokens=1000, required_tier=2, + allowed_access_levels=["public"], exclude_stale=True, + exclude_deprecated=True) + assert row["model_id"] == "kimi-k3" + assert swapped is False + assert forced is False + assert cost == std["cost"] -- 2.49.1 From 452225014fe215dedfb918c97fce8813d5be28cb Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 19:38:02 -0400 Subject: [PATCH 17/32] feat(tui): surface flex-preference knob in the monitor Show the per-decision resolved flex_preference (with swap/forced marker) in the recent-decisions table and detail popup, plus the configured routing.default_flex_preference in the quota/legend readout. Recent-decisions /metrics and the SSE feed now carry flex_preference/flex_swapped/flex_forced consistently. 615 tests pass. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- metrics.py | 4 +- tests/test_metrics_endpoint.py | 29 +++++++++++- tests/test_tui.py | 86 ++++++++++++++++++++++++++++++++++ tui.py | 29 ++++++++++-- tui_model.py | 5 ++ 5 files changed, 146 insertions(+), 7 deletions(-) diff --git a/metrics.py b/metrics.py index c9d0146..3eb112b 100644 --- a/metrics.py +++ b/metrics.py @@ -130,6 +130,7 @@ def scoring_coverage( "with_proficiency_data": total - len(missing_proficiency), "quota": quota, "warnings": warnings, + "flex_default": cfg.routing.default_flex_preference.value, } @@ -147,7 +148,8 @@ def recent_decisions( classification_source, latency_tolerance, candidates_considered, selected_model, selected_provider, runner_up_models, est_cost_usd, est_proficiency, - rejected_reason, session_key, tools, images, json_mode, streamed + rejected_reason, session_key, tools, images, json_mode, streamed, + flex_preference, flex_swapped, flex_forced FROM route_decisions ORDER BY id DESC LIMIT ? diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index 2007602..ee6efa6 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -70,11 +70,13 @@ def _seed_decision(conn: sqlite3.Connection) -> None: confidence, classifier_ms, classification_source, latency_tolerance, candidates_considered, selected_model, selected_provider, runner_up_models, est_cost_usd, est_proficiency, - session_key, tools, images, json_mode, streamed + session_key, tools, images, json_mode, streamed, + flex_preference, flex_swapped, flex_forced ) VALUES (?, 'route', 'coding_general', 2, 100, 0.95, 200, 'classifier', 'interactive', 5, 'cheap', 'neuralwatt', '[{"model_id":"dear","provider":"neuralwatt"}]', - 0.001, 0.9, 'abc123', 0, 0, 0, 0) + 0.001, 0.9, 'abc123', 0, 0, 0, 0, + 'auto', 0, 1) """, (_now().isoformat(),), ) @@ -180,6 +182,29 @@ def test_metrics_aggregations_are_populated(seeded_client): assert data["quota"] is not None +def test_metrics_recent_decisions_carry_flex_columns(seeded_client): + """Each recent_decisions row carries the three flex telemetry columns.""" + resp = seeded_client.get("/metrics") + data = resp.json() + decisions = data["recent_decisions"] + assert decisions + for row in decisions: + assert "flex_preference" in row + assert "flex_swapped" in row + assert "flex_forced" in row + assert decisions[0]["flex_preference"] == "auto" + assert decisions[0]["flex_swapped"] == 0 + assert decisions[0]["flex_forced"] == 1 + + +def test_metrics_exposes_flex_default(seeded_client): + """The configured flex default is reported (under coverage).""" + resp = seeded_client.get("/metrics") + data = resp.json() + assert "flex_default" in data["coverage"] + assert data["coverage"]["flex_default"] in ("no-flex", "auto", "prefer-flex", "force-flex") + + def test_metrics_contains_no_session_dir(seeded_client): """The JSON must never name session_dir or expose conversation text.""" body = seeded_client.get("/metrics").text diff --git a/tests/test_tui.py b/tests/test_tui.py index f6fb48c..bbb47f7 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -50,6 +50,7 @@ def _fixture() -> dict: "3/13 routable models have no reference-workload observations", "1/13 routable models have no proficiency data", ], + "flex_default": "auto", }, "recent_decisions": [ { @@ -61,6 +62,9 @@ def _fixture() -> dict: "selected_model": "deepseek-v4-flash", "selected_provider": "neuralwatt", "est_cost_usd": 0.00016, + "flex_preference": "force-flex", + "flex_swapped": 1, + "flex_forced": 1, }, { "id": 41, @@ -71,6 +75,9 @@ def _fixture() -> dict: "selected_model": "kimi-k2.7-code", "selected_provider": "neuralwatt", "est_cost_usd": 0.0136, + "flex_preference": "auto", + "flex_swapped": 0, + "flex_forced": 0, }, { "id": 40, @@ -365,6 +372,25 @@ def test_format_quota_legend_happy_path_contains_numbers(): assert "None" not in legend +def test_format_quota_legend_shows_flex_default(): + """The configured flex default is surfaced in the quota legend readout.""" + app = tui.DashboardApp(fetcher=_StubFetcher()) + legend = app._format_quota_legend( + plan=6.25, metered=1.25, frac=0.2, calls=18, + reset_date="2026-07-26", flex_default="auto", + ) + assert "flex default auto" in legend + + +def test_format_quota_legend_omits_flex_when_absent(): + """No flex default in the payload -> the legend does not claim one.""" + app = tui.DashboardApp(fetcher=_StubFetcher()) + legend = app._format_quota_legend( + plan=6.25, metered=1.25, frac=0.2, calls=18, reset_date="2026-07-26" + ) + assert "flex default" not in legend + + def test_app_run_test_quota_bar_hidden_when_plan_unconfigured(): """No plan configured -> the ProgressBar is hidden (display False). @@ -795,6 +821,66 @@ def test_build_model_recent_decisions_carry_enriched_fields(): assert "category_breakdown" in m +def test_decision_row_projects_flex_fields(): + """decision_row must project the three flex telemetry fields so they reach + both the /metrics path and the live SSE path (shared by build_model).""" + row = tui_model.decision_row( + { + "flex_preference": "prefer-flex", + "flex_swapped": 1, + "flex_forced": 0, + } + ) + assert row["flex_preference"] == "prefer-flex" + assert row["flex_swapped"] == 1 + assert row["flex_forced"] == 0 + + +def test_decision_row_missing_flex_fields_default_to_none(): + """Rows without flex columns (older payloads) yield None, not a crash.""" + row = tui_model.decision_row({"id": 1}) + assert row["flex_preference"] is None + assert row["flex_swapped"] is None + assert row["flex_forced"] is None + + +def test_build_model_threads_flex_default(): + """build_model carries the configured flex default through from coverage.""" + m = build_model(_fixture()) + assert m["flex_default"] == "auto" + + +def test_flex_indicator_labels(): + """The compact flex cell: plain preference by default, markers on a swap.""" + assert tui._flex_indicator({"flex_preference": "auto", + "flex_swapped": 0, "flex_forced": 0}) == "auto" + assert tui._flex_indicator({"flex_preference": "prefer-flex", + "flex_swapped": 1, "flex_forced": 0}) == "prefer-flex!" + assert tui._flex_indicator({"flex_preference": "force-flex", + "flex_swapped": 1, "flex_forced": 1}) == "force!" + assert tui._flex_indicator({"id": 1}) == "" + + +def test_app_decision_table_renders_flex_column(): + """The decision table renders a flex indicator per row and the configured + flex default appears in the UI.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub) + + def _assert(a): + dt = a.query_one("#decision-table") + first_row_text = " ".join(str(c) for c in dt.get_row_at(0)) + assert "force!" in first_row_text + second_row_text = " ".join(str(c) for c in dt.get_row_at(1)) + assert "auto" in second_row_text + legend = a.query_one("#quota-legend", Static) + assert "flex default auto" in str(legend.content) + + _run_app(app, _assert) + + # -------------------------------------------------------------------------- # Detail popup and live SSE decision handling (App-level tests). # -------------------------------------------------------------------------- diff --git a/tui.py b/tui.py index 0139bb4..b39d632 100644 --- a/tui.py +++ b/tui.py @@ -57,6 +57,21 @@ def _fmt_usd(v: Optional[float]) -> str: return f"{v:.6f}" +def _flex_indicator(r: dict) -> str: + """Compact per-decision flex label for the decision table. + + Shows the resolved preference by default; when a ``-flex`` swap happened + it is marked so an operator can spot ride-a-flex decisions at a glance. + Kept plain-ASCII so it renders reliably in ``App.run_test()``. + """ + pref = r.get("flex_preference") + if not pref: + return "" + if r.get("flex_swapped"): + return "force!" if r.get("flex_forced") else f"{pref}!" + return str(pref) + + class DashboardApp(App): """A terminal dashboard that reads GET /metrics and renders panels.""" @@ -263,7 +278,7 @@ class DashboardApp(App): model_table.add_columns("model", "calls", "cost $", "kWh", "gCO2eq") decision_table = self.query_one("#decision-table", DataTable) decision_table.add_columns( - "id", "kind", "category", "tier", "ctx", "selected", "est $" + "id", "kind", "category", "tier", "ctx", "selected", "est $", "flex" ) decision_table.cursor_type = "row" decision_table.zebra_stripes = True @@ -398,7 +413,7 @@ class DashboardApp(App): except Exception: # noqa: BLE001 — a Static still takes focus harmlessly pass - def _format_quota_legend(self, plan, metered, frac, calls, reset_date) -> str: + def _format_quota_legend(self, plan, metered, frac, calls, reset_date, flex_default=None) -> str: """Build the quota legend string, suppressing ``None`` values. ``metered``/``frac``/``calls`` come from ``rows.get(...)`` and may be @@ -413,6 +428,8 @@ class DashboardApp(App): f"metered {metered_fmt} / plan {plan} " f"(frac {frac_fmt} · {percent}%) · calls {calls_fmt}" ) + if flex_default is not None: + legend += f" · [rgb(128,128,128)]flex default {flex_default}[/rgb(128,128,128)]" if reset_date: legend += ( f" · [rgb(128,128,128)]window start {reset_date}" @@ -440,7 +457,10 @@ class DashboardApp(App): bar.progress = float(metered or 0) bar.display = True reset_date = rows.get("reset_date") - legend.update(self._format_quota_legend(plan, metered, frac, calls, reset_date)) + flex_default = model.get("flex_default") + legend.update( + self._format_quota_legend(plan, metered, frac, calls, reset_date, flex_default) + ) else: bar.display = False legend.update("quota not configured") @@ -515,10 +535,11 @@ class DashboardApp(App): str(r.get("required_context_tokens")), str(r.get("selected")), _fmt_usd(r.get("est_cost_usd")), + _flex_indicator(r), key=str(r.get("id")), ) if not self._last_model["recent_decisions"]: - dt.add_row("(no decisions)", "", "", "", "", "", "", key="_placeholder") + dt.add_row("(no decisions)", "", "", "", "", "", "", "", key="_placeholder") self._restore_cursor(dt, saved_decision_key) def _render_category_table(self) -> None: diff --git a/tui_model.py b/tui_model.py index 9ba49a0..af45156 100644 --- a/tui_model.py +++ b/tui_model.py @@ -80,6 +80,7 @@ def build_model(data: dict) -> dict: coverage = data.get("coverage") or {} warnings = list(coverage.get("warnings") or []) + flex_default = coverage.get("flex_default") return { "quota": quota_rows, @@ -88,6 +89,7 @@ def build_model(data: dict) -> dict: "recent_decisions": recent, "category_breakdown": build_category_breakdown(recent), "warnings": warnings, + "flex_default": flex_default, } @@ -119,6 +121,9 @@ def decision_row(r: dict) -> dict: "images": r.get("images"), "json_mode": r.get("json_mode"), "streamed": r.get("streamed"), + "flex_preference": r.get("flex_preference"), + "flex_swapped": r.get("flex_swapped"), + "flex_forced": r.get("flex_forced"), } -- 2.49.1 From 2057723f7dd2e4637cd57411269ba1f80862266a Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 19:45:15 -0400 Subject: [PATCH 18/32] fix(tui): compact wrapped keys legend instead of truncating Footer Textual's built-in Footer ellipsizes on narrow terminals, hiding keys. Replace it with a compact wrapping Static legend (9 short key/action pairs, quit deduped to one) that wraps rather than truncates at any width. 618 tests pass. --- tests/test_tui.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ tui.py | 29 ++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/tests/test_tui.py b/tests/test_tui.py index bbb47f7..fe0ab30 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -881,6 +881,62 @@ def test_app_decision_table_renders_flex_column(): _run_app(app, _assert) +def test_keys_legend_built_from_short_pairs_dedups_quit(): + """The bottom keys legend is built from compact short pairs and collapses + the three quit bindings (q/Q/ctrl+c) to a single 'quit'.""" + legend = tui._keys_legend() + assert isinstance(legend, str) + # every short key is present + for key in ("q", "r", "e", "ctrl+v", "1", "2", "3", "4", "5"): + assert f"[bold]{key}[/bold]" in legend + # short labels are present + for label in ("quit", "refresh", "detail", "mix", "model", "decisions", + "breakdown", "quota", "warnings"): + assert label in legend + # exactly one quit (q/Q/ctrl+c collapsed) + assert legend.count("quit") == 1 + + +def test_app_run_test_renders_keys_legend(): + """The app renders a #keys-legend Static (not a Footer) carrying the + full compact legend string regardless of terminal width, so no key is + truncated away.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub) + + def _assert(a): + legend = a.query_one("#keys-legend", Static) + text = str(legend.content) + # full legend is present (wrapped, not truncated) on a narrow window + for key in ("q", "r", "e", "2", "5"): + assert f"[bold]{key}[/bold]" in text + assert text.count("quit") == 1 + + _run_app(app, _assert) + + +def test_app_run_test_keys_legend_not_truncated_at_narrow_width(): + """At a narrow terminal width the legend's content is intact (wraps rather + than drops keys), unlike Textual's Footer which would ellipsize.""" + stub = _StubFetcher() + stub.payload = _fixture() + + app = tui.DashboardApp(fetcher=stub) + + async def _go(): + async with app.run_test(size=(40, 30)) as pilot: + await pilot.pause() + legend = app.query_one("#keys-legend", Static) + text = str(legend.content) + # every short key survives narrow rendering — nothing truncated + for key in ("q", "r", "e", "ctrl+v", "1", "2", "3", "4", "5"): + assert f"[bold]{key}[/bold]" in text + + asyncio.run(_go()) + + # -------------------------------------------------------------------------- # Detail popup and live SSE decision handling (App-level tests). # -------------------------------------------------------------------------- diff --git a/tui.py b/tui.py index b39d632..13b98c1 100644 --- a/tui.py +++ b/tui.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) from textual.app import App, ComposeResult from textual.containers import Vertical, VerticalScroll -from textual.widgets import DataTable, Footer, Header, ProgressBar, Static +from textual.widgets import DataTable, Header, ProgressBar, Static from textual.widgets.data_table import RowDoesNotExist, RowKey from tui_model import ( @@ -57,6 +57,27 @@ def _fmt_usd(v: Optional[float]) -> str: return f"{v:.6f}" +# Compact bottom keys legend. The 11 BINDINGS are collapsed to these short +# (key, label) pairs — duplicate quit bindings (q/Q/ctrl+c) show once. Rendered +# as a single markup string so Textual's Static wraps rather than truncates. +KEYS_LEGEND_PAIRS: list[tuple[str, str]] = [ + ("q", "quit"), + ("r", "refresh"), + ("e", "detail"), + ("ctrl+v", "mix"), + ("1", "model"), + ("2", "decisions"), + ("3", "breakdown"), + ("4", "quota"), + ("5", "warnings"), +] + + +def _keys_legend() -> str: + """Render the compact wrapping keys-legend markup string.""" + return " · ".join(f"[bold]{key}[/bold] {label}" for key, label in KEYS_LEGEND_PAIRS) + + def _flex_indicator(r: dict) -> str: """Compact per-decision flex label for the decision table. @@ -135,6 +156,10 @@ class DashboardApp(App): #quota-legend { color: $text-muted; } + #keys-legend { + color: $text-muted; + padding: 0 1; + } """ def __init__( @@ -190,7 +215,7 @@ class DashboardApp(App): yield DataTable(id="category-table") yield Static("Health / warnings", classes="panel-title") yield Static("—", id="warnings-panel") - yield Footer() + yield Static(_keys_legend(), markup=True, id="keys-legend") def on_mount(self) -> None: self._unmounted = False -- 2.49.1 From 185aa38110df6b6665343069b23017f0e75ac768 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:20:55 -0400 Subject: [PATCH 19/32] feat(tui): split keys legend into two lines --- tui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tui.py b/tui.py index 13b98c1..7112ebf 100644 --- a/tui.py +++ b/tui.py @@ -75,7 +75,9 @@ KEYS_LEGEND_PAIRS: list[tuple[str, str]] = [ def _keys_legend() -> str: """Render the compact wrapping keys-legend markup string.""" - return " · ".join(f"[bold]{key}[/bold] {label}" for key, label in KEYS_LEGEND_PAIRS) + first_line = " · ".join(f"[bold]{key}[/bold] {label}" for key, label in KEYS_LEGEND_PAIRS[:4]) + second_line = " · ".join(f"[bold]{key}[/bold] {label}" for key, label in KEYS_LEGEND_PAIRS[4:]) + return f"{first_line}\n{second_line}" def _flex_indicator(r: dict) -> str: -- 2.49.1 From 245842d3a7916ed4710e24aab131ef4e44ce8352 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:58:08 -0400 Subject: [PATCH 20/32] feat(config): add session_cache configuration section --- config.py | 25 +++++++++++++++++++++++++ config.yaml | 14 ++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/config.py b/config.py index 55371ed..b7a9517 100644 --- a/config.py +++ b/config.py @@ -343,6 +343,30 @@ class PinchConfig(StrictModel): return v +class SessionCacheConfig(StrictModel): + """Per-session classification cache (in-memory, process lifetime). + + Remembers the last task_category/task_tier decision for each session for + ``staleness_minutes``, so a long agent session skips the classifier + round-trip on every turn. Capability flags (tools/images/json) are NEVER + cached — they are read fresh from each request body. Fallback + classifications are NEVER cached. No persistence: the cache lives only in + process memory, so a restart just reclassifies once per session. + """ + + enabled: bool = False + # Minutes since a cached classification was written before it is treated + # as expired and the next turn reclassifies from scratch. + staleness_minutes: int = 20 + + @field_validator("staleness_minutes") + @classmethod + def staleness_positive(cls, v: int) -> int: + if v <= 0: + raise ValueError("session_cache.staleness_minutes must be > 0") + return v + + class DatabaseConfig(StrictModel): path: str @@ -410,6 +434,7 @@ class RouterConfig(StrictModel): escalation: EscalationConfig iteration: IterationConfig = IterationConfig() pinch: PinchConfig = PinchConfig() + session_cache: SessionCacheConfig = SessionCacheConfig() freshness: FreshnessConfig database: DatabaseConfig classifier: ClassifierConfig diff --git a/config.yaml b/config.yaml index c016e9b..002f427 100644 --- a/config.yaml +++ b/config.yaml @@ -176,6 +176,20 @@ pinch: keep_last_turns: 4 max_summarize_chars: 4000 +session_cache: + # In-memory per-session classification cache. Remembers the last + # task_category / task_tier decision for each session for a few minutes, so + # a long agent session skips the ~1-2s classifier round-trip on every turn. + # Capability flags (tools / images / json) are NEVER cached — they are read + # fresh from each request. Fallback classifications are NEVER cached. No + # persistence: a restart just reclassifies each session once. + # + # Off by default, matching every other new-and-unproven knob in this + # project: ship it, watch route_decisions.source="cached" on real traffic, + # then decide the right default. + enabled: false + staleness_minutes: 20 + routing: # Access gating is prose-only in the NeuralWatt catalog ("Private preview # (grant-gated)", "(Canary)"), so the poller parses it into access_level and -- 2.49.1 From ba5b50dff2fbbb02773d982473d62b245a3f8ab1 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:58:12 -0400 Subject: [PATCH 21/32] feat(session_cache): pure in-memory classification cache module --- session_cache.py | 53 ++++++++++++++++++++++++ tests/test_session_cache.py | 80 +++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 session_cache.py create mode 100644 tests/test_session_cache.py diff --git a/session_cache.py b/session_cache.py new file mode 100644 index 0000000..e9e826e --- /dev/null +++ b/session_cache.py @@ -0,0 +1,53 @@ +"""In-memory per-session classification cache. + +Pure module, no I/O: ``get``/``put`` operate on a module-level dict whose +lifetime matches the dispatcher process (a restart just reclassifies each +session once — a safe failure mode). Never imports dispatcher or config. +""" + +import time +from dataclasses import dataclass +from typing import Final, Optional + +# session_key -> CachedClassification +_store: Final[dict[str, "CachedClassification"]] = {} + + +@dataclass(frozen=True) +class CachedClassification: + """A session's last successful classification, with its write time.""" + + task_category: str + task_tier: int + # time.time() at write time, compared against staleness_seconds on get(). + cached_at: float + + +def get( + session_key: str, staleness_seconds: float +) -> Optional[CachedClassification]: + """Return the cached classification if fresh, else None. + + An entry older than ``staleness_seconds`` is treated exactly like a miss: + gone, not renewed. + """ + cached = _store.get(session_key) + if cached is None: + return None + if time.time() - cached.cached_at > staleness_seconds: + return None + return cached + + +def put(session_key: str, task_category: str, task_tier: int) -> None: + """Store or overwrite a session's classification with a fresh timestamp.""" + _store[session_key] = CachedClassification( + task_category=task_category, + task_tier=task_tier, + cached_at=time.time(), + ) + + +def clear() -> None: + """Drop all cached classifications (test isolation / reset helper).""" + _store.clear() diff --git a/tests/test_session_cache.py b/tests/test_session_cache.py new file mode 100644 index 0000000..853c493 --- /dev/null +++ b/tests/test_session_cache.py @@ -0,0 +1,80 @@ +"""Unit tests for the in-memory per-session classification cache (session_cache.py). + +Offline, no network. Drive the pure get/put/TTL logic directly. time.time is +monkeypatched so expiry tests are deterministic (no wall-clock dependence). +""" + +from __future__ import annotations + +import pytest + +import session_cache + + +@pytest.fixture(autouse=True) +def _clean_cache(): + session_cache.clear() + yield + session_cache.clear() + + +def _freeze(monkeypatch, t: float): + """Freeze session_cache.time.time at ``t`` seconds.""" + monkeypatch.setattr(session_cache.time, "time", lambda: t) + + +def test_get_miss_returns_none(): + assert session_cache.get("sess-none", staleness_seconds=60) is None + + +def test_get_hit_returns_fresh_entry(monkeypatch): + _freeze(monkeypatch, 1000.0) + session_cache.put("sess-1", "coding_general", 2) + got = session_cache.get("sess-1", staleness_seconds=60) + assert got is not None + assert got.task_category == "coding_general" + assert got.task_tier == 2 + assert got.cached_at == 1000.0 + + +def test_get_expired_returns_none(monkeypatch): + _freeze(monkeypatch, 1000.0) + session_cache.put("sess-2", "debugging", 3) + # Advance past the 60s staleness window. + _freeze(monkeypatch, 1000.0 + 60.0 + 0.001) + assert session_cache.get("sess-2", staleness_seconds=60) is None + + +def test_get_on_staleness_boundary_is_fresh(monkeypatch): + _freeze(monkeypatch, 1000.0) + session_cache.put("sess-3", "coding_refactor", 1) + # Exactly 60s later: not strictly greater than staleness_seconds, so fresh. + _freeze(monkeypatch, 1000.0 + 60.0) + got = session_cache.get("sess-3", staleness_seconds=60) + assert got is not None + assert got.task_category == "coding_refactor" + + +def test_put_overwrites_existing_entry(monkeypatch): + _freeze(monkeypatch, 1000.0) + session_cache.put("sess-4", "coding_general", 1) + _freeze(monkeypatch, 1100.0) + session_cache.put("sess-4", "summarization", 2) + got = session_cache.get("sess-4", staleness_seconds=60) + assert got is not None + assert got.task_category == "summarization" + assert got.task_tier == 2 + assert got.cached_at == 1100.0 + + +def test_get_staleness_is_per_session(monkeypatch): + _freeze(monkeypatch, 1000.0) + session_cache.put("sess-a", "coding_general", 2) + session_cache.put("sess-b", "debugging", 3) + _freeze(monkeypatch, 1000.0 + 61.0) + # sess-a is now stale, sess-b was written at the same time so also stale; + # write sess-b fresh to isolate per-session expiry. + _freeze(monkeypatch, 1000.0 + 62.0) + session_cache.put("sess-b", "debugging", 3) + assert session_cache.get("sess-a", staleness_seconds=60) is None + assert session_cache.get("sess-b", staleness_seconds=60) is not None -- 2.49.1 From f4557d438a64b6ee09ee86e786913f85e4fa3a80 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:58:24 -0400 Subject: [PATCH 22/32] feat(dispatcher): use session classification cache in chat_completions Adds 'cached' to Classification.source and integrates the session cache: on a hit, skip the classifier and reuse the cached category/tier via the existing override branch; on a miss, write back only successful (non-fallback) classifications. Capability flags stay fresh per request; no persistence. --- dispatcher.py | 91 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/dispatcher.py b/dispatcher.py index 72bb7d5..f44d00a 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -58,6 +58,7 @@ from capabilities import detect_capabilities, iter_image_url_values from config import FlexPreference, RouterConfig, load_config from context_prune import extract_text, prune_context import events +import session_cache from routing import ( BATCH, INTERACTIVE, @@ -212,7 +213,7 @@ class Classification(BaseModel): required_context_tokens: int confidence: float escalated: bool = False - source: Literal["classifier", "override", "fallback"] = "classifier" + source: Literal["classifier", "override", "fallback", "cached"] = "classifier" class Candidate(BaseModel): @@ -2097,25 +2098,6 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): # so the classifier sees "Context: \n---\nMessage: " # instead of judging the follow-up alone. prev_context = _previous_context(messages) - classify_started = time.perf_counter() - decision = route( - TaskRequest( - task=_last_user_text(messages), - context=prev_context, - latency_tolerance=latency, - tools_present=tools_present, - has_images=caps.has_images, - require_json_mode=caps.require_json_mode, - # Take whichever is larger: what the classifier thinks it - # needs, or what the conversation actually measures. - required_context_tokens=None, - ) - ) - # The classifier's own verdict, before the re-route below rewrites the - # Classification's source to 'override'. - classified_src = decision.classification.source - classifier_ms = _ms(classify_started) - ctx_src = "classifier" # When pinch is enabled, prune ONCE before the measured-context # decision, so the window/tier/cost choice sees the size that will # actually ship upstream rather than the raw conversation. The pruned @@ -2141,7 +2123,19 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): else: send_messages = messages measured = estimate_prompt_tokens(send_messages, tools=body.get("tools")) - if measured > decision.classification.required_context_tokens: + cached = None + if cfg.session_cache.enabled and session_key is not None: + cached = session_cache.get( + session_key, + staleness_seconds=cfg.session_cache.staleness_minutes * 60, + ) + if cached is not None: + # Cache hit: skip the classifier. Capability flags (tools/images/ + # json) were already read fresh from the body (caps) above — only + # the category/tier are reused. The required context is this + # request's own measurement. + classified_src = "cached" + classifier_ms = 0 ctx_src = "measured" decision = route( TaskRequest( @@ -2153,11 +2147,62 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): tools_present=tools_present, has_images=caps.has_images, require_json_mode=caps.require_json_mode, - task_category=decision.classification.task_category, - task_tier=decision.classification.task_tier, + task_category=cached.task_category, + task_tier=cached.task_tier, required_context_tokens=measured, ) ) + else: + # Cache miss: classify as today, then re-route on measured context + # if the conversation outgrows the classifier's own estimate. After + # the decision, a successful (non-fallback) classification is + # written back so the next turn can skip the round-trip. + classify_started = time.perf_counter() + decision = route( + TaskRequest( + task=_last_user_text(messages), + context=prev_context, + latency_tolerance=latency, + tools_present=tools_present, + has_images=caps.has_images, + require_json_mode=caps.require_json_mode, + # Take whichever is larger: what the classifier thinks it + # needs, or what the conversation actually measures. + required_context_tokens=None, + ) + ) + # The classifier's own verdict, before the re-route below rewrites + # the Classification's source to 'override'. + classified_src = decision.classification.source + classifier_ms = _ms(classify_started) + ctx_src = "classifier" + if measured > decision.classification.required_context_tokens: + ctx_src = "measured" + decision = route( + TaskRequest( + task=_last_user_text(messages), + # context omitted: the override branch (task_category + + # task_tier + required_context_tokens all provided) + # never reads req.context — it skips classify(). + latency_tolerance=latency, + tools_present=tools_present, + has_images=caps.has_images, + require_json_mode=caps.require_json_mode, + task_category=decision.classification.task_category, + task_tier=decision.classification.task_tier, + required_context_tokens=measured, + ) + ) + if ( + cfg.session_cache.enabled + and session_key is not None + and classified_src == "classifier" + ): + session_cache.put( + session_key, + task_category=decision.classification.task_category, + task_tier=decision.classification.task_tier, + ) if decision.selected is None: if ( caps.has_images -- 2.49.1 From 0a26f582b921d1cde1c84177f69bd087a6ec60b6 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:58:29 -0400 Subject: [PATCH 23/32] test(chat_completions): add session cache integration tests --- tests/test_chat_completions.py | 150 +++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/tests/test_chat_completions.py b/tests/test_chat_completions.py index 7fd5e25..b48c781 100644 --- a/tests/test_chat_completions.py +++ b/tests/test_chat_completions.py @@ -21,6 +21,7 @@ import pytest from starlette.testclient import TestClient import dispatcher +import session_cache from dispatcher import Classification, app ROOT = Path(__file__).resolve().parent.parent @@ -916,3 +917,152 @@ def test_a_pinned_non_json_model_with_json_response_format_422s(router): assert resp.status_code == 422 assert "json" in resp.json()["detail"] assert not calls, "no provider call should happen for an impossible json pin" + + +# --- session classification cache ----------------------------------------- + +def _session_messages(text="write me a function", system="You are a coding agent"): + """Messages with a stable opening (system) message, so the session + fingerprint is constant across turns — the condition a cache hit needs.""" + return [ + {"role": "system", "content": system}, + {"role": "user", "content": text}, + ] + + +def _session_key(messages=None): + return dispatcher.session_fingerprint(messages or _session_messages()) + + +@pytest.fixture +def session_cache_on(monkeypatch): + """Enable the cache and isolate the module-level store between tests.""" + session_cache.clear() + monkeypatch.setattr(dispatcher.cfg.session_cache, "enabled", True) + yield + session_cache.clear() + + +def _count_classify(monkeypatch, result=None): + """Replace classify with a counting stub returning ``result`` (or the + default real-classifier shape).""" + result = result or Classification( + task_category="coding_general", task_tier=2, + required_context_tokens=100, confidence=0.9, + ) + counts = {"calls": 0} + + def classify(task, context): + counts["calls"] += 1 + return result + + monkeypatch.setattr(dispatcher, "classify", classify) + return counts + + +def test_cache_hit_skips_classify_and_reports_cached_source(router, logbuf, session_cache_on, monkeypatch): + """A second request in the same session reuses the cached category/tier and + does not pay another classifier round-trip.""" + client, _, _ = router + counts = _count_classify(monkeypatch) + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages()}) + assert counts["calls"] == 1, "first turn classifies" + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages("follow up")}) + + assert counts["calls"] == 1, "second turn must not classify again" + routes = _lines(logbuf, "route") + assert _fields(routes[0])["src"] == "classifier", "first turn came from the classifier" + assert _fields(routes[1])["src"] == "cached", "second turn came from the cache" + + +def test_cache_miss_writes_the_successful_classification(router, session_cache_on): + """After a real (non-fallback) classification, the session's category/tier + land in the cache under its fingerprint.""" + client, _, _ = router + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages()}) + + cached = session_cache.get(_session_key(), staleness_seconds=60) + assert cached is not None + assert cached.task_category == "coding_general" + assert cached.task_tier == 2 + + +def test_cache_expiry_reclassifies(router, logbuf, session_cache_on, monkeypatch): + """Once stale, a session's next turn reclassifies from scratch.""" + client, _, _ = router + counts = _count_classify(monkeypatch) + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages()}) + assert counts["calls"] == 1 + + # Advance beyond the 20-minute TTL and send another turn. + now = [session_cache.time.time()] + monkeypatch.setattr(session_cache.time, "time", lambda: now[0] + 20 * 60 + 1) + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages("again")}) + + assert counts["calls"] == 2, "expired entry must reclassify" + assert _fields(_lines(logbuf, "route")[1])["src"] == "classifier" + + +def test_fallback_classification_is_never_cached(router, session_cache_on, monkeypatch): + """A classifier fallback must not populate the cache: the next turn still + pays the round-trip.""" + client, _, _ = router + fallback = Classification( + task_category="general_chat", task_tier=2, + required_context_tokens=0, confidence=0.0, source="fallback", + ) + counts = _count_classify(monkeypatch, result=fallback) + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages()}) + assert session_cache.get(_session_key(), staleness_seconds=60) is None, \ + "fallback must never be cached" + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages("again")}) + assert counts["calls"] == 2, "a fallback must not be served from cache" + + +def test_capability_flags_are_still_read_fresh_on_a_cache_hit(router, session_cache_on, monkeypatch): + """A cache hit reuses only category/tier; a request that suddenly carries an + image is still gated by its (freshly read) capability flags, not served + blindly from the cached decision.""" + client, _, db_path = router + _count_classify(monkeypatch) + + client.post("/v1/chat/completions", + json={"model": "auto", "messages": _session_messages()}) + # The SAME session (same system prompt) now turns into an image request: + # it must route to the vision-capable model rather than being treated as + # the cached text turn. + image_turn = [{ + "role": "user", + "content": [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], + }] + resp = client.post( + "/v1/chat/completions", + json={"model": "auto", + "messages": _session_messages()[:1] + image_turn}, + ) + assert resp.status_code == 200 + assert resp.json()["model"].startswith(CHEAP), "vision-capable model picked" + + conn = sqlite3.connect(db_path) + srcs = [r[0] for r in conn.execute( + "SELECT classification_source FROM route_decisions ORDER BY id" + )] + conn.close() + assert srcs == ["classifier", "cached"], "cached decision persisted with source=cached" -- 2.49.1 From 756341d250de2dbd8e3316287dc594d9f0638e0a Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Wed, 26 Aug 2026 20:58:34 -0400 Subject: [PATCH 24/32] docs(config): document session_cache configuration --- README.md | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e7ac0f..c399fce 100644 --- a/README.md +++ b/README.md @@ -408,7 +408,7 @@ failure — it means the checker had nothing to say, not that the model failed. | `required_context_tokens` | INTEGER | | | `confidence` | REAL | Classifier confidence | | `classifier_ms` | INTEGER | Classification latency | -| `classification_source` | TEXT | `classifier` \| `override` \| `fallback` | +| `classification_source` | TEXT | `classifier` \| `override` \| `fallback` \| `cached` | | `latency_tolerance` | TEXT | `interactive` \| `batch` | | `candidates_considered` | INTEGER | How many survived hard filters | | `selected_model` | TEXT | Null when no model was selected | @@ -730,6 +730,19 @@ Three foreground tools read the running router without spending quota: decision tree, including candidates, selected model, estimated cost, and proficiency. Use `--category`, `--tier`, and `--context` to override the classifier deterministically, or `--json` for raw output. +- **`baseline_report.py --since 2026-08-01`** — read-only retrospective that + replays recent routed decisions against two trivial counterfactuals — always + pick the cheapest eligible candidate and always pick the best-proficiency + eligible candidate — and reports total/mean cost and proficiency plus a + **dominance check** (the share of decisions where the real router picked the + cheapest eligible model), overall and per `task_category`. This automates the + README's manual "check for dominance" step: a high dominance share with a + near-zero proficiency delta means the real scoring isn't earning its + complexity for that slice of traffic. Add `--category coding_refactor` to + scope it, or `--csv` to emit parseable CSV instead of the table. It reads + only `route_decisions`, `models`, and `proficiency`, writes nothing, and + reuses `routing.select_candidates`/`routing.estimated_cost` — comparing + against the *current* catalog, not a historical snapshot. `textual` is pinned in `requirements.txt` solely for the TUI modules (`tui.py`, `tui_screens.py`, `tui_sse.py`). It is imported only by these @@ -869,11 +882,19 @@ Several settings keep it from cascading failures: - **`temperature: 0`** prevents non-reproducible classifications. At the default, the same prompt classified tier 2 then tier 1 on consecutive calls, routing to two different models. -- **Graceful fallback** routes to `fallback_tier: 2` / `fallback_category: - general_chat` when the classifier times out, errors, or returns - unparseable output. A coding agent would rather have a mid-tier answer than - an error. Escalation deliberately skips fallbacks so an unavailable local - model doesn't silently promote every request to the frontier tier. + - **Graceful fallback** routes to `fallback_tier: 2` / `fallback_category: + general_chat` when the classifier times out, errors, or returns + unparseable output. A coding agent would rather have a mid-tier answer than + an error. Escalation deliberately skips fallbacks so an unavailable local + model doesn't silently promote every request to the frontier tier. +- **Session classification cache** (`session_cache:` in `config.yaml`, + off by default) remembers the last `task_category`/`task_tier` decision per + session for `staleness_minutes` (default 20), so a long agent session skips + the classifier round-trip on every turn. It only ever short-circuits the + classifier call: capability flags (tools/images/json) are still read fresh + from each request body, fallback classifications are never cached, and there + is no persistence beyond process memory. A cache-hit decision is observable + via `route_decisions.classification_source = "cached"`. ## Pointing a Coding Agent at It @@ -954,6 +975,7 @@ Then edit `config.yaml` for your own setup — at minimum: | `classifier.base_url` | where that Ollama actually is | | `objective.plan_kwh_per_period` | your plan's quota; `/health` reports burn against it | | `objective.assumed_cache_rate` | 0.917 was measured from one client's traffic (40.7M tokens). Check yours against the provider's per-session cache-hit figures | +| `session_cache.enabled` | off by default; caches category/tier per session for `staleness_minutes` to skip repeat classifier round-trips on long agent sessions | `python seed_energy.py` is optional. It sweeps a fixed reference workload to populate `eco`, which is logged but is not an objective — routing works -- 2.49.1 From bc7601e0515453db07778876e5762027bd143d7a Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Thu, 27 Aug 2026 18:58:14 -0400 Subject: [PATCH 25/32] feat(routing): implement session classification cache and baseline routing report --- baseline_report.py | 353 ++++++++++++++++++ code_plans/baseline-routing-comparator.md | 125 +++++++ code_plans/classifier-input-scope-check.md | 65 ++++ ...-dual-use-and-classify-once-per-session.md | 6 + .../session-classification-cache-ttl.md | 178 +++++++++ code_reviews/flex-preference-knob-review.md | 171 +++++++++ ...on-cache-and-baseline-comparator-review.md | 140 +++++++ tests/test_baseline_report.py | 266 +++++++++++++ 8 files changed, 1304 insertions(+) create mode 100644 baseline_report.py create mode 100644 code_plans/baseline-routing-comparator.md create mode 100644 code_plans/classifier-input-scope-check.md create mode 100644 code_plans/session-classification-cache-ttl.md create mode 100644 code_reviews/flex-preference-knob-review.md create mode 100644 code_reviews/session-cache-and-baseline-comparator-review.md create mode 100644 tests/test_baseline_report.py diff --git a/baseline_report.py b/baseline_report.py new file mode 100644 index 0000000..841c086 --- /dev/null +++ b/baseline_report.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Read-only retrospective comparison of routing against two trivial baselines. + +Replays recent ``route_decisions`` rows against two counterfactuals that skip +routing's scoring entirely: + +- **always_cheapest** — the eligible candidate with the lowest + ``routing.estimated_cost`` for that request's shape. +- **always_best_proficiency** — the eligible candidate with the highest + ``proficiency.blended_score`` for the decision's ``task_category``, ties + broken by lowest cost. + +For each decision the eligible set is reconstructed from the current catalog by +re-running ``routing.select_candidates`` with the constraints that row actually +carried (``task_tier``, ``required_context_tokens``, ``latency_tolerance``, +and its ``tools``/``images``/``json_mode`` flags). This automates the README's +"check for dominance" step: a high dominance share with a near-zero proficiency +delta means the real scoring is not earning its complexity for that slice of +traffic. + +Read-only, adds no schema, reuses ``routing.select_candidates`` and +``routing.estimated_cost``. Baselines are recomputed against the CURRENT +catalog and proficiency table, not a historical snapshot. + +Run directly: + python baseline_report.py --since 2026-08-01 + python baseline_report.py --since 2026-08-01 --category coding_refactor + python baseline_report.py --since 2026-08-01 --csv +""" + +from __future__ import annotations + +import argparse +import csv +import sqlite3 +import sys +from collections.abc import Sequence + +from config import RouterConfig, load_config +from routing import estimated_cost, select_candidates + + +def load_candidates( + conn: sqlite3.Connection, category: str, cfg: RouterConfig, +) -> list[dict]: + """Current catalog rows joined with proficiency for ``category``. + + Mirrors dispatcher's candidate-join shape (but without the eco/energy + columns this report does not need): each row gains ``proficiency`` (the + ``blended_score`` for ``category``, or None when unmeasured) and + ``tool_proficiency`` (the blend for the tool gate's category, so the + tool filter reconstructs the same way the router gates it). + """ + rows = conn.execute( + """ + SELECT m.*, + p.blended_score AS proficiency, + tp.blended_score AS tool_proficiency + FROM models m + LEFT JOIN proficiency p + ON p.model_id = m.model_id + AND p.provider = m.provider + AND p.category = ? + LEFT JOIN proficiency tp + ON tp.model_id = m.model_id + AND tp.provider = m.provider + AND tp.category = ? + """, + (category, cfg.routing.tool_use_category), + ).fetchall() + return [dict(r) for r in rows] + + +def load_decisions( + conn: sqlite3.Connection, + since: str | None, + category: str | None, +) -> list[sqlite3.Row]: + """Recent routed decisions in the window, newest first. + + ``observed_at`` is ISO8601 UTC text, so a lexicographic comparison with a + ``YYYY-MM-DD`` prefix is a valid time filter — a decision whose timestamp + starts at or after ``since`` is inside the window. + """ + clauses: list[str] = ["kind = 'route'"] + params: list[str] = [] + if since is not None: + clauses.append("observed_at >= ?") + params.append(since) + if category is not None: + clauses.append("task_category = ?") + params.append(category) + sql = ( + "SELECT * FROM route_decisions WHERE " + + " AND ".join(clauses) + + " ORDER BY observed_at DESC" + ) + return conn.execute(sql, params).fetchall() + + +def baseline_selection( + candidates: Sequence[dict], + decision: sqlite3.Row, + cfg: RouterConfig, +) -> tuple[dict | None, dict | None]: + """Pick the always_cheapest and always_best_proficiency candidates. + + ``candidates`` is the request's reconstructed eligible set (already run + through ``routing.select_candidates``). Both baselines are pure selections + over that set: + + - cheapest: lowest ``estimated_cost`` priced for the request's own shape. + - best proficiency: highest ``proficiency`` for the decision's category, + ties broken by lowest cost, then model_id for determinism. + + Returns ``(cheapest, best)``; either may be None when the set is empty. + """ + if not candidates: + return None, None + + prompt_tokens = decision["required_context_tokens"] or 0 + cache_rate = cfg.objective.assumed_cache_rate + completion_tokens = cfg.objective.assumed_completion_tokens + + priced = [] + for row in candidates: + cost = estimated_cost(row, prompt_tokens, completion_tokens, cache_rate) + priced.append((row, cost)) + + cheapest = min( + priced, + key=lambda rc: (rc[1] if rc[1] is not None else float("inf"), rc[0]["model_id"]), + )[0] + + def prof_key(rc: tuple) -> tuple: + row, cost = rc + prof = row.get("proficiency") + return ( + -(prof if prof is not None else 0.5), + cost if cost is not None else float("inf"), + row["model_id"], + ) + + best = min(priced, key=prof_key)[0] + return cheapest, best + + +def reconstruct_decision( + conn: sqlite3.Connection, + decision: sqlite3.Row, + cfg: RouterConfig, +) -> tuple[dict | None, dict | None]: + """Re-run the decision's hard filters against the current catalog. + + Returns ``(always_cheapest, always_best_proficiency)`` candidate rows the + real decision could have picked (or None when no candidate is eligible). + """ + category = decision["task_category"] or cfg.classifier.fallback_category + filters = { + "required_context_tokens": decision["required_context_tokens"] or 0, + "required_tier": decision["task_tier"] or 1, + "latency_tolerance": decision["latency_tolerance"] + or cfg.routing.default_latency_tolerance, + "allowed_access_levels": cfg.routing.allowed_access_levels, + "exclude_stale": cfg.freshness.exclude_stale, + "exclude_deprecated": cfg.freshness.exclude_deprecated, + } + if decision["tools"]: + filters["min_tool_proficiency"] = cfg.routing.min_tool_proficiency + if decision["images"]: + filters["require_vision"] = cfg.routing.require_vision + if decision["json_mode"]: + filters["require_json_mode"] = cfg.routing.require_json_mode + + candidates = select_candidates( + load_candidates(conn, category, cfg), **filters, + ) + return baseline_selection(candidates, decision, cfg) + + +def _model_cost(row: dict | None, decision: sqlite3.Row, cfg: RouterConfig) -> float | None: + """Estimated cost of a baseline row priced for the decision's shape.""" + if row is None: + return None + return estimated_cost( + row, + decision["required_context_tokens"] or 0, + cfg.objective.assumed_completion_tokens, + cfg.objective.assumed_cache_rate, + ) + + +def _model_proficiency(row: dict | None) -> float | None: + if row is None: + return None + prof = row.get("proficiency") + return prof if prof is not None else 0.5 + + +def analyze( + conn: sqlite3.Connection, + cfg: RouterConfig, + since: str | None = None, + category: str | None = None, +) -> tuple[dict, list[dict]]: + """Compute aggregate and per-category baseline comparison. + + Returns ``(aggregate_row, category_rows)`` where each row is a plain dict + keyed by column name, ready for both the human table and the CSV writer. + """ + decisions = load_decisions(conn, since, category) + + def empty_row(cat: str | None) -> dict: + return { + "category": cat if cat is not None else "total", + "count": 0, + "actual_cost": 0.0, + "cheapest_cost": 0.0, + "best_prof_cost": 0.0, + "actual_proficiency": 0.0, + "cheapest_proficiency": 0.0, + "best_prof_proficiency": 0.0, + "dominance_count": 0, + "dominance_pct": None, + } + + by_cat: dict[str, dict] = {} + total = empty_row(None) + + for d in decisions: + cat = d["task_category"] or cfg.classifier.fallback_category + row = by_cat.setdefault(cat, empty_row(cat)) + cheapest, best = reconstruct_decision(conn, d, cfg) + + for agg in (total, row): + agg["count"] += 1 + agg["actual_cost"] += d["est_cost_usd"] or 0.0 + cc = _model_cost(cheapest, d, cfg) + bc = _model_cost(best, d, cfg) + if cc is not None: + agg["cheapest_cost"] += cc + if bc is not None: + agg["best_prof_cost"] += bc + ap = d["est_proficiency"] + agg["actual_proficiency"] += ap if ap is not None else 0.0 + cp = _model_proficiency(cheapest) + bp = _model_proficiency(best) + if cp is not None: + agg["cheapest_proficiency"] += cp + if bp is not None: + agg["best_prof_proficiency"] += bp + if cheapest is not None and cheapest["model_id"] == d["selected_model"]: + agg["dominance_count"] += 1 + + def finalize(r: dict) -> None: + if r["count"] == 0: + r["dominance_pct"] = None + return + r["actual_proficiency"] /= r["count"] + r["cheapest_proficiency"] /= r["count"] + r["best_prof_proficiency"] /= r["count"] + r["dominance_pct"] = 100.0 * r["dominance_count"] / r["count"] + + finalize(total) + for row in by_cat.values(): + finalize(row) + + cat_rows = [by_cat[c] for c in sorted(by_cat)] + return total, cat_rows + + +def format_summary(total: dict, cat_rows: list[dict]) -> str: + """Human-readable report text.""" + lines = ["baseline routing comparator"] + lines.append(f" decisions: {total['count']}") + lines.append( + f" total cost: actual ${total['actual_cost']:.4f} " + f"cheapest ${total['cheapest_cost']:.4f} " + f"best-prof ${total['best_prof_cost']:.4f}" + ) + pct = total["dominance_pct"] + dom = "n/a" if pct is None else f"{pct:.1f}%" + lines.append( + f" dominance (selected == cheapest): {total['dominance_count']}/{total['count']} " + f"({dom})" + ) + lines.append( + f" mean proficiency: actual {total['actual_proficiency']:.3f} " + f"cheapest {total['cheapest_proficiency']:.3f} " + f"best-prof {total['best_prof_proficiency']:.3f}" + ) + if cat_rows: + lines.append("") + lines.append( + f"{'category':18s}{'n':>5}{'dom%':>7}{'act$/dec':>10}" + f"{'cheap$/dec':>11}{'best$/dec':>11}{'act prof':>10}{'best prof':>10}" + ) + for r in cat_rows: + dom = "n/a" if r["dominance_pct"] is None else f"{r['dominance_pct']:.1f}" + lines.append( + f"{r['category']:18s}{r['count']:>5}{dom:>7}" + f"{r['actual_cost'] / r['count']:>10.4f}" + f"{r['cheapest_cost'] / r['count']:>11.4f}" + f"{r['best_prof_cost'] / r['count']:>11.4f}" + f"{r['actual_proficiency']:>10.3f}" + f"{r['best_prof_proficiency']:>10.3f}" + ) + return "\n".join(lines) + + +CSV_COLUMNS = [ + "category", "count", "actual_cost", "cheapest_cost", "best_prof_cost", + "actual_proficiency", "cheapest_proficiency", "best_prof_proficiency", + "dominance_count", "dominance_pct", +] + + +def write_csv(total: dict, cat_rows: list[dict], out) -> None: + """Write aggregate + per-category rows as CSV.""" + writer = csv.DictWriter(out, fieldnames=CSV_COLUMNS) + writer.writeheader() + writer.writerow(total) + for r in cat_rows: + writer.writerow(r) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--since", + help="only consider decisions observed at/after this ISO date (YYYY-MM-DD)", + ) + ap.add_argument("--category", help="only consider a single task_category") + ap.add_argument("--csv", action="store_true", help="emit CSV instead of the table") + args = ap.parse_args() + + cfg = load_config("config.yaml") + conn = sqlite3.connect(cfg.database.path) + conn.row_factory = sqlite3.Row + + total, cat_rows = analyze(conn, cfg, since=args.since, category=args.category) + + if args.csv: + write_csv(total, cat_rows, sys.stdout) + else: + print(format_summary(total, cat_rows)) + + conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/code_plans/baseline-routing-comparator.md b/code_plans/baseline-routing-comparator.md new file mode 100644 index 0000000..1ddce8d --- /dev/null +++ b/code_plans/baseline-routing-comparator.md @@ -0,0 +1,125 @@ +# Spec: retrospective baseline comparator (LLMRouter smallest_llm/largest_llm) + +**Origin.** LLMRouter ships `smallest_llm`/`largest_llm` — routers with no +scoring logic at all, used purely as a floor to prove a real router beats +"always pick cheap" or "always pick strong." This project has no size axis +(no parameter counts in the catalog), so the direct port doesn't map +cleanly — the useful part isn't "smallest/largest," it's *having a trivial +counterfactual at all*. This is also the automated form of something the +README already asks a human to do by eye: "If you see one model win +everything again, check for dominance before reaching for config." A +report script makes that a query instead of a memory. + +No code changes accompany this document — this is the spec opencode builds +from. + +--- + +## Design + +A new **read-only, standalone script**, `baseline_report.py`, modeled on +`metrics.py`'s existing pattern: takes `(conn, cfg)`, never imports +`dispatcher`, so it stays outside the import-cycle constraint `metrics.py` +was already built to respect. Runnable directly (`python baseline_report.py +--since 2026-08-01`), same shape as `leaderboard.py --check` or +`eval_proficiency.py` — no new schema, no new table, no write path. + +### What it reads + +`route_decisions` for the historical record of *what was actually decided*: +`task_category`, `task_tier`, `required_context_tokens`, +`latency_tolerance`, `tools`/`images`/`json_mode` flags, `selected_model`, +`est_cost_usd`, `est_proficiency`. Every field the two baselines below need +to reconstruct an equivalent decision is already logged here — no join to +`energy_observations` required, and deliberately so (see the limitation +below). + +### The two baselines + +For each historical decision row, re-derive the **same eligible candidate +set** the real decision would have seen — by re-running `routing.py`'s hard +filters (`select_candidates`-equivalent: tier, context, latency_tolerance, +tools/images/json_mode gates) against the **current** catalog, using the +row's own stored `task_tier`/`required_context_tokens`/`latency_tolerance`/ +capability flags as the filter inputs. This matters: a baseline of "always +pick the globally cheapest model in the catalog" is not an interesting +comparison if that model couldn't have legally served half the requests +(wrong tier, no vision, wrong latency class). The comparison that's actually +useful is "given the same hard constraints the real router respected, would +a dumber tie-break have done just as well?" + +Within that reconstructed eligible set: + +- **`always_cheapest`** (LLMRouter's `smallest_llm` analogue) — the + candidate with the lowest `routing.estimated_cost(row, prompt_tokens= + required_context_tokens, completion_tokens=, cache_rate= + )`, using the same shape assumptions the + real decision's own `est_cost_usd` was computed with. +- **`always_best_proficiency`** (LLMRouter's `largest_llm` analogue, + adapted — this catalog has no size axis, but proficiency is the actual + axis routing optimizes quality on) — the candidate with the highest + `proficiency.blended_score` for that row's `task_category`, ties broken + by lowest cost. + +Both are pure functions of already-existing code +(`routing.estimated_cost`, a live `proficiency` table read) — no new scoring +logic to build or validate. + +### Known limitation, stated plainly + +Both baselines are recomputed against the **current** catalog and **current** +proficiency table, not a historical snapshot from when each decision was +actually made. Prices, tiers, and proficiency scores drift over time (the +README documents several such shifts), so a decision from three weeks ago +is compared against today's catalog, not the one it actually saw. This is +the same trade-off LLMRouter's own benchmark pipeline makes in the other +direction — it "replays pre-recorded model executions" against a frozen +dataset rather than live catalogs. Neither approach is wrong; this project +doesn't snapshot the catalog per-decision, so "current catalog" is the +only version buildable without a new logging table, and it's directionally +fine for an aggregate report over a recent window (`--since`) where the +catalog hasn't moved much. Flagging it so nobody mistakes this for a +historically-exact replay. + +`runner_up_models` (already logged, top 3 candidates by id/provider only) +is **not** sufficient for this on its own — it carries no cost or +proficiency values and isn't guaranteed to include the two baseline winners +identified above. Re-deriving the eligible set from the catalog, as +described, is required either way. + +### Output + +``` +python baseline_report.py --since 2026-08-01 [--category coding_general] [--csv] +``` + +Aggregate summary: + +- Total actual cost vs. total `always_cheapest` cost vs. total + `always_best_proficiency` cost, over the window. +- Mean `est_proficiency` (actual) vs. mean proficiency of each baseline. +- **Dominance check**: the count/percentage of decisions where + `selected_model == always_cheapest` — this is the automated form of the + README's "check for dominance" instruction. A high percentage with near- + zero proficiency delta says the real scoring isn't earning its complexity + for that slice of traffic; a low percentage, or a percentage that's high + only where quality is genuinely tied, says it is. +- Break out by `task_category` (not just an overall number) — the README's + own finding was that dominance is categorical (`qwen3.6-35b` dominated + *before* cost-per-request and tier-from-price were fixed; different + categories now have different winners), so an aggregate-only number would + hide exactly the thing this report exists to surface. + +`--csv` mirrors LLMRouter's `aggregate_results.py --csv` for the same +reason it works there: a report like this is as much for pasting into a +follow-up conversation as for reading in a terminal. + +## Recommendation + +Build it as specified. It's pure read-over-existing-data — no new table, no +new write path, no scoring logic beyond calling `estimated_cost` and reading +`proficiency` for models the router already knows about. Run it after the +next `route_decisions` accumulates a reasonable window (a few hundred rows +across categories, going by current volume) and treat a lopsided dominance +result as the same kind of signal the README already tells you to distrust +if it isn't backed by a quality-tolerance argument. diff --git a/code_plans/classifier-input-scope-check.md b/code_plans/classifier-input-scope-check.md new file mode 100644 index 0000000..537556e --- /dev/null +++ b/code_plans/classifier-input-scope-check.md @@ -0,0 +1,65 @@ +# Finding: classifier input scope (LLMRouter "query modes" check) + +**Origin.** LLMRouter's chat interface offers three query modes — +`current_only`, `full_context` (all history), `retrieval` (top-k similar +past queries). Before considering porting any of that, the actual question +was narrower and answerable by reading the code: does this project's +classifier already read the whole conversation, or only the tail? If only +the tail, a long session that drifts category (chat → debugging) could be +mis-tiered from a stale early read. This is a verification finding, not a +design proposal — no code changes proposed here. + +--- + +## What the code actually does + +`chat_completions` (`dispatcher.py`) builds the classifier's input from two +helpers, both scoped narrowly on purpose: + +- `_last_user_text(messages)` (`dispatcher.py:1734`) — the text of the + **last** `user`-role message only. Walks backwards and returns on the + first match; never touches earlier turns. +- `_previous_context(messages)` (`dispatcher.py:1749`) — the text of the + **nearest preceding `assistant` message**, truncated to 200 characters. + Explicitly skips `system`, `user`, and `tool` roles "so that tool output + or the system prompt never contaminates the framing signal" (its own + docstring). Not the whole history — one turn of lookback, by design. + +Both are recomputed **fresh on every request** — there is no session-level +memoization of the classification input today (that's exactly the gap +[`session-classification-cache-ttl.md`](session-classification-cache-ttl.md) +proposes closing, but for the *output* — the category/tier decision — not +the input). + +## What this means for category drift + +This is already, incidentally, LLMRouter's `current_only` mode plus a +one-turn lookback — and it's the reason category drift within a long +session is *not* currently a problem: every message gets classified from +near-term signal (the current message and the immediately preceding reply), +so a session moving from `general_chat` to `debugging` over 40 turns tracks +that drift on the very next message, for free. `full_context` (feeding the +whole conversation) would not improve this — it would make classification +slower (more tokens through `max_input_chars` clamping) and arguably noisier +(early, no-longer-relevant turns diluting the signal), for no accuracy +benefit given the categories this project tracks are about the *current* +task, not the session's history as a whole. + +## The one place this now matters: the session cache + +The classifier-input-scope property described here is what +`session-classification-cache-ttl.md` explicitly trades away, on purpose, +for latency. Once that cache ships, a "cached" decision no longer re-derives +category from the current message at all — it reuses whatever was true up +to `staleness_minutes` ago. That's a deliberate, bounded trade, not a +regression introduced silently; flagging it here so the connection is +on record in both documents. + +## Recommendation + +No action needed on classifier input scope itself — it already does the +right thing, and LLMRouter's `full_context`/`retrieval` modes don't apply +here (this project's categories are per-current-task, not +per-conversation-history, and there's no embedding/retrieval infrastructure +to reuse for `retrieval` mode even if it were wanted). Retire this as +"checked, no gap found" rather than carrying it forward as an open question. diff --git a/code_plans/context-dual-use-and-classify-once-per-session.md b/code_plans/context-dual-use-and-classify-once-per-session.md index b68df22..379f724 100644 --- a/code_plans/context-dual-use-and-classify-once-per-session.md +++ b/code_plans/context-dual-use-and-classify-once-per-session.md @@ -67,6 +67,12 @@ be rare or nonexistent, this is not worth A's or B's added surface at all. ## 2. Classify once per session, not once per message +> **Settled** — see +> [`session-classification-cache-ttl.md`](session-classification-cache-ttl.md). +> Invalidation (§"Design questions to settle before implementing", item 2) +> is a minutes-based config TTL. The rest of this section is kept for the +> reasoning trail; the successor doc is the one to build from. + ### The problem, restated from the README > ~10s of local overhead on every message is a real tax for an interactive diff --git a/code_plans/session-classification-cache-ttl.md b/code_plans/session-classification-cache-ttl.md new file mode 100644 index 0000000..7aba10b --- /dev/null +++ b/code_plans/session-classification-cache-ttl.md @@ -0,0 +1,178 @@ +# Spec: session-scoped classification cache with a minutes-based staleness TTL + +**Origin.** Settles §2 of +[`context-dual-use-and-classify-once-per-session.md`](context-dual-use-and-classify-once-per-session.md) +("Classify once per session, not once per message"), which laid out the +mechanism but explicitly deferred invalidation policy pending a decision. +That decision is now made: staleness is a config value in **minutes**. Also +borrows the framing (not the mechanism) from +[LLMRouter](https://github.com/ulab-uiuc/LLMRouter)'s "routing memory" — +retrieving a past routing decision instead of reclassifying from scratch. +LLMRouter's version does similarity retrieval over embedded queries; this is +simpler and exact, since the thing being cached is per-session, not +per-query-similarity: same session, same category, until the TTL says +otherwise. This project already has one precedent for porting a llmrouter +idea directly — `PinchConfig` (`config.py:304`) is docstring-labeled "Port of +llmrouter's pinch." Same pattern applies here: keep what's provably useful, +drop what LLMRouter needed for its own broader (multi-provider, ML-router) +scope but this project doesn't. + +No code changes accompany this document — this is the spec opencode builds +from. + +--- + +## The problem, restated + +Every turn in a long agent session pays a full classifier round-trip +(~1.7s local, ~1.0s cloud per the README's own measurements) even though a +session's *category* (`coding_general`, `debugging`, etc.) rarely changes +turn to turn. What changes is mostly token count, which `chat_completions` +already measures directly via `estimate_prompt_tokens` — no model call +needed for that part. + +## Why this is safe to cache (and what must never be cached alongside it) + +**Safe to cache:** `task_category` and `task_tier`. These describe "what +kind of work is this," which is a property of the session's overall task, +not of any single turn's exact wording. + +**Never cache:** the capability flags (`tools_present`, `has_images`, +`require_json_mode`) read by `capabilities.py`. These are cheap (pure +request-body reads, no model call) and can legitimately differ turn to turn +within one session — turn 5 might attach an image, turn 6 might not. Caching +them would silently misroute a request whose actual capabilities changed. +This is not a design option to weigh; it would reproduce exactly the kind of +bug this project's fail-closed capability gates exist to prevent. The +mechanism below only ever short-circuits the classifier call, never the +capability-gate reads, which already run fresh every request regardless. + +Also never cached: `apply_escalation`. It's cheap, pure Python, and already +runs fresh on every `route()` call whether or not that call hit the +classification cache — no change needed there. + +## Mechanism + +`route()`'s override branch (`dispatcher.py`, the branch used when +`task_category`/`task_tier`/`required_context_tokens` are all supplied) +already skips `classify()` entirely — this is what the measured-context +reroute already uses within a single request. A session cache is the same +mechanism applied across requests instead of just within one: + +1. On a **cache hit** (see TTL below): skip `classify()` outright. Compute + `send_messages` (pinch, if enabled) and `measured = + estimate_prompt_tokens(...)` exactly as today, then call `route()` once + with the cached `task_category`/`task_tier` and + `required_context_tokens=measured`. This replaces today's two-step + "classify, then maybe re-route on measured context" with a single + override-branch call, since there's nothing to react to — the category + is already known. +2. On a **cache miss** (first turn in a session, or the entry expired): + unchanged from today — classify, then the existing measured-context + reroute if `measured` exceeds the classifier's own estimate. After the + decision is made, write `(task_category, task_tier, now)` into the + cache under this session's key. + +### Session identity + +Reuse `session_fingerprint(messages)` (`dispatcher.py:1645`) — already +computed once per `chat_completions` call for observation, and its own +docstring is exactly the right property for this: stable for the life of a +session (hashes the opening system-prompt message), distinct across +sessions. No new identity mechanism needed. + +### Staleness (the settled decision) + +New config section: + +```yaml +session_cache: + # Off by default, matching every other new-and-unproven knob in this + # project (pinch.enabled, routing.min_tool_proficiency): ship it, watch + # route_decisions on real traffic, then decide the right default. + enabled: false + # Minutes since the cached classification was written before it's treated + # as expired and the next turn reclassifies from scratch. + staleness_minutes: 20 +``` + +`SessionCacheConfig(StrictModel)` in `config.py`, next to `PinchConfig`, with +a `field_validator` requiring `staleness_minutes > 0` (same pattern as +`PinchConfig.budget_positive`). + +On a lookup, an entry older than `staleness_minutes` is treated exactly like +a miss: reclassify, and overwrite the cache entry with the fresh result and +a fresh timestamp. No partial-credit or extension-on-read — a stale entry is +gone, not renewed. + +**Why minutes, not turns or a context-jump heuristic** (the other two +options §2 of the original spec raised): those need their own tuning and +their own measurement pass to validate. A minutes-based TTL is one number, +directly interpretable ("stop trusting this after N minutes of session +inactivity-adjusted-by-nothing"), and cheap to change without a code +release. If real traffic later shows category drift happening *within* the +TTL window on active sessions (see the interaction note below), a +turn-count or context-jump trigger can be layered on top — but that's a +second iteration, not a blocker for this one. + +### Storage + +An in-memory dict in a new `session_cache.py`, module-level, matching the +process lifetime of the dispatcher — a restart just means every active +session's next turn reclassifies once, which is a safe failure mode (same +reasoning the original spec already used for this). No new persistence +layer. + +```python +@dataclass(frozen=True) +class CachedClassification: + task_category: str + task_tier: int + cached_at: float # time.time() + +def get(session_key: str, staleness_seconds: float) -> Optional[CachedClassification]: ... +def put(session_key: str, task_category: str, task_tier: int) -> None: ... +``` + +Pure functions, no I/O — same shape as `scoring.py`/`tiering.py`, testable +without a running dispatcher. + +### Observability + +`Classification.source` (`dispatcher.py:215`, +`Literal["classifier", "override", "fallback"]`) gains a fourth value: +`"cached"`. This was already anticipated by name in the original spec's +design questions. Set it at the `chat_completions` call site (same place +`classified_src`/`ctx_src` are already tracked locally) when a cache hit +served the decision — no changes needed in `route()`/`tiering.py` itself. +Once this Literal is updated, `/metrics`'s existing classification-source +breakdown and the TUI's decision detail popup pick it up for free — they +already group by `source`. + +## Interaction with the classifier-input-scope finding + +[`classifier-input-scope-check.md`](classifier-input-scope-check.md) (this +same batch of plans) confirms the classifier currently reads only the last +user message plus one prior assistant turn, refreshed every request — which +is incidentally the thing that lets today's per-message classification +track category drift within a session for free. This cache trades that +away on purpose, on the bet that drift is rare relative to cost: a session +that's 40 turns of `coding_general` and then turns into `docs_writing` will +serve up to `staleness_minutes` of stale category before the TTL expires +and it corrects itself. `staleness_minutes` should be picked with this +trade-off in mind, not purely for latency savings — a smaller value trades +away less accuracy for less cache benefit. This is exactly the kind of +thing `route_decisions.source = "cached"` makes observable: watch for +`"cached"` decisions whose category looks wrong in hindsight (e.g. compared +against a same-session `POST /outcome` failure) before tuning the default +up from 20 minutes. + +## Recommendation + +Build it. The mechanism is a small extension of code that already exists +(the override branch), the session-identity question is already answered by +existing code, and the staleness question is now answered by you. Ship +`session_cache.enabled: false` by default per this project's standing +pattern for new knobs, flip it on for real sessions, and watch +`route_decisions.source = "cached"` against outcomes before considering a +non-default `staleness_minutes` or a smarter invalidation trigger. diff --git a/code_reviews/flex-preference-knob-review.md b/code_reviews/flex-preference-knob-review.md new file mode 100644 index 0000000..78e061a --- /dev/null +++ b/code_reviews/flex-preference-knob-review.md @@ -0,0 +1,171 @@ +# flex-preference-knob review + +Source: manual review (this session) of `65da4a1` ("feat(routing): add +flex-preference knob for -flex variant selection") against +`.omo/plans/flex-preference-knob.md`. Full suite is green (601 passed) and +the plan's stated behavior — swap only the serving class, never the base +model, re-price on swap, log telemetry — checks out. One finding, confirmed +live against `routing.py` and against the current catalog, not just read off +the diff. + +--- + +## 1. A flex swap bypasses every hard filter except latency + +**File:** `routing.py`, `apply_flex_preference` (~line 195-269), called from +`dispatcher.route()` (~line 798-811). + +**Problem.** `get_flex_sibling` is deliberately structural — it matches a +row by `base_model_id`/`reasoning_mode`/`context_variant` + `latency_class == +"flex"` and nothing else. Its own test says why that's fine: + +```python +def test_flex_sibling_ignores_stale_or_unroutable_flex_rows(): + # The helper is purely structural: it matches by identity dims regardless + # of freshness/access. Routing-level gating is the caller's job. +``` + +But the caller — `apply_flex_preference` — never does that gating. It +checks exactly one thing before swapping: `latency_tolerance == INTERACTIVE` +(for `prefer-flex`; `force-flex` doesn't even check that). It never re-runs +`rejection_reason`/`is_eligible` against the sibling, so a swap can hand back +a row that `select_candidates` would have thrown out for being stale, +deprecated, outside `allowed_access_levels`, under-tiered, or missing a +required capability. The comment justifying the latency-only check — +"the only hard filter that distinguishes it is the latency filter" — is an +assumption about the catalog, not something the code checks, and the live +catalog already violates the premise: `glm-5.2` (`access_level: canary`) and +`glm-5.2-flex` (`access_level: public`) share `base_model_id`/ +`reasoning_mode`/`context_variant` and differ in access level, not just +latency class. Nothing stops that divergence from appearing in the other +direction (a `-flex` row going stale, deprecated, or more restricted than +its standard sibling) — the poller marks `stale`/`deprecated` per row from +each row's own `last_updated`, and access level is parsed from the +provider's free-text description per row (`poller.parse_access_level`), +independently for every `model_id`. + +Confirmed live, calling `apply_flex_preference` directly: + +```python +>>> from routing import apply_flex_preference +>>> std = {"model_id": "kimi-k3", "base_model_id": "kimi-k3", +... "reasoning_mode": "default", "context_variant": "full", +... "latency_class": "standard", "cost": 0.01, +... "proficiency_score": 0.9, "cost_score": 0.5, "composite": 0.9, +... "cost_per_1m_prompt": 1.0, "cost_per_1m_completion": 2.0} +>>> stale_flex = {**std, "model_id": "kimi-k3-flex", "latency_class": "flex", +... "availability": "stale"} +>>> apply_flex_preference(std, [std, stale_flex], "force-flex", "interactive", +... prompt_tokens=1000, completion_tokens=400, cache_rate=0.5) +({'model_id': 'kimi-k3-flex', ..., 'availability': 'stale'}, True, True, ...) +``` + +and the same for an access-restricted sibling under `prefer-flex`/`batch` +(no `force` needed — `prefer-flex` doesn't check access either): + +```python +>>> canary_flex = {**std, "model_id": "glm-5.2-short-flex", +... "latency_class": "flex", "access_level": "canary"} +>>> apply_flex_preference(std, [std, canary_flex], "prefer-flex", "batch", +... prompt_tokens=1000, completion_tokens=400, cache_rate=0.5) +({'model_id': 'glm-5.2-short-flex', ..., 'access_level': 'canary'}, True, False, ...) +``` + +Both swaps go through with no error, and `dispatcher.route()` puts the +result straight into `RouteResponse.selected` — there is no gate between +`apply_flex_preference` and the actual provider call. A restricted-access +swap dispatches to a model the account almost certainly gets a 403 from; a +stale/deprecated swap dispatches to a row the freshness filter exists +specifically to keep traffic away from. This is exactly the "silent empty +axis" failure class the project has hit before (the `6e729ad`/`1d1f3af` +seed-sweep crash, the `client_capped` over-marking) — a real check exists +elsewhere in the codebase (`rejection_reason`) and this new path just +doesn't call it. + +Not just access/freshness: the same gap covers `required_tier`, +`effective_context_window`, `min_tool_proficiency`, and the vision/json-mode +capability gates, for the same reason — none of them are re-checked on the +sibling. Access and staleness are the two proven-live cases; the others are +plausible but unconfirmed against current data. + +**Fix.** `apply_flex_preference` needs the same filter arguments +`select_candidates`/`rejection_reason` already take, and must refuse the +swap unless the sibling clears them. The cleanest way to get "every filter +except latency" without a second copy of the rule is to call +`rejection_reason` on the sibling with `latency_tolerance` forced to +`BATCH` (which always admits flex rows), so only the non-latency filters can +reject it: + +```python +def apply_flex_preference( + selected_row, all_candidates, flex_preference, latency_tolerance, + *, prompt_tokens, completion_tokens, cache_rate, + required_context_tokens, required_tier, allowed_access_levels, + exclude_stale, exclude_deprecated, min_tool_proficiency=None, + require_vision=False, require_json_mode=False, +) -> tuple[dict, bool, bool, float | None]: + ... + sibling = get_flex_sibling(selected_row, all_candidates) + if sibling is None: + return selected_row, False, False, selected_row.get("cost") + + # The sibling is a different catalog row and can independently be stale, + # deprecated, access-restricted, or under-tiered -- BATCH is used here + # only to neutralize the latency filter itself, which prefer-flex/ + # force-flex evaluate separately below. + if rejection_reason( + sibling, + required_context_tokens=required_context_tokens, + required_tier=required_tier, + latency_tolerance=BATCH, + allowed_access_levels=allowed_access_levels, + exclude_stale=exclude_stale, + exclude_deprecated=exclude_deprecated, + min_tool_proficiency=min_tool_proficiency, + require_vision=require_vision, + require_json_mode=require_json_mode, + ) is not None: + return selected_row, False, False, selected_row.get("cost") + ... +``` + +`dispatcher.route()` already builds exactly this filter set as `filters` +(line ~749-762, used for `select_candidates`) — pass it through to +`apply_flex_preference` rather than assembling a second, narrower one. + +**Test to add** (`tests/test_routing.py`, next to the existing +`test_apply_force_flex_*` cases): a flex sibling with `availability="stale"` +and one with `access_level="canary"` (allowed levels `["public"]`), under +both `prefer-flex`/batch and `force-flex`/interactive — assert no swap +(`flex_swapped is False`, original `selected_row` returned) in all four +combinations. `tests/test_route_decisions.py`'s `decision_router` fixture +only ever inserts `CHEAP_FLEX` as `public`/`active`; add a second flex +fixture row with `availability="stale"` for an end-to-end version of the +same check through `/route`. + +--- + +## Everything else checked out + +- `get_flex_sibling` is pure, O(N), matches the plan's three identity + dimensions exactly, and correctly returns `None` when the selected row is + already flex. +- `route_decisions` migration (`ensure_route_decisions`) is a proper + idempotent `ALTER TABLE ... ADD COLUMN` guarded by `PRAGMA table_info`, + matching the `proficiency_store.ensure_columns` pattern this project + already uses elsewhere. +- `flex_preference`/`flex_swapped`/`flex_forced` telemetry is wired + correctly end-to-end: `RouteResponse` → `persist_route_decision` → + `route_decisions` INSERT → SSE event payload. This was the one thing + flagged as worth re-checking when the config-default question was settled + ([[project-flex-preference-knob]]), and it holds up — confirmed by reading + the full call chain, not just the happy-path tests. +- Cost re-estimation on swap reads the sibling's own + `cost_per_1m_prompt`/`cost_per_1m_completion` via `estimated_cost`, and + carries over `proficiency_score`/`cost_score`/`composite` from the ranked + winner so the returned dict stays a well-formed `Candidate` — correct, + since a flex twin ties its standard sibling on every scored dimension by + construction. +- `no-flex`/`auto` are true no-ops (never swap, never touch an + already-flex winner for `no-flex`), matching the plan. +- Full `pytest` (601 tests) and `python config.py` both pass clean. diff --git a/code_reviews/session-cache-and-baseline-comparator-review.md b/code_reviews/session-cache-and-baseline-comparator-review.md new file mode 100644 index 0000000..cfc2609 --- /dev/null +++ b/code_reviews/session-cache-and-baseline-comparator-review.md @@ -0,0 +1,140 @@ +# session-cache-and-baseline-comparator review + +Source: manual review of the batch built from +[`session-classification-cache-ttl.md`](../code_plans/session-classification-cache-ttl.md), +[`baseline-routing-comparator.md`](../code_plans/baseline-routing-comparator.md), +and [`classifier-input-scope-check.md`](../code_plans/classifier-input-scope-check.md). +Full suite verified green independently (638 passed). One component is clean +and committed; the other has a bug confirmed live against `router.db`, not +just read off the diff — and is not actually committed yet, despite the +delivery summary saying "all deliverables are complete and merged." + +--- + +## Session classification cache — clean, matches spec + +Commits `245842d`, `ba5b50d`, `f4557d4`, `0a26f58`, `756341d`. No findings. + +Specifically verified: + +- `session_cache.py` is pure (no I/O, no imports of `dispatcher`/`config`), + TTL comparison is `>` not `>=` (a hit exactly on the boundary is still + fresh, matching the spec's "exactly 60s later... not strictly greater... + so fresh" framing), and `get` never extends an entry's life — only `put` + writes a fresh timestamp. +- `dispatcher.py`'s integration correctly reorders to compute `measured` + tokens before the cache check, issues exactly one `route()` call on a + cache hit (never a wasted classify-then-reroute), and captures + `classified_src` *before* the override reroute can overwrite + `decision.classification.source` — so the cache-write gate + (`classified_src == "classifier"`) reflects the original classification + attempt, not whatever the source ends up as after a measured-context + reroute. +- Fallback classifications are never written to the cache (tested directly: + `test_fallback_classification_is_never_cached`). +- Capability flags stay fresh on a cache hit — tested directly + (`test_capability_flags_are_still_read_fresh_on_a_cache_hit`) by sending a + same-session image turn after a cached text turn and confirming it still + routes to a vision-capable model and logs `source="cached"` for the + *routing* decision while still applying the freshly-read `has_images` + gate. This was the one property that mattered most in the spec, and it's + the one with a dedicated test. + +## Baseline routing comparator — not committed, and wrong against real data + +`baseline_report.py` and `tests/test_baseline_report.py` are untracked in +git (`git status --short` shows both as `??`), unlike every other file in +this batch. Nothing here is merged. + +### 1. `load_decisions` filters `kind = 'route'`, which excludes 99.5% of real traffic + +**File:** `baseline_report.py:85` (`load_decisions`), consumed by `analyze`. + +**Problem.** The query hardcodes `kind = 'route'`. Live `router.db`: + +``` +sqlite> SELECT kind, COUNT(*) FROM route_decisions GROUP BY kind; +chat|2209 +route|10 +``` + +`"route"` is the low-volume `/route` probe endpoint (no quota spent, mostly +used for testing). `"chat"` is `/v1/chat/completions` — the actual +dogfooding traffic this whole tool exists to audit, and it's 99.5% of the +table. Running the report against the real database does not error, it +just silently analyzes almost nothing: + +``` +$ python baseline_report.py --since 2026-08-01 +baseline routing comparator + decisions: 10 + ... +``` + +10 decisions, all of which are leftover manual test calls from this +session's own debugging, not the dogfooding traffic the tool is meant to +audit. This is exactly the failure mode the README repeatedly warns about +in its own history — "scored the rig, not the model" — reproduced in a new +tool: the test fixture +(`tests/test_baseline_report.py:96`, `INSERT INTO route_decisions (... kind +...) VALUES (?, 'route', ...)`) only ever seeds `kind='route'` rows, so the +suite is internally consistent and green without ever exercising what the +live table actually contains. 638 passing tests didn't catch this because +no test's fixture data resembled production shape here. + +**Fix.** `persist_route_decision` is called with four kinds that represent +an actual scored routing decision — `"route"` (`dispatcher.py:1628`), +`"chat"` (`:2255`, `:2278`), and `"dispatch"` (`:2664`) — versus two that +don't: `"passthrough"` (`:2297`, a client pin that bypasses scoring +entirely) and `"local_vision"` (`:2214`, logged specifically because +routing found *no* eligible cloud candidate — there's no "eligible set" to +rank a baseline within). The filter should be `kind IN ('route', 'chat', +'dispatch')`, not `kind = 'route'`. + +### 2. Rejected decisions (no `selected_model`) would pollute the aggregate once #1 is fixed + +**File:** `baseline_report.py:230-253` (`analyze`'s accumulation loop). + +**Problem.** `dispatcher.py:2255` logs a `"chat"` row with `rejected_reason` +set and no `selected_model` when routing finds nothing eligible (a 422 to +the client — no work was actually dispatched). Once finding #1's filter is +widened to include `"chat"`, these rows enter `analyze`'s loop. +`agg["actual_cost"] += d["est_cost_usd"] or 0.0` and +`agg["actual_proficiency"] += ap if ap is not None else 0.0` both silently +add `0.0` for a request that was never served, which pools "the router +refused this" together with "the router served this for free" in the +`actual_*` aggregates — understating real average cost/proficiency for any +category with rejections. `cheapest["model_id"] == d["selected_model"]` +degrades gracefully here (`None` never matches, so it just doesn't count as +dominance) but the cost/proficiency pollution does not degrade gracefully. + +Not currently visible in `router.db` — `SELECT COUNT(*) FROM +route_decisions WHERE kind IN ('route','chat','dispatch') AND +selected_model IS NULL` returns `0` on this deployment right now — but it's +a latent bug in the aggregation logic, not something that depends on luck +staying the current way, and the fix is one clause. + +**Fix.** Add `selected_model IS NOT NULL` to `load_decisions`'s `WHERE` +clause alongside the `kind` fix, so a rejected/unserved request is excluded +from both the aggregate and the dominance count, the same way this project +already excludes non-model-attributable failures elsewhere +(`model_attributable = 0` in the verification-feedback path). + +### Everything else in the file checks out + +`reconstruct_decision`'s filter-dict construction correctly gates +`min_tool_proficiency`/`require_vision`/`require_json_mode` only when the +original decision's `tools`/`images`/`json_mode` flags were set (mirroring +`capabilities.py`'s "only applies when the request carries it" rule), the +cheapest/best-proficiency tie-breaks are deterministic +(`model_id` as the final tie-break in both), and `--csv`/human-table output +both read from the same `analyze()` result so they can't drift from each +other. The "current catalog, not historical snapshot" limitation is stated +plainly in the module docstring, matching the plan. + +## Recommendation + +Fix both findings in `baseline_report.py` (one `WHERE` clause each), add a +`chat`-kind fixture row to `tests/test_baseline_report.py` so the test +suite would have caught #1, then commit. Session cache needs nothing +further. diff --git a/tests/test_baseline_report.py b/tests/test_baseline_report.py new file mode 100644 index 0000000..f402c90 --- /dev/null +++ b/tests/test_baseline_report.py @@ -0,0 +1,266 @@ +"""Tests for the baseline_report.py read-only routing comparator. + +Seeds a throwaway temp SQLite DB with a small catalog, some proficiency rows, +and a handful of route_decisions, then asserts on the aggregate analysis and +the reconstructed baselines. No network, no provider calls, no dispatcher +import — the report is pure read-over-seeded-tables. +""" + +from __future__ import annotations + +import csv +import io +import sqlite3 +from pathlib import Path + +import pytest + +import baseline_report +from config import load_config + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_SQL = (ROOT / "schema.sql").read_text() +CFG = load_config(str(ROOT / "config.yaml")) + + +@pytest.fixture() +def conn(tmp_path: Path) -> sqlite3.Connection: + c = sqlite3.connect(str(tmp_path / "test.db")) + c.row_factory = sqlite3.Row + c.executescript(SCHEMA_SQL) + return c + + +def _seed_model( + conn: sqlite3.Connection, + model_id: str, + *, + cost: float = 1.0, + tier: int = 1, + context: int = 262128, + latency: str = "standard", + access: str = "public", +) -> None: + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + cost_per_1m_prompt_cached, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'neuralwatt', ?, ?, ?, ?, 16384, ?, ?, ?, + ?, 1, 'default', 'full', + ?, 'public', 'active', '2026-08-22T00:00:00+00:00') + """, + ( + model_id, model_id, tier, context, context, + cost, cost / 3, cost / 2, + latency, access, + ), + ) + + +def _seed_proficiency( + conn: sqlite3.Connection, model_id: str, category: str, score: float, +) -> None: + conn.execute( + """ + INSERT INTO proficiency ( + model_id, provider, category, blended_score, source, last_updated + ) VALUES (?, 'neuralwatt', ?, ?, 'blended', '2026-08-22T00:00:00+00:00') + """, + (model_id, category, score), + ) + + +def _seed_decision( + conn: sqlite3.Connection, + *, + observed_at: str, + category: str, + tier: int = 1, + context: int = 1000, + latency: str = "interactive", + selected: str, + est_cost: float, + est_prof: float, + tools: int = 0, + images: int = 0, + json_mode: int = 0, +) -> None: + conn.execute( + """ + INSERT INTO route_decisions ( + observed_at, kind, task_category, task_tier, required_context_tokens, + confidence, classifier_ms, classification_source, latency_tolerance, + candidates_considered, selected_model, selected_provider, + est_cost_usd, est_proficiency, + tools, images, json_mode, streamed, + flex_preference, flex_swapped, flex_forced + ) VALUES (?, 'route', ?, ?, ?, 0.95, 200, 'classifier', ?, + 3, ?, 'neuralwatt', ?, ?, ?, ?, ?, 0, + 'auto', 0, 0) + """, + ( + observed_at, category, tier, context, latency, + selected, est_cost, est_prof, tools, images, json_mode, + ), + ) + + +def _seed_basic_catalog(conn: sqlite3.Connection) -> None: + # cheap: low proficiency, tier 1, large context + _seed_model(conn, "cheap", cost=0.10, tier=1, context=131072) + # dear: high proficiency, tier 1, large context + _seed_model(conn, "dear", cost=9.00, tier=1, context=131072) + # mid-small: cheap but small context (fails when the request needs more) + _seed_model(conn, "small", cost=0.05, tier=1, context=4096) + # low-tier: cheap and large but tier 3 (fails tier-1 requests) + _seed_model(conn, "frontier", cost=7.00, tier=3, context=262128) + _seed_proficiency(conn, "cheap", "coding_general", 0.40) + _seed_proficiency(conn, "dear", "coding_general", 0.90) + _seed_proficiency(conn, "small", "coding_general", 0.35) + _seed_proficiency(conn, "frontier", "coding_general", 0.95) + conn.commit() + + +class TestBaselineSelection: + def test_cheapest_picks_lowest_cost_eligible(self, conn) -> None: + _seed_basic_catalog(conn) + candidates = baseline_report.load_candidates(conn, "coding_general", CFG) + eligible = baseline_report.select_candidates( + candidates, + required_context_tokens=1000, + required_tier=1, + latency_tolerance="interactive", + allowed_access_levels=CFG.routing.allowed_access_levels, + exclude_stale=CFG.freshness.exclude_stale, + exclude_deprecated=CFG.freshness.exclude_deprecated, + ) + # cheap, small, dear, frontier are all eligible at tier 1 / 1000 tokens. + cheapest, best = baseline_report.baseline_selection( + eligible, decision_fake(context=1000), CFG + ) + assert cheapest["model_id"] == "small" # lowest list price + assert best["model_id"] == "frontier" # highest proficiency, ties by cost + + def test_empty_eligible_set_yields_none(self, conn) -> None: + _seed_basic_catalog(conn) + cheapest, best = baseline_report.baseline_selection([], decision_fake(1000), CFG) + assert cheapest is None + assert best is None + + +def decision_fake(context: int) -> dict: + """A minimal decision-shaped mapping for baseline/reconstruct helpers.""" + return { + "required_context_tokens": context, + "task_tier": 1, + "latency_tolerance": "interactive", + "tools": 0, + "images": 0, + "json_mode": 0, + "task_category": "coding_general", + "selected_model": None, + "est_cost_usd": 0.0, + "est_proficiency": 0.0, + } + + +class TestReconstruct: + def test_constraints_respected(self, conn) -> None: + # A decision needing a huge context must not offer "small". + _seed_basic_catalog(conn) + decision = decision_fake(context=10_000) + cheapest, best = baseline_report.reconstruct_decision(conn, decision, CFG) + # small dropped by context; frontier outranks dear on proficiency. + assert cheapest["model_id"] == "cheap" + assert best["model_id"] == "frontier" + + +class TestAnalyze: + def test_aggregate_and_dominance(self, conn) -> None: + _seed_basic_catalog(conn) + # Two decisions: one selects the cheapest eligible (small -> dominant), + # one selects dear (not dominant). + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="coding_general", + selected="small", est_cost=0.001, est_prof=0.35, + ) + _seed_decision( + conn, observed_at="2026-08-21T00:00:00+00:00", category="coding_general", + selected="dear", est_cost=0.09, est_prof=0.90, + ) + total, cat_rows = baseline_report.analyze(conn, CFG) + assert total["count"] == 2 + assert total["dominance_count"] == 1 + assert total["dominance_pct"] == 50.0 + assert len(cat_rows) == 1 + assert cat_rows[0]["category"] == "coding_general" + + def test_category_filter(self, conn) -> None: + _seed_basic_catalog(conn) + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="coding_general", + selected="cheap", est_cost=0.001, est_prof=0.40, + ) + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="translation", + selected="dear", est_cost=0.09, est_prof=0.70, + ) + total, _ = baseline_report.analyze(conn, CFG, category="translation") + assert total["count"] == 1 + assert total["actual_cost"] == pytest.approx(0.09) + + def test_since_filter(self, conn) -> None: + _seed_basic_catalog(conn) + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="coding_general", + selected="cheap", est_cost=0.001, est_prof=0.40, + ) + _seed_decision( + conn, observed_at="2026-08-25T00:00:00+00:00", category="coding_general", + selected="dear", est_cost=0.09, est_prof=0.90, + ) + total, _ = baseline_report.analyze(conn, CFG, since="2026-08-21") + assert total["count"] == 1 + assert total["actual_cost"] == pytest.approx(0.09) + + def test_empty_window_returns_zero_counts(self, conn) -> None: + _seed_basic_catalog(conn) + total, cat_rows = baseline_report.analyze(conn, CFG, since="2099-01-01") + assert total["count"] == 0 + assert total["dominance_pct"] is None + assert cat_rows == [] + + +class TestOutput: + def test_format_summary_has_dominance(self, conn) -> None: + _seed_basic_catalog(conn) + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="coding_general", + selected="small", est_cost=0.001, est_prof=0.35, + ) + total, cat_rows = baseline_report.analyze(conn, CFG) + text = baseline_report.format_summary(total, cat_rows) + assert "dominance" in text + assert "100.0%" in text + + def test_csv_parses(self, conn) -> None: + _seed_basic_catalog(conn) + _seed_decision( + conn, observed_at="2026-08-20T00:00:00+00:00", category="coding_general", + selected="small", est_cost=0.001, est_prof=0.35, + ) + total, cat_rows = baseline_report.analyze(conn, CFG) + buf = io.StringIO() + baseline_report.write_csv(total, cat_rows, buf) + buf.seek(0) + rows = list(csv.DictReader(buf)) + assert rows[0]["category"] == "total" + assert rows[1]["category"] == "coding_general" + assert rows[0]["count"] == "1" + assert rows[0]["dominance_pct"] == "100.0" -- 2.49.1 From e24006a1b9b45f26e6eacbd7df286fea5283295a Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 18:18:09 -0400 Subject: [PATCH 26/32] fix(baseline): expand traffic filter and prune rejected decisions --- baseline_report.py | 5 ++++- tests/test_baseline_report.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/baseline_report.py b/baseline_report.py index 841c086..6d91636 100644 --- a/baseline_report.py +++ b/baseline_report.py @@ -82,7 +82,10 @@ def load_decisions( ``YYYY-MM-DD`` prefix is a valid time filter — a decision whose timestamp starts at or after ``since`` is inside the window. """ - clauses: list[str] = ["kind = 'route'"] + clauses: list[str] = [ + "kind IN ('route', 'chat', 'dispatch')", + "selected_model IS NOT NULL", + ] params: list[str] = [] if since is not None: clauses.append("observed_at >= ?") diff --git a/tests/test_baseline_report.py b/tests/test_baseline_report.py index f402c90..c8a9496 100644 --- a/tests/test_baseline_report.py +++ b/tests/test_baseline_report.py @@ -170,6 +170,39 @@ def decision_fake(context: int) -> dict: } +class TestLoadDecisions: + def test_chat_rows_are_loaded(self, conn) -> None: + """The report must load scored routing decisions, not just 'route'. + + Real dogfooding traffic is recorded with kinds ``route``, ``chat`` and + ``dispatch``; only rows where a model was actually selected carry a + ``selected_model``. ``load_decisions`` should include those rows. + """ + _seed_basic_catalog(conn) + conn.execute( + """ + INSERT INTO route_decisions ( + observed_at, kind, task_category, task_tier, required_context_tokens, + confidence, classifier_ms, classification_source, latency_tolerance, + candidates_considered, selected_model, selected_provider, + est_cost_usd, est_proficiency, + tools, images, json_mode, streamed, + flex_preference, flex_swapped, flex_forced + ) VALUES (?, 'chat', ?, 1, 1000, 0.95, 200, 'classifier', 'interactive', + 3, 'cheap', 'neuralwatt', 0.001, 0.40, + 0, 0, 0, 0, + 'auto', 0, 0) + """, + ("2026-08-20T00:00:00+00:00", "coding_general"), + ) + conn.commit() + + decisions = baseline_report.load_decisions(conn, since=None, category=None) + assert len(decisions) == 1 + assert decisions[0]["kind"] == "chat" + + + class TestReconstruct: def test_constraints_respected(self, conn) -> None: # A decision needing a huge context must not offer "small". -- 2.49.1 From 32e1c30411ffe5c2fe24db5266477f1b134c0509 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 18:18:37 -0400 Subject: [PATCH 27/32] fix(events): bound SSE queues and drop stale legacy fan-out --- dispatcher.py | 2 - events.py | 79 +++------------------- tests/test_events.py | 158 ++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 171 deletions(-) diff --git a/dispatcher.py b/dispatcher.py index f44d00a..27664f4 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -1393,8 +1393,6 @@ async def _decision_event_stream(): except asyncio.TimeoutError: yield ":heartbeat\n\n" continue - if decision is events._EVICTED: - break yield _sse_data(decision) finally: events.unsubscribe_sse(decision_queue) diff --git a/events.py b/events.py index 9f01781..e2c7b30 100644 --- a/events.py +++ b/events.py @@ -14,7 +14,6 @@ ring is all that survives. The durable source of truth remains the from __future__ import annotations import asyncio -import queue import threading from collections import deque from typing import Any, Final @@ -22,14 +21,7 @@ from typing import Any, Final DEFAULT_BUFFER_SIZE: Final[int] = 100 DEFAULT_QUEUE_SIZE: Final[int] = 100 -# Sentinel value to signal a subscriber that has been evicted -# (its queue was full). The SSE stream generator detects this and -# terminates so the TUI can reconnect. -_EVICTED = object() - _buffer: deque[dict[str, Any]] = deque(maxlen=DEFAULT_BUFFER_SIZE) -# Thread-safe (queue.Queue) subscribers, used by the legacy SSE path. -_subscribers: set[queue.Queue[dict[str, Any]]] = set() # Async subscribers for the live SSE endpoint. Each entry maps the # asyncio.Queue to the event loop it was created on, so the (sync) # publisher can bridge onto it with ``call_soon_threadsafe``. @@ -68,59 +60,17 @@ def publish_decision(decision: dict[str, Any]) -> None: """ _buffer.append(decision) - with _subscribers_lock: - dead: set[queue.Queue[dict[str, Any]]] = set() - for subscriber in _subscribers: - try: - subscriber.put_nowait(decision) - except queue.Full: - dead.add(subscriber) - _subscribers.difference_update(dead) - # Evicted legacy subscribers get a sentinel so their SSE generator can - # terminate rather than block forever on the orphaned queue. - for subscriber in dead: - try: - subscriber.put_nowait(_EVICTED) - except queue.Full: - try: - subscriber.get_nowait() # discard the oldest item - subscriber.put_nowait(_EVICTED) - except queue.Empty: - pass - - # Bridge to async SSE subscribers on their own loops. Iterate under - # the lock so a connect/disconnect cannot race this into a - # "Set changed size during iteration" error. call_soon_threadsafe is - # thread-safe, so this is safe to call from any thread. - for sse_queue, loop in list(_sse_loops.items()): - try: - loop.call_soon_threadsafe(_sse_push, sse_queue, decision) - except RuntimeError: - # The loop shut down; drop its dead queue reference. - _sse_subscribers.discard(sse_queue) - _sse_loops.pop(sse_queue, None) - - -def subscribe( - replay: bool = True, - max_queue: int = DEFAULT_QUEUE_SIZE, -) -> queue.Queue[dict[str, Any]]: - """Create a thread-safe subscriber queue. - - By default the current ring-buffer contents are preloaded so a new - connection immediately sees recent decisions. The caller must arrange to - remove the queue with :func:`unsubscribe` when it disconnects. - """ - new_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=max_queue) - if replay: - for decision in _buffer: - try: - new_queue.put_nowait(decision) - except queue.Full: - break - with _subscribers_lock: - _subscribers.add(new_queue) - return new_queue + # Bridge to async SSE subscribers on their own loops. Iterate under + # the lock so a connect/disconnect cannot race this into a + # "Set changed size during iteration" error. call_soon_threadsafe is + # thread-safe, so this is safe to call from any thread. + for sse_queue, loop in list(_sse_loops.items()): + try: + loop.call_soon_threadsafe(_sse_push, sse_queue, decision) + except RuntimeError: + # The loop shut down; drop its dead queue reference. + _sse_subscribers.discard(sse_queue) + _sse_loops.pop(sse_queue, None) def subscribe_sse( @@ -148,12 +98,6 @@ def subscribe_sse( break -def unsubscribe(subscriber: queue.Queue[dict[str, Any]]) -> None: - """Remove ``subscriber`` from the fan-out set.""" - with _subscribers_lock: - _subscribers.discard(subscriber) - - def unsubscribe_sse(sse_queue: asyncio.Queue[dict[str, Any]]) -> None: """Remove an asyncio.Queue from the fan-out set.""" with _subscribers_lock: @@ -173,7 +117,6 @@ def clear() -> None: almost never call this. """ with _subscribers_lock: - _subscribers.clear() _sse_subscribers.clear() _sse_loops.clear() _buffer.clear() diff --git a/tests/test_events.py b/tests/test_events.py index ba400f1..6a3cd1e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -1,18 +1,16 @@ """Tests for the in-memory decision-event broker (events.py). -Offline, no network. Drive the thread-safe queue broker directly: publish, -subscribe-with-replay, unsubscribe, and the interaction where a decision is -published with no subscriber (must not error and must stay in the ring). +Offline, no network. Drive the broker directly: publish, replay into a new +SSE subscriber, and the interaction where a decision is published with no +subscriber (must not error and must stay in the ring). """ from __future__ import annotations -import queue import threading import pytest -import dispatcher import events @@ -33,88 +31,11 @@ def test_publish_adds_to_ring_and_stays_for_replay(): assert [d["id"] for d in events.recent_decisions()] == [1, 2] -def test_subscribe_replays_ring_into_new_queue(): - events.publish_decision(_decision(1)) - events.publish_decision(_decision(2)) - sub = events.subscribe(replay=True) - assert [d["id"] for d in _drain(sub)] == [1, 2] - - -def test_subscribe_without_replay_starts_empty(): - events.publish_decision(_decision(1)) - sub = events.subscribe(replay=False) - assert _drain(sub) == [] - - -def test_publish_fans_out_to_live_subscribers(): - sub = events.subscribe() - events.publish_decision(_decision(1)) - assert [d["id"] for d in _drain(sub)] == [1] - - -def test_unsubscribe_stops_delivery(): - sub = events.subscribe() - events.unsubscribe(sub) - events.publish_decision(_decision(1)) - assert _drain(sub) == [] - - def test_publish_with_no_subscriber_keeps_ring_and_does_not_raise(): events.publish_decision(_decision(1)) assert [d["id"] for d in events.recent_decisions()] == [1] -def test_full_subscriber_is_dropped_not_blocked(): - events.clear() - subscriber = events.subscribe(replay=False, max_queue=1) - events.publish_decision(_decision(1)) - events.publish_decision(_decision(2)) # must not raise or block; triggers eviction - # Eviction removed the subscriber from fan-out and delivered a sentinel so - # the SSE generator can terminate; the ring still contains both decisions. - assert subscriber not in events._subscribers - assert len(events.recent_decisions()) == 2 - - -def _drain(q: queue.Queue) -> list: - out = [] - while True: - try: - item = q.get_nowait() - except queue.Empty: - return out - if item is events._EVICTED: - return out - out.append(item) - - -def test_concurrent_publish_subscribe_unsubscribe_raises_no_error(): - errors: list[Exception] = [] - - def _publisher(): - for i in range(200): - try: - events.publish_decision({"id": i, "round": "pub"}) - except (RuntimeError, queue.Full) as exc: - errors.append(exc) - - def _subscriptor(): - for _ in range(10): - try: - sub = events.subscribe(replay=False) - events.unsubscribe(sub) - except RuntimeError as exc: - errors.append(exc) - - threads = [threading.Thread(target=_publisher) for _ in range(4)] + [ - threading.Thread(target=_subscriptor) for _ in range(4) - ] - for t in threads: - t.start() - for t in threads: - t.join() - assert not errors, f"unexpected errors during concurrent access: {errors}" - - def test_e2e_sse_publish_from_thread(): """Verifies that publish_decision bridges to a subscribed asyncio.Queue when called from a plain threading.Thread — the exact production shape @@ -164,25 +85,60 @@ def test_e2e_sse_publish_from_thread(): @pytest.mark.anyio -async def test_full_subscriber_eviction_terminates_decision_event_stream(monkeypatch): - """When a subscriber is evicted (full queue), the _decision_event_stream - generator breaks out of its loop instead of blocking forever.""" +async def test_sse_bridge_from_plain_thread_delivers_to_queue(): + """A decision published from a plain threading.Thread must arrive on an + asyncio.Queue registered via subscribe_sse. + + This is the production shape — persist_route_decision runs on an + anyio/Starlette worker thread, not the /events/decisions loop thread, and + calls publish_decision, which must bridge onto the subscriber's loop via + call_soon_threadsafe. Regression for finding #5 in + tui-live-routing-panel-sse-fix-review.md. + """ + import asyncio + events.clear() - - registered_queues = [] - def fake_subscribe_sse(q, replay=False): - registered_queues.append(q) - - monkeypatch.setattr(events, "subscribe_sse", fake_subscribe_sse) - monkeypatch.setattr(events, "unsubscribe_sse", lambda q: None) + sse_queue: asyncio.Queue[dict] = asyncio.Queue() + events.subscribe_sse(sse_queue, replay=False) - feed = dispatcher._decision_event_stream() - assert (await anext(feed)).startswith("retry:") - - q = registered_queues[0] - await q.put(_decision(1)) - assert await anext(feed) == dispatcher._sse_data(_decision(1)) + th = threading.Thread( + target=events.publish_decision, + args=({"id": 911, "selected_model": "x"},), + ) + th.start() + th.join(timeout=5) + assert not th.is_alive(), "publisher thread did not finish" - await q.put(events._EVICTED) - with pytest.raises(StopAsyncIteration): - await anext(feed) + decision = await asyncio.wait_for(sse_queue.get(), timeout=2.0) + assert decision == {"id": 911, "selected_model": "x"} + + events.unsubscribe_sse(sse_queue) + + +@pytest.mark.anyio +async def test_sse_push_drops_oldest_when_queue_full(): + """_sse_push must bound memory by dropping the oldest item when a + subscriber's queue is full, never the newest decision. Pushing 101 items + into a 100-slot queue must evict the first (oldest) and keep the rest.""" + import asyncio + + events.clear() + sse_queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=events.DEFAULT_QUEUE_SIZE) + + for i in range(events.DEFAULT_QUEUE_SIZE + 1): + events._sse_push(sse_queue, _decision(i)) + + assert sse_queue.qsize() == events.DEFAULT_QUEUE_SIZE + ids = [d["id"] for d in _drain_sse(sse_queue)] + assert ids == list(range(1, events.DEFAULT_QUEUE_SIZE + 1)) + + +def _drain_sse(q) -> list: + import asyncio + + out = [] + while True: + try: + out.append(q.get_nowait()) + except asyncio.QueueEmpty: + return out -- 2.49.1 From e2a2a4a8a113477fd1bf188ced13e113df07ddd1 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 19:02:47 -0400 Subject: [PATCH 28/32] fix(events): guard publish_decision against concurrent subscriber churn --- events.py | 18 ++++++++++------- tests/test_events.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/events.py b/events.py index e2c7b30..7df6619 100644 --- a/events.py +++ b/events.py @@ -60,17 +60,21 @@ def publish_decision(decision: dict[str, Any]) -> None: """ _buffer.append(decision) - # Bridge to async SSE subscribers on their own loops. Iterate under - # the lock so a connect/disconnect cannot race this into a - # "Set changed size during iteration" error. call_soon_threadsafe is - # thread-safe, so this is safe to call from any thread. - for sse_queue, loop in list(_sse_loops.items()): + # Bridge to async SSE subscribers on their own loops. Snapshot under the + # lock so a connect/disconnect cannot race this into a "Set changed size + # during iteration" error. call_soon_threadsafe is thread-safe, so the + # loop body runs outside the lock to avoid blocking the broker on a slow + # subscriber loop. + with _subscribers_lock: + subscribers = list(_sse_loops.items()) + for sse_queue, loop in subscribers: try: loop.call_soon_threadsafe(_sse_push, sse_queue, decision) except RuntimeError: # The loop shut down; drop its dead queue reference. - _sse_subscribers.discard(sse_queue) - _sse_loops.pop(sse_queue, None) + with _subscribers_lock: + _sse_subscribers.discard(sse_queue) + _sse_loops.pop(sse_queue, None) def subscribe_sse( diff --git a/tests/test_events.py b/tests/test_events.py index 6a3cd1e..fa404ce 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -142,3 +142,49 @@ def _drain_sse(q) -> list: out.append(q.get_nowait()) except asyncio.QueueEmpty: return out + + +@pytest.mark.anyio +async def test_concurrent_subscribe_unsubscribe_publish_no_error(): + """Repeated subscribe/unsubscribe concurrent with publish must not race. + + Regression pin for the lock gap in `publish_decision` (review: + resolve-review-findings-sse-lock-gap-review.md): the snapshot of + `_sse_loops` and the `except RuntimeError` mutation must both hold + `_subscribers_lock` like the other three accessors do. + + Honest caveat: this is a regression pin, not a guaranteed reproducer. + Under GIL CPython `list(dict.items())` on identity-hashed keys is a + single C-level op that does not yield the GIL mid-iteration, so the + race is unlikely to fire here even with the bug present. It exists to + exercise the concurrent churn path under the version matrix (including + a future free-threaded build) where the race is genuine. + """ + import asyncio + + events.clear() + stop = threading.Event() + errors: list[BaseException] = [] + + def _publisher(): + try: + for _ in range(2000): + if stop.is_set(): + return + events.publish_decision(_decision(1)) + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + + th = threading.Thread(target=_publisher) + th.start() + + for _ in range(500): + q: asyncio.Queue[dict] = asyncio.Queue() + events.subscribe_sse(q, replay=False) + events.unsubscribe_sse(q) + + stop.set() + th.join(timeout=5) + assert not th.is_alive(), "publisher thread did not finish" + assert errors == [], f"publisher raised: {errors}" + assert events._sse_loops == {} -- 2.49.1 From bdad7336fe5c5c7b58671e4988da29325fb5d572 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 23:14:39 -0400 Subject: [PATCH 29/32] feat(config): bake right-sized num_ctx into Modelfile-tagged model tags Ollama's OpenAI-compatible endpoint (0.22.0) silently ignores per-request num_ctx and keep_alive under every field shape tried, so the context size has to live in the model tag itself. Point classifier/verification at mistral-nemo-router:12b (8192 ctx, 8.6GB vs 14GB untagged) and local_vision at qwen3-vl-router:4b (16384 ctx), so classify+verify share one resident instance and the two-model worst case (15.9GB) leaves real headroom on a 24GB card. Document the live-measured rationale in CLAUDE.md and add inline notes in dispatcher where the limitation surfaces (_classify_once, _run_local_vision). --- CLAUDE.md | 33 +++++++++++++++++++++++++-- config.yaml | 41 ++++++++++++++++++++++++++++++---- dispatcher.py | 19 ++++++++++++++++ tests/test_config_endpoints.py | 2 +- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cbc1c94..3b972ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1057,16 +1057,45 @@ your own plan's quota (it is reported in `/health` as burn against the allowance; it does not gate anything). Ollama must be reachable with the classifier model pulled — the name must -match `classifier.model` in `config.yaml`: +match `classifier.model` in `config.yaml`, which points at a **Modelfile-tagged +variant**, not the base library tag. Ollama loads a model at its library +Modelfile's default context unless told otherwise, and the base tag never +was — measured live on a 24GB card, `mistral-nemo:12b` alone came up at +`num_ctx=32768` and cost 14GB, though `max_input_chars` + the system prompt + +`max_output_tokens` need well under a quarter of that. `num_ctx` cannot be set +per request here: verified live that Ollama's OpenAI-compatible endpoint +(0.22.0) silently accepts and ignores it under every field shape tried +(`options.num_ctx`, top-level `num_ctx`, `context_length`) — a 200 comes back +and the loaded context never changes. Only the native `/api/chat` endpoint +honors it, which the classifier cannot assume (it also serves cloud +providers), so the size has to be baked into the tag itself: ```bash ollama pull mistral-nemo:12b +printf 'FROM mistral-nemo:12b\nPARAMETER num_ctx 8192\n' > Modelfile.router +ollama create mistral-nemo-router:12b -f Modelfile.router ``` +8192 leaves 2x headroom over the real requirement and measured 8.6GB loaded +— vs. 14GB for the untagged model at its default context. `verification.model` +should name the same tag: it defaults to `classifier.model`, and since Ollama +keys a loaded model's context size at load time rather than per call, a +verifier pointed at a *different* num_ctx for the same base model would force +a reload every time a request alternates between classifying and verifying. `local_vision` ships **enabled** as a core feature (see below), so also pull -its model unless you're turning it off: +its model unless you're turning it off. It gets the same tagging treatment — +this path receives the RAW message list rather than a size-bounded one +(`dispatcher.py` calls it *before* pinch pruning), so its context is sized +more generously as a conservative cut rather than a derived minimum: ```bash ollama pull qwen3-vl:4b +printf 'FROM qwen3-vl:4b\nPARAMETER num_ctx 16384\n' > Modelfile.vision +ollama create qwen3-vl-router:4b -f Modelfile.vision ``` +Measured with both tagged models resident at once (the worst case — an +interactive session classifying while a vision-fallback request is also in +flight): 15.9GB used of 24GB, vs. 22.1GB used (1.9GB free, functionally no +headroom left for anything else on the card) with both loaded at their +untagged defaults. It does not have to be on this machine. To use one across a VPN, point `classifier.base_url` and `verification.base_url` at it and apply diff --git a/config.yaml b/config.yaml index 002f427..2733e6b 100644 --- a/config.yaml +++ b/config.yaml @@ -272,7 +272,21 @@ local_vision: enabled: true base_url: "http://localhost:11434/v1" api_key_env: null - model: "qwen3-vl:4b" + # A Modelfile-tagged variant of qwen3-vl:4b, not the base library tag. + # Measured live: the base tag comes up at Ollama's own default num_ctx + # (32768) and costs 9.4GB loaded — resident alongside the classifier's + # pre-fix 14GB, that left 1.9GB free on a 24GB card. num_ctx cannot be set + # per-request here: verified live that Ollama's OpenAI-compatible endpoint + # (0.22.0) silently ignores it under every field shape tried, so it has to + # be baked into the model tag itself: + # printf 'FROM qwen3-vl:4b\nPARAMETER num_ctx 16384\n' > Modelfile.vision + # ollama create qwen3-vl-router:4b -f Modelfile.vision + # 16384 is a conservative cut, not a derived minimum — unlike classifier + # and verification, this path gets the RAW message list (dispatcher.py + # calls it BEFORE pinch pruning), so there is no measured ceiling to size + # tightly against. Tighten once real local_vision payload sizes are + # observed. + model: "qwen3-vl-router:4b" timeout_seconds: 60 max_images: 4 max_image_bytes: 9437184 @@ -298,8 +312,11 @@ verification: # null means "whatever classifier.model is", correct only while both run on # the same local Ollama — config load REFUSES the null once the hosts differ, # because the fallback would name a model this Ollama has never heard of and - # the verifier would fail silently. - model: "mistral-nemo:12b" + # the verifier would fail silently. Stated explicitly here anyway, matching + # classifier.model exactly (see that field's comment) so both calls hit the + # SAME resident Ollama instance, loaded once at its Modelfile-tagged + # context size rather than two separately-sized copies. + model: "mistral-nemo-router:12b" # Only check answers this large. Measured on real traffic: a local check # costs ~15% of a median 193-token answer, so it would only pay if such @@ -354,7 +371,23 @@ classifier: # Unset means unauthenticated, which is the Ollama case. Name the env var # holding the key when the endpoint actually checks one. api_key_env: null - model: "mistral-nemo:12b" # must match a model `ollama list` reports + # A Modelfile-tagged variant of mistral-nemo:12b, not the base library tag + # — must match a model `ollama list` reports. Ollama loads a model at its + # library Modelfile's default context unless told otherwise, and the base + # tag never was: measured live, mistral-nemo:12b came up at num_ctx=32768 + # (14GB of a 24GB card) though max_input_chars + the system prompt + + # max_output_tokens need well under a quarter of that. num_ctx cannot be + # set per-request here: verified live that Ollama's OpenAI-compatible + # endpoint (0.22.0) silently ignores it under every field shape tried (a + # 200 comes back, the loaded context never changes), so it has to be baked + # into the tag itself: + # printf 'FROM mistral-nemo:12b\nPARAMETER num_ctx 8192\n' > Modelfile.router + # ollama create mistral-nemo-router:12b -f Modelfile.router + # 8192 leaves 2x headroom over the real requirement. verification.model + # points at this same tag (see its own comment) so classify and verify + # share one resident instance at one context size, rather than risking a + # reload thrash from two differently-sized copies of the same base model. + model: "mistral-nemo-router:12b" # A cold Ollama took 43s to answer the first classification, which blew the # old 30s ceiling and returned 503 to the client. Warm it is ~5s. The # ceiling is for a cold model load, not the steady state. diff --git a/dispatcher.py b/dispatcher.py index 27664f4..9f743c2 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -534,6 +534,14 @@ def _classifier_user_content( def _classify_once(client: OpenAI, system_prompt: str, user_content: str) -> Classification: """One classifier round-trip. Raises on anything unusable.""" categories = cfg.proficiency.categories + # Context size is NOT settable per-request here: verified live that + # Ollama's OpenAI-compatible endpoint (0.22.0) silently ignores num_ctx, + # under every field shape tried (options.num_ctx, top-level num_ctx, + # context_length) — a 200 comes back and the loaded model's context never + # changes. Only the native /api/chat endpoint honors it, which the + # classifier cannot assume (it also serves cloud providers). The context + # size is instead baked into the Ollama model tag via a Modelfile's + # PARAMETER num_ctx — see README's classifier.model setup. resp = client.chat.completions.create( model=cfg.classifier.model, messages=[ @@ -1846,6 +1854,17 @@ def _run_local_vision(messages: list[dict], lv_cfg) -> Optional[str]: logs.warning("local_vision_skip", reason="missing_api_key") return None headers["Authorization"] = f"Bearer {key}" + # Context size and keep_alive are NOT settable here: verified live that + # Ollama's OpenAI-compatible endpoint (0.22.0) silently ignores both + # options.num_ctx and a top-level keep_alive — a 200 comes back and + # neither the loaded context nor the residency timer changes. Only the + # native /api/chat endpoint honors them (used by verification's local + # check), which this path does not speak. Context size is instead baked + # into the Ollama model tag via a Modelfile's PARAMETER num_ctx — see + # README's local_vision.model setup. Getting a short keep_alive here too + # would mean rewriting this call onto the native API, including its + # image format (a separate `images` field, not OpenAI's `image_url` + # content blocks) — real surgery, left as a follow-up. try: resp = requests.post( url, diff --git a/tests/test_config_endpoints.py b/tests/test_config_endpoints.py index 48ba5f6..8fb3a80 100644 --- a/tests/test_config_endpoints.py +++ b/tests/test_config_endpoints.py @@ -202,7 +202,7 @@ def test_the_shipped_config_loads_with_the_new_keys(raw): loaded = RouterConfig(**raw) assert loaded.routing.require_vision is True assert loaded.routing.require_json_mode is True - assert loaded.local_vision.model == "qwen3-vl:4b" + assert loaded.local_vision.model == "qwen3-vl-router:4b" assert loaded.local_vision.enabled is True -- 2.49.1 From 03f62e2fd02eaa34f75f019718b71bb4a7404e44 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 23:15:44 -0400 Subject: [PATCH 30/32] feat: embedding-based pinch relevance + upstream failover with circuit breaker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent, off-by-default features built from code_plans specs in one pass. Track A — pinch embedding relevance (code_plans/pinch-embedding-relevance.md): - context_prune: pure order_by_relevance (ascending cosine, least-relevant first), shared trim_candidates helper, prune_context(relevance_order=None) param that compresses least-relevant candidates first and stops once the token deficit is covered; None is byte-for-byte today's uniform pass. - config: PinchRelevanceConfig nested under PinchConfig (model, base_url, timeout, min_candidates) + config.yaml relevance block. - dispatcher: _embed_for_relevance (single batched /v1/embeddings call, fails safe to None), _relevance_order_for (min_candidates gate), wired into both chat_completions pinch call sites. Track B — upstream failover + passive circuit breaker (code_plans/upstream-failover-and-circuit-breaker.md): - circuit_breaker: pure module mirroring session_cache.py (is_down/record_failure/ record_success/clear), injected time, exponential backoff capped. - routing: exclude_models hard filter (reason "circuit_open"), same shape as stale. - config: CircuitBreakerConfig + config.yaml block. - dispatcher: _open_circuits exclusion set in route(); non-streaming failover over runners_up on 5xx (does NOT consume attempts_used quality budget); streaming failover opens+checks each candidate before StreamingResponse so a dead replica never reaches the client as a broken stream. Both features ship off-by-default; every failure path reverts to today's behavior. 673 tests pass (up from 562); live /v1/embeddings endpoint verified with nomic-embed-text (distinct 768-dim vectors). --- circuit_breaker.py | 64 +++++ code_plans/pinch-embedding-relevance.md | 225 +++++++++++++++ .../upstream-failover-and-circuit-breaker.md | 231 ++++++++++++++++ code_reviews/magic-brainstorming-review.md | 139 ++++++++++ config.py | 79 ++++++ config.yaml | 28 ++ context_prune.py | 261 +++++++++++++----- dispatcher.py | 184 +++++++++++- routing.py | 6 + tests/test_circuit_breaker.py | 80 ++++++ tests/test_config_endpoints.py | 73 +++++ tests/test_context_prune.py | 204 +++++++++++++- tests/test_dispatcher_helpers.py | 180 ++++++++++++ tests/test_routing.py | 26 ++ 14 files changed, 1705 insertions(+), 75 deletions(-) create mode 100644 circuit_breaker.py create mode 100644 code_plans/pinch-embedding-relevance.md create mode 100644 code_plans/upstream-failover-and-circuit-breaker.md create mode 100644 code_reviews/magic-brainstorming-review.md create mode 100644 tests/test_circuit_breaker.py create mode 100644 tests/test_dispatcher_helpers.py diff --git a/circuit_breaker.py b/circuit_breaker.py new file mode 100644 index 0000000..55dd370 --- /dev/null +++ b/circuit_breaker.py @@ -0,0 +1,64 @@ +"""Passive circuit breaker for upstream model availability. + +Pure, in-memory module mirroring ``session_cache.py``: a module-level dict +whose lifetime matches the dispatcher process, with injected time for +deterministic tests. Never imports dispatcher or config. + +Recovery is passive by design: ``is_down`` only compares against +``down_until``; the next real request that would have picked a down model +becomes the natural recovery probe once the cooldown has passed. +""" + +import time +from dataclasses import dataclass +from typing import Final + +# (model_id, provider) -> CircuitState +_store: Final[dict[tuple[str, str], "CircuitState"]] = {} + + +@dataclass(frozen=True) +class CircuitState: + """A model+provider's open-circuit state.""" + + down_until: float + # What the NEXT failure's cooldown will be (doubled from this one). + cooldown_seconds: float + + +def is_down(model_id: str, provider: str, now: float) -> bool: + """True while ``now`` is before the model's cooldown expiry.""" + state = _store.get((model_id, provider)) + if state is None: + return False + return now < state.down_until + + +def record_failure( + model_id: str, + provider: str, + initial_cooldown: float, + max_cooldown: float, + backoff_multiplier: float, +) -> None: + """Record an availability failure, doubling the cooldown, capped.""" + key = (model_id, provider) + prev = _store.get(key) + if prev is None: + cooldown = initial_cooldown + else: + cooldown = min(prev.cooldown_seconds * backoff_multiplier, max_cooldown) + _store[key] = CircuitState( + down_until=time.time() + cooldown, + cooldown_seconds=cooldown, + ) + + +def record_success(model_id: str, provider: str) -> None: + """Clear an entry on a successful attempt; next failure restarts cooldown.""" + _store.pop((model_id, provider), None) + + +def clear() -> None: + """Drop all circuit state (test isolation / reset helper).""" + _store.clear() diff --git a/code_plans/pinch-embedding-relevance.md b/code_plans/pinch-embedding-relevance.md new file mode 100644 index 0000000..73ce060 --- /dev/null +++ b/code_plans/pinch-embedding-relevance.md @@ -0,0 +1,225 @@ +# Spec: embedding-based relevance scoring for pinch + +**Origin.** `context_prune.py`'s own docstring names this directly: *"Ported +from the MIT-licensed alexrudloff/llmrouter 'pinch' module... Where llmrouter +embeds every candidate message and scores cosine relevance, this module +keeps the same SAFE invariants without requiring an embedding model on the +request path."* The embedding step was cut on purpose, to keep pinch pure +and offline-testable — not because it was a bad idea. This spec is that step, +scoped to fit in without giving up the purity that cut it in the first +place. It also settles the scoping note in +[`magic-brainstorming-review.md`](../code_reviews/magic-brainstorming-review.md) +(idea #3): pinch's mechanism only ever touches old tool results, and this +spec stays inside that boundary rather than widening it. + +No code changes accompany this document — this is the spec opencode builds +from. + +--- + +## The problem, restated + +Today, `prune_context` (`context_prune.py`) decides what to trim by +**position only**. Every tool-result message before `protected_from` (the +cutoff set by `keep_last_turns`) gets compressed — elided to head+tail if +long, replaced with a placeholder if short — regardless of whether that +result has anything to do with what the current turn is actually about. A +tool result from 3 turns ago that's central to the task being finished right +now gets flattened exactly as hard as one from 20 turns ago that's +completely irrelevant, because the only signal used is *how old it is*. + +## What changes and what must never change + +**Changes:** which trim-eligible candidates get compressed. Instead of "all +of them, uniformly," it becomes "the least relevant ones first, stopping +once the token deficit is covered." A relevant-but-old tool result can now +survive untouched even though it's before the `keep_last_turns` cutoff. + +**Never changes — same invariants `context_prune.py` already documents:** +- user/assistant/system messages are still always kept verbatim. The + embedding step never scores or touches them; it only re-orders *which + tool results* get compressed, using the exact same elision/placeholder + mechanism that exists today for compressing them. +- No message is ever removed; order and role pairing are preserved. +- **Fails closed to today's exact behavior.** If relevance scoring is + disabled, unavailable, times out, or errors, pinch falls back to + compressing every trim-eligible candidate uniformly — precisely what it + does today. This is not a degraded mode with reduced functionality; it is + bit-for-bit the current, already-shipped, already-safe behavior. Nothing + about pinch's risk profile gets worse by adding this — at worst, a + request gets today's pinch instead of the smarter one. + +This is also why an embedding model is the right tool and a generative one +isn't (see `magic-brainstorming-review.md`'s idea #1/#2 findings on why +generative rewrites were rejected): an embedding call can fail or time out, +but it cannot *hallucinate a wrong ranking that looks confident* the way a +generative summary can hallucinate wrong content. The failure mode is "no +better than today," not "worse than today." + +## Mechanism + +### Pure core (testable offline, no model dependency) + +A new pure function, next to `prune_context` in `context_prune.py`: + +```python +def order_by_relevance( + query_embedding: list[float], + candidate_embeddings: list[list[float]], +) -> list[int]: + """Indexes into candidate_embeddings, LEAST relevant to query first. + + Cosine similarity, ascending. The caller compresses in this order until + the token deficit is covered, so index 0 is compressed first. + """ +``` + +Trivially unit-testable with hand-built vectors (orthogonal, parallel, +near-duplicate) — no network, no model, same testing shape as +`scoring.normalize_inverted`. + +`prune_context` gains one new optional parameter: + +```python +def prune_context( + messages: list[dict], + budget_tokens: int = ..., + keep_last_turns: int = ..., + max_summarize_chars: int = ..., + relevance_order: Optional[list[int]] = None, # NEW +) -> tuple[list[dict], dict]: +``` + +`relevance_order` indexes into the trim-eligible candidate list (tool +messages before `protected_from`, in the same order `context_prune.py` +already collects them) — same shape `order_by_relevance` returns, so the +dispatcher can pass its result straight through. + +**Cut behavior when `relevance_order` is provided:** walk it in order, +compressing each candidate with the existing elide/placeholder logic +(unchanged), tracking cumulative `tokens_saved` against +`orig_tokens - budget_tokens`. Stop as soon as the deficit is covered — +remaining (more relevant) candidates stay verbatim. If the whole list is +exhausted before the deficit is covered, every candidate has been +compressed exactly as today, so there's no scenario where this compresses +*more* than the current uniform pass. + +**When `relevance_order` is `None`:** compress every candidate, in whatever +order they're encountered — byte-for-byte the current implementation. This +is the fallback path, and it's also what happens today when the feature is +off entirely. + +### Impure edge (dispatcher.py owns it, matching every other model call) + +A new function alongside `_classify_once` / `_run_local_vision` / the local +verification call — all of which already live in `dispatcher.py` per +`context_prune.py`'s own docstring ("This module is pure... dispatcher.py +owns reading the config and deciding when to call it"): + +```python +def _embed_for_relevance(query: str, candidates: list[str], cfg) -> Optional[list[int]]: + """Returns order_by_relevance's result, or None on any failure. + + One batched embeddings call (query + all candidates in a single + request — most embedding APIs, Ollama's /api/embed included, accept a + list input) rather than N round-trips. + """ +``` + +Called only when `cfg.pinch.enabled and cfg.pinch.relevance.enabled` and the +trim-eligible candidate count is `>= cfg.pinch.relevance.min_candidates` +(below that, a network round-trip isn't worth it — ranking 1 candidate is +not a decision). Query text: the same current-turn text already computed +for classification (`_last_user_text(messages)` or equivalent — reuse, don't +recompute). On `requests.RequestException`, timeout, or an unparseable +response: `logs.warning("relevance_unavailable", ...)` and return `None`, +which the pure core already treats as "compress everything," so this can +never be the thing that breaks a request. + +### Config + +```yaml +pinch: + enabled: false + budget_tokens: 50000 + keep_last_turns: 4 + max_summarize_chars: 4000 + relevance: + # Off by default, matching every other new-and-unproven knob in this + # project — and specifically requires pinch.enabled too, since this has + # no effect otherwise. Ship it, watch route_decisions / pinch stats on + # real traffic, then decide the default. + enabled: false + # An EMBEDDING model, not a chat model — this must not point at + # classifier.model or verification.model. Pull one on the same Ollama: + # ollama pull nomic-embed-text + model: "nomic-embed-text" + base_url: "http://localhost:11434/v1" + timeout_seconds: 10 + # Below this many trim-eligible candidates, skip the embedding call + # entirely and fall back to uniform compression — a network round trip + # to rank one candidate decides nothing. + min_candidates: 2 +``` + +`PinchConfig` gains a nested `relevance: PinchRelevanceConfig = +PinchRelevanceConfig()` field, same pattern `RouterConfig` already uses for +`verification`/`local_vision`. `PinchRelevanceConfig(StrictModel)` with a +`field_validator` on `timeout_seconds` and `min_candidates` (both `> 0`), +matching every other positivity validator in `config.py`. + +## Before implementing: verify the endpoint live, don't assume it + +This session already found two cases (`classifier.num_ctx`, +`local_vision.keep_alive`) where a request-level Ollama parameter that +should have worked, per general knowledge of the API, was silently ignored +by this specific Ollama version (0.22.0) over the OpenAI-compatible surface. +`/v1/embeddings` (or native `/api/embed`) is a standard, long-supported +endpoint rather than a vendor-extension field grafted onto chat completions, +so it's a much safer bet — but "should work" was exactly the assumption that +failed twice already this session. First implementation step: `ollama pull +nomic-embed-text` and a direct `curl` against whichever endpoint +`_embed_for_relevance` will use, confirming it returns real, distinct +vectors for distinct inputs, before wiring it into `dispatcher.py`. + +## Interaction with the local-vision gap + +`local_vision`'s message list is never pinch-pruned at all today +(`dispatcher.py:2210` passes the raw `messages`, not `send_messages` — +called out separately in `magic-brainstorming-review.md`'s addendum). This +spec doesn't fix that; it only makes the *existing* pinch call sites +(`chat_completions`'s two pinch invocations) smarter. If local_vision is +later wired through pinch, it gets this relevance scoring for free, since +it's the same `prune_context` function underneath. + +## Testing + +- `order_by_relevance`: hand-built vectors — near-identical, orthogonal, + and opposite — assert the ascending-relevance ordering directly. No + model, no I/O, runs in the existing offline suite. +- `prune_context` with an explicit `relevance_order`: construct a case + where an OLDER tool result is ranked more relevant than a NEWER one still + inside the trim zone, and assert the older one survives verbatim while + the newer, less-relevant one gets compressed — this is the actual new + behavior, and it must be shown to override recency, not just coexist + with it. +- `prune_context` with `relevance_order=None`: assert byte-for-byte + identical output to a call made without the parameter at all — the + regression guard that the fallback path is truly a no-op, not just + "close enough." +- `_embed_for_relevance` failure paths (timeout, non-200, malformed + response): assert `None` is returned and nothing raises, mirroring + `tests/test_config_endpoints.py`'s and verification's own + failure-mode coverage style. + +## Recommendation + +Worth building, and lower-risk than it might look: the pure core is a small, +fully-testable addition to a module that's already designed for exactly +this extension point (the docstring names the missing piece explicitly), +the fallback is provably identical to today's shipped behavior, and the +failure mode of an embedding call (mis-ranking) is categorically safer than +the failure mode of the generative ideas this same brainstorming session +already rejected. Verify the embeddings endpoint live first — that's a +20-minute check, not a design question — then build the pure core and its +tests before touching `dispatcher.py`. diff --git a/code_plans/upstream-failover-and-circuit-breaker.md b/code_plans/upstream-failover-and-circuit-breaker.md new file mode 100644 index 0000000..3623da7 --- /dev/null +++ b/code_plans/upstream-failover-and-circuit-breaker.md @@ -0,0 +1,231 @@ +# Spec: immediate per-request failover + a passive circuit breaker for upstream model outages + +**Origin.** Found live: NeuralWatt returned `503` for `gemma-4-31b` +("All servers for model 'nvidia/Gemma-4-31B-IT-NVFP4' are currently +unavailable... Retry later") on four consecutive requests. Every one of +those requests had already computed a ranked candidate list — `cand=10` in +the route log — and none of the other nine were ever tried. Two related but +separable problems, addressed together because the second only matters once +the first exists: + +1. **No failover within a single request.** When the picked model's upstream + call itself errors (not a bad *answer* — the call never produced one), + the router surfaces the raw error to the client instead of trying the + next-ranked candidate. +2. **No memory across requests.** Even with #1 fixed, every subsequent + request during an outage would still try the dead model first and pay a + wasted round-trip before falling over — for as long as the outage lasts. + +No code changes accompany this document — this is the spec opencode builds +from. + +--- + +## Why these are a different failure class from what `iteration.py` already handles + +`iteration.py`'s retry budget (`attempts_by_tier`) is a **quality** budget: +it exists to pay for a corrective attempt after the model *answered badly* +(`truncated`, `malformed`). An upstream `5xx` means the model never got to +answer at all — this is an **availability** failure, not a quality one, and +it must not be charged against the same budget. A tier-1 interactive +request has a 0-attempt quality budget (`CLAUDE.md`'s own table), which is +correct for "don't pay to fix a bad answer on cheap work" — but it would be +wrong for that to also mean "a tier-1 request gets zero chances to route +around a dead replica." Availability failover needs its own, separate cap. + +Convenient fact found while reading the existing code: `decision.runners_up` +is already capped at 3 (`ranked[1:4]`, `dispatcher.py:838`), so "try every +candidate this request already ranked" is naturally bounded at 4 total +attempts (the primary pick + 3 runners-up) with **no new config knob +required** for the cap itself. + +## Mechanism, part 1: immediate per-request failover + +### Non-streaming path (`dispatcher.py`, the `while True:` loop around line 2415) + +This loop already exists, already tracks `current_model` and an +`alternatives` list built from `decision.runners_up` +(`dispatcher.py:2402-2409`), and already reassigns `current_model = plan.model_id` +on a quality-retry. The only change: today, `resp.status_code >= 400` +(line 2425) immediately does `raise HTTPException(...)` — bypassing the loop +entirely. Instead, treat it as an availability failure inside the same loop: + +```python +if resp.status_code >= 400: + logs.error("upstream", model=current_model, status=resp.status_code, + detail=resp.text[:200], ms=upstream_ms) + circuit_breaker.record_failure(current_model, provider) # part 2 + if not alternatives: + raise HTTPException(resp.status_code, resp.text[:500]) + current_model, _ceiling = alternatives[0] + alternatives = alternatives[1:] + continue # try the next candidate; does NOT consume `attempts_used` +``` + +Critically, this must **not** increment `attempts_used` or check it against +`budget` — that variable is the quality-retry budget from `iteration.py` and +stays reserved for verification failures, per the section above. An +availability failover loop needs its own bound, and reusing +`len(alternatives)` (already ≤3) is sufficient; no new counter needed. + +### Streaming path (`proxy()` generator, `dispatcher.py:2546` onward) + +This is the harder half, and the ordering matters. Today, the generator +opens the upstream connection and checks its status *inside* the streamed +response body's own generator function. That's too late to retry +transparently: once `StreamingResponse(proxy())` is returned from the route +handler, the client has already been sent a `200` and headers — there is no +way to swap in a different upstream after that without the client seeing a +broken stream. + +The fix is to move connection **and status check** for each candidate +*before* `StreamingResponse` is ever constructed, using the fact that +`requests.post(..., stream=True)` returns as soon as headers arrive, without +consuming the body: + +```python +def _open_upstream(model_id: str, ...) -> requests.Response: + """POST with stream=True; caller decides whether to consume or discard.""" + return requests.post(url, headers=headers, json={**upstream_body, "model": model_id}, stream=True, timeout=600) + +candidates = [target] + [c.model_id for c in decision.runners_up] # decision.runners_up already capped at 3 +upstream = None +for model_id in candidates: + attempt = _open_upstream(model_id, ...) + if attempt.status_code < 400: + upstream = attempt + target = model_id + break + logs.error("upstream", model=model_id, status=attempt.status_code, ...) + circuit_breaker.record_failure(model_id, provider) + attempt.close() # release the connection; nothing was ever sent to the client +if upstream is None: + raise HTTPException(attempt.status_code, attempt.text[:500]) +# ... proceed to construct StreamingResponse(proxy_over(upstream)) as today, +# with `proxy()` now taking the already-opened, already-healthy `upstream` +# instead of opening its own. +``` + +The `proxy()` generator keeps its existing defensive status check as a +belt-and-suspenders case (a healthy-looking connection can still fail +mid-stream — that's a much rarer, already-partially-committed situation this +spec doesn't try to solve), but the common case — a replica that's flatly +down, which is what `503 "no healthy replicas"` is — gets caught before the +client ever sees a byte. + +## Mechanism, part 2: passive circuit breaker + +A new module, `circuit_breaker.py`, matching `session_cache.py`'s exact +shape (pure, in-memory, no imports of `dispatcher`/`config`): + +```python +@dataclass(frozen=True) +class CircuitState: + down_until: float # time.time() value; now < down_until means "skip" + cooldown_seconds: float # what the NEXT failure's cooldown will be (doubled from this one) + +def is_down(model_id: str, provider: str, now: float) -> bool: ... +def record_failure(model_id: str, provider: str, initial_cooldown: float, + max_cooldown: float, backoff_multiplier: float) -> None: ... +def record_success(model_id: str, provider: str) -> None: ... # clears the entry entirely +def clear() -> None: ... # test isolation, matching session_cache.clear() +``` + +**Recovery is passive, by design — no pinger.** `is_down` only ever +compares against `down_until`; nothing proactively re-checks a down model. +The next real request that would otherwise have picked that model, once +`down_until` has passed, simply isn't excluded anymore and becomes the +natural recovery probe. If it succeeds, `record_success` clears the entry. +If it fails, `record_failure` runs again with the cooldown doubled (capped +at `max_cooldown_seconds`) — this is deliberately **not** a background +service: an active health-check would spend real billed quota probing a +model nobody is currently asking for, which is the opposite of this +project's own standing rule that a wasted attempt costs energy against a +fixed quota. Demand already provides the probe for free. + +**Where it plugs into routing.** `routing.py`'s `select_candidates` / +`rejection_reason` already hard-filters on `exclude_stale` / +`exclude_deprecated` (`routing.py:117-119`) — this is one more hard filter +of the same shape, not a new mechanism. `dispatcher.py` computes the +excluded set from `circuit_breaker` state once per request (`now = +time.time()`, check every candidate row) and passes it in, keeping +`routing.py` itself free of any import of `circuit_breaker` or `time` — +same separation `metrics.py`'s docstring already insists on to avoid an +import cycle. + +**Where it's fed.** Both failover loops in part 1 call +`circuit_breaker.record_failure(...)` on every upstream `5xx`, and record +`record_success(...)` on the eventual successful attempt for that request — +so the circuit breaker's state comes entirely from real dispatch traffic, +never a separate check. + +### Config + +```yaml +circuit_breaker: + # Off by default, matching every other new-and-unproven knob in this + # project. Unlike most of them, this one has a low-risk failure mode even + # when wrong — see the Recommendation section — so it's a reasonable + # candidate to flip on sooner than most. + enabled: false + initial_cooldown_seconds: 30 + max_cooldown_seconds: 600 + backoff_multiplier: 2.0 +``` + +`CircuitBreakerConfig(StrictModel)` with `field_validator`s requiring +`initial_cooldown_seconds > 0`, `max_cooldown_seconds >= initial_cooldown_seconds`, +and `backoff_multiplier > 1.0` (a multiplier ≤1 would never grow the +cooldown, defeating the point). + +## What this does NOT do + +- Does not retry a request whose upstream call *succeeded* but whose + *answer* was bad — that's `iteration.py`'s job, unchanged. +- Does not add a background health-check service, on purpose (see above). +- Does not change anything about `POST /outcome` or `feedback.py` — an + availability failure is not a proficiency signal about the model's + quality, so it should never be folded into `proficiency` the way a + verification failure is. + +## Testing + +- `circuit_breaker.py`: pure, offline, same shape as `tests/test_session_cache.py` + already tests `session_cache.py` — inject `now` explicitly + rather than relying on real `time.time()` in tests, so cooldown expiry is + deterministic. Cover: first failure sets `initial_cooldown_seconds`; + second consecutive failure doubles it; cooldown never exceeds + `max_cooldown_seconds`; a success clears the entry outright (next failure + after a success starts back at `initial_cooldown_seconds`, not wherever + the backoff had climbed to). +- `routing.py`: a test asserting a circuit-broken model is excluded from + `select_candidates` the same way a stale/deprecated one already is — + extend the existing test shape for those two filters rather than + inventing a new one. +- Non-streaming failover: `monkeypatch.setattr(dispatcher.requests, "post", ...)` + returning a `503` for the first candidate and `200` for the second, + asserting the response actually came from the second model and + `attempts_used`/quality budget was untouched. +- Streaming failover: same idea against `_open_upstream`, asserting the + discarded first connection's `.close()` was called and the client-visible + stream came from the second candidate with a `200` from the very first + byte (i.e., confirming the retry genuinely happened before + `StreamingResponse` was constructed, not after). + +## Recommendation + +Build part 1 (immediate failover) regardless of how part 2 is scoped — it +has essentially no downside: on a provider-wide outage where every candidate +is down, the end result is identical to today (an error, after trying every +candidate instead of one), and on a partial outage like the one that +prompted this, it turns four visible failures into four invisible +successes. This one is a reasonable candidate to ship default-on rather than +behind the project's usual off-by-default caution, precisely because its +worst case matches current behavior rather than introducing a new one. + +Build part 2 alongside it — without it, every request during an outage +still pays one wasted round-trip before failing over, for as long as the +outage lasts. Ship it behind `circuit_breaker.enabled: false` per the +project's standing pattern for new knobs, watch `route_decisions` / +`upstream` log lines for excluded-candidate behavior on real traffic, then +decide the default. diff --git a/code_reviews/magic-brainstorming-review.md b/code_reviews/magic-brainstorming-review.md new file mode 100644 index 0000000..5805f0a --- /dev/null +++ b/code_reviews/magic-brainstorming-review.md @@ -0,0 +1,139 @@ +# Review: opencode's "magic" brainstorm (closed-loop optimizer, local drafting, context-aware pinch) + +**What it was reviewing:** three feature proposals opencode (via the +llm-router `auto` model) generated when asked to brainstorm "high-magic" +next steps for this router — a `quality_monitor.py` closed-loop optimizer, a +local-draft/cloud-refine hybrid ("Local-Cloud Hybrid Synergy"), and a +model-aware upgrade to the disabled `context_prune.py` ("pinch") module. +Not a diff — nothing here had been implemented — so this checks the pitch's +claims against the actual current code (`feedback.py`, `context_prune.py`, +`iteration.py`, `dispatcher.py`) rather than against the pitch's own framing. + +## Verdict: none as pitched. One is nearly free, one has a broken cost +## mechanism, one is real but narrower than claimed. + +### 1. Closed-loop quality monitor — mostly already built; the risky half fights the project's own design + +**Claim:** `feedback.py` is "on-demand," so a new `quality_monitor.py` +background service is needed to detect drift via sliding-window failure +rates and auto-dampen a model's score or flag it for re-eval. + +**Finding.** `feedback.py:79-114` already does the sliding-window +aggregation this proposes: `unapplied_failures` groups `verifications` rows +by `(model, category)`, and `apply_failures` folds them into `proficiency` +via `add_self_eval`. The only missing piece is a schedule — turning it into +a background service is a `llm-router-feedback.timer` unit identical in +shape to the two already shipped (`deploy/llm-router-poller.timer`, +`deploy/llm-router-seed.timer`), not new logic. + +The "detect drift and auto-dampen or flag for re-eval" half is the part +actually being proposed, and it cuts against this project's own history: +three separate harness bugs (CLAUDE.md's "Harness bugs this shook out" — the +shared token budget, kimi's leading-space indentation error, unparseable +judge output) each looked exactly like a quality signal and were only +caught by a human reading per-task detail, never by a threshold. +`add_self_eval`'s running mean is deliberately smoothing for the same +reason. An automated dampening layer on top would fight that design, not +extend it. + +**Recommendation.** Ship the timer — near-zero cost, real value. Treat +"auto-dampen" as "flag for human review," not "silently adjust routing." + +### 2. Local drafting ("Local-Cloud Hybrid Synergy") — the savings mechanism doesn't check out, and it reopens a gap this project already got burned by once + +**Claim:** a local model drafts a skeleton/plan, a cloud model "refines" it, +cutting billed `completion_tokens` — the expensive side of the bill. + +**Finding.** The draft becomes part of the cloud call's *prompt*, not its +completion — "refine this draft" still requires the cloud model to generate +a full final answer as completion tokens, unless doing constrained +diff-editing, which was not proposed and is not a small addition. Given +CLAUDE.md's own measurement that a completion token costs **201x** a prompt +token ("Verification: what local compute is actually good for"), a scheme +that's really "local draft + full prompt + full cloud completion" can +easily cost *more*, not less. The pitch's central cost claim was never +checked against the codebase's own numbers. + +It also reopens open item #5 ("Local energy is not on the ledger") at a +much larger scale — full local generation instead of just classification — +before the metering that would let anyone tell if it's actually cheaper +exists. This project already spent real effort discovering "local is free" +was wrong once (the hosted classifier beat the local one on speed, +accuracy, *and* attributed energy); this proposal reintroduces the same +unverified assumption in a bigger, less reversible form. It also serializes +two model calls on the interactive path, against `iteration.py`'s explicit +stance that every retry/extra hop is a latency cost. + +**Recommendation.** Do not build. If local drafting comes back, item #5 +(real local energy metering) needs to exist first, so the cost claim can be +checked instead of assumed. + +### 3. Context-aware pinch — the most grounded idea, but narrower than pitched + +**Claim:** make `context_prune.py` ("pinch," shipped `enabled: false`) +model-aware — size the prune to the *selected* model's context window, so a +request can trade context precision for routing to a cheaper/smaller model. + +**Finding.** `context_prune.py` and `PinchConfig` are fully built and wired +into both dispatch call sites (`dispatcher.py:2107`, `:2334`, confirmed by +direct read). Making it model-aware is real but non-trivial: today pinch +runs *before* model selection with one static `budget_tokens`, and its +output already feeds the tier/cost decision (`dispatcher.py:2099-2123`) — +so "pinch to fit the chosen model" needs either a provisional-selection → +pinch-to-fit → reselect loop, or moving pinch to after a first-pass +candidate pick. It reuses tested code rather than inventing a new +subsystem, which is the strongest thing in its favor. + +One overclaim in the pitch: pinch only ever trims *tool-result* messages — +user/assistant/system content is always kept verbatim by explicit design +(`context_prune.py:9`). So "massive cost drop by shifting to a cheaper +small-window model" only applies to long agent sessions with tool-call +history. A single huge pasted document in a user message — a common way to +block a cheap model — gets none of this benefit today, and the pitch didn't +note that limit. + +**Recommendation.** If one of the three ships, this is it — but scope it +explicitly to agent/tool-heavy sessions, not "any long conversation," since +that's what the underlying mechanism can actually deliver. + +## Addendum: what this session actually shipped + +Reviewing #2 raised a GPU-capacity question that turned out to be real and +already live, not hypothetical. Measured on this box's Quadro RTX 6000 +(24GB): `mistral-nemo:12b` (classifier) and `qwen3-vl:4b` (`local_vision` +fallback), both loaded at Ollama's untagged default context (32768), +together used 22.1GB — 1.9GB free, one browser tab of GPU use away from +contention. Neither model needed anywhere near that: `max_input_chars` + +`system_prompt` + `max_output_tokens` bound the classifier well under 4k +tokens, and 32768 was never a value anyone chose — it's just what the base +model's library Modelfile defaults to. + +Fixed by baking a right-sized context into a Modelfile-tagged variant of +each model (`mistral-nemo-router:12b` @ 8192, `qwen3-vl-router:4b` @ 16384) +and pointing `classifier.model` / `verification.model` / `local_vision.model` +at the tags instead of the base ones. Verified live: 8.6GB and 6.9GB +resident respectively, 15.9GB combined worst case (both hot at once) versus +22.1GB before — free VRAM went from 1.9GB to 8.1GB in that case, and from +10.8GB to 14.6GB in the common case (classifier alone). Confirmed +empirically along the way, not assumed: Ollama's OpenAI-compatible endpoint +(0.22.0) silently ignores a per-request `num_ctx` or `keep_alive` override +under every field shape tried (`options.num_ctx`, top-level `num_ctx`, +`context_length`, top-level `keep_alive`) — a 200 comes back and nothing +changes. Only the native `/api/chat` endpoint honors either, which is why +the fix has to live in the model tag rather than in a request parameter, +and why a shorter `keep_alive` for the vision fallback specifically was +left as a follow-up (it would need that path rewritten onto the native API, +including its image format). + +This doesn't change the verdict on #2: headroom existing now is not the +same as local generation being cheaper than a cloud completion, which is +still unmeasured. It does mean the box has real spare VRAM again, which is +one less reason to reach for anything drastic to free some. + +## Overall + +None of the three should be built as pitched. #1's real, low-cost half (the +timer) is worth doing now. #2's cost mechanism doesn't hold up against this +project's own numbers and depends on an open item (#5) that isn't done yet +— hold it. #3 is the one worth real design time, once scoped down to what +pinch actually touches. diff --git a/config.py b/config.py index b7a9517..7c6c65f 100644 --- a/config.py +++ b/config.py @@ -301,6 +301,39 @@ class FreshnessConfig(StrictModel): exclude_deprecated: bool +class PinchRelevanceConfig(StrictModel): + """Embedding-model relevance scoring for pinch trimming. + + When enabled (and ``pinch.enabled`` is also true), the dispatcher embeds + the current-turn query with the old tool-result candidates and trims the + least relevant first, so a relevant-but-old result survives. Off by + default; any failure reverts to uniform trimming. This must point at an + EMBEDDING model, never ``classifier.model`` or ``verification.model``. + """ + + enabled: bool = False + # OpenAI-compatible embeddings endpoint on the same local Ollama. + model: str = "nomic-embed-text" + base_url: str = "http://localhost:11434/v1" + timeout_seconds: int = 10 + # Below this many trim-eligible candidates, skip the embedding round-trip. + min_candidates: int = 2 + + @field_validator("timeout_seconds") + @classmethod + def timeout_positive(cls, v: int) -> int: + if v <= 0: + raise ValueError("pinch.relevance.timeout_seconds must be > 0") + return v + + @field_validator("min_candidates") + @classmethod + def min_candidates_positive(cls, v: int) -> int: + if v <= 0: + raise ValueError("pinch.relevance.min_candidates must be > 0") + return v + + class PinchConfig(StrictModel): """Optional relevance-based context pruning (Port of llmrouter's pinch). @@ -317,6 +350,7 @@ class PinchConfig(StrictModel): keep_last_turns: int = 4 # Tool results longer than this many characters are summarized in place. max_summarize_chars: int = 4000 + relevance: PinchRelevanceConfig = PinchRelevanceConfig() @field_validator("budget_tokens") @classmethod @@ -367,6 +401,50 @@ class SessionCacheConfig(StrictModel): return v +class CircuitBreakerConfig(StrictModel): + """Passive circuit breaker for upstream model availability. + + When enabled, a model that returns 5xx is temporarily skipped by routing + (with exponential backoff). Recovery is passive: a real request that would + have picked it becomes the probe once the cooldown passes. Off by default. + """ + + enabled: bool = False + initial_cooldown_seconds: int = 30 + max_cooldown_seconds: int = 600 + backoff_multiplier: float = 2.0 + + @field_validator("initial_cooldown_seconds") + @classmethod + def initial_positive(cls, v: int) -> int: + if v <= 0: + raise ValueError("circuit_breaker.initial_cooldown_seconds must be > 0") + return v + + @field_validator("max_cooldown_seconds") + @classmethod + def max_at_least_initial(cls, v: int) -> int: + if v <= 0: + raise ValueError("circuit_breaker.max_cooldown_seconds must be > 0") + return v + + @model_validator(mode="after") + def max_gte_initial(self) -> "CircuitBreakerConfig": + if self.max_cooldown_seconds < self.initial_cooldown_seconds: + raise ValueError( + "circuit_breaker.max_cooldown_seconds must be >= " + "initial_cooldown_seconds" + ) + return self + + @field_validator("backoff_multiplier") + @classmethod + def multiplier_above_one(cls, v: float) -> float: + if v <= 1.0: + raise ValueError("circuit_breaker.backoff_multiplier must be > 1.0") + return v + + class DatabaseConfig(StrictModel): path: str @@ -435,6 +513,7 @@ class RouterConfig(StrictModel): iteration: IterationConfig = IterationConfig() pinch: PinchConfig = PinchConfig() session_cache: SessionCacheConfig = SessionCacheConfig() + circuit_breaker: CircuitBreakerConfig = CircuitBreakerConfig() freshness: FreshnessConfig database: DatabaseConfig classifier: ClassifierConfig diff --git a/config.yaml b/config.yaml index 2733e6b..c536550 100644 --- a/config.yaml +++ b/config.yaml @@ -175,6 +175,22 @@ pinch: budget_tokens: 50000 keep_last_turns: 4 max_summarize_chars: 4000 + relevance: + # Off by default, matching every other new-and-unproven knob in this + # project — and specifically requires pinch.enabled too, since this has no + # effect otherwise. Ship it, watch route_decisions / pinch stats on real + # traffic, then decide the default. + enabled: false + # An EMBEDDING model, not a chat model — this must not point at + # classifier.model or verification.model. Pull one on the same Ollama: + # ollama pull nomic-embed-text + model: "nomic-embed-text" + base_url: "http://localhost:11434/v1" + timeout_seconds: 10 + # Below this many trim-eligible candidates, skip the embedding call + # entirely and fall back to uniform compression — a network round trip + # to rank one candidate decides nothing. + min_candidates: 2 session_cache: # In-memory per-session classification cache. Remembers the last @@ -190,6 +206,18 @@ session_cache: enabled: false staleness_minutes: 20 +circuit_breaker: + # Passive availability circuit breaker. Off by default, matching every other + # new-and-unproven knob in this project. When enabled, a model that returns + # 5xx is temporarily excluded from routing with exponential backoff; recovery + # is passive (the next real request becomes the probe once the cooldown + # passes). Unlike most new knobs this one has a low-risk failure mode even + # when wrong, so it's a reasonable candidate to flip on sooner. + enabled: false + initial_cooldown_seconds: 30 + max_cooldown_seconds: 600 + backoff_multiplier: 2.0 + routing: # Access gating is prose-only in the NeuralWatt catalog ("Private preview # (grant-gated)", "(Canary)"), so the poller parses it into access_level and diff --git a/context_prune.py b/context_prune.py index 04a947d..a41506c 100644 --- a/context_prune.py +++ b/context_prune.py @@ -25,12 +25,45 @@ This module is pure: it takes messages and limits and returns pruned messages. from __future__ import annotations +from typing import Optional + from config import PinchConfig + # Mirrors dispatcher.CHARS_PER_TOKEN. CHARS_PER_TOKEN = 3 +def order_by_relevance( + query_embedding: list[float], + candidate_embeddings: list[list[float]], +) -> list[int]: + """Indexes into ``candidate_embeddings``, LEAST relevant to query first. + + Cosine similarity in ascending order. The caller compresses in this order + until the token deficit is covered, so index 0 is compressed first. Empty + or single-candidate inputs are safe: they return ``[]`` or ``[0]`` without + any call to an embedding model. + """ + if not candidate_embeddings: + return [] + + def _cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + norm_a = sum(x * x for x in a) ** 0.5 + norm_b = sum(y * y for y in b) ** 0.5 + if norm_a == 0.0 or norm_b == 0.0: + return 0.0 + return dot / (norm_a * norm_b) + + scored = [ + (_cosine(query_embedding, candidate), i) + for i, candidate in enumerate(candidate_embeddings) + ] + scored.sort(key=lambda pair: pair[0]) + return [i for _, i in scored] + + def estimate_tokens(text: str | None) -> int: """Crude characters-per-token estimate, consistent with the dispatcher.""" return len(text) // CHARS_PER_TOKEN if text else 0 @@ -160,11 +193,55 @@ def _with_text(message: dict, new_text: str) -> dict: return {**message, "content": new_text} +def trim_candidates( + messages: list[dict], keep_last_turns: int +) -> tuple[list[int], int, Optional[int]]: + """Indexes of trim-eligible tool messages, plus the protected_from cutoff. + + Returns ``(candidate_indexes, protected_from, protected_from_user_index)``. + ``candidate_indexes`` are the message indexes of every ``tool`` message at + ``i < protected_from``, in ascending message order. ``protected_from`` is + the exclusive cutoff below which trimming is allowed. The third value is + the protected user-turn index that defined the cutoff (``None`` when the + cutoff came from the no-user-turn / empty-protection path). + + Both ``prune_context`` and the dispatcher's relevance path call this, so + the candidate set the embedding order indexes into is byte-for-byte the + set ``prune_context`` will trim. + """ + user_indexes = _first_user_turn_indexes(messages) + if not user_indexes: + # Treat the conversation as a single ongoing turn and protect the + # trailing ``keep_last_turns`` tool results. + tool_indexes = [i for i, m in enumerate(messages) if m.get("role") == "tool"] + protected_from = ( + tool_indexes[max(len(tool_indexes) - keep_last_turns, 0)] + if tool_indexes + else len(messages) + ) + protected_user_index: Optional[int] = None + else: + num_protected_turns = min(keep_last_turns, len(user_indexes)) + if num_protected_turns <= 0: + protected_from = len(messages) + protected_user_index = None + else: + protected_from = user_indexes[len(user_indexes) - num_protected_turns] + protected_user_index = protected_from + + candidates = [ + i for i, m in enumerate(messages) + if i < protected_from and m.get("role") == "tool" + ] + return candidates, protected_from, protected_user_index + + def prune_context( messages: list[dict], budget_tokens: int = PinchConfig.model_fields["budget_tokens"].default, keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default, max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default, + relevance_order: Optional[list[int]] = None, ) -> tuple[list[dict], dict]: """Trim old tool results once a conversation exceeds ``budget_tokens``. @@ -174,6 +251,12 @@ def prune_context( equals ``len(messages)``. Order and role pairing are preserved, so the result is a valid conversation with the same shape. + ``relevance_order`` indexes the trim-eligible candidates (from + :func:`trim_candidates`) from least to most relevant. When provided, that + order is compressed first and stopping once the token deficit is covered; + remaining (more relevant) candidates stay verbatim. When ``None``, every + candidate is compressed uniformly — byte-for-byte the historical behavior. + Only runs (and only mutates anything) when the estimate actually exceeds the budget; otherwise the original list is returned untouched. """ @@ -186,78 +269,122 @@ def prune_context( "tokens_saved": 0, } - user_indexes = _first_user_turn_indexes(messages) - if not user_indexes: - # No user turn at all: treat the conversation as a single ongoing turn - # and protect the trailing ``keep_last_turns`` tool results (the ones - # the next turn needs). ``keep_last_turns`` bounds how many are - # protected; a tiny conversation protects all of them. - tool_indexes = [i for i, m in enumerate(messages) if m.get("role") == "tool"] - protected_from = ( - tool_indexes[max(len(tool_indexes) - keep_last_turns, 0)] - if tool_indexes - else len(messages) - ) - else: - num_protected_turns = min(keep_last_turns, len(user_indexes)) - if num_protected_turns <= 0: - # Nothing to protect: everything before is a candidate for trimming. - protected_from = len(messages) - else: - protected_from = user_indexes[len(user_indexes) - num_protected_turns] + candidates, protected_from, _ = trim_candidates(messages, keep_last_turns) + candidate_set = set(candidates) + # Relevance path: only the ordered subset is compressed, least-relevant + # first, stopping once the deficit is covered. Uniform path (None): + # compress every candidate unconditionally — byte-for-byte unchanged. + if relevance_order is None: + trim_indexes = None + else: + trim_indexes = [candidates[j] for j in relevance_order] + trim_indexes = [i for i in trim_indexes if i in candidate_set] + + target_save = orig_tokens - budget_tokens pruned: list[dict] = [] summarized = 0 - for i, msg in enumerate(messages): - role = msg.get("role") - if role in ("user", "assistant", "system") or i >= protected_from: - pruned.append(msg) - continue - # Only tool results are candidates here. - if role != "tool": - pruned.append(msg) - continue - content = msg.get("content") - if isinstance(content, str): - text = content - prose = content - else: - # `text` sizes the message (includes image bytes, so an image-heavy - # result still triggers pruning and still affects the length guard); - # `prose` shapes the replacement (never raw base64). - text = extract_text(msg) - prose = _text_only(msg) - head_len = 1500 - tail_len = 1500 - replaced = False - if len(text) > max_summarize_chars: - # Long tool result: keep head + tail so the shape survives, but - # only when the elision actually saves characters. A result that is - # not meaningfully longer than head+tail+marker would only grow (or - # produce a negative trim), so fall through to the placeholder path. - head = prose[:head_len] - tail = prose[-tail_len:] - trimmed = len(prose) - head_len - tail_len - marker = f"\n\n[{trimmed:,} chars trimmed...]\n\n" if trimmed > 0 else "" - elided = f"{head}{marker}{tail}" - if trimmed > 0 and len(elided) < len(prose): - summarized += 1 - pruned.append(_with_text(msg, elided)) - replaced = True - if not replaced: - # Short result (or a long one that cannot be elided to save space): - # replace with a short placeholder, but only when it is strictly - # shorter than the *combined* size — an image-only result may have - # empty prose yet large real size, and shrinking it still saves - # tokens (the image payload is stubbed by _with_text). - name = _tool_name(msg) or "tool" - placeholder = f"[{name}: result omitted]" - if len(placeholder) < len(text): - summarized += 1 - pruned.append(_with_text(msg, placeholder)) - else: + if trim_indexes is None: + # Historical uniform pass: compress every trim-eligible candidate. + for i, msg in enumerate(messages): + role = msg.get("role") + if role in ("user", "assistant", "system") or i >= protected_from: pruned.append(msg) + continue + if role != "tool": + pruned.append(msg) + continue + content = msg.get("content") + if isinstance(content, str): + text = content + prose = content + else: + text = extract_text(msg) + prose = _text_only(msg) + head_len = 1500 + tail_len = 1500 + replaced = False + if len(text) > max_summarize_chars: + head = prose[:head_len] + tail = prose[-tail_len:] + trimmed = len(prose) - head_len - tail_len + marker = f"\n\n[{trimmed:,} chars trimmed...]\n\n" if trimmed > 0 else "" + elided = f"{head}{marker}{tail}" + if trimmed > 0 and len(elided) < len(prose): + summarized += 1 + pruned.append(_with_text(msg, elided)) + replaced = True + if not replaced: + name = _tool_name(msg) or "tool" + placeholder = f"[{name}: result omitted]" + if len(placeholder) < len(text): + summarized += 1 + pruned.append(_with_text(msg, placeholder)) + else: + pruned.append(msg) + else: + # Relevance path: walk the relevance order (least relevant first) and + # decide which candidates to compress, stopping once the deficit is + # covered. Then assemble the output in message order, replacing the + # chosen candidates and leaving everything else verbatim. + to_compress: list[int] = [] + saved_so_far = 0 + for i in trim_indexes: + if saved_so_far >= target_save: + break + msg = messages[i] + content = msg.get("content") + if isinstance(content, str): + text = content + else: + text = extract_text(msg) + # Only an actually-shrinking candidate contributes savings; a + # placeholder-less tiny result would not lower the deficit. + if len(text) <= max_summarize_chars: + name = _tool_name(msg) or "tool" + placeholder = f"[{name}: result omitted]" + saved = len(text) - len(placeholder) + else: + head_len = 1500 + tail_len = 1500 + trimmed = len(text) - head_len - tail_len + saved = max(0, trimmed) + to_compress.append(i) + saved_so_far += saved + compress_set = set(to_compress) + for i, msg in enumerate(messages): + if i not in compress_set: + pruned.append(msg) + continue + content = msg.get("content") + if isinstance(content, str): + text = content + prose = content + else: + text = extract_text(msg) + prose = _text_only(msg) + head_len = 1500 + tail_len = 1500 + replaced = False + if len(text) > max_summarize_chars: + head = prose[:head_len] + tail = prose[-tail_len:] + trimmed = len(prose) - head_len - tail_len + marker = f"\n\n[{trimmed:,} chars trimmed...]\n\n" if trimmed > 0 else "" + elided = f"{head}{marker}{tail}" + if trimmed > 0 and len(elided) < len(prose): + summarized += 1 + pruned.append(_with_text(msg, elided)) + replaced = True + if not replaced: + name = _tool_name(msg) or "tool" + placeholder = f"[{name}: result omitted]" + if len(placeholder) < len(text): + summarized += 1 + pruned.append(_with_text(msg, placeholder)) + else: + pruned.append(msg) final_tokens = sum(estimate_tokens(extract_text(m)) for m in pruned) return pruned, { diff --git a/dispatcher.py b/dispatcher.py index 9f743c2..35f6618 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -56,9 +56,16 @@ from pydantic import BaseModel, Field import logs from capabilities import detect_capabilities, iter_image_url_values from config import FlexPreference, RouterConfig, load_config -from context_prune import extract_text, prune_context +from context_prune import ( + extract_text, + order_by_relevance, + prune_context, + trim_candidates, + _text_only, +) import events import session_cache +import circuit_breaker from routing import ( BATCH, INTERACTIVE, @@ -762,6 +769,7 @@ def route(req: TaskRequest) -> RouteResponse: allowed_access_levels=cfg.routing.allowed_access_levels, exclude_stale=cfg.freshness.exclude_stale, exclude_deprecated=cfg.freshness.exclude_deprecated, + exclude_models=_open_circuits(rows, cfg), min_tool_proficiency=( cfg.routing.min_tool_proficiency if req.tools_present else None ), @@ -1753,6 +1761,119 @@ def _last_user_text(messages: list[dict]) -> str: return "" +def _embed_for_relevance( + query: str, + candidates: list[str], + cfg, +) -> Optional[list[int]]: + """Rank candidates by embedding relevance to ``query``, or None on failure. + + One batched embeddings call (query + all candidates in a single request) + against the OpenAI-compatible Ollama endpoint. Returns + ``order_by_relevance``'s result (LEAST relevant first). On any failure — + RequestException, timeout, non-200, unparseable body, or missing/malformed + vectors — it logs ``relevance_unavailable`` and returns ``None``, which the + pure core already treats as "compress everything" (today's behavior). This + must never be the thing that breaks a request. + """ + rel = cfg.pinch.relevance + url = f"{rel.base_url.rstrip('/')}/embeddings" + headers: dict[str, str] = {} + try: + resp = requests.post( + url, + headers=headers, + json={"model": rel.model, "input": [query] + candidates}, + timeout=rel.timeout_seconds, + ) + except requests.RequestException as e: + logs.warning( + "relevance_unavailable", reason="request", error=type(e).__name__ + ) + return None + if resp.status_code != 200: + logs.warning( + "relevance_unavailable", reason="status", status=resp.status_code + ) + return None + try: + payload = resp.json() + except ValueError: + logs.warning("relevance_unavailable", reason="unparseable") + return None + data = payload.get("data") + if not isinstance(data, list) or len(data) < 2: + logs.warning("relevance_unavailable", reason="malformed") + return None + embeddings = [] + for entry in data: + vector = entry.get("embedding") + if not isinstance(vector, list) or not vector: + logs.warning("relevance_unavailable", reason="malformed") + return None + embeddings.append([float(x) for x in vector]) + # First is the query, the rest are the candidates. + return order_by_relevance(embeddings[0], embeddings[1:]) + + +def _relevance_order_for(messages: list[dict], cfg) -> Optional[list[int]]: + """Compute the relevance_order for prune_context, or None (uniform). + + When pinch.relevance is off, or candidate count is below min_candidates, + returns None so prune_context uses today's uniform pass. Otherwise makes + one batched embedding call and returns the least-relevant-first index list. + """ + rel = cfg.pinch.relevance + if not (cfg.pinch.enabled and rel.enabled): + return None + candidates, _, _ = trim_candidates(messages, cfg.pinch.keep_last_turns) + if len(candidates) < rel.min_candidates: + return None + query = _last_user_text(messages) + candidate_texts = [_text_only(messages[i]) for i in candidates] + return _embed_for_relevance(query, candidate_texts, cfg) + + +def _open_circuits(rows: list[dict], cfg) -> set[str]: + """Model ids currently skipped by the circuit breaker, or empty set. + + Only consulted when ``cfg.circuit_breaker.enabled``; otherwise the set is + empty so routing is unaffected. Recovery is passive: ``is_down`` just + compares against ``down_until``, and the next request once a cooldown has + passed becomes the probe that can clear the entry via ``record_success``. + """ + if not cfg.circuit_breaker.enabled: + return set() + now = time.time() + return { + row["model_id"] + for row in rows + if circuit_breaker.is_down(row.get("model_id", ""), row.get("provider", ""), now) + } + + +def _open_upstream( + model_id: str, + url: str, + headers: dict[str, str], + upstream_body: dict, + timeout: int = 600, +): + """POST with stream=True and return the Response for the caller to consume. + + The caller decides whether to consume the body or discard+close it based on + ``status_code``, so connection-opening and the status check that gates retry + happen before ``StreamingResponse`` is ever constructed. + """ + return requests.post( + url, + headers=headers, + json={**upstream_body, "model": model_id}, + stream=True, + timeout=timeout, + ) + + def _previous_context(messages: list[dict]) -> str: """The preceding assistant turn of the last user message. @@ -2129,6 +2250,7 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): budget_tokens=cfg.pinch.budget_tokens, keep_last_turns=cfg.pinch.keep_last_turns, max_summarize_chars=cfg.pinch.max_summarize_chars, + relevance_order=_relevance_order_for(messages, cfg), ) logs.debug( "pinch", @@ -2360,6 +2482,7 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): budget_tokens=cfg.pinch.budget_tokens, keep_last_turns=cfg.pinch.keep_last_turns, max_summarize_chars=cfg.pinch.max_summarize_chars, + relevance_order=_relevance_order_for(messages, cfg), ) logs.debug( "pinch", @@ -2430,7 +2553,22 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): detail=resp.text[:200], ms=upstream_ms, ) - raise HTTPException(resp.status_code, resp.text[:500]) + if cfg.circuit_breaker.enabled: + circuit_breaker.record_failure( + current_model, + provider, + cfg.circuit_breaker.initial_cooldown_seconds, + cfg.circuit_breaker.max_cooldown_seconds, + cfg.circuit_breaker.backoff_multiplier, + ) + if not alternatives: + raise HTTPException(resp.status_code, resp.text[:500]) + # Availability failover: try the next-ranked candidate. This + # deliberately does NOT consume `attempts_used` or check the + # quality budget — that is reserved for verification failures. + current_model, _ceiling = alternatives[0] + alternatives = alternatives[1:] + continue payload = resp.json() usage = payload.get("usage") or {} @@ -2543,7 +2681,7 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): # the path all agent traffic takes. slog = logs.bind() - def proxy(): + def proxy(upstream=None): stream_started = time.perf_counter() collected: dict[str, dict] = {} usage: dict = {} @@ -2554,9 +2692,6 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): finish_reason: Optional[str] = None stream_request_id: Optional[str] = None stream_tool_calls = False - upstream = requests.post( - url, headers=headers, json=upstream_body, stream=True, timeout=600 - ) if upstream.status_code >= 400: detail = upstream.text[:500] slog.error( @@ -2664,7 +2799,42 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): stream_request_id, ) - return StreamingResponse(proxy(), media_type="text/event-stream") + # Streaming failover: open the connection and check status for each + # candidate BEFORE building StreamingResponse. Once the generator returns, + # the client already has a 200 + headers, and swapping upstreams then would + # surface as a broken stream. A replica that's flatly down (503) is caught + # here, before the client ever sees a byte. + stream_candidates = [target] + if wants_routing and decision.runners_up: + stream_candidates += [c.model_id for c in decision.runners_up] + upstream = None + upstream_status = 500 + upstream_text = "" + for candidate in stream_candidates: + attempt = _open_upstream(candidate, url, headers, upstream_body) + if attempt.status_code < 400: + upstream = attempt + target = candidate + break + if cfg.circuit_breaker.enabled: + circuit_breaker.record_failure( + candidate, + provider, + cfg.circuit_breaker.initial_cooldown_seconds, + cfg.circuit_breaker.max_cooldown_seconds, + cfg.circuit_breaker.backoff_multiplier, + ) + logs.error( + "upstream", model=candidate, status=attempt.status_code, + detail=attempt.text[:200], stream=True, + ) + upstream_status = attempt.status_code + upstream_text = attempt.text[:500] + attempt.close() + if upstream is None: + raise HTTPException(upstream_status, upstream_text) + + return StreamingResponse(proxy(upstream), media_type="text/event-stream") @app.post("/dispatch", response_model=DispatchResponse) diff --git a/routing.py b/routing.py index 02295d9..fc2f406 100644 --- a/routing.py +++ b/routing.py @@ -68,6 +68,7 @@ def rejection_reason( allowed_access_levels: Sequence[str], exclude_stale: bool, exclude_deprecated: bool, + exclude_models: set[str] = frozenset(), min_tool_proficiency: float | None = None, require_vision: bool = False, require_json_mode: bool = False, @@ -119,6 +120,9 @@ def rejection_reason( if exclude_deprecated and (availability == "deprecated" or row.get("deprecated")): return "deprecated" + if row.get("model_id") in exclude_models: + return "circuit_open" + access_level = row.get("access_level", "public") if access_level not in allowed_access_levels: return f"access_level({access_level})" @@ -306,6 +310,7 @@ def select_candidates( allowed_access_levels: Sequence[str], exclude_stale: bool, exclude_deprecated: bool, + exclude_models: set[str] = frozenset(), min_tool_proficiency: float | None = None, require_vision: bool = False, require_json_mode: bool = False, @@ -322,6 +327,7 @@ def select_candidates( allowed_access_levels=allowed_access_levels, exclude_stale=exclude_stale, exclude_deprecated=exclude_deprecated, + exclude_models=exclude_models, min_tool_proficiency=min_tool_proficiency, require_vision=require_vision, require_json_mode=require_json_mode, diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py new file mode 100644 index 0000000..513e4f0 --- /dev/null +++ b/tests/test_circuit_breaker.py @@ -0,0 +1,80 @@ +"""Unit tests for the passive circuit breaker (circuit_breaker.py). + +Offline, no network. Drive the pure get/record/clear logic directly. time.time +is monkeypatched so cooldown expiry is deterministic (no wall-clock dependence). +""" + +from __future__ import annotations + +import pytest + +import circuit_breaker + + +@pytest.fixture(autouse=True) +def _clean_circuit(): + circuit_breaker.clear() + yield + circuit_breaker.clear() + + +def _freeze(monkeypatch, t: float): + monkeypatch.setattr(circuit_breaker.time, "time", lambda: t) + + +def test_is_down_miss_returns_false(): + assert circuit_breaker.is_down("m1", "p1", now=1000.0) is False + + +def test_first_failure_sets_initial_cooldown(monkeypatch): + _freeze(monkeypatch, 1000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + # Within the cooldown window it's down. + assert circuit_breaker.is_down("m1", "p1", now=1000.0) is True + # Just past the window it is not down (passive recovery probe). + assert circuit_breaker.is_down("m1", "p1", now=1000.0 + 30.0) is False + + +def test_second_consecutive_failure_doubles_cooldown(monkeypatch): + _freeze(monkeypatch, 1000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + _freeze(monkeypatch, 1000.0 + 100.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + # Second failure at t=1100: cooldown doubled to 60, down_until = 1160. + assert circuit_breaker.is_down("m1", "p1", now=1159.0) is True + assert circuit_breaker.is_down("m1", "p1", now=1160.0) is False + + +def test_cooldown_never_exceeds_max(monkeypatch): + _freeze(monkeypatch, 1000.0) + circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> 10 + _freeze(monkeypatch, 2000.0) + circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> 20 + _freeze(monkeypatch, 3000.0) + circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> capped 25 + _freeze(monkeypatch, 4000.0) + circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> capped 25 + assert circuit_breaker.is_down("m1", "p1", now=4024.0) is True + assert circuit_breaker.is_down("m1", "p1", now=4025.0) is False + + +def test_success_clears_and_next_failure_restarts_at_initial(monkeypatch): + _freeze(monkeypatch, 1000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + _freeze(monkeypatch, 2000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) # -> 60 + circuit_breaker.record_success("m1", "p1") + assert circuit_breaker.is_down("m1", "p1", now=2000.0) is False + _freeze(monkeypatch, 3000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + # Restarts at the INITIAL cooldown, not wherever backoff had climbed. + assert circuit_breaker.is_down("m1", "p1", now=3029.0) is True + assert circuit_breaker.is_down("m1", "p1", now=3030.0) is False + + +def test_state_is_keyed_by_model_and_provider(monkeypatch): + _freeze(monkeypatch, 1000.0) + circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) + # Same model, different provider is unaffected. + assert circuit_breaker.is_down("m1", "p2", now=1000.0) is False + assert circuit_breaker.is_down("m1", "p1", now=1000.0) is True diff --git a/tests/test_config_endpoints.py b/tests/test_config_endpoints.py index 8fb3a80..f8059ec 100644 --- a/tests/test_config_endpoints.py +++ b/tests/test_config_endpoints.py @@ -288,3 +288,76 @@ def test_pinch_max_summarize_chars_accepts_default(raw): loaded = RouterConfig(**raw) assert loaded.pinch.max_summarize_chars == 4000 + +def test_pinch_relevance_defaults_load(raw): + loaded = RouterConfig(**raw) + assert loaded.pinch.relevance.enabled is False + assert loaded.pinch.relevance.model == "nomic-embed-text" + assert loaded.pinch.relevance.min_candidates == 2 + + +def test_pinch_relevance_defaults_when_pinch_absent(raw): + cfg = copy.deepcopy(raw) + cfg.pop("pinch") + loaded = RouterConfig(**cfg) + assert loaded.pinch.relevance.enabled is False + assert loaded.pinch.relevance.timeout_seconds == 10 + + +def test_nonpositive_pinch_relevance_timeout_is_rejected(raw): + cfg = copy.deepcopy(raw) + cfg["pinch"]["relevance"]["timeout_seconds"] = 0 + with pytest.raises(ValueError, match="timeout_seconds"): + RouterConfig(**cfg) + + +def test_nonpositive_pinch_relevance_min_candidates_is_rejected(raw): + cfg = copy.deepcopy(raw) + cfg["pinch"]["relevance"]["min_candidates"] = 0 + with pytest.raises(ValueError, match="min_candidates"): + RouterConfig(**cfg) + + +def test_circuit_breaker_defaults_load(raw): + loaded = RouterConfig(**raw) + assert loaded.circuit_breaker.enabled is False + assert loaded.circuit_breaker.initial_cooldown_seconds == 30 + assert loaded.circuit_breaker.max_cooldown_seconds == 600 + assert loaded.circuit_breaker.backoff_multiplier == 2.0 + + +def test_nonpositive_circuit_breaker_initial_cooldown_is_rejected(raw): + cfg = copy.deepcopy(raw) + cfg["circuit_breaker"]["initial_cooldown_seconds"] = 0 + with pytest.raises(ValueError, match="initial_cooldown_seconds"): + RouterConfig(**cfg) + + +def test_circuit_breaker_max_below_initial_is_rejected(raw): + cfg = copy.deepcopy(raw) + cfg["circuit_breaker"]["initial_cooldown_seconds"] = 300 + cfg["circuit_breaker"]["max_cooldown_seconds"] = 200 + with pytest.raises(ValueError, match="max_cooldown_seconds"): + RouterConfig(**cfg) + + +def test_circuit_breaker_multiplier_at_or_below_one_is_rejected(raw): + cfg = copy.deepcopy(raw) + cfg["circuit_breaker"]["backoff_multiplier"] = 1.0 + with pytest.raises(ValueError, match="backoff_multiplier"): + RouterConfig(**cfg) + + +def test_the_shipped_config_points_classifier_and_verifier_at_the_router_tag(raw): + # Context size is not a config-level knob: Ollama's OpenAI-compatible + # endpoint (0.22.0, verified live) silently ignores num_ctx as a + # per-request field, so it has to be baked into the Ollama model tag via + # a Modelfile instead (see classifier.model's comment in config.yaml). + # What config CAN still assert is that classifier and verification point + # at the same tagged model, so they share one resident instance rather + # than two differently-sized copies of the same base model. + loaded = RouterConfig(**raw) + assert loaded.classifier.model == "mistral-nemo-router:12b" + assert loaded.verification.model == loaded.classifier.model + assert loaded.local_vision.model == "qwen3-vl-router:4b" + diff --git a/tests/test_context_prune.py b/tests/test_context_prune.py index 08dddd0..9fe6c07 100644 --- a/tests/test_context_prune.py +++ b/tests/test_context_prune.py @@ -8,7 +8,13 @@ survives so the result still parses as a conversation. from __future__ import annotations -from context_prune import estimate_tokens, extract_text, prune_context +from context_prune import ( + estimate_tokens, + extract_text, + order_by_relevance, + prune_context, + trim_candidates, +) from dispatcher import estimate_prompt_tokens @@ -383,3 +389,199 @@ def test_pinch_disabled_leaves_the_measured_size_unchanged(): ] # unpinned: identical to calling estimate_prompt_tokens(messages) directly assert estimate_prompt_tokens(messages) == estimate_prompt_tokens(list(messages)) + + +def test_order_by_relevance_near_identical_sorts_last(): + # A candidate nearly identical to the query is the MOST relevant, so it + # must sort LAST (compressed last). + query = [1.0, 0.0, 0.0] + candidates = [ + [-1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.9999, 0.0, 0.0], + ] + order = order_by_relevance(query, candidates) + assert order == [0, 1, 2] or order[-1] == 2 + + +def test_order_by_relevance_opposite_sorts_first(): + query = [1.0, 0.0, 0.0] + candidates = [ + [0.0, 1.0, 0.0], + [-1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + ] + order = order_by_relevance(query, candidates) + # The least-relevant (opposite, dot<0) is index 1, which must come first. + assert order[0] == 1 + + +def test_order_by_relevance_ascending_cosine(): + query = [1.0, 1.0] + # cosine to query: (q itself) 1.0, (0.5,0.5) ~1.0, (1,-1) 0.0, (-1,1) 0.0. + candidates = [ + [1.0, -1.0], + [0.5, 0.5], + [0.0, 1.0], + ] + order = order_by_relevance(query, candidates) + plain = [candidates[i] for i in order] + assert plain[0] == [1.0, -1.0] + + +def test_order_by_relevance_empty_and_single(): + assert order_by_relevance([1.0, 0.0], []) == [] + assert order_by_relevance([1.0, 0.0], [[1.0, 0.0]]) == [0] + + +def test_order_by_relevance_zero_norm_candidate_is_least_relevant(): + # A zero-norm candidate (no signal) gets similarity 0.0, so it sorts early + # (least relevant) without dividing by zero. + query = [1.0, 0.0] + candidates = [ + [0.0, 0.0], + [1.0, 0.0], + ] + order = order_by_relevance(query, candidates) + assert order == [0, 1] + + +def test_trim_candidates_normal_multi_turn(): + messages = [ + _user("q1"), + _tool("read", "old"), + _assistant("a1"), + _user("q2"), + _tool("read", "recent"), + _assistant("a2"), + ] + candidates, protected_from, protected_user = trim_candidates(messages, keep_last_turns=1) + # keep_last_turns=1 protects the last user turn (index 3) and everything + # at/after it. Tool message 1 (index 1) is the only trim candidate. + assert candidates == [1] + assert protected_from == 3 + assert protected_user == 3 + + +def test_trim_candidates_no_user_turn(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "call", "tool_calls": [{"id": "t1"}]}, + {"role": "tool", "name": "read", "content": "x" * 20000}, + {"role": "tool", "name": "read", "content": "y" * 20000}, + ] + candidates, protected_from, protected_user = trim_candidates(messages, keep_last_turns=1) + # No user turn: protect the trailing keep_last_turns tool results, so the + # first tool (index 2) is a candidate, the last is protected. + assert 2 in candidates + assert len(messages) - 1 not in candidates + assert protected_user is None + + +def test_trim_candidates_keep_last_turns_larger_than_turns(): + messages = [ + _user("q1"), + _tool("read", "old"), + _assistant("a1"), + ] + candidates, protected_from, _ = trim_candidates(messages, keep_last_turns=10) + assert candidates == [] + assert protected_from == 0 + + +def test_trim_candidates_zero_keep_last_turns(): + messages = [ + _user("q1"), + _tool("read", "old"), + _assistant("a1"), + ] + candidates, protected_from, _ = trim_candidates(messages, keep_last_turns=0) + # Nothing protected: both the tool message and (by position) everything + # before protected_from=len is a candidate zone; only tool candidates count. + assert candidates == [1] + assert protected_from == len(messages) + + +def test_relevance_order_overrides_recency(): + # Two old candidates; the OLDER one (index 1) is more relevant, so it is + # ranked LAST in relevance_order and must survive verbatim, while the NEWER + # (index 3) is less relevant, ranked FIRST, and gets compressed. + messages = [ + _user("q0"), + _tool("read", "old-but-relevant " * 4000), + _assistant("a1"), + _tool("read", "newer-but-irrelevant " * 4000), + _assistant("a2"), + _user("final"), + _assistant("ok"), + ] + # trim_candidates with keep_last_turns=1 protects the "final" turn; both + # tool messages (index 1 and 3) are candidates in clock order [1, 3]. + candidates, protected_from, _ = trim_candidates(messages, keep_last_turns=1) + assert candidates == [1, 3] + # Relevance order indexes into candidates: candidate 3 (newer) is LEAST + # relevant so it is compressed first; candidate 1 (older) is most relevant. + relevance_order = [1, 0] + out, stats = prune_context( + messages, + budget_tokens=100, + keep_last_turns=1, + max_summarize_chars=4000, + relevance_order=relevance_order, + ) + assert stats["pruned"] is True + # The most-relevant (older) tool result survived verbatim. + assert out[1]["content"] == messages[1]["content"] + # The least-relevant (newer) tool result was compressed. + assert "chars trimmed" in out[3]["content"] + + +def test_relevance_order_none_matches_default(): + # relevance_order=None must be byte-for-byte identical to not passing the + # parameter at all (the historical uniform pass / fallback path). + messages = [ + _user("q0"), + _tool("read", "big result " * 5000), + _assistant("a1"), + _user("final"), + _assistant("ok"), + ] + default_out, default_stats = prune_context( + messages, budget_tokens=100, keep_last_turns=0 + ) + explicit_none_out, explicit_none_stats = prune_context( + messages, + budget_tokens=100, + keep_last_turns=0, + relevance_order=None, + ) + assert explicit_none_out == default_out + assert explicit_none_stats["tokens_saved"] == default_stats["tokens_saved"] + + +def test_relevance_order_indexes_align_with_trim_candidates(): + messages = [ + _user("q0"), + _tool("search", "first-result " * 5000), + _assistant("a1"), + _tool("search", "second-result " * 5000), + _assistant("a2"), + _user("final"), + _assistant("ok"), + ] + candidates, protected_from, _ = trim_candidates(messages, keep_last_turns=1) + assert candidates == [1, 3] + # relevance_order indexes into `candidates`: index 0 means message 1. + out, stats = prune_context( + messages, + budget_tokens=2000, + keep_last_turns=1, + max_summarize_chars=4000, + relevance_order=[0], + ) + # Candidate 0 (message 1) was compressed because its content is huge. + assert "chars trimmed" in out[1]["content"] + # Candidate 1 (message 3) stayed verbatim (not in the order). + assert out[3]["content"] == messages[3]["content"] + + diff --git a/tests/test_dispatcher_helpers.py b/tests/test_dispatcher_helpers.py new file mode 100644 index 0000000..5d14daf --- /dev/null +++ b/tests/test_dispatcher_helpers.py @@ -0,0 +1,180 @@ +"""Unit tests for the new availability/relevance helpers in dispatcher.py. + +Offline. Mocks ``requests.post`` so no model/network is reached. Covers the +fail-safe ``_embed_for_relevance``, the min_candidates/disabled gates in +``_relevance_order_for``, the circuit-breaker exclusion set ``_open_circuits``, +and the connection-opening helper ``_open_upstream``. +""" + +from __future__ import annotations + +import dispatcher +import circuit_breaker + + +class _FakeResp: + def __init__(self, status_code=200, text="", json_payload=None, lines=None): + self.status_code = status_code + self._text = text + self._json = json_payload + self._lines = lines or [] + self.closed = False + + @property + def text(self): + return self._text + + def json(self): + if self._json is None: + raise ValueError("no json") + return self._json + + def close(self): + self.closed = True + + def iter_lines(self, decode_unicode=False): + yield from self._lines + + +def _cfg_with_relevance(enabled=True, min_candidates=2): + class _Rel: + enabled = True + model = "nomic-embed-text" + base_url = "http://localhost:11434/v1" + timeout_seconds = 10 + min_candidates = 2 + + if not enabled: + _Rel.enabled = False + if min_candidates != 2: + _Rel.min_candidates = min_candidates + + class _Pinch: + enabled = True + keep_last_turns = 1 + budget_tokens = 100 + max_summarize_chars = 4000 + relevance = _Rel() + + class _CB: + enabled = False + + class _Cfg: + pinch = _Pinch() + circuit_breaker = _CB() + + return _Cfg() + + +def _embedding_data(n_vectors, dim=4): + data = [] + for i in range(n_vectors): + vec = [0.0] * dim + vec[i % dim] = 1.0 + data.append({"embedding": vec, "index": i}) + return data + + +def test_embed_for_relevance_happy_path_returns_order(monkeypatch): + captured = {} + + def fake_post(url, headers=None, json=None, timeout=None): + captured["json"] = json + # Return vectors: query (idx0), candidate0 = same as query (most + # relevant), candidate1 = orthogonal (least relevant). + payload = { + "data": [ + {"embedding": [1.0, 0.0, 0.0, 0.0]}, + {"embedding": [1.0, 0.0, 0.0, 0.0]}, + {"embedding": [0.0, 1.0, 0.0, 0.0]}, + ] + } + return _FakeResp(status_code=200, json_payload=payload) + + monkeypatch.setattr(dispatcher.requests, "post", fake_post) + order = dispatcher._embed_for_relevance("hi", ["a", "b"], _cfg_with_relevance()) + # candidate1 (b) is least relevant, so it is compressed first -> index 1. + assert order == [1, 0] + assert captured["json"]["model"] == "nomic-embed-text" + + +def test_embed_for_relevance_failure_paths_return_none(monkeypatch): + cases = [ + _FakeResp(status_code=500, text="err"), # non-200 + _FakeResp(status_code=200), # unparseable (no json) + _FakeResp(status_code=200, json_payload={"data": None}), # malformed + _FakeResp(status_code=200, json_payload={"data": [{"embedding": []}]}), + ] + for resp in cases: + monkeypatch.setattr(dispatcher.requests, "post", lambda *a, _r=resp, **k: _r) + assert dispatcher._embed_for_relevance("q", ["a"], _cfg_with_relevance()) is None + + +def test_embed_for_relevance_request_exception_returns_none(monkeypatch): + def boom(*a, **k): + raise dispatcher.requests.RequestException("timeout") + + monkeypatch.setattr(dispatcher.requests, "post", boom) + assert dispatcher._embed_for_relevance("q", ["a"], _cfg_with_relevance()) is None + + +def test_relevance_order_for_returns_none_when_disabled(): + cfg = _cfg_with_relevance(enabled=False) + assert dispatcher._relevance_order_for([], cfg) is None + + +def test_relevance_order_for_below_min_candidates_skips_embedding(monkeypatch): + called = {"n": 0} + + def fake_post(*a, **k): + called["n"] += 1 + return _FakeResp(status_code=200, json_payload={"data": None}) + + monkeypatch.setattr(dispatcher.requests, "post", fake_post) + cfg = _cfg_with_relevance(enabled=True, min_candidates=2) + # One candidate only -> below min_candidates -> no embed call. + messages = [ + {"role": "user", "content": "q"}, + {"role": "tool", "name": "read", "content": "x" * 6000}, + {"role": "assistant", "content": "a"}, + ] + assert dispatcher._relevance_order_for(messages, cfg) is None + assert called["n"] == 0 + + +def test_open_circuits_empty_when_disabled(): + cfg = _cfg_with_relevance() + cfg.circuit_breaker.enabled = False + rows = [{"model_id": "m1", "provider": "p1"}] + assert dispatcher._open_circuits(rows, cfg) == set() + + +def test_open_circuits_lists_down_models(monkeypatch): + cfg = _cfg_with_relevance() + cfg.circuit_breaker.enabled = True + circuit_breaker.clear() + try: + monkeypatch.setattr(circuit_breaker.time, "time", lambda: 1000.0) + circuit_breaker.record_failure("m1", "p1", 30, 600, 2.0) + rows = [ + {"model_id": "m1", "provider": "p1"}, # down + {"model_id": "m2", "provider": "p1"}, # fine + ] + assert dispatcher._open_circuits(rows, cfg) == {"m1"} + finally: + circuit_breaker.clear() + + +def test_open_upstream_posts_stream_with_candidate_model(monkeypatch): + captured = {} + + def fake_post(url, headers=None, json=None, stream=None, timeout=None): + captured["model"] = json["model"] + captured["stream"] = stream + return _FakeResp(status_code=200) + + monkeypatch.setattr(dispatcher.requests, "post", fake_post) + dispatcher._open_upstream("candidate-model", "http://x", {"authorization": "Bearer k"}, + {"model": "ignore", "messages": []}) + assert captured["model"] == "candidate-model" + assert captured["stream"] is True diff --git a/tests/test_routing.py b/tests/test_routing.py index fe885fd..7cf12e1 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -101,6 +101,32 @@ def test_deprecated_row_is_rejected(): assert _eligible(_row(deprecated=1)) is False +# --- circuit breaker (exclude_models) ------------------------------------- + +def test_circuit_open_row_is_rejected_by_exclude_models(): + assert _eligible(_row(model_id="dead-model"), exclude_models={"dead-model"}) is False + + +def test_circuit_open_row_is_kept_when_not_excluded(): + assert _eligible(_row(model_id="dead-model"), exclude_models=set()) is True + + +def test_exclude_models_reason_is_circuit_open(): + row = _row(model_id="dead-model") + reason = rejection_reason( + row, + required_context_tokens=10_000, + required_tier=2, + latency_tolerance="interactive", + allowed_access_levels=["public"], + exclude_stale=True, + exclude_deprecated=True, + exclude_models={"dead-model"}, + ) + assert reason == "circuit_open" + + + # --- access gating -------------------------------------------------------- def test_grant_gated_rows_are_rejected_by_default(): -- 2.49.1 From e071e8bd4929e815ab30b04075bdbd98d53c82bc Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 23:28:09 -0400 Subject: [PATCH 31/32] fix(dispatcher): wire record_success into both dispatch paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code review (pinch-relevance-and-failover-review.md) found that circuit_breaker.record_success was defined and unit-tested but never called from the real dispatch path. Without it, any model with a failure history monotonically ratchets its cooldown to max_cooldown_seconds on every subsequent failure, even after long stretches of trouble-free service — the opposite of the design's passive-recovery intent. Call record_success after a successful (<400) response in the non-streaming retry loop and after the streaming pre-flight loop confirms a healthy connection, both gated by cfg.circuit_breaker.enabled matching record_failure's gating. Add an end-to-end integration test at the chat_completions level: a 503 on the first candidate, 200 on the second, asserting record_failure fired for the first and record_success for the second. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- dispatcher.py | 4 +++ tests/test_chat_completions.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/dispatcher.py b/dispatcher.py index 35f6618..a198f05 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -2570,6 +2570,8 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): alternatives = alternatives[1:] continue + if cfg.circuit_breaker.enabled: + circuit_breaker.record_success(current_model, provider) payload = resp.json() usage = payload.get("usage") or {} completion_tokens = usage.get("completion_tokens") @@ -2815,6 +2817,8 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): if attempt.status_code < 400: upstream = attempt target = candidate + if cfg.circuit_breaker.enabled: + circuit_breaker.record_success(candidate, provider) break if cfg.circuit_breaker.enabled: circuit_breaker.record_failure( diff --git a/tests/test_chat_completions.py b/tests/test_chat_completions.py index b48c781..c1e25ba 100644 --- a/tests/test_chat_completions.py +++ b/tests/test_chat_completions.py @@ -1066,3 +1066,62 @@ def test_capability_flags_are_still_read_fresh_on_a_cache_hit(router, session_ca )] conn.close() assert srcs == ["classifier", "cached"], "cached decision persisted with source=cached" + + +# --- circuit breaker failover --------------------------------------------- + +def test_breaker_records_failure_then_success_across_availability_failover( + router, monkeypatch +): + """A 503 on the routed target must record a failure, then a 200 on the + runner-up must record a success — the passive-recovery trust rebuild for + the non-streaming loop (circuit_breaker.record_success was previously + never called from the dispatch path). + """ + import circuit_breaker + + client, calls, _ = router + monkeypatch.setattr(dispatcher.cfg.circuit_breaker, "enabled", True) + + failures = [] + successes = [] + monkeypatch.setattr( + dispatcher.circuit_breaker, "record_failure", + lambda model_id, provider, *a, **kw: failures.append((model_id, provider)), + ) + monkeypatch.setattr( + dispatcher.circuit_breaker, "record_success", + lambda model_id, provider="neuralwatt": successes.append((model_id, provider)), + ) + + statuses = iter([503, 200]) + + def fake_post(url, headers=None, json=None, stream=False, timeout=None): + status = next(statuses) + calls.append({"url": url, "body": json, "stream": stream}) + if status >= 400: + return FakeResponse({"error": "down"}, status_code=status) + return FakeResponse(completion(json["model"])) + + monkeypatch.setattr(dispatcher.requests, "post", fake_post) + + try: + circuit_breaker.clear() + resp = client.post( + "/v1/chat/completions", + json={"model": "auto", "messages": _messages()}, + ) + finally: + circuit_breaker.clear() + + assert resp.status_code == 200 + # Routing picks the cheapest (CHEAP) as target; DEAR is the runner-up. + assert calls[0]["body"]["model"] == CHEAP + assert calls[1]["body"]["model"] == DEAR + assert resp.headers["X-Router-Model"] == DEAR + assert resp.json()["model"] == DEAR + + # The spies prove the wiring: a failure was recorded for the 503 target + # and a success for the 200 runner-up, without depending on is_down timing. + assert failures == [(CHEAP, "neuralwatt")] + assert successes == [(DEAR, "neuralwatt")] -- 2.49.1 From 14a7653a80f779424243c9c05f4280527c0b3e42 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Fri, 28 Aug 2026 23:28:47 -0400 Subject: [PATCH 32/32] =?UTF-8?q?docs:=20restructure=20README=20around=20p?= =?UTF-8?q?itch=20=E2=86=92=20features=20=E2=86=92=20install=20=E2=86=92?= =?UTF-8?q?=20per-command=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 450 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 256 insertions(+), 194 deletions(-) diff --git a/README.md b/README.md index c399fce..37d617f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Local LLM Model Router -## What This Is - This router sits between a coding agent (opencode, an SDK, plain curl — any OpenAI-compatible client) and the cloud LLMs it calls. It uses a local model (served via Ollama) to classify each incoming coding/documentation task — @@ -9,29 +7,9 @@ category, tier, required context size — and dispatches it to the cheapest/best-fit open-weight model on **NeuralWatt Cloud**, weighted by cost, per-category proficiency, and a per-request energy ceiling. -Point a client at `POST /v1/chat/completions` with `model: "auto"` and the -router picks per request; pin a real model id and it dispatches as asked, -still logged. See [Pointing a Coding Agent at It](#pointing-a-coding-agent-at-it). + -### NeuralWatt Cloud - -NeuralWatt is the cloud inference provider this router targets: a -multi-tenant marketplace serving a handful of open-weight base models -(DeepSeek, GLM, Qwen, Kimi, Gemma) as ~19 catalog rows, several rows per -base model for different serving classes (e.g. `-flex`/`-fast`/`-short` -variants of the same weights — see [Decision Table Schema](#decision-table-schema-sqlite)). -It bills **per kWh actually consumed, not per token**, and reports both cost -and carbon per request — that per-request telemetry is what lets this router -weight cost and eco as separate axes instead of guessing from list price. - -Neuralwatt is currently the only provider this router supports. The -`provider` column and the `(model_id, provider)` primary key in the schema -stay so a second provider can be added later without a migration. - -The local model does the classifying, so it wants a GPU, but not necessarily -*your* GPU — `classifier.base_url` takes any OpenAI-compatible endpoint, so -the usual shape is the router on your laptop and Ollama on a workstation -across a VPN. See [Setup](#setup). +Point a client at `POST /v1/chat/completions` with `model: "auto"` and the router picks per request; pin a real model id and it dispatches as asked, still logged. See [Pointing a Coding Agent at It](#pointing-a-coding-agent-at-it). **Numbers in this README are measurements, not specifications.** They come from one deployment against one provider account, and the catalog, prices, @@ -39,6 +17,260 @@ grid intensity and pool load all move. They are here because the reasoning behind a design choice is worth more than the choice, and re-running the measurement is how you check whether it still holds for you. +## Features + +- **Check routing before spending anything.** `POST /route` classifies, ranks, + and returns the selected model with no provider call and no cost. +- **Price per request from the actual shape of the traffic.** Cost is estimated + from catalog token prices scaled to the prompt size, an assumed completion + length, and an assumed cache rate — not a single fixed benchmark. +- **Fall back to local vision when no cloud row supports images.** If no + vision-capable catalog candidate survives the hard filters, the router proxies + the request to a local Ollama vision model instead of returning 422. +- **Watch decisions arrive live.** `python tui.py` opens a terminal dashboard + that follows `/events/decisions` as decisions are recorded, with no polling + delay. +- **Verify before learning.** Every routed response is structurally parsed in + the background; larger prose answers get an async local-LLM spot-check, and + failures fold back into per-model proficiency through `feedback.py`. + +## Table of Contents + +- [Features](#features) +- [Requirements](#requirements) +- [Installation](#installation) + - [Local (`venv`)](#local-venv) + - [As a systemd service](#as-a-systemd-service) +- [Usage](#usage) + - [Route without spending anything](#route-without-spending-anything) + - [Skip the classifier when you already know the shape](#skip-the-classifier-when-you-already-know-the-shape) + - [Actually dispatch and log energy](#actually-dispatch-and-log-energy) + - [Point any OpenAI-compatible client at it](#point-any-openai-compatible-client-at-it) + - [Route overnight/batch work through flex rows](#route-overnightbatch-work-through-flex-rows) + - [Ask an image question](#ask-an-image-question) + - [Force JSON output](#force-json-output) + - [Watch it live](#watch-it-live) + - [Probe routing without spending](#probe-routing-without-spending) +- [At a Glance](#at-a-glance) +- [Verification Pipeline](#verification-pipeline) + - [Structural Verification — `verification.py`](#structural-verification--verificationpy) + - [Local LLM Verification — `verification.py` (cont.)](#local-llm-verification--verificationpy-cont) + - [Feedback Loop — `feedback.py`](#feedback-loop--feedbackpy) +- [Architecture](#architecture) +- [Tech Stack](#tech-stack) +- [Pinned Dependencies (requirements.txt)](#pinned-dependencies-requirementstxt) +- [Modules](#modules) +- [Decision Table Schema (SQLite)](#decision-table-schema-sqlite) + - [`models` — one row per served model variant](#models--one-row-per-served-model-variant) + - [`proficiency` — one row per (model, provider, category)](#proficiency--one-row-per-model-provider-category) + - [`energy_observations` — per-request telemetry](#energy_observations--per-request-telemetry) + - [`verifications` — response quality observations](#verifications--response-quality-observations) + - [`route_decisions` — routing observability](#route_decisions--routing-observability) +- [Weighted Scoring](#weighted-scoring) + - [Local Vision Fallback](#local-vision-fallback) + - [What routing actually returns, and why it moves](#what-routing-actually-returns-and-why-it-moves) +- [API Endpoints](#api-endpoints) +- [Logging and Traceability](#logging-and-traceability) +- [Scheduled Jobs (systemd)](#scheduled-jobs-systemd) +- [Self-Eval Harness (`eval_proficiency.py`)](#self-eval-harness-eval_proficiencypy) +- [Classifier Reliability Notes](#classifier-reliability-notes) +- [Pointing a Coding Agent at It](#pointing-a-coding-agent-at-it) +- [Testing](#testing) +- [Setup](#setup) + - [Where Ollama lives](#where-ollama-lives) +- [Known Limitations & Open Items](#known-limitations--open-items) + +## Requirements + +- A Neuralwatt API key. +- Python 3.10+ (the test suite is verified on 3.10 and 3.14). +- An Ollama reachable from wherever this runs, with a classifier model pulled. + +Nothing else is assumed about the host — routing itself is SQLite and arithmetic. + +## Installation + +### Local (`venv`) + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +sqlite3 router.db < schema.sql +cp .env.example .env # fill in NEURALWATT_API_KEY +python poller.py # populate the catalog +python tier.py # resolve tiers +python config.py # sanity-check config loads +python -m uvicorn dispatcher:app --reload +``` + +Then edit `config.yaml` for your own setup — at minimum: + +| Key | Why | +|---|---| +| `classifier.model` | must match a model `ollama list` reports | +| `classifier.base_url` | where that Ollama actually is | +| `objective.plan_kwh_per_period` | your plan's quota; `/health` reports burn against it | +| `objective.assumed_cache_rate` | 0.917 was measured from one client's traffic (40.7M tokens). Check yours against the provider's per-session cache-hit figures | +| `session_cache.enabled` | off by default; caches category/tier per session for `staleness_minutes` to skip repeat classifier round-trips on long agent sessions | + +`python seed_energy.py` is optional. It sweeps a fixed reference workload to +populate `eco`, which is logged but is not an objective — routing works +without it. It costs real money and quota, so it is not in the path above. + +### As a systemd service + +Five user units cover continuous dispatch, catalog polling, and periodic energy reseeding. See `deploy/README.md` for full install/operate instructions. + +| Unit | Schedule | What it runs | +|---|---|---| +| `llm-router.service` | Continuous | FastAPI dispatcher | +| `llm-router-poller.timer` | 2 min after boot, then every 2 h | triggers the poller unit | +| `llm-router-poller.service` | oneshot | `poller.py` → `tier.py` | +| `llm-router-seed.timer` | Every 6 h | triggers the seed unit | +| `llm-router-seed.service` | oneshot | small `seed_energy.py` sweep | + +**The poller timer is load-bearing, not optional.** `freshness.stale_after_days` +is 3 with `exclude_stale: true` — an unpolled catalog marks every row stale +in 3 days and the router returns zero candidates for everything. + +**The seed timer spans time.** Energy attribution drifts with pool load +across hours (~50× for one model between sweeps), so a single sweep measures +one moment. The median has to span time — every 6 h sweep accumulates into a +median-across-time automatically. + +This now feeds `eco` only. Cost is priced per request from catalog prices, so +routing no longer depends on the sweep at all; disabling this timer costs you +carbon figures, not routing quality. + +## Usage + +### Route without spending anything + +Use `POST /route` when you want to see what the router would pick for a task before paying for a provider call. + +```bash +curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ + -d '{"task":"Refactor this 800-line Django view into service objects."}' +``` + +- Runs the local classifier to determine category, tier, and required context. +- Applies hard filters and ranks the surviving candidates. +- Returns the selected model, estimated cost, and estimated proficiency. +- **No provider call is made and no quota is consumed.** + +### Skip the classifier when you already know the shape + +Input to `/route` and `/dispatch` can include `task_category`, `task_tier`, and `required_context_tokens` overrides. These skip the classifier, which is useful for testing routing deterministically or for calls where you already know the task shape. + +```bash +curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ + -d '{"task":"Refactor this 800-line Django view","task_category":"coding_refactor","task_tier":2}' +``` + +- The classifier is not called, so latency is just the routing pass. +- `route_decisions.classification_source` is logged as `override`. + +### Actually dispatch and log energy + +`POST /dispatch` does the same routing work as `/route` and then calls the selected provider, streams the response if requested, and records the request as an `energy_observations` row. + +```bash +curl -s -X POST localhost:8080/dispatch -H 'content-type: application/json' \ + -d '{"task":"What is a Python context manager?"}' +``` + +- Provider response is proxied back, including streaming chunks. +- Energy, carbon, cost, and duration are scraped from SSE comments and logged. +- Structural verification runs after the response returns; the local-LLM check is async and never blocks the client. + +### Point any OpenAI-compatible client at it + +The `/v1` endpoints speak the OpenAI completions and models API, so any SDK, agent, or plain `curl` client can use the router without changes. + +```bash +curl -s localhost:8080/v1/models + +curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ + -d '{"model":"auto","messages":[{"role":"user","content":"hello"}]}' +``` + +- Ask for `model: "auto"` and the router picks per request. +- Ask for any real model id and the router dispatches directly, still logged. +- Streaming is supported: tokens pass through as they arrive while the router scrapes energy/cost from provider SSE comment lines. + +### Route overnight/batch work through flex rows + +`-flex` rows are discounted asynchronous rows that may be held during peak. Admit them for work that does not need interactive latency. + +```bash +curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ + -d '{"task":"nightly code review","latency_tolerance":"batch"}' +``` + +- `auto` excludes `-flex` rows; `auto:batch` admits them. +- `latency_tolerance: batch` on any request flips the router into the batch serving class for that call. + +### Ask an image question + +Cloud vision is not universal in the catalog, and the most economical coding rows do not declare `supports_vision`. Requests carrying `image_url` parts are routed only to vision-capable catalog rows. If none survives, the router can fall back to a local vision model instead of returning 422. + +```bash +curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ + -d '{"model":"auto","messages":[{"role":"user","content":[{"type":"text","text":"Describe this"},{"type":"image_url","image_url":{"url":"data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs="}}]}]}' +``` + +- Only inline `data:` URIs are accepted; remote `http(s)` image URLs are declined to avoid SSRF. +- Image count and total payload size are bounded before the local call is made. +- Configure the fallback in `config.yaml` under `local_vision:`. + +### Force JSON output + +Use `response_format` when you need structured output. The router treats this as a hard capability requirement and only admits rows that declare `supports_json_mode = 1`. + +```bash +curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ + -d '{"model":"auto","messages":[{"role":"user","content":"Return a JSON object with field answer"}],"response_format":{"type":"json_object"}}' +``` + +- `NULL` fails closed: an unknown flag means the capability cannot be confirmed. +- When streaming is not used, the structural verification verdict appears in the `X-Router-Verification` header. + +### Watch it live + +Three read-only ways to observe the router without spending quota: + +```bash +# Aggregate JSON health/usage summary +curl -s localhost:8080/metrics | python -m json.tool + +# Live SSE stream of routing decisions +# (tui.py reads this, or you can connect directly) +curl -s localhost:8080/events/decisions + +# Terminal dashboard with live routing feed +python tui.py +``` + +- **`GET /metrics`** returns quota burn, coverage, recent decisions, per-model totals, verdict mix, and top proficiency. Loopback-only, no auth. +- **`GET /events/decisions`** is a Server-Sent Events stream of routing decisions. It replays recent decisions, then streams new ones as they happen; `:heartbeat` keepalive comments keep the connection alive between events. +- **`python tui.py`** is a Textual dashboard with a live routing feed, a category → model breakdown panel, a detail popup (press Enter or `e`), and quota/per-model/verdict/warnings panels. It is a separate entrypoint, not a systemd unit. +- **`baseline_report.py --since 2026-08-01`** is a read-only retrospective that replays recent decisions against two trivial counterfactuals — always pick the cheapest eligible candidate and always pick the best-proficiency eligible candidate — and reports total/mean cost and proficiency plus a dominance share. Add `--category coding_refactor` to scope it, or `--csv` for parseable output. + +`textual` is pinned in `requirements.txt` solely for the TUI modules (`tui.py`, `tui_screens.py`, `tui_sse.py`). It is imported only by these modules; the FastAPI service dispatch path never touches it, so the router itself has no UI dependency. + +### Probe routing without spending + +`router_cli.py` is a one-shot shell probe that POSTs to `/route` once and prints the full decision tree. + +```bash +python router_cli.py "Refactor this Django view into service objects" +python router_cli.py "Summarize this diff" --category summarization --tier 2 +``` + +- Prints the selected model, candidates, estimated cost, and estimated proficiency. +- Use `--category`, `--tier`, and `--context` to override the classifier deterministically. +- Use `--json` for raw output. + ## At a Glance | Dimension | Detail | @@ -663,92 +895,6 @@ active constraints. That now includes "vision-capable model" or "json-mode-capable model" when the request carried images or a JSON-mode `response_format`, alongside the existing context/tier/latency/tool reasons. -## Quick Usage - -```bash -# Pick a model without spending anything -curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ - -d '{"task":"Refactor this 800-line Django view into service objects."}' - -# With explicit category/tier overrides (skips classifier) -curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ - -d '{"task":"Refactor this 800-line Django view","task_category":"coding_refactor","task_tier":2}' - -# Actually call the winner and log energy/cost -curl -s -X POST localhost:8080/dispatch -H 'content-type: application/json' \ - -d '{"task":"What is a Python context manager?"}' - -# OpenAI-compatible — works with any SDK or agent client -curl -s localhost:8080/v1/models -curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ - -d '{"model":"auto","messages":[{"role":"user","content":"hello"}]}' - -# Admit flex rows for overnight/async work -curl -s -X POST localhost:8080/route -H 'content-type: application/json' \ - -d '{"task":"nightly code review","latency_tolerance":"batch"}' - -# Image URL request (routed only to vision-capable catalog rows) -curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ - -d '{"model":"auto","messages":[{"role":"user","content":[{"type":"text","text":"Describe this"},{"type":"image_url","image_url":{"url":"data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs="}}]}]}' - -# JSON-mode request via response_format -curl -s -X POST localhost:8080/v1/chat/completions -H 'content-type: application/json' \ - -d '{"model":"auto","messages":[{"role":"user","content":"Return a JSON object with field answer"}],"response_format":{"type":"json_object"}}' - -# Monitoring: aggregate router health -curl -s localhost:8080/metrics | python -m json.tool - -# Terminal dashboard with live routing feed (requires `textual`; runs until you press q) -python tui.py - -# One-shot routing probe with no spend -python router_cli.py "Refactor this Django view into service objects" -python router_cli.py "Summarize this diff" --category summarization --tier 2 -``` - -### Monitoring - -Three foreground tools read the running router without spending quota: - -- **`GET /metrics`** — JSON summary of quota, coverage, recent routing - decisions, per-model usage, verdict mix, and top proficiency. No auth; - loopback only. -- **`GET /events/decisions`** — Server-Sent Events stream of routing - decisions for the TUI's live feed. Replays recent decisions, then streams - new ones as they are recorded; `:heartbeat` keepalive comments keep the - connection alive between events. No auth; loopback only. -- **`tui.py`** — Textual terminal dashboard with a **live routing feed** - (via `/events/decisions`, so decisions appear as they happen without - waiting for the 5s `/metrics` poll), a **category → model breakdown** - panel aggregating recent decisions by `(category, tier)` with majority - winner and share, a **detail popup** (press Enter or `e` on any decision - row to see the full JSON — runner-ups, rejected reasons, feature flags, - confidence, context size), and the existing quota / per-model / verdict - / warnings panels. Run it in a terminal with the service already up. It - is a separate entrypoint, not a systemd unit. -- **`router_cli.py ""`** — POSTs to `/route` once and prints the full - decision tree, including candidates, selected model, estimated cost, and - proficiency. Use `--category`, `--tier`, and `--context` to override the - classifier deterministically, or `--json` for raw output. -- **`baseline_report.py --since 2026-08-01`** — read-only retrospective that - replays recent routed decisions against two trivial counterfactuals — always - pick the cheapest eligible candidate and always pick the best-proficiency - eligible candidate — and reports total/mean cost and proficiency plus a - **dominance check** (the share of decisions where the real router picked the - cheapest eligible model), overall and per `task_category`. This automates the - README's manual "check for dominance" step: a high dominance share with a - near-zero proficiency delta means the real scoring isn't earning its - complexity for that slice of traffic. Add `--category coding_refactor` to - scope it, or `--csv` to emit parseable CSV instead of the table. It reads - only `route_decisions`, `models`, and `proficiency`, writes nothing, and - reuses `routing.select_candidates`/`routing.estimated_cost` — comparing - against the *current* catalog, not a historical snapshot. - -`textual` is pinned in `requirements.txt` solely for the TUI modules -(`tui.py`, `tui_screens.py`, `tui_sse.py`). It is imported only by these -modules; the FastAPI service dispatch path never touches it, so the router -itself has no UI dependency. - ## Logging and Traceability One `route` line per request says what was decided; one `dispatch` line says @@ -798,31 +944,6 @@ journalctl --user -u llm-router --grep ' id=r9116d9' # every stage of it when — systemd owns its stderr. Run uvicorn in a terminal and the lines come out clean. -## Scheduled Jobs (systemd) - -Five user units. See `deploy/README.md` for full install/operate instructions. - -| Unit | Schedule | What it runs | -|---|---|---| -| `llm-router.service` | Continuous | FastAPI dispatcher | -| `llm-router-poller.timer` | 2 min after boot, then every 2 h | triggers the poller unit | -| `llm-router-poller.service` | oneshot | `poller.py` → `tier.py` | -| `llm-router-seed.timer` | Every 6 h | triggers the seed unit | -| `llm-router-seed.service` | oneshot | small `seed_energy.py` sweep | - -**The poller timer is load-bearing, not optional.** `freshness.stale_after_days` -is 3 with `exclude_stale: true` — an unpolled catalog marks every row stale -in 3 days and the router returns zero candidates for everything. - -**The seed timer spans time.** Energy attribution drifts with pool load -across hours (~50× for one model between sweeps), so a single sweep measures -one moment. The median has to span time — every 6 h sweep accumulates into a -median-across-time automatically. - -This now feeds `eco` only. Cost is priced per request from catalog prices, so -routing no longer depends on the sweep at all; disabling this timer costs you -carbon figures, not routing quality. - ## Self-Eval Harness (`eval_proficiency.py`) ```bash @@ -949,65 +1070,6 @@ config as arguments, so the suite runs offline on a clean checkout. | `test_classifier_input.py` | Classifier framing: `_previous_context` scope, `_classifier_user_content` framing | | `test_route_decisions.py` | `route_decisions` table, inline-create helper, config gate | -## Setup - -**You need:** a Neuralwatt API key, Python 3.10+ (suite verified on 3.10 and -3.14), and an Ollama reachable from wherever this runs with a classifier model -pulled. Nothing else is assumed about the host — routing itself is SQLite and -arithmetic. - -```bash -python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt -sqlite3 router.db < schema.sql -cp .env.example .env # fill in NEURALWATT_API_KEY -python poller.py # populate the catalog -python tier.py # resolve tiers -python config.py # sanity-check config loads -python -m uvicorn dispatcher:app --reload -``` - -Then edit `config.yaml` for your own setup — at minimum: - -| Key | Why | -|---|---| -| `classifier.model` | must match a model `ollama list` reports | -| `classifier.base_url` | where that Ollama actually is | -| `objective.plan_kwh_per_period` | your plan's quota; `/health` reports burn against it | -| `objective.assumed_cache_rate` | 0.917 was measured from one client's traffic (40.7M tokens). Check yours against the provider's per-session cache-hit figures | -| `session_cache.enabled` | off by default; caches category/tier per session for `staleness_minutes` to skip repeat classifier round-trips on long agent sessions | - -`python seed_energy.py` is optional. It sweeps a fixed reference workload to -populate `eco`, which is logged but is not an objective — routing works -without it. It costs real money and quota, so it is not in the path above. - -### Where Ollama lives - -```bash -ollama pull mistral-nemo:12b # or whatever you set as classifier.model -``` - -It does not have to be on the machine running the router; the box with the -GPU usually isn't the laptop. To use one across a VPN, point **both** -endpoints at it: - -```yaml -classifier: - base_url: "http://:11434/v1" -verification: - base_url: "http://:11434" # same host, so `model` can stay null -``` - -and apply `deploy/ollama-over-vpn.conf` on the serving host — Ollama binds -`127.0.0.1` by default and will otherwise refuse. Bind it to the VPN address -rather than `0.0.0.0`: Ollama has no authentication, so anything reaching the -port can run inference and enumerate your models. - -Both endpoints move together because the verifier speaks Ollama's *native* -API and cannot follow the classifier to a cloud provider. Config load refuses -the case where they are on different hosts and `verification.model` is null, -because that combination fails silently. - ## Known Limitations & Open Items - **Leaderboard priors unfilled** — `leaderboards.yaml` ships empty. `python leaderboard.py --check` -- 2.49.1