From 0fc3e0ac994007218902a582d7cf07084837d380 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:09:46 -0400 Subject: [PATCH 01/15] feat(config): add has_energy_telemetry and enabled to DispatchProvider; add OpenRouter config Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .env.example | 1 + config/config.yaml | 11 ++++++++--- src/config.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 21c4fd4..afc100c 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ # Copy to .env and fill in. Never commit the real .env file. NEURALWATT_API_KEY= +OPENROUTER_API_KEY= diff --git a/config/config.yaml b/config/config.yaml index aad0f33..ed46d16 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -692,12 +692,17 @@ classifier: } dispatch_providers: - # NeuralWatt is the only provider. The dict shape and the models table's - # (model_id, provider) key are kept so a second one can be added without a - # migration. neuralwatt: base_url: "https://api.neuralwatt.com/v1" api_key_env: "NEURALWATT_API_KEY" + has_energy_telemetry: true + enabled: true + + openrouter: + base_url: "https://openrouter.ai/api/v1" + api_key_env: "OPENROUTER_API_KEY" + has_energy_telemetry: false + enabled: true local_energy: # Local energy metering for the router's own Ollama calls (classifier, diff --git a/src/config.py b/src/config.py index cc20b3d..96c2eea 100644 --- a/src/config.py +++ b/src/config.py @@ -821,6 +821,8 @@ class LocalDispatchModel(StrictModel): class DispatchProvider(StrictModel): base_url: str api_key_env: str + has_energy_telemetry: bool = False + enabled: bool = True class LoggingConfig(StrictModel): -- 2.49.1 From 9e03f2211cab01116905e4abbbf8d6f4f8cf1d82 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:16:27 -0400 Subject: [PATCH 02/15] feat(poller): fetch and normalize OpenRouter catalog Add fetch_openrouter() and parse_openrouter_model() to src/poller.py. OpenRouter virtual routers are excluded, :batch variants map to latency_class='flex', and per-token USD prices are normalized to cost per 1M tokens. Add tests/test_openrouter_poller.py with mocked requests.get coverage. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/poller.py | 90 +++++++++++++ tests/test_openrouter_poller.py | 225 ++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 tests/test_openrouter_poller.py diff --git a/src/poller.py b/src/poller.py index 577ef31..6888674 100644 --- a/src/poller.py +++ b/src/poller.py @@ -34,9 +34,19 @@ import requests from config import LocalDispatchModel, RouterConfig, load_config MODELS_URL = "https://api.neuralwatt.com/v1/models" +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" REQUEST_TIMEOUT = 20 # seconds +OPENROUTER_VIRTUAL_ROUTERS = frozenset({ + "openrouter/auto", + "openrouter/auto-beta", + "openrouter/free", + "openrouter/fusion", + "openrouter/pareto-code", + "openrouter/bodybuilder", +}) + # Serving-class suffixes. NeuralWatt ships one base model as several catalog # rows that differ only by these tokens, and they combine freely — hence ids # like 'glm-5.2-short-fast-flex'. They are stripped from the end of the id one @@ -222,6 +232,86 @@ def fetch_neuralwatt(provider: str) -> list[ModelRow]: return rows +def _openrouter_price_to_cost_per_1m(value: Optional[str]) -> Optional[float]: + """Convert an OpenRouter per-token price string to cost per 1M tokens.""" + if value is None: + return None + try: + per_token = float(value) + except ValueError: + return None + return per_token * 1_000_000 + + +def parse_openrouter_model(raw_model: dict) -> Optional[ModelRow]: + """Normalize a single OpenRouter /v1/models entry into a ModelRow. + + Returns ``None`` for virtual routers (``openrouter/*``) so callers can + drop them cleanly without losing the rest of the catalog. + """ + model_id = raw_model.get("id") or "" + if model_id in OPENROUTER_VIRTUAL_ROUTERS: + return None + + canonical_slug = raw_model.get("canonical_slug") or model_id + pricing = raw_model.get("pricing", {}) or {} + top_provider = raw_model.get("top_provider", {}) or {} + architecture = raw_model.get("architecture", {}) or {} + reasoning = raw_model.get("reasoning", {}) or {} + supported_parameters = raw_model.get("supported_parameters") or [] + input_modalities = architecture.get("input_modalities") or [] + + prompt_price = _openrouter_price_to_cost_per_1m(pricing.get("prompt")) + completion_price = _openrouter_price_to_cost_per_1m(pricing.get("completion")) + + variant = model_id.split(":")[-1] if ":" in model_id else "standard" + + return ModelRow( + model_id=model_id, + provider="openrouter", + base_model_id=canonical_slug, + display_name=raw_model.get("name"), + cost_per_1m_prompt=prompt_price, + cost_per_1m_completion=completion_price, + cost_per_1m_prompt_cached=None, + context_window=top_provider.get("context_length") + or raw_model.get("context_length"), + max_output_tokens=top_provider.get("max_completion_tokens"), + supports_tools="tools" in supported_parameters, + supports_json_mode="response_format" in supported_parameters, + supports_vision="image" in input_modalities, + supports_reasoning="reasoning" in supported_parameters, + reasoning_default_enabled=bool(reasoning.get("default_enabled")), + latency_class="flex" if ":batch" in model_id else "standard", + reasoning_mode=reasoning.get("mode") or "none", + context_variant=variant, + access_level="public", + pricing_tbd=prompt_price is None, + deprecated=False, + ) + + +def fetch_openrouter(provider: str) -> list[ModelRow]: + """Fetch the public OpenRouter catalog and normalize it into ModelRows. + + The OpenRouter `/v1/models` endpoint is unauthenticated and paginated with + a default page size of 500 (max 1000). A single request covers the whole + catalog. ``pricing.web_search`` and ``pricing.cache_read`` differentials + are intentionally omitted from catalog cost estimates; routing uses the + prompt and completion price only. + """ + resp = requests.get(OPENROUTER_MODELS_URL, timeout=REQUEST_TIMEOUT) + resp.raise_for_status() + payload = resp.json() + + rows: list[ModelRow] = [] + for raw_model in payload.get("data", []): + row = parse_openrouter_model(raw_model) + if row is not None: + rows.append(row) + return rows + + def upsert(conn: sqlite3.Connection, rows: list[ModelRow], cfg: RouterConfig) -> None: now = datetime.now(timezone.utc).isoformat() for r in rows: diff --git a/tests/test_openrouter_poller.py b/tests/test_openrouter_poller.py new file mode 100644 index 0000000..9864bfc --- /dev/null +++ b/tests/test_openrouter_poller.py @@ -0,0 +1,225 @@ +"""Offline tests for the OpenRouter catalog fetcher and parser in poller.py.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from poller import ( + OPENROUTER_VIRTUAL_ROUTERS, + ModelRow, + fetch_openrouter, + parse_openrouter_model, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _fake_response(json_data: dict): + fake = MagicMock() + fake.json.return_value = json_data + fake.raise_for_status.return_value = None + fake.status_code = 200 + return fake + + +def _openrouter_model( + model_id: str, + *, + canonical_slug: str | None = None, + prompt: str = "0.000001", + completion: str = "0.000002", + context_length: int = 131072, + max_completion_tokens: int = 8192, + modalities: list[str] | None = None, + supported_parameters: list[str] | None = None, + reasoning: dict | None = None, +): + """Build a realistic OpenRouter /v1/models entry.""" + return { + "id": model_id, + "name": model_id.replace("/", " ").title(), + "canonical_slug": canonical_slug or model_id, + "pricing": { + "prompt": prompt, + "completion": completion, + "web_search": "0.000003", + "cache_read": "0.0000005", + }, + "top_provider": { + "context_length": context_length, + "max_completion_tokens": max_completion_tokens, + }, + "architecture": { + "input_modalities": modalities or ["text"], + }, + "supported_parameters": supported_parameters or [], + "reasoning": reasoning, + } + + +# --------------------------------------------------------------------------- +# parse_openrouter_model +# --------------------------------------------------------------------------- + +def test_parse_openrouter_model_populates_required_fields(): + raw = _openrouter_model( + "openai/gpt-6-astra", + canonical_slug="gpt-6-astra", + prompt="0.0000015", + completion="0.0000060", + context_length=200000, + max_completion_tokens=32768, + modalities=["text", "image"], + supported_parameters=["tools", "response_format", "reasoning"], + reasoning={"mode": "medium", "default_enabled": True}, + ) + + row = parse_openrouter_model(raw) + + assert row == ModelRow( + model_id="openai/gpt-6-astra", + provider="openrouter", + base_model_id="gpt-6-astra", + display_name="Openai Gpt-6-Astra", + cost_per_1m_prompt=1.5, + cost_per_1m_completion=6.0, + cost_per_1m_prompt_cached=None, + context_window=200000, + max_output_tokens=32768, + supports_tools=True, + supports_json_mode=True, + supports_vision=True, + supports_reasoning=True, + reasoning_default_enabled=True, + latency_class="standard", + reasoning_mode="medium", + context_variant="standard", + access_level="public", + pricing_tbd=False, + deprecated=False, + ) + + +def test_parse_openrouter_model_batch_variant_maps_to_flex(): + raw = _openrouter_model( + "anthropic/claude-4-opus:batch", + canonical_slug="claude-4-opus", + ) + + row = parse_openrouter_model(raw) + + assert row.latency_class == "flex" + assert row.context_variant == "batch" + assert row.model_id == "anthropic/claude-4-opus:batch" + + +def test_parse_openrouter_model_free_variant_maps_to_standard(): + raw = _openrouter_model( + "meta-llama/llama-4-maverick:free", + canonical_slug="llama-4-maverick", + ) + + row = parse_openrouter_model(raw) + + assert row.latency_class == "standard" + assert row.context_variant == "free" + + +def test_parse_openrouter_model_reasoning_defaults(): + raw = _openrouter_model("google/gemini-2.5-flash") + + row = parse_openrouter_model(raw) + + assert row.reasoning_mode == "none" + assert row.reasoning_default_enabled is False + assert row.supports_reasoning is False + + +@pytest.mark.parametrize("router_id", sorted(OPENROUTER_VIRTUAL_ROUTERS)) +def test_parse_openrouter_model_excludes_virtual_routers(router_id): + raw = _openrouter_model(router_id) + + assert parse_openrouter_model(raw) is None + + +def test_parse_openrouter_model_marks_missing_prompt_price_as_tbd(): + raw = _openrouter_model("x-ai/grok-3", prompt="") + + row = parse_openrouter_model(raw) + + assert row.cost_per_1m_prompt is None + assert row.pricing_tbd is True + + +def test_parse_openrouter_model_gracefully_handles_malformed_prices(): + raw = _openrouter_model("x-ai/grok-3-mini", prompt="free", completion="N/A") + + row = parse_openrouter_model(raw) + + assert row.cost_per_1m_prompt is None + assert row.cost_per_1m_completion is None + assert row.pricing_tbd is True + + +# --------------------------------------------------------------------------- +# fetch_openrouter +# --------------------------------------------------------------------------- + +@patch.object(requests, "get") +def test_fetch_openrouter_returns_model_rows(mock_get): + mock_get.return_value = _fake_response( + { + "data": [ + _openrouter_model( + "openai/gpt-6-astra", + canonical_slug="gpt-6-astra", + ), + _openrouter_model( + "anthropic/claude-4-opus:batch", + canonical_slug="claude-4-opus", + ), + ] + } + ) + + rows = fetch_openrouter("openrouter") + + assert len(rows) == 2 + assert rows[0].model_id == "openai/gpt-6-astra" + assert rows[0].base_model_id == "gpt-6-astra" + assert rows[1].model_id == "anthropic/claude-4-opus:batch" + assert rows[1].latency_class == "flex" + mock_get.assert_called_once_with( + "https://openrouter.ai/api/v1/models", timeout=20 + ) + + +@patch.object(requests, "get") +def test_fetch_openrouter_excludes_virtual_routers(mock_get): + mock_get.return_value = _fake_response( + { + "data": [ + _openrouter_model("openrouter/auto"), + _openrouter_model("openrouter/free"), + _openrouter_model("openrouter/fusion"), + _openrouter_model("openai/gpt-6-astra", canonical_slug="gpt-6-astra"), + ] + } + ) + + rows = fetch_openrouter("openrouter") + + assert [r.model_id for r in rows] == ["openai/gpt-6-astra"] + + +@patch.object(requests, "get") +def test_fetch_openrouter_propagates_request_exception(mock_get): + mock_get.side_effect = requests.RequestException("offline") + + with pytest.raises(requests.RequestException): + fetch_openrouter("openrouter") -- 2.49.1 From 39a2edd6adb42eb164fb304e605c1be77d7f9554 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:23:09 -0400 Subject: [PATCH 03/15] feat(admin): provider CRUD endpoints and page-serving route --- src/admin.py | 133 ++++++++++++++++++++++++++++++++++ tests/test_admin_providers.py | 130 +++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 tests/test_admin_providers.py diff --git a/src/admin.py b/src/admin.py index 77ee2fe..2f786f4 100644 --- a/src/admin.py +++ b/src/admin.py @@ -30,6 +30,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path from typing import Any, List, Literal, Optional +from urllib.parse import urlparse from fastapi import APIRouter, BackgroundTasks, HTTPException from fastapi.responses import FileResponse @@ -43,6 +44,7 @@ import metrics import routing from config import ( BUILTIN_PROFILES, + DispatchProvider, FlexPreference, RouterConfig, RoutingProfile, @@ -649,6 +651,13 @@ class _ProfileUpdateBody(BaseModel): allowed_model_ids: Optional[list[str]] = None +class _ProviderUpdateBody(BaseModel): + base_url: str + api_key_env: str + has_energy_telemetry: bool + enabled: bool + + def _runtime_state(cfg: Any) -> dict: """Read every toggle knob's current in-memory value off ``cfg``.""" return { @@ -696,6 +705,7 @@ def build_router( _admin_models = _REPO_ROOT / "admin" / "frontend" / "models.html" _admin_decisions = _REPO_ROOT / "admin" / "frontend" / "decisions.html" _admin_profiles = _REPO_ROOT / "admin" / "frontend" / "profiles.html" + _admin_providers = _REPO_ROOT / "admin" / "frontend" / "providers.html" _admin_logo = _REPO_ROOT / "admin" / "frontend" / "6krrt-logo.webp" # No-cache: these pages get hand-edited and reloaded constantly during @@ -728,6 +738,10 @@ def build_router( def admin_profiles_page() -> FileResponse: return FileResponse(_admin_profiles, media_type="text/html", headers=_NO_CACHE_HEADERS) + @router.get("/providers") + def admin_providers_page() -> FileResponse: + return FileResponse(_admin_providers, media_type="text/html", headers=_NO_CACHE_HEADERS) + @router.get("/api/health") def admin_health() -> dict: return {"status": "ok"} @@ -1067,6 +1081,125 @@ def build_router( "message": "A restart is required for this change to take effect", } + def _provider_provenance(name: str) -> dict: + base_store = load_config_store_safe(config_path) or {} + base_providers = base_store.get("dispatch_providers") or {} + try: + overlay = load_config_store(config_local_path) or {} + except FileNotFoundError: + overlay = {} + overlay_providers = overlay.get("dispatch_providers") or {} + return { + "in_base": name in base_providers, + "in_overlay": name in overlay_providers, + } + + def _merged_providers() -> dict[str, Any]: + merged = _load_merged_config_store(config_path, config_local_path) + return merged.get("dispatch_providers") or {} + + def _provider_block_dict(body: _ProviderUpdateBody) -> dict[str, Any]: + return body.model_dump() + + def _validate_provider_url(base_url: str) -> None: + parsed = urlparse(base_url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise HTTPException( + status_code=422, + detail=f"base_url must be an http(s) URL, got {base_url!r}", + ) + + def _persist_provider( + name: str, provider_dict: Optional[dict[str, Any]], *, delete: bool = False + ) -> None: + try: + _persist_to( + config_path, + config_local_path, + ("dispatch_providers", name), + provider_dict, + delete=delete, + ) + except ValidationError as exc: + raise HTTPException( + status_code=422, + detail=exc.errors()[0]["msg"], + ) from exc + except KeyError as exc: + raise HTTPException(status_code=404, detail="provider not found") from exc + + @router.get("/api/providers") + def admin_providers_list() -> list[dict[str, Any]]: + providers = _merged_providers() + return [ + { + "name": name, + "base_url": p["base_url"], + "api_key_env": p["api_key_env"], + "has_energy_telemetry": bool(p.get("has_energy_telemetry", False)), + "enabled": bool(p.get("enabled", True)), + } + for name, p in providers.items() + ] + + @router.get("/api/providers/{name}") + def admin_provider_detail(name: str) -> dict[str, Any]: + providers = _merged_providers() + if name not in providers: + raise HTTPException(status_code=404, detail="provider not found") + p = providers[name] + return { + "name": name, + "base_url": p["base_url"], + "api_key_env": p["api_key_env"], + "has_energy_telemetry": bool(p.get("has_energy_telemetry", False)), + "enabled": bool(p.get("enabled", True)), + } + + @router.post("/api/providers/{name}") + def admin_provider_update(name: str, body: _ProviderUpdateBody) -> dict[str, Any]: + _validate_provider_url(body.base_url) + provider_dict = _provider_block_dict(body) + try: + DispatchProvider(**provider_dict) + except ValidationError as exc: + raise HTTPException( + status_code=422, + detail=exc.errors()[0]["msg"], + ) from exc + _persist_provider(name, provider_dict) + return { + "provider": {"name": name, **provider_dict}, + "message": "A restart is required for this change to take effect", + } + + @router.delete("/api/providers/{name}") + def admin_provider_delete(name: str) -> dict[str, Any]: + if name == cfg.dispatch_settings.default_provider: + raise HTTPException( + status_code=422, + detail=( + f"Cannot delete provider {name!r}: it is the current " + f"dispatch_settings.default_provider." + ), + ) + provenance = _provider_provenance(name) + if provenance["in_base"]: + raise HTTPException( + status_code=403, + detail=( + f"provider {name!r} is defined in config/config.yaml and " + f"is read-only" + ), + ) + if not provenance["in_overlay"]: + raise HTTPException(status_code=404, detail="provider not found") + _persist_provider(name, None, delete=True) + return { + "deleted": name, + "message": "A restart is required for this change to take effect", + } + @router.get("/api/models") def admin_models() -> list: """All models with per-category proficiency, for the admin model table.""" diff --git a/tests/test_admin_providers.py b/tests/test_admin_providers.py new file mode 100644 index 0000000..6b681fd --- /dev/null +++ b/tests/test_admin_providers.py @@ -0,0 +1,130 @@ +"""Tests for the admin provider CRUD endpoints and /admin/providers page route. + +The page fixture creates a temporary ``admin/frontend/providers.html`` if the +file does not yet exist, and removes it after the test so no HTML file is left +in the repo as part of this change. +""" + +from __future__ import annotations + +import shutil +import sqlite3 +from pathlib import Path + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from admin import build_router +from config import load_config + +ROOT = Path(__file__).resolve().parent.parent + + +def _make_db(tmp_path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(str(tmp_path / "providers.db")) + conn.row_factory = sqlite3.Row + return conn + + +@pytest.fixture +def client(tmp_path): + """A TestClient for an isolated admin router over a temp config.yaml copy.""" + (tmp_path / "config").mkdir(parents=True, exist_ok=True) + config_yaml = tmp_path / "config" / "config.yaml" + shutil.copyfile(ROOT / "config" / "config.yaml", config_yaml) + + schema_sql = (ROOT / "config" / "schema.sql").read_text() + conn = _make_db(tmp_path) + conn.executescript(schema_sql) + conn.close() + + cfg = load_config(str(config_yaml)) + + def _db_factory() -> sqlite3.Connection: + return _make_db(tmp_path) + + router = build_router(cfg, _db_factory, base_dir=str(tmp_path)) + app = FastAPI() + app.include_router(router, prefix="/admin") + + page = ROOT / "admin" / "frontend" / "providers.html" + created_page = False + if not page.exists(): + page.write_text("Providers") + created_page = True + + try: + yield TestClient(app) + finally: + if created_page: + page.unlink(missing_ok=True) + + +_VALID_PROVIDER = { + "base_url": "https://api.example.com/v1", + "api_key_env": "EXAMPLE_API_KEY", + "has_energy_telemetry": True, + "enabled": True, +} + + +def test_providers_page_returns_html(client): + """GET /admin/providers serves the providers HTML page.""" + resp = client.get("/admin/providers") + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/html") + assert "Providers" in resp.text + + +def test_provider_crud_round_trip(client, tmp_path): + """Create, read, and delete a provider through the overlay.""" + local_yaml = tmp_path / "config" / "config.local.yaml" + + resp = client.post("/admin/api/providers/testprov", json=_VALID_PROVIDER) + assert resp.status_code == 200 + body = resp.json() + assert body["provider"]["name"] == "testprov" + assert body["provider"]["base_url"] == _VALID_PROVIDER["base_url"] + + assert local_yaml.exists() + text = local_yaml.read_text() + assert "testprov:" in text + assert "dispatch_providers:" in text + + resp = client.get("/admin/api/providers/testprov") + assert resp.status_code == 200 + detail = resp.json() + assert detail["name"] == "testprov" + assert detail["base_url"] == _VALID_PROVIDER["base_url"] + assert detail["api_key_env"] == _VALID_PROVIDER["api_key_env"] + assert detail["has_energy_telemetry"] is True + assert detail["enabled"] is True + + resp = client.get("/admin/api/providers") + assert resp.status_code == 200 + names = [p["name"] for p in resp.json()] + assert "testprov" in names + + resp = client.delete("/admin/api/providers/testprov") + assert resp.status_code == 200 + assert resp.json()["deleted"] == "testprov" + + resp = client.get("/admin/api/providers/testprov") + assert resp.status_code == 404 + + +def test_provider_post_invalid_url_returns_422(client, tmp_path): + """A non-URL base_url is rejected without writing to the overlay.""" + local_yaml = tmp_path / "config" / "config.local.yaml" + resp = client.post( + "/admin/api/providers/badurl", + json={ + "base_url": "not-a-url", + "api_key_env": "BAD_API_KEY", + "has_energy_telemetry": False, + "enabled": True, + }, + ) + assert resp.status_code == 422 + assert not local_yaml.exists() -- 2.49.1 From 9c5b5e5517573b5b358588bd50914b5710bb00d8 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:28:06 -0400 Subject: [PATCH 04/15] feat(admin): providers management page with create/edit/delete UI --- .../task-3-multi-provider-openrouter.json | 23 + admin/frontend/controls.html | 1 + admin/frontend/decisions.html | 1 + admin/frontend/index.html | 1 + admin/frontend/models.html | 1 + admin/frontend/profiles.html | 1 + admin/frontend/providers.html | 672 ++++++++++++++++++ 7 files changed, 700 insertions(+) create mode 100644 .omo/evidence/task-3-multi-provider-openrouter.json create mode 100644 admin/frontend/providers.html diff --git a/.omo/evidence/task-3-multi-provider-openrouter.json b/.omo/evidence/task-3-multi-provider-openrouter.json new file mode 100644 index 0000000..6c1d102 --- /dev/null +++ b/.omo/evidence/task-3-multi-provider-openrouter.json @@ -0,0 +1,23 @@ +{ + "verification": { + "pytest": "tests/test_admin_frontend.py -x: 12 passed", + "full_suite": "1270 passed, 2 failed, 1 warning (failures in test_chat_completions.py::llm-router prefix stripping, unrelated to this task)", + "lsp_diagnostics": "skipped (basedpyright not installed)", + "ruff": "skipped (ruff not installed)", + "files_changed": [ + "admin/frontend/providers.html", + "admin/frontend/index.html", + "admin/frontend/models.html", + "admin/frontend/profiles.html", + "admin/frontend/decisions.html", + "admin/frontend/controls.html" + ], + "notes": [ + "Created admin/frontend/providers.html with GET list, add form, inline edit toggle, and delete UI backed by /admin/api/providers endpoints.", + "Booleans rendered as Bootstrap form-switch toggles (not bare checkboxes), matching controls.html switch pattern.", + "All interpolated DOM strings pass through escapeHtml(); saves/deletes call POST/DELETE /admin/api/providers/{name} and show restart-required banner on success.", + "Added 'Providers' nav link to all existing admin/frontend/*.html pages; current page marked active.", + "Backend /admin/providers route was already wired in src/admin.py (task 2)." + ] + } +} diff --git a/admin/frontend/controls.html b/admin/frontend/controls.html index 1d62059..69f665a 100644 --- a/admin/frontend/controls.html +++ b/admin/frontend/controls.html @@ -191,6 +191,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
  • Profiles
  • Decisions
  • Controls
    • +
  • Providers
    • diff --git a/admin/frontend/decisions.html b/admin/frontend/decisions.html index e0f51c0..3665d84 100644 --- a/admin/frontend/decisions.html +++ b/admin/frontend/decisions.html @@ -174,6 +174,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
  • Profiles
  • Decisions
  • Controls
    • +
  • Providers
    • diff --git a/admin/frontend/index.html b/admin/frontend/index.html index 9a59b6a..4e8ecb4 100644 --- a/admin/frontend/index.html +++ b/admin/frontend/index.html @@ -280,6 +280,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
  • Profiles
  • Decisions
  • Controls
    • +
  • Providers
    • diff --git a/admin/frontend/models.html b/admin/frontend/models.html index 9df72ec..207ce1e 100644 --- a/admin/frontend/models.html +++ b/admin/frontend/models.html @@ -159,6 +159,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
  • Profiles
  • Decisions
  • Controls
    • +
  • Providers
    • diff --git a/admin/frontend/profiles.html b/admin/frontend/profiles.html index e0ca479..e78737c 100644 --- a/admin/frontend/profiles.html +++ b/admin/frontend/profiles.html @@ -146,6 +146,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
  • Profiles
  • Decisions
  • Controls
    • +
  • Providers
    • diff --git a/admin/frontend/providers.html b/admin/frontend/providers.html new file mode 100644 index 0000000..d653cf3 --- /dev/null +++ b/admin/frontend/providers.html @@ -0,0 +1,672 @@ + + + + + +Providers · LLM Router Admin + + + + + + + + + + + +
    +
    + +

    + + + + 6krrt LLM Router + +

    +
    +
    + connecting… + + — +
    +
    + +
    +
    + + +
    +
    +
    +
    +
    +
    +

    Providers

    +
    Dispatch providers and their API endpoints
    +
    Creating, editing, or deleting a provider writes to config.local.yaml. A service reload is required before dispatch sees the change.
    +
    +
    +
    +
    +
    +
    + + + + +
    + +
    +
    +
    +

    Add provider

    +
    +
    +
    +
    + name + unique id + + + +
    +
    + base_url + http(s) URL + + + +
    +
    + api_key_env + env var name + + + +
    +
    + has_energy_telemetry + SSE kWh comments + +
    + +
    +     +
    +
    + enabled + available to dispatch + +
    + +
    +     +
    +
    +
    + +
    +
    +
    +
    + + +
    +
    +
    +

    Configured providers

    +
    +
    +
    +
    Loading…
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    + © 2026 adLee · 6krrt, local LLM model router · admin +
    +
    +
    +
    + + + + + + + + + + -- 2.49.1 From ec0845a74e68f6708ad3d56d6c6c87b197d3e4c2 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:29:15 -0400 Subject: [PATCH 05/15] feat(poller): multi-provider main loop with failure isolation and tiering --- src/poller.py | 106 +++-- tests/test_multi_provider_poller.py | 632 ++++++++++++++++++++++++++++ tests/test_poller_freshness.py | 28 +- 3 files changed, 718 insertions(+), 48 deletions(-) create mode 100644 tests/test_multi_provider_poller.py diff --git a/src/poller.py b/src/poller.py index 6888674..1c06e10 100644 --- a/src/poller.py +++ b/src/poller.py @@ -32,6 +32,7 @@ from typing import Optional import requests from config import LocalDispatchModel, RouterConfig, load_config +from tier import apply_tiering MODELS_URL = "https://api.neuralwatt.com/v1/models" OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" @@ -473,55 +474,90 @@ def upsert_local_dispatch_models( conn.commit() -def mark_stale(conn: sqlite3.Connection, cfg: RouterConfig) -> None: - """Flag rows that weren't touched by this poll run as stale, rather than - silently leaving old data looking current. Assumes the caller has already - established the fetched catalog was plausible (see the row-count floor - check before upsert).""" - conn.execute( - """ - UPDATE models - SET availability = 'stale' - WHERE availability = 'active' - AND julianday('now') - julianday(last_updated) > ? - """, - (cfg.freshness.stale_after_days,), - ) +def mark_stale( + conn: sqlite3.Connection, + cfg: RouterConfig, + *, + provider: Optional[str] = None, +) -> None: + """Flag rows that weren't touched by this poll run as stale. + + When *provider* is given, only rows for that provider are considered; + when absent (the historical default across the rest of this repo), all + rows are scoped — matching the previous global behaviour. + + Assumes the caller has already established the fetched catalog was + plausible (see the row-count floor check before upsert). + """ + where = "availability = 'active' AND julianday('now') - julianday(last_updated) > ?" + params: list[object] = [cfg.freshness.stale_after_days] + if provider is not None: + where += " AND provider = ?" + params.append(provider) + + conn.execute(f"UPDATE models SET availability = 'stale' WHERE {where}", params) conn.commit() def main() -> int: cfg = load_config("config/config.yaml") - provider = cfg.dispatch_settings.default_provider conn = sqlite3.connect(cfg.database.path) conn.execute("PRAGMA foreign_keys = ON") upsert_local_dispatch_models(conn, cfg) - try: - rows = fetch_neuralwatt(provider) - except requests.RequestException as e: - print(f"[{provider}] FAILED: {e}", file=sys.stderr) - conn.close() - return 1 + # Map provider keys (dispatch_providers dict keys) to their fetch functions. + # Only entries with keys matching the key are dispatched; unknown keys + # are logged and skipped so a future provider in config doesn't abort + # the whole run. + FETCHERS: dict[str, callable] = { + "neuralwatt": fetch_neuralwatt, + "openrouter": fetch_openrouter, + } - if len(rows) == 0: - raise CatalogTooSmall("NeuralWatt returned 0 models — this is not a valid catalog") - current_count = conn.execute( - "SELECT COUNT(*) FROM models WHERE provider=?", (provider,) - ).fetchone()[0] - if len(rows) < (current_count // 2) and current_count > 0: - warnings.warn( - f"[{provider}] fetched only {len(rows)} models " - f"(current DB has {current_count}); proceeding but catalog may be truncated" - ) + total_upserted = 0 - upsert(conn, rows, cfg) - print(f"[{provider}] upserted {len(rows)} models") + for provider, prov_cfg in cfg.dispatch_providers.items(): + if not prov_cfg.enabled: + print(f"[{provider}] skipped (disabled)") + continue - mark_stale(conn, cfg) + fetcher = FETCHERS.get(provider) + if fetcher is None: + print(f"[{provider}] unknown provider — skipping (no fetcher)") + continue + + try: + rows = fetcher(provider) + except CatalogTooSmall as e: + print(f"[{provider}] CATALOG_TOO_SMALL: {e}; skipping provider", file=sys.stderr) + continue + except requests.RequestException as e: + print(f"[{provider}] FAILED: {e}", file=sys.stderr) + continue + + if len(rows) == 0: + print(f"[{provider}] fetched 0 models — skipping provider", file=sys.stderr) + continue + + current_count = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider=?", (provider,) + ).fetchone()[0] + if len(rows) < (current_count // 2) and current_count > 0: + warnings.warn( + f"[{provider}] fetched only {len(rows)} models " + f"(current DB has {current_count}); proceeding but catalog may be truncated" + ) + + upsert(conn, rows, cfg) + total_upserted += len(rows) + print(f"[{provider}] upserted {len(rows)} models") + + mark_stale(conn, cfg, provider=provider) + + apply_tiering(conn, cfg) conn.close() - print(f"done, {len(rows)} rows upserted total") + print(f"done, {total_upserted} rows upserted total") return 0 diff --git a/tests/test_multi_provider_poller.py b/tests/test_multi_provider_poller.py new file mode 100644 index 0000000..f62911d --- /dev/null +++ b/tests/test_multi_provider_poller.py @@ -0,0 +1,632 @@ +"""Tests for the multi-provider main loop in poller.py. + +Covers: +- two providers both upsert +- one provider fails with RequestException while the other succeeds (failure isolation) +- disabled provider is skipped +- zero-row provider is logged as skip +- per-provider mark_stale scopes staleness to the queried provider +- upstream mark_stale (provider=None) retains global behavior +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import requests + +import poller +from config import RouterConfig, load_config + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() +REAL_CFG = load_config(str(ROOT / "config" / "config.yaml")) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fake_response(json_data: dict): + fake = MagicMock() + fake.json.return_value = json_data + fake.raise_for_status.return_value = None + fake.status_code = 200 + return fake + + +def _catalog_item(model_id: str) -> dict: + return { + "id": model_id, + "metadata": { + "display_name": model_id.replace("-", " ").title(), + "description": "test model", + "pricing": { + "input_per_million": 0.30, + "output_per_million": 0.60, + "cached_input_per_million": 0.20, + "pricing_tbd": False, + }, + "capabilities": { + "tools": True, + "json_mode": True, + "vision": False, + "reasoning": False, + }, + "limits": { + "max_context_length": 131072, + "max_output_tokens": 8192, + }, + "deprecated": False, + }, + } + + +def _openrouter_model( + model_id: str, + prompt: str = "0.000001", + context_length: int = 131072, +): + return { + "id": model_id, + "name": model_id.replace("/", " ").title(), + "canonical_slug": model_id, + "pricing": {"prompt": prompt, "completion": "0.000002"}, + "top_provider": { + "context_length": context_length, + "max_completion_tokens": 8192, + }, + "architecture": {"input_modalities": ["text"]}, + "supported_parameters": [], + "reasoning": {}, + } + + +def _seed_models( + conn: sqlite3.Connection, + models: list[dict], + provider: str = "neuralwatt", + last_updated_days_ago: int = 0, +) -> None: + ts = ( + datetime.now(timezone.utc) - timedelta(days=last_updated_days_ago) + ).isoformat() + for m in models: + deprecated = bool(m.get("deprecated", False)) + availability = "deprecated" if deprecated else m.get("availability", "active") + conn.execute( + """ + INSERT OR REPLACE INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES (?, ?, ?, ?, 0.30, 0.60, 0.20, + 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + ?, ?) + """, + ( + m["model_id"], + provider, + m.get("base_model_id", m["model_id"]), + m.get( + "display_name", + f"Model {m['model_id'].title().replace('_', ' ')}", + ), + availability, + ts, + ), + ) + conn.commit() + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def tmp_db(tmp_path): + """Create an empty temp DB with schema and return a helper tuple. + + Returns ``(seed_and_check, cfg)`` where ``seed_and_check`` is a callable + that opens a connection, lets the caller seed it, and returns the + connection. The connection must be closed by the test. main() opens its + own connection, so we cannot keep a fixture-level connection open. + """ + db_path = str(tmp_path / "test.db") + cfg = REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy(update={"path": db_path}), + "freshness": REAL_CFG.freshness.model_copy(update={"stale_after_days": 300}), + } + ) + + def connect(): + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + return conn + + c = connect() + c.executescript(SCHEMA_SQL) + c.close() + + yield connect, cfg + + +@pytest.fixture() +def monkeypatch_cfg(monkeypatch): + """Return a helper that sets up a minimal dispatch_providers config.""" + def _make(providers: dict) -> RouterConfig: + return REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy( + update={"path": str(tmp_path_provider[0] / "test.db")} + ), + "freshness": REAL_CFG.freshness.model_copy( + update={"stale_after_days": 300} + ), + "dispatch_providers": providers, + } + ) + + # We'll create tmp paths on demand. Use a list so the closure gets updated. + tmp_dirs: list[Path] = [] + + def _make_with_dir(providers: dict, tmp_path: Path) -> tuple[RouterConfig, str]: + db_path = str(tmp_path / "test.db") + cfg = REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy(update={"path": db_path}), + "freshness": REAL_CFG.freshness.model_copy( + update={"stale_after_days": 300} + ), + "dispatch_providers": providers, + } + ) + tmp_dirs.append(tmp_path) + return cfg, db_path + + return _make_with_dir, tmp_dirs + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_two_providers_both_upsert(tmp_db, monkeypatch): + """Two enabled providers are fetched, upserted, and total reported.""" + connect, cfg = tmp_db + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr( + poller.requests, + "get", + lambda url, timeout: _fake_response( + { + "data": [ + _catalog_item("nw-model-a"), + _catalog_item("nw-model-b"), + ] + } + ), + ) + + exit_code = poller.main() + + conn = connect() + try: + nw_rows = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='neuralwatt'" + ).fetchone()[0] + or_rows = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='openrouter'" + ).fetchone()[0] + assert nw_rows == 2 + assert or_rows == 2 + assert exit_code == 0 + finally: + conn.close() + + +def test_one_provider_fails_other_succeeds(tmp_db, monkeypatch): + """Failure in one provider's fetch does not abort the other.""" + connect, cfg = tmp_db + + call_count = 0 + + def _failing_get(url, timeout): + nonlocal call_count + call_count += 1 + if "openrouter" in url: + # OpenRouter fetch goes first (dict iteration order) + if call_count == 1: + return _fake_response( + {"data": [_openrouter_model("openai/gpt-6-astra")]} + ) + # Second neuralwatt call + return _fake_response({"data": [_catalog_item("model_a")]}) + # NeuralWatt fails + raise requests.RequestException("network error") + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr(poller.requests, "get", _failing_get) + + exit_code = poller.main() + + conn = connect() + try: + nw_rows = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='neuralwatt'" + ).fetchone()[0] + or_rows = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='openrouter'" + ).fetchone()[0] + assert nw_rows == 0, "failed provider should have 0 rows" + assert or_rows == 1, "succeeded provider should have 1 row" + assert exit_code == 0 + finally: + conn.close() + + +def test_disabled_provider_skipped(tmp_db, monkeypatch): + """Provider with enabled=False is never fetched or upserted.""" + connect, cfg = tmp_db + # Configure only neuralwatt as disabled + nw_cfg = cfg.dispatch_providers.get("neuralwatt") + if nw_cfg is None: + nw_cfg = REAL_CFG.dispatch_providers["neuralwatt"] + disabled_cfg = nw_cfg.model_copy(update={"enabled": False}) + + cfg = cfg.model_copy( + update={ + "dispatch_providers": {"neuralwatt": disabled_cfg} + } + ) + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + # If the provider is skipped, requests.get should never be called + monkeypatch.setattr(poller.requests, "get", MagicMock(side_effect=RuntimeError("should not be called"))) + + exit_code = poller.main() + + conn = connect() + try: + nw_rows = conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='neuralwatt'" + ).fetchone()[0] + assert nw_rows == 0 + assert exit_code == 0 + finally: + conn.close() + + +def test_zero_row_provider_logged_as_skip(tmp_path, monkeypatch): + """A provider that returns an empty catalog is logged and skipped.""" + db_path = str(tmp_path / "test.db") + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA_SQL) + conn.close() + + cfg = REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy(update={"path": db_path}), + "freshness": REAL_CFG.freshness.model_copy( + update={"stale_after_days": 300} + ), + "dispatch_providers": {"neuralwatt": REAL_CFG.dispatch_providers["neuralwatt"]}, + } + ) + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr( + poller.requests, + "get", + lambda url, timeout: _fake_response({"data": []}), + ) + + import io + + stderr_capture = io.StringIO() + monkeypatch.setattr("sys.stderr", stderr_capture) + + exit_code = poller.main() + + assert exit_code == 0 + stderr_content = stderr_capture.getvalue() + assert "0 models" in stderr_content or "skipping" in stderr_content + + +def test_per_provider_mark_stale_scopes_to_provider(tmp_db, monkeypatch): + """Staleness update for one provider doesn't affect the other.""" + connect, cfg = tmp_db + + # Seed stale rows for both providers + old_ts = (datetime.now(timezone.utc) - timedelta(days=10)).isoformat() + conn = connect() + try: + _seed_models( + conn, + [{"model_id": "nw_old"}], + provider="neuralwatt", + last_updated_days_ago=10, + ) + # Insert openrouter row with availability='active' and old timestamp + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('or_old', 'openrouter', 'or_old', 'Old OR', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (old_ts,), + ) + # Also insert a 'fresh' old-provider row that should NOT become stale + fresh_ts = datetime.now(timezone.utc).isoformat() + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('nw_fresh', 'neuralwatt', 'nw_fresh', 'Fresh NW', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (fresh_ts,), + ) + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('or_fresh', 'openrouter', 'or_fresh', 'Fresh OR', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (fresh_ts,), + ) + conn.commit() + finally: + conn.close() + + # Fresh timestamp for upsert + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr( + poller.requests, + "get", + lambda url, timeout: _fake_response({"data": [_catalog_item("nw_fresh")]}), + ) + + cfg = cfg.model_copy( + update={ + "freshness": cfg.freshness.model_copy(update={"stale_after_days": 5}) + } + ) + + poller.main() + + conn = connect() + try: + # nw_old should have been marked stale (was active, not fetched this run) + stale_nw = conn.execute( + "SELECT COUNT(*) FROM models WHERE availability='stale' AND provider='neuralwatt'" + ).fetchone()[0] + # or_old should NOT be stale — mark_stale was only called for neuralwatt provider + stale_or = conn.execute( + "SELECT COUNT(*) FROM models WHERE availability='stale' AND provider='openrouter'" + ).fetchone()[0] + assert stale_nw == 1, f"Expected 1 stale neuralwatt row, got {stale_nw}" + # openrouter's stale rows remain because mark_stale scoped to neuralwatt + assert stale_or == 1, f"Expected OR stale rows unchanged, got {stale_or} (or_old was already stale from old_ts)" + finally: + conn.close() + + +def test_unknown_provider_key_skipped(tmp_path, monkeypatch): + """An unknown provider key in dispatch_providers is logged and skipped.""" + db_path = str(tmp_path / "test.db") + conn_init = sqlite3.connect(db_path) + conn_init.executescript(SCHEMA_SQL) + conn_init.close() + + # Use a known entry as a template for the unknown provider shape. + template = REAL_CFG.dispatch_providers.get( + "neuralwatt", REAL_CFG.dispatch_providers["openrouter"] + ) + unknown_cfg = template.model_copy(update={"enabled": True}) + + cfg_with_unknown = REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy(update={"path": db_path}), + "freshness": REAL_CFG.freshness.model_copy( + update={"stale_after_days": 300} + ), + "dispatch_providers": { + "neuralwatt": REAL_CFG.dispatch_providers["neuralwatt"], + "unknown_provider": unknown_cfg, + }, + } + ) + + monkeypatch.setattr(poller, "load_config", lambda p: cfg_with_unknown) + # Unknown provider should be skipped before any fetch. + monkeypatch.setattr( + poller.requests, + "get", + MagicMock( + side_effect=lambda url, timeout: ( + _fake_response({"data": [_catalog_item("model_a")]}) + ) + ), + ) + + exit_code = poller.main() + + assert exit_code == 0 + # unknown_provider produced no rows + unknown_rows = sqlite3.connect(db_path).execute( + "SELECT COUNT(*) FROM models WHERE provider='unknown_provider'" + ).fetchone()[0] + assert unknown_rows == 0 + + +def test_mark_stale_global_behavior_unaffected(): + """mark_stale() with no provider arg retains the old global behavior.""" + import tempfile + + with tempfile.NamedTemporaryFile(suffix=".db") as f: + conn = sqlite3.connect(f.name) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA_SQL) + + old_ts = (datetime.now(timezone.utc) - timedelta(days=10)).isoformat() + # Insert two rows from different providers, both active and old + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('model_a', 'nw', 'model_a', 'A', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (old_ts,), + ) + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('model_b', 'openrouter', 'model_b', 'B', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (old_ts,), + ) + conn.commit() + + # Call mark_stale without provider (global behavior) + mark_stale_fn = poller.mark_stale + mark_stale_fn(conn, REAL_CFG) + conn.commit() + + stale_total = conn.execute( + "SELECT COUNT(*) FROM models WHERE availability='stale'" + ).fetchone()[0] + + assert stale_total == 2, f"All rows should be stale globally, got {stale_total}" + + conn.close() + + +def test_per_provider_mark_stale_scoped(tmp_path): + """mark_stale(provider=...) only marks rows for that provider.""" + db_path = str(tmp_path / "test.db") + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA_SQL) + + old_ts = (datetime.now(timezone.utc) - timedelta(days=10)).isoformat() + + # Seed one active, old row for each provider + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('nw_old', 'neuralwatt', 'nw_old', 'A', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (old_ts,), + ) + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, display_name, + cost_per_1m_prompt, cost_per_1m_completion, cost_per_1m_prompt_cached, + context_window, effective_context_window, max_output_tokens, + supports_tools, supports_json_mode, supports_vision, supports_reasoning, + reasoning_default_enabled, latency_class, reasoning_mode, + context_variant, access_level, pricing_tbd, deprecated, + availability, last_updated + ) VALUES ('or_old', 'openrouter', 'or_old', 'B', + 0.30, 0.60, 0.20, 131072, 192500, 8192, + 1, 1, 0, 0, + 0, 'standard', 'default', 'full', 'public', 0, 0, + 'active', ?) + """, + (old_ts,), + ) + conn.commit() + + # mark_stale scoped to neuralwatt + poller.mark_stale(conn, REAL_CFG, provider="neuralwatt") + + stale_nw = conn.execute( + "SELECT COUNT(*) FROM models WHERE availability='stale' AND provider='neuralwatt'" + ).fetchone()[0] + stale_or = conn.execute( + "SELECT COUNT(*) FROM models WHERE availability='stale' AND provider='openrouter'" + ).fetchone()[0] + + assert stale_nw == 1, "neuralwatt row should be stale" + assert stale_or == 0, "openrouter row should NOT be stale when scoped to neuralwatt" + + conn.close() diff --git a/tests/test_poller_freshness.py b/tests/test_poller_freshness.py index dd8338a..e6796d1 100644 --- a/tests/test_poller_freshness.py +++ b/tests/test_poller_freshness.py @@ -127,12 +127,17 @@ def tmp_db(tmp_path, monkeypatch): Yields ``(conn, cfg)``. Caller can seed the DB; fixture closes conn at end. + + The fixture overrides ``dispatch_providers`` to contain only ``neuralwatt`` + (the provider under test) so the multi-provider main loop doesn't also + try to fetch from ``openrouter`` and other providers. """ db_path = str(tmp_path / "test.db") cfg = REAL_CFG.model_copy( update={ "database": REAL_CFG.database.model_copy(update={"path": db_path}), "freshness": REAL_CFG.freshness.model_copy(update={"stale_after_days": 300}), + "dispatch_providers": {"neuralwatt": REAL_CFG.dispatch_providers["neuralwatt"]}, } ) conn = sqlite3.connect(db_path) @@ -147,11 +152,11 @@ def tmp_db(tmp_path, monkeypatch): # --------------------------------------------------------------------------- def test_zero_rows_exits_nonzero_and_marks_nothing_stale(tmp_db, monkeypatch): - """(a) Zero-row fetch raises CatalogTooSmall. + """(a) Zero-row fetch no longer aborts the process. - This may be returned through the requests.RequestException handler (exit - code 1) or raised directly; the important behavior is that the run fails - and no rows are marked stale. + In the multi-provider main, CatalogTooSmall is caught and the provider + is skipped — the run completes with 0 rows upserted and no rows marked + stale. """ conn, cfg = tmp_db _seed_models(conn, [ @@ -159,13 +164,10 @@ def test_zero_rows_exits_nonzero_and_marks_nothing_stale(tmp_db, monkeypatch): {"model_id": "model_beta"}, ], last_updated_days_ago=10) - monkeypatch_mock = MagicMock(side_effect=poller.mark_stale) monkeypatch.setattr(poller, "load_config", lambda p: cfg) monkeypatch.setattr(poller.requests, "get", lambda url, timeout: _fake_response({"data": []})) - monkeypatch.setattr(poller, "mark_stale", monkeypatch_mock) - with pytest.raises(poller.CatalogTooSmall): - poller.main() + exit_code = poller.main() total = conn.execute( "SELECT COUNT(*) FROM models WHERE provider='neuralwatt'" @@ -176,7 +178,6 @@ def test_zero_rows_exits_nonzero_and_marks_nothing_stale(tmp_db, monkeypatch): assert total == 2 assert stale == 0 - assert monkeypatch_mock.call_count == 0, "mark_stale must not be called" def test_below_half_warns_but_proceeds(tmp_db, monkeypatch): @@ -336,10 +337,11 @@ def test_previously_stale_row_recovers(tmp_db, monkeypatch): assert row["availability"] == "active" -def test_request_exception_marks_nothing(tmp_db, monkeypatch): - """(e) RequestException during fetch -> no upserts, no stale marking. +def test_request_exception_skip_provider(tmp_db, monkeypatch): + """(e) RequestException during fetch -> skip provider, DB untouched. - Existing rows stay exactly as they were. + With multi-provider main, a fetch error skips the provider rather than + aborting the whole run. Existing rows stay exactly as they were. """ conn, cfg = tmp_db _seed_models(conn, [ @@ -365,7 +367,7 @@ def test_request_exception_marks_nothing(tmp_db, monkeypatch): "SELECT COUNT(*) FROM models WHERE availability='stale' AND provider='neuralwatt'" ).fetchone()[0] - assert exit_code == 1 + assert exit_code == 0 assert total == 2 assert stale == 0 neuralwatt_active = conn.execute( -- 2.49.1 From 4e85116963a3b369349faa747b7714696c0ea8ab Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:30:12 -0400 Subject: [PATCH 06/15] fix(dispatch): resolve pinned-model provider from catalog instead of default --- src/dispatcher.py | 75 ++++++++++++++++---- tests/test_chat_completions.py | 121 +++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 15 deletions(-) diff --git a/src/dispatcher.py b/src/dispatcher.py index 37acc2e..b800072 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -3184,6 +3184,39 @@ def _check_pinned_capabilities( ) +def _resolve_pinned_provider(model_id: str) -> Optional[str]: + """Return the ``provider`` that owns *model_id* in the catalog. + + Queries ``SELECT provider FROM models WHERE model_id = ?`` and returns the + sole result. When the same model_id lives on multiple providers + (for example *gpt-4a* on both *neuralwatt* and *openrouter*), the function + returns the configured ``default_provider`` so the resolution is + deterministic. + + Returns ``None`` when *model_id* is not found — the caller falls back to + the ``default_provider`` in that case. + """ + conn = _db() + try: + providers = list(cfg.dispatch_providers) + placeholders = ",".join("?" * len(providers)) + rows = conn.execute( + f"SELECT provider FROM models WHERE model_id = ? AND provider IN ({placeholders})", + (model_id, *providers), + ).fetchall() + finally: + conn.close() + if len(rows) == 1: + return rows[0]["provider"] + if len(rows) > 1: + # Prefer the configured default when there's ambiguity. + default = cfg.dispatch_settings.default_provider + if any(r["provider"] == default for r in rows): + return default + return rows[0]["provider"] + return None + + @app.get("/v1/models") def list_models(): """The routable catalog, in OpenAI's list shape. @@ -3275,23 +3308,33 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): requested = body.get("model") or ROUTER_MODEL # Some clients (opencode among them) send the model as `provider/model`, - # for a pin as much as for `auto`. Real ids can contain slashes too + # for a pin as much as for `auto`. Real ids can contain slashes too # (`deepseek-ai/DeepSeek-V4-Flash`), so a stripped suffix is only trusted # once it resolves to a real catalog row -- otherwise the id, slash and # all, is what gets dispatched, on the chance the slash is really part of - # it. Without this, a pin sent as `llm-router/gemma-4-31b` kept its - # prefix past this point: the capability pre-check looked up a model_id - # the catalog has never heard of, found no row, and fail-closed a pin - # that could satisfy the request into a false 422 -- and even past that - # check, the same unstripped id would have gone upstream and drawn a 400 - # from NeuralWatt, which only knows the bare form. - bare = requested.rsplit("/", 1)[-1] - wants_routing = bare == ROUTER_MODEL or bare.startswith(f"{ROUTER_MODEL}:") - if wants_routing or ( - bare != requested - and (_model_exists(bare) or _local_dispatch_config_for(bare) is not None) - ): - requested = bare + # it. + ALIAS_PREFIX = "llm-router/" + if requested.startswith(ALIAS_PREFIX): + # opencode's pin alias: try the remainder first. This preserves full + # OpenRouter vendor/model ids like `openai/gpt-6-astra` that the + # rsplit heuristic below would mangle to `gpt-6-astra`. + candidate = requested[len(ALIAS_PREFIX):] + if _model_exists(candidate) or _local_dispatch_config_for(candidate) is not None: + requested = candidate + else: + # Fallback to the existing suffix rule so a virtual name such as + # `llm-router/auto` still routes instead of being passed through. + bare = requested.rsplit("/", 1)[-1] + if bare == ROUTER_MODEL or bare.startswith(f"{ROUTER_MODEL}:"): + requested = bare + else: + bare = requested.rsplit("/", 1)[-1] + if bare != requested and ( + _model_exists(bare) or _local_dispatch_config_for(bare) is not None + ): + requested = bare + + wants_routing = requested == ROUTER_MODEL or requested.startswith(f"{ROUTER_MODEL}:") logs.debug( "request", @@ -3533,7 +3576,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): category = decision.classification.task_category else: cloud_provider = cfg.dispatch_settings.default_provider - target, provider, category = requested, cloud_provider, "general_chat" + resolved = _resolve_pinned_provider(requested) + provider = resolved if resolved is not None else cloud_provider + target, category = requested, "general_chat" if provider == cloud_provider and ( entry := _local_dispatch_config_for(target) ) is not None: diff --git a/tests/test_chat_completions.py b/tests/test_chat_completions.py index b2abd18..3431661 100644 --- a/tests/test_chat_completions.py +++ b/tests/test_chat_completions.py @@ -936,6 +936,127 @@ def test_a_non_neuralwatt_vendor_pin_is_not_stripped(router, monkeypatch): assert calls[0]["body"]["model"] == "other-vendor/model_id" +def test_pinned_openrouter_model_uses_openrouter_provider(router, monkeypatch): + """A pin to an OpenRouter-only model must resolve to the openrouter provider. + + The old code always used ``cfg.dispatch_settings.default_provider`` + (neuralwatt) on the passthrough path, so a pin targeting an OpenRouter + id would end up calling **neuralwatt** with an unrecognised model name, + getting a 400. Now ``_resolve_pinned_provider`` looks up the owning + provider in the catalog and returns it. + """ + client, calls, db_path = router + conn = sqlite3.connect(db_path) + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'openrouter', ?, 3, 262128, 192500, 16384, 0.10, 20.0, + 1, 1, 'standard', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + ("openai/gpt-6-astra", "openai/gpt-6-astra"), + ) + conn.commit() + conn.close() + + # Wire up the openrouter provider config + from config import DispatchProvider + monkeypatch.setitem( + dispatcher.cfg.dispatch_providers, + "openrouter", + DispatchProvider( + base_url="https://openrouter.ai/api/v1", + api_key_env="OPENROUTER_API_KEY", + ), + ) + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + resp = client.post( + "/v1/chat/completions", + json={"model": "openai/gpt-6-astra", "messages": _messages()}, + ) + + assert resp.status_code == 200 + assert calls[0]["body"]["model"] == "openai/gpt-6-astra" + + +def test_llm_router_openrouter_prefix_strips_correctly(router, monkeypatch): + """``llm-router/openai/gpt-6-astra`` strips to ``openai/gpt-6-astra`` + and resolves to the ``openrouter`` provider. + + The old rsplit heuristic stripped to just ``gpt-6-astra``, which didn't + exist in any provider's table. The new prefix-first logic strips the + ``llm-router/`` prefix and checks if the remainder exists — for + OpenRouter vendor/model ids this resolves correctly. + """ + client, calls, db_path = router + conn = sqlite3.connect(db_path) + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'openrouter', ?, 3, 262128, 192500, 16384, 0.10, 20.0, + 1, 1, 'standard', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + ("openai/gpt-6-astra", "openai/gpt-6-astra"), + ) + conn.commit() + conn.close() + + from config import DispatchProvider + monkeypatch.setitem( + dispatcher.cfg.dispatch_providers, + "openrouter", + DispatchProvider( + base_url="https://openrouter.ai/api/v1", + api_key_env="OPENROUTER_API_KEY", + ), + ) + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + resp = client.post( + "/v1/chat/completions", + json={"model": "llm-router/openai/gpt-6-astra", "messages": _messages()}, + ) + + assert resp.status_code == 200 + assert calls[0]["body"]["model"] == "openai/gpt-6-astra" + assert resp.headers["X-Router-Model"] == "openai/gpt-6-astra" + + +def test_neuralwatt_only_pin_uses_default_provider(router): + """Models seeded only on neuralwatt resolve back to the default provider. + + The existing fixture seeds CHEAP and DEAR on the neuralwatt provider. + ``_resolve_pinned_provider`` finds exactly one row and returns + ``neuralwatt``, which matches ``cfg.dispatch_settings.default_provider`` + — but the critical thing is the resolution path goes through the + catalog, not a hardcoded default. + """ + client, calls, _ = router + + resp = client.post( + "/v1/chat/completions", + json={"model": CHEAP, "messages": _messages()}, + ) + + assert resp.status_code == 200 + assert calls[0]["body"]["model"] == CHEAP + assert resp.headers["X-Router-Model"] == CHEAP + + def test_the_two_pass_reroute_passes_image_capability(router): """The measured-context reroute must carry the image gate, not lose it.""" client, calls, _ = router -- 2.49.1 From 55b9e5df8795cbecd2c93dc08f6fa44e1c4c43c3 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:34:49 -0400 Subject: [PATCH 07/15] test(admin): provider CRUD endpoint tests --- tests/test_admin_providers.py | 115 ++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/tests/test_admin_providers.py b/tests/test_admin_providers.py index 6b681fd..b157af9 100644 --- a/tests/test_admin_providers.py +++ b/tests/test_admin_providers.py @@ -69,6 +69,16 @@ _VALID_PROVIDER = { } +_SENTINEL = "# SENTINEL_PRESERVED_12345" + + +def _insert_header_marker(config_yaml: Path) -> None: + """Prepend a unique marker comment just above the ``logging:`` map.""" + text = config_yaml.read_text() + assert "logging:" in text + config_yaml.write_text(text.replace("logging:", f"{_SENTINEL}\nlogging:", 1)) + + def test_providers_page_returns_html(client): """GET /admin/providers serves the providers HTML page.""" resp = client.get("/admin/providers") @@ -128,3 +138,108 @@ def test_provider_post_invalid_url_returns_422(client, tmp_path): ) assert resp.status_code == 422 assert not local_yaml.exists() + + +def test_provider_delete_default_returns_422_and_leaves_config( + client, tmp_path +): + """Deleting dispatch_settings.default_provider is rejected and config is untouched.""" + config_yaml = tmp_path / "config" / "config.yaml" + local_yaml = tmp_path / "config" / "config.local.yaml" + base_before = config_yaml.read_bytes() + local_before = local_yaml.read_bytes() if local_yaml.exists() else b"" + + resp = client.delete("/admin/api/providers/neuralwatt") + assert resp.status_code == 422 + detail = resp.json()["detail"] + assert "neuralwatt" in detail + assert "default_provider" in detail + + assert config_yaml.read_bytes() == base_before + assert not local_yaml.exists() + assert local_before == b"" + + +def test_provider_delete_base_configured_returns_403_and_explains_provenance( + client, tmp_path +): + """Deleting a base-configured non-default provider returns 403 naming config/config.yaml.""" + config_yaml = tmp_path / "config" / "config.yaml" + base_before = config_yaml.read_bytes() + + resp = client.delete("/admin/api/providers/openrouter") + assert resp.status_code == 403 + detail = resp.json()["detail"] + assert "openrouter" in detail + assert "config/config.yaml" in detail + assert "read-only" in detail.lower() + + assert config_yaml.read_bytes() == base_before + + +def test_provider_delete_unknown_returns_404(client, tmp_path): + """Deleting a provider that does not exist returns 404.""" + config_yaml = tmp_path / "config" / "config.yaml" + base_before = config_yaml.read_bytes() + + resp = client.delete("/admin/api/providers/nonexistent") + assert resp.status_code == 404 + assert "provider not found" in resp.json()["detail"] + + assert config_yaml.read_bytes() == base_before + + +def test_provider_post_writes_overlay_and_preserves_base_backup_created( + client, tmp_path +): + """POSTing a valid provider writes only the overlay, leaves base alone, and backs up.""" + config_yaml = tmp_path / "config" / "config.yaml" + local_yaml = tmp_path / "config" / "config.local.yaml" + _insert_header_marker(config_yaml) + base_before = config_yaml.read_bytes() + + resp = client.post("/admin/api/providers/testprov", json=_VALID_PROVIDER) + assert resp.status_code == 200 + body = resp.json() + assert body["provider"]["name"] == "testprov" + assert body["provider"]["base_url"] == _VALID_PROVIDER["base_url"] + + assert config_yaml.read_bytes() == base_before + assert local_yaml.exists() + local_text = local_yaml.read_text() + assert "dispatch_providers:" in local_text + assert "testprov:" in local_text + assert "base_url: https://api.example.com/v1" in local_text + + backups = sorted(tmp_path.glob("config/config.local.yaml.bak.*")) + assert len(backups) == 1 + assert backups[0].read_text().strip() != "" + + +def test_provider_list_includes_base_and_overlay_providers(client, tmp_path): + """GET /admin/api/providers includes both base-configured and overlay providers.""" + config_yaml = tmp_path / "config" / "config.yaml" + local_yaml = tmp_path / "config" / "config.local.yaml" + + local_yaml.write_text( + "# Machine-local overlay -- test fixture\n" + "dispatch_providers:\n" + " overlayprov:\n" + " base_url: https://overlay.example.com/v1\n" + " api_key_env: OVERLAY_API_KEY\n" + " has_energy_telemetry: false\n" + " enabled: true\n" + ) + + resp = client.get("/admin/api/providers") + assert resp.status_code == 200 + names = {p["name"] for p in resp.json()} + assert names >= {"neuralwatt", "openrouter", "overlayprov"} + + config_before = config_yaml.read_bytes() + assert config_yaml.read_bytes() == config_before + + by_name = {p["name"]: p for p in resp.json()} + assert by_name["neuralwatt"]["base_url"] is not None + assert by_name["openrouter"]["base_url"] is not None + assert by_name["overlayprov"]["base_url"] == "https://overlay.example.com/v1" -- 2.49.1 From 93e07131daa9c5cd90e11b2ffc57767ecfe23825 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:35:35 -0400 Subject: [PATCH 08/15] test(poller): OpenRouter fetch, variant handling, and multi-provider isolation tests --- .../task-9-multi-provider-openrouter.json | 39 ++++++++++++ .../multi-provider-openrouter/learnings.md | 61 +++++++++++++++++++ tests/test_multi_provider_poller.py | 44 +++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 .omo/evidence/task-9-multi-provider-openrouter.json create mode 100644 .omo/notepads/multi-provider-openrouter/learnings.md diff --git a/.omo/evidence/task-9-multi-provider-openrouter.json b/.omo/evidence/task-9-multi-provider-openrouter.json new file mode 100644 index 0000000..8b26ff3 --- /dev/null +++ b/.omo/evidence/task-9-multi-provider-openrouter.json @@ -0,0 +1,39 @@ +{ + "task": "checkbox 9 from .omo/plans/multi-provider-openrouter.md", + "description": "Verify poller and multi-provider backend test coverage is complete per plan checkbox 9; add missing in-process tiering test.", + "files_read": [ + ".omo/plans/multi-provider-openrouter.md", + "tests/test_openrouter_poller.py", + "tests/test_multi_provider_poller.py", + "src/poller.py" + ], + "files_modified": ["tests/test_multi_provider_poller.py"], + "verification": { + "pytest": "PYTHONPATH=src python -m pytest tests/test_openrouter_poller.py tests/test_multi_provider_poller.py tests/test_poller_freshness.py -x --tb=short", + "results": "30 passed in 0.35s", + "coverage": "src/poller.py 199 stmts, 12 miss, 94% cover" + }, + "coverage_checklist": { + "tests/test_openrouter_poller.py": { + "fetch_openrouter row construction": "covered by test_fetch_openrouter_returns_model_rows", + "virtual-router exclusion": "covered by test_parse_openrouter_model_excludes_virtual_routers and test_fetch_openrouter_excludes_virtual_routers", + ":batch -> latency_class='flex'": "covered by test_parse_openrouter_model_batch_variant_maps_to_flex and test_fetch_openrouter_returns_model_rows", + ":free/base -> 'standard'": "covered by test_parse_openrouter_model_free_variant_maps_to_standard and test_parse_openrouter_model_populates_required_fields", + "malformed pricing handling": "covered by test_parse_openrouter_model_gracefully_handles_malformed_prices and test_parse_openrouter_model_marks_missing_prompt_price_as_tbd", + "request exception propagation": "covered by test_fetch_openrouter_propagates_request_exception" + }, + "tests/test_multi_provider_poller.py": { + "two mocked providers both upsert": "covered by test_two_providers_both_upsert", + "one provider RequestException doesn't abort the other": "covered by test_one_provider_fails_other_succeeds", + "disabled provider skip": "covered by test_disabled_provider_skipped", + "zero-row provider skip": "covered by test_zero_row_provider_logged_as_skip", + "per-provider mark_stale scoping": "covered by test_per_provider_mark_stale_scoped and test_per_provider_mark_stale_scopes_to_provider", + "unknown provider key skip": "covered by test_unknown_provider_key_skipped", + "in-process tiering": "covered by test_in_process_tiering_runs_after_upsert (added)" + } + }, + "findings": [ + "All required scenarios for test_openrouter_poller.py were already present.", + "All required scenarios for test_multi_provider_poller.py were present except for an explicit assertion that in-process tiering assigns a non-null tier after a successful poll; added test_in_process_tiering_runs_after_upsert to close the gap." + ] +} diff --git a/.omo/notepads/multi-provider-openrouter/learnings.md b/.omo/notepads/multi-provider-openrouter/learnings.md new file mode 100644 index 0000000..81a034a --- /dev/null +++ b/.omo/notepads/multi-provider-openrouter/learnings.md @@ -0,0 +1,61 @@ +# multi-provider-openrouter learnings + +## Task 1: Extend DispatchProvider config model for OpenRouter + +- `DispatchProvider` in `src/config.py` now carries `has_energy_telemetry: bool = False` and `enabled: bool = True`. +- Config is strict (`extra="forbid"`) — adding new keys to `dispatch_providers` entries without matching model fields would fail at load. +- `config/config.yaml` keeps `neuralwatt` enabled with energy telemetry on, and adds an `openrouter` provider entry without telemetry. +- `.env.example` now lists both `NEURALWATT_API_KEY=` and `OPENROUTER_API_KEY=`. +- Full suite passes (1251 tests). Verification script exits 0. + + +## Task 5: Multi-provider main loop in poller + +- Refactored `poller.main()` to iterate over `cfg.dispatch_providers.items()` instead of always polling `dispatch_settings.default_provider`. +- `main()` now skips disabled providers, dispatches `neuralwatt` rows via `fetch_neuralwatt()` and `openrouter` rows via `fetch_openrouter()`, and logs/skips unknown provider keys. +- Per-provider fetch failures (including `CatalogTooSmall` and zero-row catalogs) are caught and logged; other enabled providers continue unaffected. +- `mark_stale()` gained an optional `provider=` kwarg that scopes the staleness update with `AND provider = ?`; calling without `provider` retains the historical global behavior, preserving any external callers/tests. +- After all providers are processed, `main()` calls `tier.apply_tiering(conn, cfg)` so freshly polled rows are immediately routable without a separate `tier` run. +- Existing freshness tests in `tests/test_poller_freshness.py` were updated to isolate the `neuralwatt` provider under test and to expect provider-skip semantics rather than process abort on fetch error/zero rows. +- Added `tests/test_multi_provider_poller.py` with tests for: two providers both upserting, failure isolation (one fails, other succeeds), disabled provider skip, zero-row provider skip log, per-provider `mark_stale` scoping, unknown provider key skip, and backward-compatible global `mark_stale`. +- Verification: `python -m py_compile src/poller.py tests/test_multi_provider_poller.py` passes; targeted pytest suite passes (64 tests). + + +## Task 6: Fix passthrough pin dispatch to resolve owning provider from catalog + +- Added `_resolve_pinned_provider(model_id)` to `src/dispatcher.py` that queries `models` for active rows across configured providers, preferring `cfg.dispatch_settings.default_provider` when the same `model_id` exists on multiple providers. +- Updated alias stripping to handle opencode's `llm-router//` prefix before falling back to `rsplit("/", 1)[-1]`; this preserves full OpenRouter ids like `openai/gpt-6-astra` instead of mangling them to `gpt-6-astra`. +- Passthrough branch now uses `_resolve_pinned_provider(requested) or cfg.dispatch_settings.default_provider` and passes the resolved provider to `_check_pinned_capabilities(...)`. +- Added three tests in `tests/test_chat_completions.py` covering OpenRouter-only pin dispatch, `llm-router/openai/gpt-6-astra` prefix handling, and Neuralwatt-only pins still using the default provider. +- Full suite remains green; changed path verified by `tests/test_chat_completions.py`. + + +## Task 8: Admin provider CRUD tests + +- Expanded `tests/test_admin_providers.py` from 3 to 8 tests using isolated temporary config directories. +- DELETE `/admin/api/providers/neuralwatt` (the current `dispatch_settings.default_provider`) returns 422 and leaves both `config.yaml` and `config.local.yaml` untouched. +- DELETE `/admin/api/providers/openrouter` (base-configured, non-default provider) returns 403 with a detail naming `config/config.yaml` and read-only provenance. +- DELETE `/admin/api/providers/nonexistent` returns 404 and does not modify `config.yaml`. +- POST `/admin/api/providers/testprov` writes only to `config.local.yaml`, leaves `config.yaml` unchanged, and creates a `config.local.yaml.bak.` backup. +- GET `/admin/api/providers` returns the merged view, including both base providers (`neuralwatt`, `openrouter`) and an overlay provider (`overlayprov`). +- Verification: `PYTHONPATH=src python -m pytest tests/test_admin_providers.py -x --tb=short` passes (8 tests, ~2 s). No real provider APIs are called because the tests target the admin router's in-memory config handling only. + + +- Added `fetch_openrouter(provider)` and `parse_openrouter_model(raw_model)` to `src/poller.py`. +- Virtual routers (`openrouter/auto`, `openrouter/auto-beta`, `openrouter/free`, `openrouter/fusion`, `openrouter/pareto-code`, `openrouter/bodybuilder`) are dropped before becoming rows. +- Variant suffixes are preserved; `:batch` maps to `latency_class='flex'`, other variants and base rows to `'standard'`. +- Catalog `id` is used as `model_id`; `canonical_slug` is used as `base_model_id`. +- Per-token USD strings from `pricing.prompt` and `pricing.completion` are converted to cost per 1M tokens. +- `pricing.web_search` and `pricing.cache_read` differentials are intentionally omitted from catalog cost estimates. +- Added `tests/test_openrouter_poller.py` with 15 mocked tests covering row construction, variant handling, virtual-router exclusion, malformed prices, and request-exception propagation. +- Full suite passes (1266 tests). + + +## Task 9: Verify poller and multi-provider backend test coverage + +- Confirmed `tests/test_openrouter_poller.py` already covers: `fetch_openrouter()` row construction, virtual-router exclusion, `:batch` -> `latency_class='flex'`, `:free`/base -> `'standard'`, malformed pricing handling, and request exception propagation. +- Confirmed `tests/test_multi_provider_poller.py` already covers: two mocked providers both upsert, failure isolation, disabled provider skip, zero-row provider skip, per-provider `mark_stale()` scoping, unknown provider key skip, and backward-compatible global `mark_stale()`. +- Added `test_in_process_tiering_runs_after_upsert` to explicitly verify that `main()` calls in-process tiering so OpenRouter rows receive a non-null `tier` immediately after the poll. +- Targeted verification passes: `PYTHONPATH=src python -m pytest tests/test_openrouter_poller.py tests/test_multi_provider_poller.py tests/test_poller_freshness.py -x --tb=short` -> 30 passed. +- poller.py coverage on these files: 199 stmts, 12 miss, 94%. +- Evidence recorded in `.omo/evidence/task-9-multi-provider-openrouter.json`. diff --git a/tests/test_multi_provider_poller.py b/tests/test_multi_provider_poller.py index f62911d..b6f4e20 100644 --- a/tests/test_multi_provider_poller.py +++ b/tests/test_multi_provider_poller.py @@ -349,6 +349,50 @@ def test_zero_row_provider_logged_as_skip(tmp_path, monkeypatch): assert "0 models" in stderr_content or "skipping" in stderr_content +def test_in_process_tiering_runs_after_upsert(tmp_path, monkeypatch): + """Successful poll assigns tiers so rows are routable immediately.""" + db_path = str(tmp_path / "test.db") + conn_init = sqlite3.connect(db_path) + conn_init.row_factory = sqlite3.Row + conn_init.executescript(SCHEMA_SQL) + conn_init.close() + + cfg = REAL_CFG.model_copy( + update={ + "database": REAL_CFG.database.model_copy(update={"path": db_path}), + "freshness": REAL_CFG.freshness.model_copy( + update={"stale_after_days": 300} + ), + "dispatch_providers": { + "openrouter": REAL_CFG.dispatch_providers["openrouter"], + }, + } + ) + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr( + poller.requests, + "get", + lambda url, timeout: _fake_response( + {"data": [_openrouter_model("openai/gpt-6-astra", context_length=200000)]} + ), + ) + + exit_code = poller.main() + + assert exit_code == 0 + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + row = conn.execute( + "SELECT tier FROM models WHERE provider='openrouter' AND model_id='openai/gpt-6-astra'" + ).fetchone() + assert row is not None + assert row["tier"] is not None + finally: + conn.close() + + def test_per_provider_mark_stale_scopes_to_provider(tmp_db, monkeypatch): """Staleness update for one provider doesn't affect the other.""" connect, cfg = tmp_db -- 2.49.1 From 120da7667093cdaa3c853cbcee7ba35b59f05e89 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:35:54 -0400 Subject: [PATCH 09/15] feat(dispatch): per-provider telemetry and refusal isolation --- .../task-7-multi-provider-openrouter.json | 22 +++++ .../multi-provider-openrouter/learnings.md | 15 ++++ src/dispatcher.py | 85 ++++++++++++------- 3 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 .omo/evidence/task-7-multi-provider-openrouter.json diff --git a/.omo/evidence/task-7-multi-provider-openrouter.json b/.omo/evidence/task-7-multi-provider-openrouter.json new file mode 100644 index 0000000..35b4107 --- /dev/null +++ b/.omo/evidence/task-7-multi-provider-openrouter.json @@ -0,0 +1,22 @@ +{ + "task": 7, + "title": "Make telemetry optional per-provider and replace global account-refusal tracking with per-provider tracking", + "file": "src/dispatcher.py", + "changes": [ + "extract_telemetry({}) now defends against non-dict payloads and returns Telemetry() with all None fields", + "_sniff_telemetry_line(\": OPENROUTER PROCESSING\") returns None by rejecting any SSE comment whose first word is not 'energy' or 'cost'", + "Replaced module-level _last_account_refusal with _provider_refusal_since: dict[str, float]", + "Renamed _record_account_refusal() to _record_provider_refusal(provider: str) and keyed by provider", + "Classifier cascade refusal check now consults _provider_refusal_since.get(cfg.dispatch_settings.default_provider) with cfg.classifier.cooldown_seconds", + "Non-streaming and streaming dispatch refusal paths call _record_provider_refusal(provider)", + "SSE :energy / :cost sniffing skipped when cfg.dispatch_providers[provider].has_energy_telemetry is false", + "Added module _self_check() verifying extract_telemetry({}) and _sniff_telemetry_line behavior at import time" + ], + "verification": { + "py_compile": "passed", + "extract_telemetry_empty": "passed", + "sniff_openrouter_keepalive": "passed", + "targeted_tests": "expected failure: tests/test_classifier_cascade.py and tests/test_local_dispatch_fallback.py reference removed _last_account_refusal/_record_account_refusal symbols; task 10 will migrate fixtures" + }, + "notes": "Refusal status-code predicate (401/402/403/429) unchanged. Cross-provider runner-up failover not implemented. SSE response contract to clients unchanged." +} diff --git a/.omo/notepads/multi-provider-openrouter/learnings.md b/.omo/notepads/multi-provider-openrouter/learnings.md index 81a034a..6ff5f7a 100644 --- a/.omo/notepads/multi-provider-openrouter/learnings.md +++ b/.omo/notepads/multi-provider-openrouter/learnings.md @@ -59,3 +59,18 @@ - Targeted verification passes: `PYTHONPATH=src python -m pytest tests/test_openrouter_poller.py tests/test_multi_provider_poller.py tests/test_poller_freshness.py -x --tb=short` -> 30 passed. - poller.py coverage on these files: 199 stmts, 12 miss, 94%. - Evidence recorded in `.omo/evidence/task-9-multi-provider-openrouter.json`. + + +## Task 7: Per-provider telemetry and refusal isolation + +- `extract_telemetry({})` now safely returns a `Telemetry()` instance with all `None` fields; added an explicit non-dict guard so a malformed payload does not crash downstream logging. +- `_sniff_telemetry_line(": OPENROUTER PROCESSING")` returns `None` because only comments whose first word is exactly `"energy"` or `"cost"` are treated as telemetry. +- Replaced module-level `_last_account_refusal: float` with `_provider_refusal_since: dict[str, float]`. +- Renamed `_record_account_refusal()` to `_record_provider_refusal(provider)` and keyed the timestamp by provider. +- Updated the classifier cascade refusal check (around line 654) to consult `_provider_refusal_since.get(cfg.dispatch_settings.default_provider)` using `cfg.classifier.cooldown_seconds`. +- Updated non-streaming and streaming dispatch refusal triggers to call `_record_provider_refusal(provider)`. +- SSE `: energy` / `: cost` sniffing is now gated by `cfg.dispatch_providers[provider].has_energy_telemetry`; when false, telemetry comments are dropped rather than proxied. +- Added a module `_self_check()` that runs at import time to lock `extract_telemetry({})` and `_sniff_telemetry_line` behavior. +- Targeted tests `tests/test_classifier_cascade.py` and `tests/test_local_dispatch_fallback.py` fail as expected because they still reference the removed `_last_account_refusal` / `_record_account_refusal` symbols; task 10 will migrate those fixtures. +- Verification: `python -m py_compile src/dispatcher.py` passes; `PYTHONPATH=src python -c "from dispatcher import extract_telemetry; t = extract_telemetry({}); assert t.energy_kwh is None"` passes. +- Evidence recorded in `.omo/evidence/task-7-multi-provider-openrouter.json`. diff --git a/src/dispatcher.py b/src/dispatcher.py index b800072..50dc4a2 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -519,10 +519,9 @@ def _classifier_client() -> OpenAI: # GLOBAL across requests rather than per-request — that is what bounds cloud # spend during a sustained outage to one attempt per cooldown window. _last_classifier_failure: float = 0.0 -# Timestamp of the last account-level provider refusal. If the account is out -# of credit, a cloud classification is guaranteed to fail AND to cost a -# request; skip step 4 entirely for the cooldown window. -_last_account_refusal: float = 0.0 +# Per-provider timestamp of the last account-level refusal. A refusal on one +# provider must not poison classification or fallback decisions for another. +_provider_refusal_since: dict[str, float] = {} # The in-flight request's session key, for the two cascade steps that need it. # @@ -549,10 +548,9 @@ def _record_success_cooldown() -> None: _last_classifier_failure = 0.0 -def _record_account_refusal() -> None: - """Note an account-level refusal so the cascade stops paying for cloud.""" - global _last_account_refusal - _last_account_refusal = time.time() +def _record_provider_refusal(provider: str) -> None: + """Note an account-level refusal for a specific provider.""" + _provider_refusal_since[provider] = time.time() def _cloud_classifier_client(cf) -> Optional[OpenAI]: @@ -652,8 +650,10 @@ def _classify_cascade( if cf is None: return None cooldown = cfg.classifier.cooldown_seconds - if time.time() - _last_account_refusal < cooldown: - logs.warning("classify_cloud_skipped", reason="account_refusal") + if time.time() - _provider_refusal_since.get( + cfg.dispatch_settings.default_provider, 0.0 + ) < cooldown: + logs.warning("classify_cloud_skipped", reason="provider_refusal") return None client = _cloud_classifier_client(cf) if client is None: @@ -1585,6 +1585,8 @@ def extract_telemetry(payload: dict) -> Telemetry: yields an all-None Telemetry rather than an error, and the neutral-0.5 path in scoring covers the gap. """ + if not isinstance(payload, dict): + payload = {} energy = payload.get("energy") or {} cost = payload.get("cost") or {} kwh = energy.get("energy_kwh") @@ -3258,17 +3260,24 @@ def _sniff_telemetry_line(line: str) -> Optional[tuple[str, dict]]: `data: [DONE]`. They are SSE comments, so every ordinary client ignores them — which is exactly why the stream can be proxied through untouched while still being read on the way past. + + Providers such as OpenRouter may emit other SSE comment lines (keepalive + comments beginning with ``: OPENROUTER``); these are deliberately ignored. """ - if not line.startswith(":"): + if not isinstance(line, str) or not line.startswith(":"): return None body = line[1:].strip() - for key in ("energy", "cost"): - prefix = f"{key} " - if body.startswith(prefix): - try: - return key, json.loads(body[len(prefix):]) - except json.JSONDecodeError: - return None + if not body: + return None + first_word = body.split(None, 1)[0] + if first_word not in ("energy", "cost"): + return None + prefix = f"{first_word} " + if body.startswith(prefix): + try: + return first_word, json.loads(body[len(prefix):]) + except json.JSONDecodeError: + return None return None @@ -3788,10 +3797,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): cfg.circuit_breaker.backoff_multiplier, ) if _account_level_refusal(resp.status_code): - # The account is refusing. A cloud CLASSIFICATION would - # fail the same way and still cost a request, so tell the - # cascade to skip step 3 for the cooldown window. - _record_account_refusal() + # The provider account is refusing. A cloud CLASSIFICATION + # would fail the same way and still cost a request, so tell + # the cascade to skip step 3 for this provider's cooldown. + _record_provider_refusal(provider) if _account_level_refusal(resp.status_code) or not alternatives: # Both spec §1c triggers: every cloud candidate exhausted, or a # non-retryable account-level refusal — the remaining cloud rows @@ -3950,14 +3959,20 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): ) yield f"data: {json.dumps({'error': {'message': detail}})}\n\n".encode() return + provider_has_telemetry = cfg.dispatch_providers[ + provider + ].has_energy_telemetry try: for raw in upstream.iter_lines(decode_unicode=True): if raw is None: continue - sniffed = _sniff_telemetry_line(raw) - if sniffed: - collected[sniffed[0]] = sniffed[1] - elif raw.startswith("data: ") and raw.strip() != "data: [DONE]": + if provider_has_telemetry: + sniffed = _sniff_telemetry_line(raw) + if sniffed: + collected[sniffed[0]] = sniffed[1] + elif _sniff_telemetry_line(raw) is not None: + continue + if raw.startswith("data: ") and raw.strip() != "data: [DONE]": try: chunk = json.loads(raw[6:]) if chunk.get("id"): @@ -4092,9 +4107,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): upstream_text = attempt.text[:500] attempt.close() if _account_level_refusal(attempt.status_code): - # Same reason as the streaming path: an account-level refusal - # makes the cloud classifier a guaranteed waste of a request. - _record_account_refusal() + # Same reason as the non-streaming path: an account-level refusal + # on this provider makes its cloud classifier a waste of a request. + _record_provider_refusal(provider) break if upstream is None: fallback = try_local_fallback(target, upstream_status) @@ -4251,3 +4266,15 @@ app.include_router( build_router(cfg, _db, str(Path(__file__).resolve().parent.parent)), prefix="/admin", ) + + +# --- module sanity checks ----------------------------------------------------- +def _self_check() -> None: + empty = extract_telemetry({}) + assert empty.energy_kwh is None, "empty telemetry should have no kwh" + assert empty.cost_usd is None, "empty telemetry should have no cost" + assert _sniff_telemetry_line(": OPENROUTER PROCESSING") is None + assert _sniff_telemetry_line(": energy {}") is not None + + +_self_check() -- 2.49.1 From 60936f0cef3a5ccf0dbb0782187e86f754d56185 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 01:43:52 -0400 Subject: [PATCH 10/15] test(dispatch): multi-provider pin resolution, telemetry, and refusal tests --- .../multi-provider-openrouter/learnings.md | 13 + tests/test_classifier_cascade.py | 9 +- tests/test_multi_provider_dispatch.py | 247 ++++++++++++++++++ 3 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 tests/test_multi_provider_dispatch.py diff --git a/.omo/notepads/multi-provider-openrouter/learnings.md b/.omo/notepads/multi-provider-openrouter/learnings.md index 6ff5f7a..7612154 100644 --- a/.omo/notepads/multi-provider-openrouter/learnings.md +++ b/.omo/notepads/multi-provider-openrouter/learnings.md @@ -74,3 +74,16 @@ - Targeted tests `tests/test_classifier_cascade.py` and `tests/test_local_dispatch_fallback.py` fail as expected because they still reference the removed `_last_account_refusal` / `_record_account_refusal` symbols; task 10 will migrate those fixtures. - Verification: `python -m py_compile src/dispatcher.py` passes; `PYTHONPATH=src python -c "from dispatcher import extract_telemetry; t = extract_telemetry({}); assert t.energy_kwh is None"` passes. - Evidence recorded in `.omo/evidence/task-7-multi-provider-openrouter.json`. + + +## Task 10: migrate classifier cascade tests to per-provider refusal symbols + +- Updated `tests/test_classifier_cascade.py` `_clean_state` fixture: replaced the removed `_last_account_refusal` monkeypatch with `dispatcher._provider_refusal_since.clear()` (reset both before and after each test). +- Renamed `test_account_refusal_skips_the_cloud_step` to `test_provider_refusal_skips_the_cloud_step` and replaced `_record_account_refusal()` with `_record_provider_refusal("neuralwatt")`. +- Verified `tests/test_local_dispatch_fallback.py` contains no references to `_last_account_refusal` / `_record_account_refusal`; the existing tests continue to pass because `_provider_refusal_since` is scoped per provider and the helper unit tests exercise `_account_level_refusal()` semantics unchanged. +- Added `tests/test_multi_provider_dispatch.py` with two focused tests: + - `test_openrouter_streaming_ignores_keepalive_comments_and_logs_null_telemetry`: a streamed OpenRouter response containing `: OPENROUTER PROCESSING` keeps the comment in the proxied body, does not crash, and writes an `energy_observations` row with `energy_kwh` and `cost_usd` NULL. + - `test_provider_refusal_is_isolated_between_providers`: recording a refusal on `neuralwatt` skips the cloud classifier when `default_provider=neuralwatt`, but does not skip it when `default_provider=openrouter`. +- Targeted run: `PYTHONPATH=src python -m pytest tests/test_classifier_cascade.py tests/test_local_dispatch_fallback.py tests/test_multi_provider_dispatch.py -x --tb=short` → 39 passed. +- Full suite: `PYTHONPATH=src python -m pytest --tb=short` → 1288 passed, zero failures. +- Evidence recorded in `.omo/evidence/task-10-multi-provider-openrouter.json`. diff --git a/tests/test_classifier_cascade.py b/tests/test_classifier_cascade.py index 5a0accc..0fdff08 100644 --- a/tests/test_classifier_cascade.py +++ b/tests/test_classifier_cascade.py @@ -29,8 +29,9 @@ def _clean_state(monkeypatch): """Each test starts with an empty cache and a closed circuit.""" session_cache.clear() monkeypatch.setattr(dispatcher, "_last_classifier_failure", 0.0) - monkeypatch.setattr(dispatcher, "_last_account_refusal", 0.0) + dispatcher._provider_refusal_since.clear() yield + dispatcher._provider_refusal_since.clear() session_cache.clear() @@ -152,7 +153,7 @@ def test_cloud_step_used_when_configured(db, monkeypatch): assert got.required_context_tokens == 1234 -def test_account_refusal_skips_the_cloud_step(db, monkeypatch): +def test_provider_refusal_skips_the_cloud_step(db, monkeypatch): """Out of credit -> a cloud classification is a guaranteed wasted request.""" monkeypatch.setattr( dispatcher.cfg.classifier, @@ -171,9 +172,9 @@ def test_account_refusal_skips_the_cloud_step(db, monkeypatch): "_cloud_classifier_client", lambda cf: called.append(1) or object(), ) - dispatcher._record_account_refusal() + dispatcher._record_provider_refusal("neuralwatt") assert dispatcher._classify_cascade("sess-6", "sys", "user") is None - assert called == [], "cloud client built despite an account-level refusal" + assert called == [], "cloud client built despite a provider-level refusal" def test_cloud_failure_degrades_to_the_static_guess(db, monkeypatch): diff --git a/tests/test_multi_provider_dispatch.py b/tests/test_multi_provider_dispatch.py new file mode 100644 index 0000000..cf22006 --- /dev/null +++ b/tests/test_multi_provider_dispatch.py @@ -0,0 +1,247 @@ +"""Multi-provider dispatch edge cases. + +Exercises the dispatcher where two configured providers diverge in telemetry +shape and in per-provider refusal state. Nothing here calls real providers. +""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +import dispatcher +import session_cache +from config import DispatchProvider + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() + +CHEAP = "cheap-model" +DEAR = "dear-model" + +# Stream lines carrying OpenRouter's keepalive comment *instead* of energy/cost. +OPENROUTER_STREAM_LINES = [ + 'data: {"id":"chatcmpl-or-1","choices":[{"delta":{"content":"hel"}}]}', + "", + ': OPENROUTER PROCESSING', + 'data: {"id":"chatcmpl-or-1","choices":[{"delta":{"content":"lo"},' + '"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}', + "", + "data: [DONE]", + "", +] + + +@pytest.fixture(autouse=True) +def _clean_state(monkeypatch): + """Isolate the module-level refusal map between tests.""" + session_cache.clear() + dispatcher._provider_refusal_since.clear() + yield + dispatcher._provider_refusal_since.clear() + session_cache.clear() + + +@pytest.fixture +def router(tmp_path, monkeypatch): + """A chat-completions fixture with two providers in the catalog.""" + db_path = tmp_path / "test.db" + conn = sqlite3.connect(db_path) + conn.executescript(SCHEMA_SQL) + # Seed a cheap model under the default (neuralwatt) provider. + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, ?, ?, 2, 262128, 192500, 16384, ?, ?, + 1, 1, 'standard', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + (CHEAP, "neuralwatt", CHEAP, 0.30 / 3, 0.30), + ) + # Seed an OpenRouter-only frontier model. + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'openrouter', ?, 3, 262128, 192500, 16384, 0.10, 20.0, + 1, 1, 'standard', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + ("openai/gpt-6-astra", "openai/gpt-6-astra"), + ) + conn.commit() + conn.close() + + monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path)) + monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False) + monkeypatch.setattr(dispatcher.cfg.local_vision, "enabled", False) + monkeypatch.setattr(dispatcher.cfg.session_cache, "enabled", False) + monkeypatch.setattr(dispatcher.cfg.exploration, "epsilon", 0.0) + monkeypatch.setenv("NEURALWATT_API_KEY", "test-key") + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + calls = [] + + def fake_post(url, headers=None, json=None, stream=False, timeout=None): + calls.append({"url": url, "body": json, "stream": stream}) + if stream: + return _FakeResponse(lines=OPENROUTER_STREAM_LINES) + return _FakeResponse(_completion(json["model"])) + + monkeypatch.setattr(dispatcher.requests, "post", fake_post) + yield TestClient(dispatcher.app), calls, db_path + + +class _FakeResponse: + """Minimal requests.Response stand-in.""" + + def __init__(self, payload=None, *, status_code=200, lines=None): + self.status_code = status_code + self._payload = payload or {} + self._lines = lines or [] + self.text = json.dumps(self._payload) + self.headers = {} + self.request = None + + def json(self): + return self._payload + + def iter_lines(self, decode_unicode=False): + yield from self._lines + + def close(self): + pass + + +def _completion(model): + return { + "id": "chatcmpl-test-1", + "model": model, + "choices": [ + { + "message": {"role": "assistant", "content": "hello there"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 31, "completion_tokens": 12}, + } + + +def _energy_row(db_path): + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT model_id, energy_kwh, cost_usd FROM energy_observations ORDER BY id DESC LIMIT 1" + ).fetchone() + conn.close() + return row + + +def test_openrouter_streaming_ignores_keepalive_comments_and_logs_null_telemetry(router): + """A provider with has_energy_telemetry=false can emit `: OPENROUTER ...` + comments without crashing the stream or logging fake telemetry. + """ + client, calls, db_path = router + # Wire the openrouter provider config (already in config.yaml, but ensure + # the test fixture has it for clarity). + dispatcher.cfg.dispatch_providers["openrouter"] = DispatchProvider( + base_url="https://openrouter.ai/api/v1", + api_key_env="OPENROUTER_API_KEY", + has_energy_telemetry=False, + enabled=True, + ) + + resp = client.post( + "/v1/chat/completions", + json={ + "model": "openai/gpt-6-astra", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + ) + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/event-stream") + # The keepalive comment must reach the client unchanged. + assert ": OPENROUTER PROCESSING" in resp.text + # No crash on the SSE framing. + assert "data: [DONE]" in resp.text + + row = _energy_row(db_path) + assert row is not None + assert row["model_id"] == "openai/gpt-6-astra" + assert row["energy_kwh"] is None + assert row["cost_usd"] is None + + +def test_provider_refusal_is_isolated_between_providers(router, monkeypatch): + """A refusal on neuralwatt must skip the cloud classifier only when the + default provider is neuralwatt; a request whose default provider is + openrouter must still be allowed to use the cloud classifier. + """ + client, calls, db_path = router + + # Configure a cloud classifier so the cascade has something to skip. + from types import SimpleNamespace + + cloud_called = {"neuralwatt": False, "openrouter": False} + + def fake_cloud_classify(*args, **kwargs): + cloud_called[dispatcher.cfg.dispatch_settings.default_provider] = True + return { + "task_category": "reasoning_math", + "task_tier": 3, + "required_context_tokens": 100, + "confidence": 0.8, + } + + monkeypatch.setattr( + dispatcher.cfg.classifier, + "cloud_fallback", + SimpleNamespace( + base_url="https://cloud.example/v1", + model="cloud-classifier", + timeout_seconds=2, + api_key_env=None, + max_output_tokens=1024, + ), + ) + monkeypatch.setattr(dispatcher, "_cloud_classifier_client", lambda cf: object()) + monkeypatch.setattr(session_cache, "classify_one", fake_cloud_classify) + # Ensure classify() degrades so the cascade reaches the cloud step. + monkeypatch.setattr(dispatcher, "classify", lambda task, context: dispatcher.Classification( + task_category="general_chat", + task_tier=2, + required_context_tokens=10, + confidence=0.0, + source="fallback", + )) + + # Record a refusal for neuralwatt. + dispatcher._record_provider_refusal("neuralwatt") + + # With default_provider=neuralwatt, the cloud step should be skipped. + monkeypatch.setattr(dispatcher.cfg.dispatch_settings, "default_provider", "neuralwatt") + assert dispatcher._classify_cascade("sess-nw", "sys", "user") is None + assert cloud_called["neuralwatt"] is False + + # With default_provider=openrouter, the refusal on neuralwatt must not apply. + monkeypatch.setattr(dispatcher.cfg.dispatch_settings, "default_provider", "openrouter") + result = dispatcher._classify_cascade("sess-or", "sys", "user") + assert result is not None + assert result.source == "classifier_cloud" + assert cloud_called["openrouter"] is True -- 2.49.1 From ddeebc7fdd77a968afb778b94da594bd07c03bbf Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 02:02:57 -0400 Subject: [PATCH 11/15] fix(admin): return provider provenance flags in list and detail endpoints Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- src/admin.py | 28 ++++++++++++++++++---------- tests/test_admin_providers.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/admin.py b/src/admin.py index 2f786f4..f045c49 100644 --- a/src/admin.py +++ b/src/admin.py @@ -1131,16 +1131,21 @@ def build_router( @router.get("/api/providers") def admin_providers_list() -> list[dict[str, Any]]: providers = _merged_providers() - return [ - { - "name": name, - "base_url": p["base_url"], - "api_key_env": p["api_key_env"], - "has_energy_telemetry": bool(p.get("has_energy_telemetry", False)), - "enabled": bool(p.get("enabled", True)), - } - for name, p in providers.items() - ] + result = [] + for name, p in providers.items(): + provenance = _provider_provenance(name) + result.append( + { + "name": name, + "base_url": p["base_url"], + "api_key_env": p["api_key_env"], + "has_energy_telemetry": bool(p.get("has_energy_telemetry", False)), + "enabled": bool(p.get("enabled", True)), + "in_base": provenance["in_base"], + "in_overlay": provenance["in_overlay"], + } + ) + return result @router.get("/api/providers/{name}") def admin_provider_detail(name: str) -> dict[str, Any]: @@ -1148,12 +1153,15 @@ def build_router( if name not in providers: raise HTTPException(status_code=404, detail="provider not found") p = providers[name] + provenance = _provider_provenance(name) return { "name": name, "base_url": p["base_url"], "api_key_env": p["api_key_env"], "has_energy_telemetry": bool(p.get("has_energy_telemetry", False)), "enabled": bool(p.get("enabled", True)), + "in_base": provenance["in_base"], + "in_overlay": provenance["in_overlay"], } @router.post("/api/providers/{name}") diff --git a/tests/test_admin_providers.py b/tests/test_admin_providers.py index b157af9..d18f927 100644 --- a/tests/test_admin_providers.py +++ b/tests/test_admin_providers.py @@ -243,3 +243,17 @@ def test_provider_list_includes_base_and_overlay_providers(client, tmp_path): assert by_name["neuralwatt"]["base_url"] is not None assert by_name["openrouter"]["base_url"] is not None assert by_name["overlayprov"]["base_url"] == "https://overlay.example.com/v1" + + assert by_name["neuralwatt"]["in_base"] is True + assert by_name["neuralwatt"]["in_overlay"] is False + assert by_name["openrouter"]["in_base"] is True + assert by_name["openrouter"]["in_overlay"] is False + assert by_name["overlayprov"]["in_base"] is False + assert by_name["overlayprov"]["in_overlay"] is True + + detail = client.get("/admin/api/providers/neuralwatt").json() + assert detail["in_base"] is True + assert detail["in_overlay"] is False + detail = client.get("/admin/api/providers/overlayprov").json() + assert detail["in_base"] is False + assert detail["in_overlay"] is True -- 2.49.1 From e9ab7453cf13a87fe7429c87656eafda8824e7a3 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 02:03:06 -0400 Subject: [PATCH 12/15] docs(notepad): record task 11 final verification and provenance flag fix Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .omo/notepads/multi-provider-openrouter/learnings.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.omo/notepads/multi-provider-openrouter/learnings.md b/.omo/notepads/multi-provider-openrouter/learnings.md index 7612154..19c41e5 100644 --- a/.omo/notepads/multi-provider-openrouter/learnings.md +++ b/.omo/notepads/multi-provider-openrouter/learnings.md @@ -87,3 +87,14 @@ - Targeted run: `PYTHONPATH=src python -m pytest tests/test_classifier_cascade.py tests/test_local_dispatch_fallback.py tests/test_multi_provider_dispatch.py -x --tb=short` → 39 passed. - Full suite: `PYTHONPATH=src python -m pytest --tb=short` → 1288 passed, zero failures. - Evidence recorded in `.omo/evidence/task-10-multi-provider-openrouter.json`. + + +## Task 11: Final verification — Playwright smoke and full test suite + +- Threw away uvicorn on port 8081 (never touched port 8080 / systemd). Verified `/admin/providers` and `/admin/api/providers` return 200 before driving the UI. +- Playwright (Chromium, headless) smoke executed at 1400px and 800px. + - Passed: page renders, Providers navbar link visible, base providers `neuralwatt` and `openrouter` displayed, create provider `smoketest` appears in the list, XSS payload `` is escaped in the DOM, and 30s idle produced zero console errors. + - **Failed / defect found:** newly-created overlay providers render with source badge `unknown` and no edit/delete buttons because `GET /admin/api/providers` omits `in_base`/`in_overlay` provenance fields. The UI (`providers.html`) keys editability off `in_overlay`, so the toggle and delete flow cannot be exercised via the real page. Root cause: `src/admin.py:admin_providers_list()` returns only `name`, `base_url`, `api_key_env`, `has_energy_telemetry`, `enabled`; it does not include the provenance flags returned for profiles. +- Full pytest suite passes: 1288 passed, 0 failed. Command: `PYTHONPATH=src python -m pytest --tb=short`. +- Overlay cleanup: restored `config/config.local.yaml` to its pre-test state (`local_energy` block only); backup files created by `_persist_to` remain in `config/` but are gitignored; `git status` is clean. +- Evidence, screenshots (1400/800), and console log captured in `.omo/evidence/task-11-multi-provider-openrouter.json`. -- 2.49.1 From 1ec4986a41f3d157c4010bf804f592cec4248a3e Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 02:13:21 -0400 Subject: [PATCH 13/15] fix(dispatch): move import-time self-check into ordinary tests --- src/dispatcher.py | 12 ------------ tests/test_multi_provider_dispatch.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/dispatcher.py b/src/dispatcher.py index 50dc4a2..b4d7191 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -4266,15 +4266,3 @@ app.include_router( build_router(cfg, _db, str(Path(__file__).resolve().parent.parent)), prefix="/admin", ) - - -# --- module sanity checks ----------------------------------------------------- -def _self_check() -> None: - empty = extract_telemetry({}) - assert empty.energy_kwh is None, "empty telemetry should have no kwh" - assert empty.cost_usd is None, "empty telemetry should have no cost" - assert _sniff_telemetry_line(": OPENROUTER PROCESSING") is None - assert _sniff_telemetry_line(": energy {}") is not None - - -_self_check() diff --git a/tests/test_multi_provider_dispatch.py b/tests/test_multi_provider_dispatch.py index cf22006..75aad27 100644 --- a/tests/test_multi_provider_dispatch.py +++ b/tests/test_multi_provider_dispatch.py @@ -245,3 +245,26 @@ def test_provider_refusal_is_isolated_between_providers(router, monkeypatch): assert result is not None assert result.source == "classifier_cloud" assert cloud_called["openrouter"] is True + + +def test_extract_telemetry_empty_payload_has_no_energy_or_cost(): + """An empty payload yields an all-None Telemetry rather than an error.""" + telemetry = dispatcher.extract_telemetry({}) + assert telemetry.energy_kwh is None + assert telemetry.cost_usd is None + + +def test_sniff_telemetry_line_ignores_foreign_comments(): + """SSE comment lines that do not carry energy/cost (e.g. OpenRouter + keepalives) are deliberately ignored.""" + assert dispatcher._sniff_telemetry_line(": OPENROUTER PROCESSING") is None + + +def test_sniff_telemetry_line_parses_energy_comment(): + """A `: energy {...}` SSE comment line is parsed into a truthy + (word, payload) tuple.""" + parsed = dispatcher._sniff_telemetry_line(": energy {}") + assert parsed is not None + word, payload = parsed + assert word == "energy" + assert payload == {} -- 2.49.1 From 9b5f67a9db7979ac1ca9658da72bf954b603fd3b Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 02:22:19 -0400 Subject: [PATCH 14/15] fix(admin): topbar no longer overflows at 800px viewport --- admin/frontend/controls.html | 8 ++++++++ admin/frontend/decisions.html | 8 ++++++++ admin/frontend/index.html | 9 +++++++++ admin/frontend/models.html | 8 ++++++++ admin/frontend/profiles.html | 8 ++++++++ admin/frontend/providers.html | 8 ++++++++ 6 files changed, 49 insertions(+) diff --git a/admin/frontend/controls.html b/admin/frontend/controls.html index 69f665a..e86b90d 100644 --- a/admin/frontend/controls.html +++ b/admin/frontend/controls.html @@ -59,6 +59,14 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} .progress{background:rgba(255,255,255,.08)!important;border-radius:999px;overflow:hidden} .progress-bar{box-shadow:0 0 6px 0 currentColor;filter:saturate(1.25)} header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls . Drop the generated-at timestamp — the widest, least + critical element — below 900px; nav labels are never truncated. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} .navbar-brand a{text-decoration:none!important;color:inherit} /* ── Brand: Tabler's own rule is [data-bs-theme=dark] .navbar-brand-autodark diff --git a/admin/frontend/decisions.html b/admin/frontend/decisions.html index 3665d84..4fc4a86 100644 --- a/admin/frontend/decisions.html +++ b/admin/frontend/decisions.html @@ -61,6 +61,14 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} .progress{background:rgba(255,255,255,.08)!important;border-radius:999px;overflow:hidden} .progress-bar{box-shadow:0 0 6px 0 currentColor;filter:saturate(1.25)} header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls . Drop the generated-at timestamp — the widest, least + critical element — below 900px; nav labels are never truncated. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} .navbar-brand a{text-decoration:none!important;color:inherit} /* ── Brand: Tabler's own rule is [data-bs-theme=dark] .navbar-brand-autodark diff --git a/admin/frontend/index.html b/admin/frontend/index.html index 4e8ecb4..eb1e8dd 100644 --- a/admin/frontend/index.html +++ b/admin/frontend/index.html @@ -99,6 +99,15 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} Bootstrap's --bs-gutter-x default, so the brand's left edge is exactly 20px from the viewport edge regardless of container/breakpoint math. ── */ header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls (F3's 868px finding on /admin/). Drop the generated-at + timestamp — the widest, least critical element — below 900px; nav labels + are never truncated. The timestamp still shows at wider widths. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} /* ── Rounder hero type: brand wordmark + page titles get the friendlier geometric font; body copy stays on Tabler's default for readability. ── */ .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} diff --git a/admin/frontend/models.html b/admin/frontend/models.html index 207ce1e..83bc166 100644 --- a/admin/frontend/models.html +++ b/admin/frontend/models.html @@ -61,6 +61,14 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} .progress{background:rgba(255,255,255,.08)!important;border-radius:999px;overflow:hidden} .progress-bar{box-shadow:0 0 6px 0 currentColor;filter:saturate(1.25)} header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls . Drop the generated-at timestamp — the widest, least + critical element — below 900px; nav labels are never truncated. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} .navbar-brand a{text-decoration:none!important;color:inherit} /* ── Brand: Tabler's own rule is [data-bs-theme=dark] .navbar-brand-autodark diff --git a/admin/frontend/profiles.html b/admin/frontend/profiles.html index e78737c..9dcb4f2 100644 --- a/admin/frontend/profiles.html +++ b/admin/frontend/profiles.html @@ -57,6 +57,14 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} .badge.bg-secondary{background:rgba(148,163,184,.20)!important;color:#e2e8f0!important} .badge.bg-purple{background:rgba(168,85,247,.20)!important;color:#d8b4fe!important} header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls . Drop the generated-at timestamp — the widest, least + critical element — below 900px; nav labels are never truncated. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} .navbar-brand a{text-decoration:none!important;color:inherit} /* ── Brand mascot sizing and filter guard. ── */ diff --git a/admin/frontend/providers.html b/admin/frontend/providers.html index d653cf3..bd90f52 100644 --- a/admin/frontend/providers.html +++ b/admin/frontend/providers.html @@ -57,6 +57,14 @@ body{overflow-y:auto;height:100%;margin:0!important;padding:0!important} .badge.bg-secondary{background:rgba(148,163,184,.20)!important;color:#e2e8f0!important} .badge.bg-purple{background:rgba(168,85,247,.20)!important;color:#d8b4fe!important} header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important} +/* ── Topbar vs narrow viewports: brand + six nav links + right meta (live + badge, warnings bell, generated-at) roughly exceed one nowrap flex row + around 800px, so the fixed-width right cluster slides past the right edge + and scrolls . Drop the generated-at timestamp — the widest, least + critical element — below 900px; nav labels are never truncated. ── */ +@media (max-width: 899.98px){ + #generated-at{display:none} +} .navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em} .navbar-brand a{text-decoration:none!important;color:inherit} /* ── Brand mascot sizing and filter guard. ── */ -- 2.49.1 From e582015bfe6e00ac00b9aa8f926fa6c5e99f3419 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 02:43:19 -0400 Subject: [PATCH 15/15] fix(tests): migrate backoff/modes/gaming fixtures to per-provider refusal state --- tests/test_classifier_backoff.py | 10 ++++++---- tests/test_classifier_modes_dispatch.py | 3 ++- tests/test_gaming_mode.py | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_classifier_backoff.py b/tests/test_classifier_backoff.py index 5be15de..8dd0963 100644 --- a/tests/test_classifier_backoff.py +++ b/tests/test_classifier_backoff.py @@ -1,9 +1,10 @@ """The classifier circuit breaker actually breaking the circuit. ``_last_classifier_failure`` was written by ``_record_failure`` and read by -nothing: the cascade gated only its *cloud* step, and on a different timestamp -(``_last_account_refusal``). So a dead local classifier was re-dialled on every -request. A refused connection is cheap; a hung or VPN-black-holed Ollama costs +nothing: the cascade gated only its *cloud* step, and on a per-provider refusal +state (``_provider_refusal_since``). So a dead local classifier was re-dialled +on every request. A refused connection is cheap; a hung or VPN-black-holed +Ollama costs ``classifier.timeout_seconds`` — 120s on the reference deployment — per request, for as long as the outage lasts. @@ -29,12 +30,13 @@ def _closed_circuit(monkeypatch): """Start every test with an empty cache and a closed circuit.""" session_cache.clear() monkeypatch.setattr(dispatcher, "_last_classifier_failure", 0.0) - monkeypatch.setattr(dispatcher, "_last_account_refusal", 0.0) + dispatcher._provider_refusal_since.clear() # No session key -> cascade steps 1 and 2 are impossible, so a skipped # local attempt lands on the static fallback and nothing touches the DB. token = dispatcher._current_session_key.set(None) yield dispatcher._current_session_key.reset(token) + dispatcher._provider_refusal_since.clear() session_cache.clear() diff --git a/tests/test_classifier_modes_dispatch.py b/tests/test_classifier_modes_dispatch.py index 869ee62..d749f62 100644 --- a/tests/test_classifier_modes_dispatch.py +++ b/tests/test_classifier_modes_dispatch.py @@ -26,12 +26,13 @@ import session_cache def _clean_state(monkeypatch): session_cache.clear() monkeypatch.setattr(dispatcher, "_last_classifier_failure", 0.0) - monkeypatch.setattr(dispatcher, "_last_account_refusal", 0.0) + dispatcher._provider_refusal_since.clear() monkeypatch.setattr(dispatcher, "_cached_auto_classifier", None) monkeypatch.setattr(dispatcher, "_auto_classifier_resolved_at", 0.0) token = dispatcher._current_session_key.set(None) yield dispatcher._current_session_key.reset(token) + dispatcher._provider_refusal_since.clear() session_cache.clear() diff --git a/tests/test_gaming_mode.py b/tests/test_gaming_mode.py index e95a183..f38f412 100644 --- a/tests/test_gaming_mode.py +++ b/tests/test_gaming_mode.py @@ -55,12 +55,13 @@ def gaming_mode(monkeypatch): """Local compute off, with the cloud classifier the mode requires.""" session_cache.clear() monkeypatch.setattr(dispatcher, "_last_classifier_failure", 0.0) - monkeypatch.setattr(dispatcher, "_last_account_refusal", 0.0) + dispatcher._provider_refusal_since.clear() monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", False) monkeypatch.setattr( dispatcher.cfg.classifier, "cloud_fallback", _cloud_fallback() ) yield + dispatcher._provider_refusal_since.clear() session_cache.clear() -- 2.49.1