From 54cfd2e8b70244ab6824036f3eaa0cf23877a985 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:37:49 -0400 Subject: [PATCH 01/13] feat(routing): credit_attenuation_multiplier pure function --- src/routing.py | 28 ++++++++++++++++++++ tests/test_routing.py | 59 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/src/routing.py b/src/routing.py index f7ea38a..1591960 100644 --- a/src/routing.py +++ b/src/routing.py @@ -549,6 +549,34 @@ def within_budget(row: dict, max_energy_kwh: float | None) -> bool: return energy is None or energy <= max_energy_kwh +def credit_attenuation_multiplier( + balance_usd: float | None, + *, + soft_floor_usd: float, + zero_floor_usd: float, + max_multiplier: float, +) -> float: + """Return a cost multiplier that nudges routing away from a low balance. + + ``None`` or a balance at/above ``soft_floor_usd`` returns 1.0: the provider + participates normally. A balance at/below ``zero_floor_usd`` returns + ``max_multiplier``. Between the two floors the multiplier rises linearly + from 1.0 to ``max_multiplier``. + + Unknown balance is treated as healthy. This matches the "NULL does not + disqualify" precedent used for ``min_tool_proficiency`` in + ``rejection_reason``: a provider whose balance we cannot read is left out + of the nudge rather than pushed toward the maximum attenuation. The routing + rule is fail-neutral for missing information. + """ + if balance_usd is None or balance_usd >= soft_floor_usd: + return 1.0 + if balance_usd <= zero_floor_usd: + return max_multiplier + frac = (soft_floor_usd - balance_usd) / (soft_floor_usd - zero_floor_usd) + return 1.0 + frac * (max_multiplier - 1.0) + + def rank_candidates( rows: Sequence[dict], *, diff --git a/tests/test_routing.py b/tests/test_routing.py index d1a815e..a348329 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -10,6 +10,7 @@ from config import FlexPreference from routing import ( apply_flex_preference, capability_gate_reason, + credit_attenuation_multiplier, get_flex_sibling, is_eligible, rank_candidates, @@ -470,6 +471,64 @@ def test_reasons_are_single_tokens(): assert " " not in reason +# --- credit attenuation multiplier ---------------------------------------- + + +def test_credit_attenuation_unknown_balance_is_healthy(): + assert credit_attenuation_multiplier( + None, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == 1.0 + + +def test_credit_attenuation_above_soft_floor_is_healthy(): + assert credit_attenuation_multiplier( + 10.0, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == 1.0 + + +def test_credit_attenuation_at_soft_floor_is_healthy(): + assert credit_attenuation_multiplier( + 5.0, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == 1.0 + + +def test_credit_attenuation_at_zero_floor_is_max(): + assert credit_attenuation_multiplier( + 0.0, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == 5.0 + + +def test_credit_attenuation_below_zero_floor_is_max(): + assert credit_attenuation_multiplier( + -1.0, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == 5.0 + + +def test_credit_attenuation_midpoint_linear(): + # balance 2.5 is halfway between soft=5 and zero=0 with max=5. + # frac = (5 - 2.5) / (5 - 0) = 0.5 + # result = 1.0 + 0.5 * (5 - 1.0) = 3.0 + assert credit_attenuation_multiplier( + 2.5, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=5.0 + ) == pytest.approx(3.0) + + +def test_credit_attenuation_multiplier_range_independence(): + # Same fractional position with max_multiplier=2.0 should yield 1.5. + assert credit_attenuation_multiplier( + 2.5, soft_floor_usd=5.0, zero_floor_usd=0.0, max_multiplier=2.0 + ) == pytest.approx(1.5) + + +def test_credit_attenuation_equal_floors_is_safe(): + # With soft == zero, the guard order (>= soft first, <= zero second) + # means the interpolation branch is unreachable regardless of balance. + # This construction-safety test hits the <= zero branch. + assert credit_attenuation_multiplier( + 0.0, soft_floor_usd=5.0, zero_floor_usd=5.0, max_multiplier=5.0 + ) == 5.0 + + # --- capability_gate_reason (the extracted flag rule) ----------------------- # # rejection_reason delegates its vision/json-mode arm to this function, and -- 2.49.1 From 2c4cee0fe929397c40aff1872edea59a7e0a416a Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:38:03 -0400 Subject: [PATCH 02/13] feat(config): DispatchProvider.balance_url with fail-closed parser/telemetry validators --- src/config.py | 49 ++++++++++++ tests/test_balance_config.py | 141 +++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 tests/test_balance_config.py diff --git a/src/config.py b/src/config.py index f112d08..24e2c91 100644 --- a/src/config.py +++ b/src/config.py @@ -884,9 +884,27 @@ class LocalDispatchModel(StrictModel): class DispatchProvider(StrictModel): base_url: str api_key_env: str + balance_url: Optional[str] = None has_energy_telemetry: bool = False enabled: bool = True + @field_validator("balance_url") + @classmethod + def balance_url_requires_https(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + if urlparse(v).scheme != "https": + raise ValueError( + f"dispatch_providers[...].balance_url must use https://, got {v!r}" + ) + return v + + +# Names-only balance-parser registry. poller imports config, so config cannot +# import poller's parser dict; the poller implements the parser and tests pin +# the two sets equal. +PROVIDERS_WITH_BALANCE_PARSERS: frozenset[str] = frozenset({"openrouter"}) + class LoggingConfig(StrictModel): # log_path is gone. Nothing ever wrote a file: the dispatcher logs to @@ -1230,6 +1248,37 @@ class RouterConfig(StrictModel): """ return self._dispatch_meterable_cache + @model_validator(mode="after") + def balance_url_on_provider_with_parser_and_no_telemetry( + self, + ) -> "RouterConfig": + """balance_url may only be set for providers with a parser and no telemetry. + + A balance_url asks the poller to query a provider account endpoint. + Providers that already report balance per completion via + has_energy_telemetry would create two contradictory sources; fail at + load rather than silently picking one. A provider whose key has no + parser implementation would fail quietly every poll cycle. + """ + for provider, prov_cfg in self.dispatch_providers.items(): + if prov_cfg.balance_url is None: + continue + if getattr(prov_cfg, "has_energy_telemetry", False): + raise ValueError( + f"dispatch_providers[{provider!r}] has both " + f"balance_url and has_energy_telemetry=true. " + f"Telemetry providers already report balance per completion; " + f"configure one source or the other." + ) + if provider not in PROVIDERS_WITH_BALANCE_PARSERS: + valid = sorted(PROVIDERS_WITH_BALANCE_PARSERS) + raise ValueError( + f"dispatch_providers[{provider!r}].balance_url is set, " + f"but {provider!r} has no balance parser implementation. " + f"Providers with parsers: {valid}" + ) + return self + @model_validator(mode="after") def verifier_model_is_stated_once_the_hosts_differ(self) -> "RouterConfig": """A remote classifier must not lend its model name to the verifier. diff --git a/tests/test_balance_config.py b/tests/test_balance_config.py new file mode 100644 index 0000000..3118976 --- /dev/null +++ b/tests/test_balance_config.py @@ -0,0 +1,141 @@ +"""QA for DispatchProvider.balance_url + fail-closed validators.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from config import ( + PROVIDERS_WITH_BALANCE_PARSERS, + DispatchProvider, + RouterConfig, + _merge_overlay, + load_config, +) + + +CONFIG_PATH = Path("config/config.yaml") + + +def _base_cfg_dict() -> dict: + """Return the real base config dict so mutants start from a valid tree.""" + raw = load_config(CONFIG_PATH).model_dump() + # The real config currently has only "neuralwatt" in dispatch_providers. + return raw + + +def _provider_dict(**overrides) -> dict: + base = {"base_url": "https://example.com/v1", "api_key_env": "EXAMPLE_KEY"} + base.update(overrides) + return base + + +class TestDispatchProviderBalanceUrl: + def test_none_is_allowed(self): + p = DispatchProvider(base_url="https://x/v1", api_key_env="X") + assert p.balance_url is None + + def test_https_url_is_allowed(self): + p = DispatchProvider( + base_url="https://x/v1", + api_key_env="X", + balance_url="https://x/credits", + ) + assert p.balance_url == "https://x/credits" + + def test_http_url_rejected(self): + with pytest.raises(ValidationError) as exc_info: + DispatchProvider( + base_url="https://x/v1", + api_key_env="X", + balance_url="http://insecure.example/credits", + ) + assert "https://" in str(exc_info.value) + assert "http://insecure.example/credits" in str(exc_info.value) + + def test_unknown_provider_key_fails(self): + """(a) balance_url on a provider without a parser is an error.""" + raw = _base_cfg_dict() + raw["dispatch_providers"]["unknown_provider"] = _provider_dict( + balance_url="https://unknown.example/credits", + ) + with pytest.raises(ValidationError) as exc_info: + RouterConfig(**raw) + err = str(exc_info.value) + assert "unknown_provider" in err + assert "no balance parser implementation" in err + assert "neuralwatt" not in err + + def test_telemetry_provider_with_balance_url_fails(self): + """(b) balance_url + has_energy_telemetry is a double-source error.""" + raw = _base_cfg_dict() + raw["dispatch_providers"]["neuralwatt"]["balance_url"] = ( + "https://neuralwatt.example/credits" + ) + raw["dispatch_providers"]["neuralwatt"]["has_energy_telemetry"] = True + with pytest.raises(ValidationError) as exc_info: + RouterConfig(**raw) + err = str(exc_info.value) + assert "neuralwatt" in err + assert "has_energy_telemetry=true" in err + assert "Telemetry providers already report balance per completion" in err + + def test_openrouter_balance_url_loads(self): + """(c) openrouter with a balance URL loads fine.""" + raw = _base_cfg_dict() + raw["dispatch_providers"]["openrouter"] = _provider_dict( + balance_url="https://openrouter.ai/api/v1/credits", + ) + cfg = RouterConfig(**raw) + assert cfg.dispatch_providers["openrouter"].balance_url == ( + "https://openrouter.ai/api/v1/credits" + ) + + def test_http_balance_url_rejected_at_provider_level(self): + """(d) http:// balance URL fails the field validator.""" + raw = _base_cfg_dict() + raw["dispatch_providers"]["openrouter"] = _provider_dict( + balance_url="http://openrouter.example/credits", + ) + with pytest.raises(ValidationError) as exc_info: + RouterConfig(**raw) + assert "https://" in str(exc_info.value) + + +class TestRegistryAndRealConfig: + def test_registry_is_openrouter_only(self): + assert PROVIDERS_WITH_BALANCE_PARSERS == frozenset({"openrouter"}) + + def test_real_config_still_loads(self): + cfg = load_config(CONFIG_PATH) + assert "neuralwatt" in cfg.dispatch_providers + assert cfg.dispatch_providers["neuralwatt"].balance_url is None + + +class TestAdminRoundTripPreservation: + """(e2) _merge_overlay preserves base-only balance_url through provider edits.""" + + def test_overlay_preserves_balance_url(self): + base = { + "dispatch_providers": { + "openrouter": _provider_dict( + balance_url="https://openrouter.ai/api/v1/credits", + ), + }, + } + overlay = { + "dispatch_providers": { + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY_EDITED", + "timeout_seconds": 30, + }, + }, + } + merged = _merge_overlay(base, overlay) + assert merged["dispatch_providers"]["openrouter"]["balance_url"] == ( + "https://openrouter.ai/api/v1/credits" + ) + assert ( + merged["dispatch_providers"]["openrouter"]["api_key_env"] + == "OPENROUTER_API_KEY_EDITED" + ) -- 2.49.1 From 2f3f9607bb6209c58ecacd8026e9f9f64e850030 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:40:23 -0400 Subject: [PATCH 03/13] feat(schema): provider_balance_observations table + poller record_balance --- config/schema.sql | 11 +++++ src/poller.py | 34 ++++++++++++++ tests/test_provider_balance.py | 86 ++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 tests/test_provider_balance.py diff --git a/config/schema.sql b/config/schema.sql index de9b6ae..7d71e6d 100644 --- a/config/schema.sql +++ b/config/schema.sql @@ -307,3 +307,14 @@ CREATE TABLE IF NOT EXISTS local_energy_observations ( ); CREATE INDEX IF NOT EXISTS idx_local_energy_model ON local_energy_observations (model_id); CREATE INDEX IF NOT EXISTS idx_local_energy_request ON local_energy_observations (request_id); + +-- Account-level balance observations per dispatch provider (separate from +-- energy_observations because this is a prepaid-account pool, not a +-- per-completion telemetry measure). +CREATE TABLE IF NOT EXISTS provider_balance_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + balance_usd REAL NOT NULL, + observed_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_provider_balance ON provider_balance_observations (provider, observed_at); diff --git a/src/poller.py b/src/poller.py index 1c06e10..62f4508 100644 --- a/src/poller.py +++ b/src/poller.py @@ -380,6 +380,40 @@ def upsert(conn: sqlite3.Connection, rows: list[ModelRow], cfg: RouterConfig) -> conn.commit() +def _ensure_provider_balance_table(conn: sqlite3.Connection) -> None: + """Idempotently create the provider_balance_observations table and index. + + Live router.db files that predate this feature lack the table; the + CREATE TABLE IF NOT EXISTS / CREATE INDEX IF NOT EXISTS DDL is safe to + re-run on every poll. + """ + conn.execute(""" + CREATE TABLE IF NOT EXISTS provider_balance_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + balance_usd REAL NOT NULL, + observed_at TEXT NOT NULL + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_provider_balance " + "ON provider_balance_observations (provider, observed_at)" + ) + conn.commit() + + +def record_balance( + conn: sqlite3.Connection, provider: str, balance_usd: float +) -> None: + """Write a provider account-balance observation row.""" + conn.execute( + "INSERT INTO provider_balance_observations (provider, balance_usd, observed_at) " + "VALUES (?, ?, ?)", + (provider, balance_usd, datetime.now(timezone.utc).isoformat()), + ) + conn.commit() + + def _ensure_models_eligible_categories(conn: sqlite3.Connection) -> None: """Idempotently add the models.eligible_categories column. diff --git a/tests/test_provider_balance.py b/tests/test_provider_balance.py new file mode 100644 index 0000000..575e8aa --- /dev/null +++ b/tests/test_provider_balance.py @@ -0,0 +1,86 @@ +"""Tests for the provider_balance_observations table and poller helpers. + +All tests seed a throwaway SQLite DB from schema.sql and assert on queried +rows, not on stdout strings. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +import poller + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() + + +def _make_db() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.execute("PRAGMA foreign_keys = ON") + conn.executescript(SCHEMA_SQL) + return conn + + +def _table_columns(conn: sqlite3.Connection, table: str) -> list[str]: + return [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] + + +def test_provider_balance_observations_schema() -> None: + """QA (a): schema.sql creates the expected columns.""" + conn = _make_db() + assert _table_columns(conn, "provider_balance_observations") == [ + "id", + "provider", + "balance_usd", + "observed_at", + ] + + +def test_record_balance_round_trip() -> None: + """QA (b): record_balance inserts UTC ISO8601 with correct values.""" + conn = _make_db() + before = datetime.now(timezone.utc) + poller.record_balance(conn, "openrouter", 42.5) + after = datetime.now(timezone.utc) + + row = conn.execute( + "SELECT provider, balance_usd, observed_at " + "FROM provider_balance_observations" + ).fetchone() + assert row is not None + assert row[0] == "openrouter" + assert row[1] == pytest.approx(42.5) + observed = datetime.fromisoformat(row[2]) + assert observed.tzinfo is not None + assert before <= observed <= after + + +def test_ensure_provider_balance_table_idempotent() -> None: + """QA (c): _ensure_provider_balance_table runs twice on an empty DB.""" + conn = sqlite3.connect(":memory:") + conn.execute("PRAGMA foreign_keys = ON") + poller._ensure_provider_balance_table(conn) + poller._ensure_provider_balance_table(conn) + assert _table_columns(conn, "provider_balance_observations") == [ + "id", + "provider", + "balance_usd", + "observed_at", + ] + idx = conn.execute( + "SELECT name FROM sqlite_master WHERE type='index' AND name='idx_provider_balance'" + ).fetchone() + assert idx is not None + + +def test_record_balance_fails_without_table() -> None: + """QA (d): no hidden auto-create; missing table raises OperationalError.""" + conn = sqlite3.connect(":memory:") + conn.execute("PRAGMA foreign_keys = ON") + # Table intentionally not created. + with pytest.raises(sqlite3.OperationalError): + poller.record_balance(conn, "openrouter", 1.0) -- 2.49.1 From 2663e867bedf27881273e0cabc94caa4be6d87bb Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:50:27 -0400 Subject: [PATCH 04/13] feat(config): objective.credit_attenuation block, off by default --- config/config.yaml | 21 +++++++ src/config.py | 62 +++++++++++++++++++ tests/test_credit_attenuation_config.py | 80 +++++++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 tests/test_credit_attenuation_config.py diff --git a/config/config.yaml b/config/config.yaml index 76cd164..8807b55 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -104,6 +104,27 @@ objective: # peak, far below the dozens/hour a deprecation flare produces. rejection_warning_min_count: 6 + # Credit-aware routing attenuation (OFF BY DEFAULT). + # When enabled, a provider configured with a balance_url (e.g. OpenRouter's + # prepaid account balance) gets its *comparison cost* inflated inside the + # quality-first ranker as its account balance nears zero. Quality bands still + # win; this only shifts ties. Providers whose balance comes from per-request + # energy telemetry allowance_remaining_usd (e.g. NeuralWatt's overage-billed + # subscription) are ALWAYS multiplier 1.0 regardless of their reading, so a + # low soft_floor_usd never biases routing toward an attenuated provider just + # because a telemetry provider's allowance reads near zero under normal use. + # + # This knob is deliberately NOT exposed in the admin UI persisted-config + # allowlist (_CONFIG_ALLOWLIST in admin.py) or _ProviderUpdateBody; enabling + # or tuning it requires editing this file and restarting llm-router.service + # (the dispatcher's module-level cfg binds at import). + credit_attenuation: + enabled: false + soft_floor_usd: 5.0 # balance >= this -> multiplier 1.0 + zero_floor_usd: 0.0 # balance <= this -> max_multiplier + max_multiplier: 5.0 # maximum cost-inflation at/below zero floor + refresh_seconds: 300 # cache duration for resolved multipliers + context: safety_factor: 0.75 # fraction of advertised context treated as usable default_output_reserve_tokens: 4096 diff --git a/src/config.py b/src/config.py index 24e2c91..94be07c 100644 --- a/src/config.py +++ b/src/config.py @@ -36,6 +36,67 @@ class StrictModel(BaseModel): model_config = ConfigDict(extra="forbid") +class CreditAttenuationConfig(StrictModel): + """Optional cost-inflation tiebreak for providers with a polled account balance. + + When enabled, a provider whose prepaid balance is near depletion gets its + comparison cost multiplied inside the quality-first ranking. Quality bands + still win; the multiplier only breaks ties. Telemetry providers (those + billed per-completion via ``allowance_remaining_usd``) are always treated + as multiplier 1.0 — their "low" reading is normal overage-invoice noise, + not a depleting pool, so attenuation never applies to them. + """ + + enabled: bool = False + soft_floor_usd: float = 5.0 + zero_floor_usd: float = 0.0 + max_multiplier: float = 5.0 + refresh_seconds: int = 300 + + @field_validator("soft_floor_usd") + @classmethod + def soft_floor_positive(cls, v: float) -> float: + if v <= 0: + raise ValueError("objective.credit_attenuation.soft_floor_usd must be > 0") + return v + + @field_validator("zero_floor_usd") + @classmethod + def zero_floor_non_negative(cls, v: float) -> float: + if v < 0: + raise ValueError( + "objective.credit_attenuation.zero_floor_usd must be >= 0" + ) + return v + + @field_validator("max_multiplier") + @classmethod + def max_multiplier_above_one(cls, v: float) -> float: + if v <= 1.0: + raise ValueError( + "objective.credit_attenuation.max_multiplier must be > 1.0" + ) + return v + + @field_validator("refresh_seconds") + @classmethod + def refresh_seconds_positive(cls, v: int) -> int: + if v <= 0: + raise ValueError( + "objective.credit_attenuation.refresh_seconds must be > 0" + ) + return v + + @model_validator(mode="after") + def zero_below_soft(self) -> "CreditAttenuationConfig": + if self.zero_floor_usd >= self.soft_floor_usd: + raise ValueError( + "objective.credit_attenuation.zero_floor_usd " + "must be < soft_floor_usd" + ) + return self + + class Objective(StrictModel): """What the router optimizes: quality, bounded by cost. @@ -57,6 +118,7 @@ class Objective(StrictModel): rejection_warning_window_hours: Optional[int] = None rejection_warning_baseline_hours: Optional[int] = None rejection_warning_min_count: Optional[int] = None + credit_attenuation: CreditAttenuationConfig = CreditAttenuationConfig() @field_validator("quality_tolerance") @classmethod diff --git a/tests/test_credit_attenuation_config.py b/tests/test_credit_attenuation_config.py new file mode 100644 index 0000000..3f516cb --- /dev/null +++ b/tests/test_credit_attenuation_config.py @@ -0,0 +1,80 @@ +"""QA for Objective.credit_attenuation config block.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from config import ( + CreditAttenuationConfig, + Objective, + RouterConfig, + load_config, +) + + +CONFIG_PATH = Path("config/config.yaml") + +class TestCreditAttenuationConfigDefaults: + def test_default_constructed_disabled(self): + """(a) Objective() default constructs with credit_attenuation disabled.""" + obj = Objective() + assert obj.credit_attenuation.enabled is False + assert obj.credit_attenuation.soft_floor_usd == 5.0 + assert obj.credit_attenuation.zero_floor_usd == 0.0 + assert obj.credit_attenuation.max_multiplier == 5.0 + assert obj.credit_attenuation.refresh_seconds == 300 + + +class TestCreditAttenuationValidators: + def test_zero_floor_eq_soft_floor_rejected(self): + """(b) zero_floor_usd == soft_floor_usd raises.""" + with pytest.raises(ValidationError) as exc_info: + CreditAttenuationConfig(soft_floor_usd=5.0, zero_floor_usd=5.0) + assert "zero_floor_usd" in str(exc_info.value) + assert "soft_floor_usd" in str(exc_info.value) + + def test_max_multiplier_not_above_one_rejected(self): + """(c) max_multiplier=1.0 raises.""" + with pytest.raises(ValidationError) as exc_info: + CreditAttenuationConfig(max_multiplier=1.0) + assert "max_multiplier" in str(exc_info.value) + + def test_refresh_seconds_zero_rejected(self): + """(d) refresh_seconds=0 raises.""" + with pytest.raises(ValidationError) as exc_info: + CreditAttenuationConfig(refresh_seconds=0) + assert "refresh_seconds" in str(exc_info.value) + + def test_soft_floor_zero_rejected(self): + """(e) soft_floor_usd=0 raises.""" + with pytest.raises(ValidationError) as exc_info: + CreditAttenuationConfig(soft_floor_usd=0.0) + assert "soft_floor_usd" in str(exc_info.value) + + def test_zero_floor_negative_rejected(self): + """(f) zero_floor_usd<0 raises.""" + with pytest.raises(ValidationError) as exc_info: + CreditAttenuationConfig(zero_floor_usd=-1.0) + assert "zero_floor_usd" in str(exc_info.value) + + +class TestCreditAttenuationRealConfig: + def test_real_config_has_attenuation_disabled(self): + """(g) real load_config returns credit_attenuation.enabled False.""" + cfg = load_config(Path("config/config.yaml")) + assert cfg.objective.credit_attenuation.enabled is False + assert cfg.objective.credit_attenuation.soft_floor_usd == 5.0 + assert cfg.objective.credit_attenuation.zero_floor_usd == 0.0 + assert cfg.objective.credit_attenuation.max_multiplier == 5.0 + assert cfg.objective.credit_attenuation.refresh_seconds == 300 + + +class TestCreditAttenuationExtraForbid: + def test_unknown_key_inside_block_raises(self): + """(h) unknown key inside credit_attenuation raises via extra-forbid.""" + raw = load_config(CONFIG_PATH).model_dump() + raw["objective"]["credit_attenuation"]["hard_floor_usd"] = 1.0 + with pytest.raises(ValidationError) as exc_info: + RouterConfig(**raw) + assert "hard_floor_usd" in str(exc_info.value) -- 2.49.1 From ad65fad8e343ee8b7d973b34e07bfed61ee6e8af Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:57:18 -0400 Subject: [PATCH 05/13] feat(routing): provider cost multipliers as an effective-cost rank tiebreak --- src/routing.py | 40 ++++++++++++++++--- tests/test_routing.py | 89 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/src/routing.py b/src/routing.py index 1591960..221b96a 100644 --- a/src/routing.py +++ b/src/routing.py @@ -585,6 +585,7 @@ def rank_candidates( prompt_tokens: int = 0, completion_tokens: int = 500, cache_rate: float = 0.84, + provider_cost_multipliers: dict[str, float] | None = None, ) -> list[dict]: """Order candidates: best quality first, cheapest among equals. @@ -616,6 +617,16 @@ def rank_candidates( Returns each row plus ``proficiency_score``, ``cost_score`` (reported for visibility only, no longer part of the decision) and ``composite``, which is now simply the effective quality after the tolerance band. + + ``provider_cost_multipliers`` is an optional per-provider cost inflation + map used ONLY inside the quality-band tiebreak. A provider mapped to + ``5.0`` is treated as five times as expensive for ranking, but its + ``cost`` and ``cost_score`` remain the honest catalog estimate. This is + the credit-attenuation hook: it can nudge a near-tie toward a provider + with a healthier account balance, but it can never override a genuine + quality gap because the band is computed before the cost key. When the + argument is ``None`` or omitted the sort falls back to the raw cost and + behavior is identical to the pre-attenuation implementation. """ affordable = [r for r in rows if within_budget(r, max_energy_per_request)] @@ -624,18 +635,35 @@ def rank_candidates( # has no price for keeps whatever measured cost it arrived with rather # than losing the field — a missing list price is not free. estimates = [] + effective_estimates = [] for r in affordable: est = estimated_cost(r, prompt_tokens, completion_tokens, cache_rate) - estimates.append(r.get("cost") if est is None else est) + raw = r.get("cost") if est is None else est + estimates.append(raw) + if provider_cost_multipliers and raw is not None: + multiplier = provider_cost_multipliers.get(r.get("provider"), 1.0) + effective_estimates.append(raw * multiplier) + else: + effective_estimates.append(raw) # cost_score is retained purely so callers can still see the spread; it # does not enter the ordering. cost_scores = cost_score(estimates) ranked = [] - for row, c_s, est in zip(affordable, cost_scores, estimates): + for row, c_s, raw, effective in zip( + affordable, cost_scores, estimates, effective_estimates + ): p_s = proficiency_score(row.get("proficiency")) - ranked.append({**row, "cost": est, "cost_score": c_s, - "proficiency_score": p_s, "composite": p_s}) + ranked.append( + { + **row, + "cost": raw, + "effective_cost": effective, + "cost_score": c_s, + "proficiency_score": p_s, + "composite": p_s, + } + ) if not ranked: return [] @@ -657,7 +685,9 @@ def rank_candidates( ranked.sort( key=lambda r: ( band(r), - r["cost"] if r.get("cost") is not None else float("inf"), + r["effective_cost"] + if r.get("effective_cost") is not None + else float("inf"), r["model_id"], ) ) diff --git a/tests/test_routing.py b/tests/test_routing.py index a348329..33ec166 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -1077,3 +1077,92 @@ def test_through_is_eligible_via_filters(): task_category="summarization", ) is False + +# QA cases (a)-(g) for provider_cost_multipliers effective_cost tiebreak. + +def _ranked_ids(rows, **kwargs): + return [r["model_id"] for r in rank_candidates(rows, **kwargs)] + + +def test_provider_cost_multiplier_flips_tiebreak_toward_healthy_provider(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.80), + _row(model_id="B", provider="provB", cost=0.10, proficiency=0.80), + ] + assert _ranked_ids(rows) == ["A", "B"] + assert _ranked_ids( + rows, provider_cost_multipliers={"provA": 5.0} + ) == ["B", "A"] + + +def test_provider_cost_multiplier_cannot_override_quality_band_gap(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.95), + _row(model_id="B", provider="provB", cost=0.01, proficiency=0.80), + ] + assert _ranked_ids( + rows, provider_cost_multipliers={"provA": 5.0}, quality_tolerance=0.1 + ) == ["A", "B"] + + +def test_ranked_rows_carry_raw_cost_and_effective_cost(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.80), + ] + ranked = rank_candidates(rows, provider_cost_multipliers={"provA": 5.0}) + assert len(ranked) == 1 + assert ranked[0]["cost"] == pytest.approx(0.10) + assert ranked[0]["cost_score"] == pytest.approx(1.0) + assert ranked[0]["effective_cost"] == pytest.approx(0.50) + + +def test_provider_multiplier_applies_to_fallback_cost_when_estimate_missing(): + rows = [ + _row( + model_id="A", + provider="provA", + cost=0.20, + proficiency=0.80, + prompt_cost_1m=None, + completion_cost_1m=None, + ), + ] + ranked = rank_candidates( + rows, + prompt_tokens=0, + completion_tokens=0, + provider_cost_multipliers={"provA": 3.0}, + ) + assert ranked[0]["cost"] == pytest.approx(0.20) + assert ranked[0]["effective_cost"] == pytest.approx(0.60) + + +def test_provider_cost_multiplier_unknown_provider_ignored(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.80), + ] + ranked = rank_candidates( + rows, provider_cost_multipliers={"provUnknown": 99.0} + ) + assert ranked[0]["effective_cost"] == pytest.approx(0.10) + + +def test_provider_cost_multiplier_defaults_to_one_for_unmapped_provider(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.80), + _row(model_id="B", provider="provB", cost=0.10, proficiency=0.80), + ] + assert _ranked_ids( + rows, provider_cost_multipliers={"provA": 5.0} + ) == ["B", "A"] + + +def test_provider_cost_multiplier_empty_dict_matches_no_argument(): + rows = [ + _row(model_id="A", provider="provA", cost=0.10, proficiency=0.80), + _row(model_id="B", provider="provB", cost=0.10, proficiency=0.80), + ] + assert _ranked_ids(rows) == _ranked_ids( + rows, provider_cost_multipliers={} + ) + -- 2.49.1 From b9f70c1745733f2daf0500aff5a14b9ec3b6b3e5 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 12:58:21 -0400 Subject: [PATCH 06/13] fix(metrics): per-provider balance, burn, and runway instead of a global series --- src/metrics.py | 233 ++++++++++---- tests/test_metrics.py | 549 ++++++++++++++++++++++++++++----- tests/test_metrics_endpoint.py | 53 +++- 3 files changed, 675 insertions(+), 160 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index b34c1c6..5e430cc 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -96,33 +96,26 @@ def _billing_period_start(billing_reset_day: int, today: Optional[date] = None) return date(prev_year, prev_month, billing_reset_day).isoformat() -def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: - """Latest account balance and its burn/runway from ``allowance_remaining_usd``. +def _balance_series_stats( + samples: list[tuple[datetime, float]], + *, + latest: Optional[tuple[str, float]], + burn_window_hours: float, + warning_hours: float, + min_samples: int, + min_hours: float, +) -> dict[str, Any]: + """Compute balance/burn/runway for one provider-scoped series. - Pure read of ``energy_observations`` — no dispatcher import. The burn - rate is intentionally conservative: only the most recent + ``samples`` are the in-window (observed_at, balance) points, ordered + oldest-first; rows with unparseable timestamps are already skipped by the + caller. ``latest`` is the provider's most recent balance row regardless of + window (``(observed_at, balance_usd)``) and may be None. + + The burn rate is intentionally conservative: only the latest monotonically-decreasing balance segment is used, and two guards stop a fresh credit top-up from producing a wild extrapolation. """ - # Read optional knobs from config; treat None as unset and use code defaults. - # Test configs use SimpleNamespace without these attributes, so getattr - # must have a fallback and then a second default when the attr is None. - burn_window_hours = getattr(cfg.objective, "quota_burn_window_hours", None) - if burn_window_hours is None: - burn_window_hours = 24 - - warning_hours = getattr(cfg.objective, "quota_runway_warning_hours", None) - if warning_hours is None: - warning_hours = 6 - - min_samples = getattr(cfg.objective, "quota_burn_min_segment_samples", None) - if min_samples is None: - min_samples = 3 - - min_hours = getattr(cfg.objective, "quota_burn_min_segment_hours", None) - if min_hours is None: - min_hours = 0.5 - result: dict[str, Any] = { "balance_usd": None, "balance_at": None, @@ -133,49 +126,20 @@ def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: "runway_note": None, } - # 1. Latest non-NULL allowance across ALL rows (no time-window filter). - balance_row = conn.execute( - """ - SELECT observed_at, allowance_remaining_usd - FROM energy_observations - WHERE allowance_remaining_usd IS NOT NULL - ORDER BY observed_at DESC, id DESC - LIMIT 1 - """ - ).fetchone() - if balance_row is not None: - result["balance_usd"] = float(balance_row["allowance_remaining_usd"]) - result["balance_at"] = balance_row["observed_at"] + if latest is not None: + result["balance_at"] = latest[0] + result["balance_usd"] = latest[1] - # 2. In-window rows for burn estimation. - rows = conn.execute( - """ - SELECT observed_at, allowance_remaining_usd - FROM energy_observations - WHERE allowance_remaining_usd IS NOT NULL - AND julianday(observed_at) > julianday('now', '-' || ? || ' hours') - ORDER BY observed_at ASC, id ASC - """, - (str(burn_window_hours),), - ).fetchall() - - if not rows: + if not samples: result["runway_note"] = ( "burn estimate unavailable: no decreasing balance samples in the current window" ) return result - # 3. Split into monotonically-decreasing segments at every balance INCREASE. - # A top-up (credit jump) starts a new segment; only the latest survives. + # Split into monotonically-decreasing segments at every balance INCREASE. + # A top-up (credit jump) starts a new segment; only the latest survives. segments: list[list[tuple[datetime, float]]] = [[]] - for row in rows: - raw_ts = row["observed_at"] - try: - ts = datetime.fromisoformat(raw_ts) - except ValueError: - # Defensive: malformed timestamp would otherwise break /metrics. - continue - value = float(row["allowance_remaining_usd"]) + for ts, value in samples: current = segments[-1] if not current: current.append((ts, value)) @@ -191,7 +155,6 @@ def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: ) return result - # 4. Guarded burn-rate computation. if len(latest_segment) < min_samples: result["runway_note"] = ( f"burn estimate unavailable: segment after last balance increase has only " @@ -225,7 +188,6 @@ def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: burn_rate = round(total_decrease / elapsed_hours, 6) result["burn_rate_usd_per_hour"] = burn_rate - # 5. Projected runway and warning. balance = result["balance_usd"] if balance is not None and burn_rate > 0.0: projected = balance / burn_rate @@ -235,6 +197,138 @@ def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: return result +def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: + """Per-provider balance, burn and runway. + + Returns ``{"by_provider": {provider: {balance_usd, balance_at, + balance_source, burn_window_hours, burn_rate_usd_per_hour, + projected_hours_remaining, runway_low_warning, runway_note}}, + "total_balance_usd": float|None}`` with one entry per + ``cfg.dispatch_providers`` key. + + The two-source rule: + + * ``has_energy_telemetry`` providers read + ``energy_observations.allowance_remaining_usd`` scoped to their provider. + * Every other provider reads ``provider_balance_observations`` scoped to + its provider (the polled account-balance table), guarded with + ``sqlite3.OperationalError`` so a live DB predating that table degrades + to an all-None entry instead of failing ``/metrics``. + + ``balance_source`` is CONFIG-derived and present even on all-None entries, + so a consumer can tell a prepaid-pool depletion story from an + overage-allowance accounting figure without re-deriving config: + ``"telemetry"`` (per-completion allowance), ``"polled"`` (balance_url + configured), ``"unconfigured"`` (neither). + """ + # Read optional knobs from config; treat None as unset and use code defaults. + # Test configs use SimpleNamespace without these attributes, so getattr + # must have a fallback and then a second default when the attr is None. + burn_window_hours = getattr(cfg.objective, "quota_burn_window_hours", None) + if burn_window_hours is None: + burn_window_hours = 24 + + warning_hours = getattr(cfg.objective, "quota_runway_warning_hours", None) + if warning_hours is None: + warning_hours = 6 + + min_samples = getattr(cfg.objective, "quota_burn_min_segment_samples", None) + if min_samples is None: + min_samples = 3 + + min_hours = getattr(cfg.objective, "quota_burn_min_segment_hours", None) + if min_hours is None: + min_hours = 0.5 + + providers = getattr(cfg, "dispatch_providers", None) or {} + by_provider: dict[str, dict[str, Any]] = {} + + for provider, pc in providers.items(): + has_telemetry = getattr(pc, "has_energy_telemetry", False) + balance_url = getattr(pc, "balance_url", None) + + if has_telemetry: + balance_source = "telemetry" + latest_sql = """ + SELECT observed_at, allowance_remaining_usd + FROM energy_observations + WHERE provider = ? + AND allowance_remaining_usd IS NOT NULL + ORDER BY observed_at DESC, id DESC + LIMIT 1 + """ + window_sql = """ + SELECT observed_at, allowance_remaining_usd + FROM energy_observations + WHERE provider = ? + AND allowance_remaining_usd IS NOT NULL + AND julianday(observed_at) > julianday('now', '-' || ? || ' hours') + ORDER BY observed_at ASC, id ASC + """ + else: + balance_source = "polled" if balance_url else "unconfigured" + latest_sql = """ + SELECT observed_at, balance_usd + FROM provider_balance_observations + WHERE provider = ? + ORDER BY observed_at DESC, id DESC + LIMIT 1 + """ + window_sql = """ + SELECT observed_at, balance_usd + FROM provider_balance_observations + WHERE provider = ? + AND julianday(observed_at) > julianday('now', '-' || ? || ' hours') + ORDER BY observed_at ASC, id ASC + """ + + latest: Optional[tuple[str, float]] = None + raw_rows: list[sqlite3.Row] = [] + try: + row = conn.execute(latest_sql, (provider,)).fetchone() + if row is not None and row[1] is not None: + latest = (row["observed_at"], float(row[1])) + raw_rows = conn.execute( + window_sql, + (provider, str(burn_window_hours)), + ).fetchall() + except sqlite3.OperationalError: + # Live DBs that predate the provider_balance_observations table + # must not 500 /metrics; report an all-None entry instead. + latest = None + raw_rows = [] + + samples: list[tuple[datetime, float]] = [] + for row in raw_rows: + try: + ts = datetime.fromisoformat(row["observed_at"]) + except ValueError: + # Defensive: malformed timestamp would otherwise break /metrics. + continue + samples.append((ts, float(row[1]))) + + entry = _balance_series_stats( + samples, + latest=latest, + burn_window_hours=burn_window_hours, + warning_hours=warning_hours, + min_samples=min_samples, + min_hours=min_hours, + ) + entry["balance_source"] = balance_source + by_provider[provider] = entry + + known_balances = [ + entry["balance_usd"] + for entry in by_provider.values() + if entry["balance_usd"] is not None + ] + total_balance_usd = round(sum(known_balances), 6) if known_balances else None + + return {"by_provider": by_provider, "total_balance_usd": total_balance_usd} + + + def quota_burn( conn: sqlite3.Connection, cfg: Any, @@ -249,10 +343,11 @@ def quota_burn( Returns a dict with ``plan_kwh``, ``metered_kwh_30d``, ``metered_kwh_period``, ``metered_fraction_of_plan``, ``metered_calls_30d``, ``reset_date`` (the billing-period start), - ``window_start_30d`` (the rolling 30-day window start), ``note`` and the - balance/burn/runway fields from ``quota_balance_and_burn``. - When ``cfg.objective.billing_reset_day`` is set, also returns - ``next_reset_date`` — the upcoming billing-period reset day. + ``window_start_30d`` (the rolling 30-day window start), ``note``, plus the + per-provider ``by_provider`` mapping and ``total_balance_usd`` from + ``quota_balance_and_burn``. When ``cfg.objective.billing_reset_day`` is + set, also returns ``next_reset_date`` — the upcoming billing-period reset + day. """ # This gate removes the report only; it is intentionally not used to refuse # or alter request dispatch — routing decisions remain independent of quota. @@ -302,9 +397,11 @@ def quota_burn( "note": "router-metered only; traffic bypassing the router is not counted", } - # Merge the balance/burn/runway block; quota_balance_and_burn also supplies - # the burn_window_hours default. - result.update(quota_balance_and_burn(conn, cfg)) + # Merge the per-provider balance/burn/runway block. The old flat keys are + # deliberately gone: every consumer reads by_provider. + balance = quota_balance_and_burn(conn, cfg) + result["by_provider"] = balance["by_provider"] + result["total_balance_usd"] = balance["total_balance_usd"] if reset_day is not None: result["next_reset_date"] = _next_reset_date(reset_day) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index c01b986..a91c15c 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -37,6 +37,7 @@ from metrics import ( local_energy_summary, per_model, pinch_summary, + quota_balance_and_burn, quota_burn, recent_decisions, rejection_warnings, @@ -223,20 +224,43 @@ def test_quota_burn_empty_db(tmp_path): assert result["metered_calls_30d"] == 0 -# --- quota_burn balance / burn / runway tests -------------------------------- +# Internal mirror of the provider_balance_observations DDL (schema.sql owns the +# canonical version). Local creation keeps these tests hermetic against a +# transiently older schema.sql in this shared worktree. +_PROVIDER_BALANCE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS provider_balance_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + balance_usd REAL NOT NULL, + observed_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_provider_balance + ON provider_balance_observations (provider, observed_at); +""" + + +def _telemetry_provider_cfg(**objective_overrides) -> SimpleNamespace: + """Build a cfg whose only dispatch provider is telemetry-backed.""" + return SimpleNamespace( + objective=SimpleNamespace(**objective_overrides), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + }, + ) + + +# --- quota_burn balance / burn / runway tests (per-provider) ------------------ def test_quota_burn_with_top_up_ignores_credit_jump(tmp_path): """A credit top-up splits the window; burn uses only the latest segment. - Series before top-up: 3.00 → 2.00 → 1.00. Then balance jumps to 10.00 and - resumes decreasing 10.00 → 9.25 → 8.50. Burn must come from the post-top-up + Series before top-up: 3.00 -> 2.00 -> 1.00. Then balance jumps to 10.00 and + resumes decreasing 10.00 -> 9.25 -> 8.50. Burn must come from the post-top-up segment (1.50 USD / 1.5 h = 1.0 USD/h), not from the overall MAX-MIN which would invent a phantom burn of ~7.00 USD. """ - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() rows = [ @@ -257,19 +281,20 @@ def test_quota_burn_with_top_up_ignores_credit_jump(tmp_path): conn.commit() result = quota_burn(conn, cfg) - assert result["balance_usd"] == pytest.approx(8.50) - assert result["balance_at"] == rows[-1][2].isoformat() - assert result["burn_window_hours"] == 24 - assert result["burn_rate_usd_per_hour"] == pytest.approx(1.0) - assert result["projected_hours_remaining"] == pytest.approx(8.5) - assert result["runway_low_warning"] is False + entry = result["by_provider"]["neuralwatt"] + assert entry["balance_usd"] == pytest.approx(8.50) + assert entry["balance_at"] == rows[-1][2].isoformat() + assert entry["balance_source"] == "telemetry" + assert entry["burn_window_hours"] == 24 + assert entry["burn_rate_usd_per_hour"] == pytest.approx(1.0) + assert entry["projected_hours_remaining"] == pytest.approx(8.5) + assert entry["runway_low_warning"] is False + assert result["total_balance_usd"] == pytest.approx(8.50) def test_quota_burn_with_top_up_immediately_before_window_end_returns_none(tmp_path): - """Top-up with only a short final tail → burn is unavailable, never wild.""" - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + """Top-up with only a short final tail -> burn is unavailable, never wild.""" + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() rows = [ @@ -290,18 +315,17 @@ def test_quota_burn_with_top_up_immediately_before_window_end_returns_none(tmp_p conn.commit() result = quota_burn(conn, cfg) - assert result["burn_rate_usd_per_hour"] is None - assert result["runway_note"] is not None - assert "burn estimate unavailable" in result["runway_note"] - assert result["projected_hours_remaining"] is None - assert result["runway_low_warning"] is False + entry = result["by_provider"]["neuralwatt"] + assert entry["burn_rate_usd_per_hour"] is None + assert entry["runway_note"] is not None + assert "burn estimate unavailable" in entry["runway_note"] + assert entry["projected_hours_remaining"] is None + assert entry["runway_low_warning"] is False def test_quota_burn_all_null_allowance_degrades_gracefully(tmp_path): - """All allowance_remaining_usd NULL → no balance, no burn, no false alarm.""" - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + """All allowance_remaining_usd NULL -> no balance, no burn, no false alarm.""" + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() rows = [ @@ -319,20 +343,21 @@ def test_quota_burn_all_null_allowance_degrades_gracefully(tmp_path): result = quota_burn(conn, cfg) assert result is not None - assert result["balance_usd"] is None - assert result["balance_at"] is None - assert result["burn_rate_usd_per_hour"] is None - assert result["projected_hours_remaining"] is None - assert result["runway_low_warning"] is False - assert result["runway_note"] is not None + entry = result["by_provider"]["neuralwatt"] + assert entry["balance_usd"] is None + assert entry["balance_at"] is None + assert entry["balance_source"] == "telemetry" + assert entry["burn_rate_usd_per_hour"] is None + assert entry["projected_hours_remaining"] is None + assert entry["runway_low_warning"] is False + assert entry["runway_note"] is not None + assert result["total_balance_usd"] is None assert result["metered_kwh_30d"] == pytest.approx(5.0) def test_quota_burn_runway_warning_when_below_threshold(tmp_path): """Low balance and positive burn below warning threshold triggers warning.""" - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() rows = [ @@ -351,10 +376,56 @@ def test_quota_burn_runway_warning_when_below_threshold(tmp_path): conn.commit() result = quota_burn(conn, cfg) - assert result["burn_rate_usd_per_hour"] == pytest.approx(0.4) - assert result["projected_hours_remaining"] == pytest.approx(1.5) - assert result["runway_low_warning"] is True - assert result["burn_window_hours"] == 24 + entry = result["by_provider"]["neuralwatt"] + assert entry["burn_rate_usd_per_hour"] == pytest.approx(0.4) + assert entry["projected_hours_remaining"] == pytest.approx(1.5) + assert entry["runway_low_warning"] is True + assert entry["burn_window_hours"] == 24 + + +def test_quota_burn_min_samples_guard_per_provider(tmp_path): + """A post-top-up segment with only 2 samples cannot compute a burn rate.""" + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) + conn = _make_db(tmp_path) + now = _now() + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 10.0, (now - timedelta(hours=2)).isoformat()), + ) + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 9.0, (now - timedelta(minutes=30)).isoformat()), + ) + conn.commit() + + result = quota_burn(conn, cfg) + entry = result["by_provider"]["neuralwatt"] + assert entry["burn_rate_usd_per_hour"] is None + assert "need 3" in entry["runway_note"] + + +def test_quota_burn_min_hours_guard_per_provider(tmp_path): + """A 3-sample segment spanning only 10 minutes cannot compute a burn rate.""" + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) + conn = _make_db(tmp_path) + now = _now() + for value, minutes_ago in ((10.0, 10), (9.0, 7), (8.0, 1)): + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", value, (now - timedelta(minutes=minutes_ago)).isoformat()), + ) + conn.commit() + + result = quota_burn(conn, cfg) + entry = result["by_provider"]["neuralwatt"] + assert entry["burn_rate_usd_per_hour"] is None + assert "need at least 0.5 h" in entry["runway_note"] def test_quota_burn_billing_period_kwh_excludes_rolling_window(tmp_path): @@ -363,9 +434,7 @@ def test_quota_burn_billing_period_kwh_excludes_rolling_window(tmp_path): Rows are pinned on either side of each boundary so the expected values can be computed from first principles without relying on a helper. """ - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() today = now.date() @@ -406,20 +475,19 @@ def test_quota_burn_billing_period_kwh_excludes_rolling_window(tmp_path): ) result = quota_burn(conn, cfg) + assert "by_provider" in result + assert "total_balance_usd" in result assert "metered_kwh_period" in result assert "window_start_30d" in result assert result["metered_kwh_period"] == pytest.approx(expected_period) assert result["metered_kwh_30d"] == pytest.approx(expected_rolling) assert result["metered_fraction_of_plan"] == pytest.approx(expected_period / 6.25) + assert result["by_provider"]["neuralwatt"]["balance_source"] == "telemetry" -def test_quota_burn_includes_balance_and_runway_keys(tmp_path): - """quota_burn returns the full balance/burn/runway key set, and the - standalone quota_balance_and_burn helper matches it. - """ - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) +def test_quota_burn_per_provider_shape_matches_standalone_helper(tmp_path): + """quota_burn carries by_provider/total_balance_usd and no flat keys.""" + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) now = _now() rows = [ @@ -438,50 +506,35 @@ def test_quota_burn_includes_balance_and_runway_keys(tmp_path): conn.commit() result = quota_burn(conn, cfg) - for key in ( - "plan_kwh", - "metered_kwh_30d", - "metered_kwh_period", - "metered_fraction_of_plan", - "metered_calls_30d", - "note", - "reset_date", - "window_start_30d", + flat_balance_keys = { "balance_usd", "balance_at", - "burn_window_hours", "burn_rate_usd_per_hour", "projected_hours_remaining", "runway_low_warning", "runway_note", - "next_reset_date", - ): - assert key in result, f"missing key {key!r}" - - from metrics import quota_balance_and_burn - - balance_only = quota_balance_and_burn(conn, cfg) - for key in ( - "balance_usd", - "balance_at", "burn_window_hours", - "burn_rate_usd_per_hour", - "projected_hours_remaining", - "runway_low_warning", - "runway_note", - ): - assert key in balance_only, f"missing key {key!r}" - assert balance_only[key] == result[key] + } + for key in flat_balance_keys: + assert key not in result, f"flat key {key!r} must not be top-level in quota_burn" + + assert isinstance(result["by_provider"], dict) + assert "neuralwatt" in result["by_provider"] + entry = result["by_provider"]["neuralwatt"] + for key in flat_balance_keys | {"balance_source"}: + assert key in entry, f"per-provider key {key!r} missing" + assert result["total_balance_usd"] == pytest.approx(0.60) + + standalone = quota_balance_and_burn(conn, cfg) + assert standalone["by_provider"]["neuralwatt"] == entry + assert standalone["total_balance_usd"] == result["total_balance_usd"] def test_quota_burn_reset_date_is_billing_period_start(tmp_path): """reset_date is the billing-period start; window_start_30d is rolling 30 d.""" - cfg = SimpleNamespace( - objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6) - ) + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25, billing_reset_day=6) conn = _make_db(tmp_path) - now = datetime.now(timezone.utc) - today = now.date() + today = _now().date() reset_day = 6 if today.day >= reset_day: expected_period_start = date(today.year, today.month, reset_day).isoformat() @@ -495,6 +548,8 @@ def test_quota_burn_reset_date_is_billing_period_start(tmp_path): result = quota_burn(conn, cfg) assert result["reset_date"] == expected_period_start assert result["window_start_30d"] == expected_rolling_start + assert "by_provider" in result + assert "neuralwatt" in result["by_provider"] cfg_unconfigured = SimpleNamespace( objective=SimpleNamespace(plan_kwh_per_period=6.25) @@ -503,6 +558,340 @@ def test_quota_burn_reset_date_is_billing_period_start(tmp_path): assert "reset_date" in result_unconfigured assert result_unconfigured["reset_date"] is None assert result_unconfigured["window_start_30d"] == expected_rolling_start + assert result_unconfigured["by_provider"] == {} + assert result_unconfigured["total_balance_usd"] is None + + +# --- per-provider QA cases ----------------------------------------------------- + + +def test_quota_balance_interleaved_two_providers_is_independent(tmp_path): + """THE regression test: interleaved two-provider series compute separately. + + Two providers' rows are interleaved in time so a global view would mix + them. energy_observations also carries poison allowance rows for + openrouter (genuinely possible: dispatch rows carry foreign-shaped + allowance data) that are NEWER than every neuralwatt row — a global + "latest allowance" read would report 80.0 for everyone. Each provider's + entry must instead trace to its own source rows, with DIFFERENT burn + rates so a merged/globally-latest regression cannot pass silently. + """ + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + "openrouter": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + now = _now() + + # NeuralWatt telemetry series: burn 1.0 USD/h, balance 2.0, runway 2 h. + for bal, at in ( + (5.0, now - timedelta(hours=4)), + (4.0, now - timedelta(hours=3)), + (3.0, now - timedelta(hours=2)), + (2.0, now - timedelta(hours=1)), + ): + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", bal, at.isoformat()), + ) + + # Poison: openrouter allowance rows in energy_observations NEWER than every + # neuralwatt row. A global-latest implementation would report 80.0. + for bal, at in ( + (100.0, now - timedelta(minutes=50)), + (90.0, now - timedelta(minutes=35)), + (80.0, now - timedelta(minutes=10)), + ): + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'openrouter', 0.001, ?, ?)", + ("m", bal, at.isoformat()), + ) + + # OpenRouter's real source, offset 30 min from the neuralwatt rows so the + # series visibly interleave: burn 4.0 USD/h, balance 8.0, runway 2 h. + for bal, at in ( + (20.0, now - timedelta(minutes=210)), + (16.0, now - timedelta(minutes=150)), + (12.0, now - timedelta(minutes=90)), + (8.0, now - timedelta(minutes=30)), + ): + conn.execute( + "INSERT INTO provider_balance_observations " + "(provider, balance_usd, observed_at) " + "VALUES ('openrouter', ?, ?)", + (bal, at.isoformat()), + ) + conn.commit() + + result = quota_balance_and_burn(conn, cfg) + by_provider = result["by_provider"] + + nw = by_provider["neuralwatt"] + assert nw["balance_usd"] == pytest.approx(2.0) + assert nw["balance_at"] == (now - timedelta(hours=1)).isoformat() + assert nw["burn_rate_usd_per_hour"] == pytest.approx(1.0) + assert nw["projected_hours_remaining"] == pytest.approx(2.0) + assert nw["runway_low_warning"] is True + assert nw["balance_source"] == "telemetry" + + orr = by_provider["openrouter"] + assert orr["balance_usd"] == pytest.approx(8.0) + assert orr["balance_at"] == (now - timedelta(minutes=30)).isoformat() + assert orr["burn_rate_usd_per_hour"] == pytest.approx(4.0) + assert orr["projected_hours_remaining"] == pytest.approx(2.0) + assert orr["runway_low_warning"] is True + + # The two providers must differ where their series differ; the poison + # rows (80.0, or any burn computed over a mixed series) must not appear. + assert nw["balance_usd"] != orr["balance_usd"] + assert nw["burn_rate_usd_per_hour"] != orr["burn_rate_usd_per_hour"] + assert result["total_balance_usd"] == pytest.approx(10.0) + + +def test_quota_balance_polled_series_equivalence(tmp_path): + """A polled series computes the same burn/runway as telemetry would.""" + conn = _make_db(tmp_path) + now = _now() + for bal, hours_ago in ((5.0, 3), (4.0, 2), (3.0, 1)): + conn.execute( + "INSERT INTO provider_balance_observations " + "(provider, balance_usd, observed_at) " + "VALUES ('openrouter', ?, ?)", + (bal, (now - timedelta(hours=hours_ago)).isoformat()), + ) + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", bal, (now - timedelta(hours=hours_ago)).isoformat()), + ) + conn.commit() + + polled_cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25), + dispatch_providers={ + "openrouter": SimpleNamespace( + has_energy_telemetry=False, + balance_url="https://openrouter.ai/api/v1/credits", + ), + }, + ) + telemetry_cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25) + + polled = quota_balance_and_burn(conn, polled_cfg)["by_provider"]["openrouter"] + telemetry = quota_balance_and_burn(conn, telemetry_cfg)["by_provider"]["neuralwatt"] + + assert polled["balance_usd"] == telemetry["balance_usd"] == pytest.approx(3.0) + assert polled["balance_at"] == telemetry["balance_at"] + assert polled["burn_rate_usd_per_hour"] == pytest.approx(1.0) + assert polled["burn_rate_usd_per_hour"] == telemetry["burn_rate_usd_per_hour"] + assert polled["projected_hours_remaining"] == telemetry["projected_hours_remaining"] + assert polled["runway_low_warning"] == telemetry["runway_low_warning"] + assert polled["balance_source"] == "polled" + assert telemetry["balance_source"] == "telemetry" + + +def test_quota_balance_top_up_guard_per_provider(tmp_path): + """A top-up for one provider resets only that provider's segment.""" + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + "openrouter": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + now = _now() + + # NeuralWatt: top-up to 4.0 at -60m; the post-top-up segment (4.0 -> 2.0 + # over 1 h) burns 2.0 USD/h. Without the reset, the flat 3.0 -> 2.0 span + # would report 0.5 USD/h. + for bal, at in ( + (3.0, now - timedelta(hours=2)), + (2.0, now - timedelta(minutes=90)), + (4.0, now - timedelta(minutes=60)), # top-up + (3.0, now - timedelta(minutes=40)), + (2.6, now - timedelta(minutes=20)), + (2.0, now), + ): + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", bal, at.isoformat()), + ) + + # OpenRouter: monotonic decrease, burn (8 - 2) / 2 h = 3.0 USD/h. + for bal, at in ( + (8.0, now - timedelta(hours=2)), + (5.0, now - timedelta(hours=1)), + (2.0, now), + ): + conn.execute( + "INSERT INTO provider_balance_observations " + "(provider, balance_usd, observed_at) " + "VALUES ('openrouter', ?, ?)", + (bal, at.isoformat()), + ) + conn.commit() + + result = quota_balance_and_burn(conn, cfg) + assert result["by_provider"]["neuralwatt"]["burn_rate_usd_per_hour"] == pytest.approx(2.0) + assert result["by_provider"]["openrouter"]["burn_rate_usd_per_hour"] == pytest.approx(3.0) + + +def test_quota_balance_no_data_provider_all_none(tmp_path): + """A configured provider with no rows gets an all-None entry and a note. + + total_balance_usd sums only the providers that report a balance. + """ + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=6), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + "openrouter": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 5.0, _now().isoformat()), + ) + conn.commit() + + result = quota_balance_and_burn(conn, cfg) + assert result["by_provider"]["neuralwatt"]["balance_usd"] == pytest.approx(5.0) + openrouter = result["by_provider"]["openrouter"] + assert openrouter["balance_usd"] is None + assert openrouter["runway_note"] is not None + assert result["total_balance_usd"] == pytest.approx(5.0) + + +def test_quota_balance_total_none_when_no_provider_reports(tmp_path): + """total_balance_usd is None when no provider reports a balance.""" + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25), + dispatch_providers={ + "openrouter": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + result = quota_balance_and_burn(conn, cfg) + assert result["by_provider"]["openrouter"]["balance_usd"] is None + assert result["total_balance_usd"] is None + + +def test_quota_balance_providers_absent_empty(tmp_path): + """A cfg without dispatch_providers yields an empty by_provider mapping.""" + cfg = SimpleNamespace(objective=SimpleNamespace(plan_kwh_per_period=6.25)) + conn = _make_db(tmp_path) + result = quota_balance_and_burn(conn, cfg) + assert result["by_provider"] == {} + assert result["total_balance_usd"] is None + + +def test_quota_balance_source_correctness(tmp_path): + """balance_source reflects the config-derived source, even all-None.""" + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + "openrouter": SimpleNamespace( + has_energy_telemetry=False, + balance_url="https://openrouter.ai/api/v1/credits", + ), + "bare": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + result = quota_balance_and_burn(conn, cfg) + by_provider = result["by_provider"] + assert by_provider["neuralwatt"]["balance_source"] == "telemetry" + assert by_provider["openrouter"]["balance_source"] == "polled" + assert by_provider["bare"]["balance_source"] == "unconfigured" + # Present on all-None entries, not just on providers with data. + for provider in ("neuralwatt", "openrouter", "bare"): + assert "balance_source" in by_provider[provider] + + +def test_quota_balance_missing_provider_balance_table_guard(tmp_path): + """A live DB predating provider_balance_observations must not 500 /metrics. + + The T1 schema landed after live DBs were created; the guard degrades the + polled provider to an all-None entry while telemetry providers keep + working. + """ + cfg = SimpleNamespace( + objective=SimpleNamespace(plan_kwh_per_period=6.25), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True), + "openrouter": SimpleNamespace(has_energy_telemetry=False), + }, + ) + conn = _make_db(tmp_path) + now = _now() + for bal, hours_ago in ((5.0, 3), (4.0, 2), (3.0, 1)): + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", bal, (now - timedelta(hours=hours_ago)).isoformat()), + ) + conn.commit() + conn.execute("DROP TABLE provider_balance_observations") + conn.commit() + + result = quota_balance_and_burn(conn, cfg) + assert result["by_provider"]["neuralwatt"]["balance_usd"] == pytest.approx(3.0) + openrouter = result["by_provider"]["openrouter"] + assert openrouter["balance_usd"] is None + assert openrouter["runway_note"] is not None + + +def test_quota_balance_malformed_timestamp_skipped(tmp_path): + """A malformed observed_at degrades to fewer samples, never a crash. + + With the malformed row (6.0) skipped, the neuralwatt series holds only two + valid samples, so the min-samples guard fires. If the malformed row were + counted, three samples over 2 h would produce a 4.0 USD/h burn. + """ + cfg = _telemetry_provider_cfg(plan_kwh_per_period=6.25) + conn = _make_db(tmp_path) + now = _now() + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 10.0, (now - timedelta(hours=3)).isoformat()), + ) + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 6.0, ""), + ) + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, energy_kwh, allowance_remaining_usd, observed_at) " + "VALUES (?, 'neuralwatt', 0.001, ?, ?)", + ("m", 2.0, (now - timedelta(hours=1)).isoformat()), + ) + conn.commit() + + result = quota_balance_and_burn(conn, cfg) + entry = result["by_provider"]["neuralwatt"] + assert entry["balance_usd"] == pytest.approx(2.0) + assert entry["burn_rate_usd_per_hour"] is None + assert entry["runway_note"] is not None def test_local_energy_summary_reset_date_is_billing_period_start(tmp_path): diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index 4c8f408..057cb86 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -226,24 +226,27 @@ def test_metrics_contains_no_session_dir(seeded_client): assert "session_dir" not in body -def test_metrics_quota_carries_balance_and_runway_keys(seeded_client): - """GET /metrics quota section includes the new balance/burn/runway keys - alongside the existing kWh/keys. With no allowance seeded, kWh fields are - present and balance/burn values are None or empty, but keys must exist. - """ +def test_metrics_quota_carries_per_provider_balance_and_total(seeded_client): + """GET /metrics quota carries by_provider + total_balance_usd, no flat keys.""" resp = seeded_client.get("/metrics") assert resp.status_code == 200 data = resp.json() assert data["quota"] is not None quota = data["quota"] - legacy_keys = ( + for key in ( "plan_kwh", "metered_kwh_30d", + "metered_kwh_period", "metered_fraction_of_plan", "metered_calls_30d", "note", - ) - new_keys = ( + "window_start_30d", + ): + assert key in quota, f"missing quota key {key!r}" + + assert "by_provider" in quota + assert "total_balance_usd" in quota + flat_balance_keys = ( "balance_usd", "balance_at", "burn_window_hours", @@ -251,11 +254,37 @@ def test_metrics_quota_carries_balance_and_runway_keys(seeded_client): "projected_hours_remaining", "runway_low_warning", "runway_note", - "window_start_30d", - "metered_kwh_period", ) - for key in legacy_keys + new_keys: - assert key in quota, f"missing quota key {key!r}" + for key in flat_balance_keys: + assert key not in quota, f"flat key {key!r} must not be top-level in quota" + + for provider, entry in quota["by_provider"].items(): + for key in flat_balance_keys + ("balance_source",): + assert key in entry, f"per-provider key {key!r} missing for {provider}" + + +def test_metrics_quota_survives_missing_provider_balance_table(tmp_path, monkeypatch): + """A live DB predating provider_balance_observations must not 500 /metrics.""" + conn = _make_db(tmp_path) + _seed_models(conn) + _seed_decision(conn) + conn.execute("DROP TABLE provider_balance_observations") + conn.commit() + conn.close() + + monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db")) + monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False) + monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False) + monkeypatch.setattr(dispatcher.cfg.local_energy, "enabled", False) + monkeypatch.setenv("NEURALWATT_API_KEY", "test-key") + + with TestClient(dispatcher.app) as client: + resp = client.get("/metrics") + assert resp.status_code == 200 + quota = resp.json()["quota"] + assert "by_provider" in quota + for entry in quota["by_provider"].values(): + assert entry["balance_usd"] is None def test_metrics_local_energy_omitted_when_disabled(seeded_client): -- 2.49.1 From aa593005752c96c018541a5cded1715cc3605e70 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:01:03 -0400 Subject: [PATCH 07/13] feat(poller): poll provider account balances inside the catalog loop --- config/config.yaml | 3 + src/poller.py | 43 +++++- tests/test_multi_provider_poller.py | 201 ++++++++++++++++++++++++++-- 3 files changed, 231 insertions(+), 16 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 8807b55..f4ff13a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -756,6 +756,9 @@ dispatch_providers: openrouter: base_url: "https://openrouter.ai/api/v1" api_key_env: "OPENROUTER_API_KEY" + # Account-level balance poll; OpenRouter reports per-completion energy via + # a separate allowance_remaining field, so this URL is for the prepaid pool. + balance_url: "https://openrouter.ai/api/v1/credits" has_energy_telemetry: false enabled: true diff --git a/src/poller.py b/src/poller.py index 62f4508..a013e0e 100644 --- a/src/poller.py +++ b/src/poller.py @@ -22,16 +22,17 @@ migration. from __future__ import annotations +import os import sqlite3 import sys import warnings from dataclasses import dataclass from datetime import datetime, timezone -from typing import Optional +from typing import Callable, Optional import requests -from config import LocalDispatchModel, RouterConfig, load_config +from config import DispatchProvider, LocalDispatchModel, RouterConfig, load_config from tier import apply_tiering MODELS_URL = "https://api.neuralwatt.com/v1/models" @@ -58,6 +59,15 @@ SUFFIX_FAST = "fast" SUFFIX_SHORT = "short" SERVING_SUFFIXES = frozenset({SUFFIX_FLEX, SUFFIX_FAST, SUFFIX_SHORT}) +# Provider-specific account-balance parsers. The registry lives inside the +# poller because poller imports config, and config importing this dict would +# create a circular dependency. Tests pin this set equal to +# config.PROVIDERS_WITH_BALANCE_PARSERS. +_BALANCE_PARSERS: dict[str, Callable[[dict], float]] = { + "openrouter": lambda payload: payload["data"]["total_credits"] + - payload["data"]["total_usage"] +} + class CatalogTooSmall(requests.RequestException): """Raised when the fetched catalog has zero rows — not a transient error, @@ -67,6 +77,14 @@ class CatalogTooSmall(requests.RequestException): exit with code 1 without any extra machinery.""" +def _auth_header(prov_cfg: DispatchProvider) -> Optional[dict[str, str]]: + """Return a Bearer header if the provider's API key env var is set.""" + key = os.environ.get(prov_cfg.api_key_env) + if not key: + return None + return {"Authorization": f"Bearer {key}"} + + def parse_serving_class(model_id: str) -> tuple[str, str, str]: """Derive (latency_class, reasoning_mode, context_variant) from a model id. @@ -539,6 +557,7 @@ def main() -> int: conn.execute("PRAGMA foreign_keys = ON") upsert_local_dispatch_models(conn, cfg) + _ensure_provider_balance_table(conn) # Map provider keys (dispatch_providers dict keys) to their fetch functions. # Only entries with keys matching the key are dispatched; unknown keys @@ -556,6 +575,26 @@ def main() -> int: print(f"[{provider}] skipped (disabled)") continue + if prov_cfg.balance_url: + headers = _auth_header(prov_cfg) + if headers is None: + print( + f"[{provider}] balance poll skipped: " + f"{prov_cfg.api_key_env} not set", + file=sys.stderr, + ) + else: + try: + resp = requests.get( + prov_cfg.balance_url, headers=headers, timeout=REQUEST_TIMEOUT + ) + resp.raise_for_status() + balance = _BALANCE_PARSERS[provider](resp.json()) + record_balance(conn, provider, balance) + print(f"[{provider}] balance ${balance:.2f}") + except (requests.RequestException, KeyError, TypeError, ValueError) as e: + print(f"[{provider}] balance poll FAILED: {e}", file=sys.stderr) + fetcher = FETCHERS.get(provider) if fetcher is None: print(f"[{provider}] unknown provider — skipping (no fetcher)") diff --git a/tests/test_multi_provider_poller.py b/tests/test_multi_provider_poller.py index b6f4e20..30a2423 100644 --- a/tests/test_multi_provider_poller.py +++ b/tests/test_multi_provider_poller.py @@ -244,20 +244,13 @@ def test_one_provider_fails_other_succeeds(tmp_db, monkeypatch): """Failure in one provider's fetch does not abort the other.""" connect, cfg = tmp_db - call_count = 0 - - def _failing_get(url, timeout): - nonlocal call_count - call_count += 1 - if "openrouter" in url: - # OpenRouter fetch goes first (dict iteration order) - if call_count == 1: - return _fake_response( - {"data": [_openrouter_model("openai/gpt-6-astra")]} - ) - # Second neuralwatt call - return _fake_response({"data": [_catalog_item("model_a")]}) - # NeuralWatt fails + def _failing_get(url, timeout, **kwargs): + if "/credits" in url: + # OpenRouter account-balance poll happens before the catalog fetch. + return _fake_response({"data": {"total_credits": 50.0, "total_usage": 0.0}}) + if "openrouter.ai" in url: + return _fake_response({"data": [_openrouter_model("openai/gpt-6-astra")]}) + # NeuralWatt catalog fails raise requests.RequestException("network error") monkeypatch.setattr(poller, "load_config", lambda p: cfg) @@ -674,3 +667,183 @@ def test_per_provider_mark_stale_scoped(tmp_path): assert stale_or == 0, "openrouter row should NOT be stale when scoped to neuralwatt" conn.close() + + +# --------------------------------------------------------------------------- +# Balance polling tests +# --------------------------------------------------------------------------- + + +OPENROUTER_BALANCE_URL = REAL_CFG.dispatch_providers["openrouter"].balance_url + + +def _url_keyed_get(url: str, timeout: object, **kwargs: object) -> MagicMock: + """Dispatch mocked HTTP responses by URL for balance + catalog tests.""" + if "/credits" in url: + return _fake_response( + {"data": {"total_credits": 50.0, "total_usage": 7.0}} + ) + if "openrouter.ai" in url: + return _fake_response({"data": [_openrouter_model("openai/gpt-6-astra")]}) + if "neuralwatt" in url: + return _fake_response({"data": [_catalog_item("nw-model-a")]}) + raise RuntimeError(f"unexpected URL in test mock: {url}") + + +def test_balance_poll_success_records_row(tmp_db, monkeypatch): + """An OpenRouter balance poll writes one provider_balance_observations row.""" + connect, cfg = tmp_db + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr(poller.requests, "get", _url_keyed_get) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") + + exit_code = poller.main() + assert exit_code == 0 + + conn = connect() + try: + row = conn.execute( + "SELECT provider, balance_usd FROM provider_balance_observations " + "WHERE provider = ?", + ("openrouter",), + ).fetchone() + assert row is not None, "expected a balance row for openrouter" + assert row["provider"] == "openrouter" + assert row["balance_usd"] == 43.0 + finally: + conn.close() + + +def test_balance_parser_registry_matches_config_registry(): + """poller._BALANCE_PARSERS keys equal config.PROVIDERS_WITH_BALANCE_PARSERS.""" + from config import PROVIDERS_WITH_BALANCE_PARSERS + + assert set(poller._BALANCE_PARSERS) == PROVIDERS_WITH_BALANCE_PARSERS + + +def test_balance_poll_failure_is_isolated(tmp_db, monkeypatch): + """A balance-poll exception does not abort the rest of the poll run.""" + connect, cfg = tmp_db + + def _get(url, timeout, **kwargs): + if "/credits" in url: + raise requests.ConnectionError("balance endpoint unreachable") + if "openrouter.ai" in url: + return _fake_response({"data": [_openrouter_model("openai/gpt-6-astra")]}) + if "neuralwatt" in url: + return _fake_response({"data": [_catalog_item("nw-model-a")]}) + raise RuntimeError(f"unexpected URL: {url}") + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr(poller.requests, "get", _get) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") + + import io + + stderr_capture = io.StringIO() + monkeypatch.setattr("sys.stderr", stderr_capture) + + exit_code = poller.main() + assert exit_code == 0 + assert "[openrouter] balance poll FAILED" in stderr_capture.getvalue() + + conn = connect() + try: + # NeuralWatt catalog still upserted + assert conn.execute( + "SELECT COUNT(*) FROM models WHERE provider='neuralwatt'" + ).fetchone()[0] == 1 + # No balance row because the poll failed + assert conn.execute( + "SELECT COUNT(*) FROM provider_balance_observations" + ).fetchone()[0] == 0 + finally: + conn.close() + + +def test_balance_poll_skipped_when_api_key_missing(tmp_db, monkeypatch): + """Without OPENROUTER_API_KEY the balance poll is skipped but the run continues.""" + connect, cfg = tmp_db + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr(poller.requests, "get", _url_keyed_get) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + + import io + + stderr_capture = io.StringIO() + monkeypatch.setattr("sys.stderr", stderr_capture) + + exit_code = poller.main() + assert exit_code == 0 + assert "[openrouter] balance poll skipped: OPENROUTER_API_KEY not set" in stderr_capture.getvalue() + + conn = connect() + try: + assert conn.execute( + "SELECT COUNT(*) FROM provider_balance_observations" + ).fetchone()[0] == 0 + finally: + conn.close() + + +def test_balance_parser_drift_caught_not_fatal(tmp_db, monkeypatch): + """A malformed credits response is logged and the run continues.""" + connect, cfg = tmp_db + + def _get(url, timeout, **kwargs): + if "/credits" in url: + return _fake_response({"data": {}}) + if "openrouter.ai" in url: + return _fake_response({"data": [_openrouter_model("openai/gpt-6-astra")]}) + if "neuralwatt" in url: + return _fake_response({"data": [_catalog_item("nw-model-a")]}) + raise RuntimeError(f"unexpected URL: {url}") + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + monkeypatch.setattr(poller.requests, "get", _get) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") + + import io + + stderr_capture = io.StringIO() + monkeypatch.setattr("sys.stderr", stderr_capture) + + exit_code = poller.main() + assert exit_code == 0 + assert "[openrouter] balance poll FAILED" in stderr_capture.getvalue() + + +def test_disabled_provider_with_balance_url_gets_no_http_call(tmp_db, monkeypatch): + """An enabled=False provider with balance_url configured is skipped entirely.""" + connect, cfg = tmp_db + + or_cfg = REAL_CFG.dispatch_providers["openrouter"] + disabled_or = or_cfg.model_copy(update={"enabled": False}) + cfg = cfg.model_copy( + update={ + "dispatch_providers": { + "neuralwatt": REAL_CFG.dispatch_providers["neuralwatt"], + "openrouter": disabled_or, + } + } + ) + + monkeypatch.setattr(poller, "load_config", lambda p: cfg) + call_log: list[str] = [] + + def _get(url, timeout, **kwargs): + call_log.append(url) + if "neuralwatt" in url: + return _fake_response({"data": [_catalog_item("nw-model-a")]}) + if "openrouter.ai" in url: + return _fake_response({"data": [_openrouter_model("openai/gpt-6-astra")]}) + raise RuntimeError(f"unexpected URL: {url}") + + monkeypatch.setattr(poller.requests, "get", _get) + + exit_code = poller.main() + assert exit_code == 0 + assert not any("/credits" in u for u in call_log) + assert not any("openrouter.ai" in u for u in call_log) -- 2.49.1 From 98b77a20286953804f7bff968bea711db80847f1 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:11:43 -0400 Subject: [PATCH 08/13] fix(metrics): quota report carries by_provider and total_balance_usd; flat keys removed --- src/metrics.py | 78 +++++++++++++++++++++++++++++--------- tests/test_tui_warnings.py | 3 ++ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 5e430cc..fc17b48 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -357,28 +357,70 @@ def quota_burn( plan = cfg.objective.plan_kwh_per_period window_start_30d = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat() - row = conn.execute( - """ - SELECT COALESCE(SUM(energy_kwh), 0) kwh, COUNT(*) n - FROM energy_observations - WHERE julianday(observed_at) > julianday('now', '-30 days') - """ - ).fetchone() - metered_kwh_30d = round(float(row["kwh"]), 5) - metered_calls_30d = row["n"] + providers = getattr(cfg, "dispatch_providers", None) + if providers is None: + # SimpleNamespace fixture compatibility: cfg has no dispatch_providers, + # so keep the historical unscoped SUM/COUNT behavior. + telemetry_providers: Optional[list[str]] = None + else: + telemetry_providers = [ + name + for name, pc in providers.items() + if getattr(pc, "has_energy_telemetry", False) + ] + + if telemetry_providers == []: + # No provider can meter energy. Both sums are honestly zero; do not + # run the queries with an empty IN () clause (SQLite syntax error). + metered_kwh_30d = 0.0 + metered_calls_30d = 0 + else: + if telemetry_providers: + telemetry_placeholders = ",".join("?" * len(telemetry_providers)) + thirty_where = ( + "julianday(observed_at) > julianday('now', '-30 days') " + f"AND provider IN ({telemetry_placeholders})" + ) + thirty_params = tuple(telemetry_providers) + else: + thirty_where = "julianday(observed_at) > julianday('now', '-30 days')" + thirty_params = () + row = conn.execute( + f""" + SELECT COALESCE(SUM(energy_kwh), 0) kwh, COUNT(*) n + FROM energy_observations + WHERE {thirty_where} + """, + thirty_params, + ).fetchone() + metered_kwh_30d = round(float(row["kwh"]), 5) + metered_calls_30d = row["n"] reset_day = getattr(cfg.objective, "billing_reset_day", None) if reset_day is not None: period_start = _billing_period_start(reset_day) - period_row = conn.execute( - """ - SELECT COALESCE(SUM(energy_kwh), 0) kwh - FROM energy_observations - WHERE julianday(observed_at) >= julianday(?) - """, - (period_start,), - ).fetchone() - metered_kwh_period = round(float(period_row["kwh"]), 5) + if telemetry_providers == []: + metered_kwh_period = 0.0 + else: + if telemetry_providers: + period_placeholders = ",".join("?" * len(telemetry_providers)) + period_where = ( + "julianday(observed_at) >= julianday(?) " + f"AND provider IN ({period_placeholders})" + ) + period_params = (period_start, *telemetry_providers) + else: + period_where = "julianday(observed_at) >= julianday(?)" + period_params = (period_start,) + period_row = conn.execute( + f""" + SELECT COALESCE(SUM(energy_kwh), 0) kwh + FROM energy_observations + WHERE {period_where} + """, + period_params, + ).fetchone() + metered_kwh_period = round(float(period_row["kwh"]), 5) metered_fraction_of_plan = round(metered_kwh_period / plan, 4) reset_date = period_start else: diff --git a/tests/test_tui_warnings.py b/tests/test_tui_warnings.py index 965d129..daf0264 100644 --- a/tests/test_tui_warnings.py +++ b/tests/test_tui_warnings.py @@ -64,6 +64,9 @@ CFG = SimpleNamespace( default_flex_preference=SimpleNamespace(value="auto"), ), objective=SimpleNamespace(plan_kwh_per_period=6.25, billing_reset_day=None), + dispatch_providers={ + "neuralwatt": SimpleNamespace(has_energy_telemetry=True, balance_url=None) + }, escalation=SimpleNamespace(enabled=True), classifier=SimpleNamespace(degraded_warn_min=20, degraded_warn_threshold=0.5), ) -- 2.49.1 From a5e1b9d1a5d05836041cd4cd845e41430f655852 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:17:25 -0400 Subject: [PATCH 09/13] feat(dispatcher): credit-aware routing attenuation behind opt-in config --- src/dispatcher.py | 62 ++++ tests/test_credit_attenuation_routing.py | 437 +++++++++++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 tests/test_credit_attenuation_routing.py diff --git a/src/dispatcher.py b/src/dispatcher.py index b406e18..ca43459 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -80,6 +80,7 @@ from metrics import ( local_energy_summary, per_model, pinch_summary, + quota_balance_and_burn, quota_burn, recent_decisions, scoring_coverage, @@ -90,6 +91,7 @@ from routing import ( apply_flex_preference, capability_gate_reason, cheapest_classifier_candidate, + credit_attenuation_multiplier, parse_eligible_categories, rank_candidates, rejection_reason, @@ -668,6 +670,59 @@ def _resolve_auto_classifier() -> Optional[dict]: return picked +_provider_cost_multipliers_cache: Optional[dict[str, float]] = None +_provider_cost_multipliers_resolved_at: float = 0.0 + + +def _provider_cost_multipliers() -> Optional[dict[str, float]]: + """Cached per-provider cost multipliers for credit attenuation. + + Returns ``None`` when the feature is disabled so the routing path performs + no database work and behaves identically to the pre-attenuation code. + + When enabled, only providers configured with ``balance_url`` (the polled + account-balance path, e.g. OpenRouter) receive a multiplier; telemetry + providers such as NeuralWatt, whose ``allowance_remaining_usd`` reflects + overage accounting rather than a depleting prepaid pool, are intentionally + omitted so their normal near-zero readings never bias routing. + """ + global _provider_cost_multipliers_cache, _provider_cost_multipliers_resolved_at + att = cfg.objective.credit_attenuation + if not att.enabled: + return None + + now = time.time() + if ( + _provider_cost_multipliers_cache is not None + and now - _provider_cost_multipliers_resolved_at < att.refresh_seconds + ): + return _provider_cost_multipliers_cache + + conn = _db() + try: + balances = quota_balance_and_burn(conn, cfg) + finally: + conn.close() + + multipliers: dict[str, float] = {} + for provider, entry in balances["by_provider"].items(): + prov_cfg = cfg.dispatch_providers.get(provider) + if prov_cfg is not None and getattr(prov_cfg, "balance_url", None): + multipliers[provider] = credit_attenuation_multiplier( + entry.get("balance_usd"), + soft_floor_usd=att.soft_floor_usd, + zero_floor_usd=att.zero_floor_usd, + max_multiplier=att.max_multiplier, + ) + + if any(v != 1.0 for v in multipliers.values()): + logs.info("credit_attenuation_resolved", provider_multipliers=multipliers) + + _provider_cost_multipliers_cache = multipliers + _provider_cost_multipliers_resolved_at = now + return multipliers + + class _ClassifierSkipped(Exception): """The local classifier was not attempted at all -- gaming mode, or an already-open backoff circuit (see _local_classifier_skip_reason). @@ -1481,6 +1536,7 @@ def route( model=row["model_id"], reason=rejection_reason(row, **filters), ) + provider_cost_multipliers = _provider_cost_multipliers() ranked = rank_candidates( eligible, quality_tolerance=cfg.objective.quality_tolerance, @@ -1489,6 +1545,7 @@ def route( prompt_tokens=classification.required_context_tokens, completion_tokens=cfg.objective.assumed_completion_tokens, cache_rate=cfg.objective.assumed_cache_rate, + provider_cost_multipliers=provider_cost_multipliers, ) if logs.enabled_for_debug(): @@ -1499,6 +1556,11 @@ def route( model=row["model_id"], prof=row["proficiency_score"], est_usd=row.get("cost"), + provider_cost_multipliers=provider_cost_multipliers.get( + row.get("provider") + ) + if provider_cost_multipliers + else None, kwh=row.get("energy"), ) diff --git a/tests/test_credit_attenuation_routing.py b/tests/test_credit_attenuation_routing.py new file mode 100644 index 0000000..58dd14b --- /dev/null +++ b/tests/test_credit_attenuation_routing.py @@ -0,0 +1,437 @@ +"""Dispatcher-level credit attenuation wiring tests. + +These tests exercise the cached resolver and its call sites on the real +``dispatcher.route_endpoint`` TestClient path. They are intentionally +end-to-end for the wiring: pure ``rank_candidates`` cases live in +``tests/test_routing.py``. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +import dispatcher +from config import load_config + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() +CFG = load_config(str(ROOT / "config" / "config.yaml")) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _oldest() -> str: + return (datetime.now(timezone.utc) - timedelta(days=365)).isoformat() + + +def _hours_ago(hours: float) -> str: + return (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat() + + +def _make_db(tmp_path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(str(tmp_path / "test.db")) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA_SQL) + return conn + + +def _seed_two_provider_models(conn: sqlite3.Connection) -> None: + """Two providers, two models: near-identical profile so the tiebreak decides.""" + rows = [ + # (model_id, provider, tier, context_window, cost_per_1m_prompt, + # cost_per_1m_completion, proficiency_category, blended_score) + ( + "nw-cheap", + "neuralwatt", + 2, + 131072, + 0.30, + 0.30, + ), + ( + "or-cheap", + "openrouter", + 2, + 131072, + 0.31, + 0.31, + ), + ] + for model_id, provider, tier, ctx, prompt_cost, completion_cost in rows: + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + 1, 'standard', 'default', 'full', 'public', 'active', + '2026-08-22T00:00:00+00:00') + """, + ( + model_id, + provider, + model_id, + tier, + ctx, + ctx, + 16384, + prompt_cost, + completion_cost, + 1, + ), + ) + conn.execute( + """ + INSERT INTO proficiency + (model_id, provider, category, blended_score, source, last_updated) + VALUES (?, ?, 'coding_general', 0.80, 'self_eval_thin', + '2026-01-01T00:00:00+00:00') + """, + (model_id, provider), + ) + conn.commit() + + +def _seed_provider_balance(conn: sqlite3.Connection, provider: str, balance: float) -> None: + conn.execute( + "INSERT INTO provider_balance_observations (provider, balance_usd, observed_at) " + "VALUES (?, ?, ?)", + (provider, balance, _hours_ago(0.1)), + ) + conn.commit() + + +def _seed_energy_allowance( + conn: sqlite3.Connection, + provider: str, + model_id: str, + allowance_remaining_usd: float, +) -> None: + conn.execute( + "INSERT INTO energy_observations " + "(model_id, provider, task_category, completion_tokens, energy_kwh, " + "cost_usd, carbon_g_co2eq, attribution_ratio, observed_at, " + "allowance_remaining_usd) " + "VALUES (?, ?, 'coding_general', 100, 5.0e-05, 0.001, " + "2.4e-03, 0.25, ?, ?)", + (model_id, provider, _hours_ago(0.1), allowance_remaining_usd), + ) + conn.commit() + + +def _route_response(client: TestClient, task: str = "fix a bug") -> dict: + resp = client.post( + "/route", + json={ + "task": task, + "task_category": "coding_general", + "task_tier": 2, + "required_context_tokens": 1000, + }, + ) + assert resp.status_code == 200, resp.text + return resp.json() + + +@pytest.fixture(autouse=True) +def _reset_provider_multiplier_cache(): + """Every dispatcher-attenuation test starts with a cold cache. + + This also prevents the resolver cache from leaking between tests. + """ + setattr(dispatcher, "_provider_cost_multipliers_cache", None) + setattr(dispatcher, "_provider_cost_multipliers_resolved_at", 0.0) + yield + setattr(dispatcher, "_provider_cost_multipliers_cache", None) + setattr(dispatcher, "_provider_cost_multipliers_resolved_at", 0.0) + + +@pytest.fixture +def route_client(tmp_path, monkeypatch): + """A TestClient wired to a temp DB seeded with two cross-provider models.""" + conn = _make_db(tmp_path) + _seed_two_provider_models(conn) + conn.close() + + monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db")) + monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False) + monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False) + monkeypatch.setattr(dispatcher.cfg.exploration, "enabled", False) + monkeypatch.setenv("NEURALWATT_API_KEY", "test-key") + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key") + + with TestClient(dispatcher.app) as client: + yield client + + +# (a) OFF (default): the resolver never touches the DB, even if a call would raise. + +def test_credit_attenuation_off_by_default_no_db_read( + route_client, monkeypatch +): + """With attenuation disabled, quota_balance_and_burn is never called.""" + calls = [] + + def explode(*args, **kwargs): + calls.append((args, kwargs)) + raise RuntimeError("resolver should not have been called") + + monkeypatch.setattr(dispatcher, "quota_balance_and_burn", explode) + + decision = _route_response(route_client) + assert decision["selected"] is not None + assert calls == [] + + +# (b) ON + all balances healthy → selection matches the no-attenuation baseline. + +def test_credit_attenuation_on_healthy_balances_matches_baseline( + route_client, monkeypatch +): + """Healthy OpenRouter balance keeps the original order (NeuralWatt first).""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 + ) + + decision = _route_response(route_client) + assert decision["selected"]["provider"] == "neuralwatt" + + +# (c) ON + synthetic low OpenRouter balance flips the tiebreak toward the +# healthier NeuralWatt provider. + +def test_credit_attenuation_on_low_openrouter_balance_flips_to_healthy_provider( + route_client, monkeypatch +): + """Low OpenRouter balance inflates its effective cost; NeuralWatt wins.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 + ) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "zero_floor_usd", 0.0 + ) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "max_multiplier", 5.0 + ) + + # No seeded balance row for OpenRouter → balance is None → multiplier 1.0, + # so the baseline winner stays neuralwatt. To flip, give OpenRouter a very + # low but nonzero balance. + db_path = dispatcher.cfg.database.path + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + _seed_provider_balance(conn, "openrouter", 0.01) + conn.close() + + decision = _route_response(route_client) + assert decision["selected"]["provider"] == "neuralwatt" + + # Sanity-check via direct rank call: without multipliers openrouter would win + # by model_id tiebreak (its effective cost is essentially equal). The weak + # assertion above relies on cost ordering; verify the multiplier is present. + from routing import rank_candidates + + test_rows = [ + { + "model_id": "nw-cheap", + "provider": "neuralwatt", + "tier": 2, + "cost": 0.003, + "energy": 1.0e-5, + "effective_context_window": 131072, + "availability": "active", + "deprecated": 0, + "access_level": "public", + "latency_class": "standard", + "reasoning_mode": "default", + "context_variant": "full", + "supports_vision": 1, + "supports_json_mode": 1, + "proficiency": 0.80, + }, + { + "model_id": "or-cheap", + "provider": "openrouter", + "tier": 2, + "cost": 0.0031, + "energy": 1.0e-5, + "effective_context_window": 131072, + "availability": "active", + "deprecated": 0, + "access_level": "public", + "latency_class": "standard", + "reasoning_mode": "default", + "context_variant": "full", + "supports_vision": 1, + "supports_json_mode": 1, + "proficiency": 0.80, + }, + ] + ranked = rank_candidates(test_rows, provider_cost_multipliers={"openrouter": 5.0}) + assert ranked[0]["model_id"] == "nw-cheap" + + +# (c2) Billing-shape guard: low NeuralWatt allowance (telemetry provider) is +# ignored; OpenRouter stays attenuated. + +def test_credit_attenuation_ignores_telemetry_provider_allowance( + route_client, monkeypatch +): + """NeuralWatt's per-completion allowance is never attenuated.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 + ) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "zero_floor_usd", 0.0 + ) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "max_multiplier", 5.0 + ) + + db_path = dispatcher.cfg.database.path + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + # NeuralWatt reading near zero (mirrors live overage-invoice noise). + _seed_energy_allowance(conn, "neuralwatt", "nw-cheap", -0.004) + # OpenRouter prepaid pool nearly empty. + _seed_provider_balance(conn, "openrouter", 0.01) + conn.close() + + decision = _route_response(route_client) + # OpenRouter gets max_multiplier (~5.0); NeuralWatt is pinned at 1.0 despite + # its negative allowance, so NeuralWatt wins. + assert decision["selected"]["provider"] == "neuralwatt" + + +# (d) Resolver caches: monkeypatch the binding; only the first call resolves, +# the immediate second call returns cached value. + +def test_credit_attenuation_resolver_caches_within_refresh_window( + route_client, monkeypatch +): + """Two route calls within refresh_seconds resolve balances once.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + + calls = [] + + def counting_quota(*args, **kwargs): + calls.append((args, kwargs)) + return { + "by_provider": { + "openrouter": { + "balance_usd": 10.0, + "balance_source": "polled", + }, + "neuralwatt": { + "balance_usd": 10.0, + "balance_source": "telemetry", + }, + }, + "total_balance_usd": 20.0, + } + + monkeypatch.setattr(dispatcher, "quota_balance_and_burn", counting_quota) + + _route_response(route_client) + _route_response(route_client) + assert len(calls) == 1 + + +# (e) Only balance_url providers appear in the multiplier dict. + +def test_provider_cost_multipliers_only_for_balance_url_providers( + route_client, monkeypatch +): + """The resolver returns a dict containing only providers with balance_url.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + + def fake_quota(*args, **kwargs): + return { + "by_provider": { + "openrouter": { + "balance_usd": 1.0, + "balance_source": "polled", + }, + "neuralwatt": { + "balance_usd": -0.004, + "balance_source": "telemetry", + }, + }, + "total_balance_usd": 0.996, + } + + monkeypatch.setattr(dispatcher, "quota_balance_and_burn", fake_quota) + + multipliers = dispatcher._provider_cost_multipliers() + assert "openrouter" in multipliers + assert "neuralwatt" not in multipliers + + +# (f) Disabled feature returns None. + +def test_provider_cost_multipliers_none_when_disabled( + route_client, monkeypatch +): + """When credit_attenuation.enabled is False the resolver returns None.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", False) + + def explode(*args, **kwargs): + raise RuntimeError("should not resolve when disabled") + + monkeypatch.setattr(dispatcher, "quota_balance_and_burn", explode) + assert dispatcher._provider_cost_multipliers() is None + + +# (g) Stale cache refreshes outside refresh_seconds. + +def test_credit_attenuation_resolver_refreshes_after_refresh_seconds( + route_client, monkeypatch +): + """A long-past resolution timestamp triggers a fresh DB read.""" + monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) + monkeypatch.setattr( + dispatcher.cfg.objective.credit_attenuation, "refresh_seconds", 60 + ) + + calls = [] + + def counting_quota(*args, **kwargs): + calls.append((args, kwargs)) + return { + "by_provider": { + "openrouter": { + "balance_usd": 10.0, + "balance_source": "polled", + }, + "neuralwatt": { + "balance_usd": 10.0, + "balance_source": "telemetry", + }, + }, + "total_balance_usd": 20.0, + } + + monkeypatch.setattr(dispatcher, "quota_balance_and_burn", counting_quota) + + # Warm the cache, then age it past the refresh window. + dispatcher._provider_cost_multipliers() + assert len(calls) == 1 + dispatcher._provider_cost_multipliers_resolved_at = ( + dispatcher._provider_cost_multipliers_resolved_at - 120 + ) + dispatcher._provider_cost_multipliers() + assert len(calls) == 2 -- 2.49.1 From 01a52204ba8fb68a1ba56ddf0a21d0d5bb6688f5 Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:24:41 -0400 Subject: [PATCH 10/13] feat(tui): per-provider quota panel rows and binding-provider lead --- src/tui.py | 62 +++++++++++---- src/tui_model.py | 41 ++++++++-- tests/test_tui.py | 199 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 251 insertions(+), 51 deletions(-) diff --git a/src/tui.py b/src/tui.py index 8f3a761..6338fdd 100644 --- a/src/tui.py +++ b/src/tui.py @@ -65,6 +65,51 @@ def _fmt_runway(hours: float) -> str: return f"~{hours / 24:.1f}d" +def _quota_lead_values(rows: dict) -> tuple[Optional[float], Optional[float], Optional[float], bool, Optional[str]]: + providers = sorted( + {label[: -len(" balance_usd")] for label in rows if label.endswith(" balance_usd")} + ) + eligible = [ + p for p in providers if rows.get(f"{p} balance_source") == "polled" + ] + candidates = [] + for provider in eligible: + balance_value = rows.get(f"{provider} balance_usd") + balance = _numeric_suffix_value(balance_value) + burn_value = rows.get(f"{provider} burn_rate_usd_per_hour") + burn = _numeric_suffix_value(burn_value) + runway_value = rows.get(f"{provider} projected_hours_remaining") + runway = _numeric_suffix_value(runway_value) + low = bool(rows.get(f"{provider} runway_low_warning")) + note = rows.get(f"{provider} runway_note") + candidates.append((provider, balance, burn, runway, low, note)) + + if not candidates: + return (None, None, None, False, None) + + def _key(item): + _provider, balance, _burn, runway, _low, _note = item + runway_sort = runway if isinstance(runway, (int, float)) else float("inf") + balance_sort = balance if isinstance(balance, (int, float)) else float("inf") + return (runway_sort, balance_sort, item[0]) + + _, balance, burn, runway, low, note = min(candidates, key=_key) + return (balance, burn, runway, low, note) + + +def _numeric_suffix_value(value: object) -> Optional[float]: + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value.split()[0]) + except (IndexError, ValueError): + return None + return None + + def _format_quota_lead( balance: Optional[float], burn: Optional[float], @@ -616,21 +661,10 @@ class DashboardApp(App): lead = self.query_one("#quota-lead", Static) note = self.query_one("#quota-note", Static) if rows: - balance = rows.get("balance_usd") - burn = rows.get("burn_rate_usd_per_hour") - lead.update( - _format_quota_lead( - balance, - burn, - rows.get("projected_hours_remaining"), - bool(rows.get("runway_low_warning")), - ) - ) - # Explain an absent burn rate rather than leaving a bare "n/a". + balance, burn, runway, low, note_text = _quota_lead_values(rows) + lead.update(_format_quota_lead(balance, burn, runway, low)) if balance is not None and burn is None: - note.update( - str(rows.get("runway_note") or "burn estimate unavailable") - ) + note.update(str(note_text or "burn estimate unavailable")) note.display = True else: note.update("") diff --git a/src/tui_model.py b/src/tui_model.py index 9ef672e..e682fe4 100644 --- a/src/tui_model.py +++ b/src/tui_model.py @@ -55,13 +55,44 @@ def build_model(data: dict) -> dict: {"label": "calls", "value": quota.get("metered_calls_30d")}, {"label": "reset_date", "value": quota.get("reset_date")}, {"label": "next_reset_date", "value": quota.get("next_reset_date")}, - {"label": "balance_usd", "value": quota.get("balance_usd")}, - {"label": "burn_rate_usd_per_hour", "value": quota.get("burn_rate_usd_per_hour")}, - {"label": "projected_hours_remaining", "value": quota.get("projected_hours_remaining")}, - {"label": "runway_low_warning", "value": quota.get("runway_low_warning")}, - {"label": "runway_note", "value": quota.get("runway_note")}, {"label": "window_start_30d", "value": quota.get("window_start_30d")}, + {"label": "total_balance_usd", "value": quota.get("total_balance_usd")}, ] + for provider, entry in sorted((quota.get("by_provider") or {}).items()): + balance = entry.get("balance_usd") + balance_at = entry.get("balance_at") + balance_label = f"{provider} balance_usd" + if balance is None: + balance_value = None + elif balance_at: + balance_value = f"{balance} @ {balance_at}" + else: + balance_value = str(balance) + quota_rows.append({"label": balance_label, "value": balance_value}) + quota_rows.append( + {"label": f"{provider} balance_source", "value": entry.get("balance_source")} + ) + quota_rows.append( + { + "label": f"{provider} burn_rate_usd_per_hour", + "value": entry.get("burn_rate_usd_per_hour"), + } + ) + quota_rows.append( + { + "label": f"{provider} projected_hours_remaining", + "value": entry.get("projected_hours_remaining"), + } + ) + quota_rows.append( + { + "label": f"{provider} runway_low_warning", + "value": entry.get("runway_low_warning"), + } + ) + quota_rows.append( + {"label": f"{provider} runway_note", "value": entry.get("runway_note")} + ) else: quota_rows = [] diff --git a/tests/test_tui.py b/tests/test_tui.py index 9873d7f..609a4f5 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -35,13 +35,19 @@ def _fixture() -> dict: "metered_calls_30d": 18, "reset_date": "2026-07-26", "note": "router-metered only", - "balance_usd": 8.50, - "balance_at": "2026-08-23T09:58:00+00:00", - "burn_window_hours": 24, - "burn_rate_usd_per_hour": 1.0, - "projected_hours_remaining": 8.5, - "runway_low_warning": False, - "runway_note": None, + "total_balance_usd": 8.50, + "by_provider": { + "neuralwatt": { + "balance_usd": 8.50, + "balance_at": "2026-08-23T09:58:00+00:00", + "balance_source": "telemetry", + "burn_window_hours": 24, + "burn_rate_usd_per_hour": 1.0, + "projected_hours_remaining": 8.5, + "runway_low_warning": False, + "runway_note": None, + } + }, "window_start_30d": "2026-07-24", "metered_kwh_period": 0.5, }, @@ -56,13 +62,19 @@ def _fixture() -> dict: "metered_calls_30d": 18, "reset_date": "2026-07-26", "note": "router-metered only", - "balance_usd": 8.50, - "balance_at": "2026-08-23T09:58:00+00:00", - "burn_window_hours": 24, - "burn_rate_usd_per_hour": 1.0, - "projected_hours_remaining": 8.5, - "runway_low_warning": False, - "runway_note": None, + "total_balance_usd": 8.50, + "by_provider": { + "neuralwatt": { + "balance_usd": 8.50, + "balance_at": "2026-08-23T09:58:00+00:00", + "balance_source": "telemetry", + "burn_window_hours": 24, + "burn_rate_usd_per_hour": 1.0, + "projected_hours_remaining": 8.5, + "runway_low_warning": False, + "runway_note": None, + } + }, "window_start_30d": "2026-07-24", "metered_kwh_period": 0.5, }, @@ -227,21 +239,33 @@ def test_build_model_quota_panel_includes_reset_date(): assert by_label["window_start_30d"] == "2026-07-24" -def test_build_model_quota_panel_includes_balance_and_runway_keys(): - """The new quota payload keys must reach the TUI data model so the - quota panel can surface balance, burn rate, and runway alongside the - existing plan/metered/reset rows.""" +def test_build_model_quota_panel_includes_total_balance_and_provider_rows(): + """The new per-provider quota shape must reach the TUI data model.""" m = build_model(_fixture()) rows = m["quota"] by_label = {r["label"]: r["value"] for r in rows} - assert by_label["balance_usd"] == 8.50 - assert by_label["burn_rate_usd_per_hour"] == 1.0 - assert by_label["projected_hours_remaining"] == 8.5 - assert by_label["runway_low_warning"] is False + assert by_label["total_balance_usd"] == 8.50 + assert by_label["neuralwatt balance_usd"] == "8.5 @ 2026-08-23T09:58:00+00:00" + assert by_label["neuralwatt balance_source"] == "telemetry" + assert by_label["neuralwatt burn_rate_usd_per_hour"] == 1.0 + assert by_label["neuralwatt projected_hours_remaining"] == 8.5 + assert by_label["neuralwatt runway_low_warning"] is False assert by_label["window_start_30d"] == "2026-07-24" assert by_label["reset_date"] == "2026-07-26" +def test_build_model_quota_panel_drops_flat_balance_keys(): + """Old flat balance keys must not leak into the TUI row list.""" + m = build_model(_fixture()) + labels = {r["label"] for r in m["quota"]} + assert "balance_usd" not in labels + assert "balance_at" not in labels + assert "burn_rate_usd_per_hour" not in labels + assert "projected_hours_remaining" not in labels + assert "runway_low_warning" not in labels + assert "runway_note" not in labels + + def test_build_model_per_model_lists_seeded_models(): m = build_model(_fixture()) rows = m["per_model"] @@ -1720,6 +1744,115 @@ def test_format_quota_lead_states(): assert "[rgb(200,80,80)]runway ~1.5h" in low +def test_quota_lead_values_binds_only_to_polled_providers(): + rows = { + "openrouter balance_usd": "50.0 @ 2026-08-23T10:00:00+00:00", + "openrouter balance_source": "polled", + "openrouter burn_rate_usd_per_hour": 5.0, + "openrouter projected_hours_remaining": 10.0, + "openrouter runway_low_warning": False, + "openrouter runway_note": None, + "neuralwatt balance_usd": "-0.004 @ 2026-08-23T10:00:00+00:00", + "neuralwatt balance_source": "telemetry", + "neuralwatt burn_rate_usd_per_hour": 1.0, + "neuralwatt projected_hours_remaining": -0.004, + "neuralwatt runway_low_warning": True, + "neuralwatt runway_note": "allowance reads negative", + } + balance, burn, runway, low, note = tui._quota_lead_values(rows) + assert balance == 50.0 + assert burn == 5.0 + assert runway == 10.0 + assert low is False + assert note is None + + +def test_quota_lead_values_uses_lowest_runway_among_polled(): + rows = { + "openrouter balance_usd": 50.0, + "openrouter balance_source": "polled", + "openrouter burn_rate_usd_per_hour": 5.0, + "openrouter projected_hours_remaining": 10.0, + "openrouter runway_low_warning": False, + "openrouter runway_note": None, + "provider-b balance_usd": 20.0, + "provider-b balance_source": "polled", + "provider-b burn_rate_usd_per_hour": 5.0, + "provider-b projected_hours_remaining": 3.0, + "provider-b runway_low_warning": True, + "provider-b runway_note": "low", + } + balance, burn, runway, low, note = tui._quota_lead_values(rows) + assert balance == 20.0 + assert runway == 3.0 + assert low is True + assert note == "low" + + +def test_quota_lead_values_falls_back_to_lowest_balance_when_no_runway(): + rows = { + "openrouter balance_usd": 50.0, + "openrouter balance_source": "polled", + "openrouter burn_rate_usd_per_hour": None, + "openrouter projected_hours_remaining": None, + "openrouter runway_low_warning": False, + "openrouter runway_note": "recent top-up", + "provider-b balance_usd": 20.0, + "provider-b balance_source": "polled", + "provider-b burn_rate_usd_per_hour": None, + "provider-b projected_hours_remaining": None, + "provider-b runway_low_warning": False, + "provider-b runway_note": None, + } + balance, burn, runway, low, note = tui._quota_lead_values(rows) + assert balance == 20.0 + assert burn is None + assert runway is None + + +def test_quota_lead_values_returns_none_for_all_telemetry_providers(): + rows = { + "neuralwatt balance_usd": -0.004, + "neuralwatt balance_source": "telemetry", + "neuralwatt burn_rate_usd_per_hour": 1.0, + "neuralwatt projected_hours_remaining": -0.004, + "neuralwatt runway_low_warning": True, + "neuralwatt runway_note": "negative allowance", + } + assert tui._quota_lead_values(rows) == (None, None, None, False, None) + + +def test_quota_lead_values_returns_none_for_old_flat_payload(): + rows = { + "balance_usd": 8.5, + "burn_rate_usd_per_hour": 1.0, + "projected_hours_remaining": 8.5, + "runway_low_warning": False, + "runway_note": None, + } + assert tui._quota_lead_values(rows) == (None, None, None, False, None) + + +def test_quota_lead_values_sorted_name_tiebreak(): + rows = { + "openrouter balance_usd": 50.0, + "openrouter balance_source": "polled", + "openrouter burn_rate_usd_per_hour": 1.0, + "openrouter projected_hours_remaining": 5.0, + "openrouter runway_low_warning": False, + "openrouter runway_note": None, + "provider-a balance_usd": 50.0, + "provider-a balance_source": "polled", + "provider-a burn_rate_usd_per_hour": 1.0, + "provider-a projected_hours_remaining": 5.0, + "provider-a runway_low_warning": False, + "provider-a runway_note": None, + } + balance, burn, runway, low, note = tui._quota_lead_values(rows) + assert balance == 50.0 + assert runway == 5.0 + + def test_quota_panel_burn_unavailable_renders_note_verbatim(): """When burn is None the note renders VERBATIM — never blank, 0, or None. @@ -1730,11 +1863,12 @@ def test_quota_panel_burn_unavailable_renders_note_verbatim(): data = _fixture() note = "burn estimate unavailable: segment after last balance increase spans only 9 minutes" for block in (data["quota"], data["coverage"]["quota"]): - block["balance_usd"] = 7.5 - block["burn_rate_usd_per_hour"] = None - block["projected_hours_remaining"] = None - block["runway_low_warning"] = False - block["runway_note"] = note + block["by_provider"]["neuralwatt"]["balance_source"] = "polled" + block["by_provider"]["neuralwatt"]["balance_usd"] = 7.5 + block["by_provider"]["neuralwatt"]["burn_rate_usd_per_hour"] = None + block["by_provider"]["neuralwatt"]["projected_hours_remaining"] = None + block["by_provider"]["neuralwatt"]["runway_low_warning"] = False + block["by_provider"]["neuralwatt"]["runway_note"] = note stub.payload = data app = tui.DashboardApp(fetcher=stub) @@ -1758,11 +1892,12 @@ def test_quota_panel_note_hidden_when_burn_is_available(): stub = _StubFetcher() data = _fixture() for block in (data["quota"], data["coverage"]["quota"]): - block["balance_usd"] = 12.0 - block["burn_rate_usd_per_hour"] = 0.5 - block["projected_hours_remaining"] = 24.0 - block["runway_low_warning"] = False - block["runway_note"] = None + block["by_provider"]["neuralwatt"]["balance_source"] = "polled" + block["by_provider"]["neuralwatt"]["balance_usd"] = 12.0 + block["by_provider"]["neuralwatt"]["burn_rate_usd_per_hour"] = 0.5 + block["by_provider"]["neuralwatt"]["projected_hours_remaining"] = 24.0 + block["by_provider"]["neuralwatt"]["runway_low_warning"] = False + block["by_provider"]["neuralwatt"]["runway_note"] = None stub.payload = data app = tui.DashboardApp(fetcher=stub) -- 2.49.1 From c1988392f63cc9c0ed72aad5199aa464b4ef5e0b Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:30:06 -0400 Subject: [PATCH 11/13] feat(admin): per-provider balance and runway rows in quota chip and modal --- admin/frontend/index.html | 248 ++++++++++++++++++++--------------- tests/test_admin_frontend.py | 4 +- 2 files changed, 142 insertions(+), 110 deletions(-) diff --git a/admin/frontend/index.html b/admin/frontend/index.html index 86de9c8..36ed527 100644 --- a/admin/frontend/index.html +++ b/admin/frontend/index.html @@ -668,74 +668,113 @@ function renderQuotaModal(quota) { el.innerHTML = `
No quota plan configured
`; return; } - // ── balance-led rendering ─────────────────────────────────────────── - if (quota.balance_usd != null && typeof quota.balance_usd === 'number') { - const balanceStr = '$' + quota.balance_usd.toFixed(2); - const burn = quota.burn_rate_usd_per_hour ?? null; - const proj = quota.projected_hours_remaining ?? null; - const low = !!quota.runway_low_warning; - const hours = proj != null ? '~' + Math.round(proj) + 'h runway' : 'runway unknown'; - const hourColor = low ? 'var(--tblr-danger)' : !burn ? 'var(--tblr-warning)' : 'var(--tblr-success)'; - const burnLabel = burn != null ? '$' + (Math.round(burn * 100) / 100).toFixed(2) + '/hr' : (quota.runway_note ? '' : 'n/a'); - - // Build detail rows - const detailRows = []; - if (proj != null) detailRows.push('Projected'); - if (burn != null && !quota.runway_note) detailRows.push('Burn rate'); - detailRows.push('Plan'); - detailRows.push('Metered'); - detailRows.push('Reset'); - - let rowIdx = -1; - const runways = ['Low Runway', 'Burn N/A']; - const meteredLabel = quota.reset_date - ? `Metered (period since ${escapeHtml(quota.reset_date)})` - : 'Metered (30d window)'; - - el.innerHTML = ` -
-
${escapeHtml(balanceStr)}
-
${hours}
- ${low ? '
Low Runway
' : ''} -
-
- ${burn != null && !quota.runway_note ? 'Burn rate~' + escapeHtml(burnLabel) + '/hr' : ''} - ${quota.runway_note ? 'Burn' + escapeHtml(quota.runway_note) + '' : ''} - ${proj != null ? 'Projected~' + Math.round(proj) + 'h' : ''} - Plan${quota.plan_kwh} kWh - ${escapeHtml(meteredLabel)}${quota.metered_kwh_period != null ? quota.metered_kwh_period + ' kWh' : quota.metered_kwh_30d + ' kWh'} - Fraction${(quota.metered_fraction_of_plan * 100).toFixed(1)}% - Reset${quota.next_reset_date || 'not configured'} -
`; - return; - } - // ── fallback: percentage-only rendering ───────────────────────────── + const meteredLabel = quota.reset_date + ? `Metered (period since ${escapeHtml(quota.reset_date)})` + : 'Metered (30d window)'; const pct = Math.min(quota.metered_fraction_of_plan * 100, 100); - const color = pct > 90 ? 'var(--tblr-danger)' : pct > 75 ? 'var(--tblr-warning)' : 'var(--tblr-success)'; + const heroColor = pct > 90 ? 'var(--tblr-danger)' : pct > 75 ? 'var(--tblr-warning)' : 'var(--tblr-success)'; const displayPct = (quota.metered_fraction_of_plan * 100).toFixed(1); const calls = Number(quota.metered_calls_30d || 0).toLocaleString(); - const pctInside = pct > 22; - const pctStyle = pctInside - ? `left:calc(${pct}% - 30px);color:#fff` - : `left:calc(${pct}% + 8px);color:${color}`; + + // ── per-provider balance rows (reusing .bar-row / .bars-scroll) ───── + const byProvider = quota.by_provider || {}; + const providers = Object.keys(byProvider).sort(); + let providerRowsHtml = ''; + let totalBalance = quota.total_balance_usd; + if (providers.length) { + providerRowsHtml = providers.map(name => { + const p = byProvider[name] || {}; + const balance = p.balance_usd; + const burn = p.burn_rate_usd_per_hour ?? null; + const proj = p.projected_hours_remaining ?? null; + const low = !!p.runway_low_warning; + const balanceAt = p.balance_at || null; + const source = p.balance_source || ''; + const lineParts = []; + if (balance != null && typeof balance === 'number') { + lineParts.push('$' + balance.toFixed(2)); + } else { + lineParts.push('balance unknown'); + } + if (burn != null && typeof burn === 'number') { + lineParts.push('~$' + burn.toFixed(2) + '/hr'); + } + if (proj != null && typeof proj === 'number') { + lineParts.push('~' + Math.round(proj) + 'h'); + } + const suffix = formatBalanceStaleness(balanceAt, source); + const accent = low ? 'var(--tblr-danger)' : (balance == null || burn == null) ? 'var(--tblr-warning)' : 'var(--tblr-success)'; + return `
+ ${escapeHtml(name)} + ${lineParts.join(' · ')}${suffix ? ' · ' + escapeHtml(suffix) + '' : ''} +
`; + }).join(''); + } + + let totalRow = ''; + if (totalBalance != null && typeof totalBalance === 'number') { + totalRow = `
+ Total + $${totalBalance.toFixed(2)} +
`; + } el.innerHTML = ` -
-
-
${calls} calls (30d)
-
${displayPct}%
-
-
-
- ${displayPct}% -
-
- Plan${quota.plan_kwh} kWh - Metered (30d)${quota.metered_kwh_30d} kWh - Resets${quota.next_reset_date || 'not configured'} -
-
${escapeHtml(quota.note || '')}
-
`; +
+
${displayPct}%
+
${quota.plan_kwh} kWh plan · ${calls} calls (30d)
+
+
+ Plan${quota.plan_kwh} kWh + ${escapeHtml(meteredLabel)}${quota.metered_kwh_period != null ? quota.metered_kwh_period + ' kWh' : quota.metered_kwh_30d + ' kWh'} + Fraction${displayPct}% + Reset${quota.next_reset_date || 'not configured'} +
+ ${providers.length ? ` +
Per-provider balance & runway
+
+ ${providerRowsHtml} + ${totalRow} +
` : ''} +
${escapeHtml(quota.note || '')}
`; +} + +function formatBalanceStaleness(balanceAt, source) { + if (!balanceAt) return ''; + try { + const then = new Date(balanceAt); + const ms = Date.now() - then.getTime(); + if (Number.isNaN(ms) || ms < 0) return ''; + const minutes = Math.floor(ms / 60000); + if (minutes < 1) return 'just now'; + if (minutes < 60) return `sampled ${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `sampled ${hours}h ago`; + const days = Math.floor(hours / 24); + return `sampled ${days}d ago`; + } catch (_) { + return ''; + } +} + +function _quotaChipAccent(low, hasKnownBurn) { + if (low) { + return { + '--chip-accent': 'var(--tblr-danger)', + background: 'linear-gradient(135deg, rgba(239,68,68,.12), rgba(59,130,246,.08))', + borderColor: 'rgba(239,68,68,.35)', + }; + } + if (!hasKnownBurn) { + return { + '--chip-accent': 'var(--tblr-warning)', + background: 'linear-gradient(135deg, rgba(245,158,11,.12), rgba(59,130,246,.08))', + borderColor: 'rgba(245,158,11,.38)', + }; + } + return { + '--chip-accent': 'var(--tblr-success)', + }; } function renderQuotaChip(quota) { @@ -748,64 +787,55 @@ function renderQuotaChip(quota) { return; } chip.hidden = false; - // ── balance-led display ------------------------------------------------ - if (quota.balance_usd != null) { - const burn = quota.burn_rate_usd_per_hour ?? null; - const proj = quota.projected_hours_remaining ?? null; - const low = !!quota.runway_low_warning; - const hours = proj != null ? Math.round(proj) : null; - let chipStyle = {}; + // ── per-provider aggregate display ----------------------------------- + const byProvider = quota.by_provider || {}; + const providers = Object.keys(byProvider); + const entries = providers.map(name => byProvider[name]).filter(Boolean); + const knownBalances = entries.map(p => p.balance_usd).filter(v => v != null && typeof v === 'number'); + const totalBalance = knownBalances.length ? knownBalances.reduce((a, b) => a + b, 0) : null; - if (low) { - // Red: runway critically low - chipStyle = { - '--chip-accent': 'var(--tblr-danger)', - background: 'linear-gradient(135deg, rgba(239,68,68,.12), rgba(59,130,246,.08))', - borderColor: 'rgba(239,68,68,.35)', - }; - } else if (!burn) { - // Amber: burn unknown, not alarming - chipStyle = { - '--chip-accent': 'var(--tblr-warning)', - background: 'linear-gradient(135deg, rgba(245,158,11,.12), rgba(59,130,246,.08))', - borderColor: 'rgba(245,158,11,.38)', - }; - } else if (chipStyle._unset !== true) { - // Green: healthy runway - chipStyle = { - '--chip-accent': 'var(--tblr-success)', - }; - } + if (totalBalance != null) { + const low = entries.some(p => !!p.runway_low_warning); + const hasKnownBurn = entries.some(p => p.burn_rate_usd_per_hour != null); + const worstProjEntry = entries + .filter(p => p.projected_hours_remaining != null) + .sort((a, b) => Number(a.projected_hours_remaining) - Number(b.projected_hours_remaining))[0]; + const hours = worstProjEntry ? Math.round(Number(worstProjEntry.projected_hours_remaining)) : null; + const chipStyle = _quotaChipAccent(low, hasKnownBurn); + const balanceStr = '$' + totalBalance.toFixed(2); - const balanceStr = typeof quota.balance_usd === 'number' - ? '$' + quota.balance_usd.toFixed(2) - : '$' + String(quota.balance_usd); + text.textContent = hours != null ? balanceStr + ' · ~' + hours + 'h' : balanceStr; - if (hours != null) { - text.textContent = balanceStr + ' · ~' + hours + 'h'; - } else if (burn != null) { - text.textContent = balanceStr; - } else { - text.textContent = balanceStr; - } - - // Apply chip accent colours (border + text tint) for (const [k, v] of Object.entries(chipStyle)) { - if (k !== '_unset') chip.style.setProperty(k, v); + chip.style.setProperty(k, v); } if (chipStyle['--chip-accent']) { text.style.color = chipStyle['--chip-accent']; } - // Build operator-readable tooltip - let title = 'Account balance'; - if (burn != null) title = '$' + quota.balance_usd.toFixed(2) + ' · ~' + (Math.round(burn * 100) / 100) + '/hr'; - if (hours != null) title += ' · ~' + hours + 'h runway'; - text.setAttribute('title', title); + // Tooltip lists every provider plus its staleness + const tooltipLines = providers.map(name => { + const p = byProvider[name] || {}; + const bal = p.balance_usd; + const burn = p.burn_rate_usd_per_hour; + const parts = [escapeHtml(name)]; + if (bal != null && typeof bal === 'number') { + parts.push('$' + bal.toFixed(2)); + } else { + parts.push('balance unknown'); + } + if (burn != null && typeof burn === 'number') parts.push('~$' + burn.toFixed(2) + '/hr'); + const proj = p.projected_hours_remaining; + if (proj != null && typeof proj === 'number') parts.push('~' + Math.round(proj) + 'h'); + const suffix = formatBalanceStaleness(p.balance_at, p.balance_source); + if (suffix) parts.push(suffix); + return parts.join(' · '); + }); + text.setAttribute('title', tooltipLines.join('\n')); fill.style.width = `${Math.min(quota.metered_fraction_of_plan * 100, 100)}%`; return; } - // ── fallback: percentage only ------------------------------------------------ + // ── fallback: percentage only ---------------------------------------- const pct = Math.min(quota.metered_fraction_of_plan * 100, 100); fill.style.width = `${pct}%`; text.textContent = `${(quota.metered_fraction_of_plan * 100).toFixed(1)}%`; diff --git a/tests/test_admin_frontend.py b/tests/test_admin_frontend.py index 60c81dd..f78609a 100644 --- a/tests/test_admin_frontend.py +++ b/tests/test_admin_frontend.py @@ -176,10 +176,12 @@ def test_quota_modal_has_not_configured_fallback(): def test_admin_quota_chip_references_balance_and_runway(): - """The index page references the new balance / runway quota fields.""" + """The index page references per-provider balance / runway quota fields and the provider rows container.""" index_path = ROOT / "admin" / "frontend" / "index.html" html = index_path.read_text() assert "balance_usd" in html and "runway_low_warning" in html + assert 'id="quota-provider-rows"' in html + assert "balance_at" in html def test_builtin_profile_cards_offer_duplicate(admin_client): -- 2.49.1 From 080648feb63b93d20fe84217bd421571c6534b3b Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 13:43:11 -0400 Subject: [PATCH 12/13] docs(admin,claude): per-provider quota measurement and credit attenuation --- CLAUDE.md | 32 ++++++++++++++++++++++++++++---- docs/admin-portal.md | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e14b6cc..51cd028 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,9 +117,10 @@ sweep, and refreshes whenever the poller runs. is **never** refused for exceeding it — it **gates nothing**. Overage is billed against the account's credit balance (`allowance_remaining_usd` from the provider). `/metrics` now reports balance, estimated burn rate, and projected -runway (hours remaining) derived from the provider-reported -`allowance_remaining_usd` over a configurable window, using a configurable -minimum segment length and sample count to avoid wild extrapolations. +runway **per provider** inside `quota.by_provider`, plus a `total_balance_usd` +field that sums whatever each provider reports. The flat balance/burn/runway +keys at the top level of `quota` were removed; every consumer was updated in the +same change, so there are no deprecated aliases. Three signals said `deepseek-v4-flash` — catalog token price (7.8x cheaper), NeuralWatt's own published per-request energy (~10x lower), and a live 70k @@ -222,7 +223,7 @@ rather than from months of history. - `poller.py` — also seeds/updates `provider='ollama-local'` rows from `config.yaml` each poll so local rows stay current even when NeuralWatt is unreachable. - `logs.py` — per-request trace id (ContextVar), logfmt, journald priority prefixes; `logs.bind()` survives StreamingResponse generators. [operations](docs/operations.md). - `metrics.py` / `GET /metrics` — read-only observability; takes `(conn, cfg)`, never imports `dispatcher`. Also carries the three detectors added after the incidents below: capability sub-ceilings, the reactive rejection detector, and the classifier-degradation share. [api](docs/api.md). -- `tui.py` — Textual dashboard over `/metrics` + `/events/decisions`; live feed, category→model panel, detail popup; data layer split into `tui_model.py`. The decision table leads with a `time` column and carries `profile` plus an `E` flag for exploratory picks; the quota panel leads with balance and runway. [architecture](docs/architecture.md). +- `tui.py` — Textual dashboard over `/metrics` + `/events/decisions`; live feed, category→model panel, detail popup; data layer split into `tui_model.py`. The decision table leads with a `time` column and carries `profile` plus an `E` flag for exploratory picks; the quota panel now shows one row group per provider plus a `total_balance_usd` line, with a lead calculated only from providers whose balance is polled from a prepaid account. [architecture](docs/architecture.md). - `tests/test_tui_schema_drift.py` — the tripwire that keeps the two honest. A new `route_decisions` column must be registered as surfaced or deliberately-not, or the test fails **naming the column**. Five columns had already reached the schema without reaching the dashboard; `ROUTE_DECISIONS_COLUMNS` in `tests/test_route_decisions.py` had itself drifted. - `tests/test_tui_warnings.py` — the same idea for warnings. Every class `/metrics` can emit must render in `#warnings-panel`, and every emitted warning must be registered — the second failing with the RAW text, because the point is that nobody knew the class existed. **Its fixture is a coupled system**: adding a seed can silence an existing class (a small-context seed once killed the escalation hazard by dragging the p95 down), which is why both directions are asserted. - `router_cli.py` — one-shot `/route` probe (no spend), raw JSON with `--json`. [api](docs/api.md). @@ -763,6 +764,29 @@ include `request_id` and `session_dir` in `local_energy_observations`, so a client report on a local answer resolves to the same `(model_id, provider, task_category)` provider-agnostic record as a cloud one. +## Routing notes + +Ranking is quality-first, cost as a tiebreak; cost is never allowed to override +a real quality gap. The optional `objective.credit_attenuation` block extends +that tiebreak without changing it: when the block is enabled, a per-provider +multiplier is applied to a candidate's comparison cost only, producing an +`effective_cost` that breaks ties. The multiplier is derived from the provider's +polled account balance (the `balance_url` path, such as OpenRouter), so a low +prepaid balance can nudge a near-tie toward a healthier provider. The logged +`est_cost_usd` and the decision history stay as raw catalog estimates. The +multiplier is 1.0 for providers whose balance comes from per-completion +`allowance_remaining_usd` telemetry (NeuralWatt), so normal overage readings do +not bias routing. + +Two semantics matter when reading the numbers. `total_balance_usd` is a sum of +heterogeneous provider-reported readings: OpenRouter's prepaid credits plus +NeuralWatt's overage allowance, which normally reads near -$0.004. It can be +negative and it is not a single spendable figure. `credit_attenuation.enabled` +deliberately lives only in the config file; it is absent from the admin +persisted-config allowlist and from provider edits. Turning it on or off +requires editing `config/config.yaml` and `systemctl --user restart +llm-router.service`, because the dispatcher's `cfg` binds at import time. + ## What's NOT built yet — pick up here Built: session-directory attribution, the local energy ledger, local model diff --git a/docs/admin-portal.md b/docs/admin-portal.md index 956b422..1ac56f1 100644 --- a/docs/admin-portal.md +++ b/docs/admin-portal.md @@ -179,7 +179,7 @@ the attributable set, so `POST /outcome` keeps training proficiency normally. **Read-only dashboards** — `GET /admin/api/snapshot` exposes data for: -- quota burn against `objective.plan_kwh_per_period` +- quota: energy burn against `objective.plan_kwh_per_period`, plus per-provider balance, burn rate, and runway under `by_provider`, and a `total_balance_usd` sum (old flat balance/burn/runway keys removed) - per-model usage from `energy_observations` - live routing decisions from `route_decisions` - verdict mix and scoring coverage -- 2.49.1 From 8b44cefe0de872f36b34eb39e10b1a3a5a0c8c8d Mon Sep 17 00:00:00 2001 From: adlee-was-taken Date: Sun, 6 Sep 2026 14:12:54 -0400 Subject: [PATCH 13/13] =?UTF-8?q?fix(test):=20reprice=20attenuation=20fixt?= =?UTF-8?q?ure=20so=20OFF=E2=86=92ON=20flip=20is=20observable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/metrics.py | 8 +- tests/test_credit_attenuation_routing.py | 122 ++++++++++------------- 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index fc17b48..bded171 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -10,9 +10,10 @@ optionally a ``RouterConfig`` instance; none rely on module-level globals. Functions --------- quota_burn — kWh metered in the last 30 d and in the current billing - period, against the plan allowance; also reports the account - credit balance, burn rate and runway from - allowance_remaining_usd + period, against the plan allowance; also reports per-provider + credit balance, burn rate and runway (from + allowance_remaining_usd telemetry or the polled + provider_balance_observations table) plus total_balance_usd scoring_coverage — which scoring axes actually have data capability_ceilings — vision / json_mode context-window sub-ceilings capability_demand_warnings — demand-relative warnings for those sub-ceilings @@ -328,7 +329,6 @@ def quota_balance_and_burn(conn: sqlite3.Connection, cfg: Any) -> dict: return {"by_provider": by_provider, "total_balance_usd": total_balance_usd} - def quota_burn( conn: sqlite3.Connection, cfg: Any, diff --git a/tests/test_credit_attenuation_routing.py b/tests/test_credit_attenuation_routing.py index 58dd14b..4b28724 100644 --- a/tests/test_credit_attenuation_routing.py +++ b/tests/test_credit_attenuation_routing.py @@ -16,19 +16,9 @@ import pytest from starlette.testclient import TestClient import dispatcher -from config import load_config ROOT = Path(__file__).resolve().parent.parent SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text() -CFG = load_config(str(ROOT / "config" / "config.yaml")) - - -def _now() -> datetime: - return datetime.now(timezone.utc) - - -def _oldest() -> str: - return (datetime.now(timezone.utc) - timedelta(days=365)).isoformat() def _hours_ago(hours: float) -> str: @@ -43,10 +33,18 @@ def _make_db(tmp_path: Path) -> sqlite3.Connection: def _seed_two_provider_models(conn: sqlite3.Connection) -> None: - """Two providers, two models: near-identical profile so the tiebreak decides.""" + """Two providers, two models: near-identical profile so the tiebreak decides. + + OpenRouter is priced slightly BELOW NeuralWatt (0.29 vs 0.30 per 1M), so + the raw-cost order picks or-cheap. The attenuation multipliers must + overcome that gap to flip the selection to neuralwatt — which is what + makes tests (b)/(c)/(c2) observable rather than tautological: with the + wiring broken (multipliers never reaching rank_candidates), openrouter + wins and tests (c)/(c2) FAIL. + """ rows = [ - # (model_id, provider, tier, context_window, cost_per_1m_prompt, - # cost_per_1m_completion, proficiency_category, blended_score) + # (model_id, provider, tier, context_window, + # cost_per_1m_prompt, cost_per_1m_completion) ( "nw-cheap", "neuralwatt", @@ -60,8 +58,8 @@ def _seed_two_provider_models(conn: sqlite3.Connection) -> None: "openrouter", 2, 131072, - 0.31, - 0.31, + 0.29, + 0.29, ), ] for model_id, provider, tier, ctx, prompt_cost, completion_cost in rows: @@ -199,14 +197,30 @@ def test_credit_attenuation_off_by_default_no_db_read( def test_credit_attenuation_on_healthy_balances_matches_baseline( route_client, monkeypatch ): - """Healthy OpenRouter balance keeps the original order (NeuralWatt first).""" + """Healthy OpenRouter balance keeps the raw-cost order (OpenRouter first). + + OpenRouter is priced slightly cheaper, so the OFF baseline selects it. + Enabling attenuation with a healthy balance must NOT change that — the + multiplier is 1.0 and the winner stays or-cheap. + """ + db_path = dispatcher.cfg.database.path + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + _seed_provider_balance(conn, "openrouter", 50.0) + conn.close() + + # OFF baseline: the cheaper model wins. + decision = _route_response(route_client) + assert decision["selected"]["provider"] == "openrouter" + assert decision["selected"]["model_id"] == "or-cheap" + + # ON + healthy balance: multiplier 1.0, same winner. monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) monkeypatch.setattr( dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 ) - decision = _route_response(route_client) - assert decision["selected"]["provider"] == "neuralwatt" + assert decision["selected"]["provider"] == "openrouter" # (c) ON + synthetic low OpenRouter balance flips the tiebreak toward the @@ -215,7 +229,13 @@ def test_credit_attenuation_on_healthy_balances_matches_baseline( def test_credit_attenuation_on_low_openrouter_balance_flips_to_healthy_provider( route_client, monkeypatch ): - """Low OpenRouter balance inflates its effective cost; NeuralWatt wins.""" + """A near-zero OpenRouter balance must flip the pick to NeuralWatt. + + OpenRouter is priced cheaper than NeuralWatt, so the only way neuralwatt + can win is a >1.0 multiplier applied to or-cheap's comparison cost inside + route(). If the dispatcher stopped passing provider_cost_multipliers into + rank_candidates, openrouter (raw-cheaper) would win and this test FAILS. + """ monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) monkeypatch.setattr( dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 @@ -227,61 +247,17 @@ def test_credit_attenuation_on_low_openrouter_balance_flips_to_healthy_provider( dispatcher.cfg.objective.credit_attenuation, "max_multiplier", 5.0 ) - # No seeded balance row for OpenRouter → balance is None → multiplier 1.0, - # so the baseline winner stays neuralwatt. To flip, give OpenRouter a very - # low but nonzero balance. db_path = dispatcher.cfg.database.path conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row + # 0.01 is just above the zero floor: multiplier ≈ 4.99, well past the + # ~3% raw-price gap between or-cheap and nw-cheap. _seed_provider_balance(conn, "openrouter", 0.01) conn.close() decision = _route_response(route_client) assert decision["selected"]["provider"] == "neuralwatt" - - # Sanity-check via direct rank call: without multipliers openrouter would win - # by model_id tiebreak (its effective cost is essentially equal). The weak - # assertion above relies on cost ordering; verify the multiplier is present. - from routing import rank_candidates - - test_rows = [ - { - "model_id": "nw-cheap", - "provider": "neuralwatt", - "tier": 2, - "cost": 0.003, - "energy": 1.0e-5, - "effective_context_window": 131072, - "availability": "active", - "deprecated": 0, - "access_level": "public", - "latency_class": "standard", - "reasoning_mode": "default", - "context_variant": "full", - "supports_vision": 1, - "supports_json_mode": 1, - "proficiency": 0.80, - }, - { - "model_id": "or-cheap", - "provider": "openrouter", - "tier": 2, - "cost": 0.0031, - "energy": 1.0e-5, - "effective_context_window": 131072, - "availability": "active", - "deprecated": 0, - "access_level": "public", - "latency_class": "standard", - "reasoning_mode": "default", - "context_variant": "full", - "supports_vision": 1, - "supports_json_mode": 1, - "proficiency": 0.80, - }, - ] - ranked = rank_candidates(test_rows, provider_cost_multipliers={"openrouter": 5.0}) - assert ranked[0]["model_id"] == "nw-cheap" + assert decision["selected"]["model_id"] == "nw-cheap" # (c2) Billing-shape guard: low NeuralWatt allowance (telemetry provider) is @@ -290,7 +266,16 @@ def test_credit_attenuation_on_low_openrouter_balance_flips_to_healthy_provider( def test_credit_attenuation_ignores_telemetry_provider_allowance( route_client, monkeypatch ): - """NeuralWatt's per-completion allowance is never attenuated.""" + """NeuralWatt's per-completion allowance is never attenuated. + + OpenRouter is raw-cheaper, so neuralwatt winning here requires BOTH: + OpenRouter attenuated (low polled balance → ~5x comparison cost) AND + NeuralWatt left at multiplier 1.0 despite its negative telemetry + allowance. If the balance_url billing-shape guard broke and telemetry + providers were attenuated too, nw-cheap (~1.5 effective) would lose to + or-cheap (~1.45 effective) and this test FAILS. If the multiplier + passthrough broke, or-cheap wins raw and this test FAILS. + """ monkeypatch.setattr(dispatcher.cfg.objective.credit_attenuation, "enabled", True) monkeypatch.setattr( dispatcher.cfg.objective.credit_attenuation, "soft_floor_usd", 5.0 @@ -312,9 +297,8 @@ def test_credit_attenuation_ignores_telemetry_provider_allowance( conn.close() decision = _route_response(route_client) - # OpenRouter gets max_multiplier (~5.0); NeuralWatt is pinned at 1.0 despite - # its negative allowance, so NeuralWatt wins. assert decision["selected"]["provider"] == "neuralwatt" + assert decision["selected"]["model_id"] == "nw-cheap" # (d) Resolver caches: monkeypatch the binding; only the first call resolves, -- 2.49.1