diff --git a/admin/frontend/index.html b/admin/frontend/index.html index 161837a..ea85895 100644 --- a/admin/frontend/index.html +++ b/admin/frontend/index.html @@ -311,7 +311,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
-
@@ -324,6 +324,18 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
+
+
+
+

Pinch savings

+
+
+
Loading…
+
+
+
+ +
@@ -540,6 +552,7 @@ async function loadSnapshot() { renderQuotaChip(data.quota); renderDecisions(data.recent_decisions); renderLocalEnergy(data.local_energy); + renderPinch(data.pinch); renderCloudEnergy(data.per_model); renderBars(data.per_model); renderVerdict(data.verdict_mix); @@ -765,6 +778,27 @@ function renderLocalEnergy(summary) { ${typeHtml}`; } +function renderPinch(summary) { + const el = document.getElementById('pinch-content'); + if (!summary) { + el.innerHTML = '
Pinch is disabled
'; + return; + } + const share = Number(summary.share_pruned || 0); + const totalSaved = Number(summary.total_tokens_saved || 0).toLocaleString(); + const medianSaved = summary.median_tokens_saved == null + ? 'n/a' + : Number(summary.median_tokens_saved).toLocaleString(); + const dollars = Number(summary.dollars_saved_usd_30d || 0).toFixed(6); + el.innerHTML = ` +
+
share pruned
${(share * 100).toFixed(1)}%
+
tokens saved
${totalSaved}
+
median saved
${medianSaved}
+
30d dollars saved
$${dollars}
+
`; +} + function renderCloudEnergy(perModel) { const el = document.getElementById('cloud-energy-content'); const data = perModel || []; diff --git a/config/schema.sql b/config/schema.sql index 85825fa..28a8235 100644 --- a/config/schema.sql +++ b/config/schema.sql @@ -234,7 +234,10 @@ CREATE TABLE IF NOT EXISTS route_decisions ( flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex' -- |'force-flex' (resolved) flex_swapped INTEGER, -- 0/1 post-rank flex swap applied - flex_forced INTEGER -- 0/1 swap bypassed interactive + flex_forced INTEGER, -- 0/1 swap bypassed interactive + pinch_original_tokens INTEGER, -- estimated tokens before pruning + pinch_final_tokens INTEGER -- tokens sent after pruning (may + -- equal original when no pruning) ); CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider); diff --git a/plans/pinch-instrumentation-and-token-accounting.md b/plans/pinch-instrumentation-and-token-accounting.md index 70b9c82..9be61fd 100644 --- a/plans/pinch-instrumentation-and-token-accounting.md +++ b/plans/pinch-instrumentation-and-token-accounting.md @@ -129,10 +129,28 @@ compare them to. it already adds `request_id` and `exploration` to this table.* 3. **Surface it in `/metrics`.** A `pinch` block alongside the existing aggregates: share of requests pruned, median and total `tokens_saved`, and - estimated dollars saved (price the delta at the selected model's cached - prompt rate — `metrics.py` already joins what it needs). This is the number - that says whether pinch earns its place, and there is currently no way to - ask for it. + estimated dollars saved. **Price the saved tokens at the same BLENDED rate + `routing.estimated_cost` uses**, not at the cached rate: + + ``` + saved_usd = saved_tokens + * ( (1 - cache_rate) * cost_per_1m_prompt + + cache_rate * COALESCE(cost_per_1m_prompt_cached, cost_per_1m_prompt) ) + / 1_000_000 + ``` + + *(Corrected 2026-09-01. An earlier draft said "price the delta at the + cached prompt rate", which is wrong twice over: `cost_per_1m_prompt_cached` + is already the discounted per-1M price, so multiplying by `cache_rate` + applies the discount a second time, and it silently drops the + `(1 - cache_rate)` fraction billed at full price. Match + `routing.estimated_cost` — `src/routing.py:363-374` — or the dashboard will + report a savings figure that disagrees with the cost model the router + actually ranks on. Note the COALESCE: `cost_per_1m_prompt_cached` can be + NULL, and routing falls back to the full prompt price in that case.)* + + This is the number that says whether pinch earns its place, and there is + currently no way to ask for it. ### Commit 2 — count what gets billed diff --git a/requirements.txt b/requirements.txt index 839092c..b2494ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ openai==3.0.0 python-dotenv==1.2.2 pytest==9.1.1 pytest-cov==7.1.0 +pytest-xdist==3.8.0 # Deliberate UI-bring-your-own-tool dependency: textual is the terminal # dashboard framework for the monitoring TUI (tui.py), a SEPARATE entrypoint. # It is only imported by tui.py — never by the service dispatch path, so the diff --git a/src/admin.py b/src/admin.py index 0928d6c..c184e1e 100644 --- a/src/admin.py +++ b/src/admin.py @@ -653,6 +653,7 @@ def build_router( "verdict_mix": metrics.verdict_mix(conn), "top_proficiency": metrics.top_proficiency(conn, "coding_general"), "local_energy": metrics.local_energy_summary(conn, cfg), + "pinch": metrics.pinch_summary(conn, cfg), "health": _health_check(conn, cfg), "generated_at": datetime.now(timezone.utc).isoformat(), } diff --git a/src/context_prune.py b/src/context_prune.py index a41506c..65eaa0a 100644 --- a/src/context_prune.py +++ b/src/context_prune.py @@ -242,6 +242,7 @@ def prune_context( keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default, max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default, relevance_order: Optional[list[int]] = None, + extra_fixed_tokens: int = 0, ) -> tuple[list[dict], dict]: """Trim old tool results once a conversation exceeds ``budget_tokens``. @@ -260,7 +261,7 @@ def prune_context( Only runs (and only mutates anything) when the estimate actually exceeds the budget; otherwise the original list is returned untouched. """ - orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens if orig_tokens <= budget_tokens: return messages, { "pruned": False, diff --git a/src/dispatcher.py b/src/dispatcher.py index 31e8e31..57dbb98 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -91,6 +91,7 @@ from verification import ( from metrics import ( local_energy_summary, per_model, + pinch_summary, quota_burn, recent_decisions, scoring_coverage, @@ -328,7 +329,9 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None: streamed INTEGER, flex_preference TEXT, flex_swapped INTEGER, - flex_forced INTEGER + flex_forced INTEGER, + pinch_original_tokens INTEGER, + pinch_final_tokens INTEGER ) """ ) @@ -346,6 +349,8 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None: ("flex_preference", "TEXT"), ("flex_swapped", "INTEGER"), ("flex_forced", "INTEGER"), + ("pinch_original_tokens", "INTEGER"), + ("pinch_final_tokens", "INTEGER"), ): if name not in existing: conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}") @@ -989,6 +994,8 @@ def persist_route_decision( flex_preference=None, flex_swapped=0, flex_forced=0, + pinch_original_tokens=None, + pinch_final_tokens=None, ) -> None: """Record one routing decision to route_decisions, best-effort and gated. @@ -1078,8 +1085,9 @@ def persist_route_decision( selected_model, selected_provider, runner_up_models, est_cost_usd, est_proficiency, rejected_reason, session_key, tools, images, json_mode, streamed, - flex_preference, flex_swapped, flex_forced - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + flex_preference, flex_swapped, flex_forced, + pinch_original_tokens, pinch_final_tokens + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( observed_at, @@ -1106,6 +1114,8 @@ def persist_route_decision( flex_preference, int(bool(flex_swapped)), int(bool(flex_forced)), + pinch_original_tokens, + pinch_final_tokens, ), ) decision_id = int(cursor.lastrowid) @@ -1142,6 +1152,8 @@ def persist_route_decision( "flex_preference": flex_preference, "flex_swapped": int(bool(flex_swapped)), "flex_forced": int(bool(flex_forced)), + "pinch_original_tokens": pinch_original_tokens, + "pinch_final_tokens": pinch_final_tokens, } ) except Exception as e: # noqa: BLE001 - best-effort must never raise @@ -1462,6 +1474,7 @@ def metrics_endpoint(): "verdict_mix": verdict_mix(conn), "top_proficiency": top_proficiency(conn, "coding_general"), "local_energy": local_energy_summary(conn, cfg), + "pinch": pinch_summary(conn, cfg), "generated_at": datetime.now(timezone.utc).isoformat(), } finally: @@ -1933,12 +1946,21 @@ def _embed_for_relevance( return order_by_relevance(embeddings[0], embeddings[1:]) -def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optional[list[int]]: +def _relevance_order_for( + messages: list[dict], + cfg, + budget_tokens: int, + extra_fixed_tokens: int = 0, +) -> Optional[list[int]]: """Compute the relevance_order for prune_context, or None (uniform). When pinch.relevance is off, or candidate count is below min_candidates, returns None so prune_context uses today's uniform pass. Otherwise makes one batched embedding call and returns the least-relevant-first index list. + + ``extra_fixed_tokens`` accounts for overhead that ``prune_context``'s + budget guard also adds to ``orig_tokens`` (e.g. tool-definition size), + so the skip-budget check in here matches the same total. """ rel = cfg.pinch.relevance if not (cfg.pinch.enabled and rel.enabled): @@ -1947,8 +1969,10 @@ def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optio if len(candidates) < rel.min_candidates: return None # Skip the embed when the conversation is under budget — prune_context would - # be a no-op, so ranking is pointless. - orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + # be a no-op, so ranking is pointless. Same total as prune_context includes + # extra_fixed_tokens (e.g. tool definitions) so relevance doesn't silently + # decline on tool-heavy requests. + orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens if orig_tokens <= budget_tokens: return None query = _last_user_text(messages) @@ -2355,6 +2379,11 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): session_key = session_fingerprint(messages) streamed = bool(body.get("stream")) + # Pinch stats are filled by the routed and passthrough prune paths (when + # pinch is enabled) and read by all persist sites. A single declaration + # keeps the variable in scope regardless of branch. + pinch_stats = None + # What this request needs, read from the body once and used by both the # routed and pass-through branches. Stated by the caller, not guessed at: # a tools array, an image_url part and a response_format each say exactly @@ -2404,20 +2433,26 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): # messages, so the classification turn above (which used # `_previous_context` from the full messages) is undisturbed. if cfg.pinch.enabled: + _tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN send_messages, pinch_stats = prune_context( list(messages), budget_tokens=cfg.pinch.budget_tokens, keep_last_turns=cfg.pinch.keep_last_turns, max_summarize_chars=cfg.pinch.max_summarize_chars, - relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens), - ) - logs.debug( - "pinch", - pruned=pinch_stats["pruned"], - saved=pinch_stats["tokens_saved"], - orig=pinch_stats["original_tokens"], - final=pinch_stats["final_tokens"], + relevance_order=_relevance_order_for( + messages, cfg, budget_tokens=cfg.pinch.budget_tokens, + extra_fixed_tokens=_tools_overhead, + ), + extra_fixed_tokens=_tools_overhead, ) + if pinch_stats.get("pruned"): + logs.info( + "pinch", + pruned=pinch_stats["pruned"], + saved=pinch_stats["tokens_saved"], + orig=pinch_stats["original_tokens"], + final=pinch_stats["final_tokens"], + ) else: send_messages = messages measured = estimate_prompt_tokens(send_messages, tools=body.get("tools")) @@ -2559,6 +2594,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): images=int(caps.has_images), json_mode=int(caps.require_json_mode), streamed=streamed, + pinch_original_tokens=pinch_stats.get("original_tokens") + if pinch_stats is not None else None, + pinch_final_tokens=pinch_stats.get("final_tokens") + if pinch_stats is not None else None, ) raise HTTPException( 422, @@ -2583,6 +2622,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): streamed=streamed, classification_source=classified_src, classifier_ms=classifier_ms, + pinch_original_tokens=pinch_stats.get("original_tokens") + if pinch_stats is not None else None, + pinch_final_tokens=pinch_stats.get("final_tokens") + if pinch_stats is not None else None, ) target = decision.selected.model_id provider = decision.selected.provider @@ -2592,6 +2635,31 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): # Not a routing decision at all, and worth saying so plainly: a client # pinned to one model gets none of the filtering or ranking below. logs.info("passthrough", model=target, stream=bool(body.get("stream"))) + send_messages = messages + if cfg.pinch.enabled: + # Same pruning as the routed path, but it must run before persistence + # so a pruned passthrough request records real pinch tokens rather + # than NULL. + _tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN + send_messages, pinch_stats = prune_context( + list(messages), + budget_tokens=cfg.pinch.budget_tokens, + keep_last_turns=cfg.pinch.keep_last_turns, + max_summarize_chars=cfg.pinch.max_summarize_chars, + relevance_order=_relevance_order_for( + messages, cfg, budget_tokens=cfg.pinch.budget_tokens, + extra_fixed_tokens=_tools_overhead, + ), + extra_fixed_tokens=_tools_overhead, + ) + if pinch_stats.get("pruned"): + logs.info( + "pinch", + pruned=pinch_stats["pruned"], + saved=pinch_stats["tokens_saved"], + orig=pinch_stats["original_tokens"], + final=pinch_stats["final_tokens"], + ) persist_route_decision( "passthrough", selected_model=target, @@ -2605,6 +2673,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): flex_preference=cfg.routing.default_flex_preference.value, flex_swapped=0, flex_forced=0, + pinch_original_tokens=pinch_stats.get("original_tokens") + if pinch_stats is not None else None, + pinch_final_tokens=pinch_stats.get("final_tokens") + if pinch_stats is not None else None, ) # A pinned model id is dispatched as asked, but a pin that cannot @@ -2625,31 +2697,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks): # top (needed by the passthrough/local-vision branches), only the working # directory is derived here because only the observation path uses it. session_dir = session_directory(messages) - if not wants_routing: - # The routed path pruned send_messages once, before its measured-context - # decision, and the pruned list is reused here. A passthrough request - # never went through that path, so it prunes now — same gating as - # before (only when pinch is enabled). - send_messages = messages - if cfg.pinch.enabled: - # Optional relevance-based context pruning: when the conversation - # exceeds budget_tokens, trim old tool results BEFORE any paid token - # is sent upstream. User/assistant/system messages are always kept; - # only the classifier input is exempt (it already clamps to head+tail). - send_messages, pinch_stats = prune_context( - list(messages), - budget_tokens=cfg.pinch.budget_tokens, - keep_last_turns=cfg.pinch.keep_last_turns, - max_summarize_chars=cfg.pinch.max_summarize_chars, - relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens), - ) - logs.debug( - "pinch", - pruned=pinch_stats["pruned"], - saved=pinch_stats["tokens_saved"], - orig=pinch_stats["original_tokens"], - final=pinch_stats["final_tokens"], - ) + # send_messages is already set: by the routed path's prune block above, or + # by the passthrough path's prune block. Both branches default to messages + # when pinch is disabled. upstream_body = {**body, "model": target, "messages": list(send_messages)} streaming = bool(body.get("stream")) if streaming: diff --git a/src/metrics.py b/src/metrics.py index 6b45d57..c40e29a 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -16,6 +16,7 @@ recent_decisions — last N rows from the route_decisions observability table per_model — per-model aggregates over energy_observations (last 30 d) verdict_mix — counts by verdict from verifications (last N days) top_proficiency — top models by blended_score for a category +pinch_summary — context-pruning savings over the last 30 d """ from __future__ import annotations @@ -422,7 +423,8 @@ def recent_decisions( candidates_considered, selected_model, selected_provider, runner_up_models, est_cost_usd, est_proficiency, rejected_reason, session_key, tools, images, json_mode, streamed, - flex_preference, flex_swapped, flex_forced + flex_preference, flex_swapped, flex_forced, + pinch_original_tokens, pinch_final_tokens FROM route_decisions ORDER BY id DESC LIMIT ? @@ -518,7 +520,7 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]: COALESCE(SUM(cost_usd), 0) cost FROM local_energy_observations WHERE julianday(observed_at) > julianday('now', '-30 days') - GROUP BY call_type + GROUP BY call_type """ ).fetchall() by_type = { @@ -540,3 +542,99 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]: if reset_day is not None: result["next_reset_date"] = _next_reset_date(reset_day) return result + + +def pinch_summary(conn: sqlite3.Connection, cfg: Any) -> Optional[dict]: + """Aggregate context-pruning savings over the last 30 days. + + Returns None when pinch is disabled so callers can omit the section. + Computes the number of pruned calls, share of pruned calls, total and + median tokens saved, and an estimated dollar saving priced at the blended + prompt rate the router uses for cost estimation. + """ + if not cfg.pinch.enabled: + return None + + cache_rate = cfg.objective.assumed_cache_rate + + totals = conn.execute( + """ + SELECT COUNT(*) AS calls_30d, + SUM(CASE + WHEN pinch_original_tokens IS NOT NULL + AND pinch_final_tokens IS NOT NULL + AND pinch_original_tokens > pinch_final_tokens + THEN 1 ELSE 0 + END) AS pruned_calls_30d, + SUM(CASE + WHEN pinch_original_tokens IS NOT NULL + AND pinch_final_tokens IS NOT NULL + THEN pinch_original_tokens - pinch_final_tokens + ELSE 0 + END) AS total_tokens_saved + FROM route_decisions + WHERE julianday(observed_at) > julianday('now', '-30 days') + """ + ).fetchone() + + calls_30d = totals["calls_30d"] or 0 + pruned_calls_30d = totals["pruned_calls_30d"] or 0 + share_pruned = pruned_calls_30d / calls_30d if calls_30d else 0.0 + total_tokens_saved = totals["total_tokens_saved"] or 0 + + saved_rows = conn.execute( + """ + SELECT pinch_original_tokens - pinch_final_tokens AS tokens_saved + FROM route_decisions + WHERE julianday(observed_at) > julianday('now', '-30 days') + AND pinch_original_tokens IS NOT NULL + AND pinch_final_tokens IS NOT NULL + AND pinch_original_tokens > pinch_final_tokens + """ + ).fetchall() + median_tokens_saved = _percentile( + [r["tokens_saved"] for r in saved_rows], 50 + ) + + dollar_rows = conn.execute( + """ + SELECT rd.pinch_original_tokens - rd.pinch_final_tokens AS tokens_saved, + m.cost_per_1m_prompt, + m.cost_per_1m_prompt_cached + FROM route_decisions rd + JOIN models m + ON rd.selected_model = m.model_id + AND rd.selected_provider = m.provider + WHERE julianday(rd.observed_at) > julianday('now', '-30 days') + AND rd.pinch_original_tokens IS NOT NULL + AND rd.pinch_final_tokens IS NOT NULL + AND rd.pinch_original_tokens > rd.pinch_final_tokens + AND rd.selected_model IS NOT NULL + AND m.cost_per_1m_prompt IS NOT NULL + """ + ).fetchall() + + dollars_saved_usd_30d = 0.0 + for r in dollar_rows: + saved = r["tokens_saved"] + prompt_price = r["cost_per_1m_prompt"] + cached_price = r["cost_per_1m_prompt_cached"] + if cached_price is None: + cached_price = prompt_price + blended = (1.0 - cache_rate) * prompt_price + cache_rate * cached_price + dollars_saved_usd_30d += saved * blended / 1_000_000 + + result = { + "calls_30d": calls_30d, + "pruned_calls_30d": pruned_calls_30d, + "share_pruned": round(share_pruned, 4), + "total_tokens_saved": total_tokens_saved, + "median_tokens_saved": median_tokens_saved, + "dollars_saved_usd_30d": round(dollars_saved_usd_30d, 6), + } + reset_day = getattr(cfg.objective, "billing_reset_day", None) + if reset_day is not None: + result["reset_date"] = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat() + result["next_reset_date"] = _next_reset_date(reset_day) + return result + diff --git a/src/tui.py b/src/tui.py index 5df3880..314914a 100644 --- a/src/tui.py +++ b/src/tui.py @@ -254,6 +254,13 @@ class DashboardApp(App): def _on_interval(self) -> None: if getattr(self, "_unmounted", False): return + # Skip rendering while the dashboard screen is still composing + # (the DataTable isn't in the DOM yet). The next interval tick will + # catch it once the widget is mounted. + try: + self.query_one("#model-table", DataTable) + except Exception: # noqa: BLE001 — widget not yet available + return self._refresh() def action_refresh(self) -> None: diff --git a/src/tui_model.py b/src/tui_model.py index 76d615b..7fe279d 100644 --- a/src/tui_model.py +++ b/src/tui_model.py @@ -59,6 +59,17 @@ def build_model(data: dict) -> dict: else: quota_rows = [] + pinch = data.get("pinch") + if pinch: + pinch_rows = [ + {"label": "share_pruned", "value": pinch.get("share_pruned")}, + {"label": "total_tokens_saved", "value": pinch.get("total_tokens_saved")}, + {"label": "median_tokens_saved", "value": pinch.get("median_tokens_saved")}, + {"label": "dollars_saved_usd_30d", "value": pinch.get("dollars_saved_usd_30d")}, + ] + else: + pinch_rows = [] + per_model = [ { "model": r.get("model_id"), @@ -85,6 +96,7 @@ def build_model(data: dict) -> dict: return { "quota": quota_rows, + "pinch": pinch_rows, "per_model": per_model, "verdict_mix": verdict_mix, "recent_decisions": recent, diff --git a/tests/test_admin_health.py b/tests/test_admin_health.py index dce39bc..7b5e664 100644 --- a/tests/test_admin_health.py +++ b/tests/test_admin_health.py @@ -155,7 +155,9 @@ def test_admin_snapshot_has_all_top_level_keys(seeded_client): "per_model", "verdict_mix", "top_proficiency", + "local_energy", "health", + "pinch", "generated_at", ): assert key in data, f"missing top-level key {key!r}" diff --git a/tests/test_admin_snapshot.py b/tests/test_admin_snapshot.py index fde2fe8..599ffe0 100644 --- a/tests/test_admin_snapshot.py +++ b/tests/test_admin_snapshot.py @@ -31,6 +31,7 @@ EXHAUSTIVE_KEYS = ( "top_proficiency", "local_energy", "health", + "pinch", "generated_at", ) diff --git a/tests/test_context_prune.py b/tests/test_context_prune.py index 9fe6c07..189e044 100644 --- a/tests/test_context_prune.py +++ b/tests/test_context_prune.py @@ -536,6 +536,89 @@ def test_relevance_order_overrides_recency(): assert "chars trimmed" in out[3]["content"] +def test_extra_fixed_tokens_tips_budget_to_prune(): + # A conversation whose message tokens sit just under the budget, but tool + # overhead pushes the accounting total over. Without overhead it is a no-op; + # with overhead it prunes. + messages = [ + _user("question"), + _tool("read", "x" * 540), # 540 chars -> 180 tokens + _assistant("answer"), + _user("follow up"), + ] + msg_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + assert msg_tokens == 187 + + out, stats = prune_context( + messages, + budget_tokens=200, + keep_last_turns=0, + extra_fixed_tokens=0, + ) + assert stats["pruned"] is False + assert stats["original_tokens"] == msg_tokens + + out, stats = prune_context( + messages, + budget_tokens=200, + keep_last_turns=0, + extra_fixed_tokens=100, + ) + assert stats["pruned"] is True + assert stats["original_tokens"] == msg_tokens + 100 + # The tool result was trimmed to cover the deficit (187 + 100 - 200 = 87). + assert stats["tokens_saved"] >= 87 + tool_out = [m for m in out if m.get("role") == "tool"][0] + assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"] + + +def test_extra_fixed_tokens_zero_preserves_behavior(): + # The default and explicit zero must be byte-for-byte identical. + messages = [ + _user("question"), + _tool("read", "big result " * 500), + _assistant("answer"), + _user("follow up"), + _assistant("ok"), + ] + default_out, default_stats = prune_context( + messages, budget_tokens=100, keep_last_turns=0 + ) + explicit_zero_out, explicit_zero_stats = prune_context( + messages, + budget_tokens=100, + keep_last_turns=0, + extra_fixed_tokens=0, + ) + assert explicit_zero_out == default_out + assert explicit_zero_stats == default_stats + + +def test_tool_content_never_summarized(): + # Even when extra_fixed_tokens forces pruning, user/assistant/system + # messages survive verbatim; only tool results are candidates. + messages = [ + _user("what is the answer"), + _assistant("i'll check"), + _tool("search", "big tool result " * 500), + _assistant("the answer is 42"), + _user("thanks"), + ] + out, stats = prune_context( + messages, + budget_tokens=100, + keep_last_turns=0, + extra_fixed_tokens=1000, + ) + assert stats["pruned"] is True + assert len(out) == len(messages) + for orig, pruned in zip(messages, out): + if orig.get("role") in ("user", "assistant", "system"): + assert pruned["content"] == orig["content"] + tool_out = [m for m in out if m.get("role") == "tool"][0] + assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"] + + def test_relevance_order_none_matches_default(): # relevance_order=None must be byte-for-byte identical to not passing the # parameter at all (the historical uniform pass / fallback path). diff --git a/tests/test_eval_scoring.py b/tests/test_eval_scoring.py index 7bac321..d413265 100644 --- a/tests/test_eval_scoring.py +++ b/tests/test_eval_scoring.py @@ -12,6 +12,8 @@ point of testing it. import pytest +import eval_proficiency + from eval_proficiency import ( judge_for, parse_check_results, @@ -49,7 +51,8 @@ def test_code_that_does_not_parse_scores_zero_and_says_so(): assert "did not run" in detail -def test_infinite_loop_is_bounded_not_hung(): +def test_infinite_loop_is_bounded_not_hung(monkeypatch): + monkeypatch.setattr(eval_proficiency, "CODE_TIMEOUT_SECONDS", 1) score, detail = score_code("def f(x):\n while True: pass", ["f(1)==1"]) assert (score, detail) == (0.0, "timeout") diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 31977e9..740674b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -32,6 +32,7 @@ from metrics import ( _next_reset_date, context_ceilings, per_model, + pinch_summary, quota_burn, recent_decisions, scoring_coverage, @@ -626,6 +627,119 @@ def test_top_proficiency_empty_for_missing_category(tmp_path): # --- /health endpoint compatibility ------------------------------------------- +def _seed_pinch_models(conn: sqlite3.Connection) -> None: + """Insert two cheap/dear models plus a cheap-no-cached-price variant.""" + fresh = _now().isoformat() + rows = [ + # (model_id, tier, context, cost_per_1m_prompt, cost_per_1m_prompt_cached) + ("cheap", 2, 262128, 1.0, 0.5), + ("dear", 2, 262128, 5.0, None), + ("tiny", 1, 131072, 0.1, 0.1), + ] + for model_id, tier, context, cost, cached in rows: + conn.execute( + """ + INSERT INTO models ( + model_id, provider, base_model_id, tier, context_window, + effective_context_window, max_output_tokens, + cost_per_1m_prompt, cost_per_1m_completion, + cost_per_1m_prompt_cached, + supports_vision, supports_json_mode, + latency_class, reasoning_mode, context_variant, + access_level, availability, last_updated + ) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?, ?, + 1, 1, 'standard', 'default', 'full', 'public', 'active', + ?) + """, + (model_id, model_id, tier, context, cost, cost / 3, cached, fresh), + ) + conn.commit() + + +def _insert_pinch_decision(conn, *, model, provider, orig, final): + """Insert a route_decisions row with observed_at = now and given pinch values.""" + conn.execute( + """ + INSERT INTO route_decisions ( + observed_at, kind, task_category, task_tier, + required_context_tokens, confidence, classifier_ms, + classification_source, latency_tolerance, candidates_considered, + selected_model, selected_provider, runner_up_models, + est_cost_usd, est_proficiency, rejected_reason, session_key, + tools, images, json_mode, streamed, + pinch_original_tokens, pinch_final_tokens + ) VALUES (?, 'chat', 'coding_general', 2, 100, 0.9, 100, + 'classifier', 'interactive', 3, ?, ?, NULL, + 0.001, 0.9, NULL, 'sess', 0, 0, 0, 0, ?, ?) + """, + (_now().isoformat(), model, provider, orig, final), + ) + conn.commit() + + +def test_pinch_summary_aggregates_correctly(tmp_path, monkeypatch): + """Hand-computed aggregates over a seeded 30-day window.""" + cfg = load_config(str(ROOT / "config" / "config.yaml")) + monkeypatch.setattr(cfg.pinch, "enabled", True) + cache_rate = cfg.objective.assumed_cache_rate + + conn = _make_db(tmp_path) + _seed_pinch_models(conn) + + # Two pruned cheap rows: saved tokens 50 and 30. + _insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=100, final=50) + _insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=80, final=50) + # One non-pruned cheap row (final == orig). + _insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=60, final=60) + # One pruned dear row with NULL cost_per_1m_prompt_cached -> falls back to prompt price. + _insert_pinch_decision(conn, model="dear", provider="neuralwatt", orig=200, final=100) + # One rejection row with NULL selected_model -> counted in share/tokens but excluded from dollars. + _insert_pinch_decision(conn, model=None, provider=None, orig=1000, final=500) + + result = pinch_summary(conn, cfg) + assert result is not None + assert result["calls_30d"] == 5 + assert result["pruned_calls_30d"] == 4 + assert result["share_pruned"] == round(4 / 5, 4) + # saved tokens: 50 + 30 + 100 + 500 = 680 + assert result["total_tokens_saved"] == 680 + # median of [30, 50, 100, 500] = 75 + assert result["median_tokens_saved"] == 75 + + # dollar math: cheap saved 80 tokens at blended rate over cached prompt. + cheap_blended = (1 - cache_rate) * 1.0 + cache_rate * 0.5 + cheap_dollars = 80 * cheap_blended / 1_000_000 + # dear saved 100 tokens; cached price NULL -> fallback to prompt price 5.0. + dear_dollars = 100 * 5.0 / 1_000_000 + # rejection row with NULL selected_model is excluded from dollar math. + assert result["dollars_saved_usd_30d"] == round(cheap_dollars + dear_dollars, 6) + + +def test_pinch_summary_zero_rows(tmp_path, monkeypatch): + """Enabled with no decisions in the window returns zeros and None median.""" + cfg = load_config(str(ROOT / "config" / "config.yaml")) + monkeypatch.setattr(cfg.pinch, "enabled", True) + conn = _make_db(tmp_path) + result = pinch_summary(conn, cfg) + assert result is not None + assert result["calls_30d"] == 0 + assert result["pruned_calls_30d"] == 0 + assert result["share_pruned"] == 0.0 + assert result["total_tokens_saved"] == 0 + assert result["median_tokens_saved"] is None + assert result["dollars_saved_usd_30d"] == 0.0 + + +def test_pinch_summary_disabled_returns_none(tmp_path, monkeypatch): + """When cfg.pinch.enabled is False, None is returned.""" + conn = _make_db(tmp_path) + cfg = SimpleNamespace(pinch=SimpleNamespace(enabled=False)) + assert pinch_summary(conn, cfg) is None + + +# --- /health endpoint compatibility ------------------------------------------- + + def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch): """/health still returns the same SHAPE after moving functions to metrics.""" db_path = tmp_path / "test.db" diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py index f7fe059..5516cdf 100644 --- a/tests/test_metrics_endpoint.py +++ b/tests/test_metrics_endpoint.py @@ -161,6 +161,7 @@ def test_metrics_endpoint_has_all_top_level_keys(seeded_client): "verdict_mix", "top_proficiency", "local_energy", + "pinch", "generated_at", ): assert key in data, f"missing top-level key {key!r}" diff --git a/tests/test_route_decisions.py b/tests/test_route_decisions.py index 75ff0bd..f3b1e0e 100644 --- a/tests/test_route_decisions.py +++ b/tests/test_route_decisions.py @@ -58,6 +58,8 @@ ROUTE_DECISIONS_COLUMNS = [ "flex_preference", "flex_swapped", "flex_forced", + "pinch_original_tokens", + "pinch_final_tokens", ] @@ -205,6 +207,54 @@ def test_ensure_route_decisions_allows_insert(tmp_path): conn.close() +def test_persist_writes_pinch_columns(tmp_path, monkeypatch): + """persist_route_decision writes pinch_original_tokens and pinch_final_tokens + when provided, and defaults them to NULL when omitted.""" + db_path = tmp_path / "pinch.db" + conn = sqlite3.connect(db_path) + conn.executescript(_schema_minus_route_decisions()) + conn.close() + + monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path)) + monkeypatch.setattr(dispatcher.cfg.logging, "log_route_decisions", True) + + dispatcher.persist_route_decision( + "route", + classification=Classification( + task_category="coding_general", task_tier=2, + required_context_tokens=100, confidence=0.9, + ), + latency_tolerance="interactive", + selected_model="deepseek-v4-flash", + selected_provider="neuralwatt", + pinch_original_tokens=100, + pinch_final_tokens=50, + ) + + dispatcher.persist_route_decision( + "route", + classification=Classification( + task_category="coding_general", task_tier=2, + required_context_tokens=100, confidence=0.9, + ), + latency_tolerance="interactive", + selected_model="deepseek-v4-flash", + selected_provider="neuralwatt", + ) + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + rows = conn.execute( + "SELECT pinch_original_tokens, pinch_final_tokens FROM route_decisions ORDER BY id" + ).fetchall() + assert len(rows) == 2 + assert rows[0]["pinch_original_tokens"] == 100 + assert rows[0]["pinch_final_tokens"] == 50 + assert rows[1]["pinch_original_tokens"] is None + assert rows[1]["pinch_final_tokens"] is None + conn.close() + + def test_persist_ensure_on_write_fixes_live_db_missing_table(tmp_path, monkeypatch): """A live router.db without route_decisions gets it on the WRITE path. @@ -600,6 +650,53 @@ def test_passthrough_persists_one_row_with_no_nameerror(decision_router): assert r["session_key"] is not None +def test_passthrough_records_pinch_columns_when_pruned(decision_router, monkeypatch): + """Regression: passthrough pruning must run before persistence so pinch stats + are recorded rather than NULL.""" + client, db_path = decision_router + monkeypatch.setattr(dispatcher.cfg.pinch, "enabled", True) + monkeypatch.setattr(dispatcher.cfg.pinch, "budget_tokens", 1) + monkeypatch.setattr(dispatcher.cfg.pinch, "keep_last_turns", 1) + + # Long conversation plus tools to exceed the tiny budget. + long_text = "word " * 500 + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": long_text}, + {"role": "assistant", "content": "acknowledged"}, + {"role": "user", "content": "now summarize"}, + ] + resp = client.post( + "/v1/chat/completions", + json={ + "model": DEAR, + "messages": messages, + "stream": False, + "tools": [ + { + "type": "function", + "function": { + "name": "noop", + "description": "A tool with a long description to inflate overhead", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + # Routing succeeds because the fixture stubs the provider call; persistence + # happens before forwarding, so the row is already written. + assert resp.status_code == 200 + + rows = _rows(db_path) + passthrough_rows = [r for r in rows if r["kind"] == "passthrough"] + assert len(passthrough_rows) == 1 + r = passthrough_rows[0] + assert r["pinch_original_tokens"] is not None + assert r["pinch_final_tokens"] is not None + assert r["pinch_original_tokens"] > r["pinch_final_tokens"] + + def test_local_vision_success_persists_one_local_row(decision_router, monkeypatch): client, db_path = decision_router _drop_cheap(db_path) diff --git a/tests/test_tui.py b/tests/test_tui.py index 005eafd..ede4d21 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -114,6 +114,14 @@ def _fixture() -> dict: "blended_score": 1.0, "source": "self_eval_thin", "self_eval_samples": 3} ], + "pinch": { + "calls_30d": 120, + "pruned_calls_30d": 30, + "share_pruned": 0.25, + "total_tokens_saved": 50000, + "median_tokens_saved": 1200, + "dollars_saved_usd_30d": 0.0125, + }, "generated_at": "2026-08-23T10:01:00+00:00", } @@ -134,6 +142,23 @@ def test_build_model_quota_panel(): assert "calls=18" in joined +def test_build_model_pinch_panel(): + m = build_model(_fixture()) + rows = m["pinch"] + by_label = {r["label"]: r["value"] for r in rows} + assert by_label["share_pruned"] == 0.25 + assert by_label["total_tokens_saved"] == 50000 + assert by_label["median_tokens_saved"] == 1200 + assert by_label["dollars_saved_usd_30d"] == 0.0125 + + +def test_build_model_pinch_panel_empty_when_none(): + data = _fixture() + data["pinch"] = None + m = build_model(data) + assert m["pinch"] == [] + + def test_build_model_quota_panel_includes_reset_date(): m = build_model(_fixture()) rows = m["quota"]