feat(pinch): instrument savings and count tool overhead in the budget #18

Merged
alee merged 4 commits from feat/pinch-instrumentation-and-token-accounting into main 2026-09-02 03:16:53 +00:00
18 changed files with 600 additions and 49 deletions

View File

@@ -311,7 +311,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
<div class="container-xl">
<div class="row row-cards">
<!-- Row 1: Local compute (own meter) + Cloud compute (NeuralWatt twin)
<!-- Row 1: Local compute (own meter) + Pinch savings
side-by-side above the wider model-usage section. Card bodies
rebuild every poll; they share no chart lifecycle. -->
<div class="col-xl-6">
@@ -324,6 +324,18 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
</div>
</div>
</div>
<div class="col-xl-6">
<div class="card h-100">
<div class="card-header d-flex align-items-center justify-content-between">
<h3 class="card-title mb-0"><span class="me-2" data-icon="bar-chart"></span>Pinch savings</h3>
</div>
<div class="card-body p-3" id="pinch-content">
<div class="empty">Loading&hellip;</div>
</div>
</div>
</div>
<!-- Row 1.5: Cloud compute (NeuralWatt twin) -->
<div class="col-xl-6">
<div class="card h-100">
<div class="card-header d-flex align-items-center justify-content-between">
@@ -540,6 +552,7 @@ async function loadSnapshot() {
renderQuotaChip(data.quota);
renderDecisions(data.recent_decisions);
renderLocalEnergy(data.local_energy);
renderPinch(data.pinch);
renderCloudEnergy(data.per_model);
renderBars(data.per_model);
renderVerdict(data.verdict_mix);
@@ -765,6 +778,27 @@ function renderLocalEnergy(summary) {
${typeHtml}`;
}
function renderPinch(summary) {
const el = document.getElementById('pinch-content');
if (!summary) {
el.innerHTML = '<div class="empty">Pinch is disabled</div>';
return;
}
const share = Number(summary.share_pruned || 0);
const totalSaved = Number(summary.total_tokens_saved || 0).toLocaleString();
const medianSaved = summary.median_tokens_saved == null
? 'n/a'
: Number(summary.median_tokens_saved).toLocaleString();
const dollars = Number(summary.dollars_saved_usd_30d || 0).toFixed(6);
el.innerHTML = `
<div class="d-flex gap-4">
<div><div class="text-muted small">share pruned</div><div class="h4 m-0">${(share * 100).toFixed(1)}%</div></div>
<div><div class="text-muted small">tokens saved</div><div class="h4 m-0">${totalSaved}</div></div>
<div><div class="text-muted small">median saved</div><div class="h4 m-0">${medianSaved}</div></div>
<div><div class="text-muted small">30d dollars saved</div><div class="h4 m-0">$${dollars}</div></div>
</div>`;
}
function renderCloudEnergy(perModel) {
const el = document.getElementById('cloud-energy-content');
const data = perModel || [];

View File

@@ -234,7 +234,10 @@ CREATE TABLE IF NOT EXISTS route_decisions (
flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex'
-- |'force-flex' (resolved)
flex_swapped INTEGER, -- 0/1 post-rank flex swap applied
flex_forced INTEGER -- 0/1 swap bypassed interactive
flex_forced INTEGER, -- 0/1 swap bypassed interactive
pinch_original_tokens INTEGER, -- estimated tokens before pruning
pinch_final_tokens INTEGER -- tokens sent after pruning (may
-- equal original when no pruning)
);
CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider);

View File

@@ -129,10 +129,28 @@ compare them to.
it already adds `request_id` and `exploration` to this table.*
3. **Surface it in `/metrics`.** A `pinch` block alongside the existing
aggregates: share of requests pruned, median and total `tokens_saved`, and
estimated dollars saved (price the delta at the selected model's cached
prompt rate — `metrics.py` already joins what it needs). This is the number
that says whether pinch earns its place, and there is currently no way to
ask for it.
estimated dollars saved. **Price the saved tokens at the same BLENDED rate
`routing.estimated_cost` uses**, not at the cached rate:
```
saved_usd = saved_tokens
* ( (1 - cache_rate) * cost_per_1m_prompt
+ cache_rate * COALESCE(cost_per_1m_prompt_cached, cost_per_1m_prompt) )
/ 1_000_000
```
*(Corrected 2026-09-01. An earlier draft said "price the delta at the
cached prompt rate", which is wrong twice over: `cost_per_1m_prompt_cached`
is already the discounted per-1M price, so multiplying by `cache_rate`
applies the discount a second time, and it silently drops the
`(1 - cache_rate)` fraction billed at full price. Match
`routing.estimated_cost` — `src/routing.py:363-374` — or the dashboard will
report a savings figure that disagrees with the cost model the router
actually ranks on. Note the COALESCE: `cost_per_1m_prompt_cached` can be
NULL, and routing falls back to the full prompt price in that case.)*
This is the number that says whether pinch earns its place, and there is
currently no way to ask for it.
### Commit 2 — count what gets billed

View File

@@ -14,6 +14,7 @@ openai==3.0.0
python-dotenv==1.2.2
pytest==9.1.1
pytest-cov==7.1.0
pytest-xdist==3.8.0
# Deliberate UI-bring-your-own-tool dependency: textual is the terminal
# dashboard framework for the monitoring TUI (tui.py), a SEPARATE entrypoint.
# It is only imported by tui.py — never by the service dispatch path, so the

View File

@@ -653,6 +653,7 @@ def build_router(
"verdict_mix": metrics.verdict_mix(conn),
"top_proficiency": metrics.top_proficiency(conn, "coding_general"),
"local_energy": metrics.local_energy_summary(conn, cfg),
"pinch": metrics.pinch_summary(conn, cfg),
"health": _health_check(conn, cfg),
"generated_at": datetime.now(timezone.utc).isoformat(),
}

View File

@@ -242,6 +242,7 @@ def prune_context(
keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default,
max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default,
relevance_order: Optional[list[int]] = None,
extra_fixed_tokens: int = 0,
) -> tuple[list[dict], dict]:
"""Trim old tool results once a conversation exceeds ``budget_tokens``.
@@ -260,7 +261,7 @@ def prune_context(
Only runs (and only mutates anything) when the estimate actually exceeds
the budget; otherwise the original list is returned untouched.
"""
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
if orig_tokens <= budget_tokens:
return messages, {
"pruned": False,

View File

@@ -91,6 +91,7 @@ from verification import (
from metrics import (
local_energy_summary,
per_model,
pinch_summary,
quota_burn,
recent_decisions,
scoring_coverage,
@@ -328,7 +329,9 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
streamed INTEGER,
flex_preference TEXT,
flex_swapped INTEGER,
flex_forced INTEGER
flex_forced INTEGER,
pinch_original_tokens INTEGER,
pinch_final_tokens INTEGER
)
"""
)
@@ -346,6 +349,8 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
("flex_preference", "TEXT"),
("flex_swapped", "INTEGER"),
("flex_forced", "INTEGER"),
("pinch_original_tokens", "INTEGER"),
("pinch_final_tokens", "INTEGER"),
):
if name not in existing:
conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}")
@@ -989,6 +994,8 @@ def persist_route_decision(
flex_preference=None,
flex_swapped=0,
flex_forced=0,
pinch_original_tokens=None,
pinch_final_tokens=None,
) -> None:
"""Record one routing decision to route_decisions, best-effort and gated.
@@ -1078,8 +1085,9 @@ def persist_route_decision(
selected_model, selected_provider, runner_up_models,
est_cost_usd, est_proficiency, rejected_reason, session_key,
tools, images, json_mode, streamed,
flex_preference, flex_swapped, flex_forced
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
flex_preference, flex_swapped, flex_forced,
pinch_original_tokens, pinch_final_tokens
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
observed_at,
@@ -1106,6 +1114,8 @@ def persist_route_decision(
flex_preference,
int(bool(flex_swapped)),
int(bool(flex_forced)),
pinch_original_tokens,
pinch_final_tokens,
),
)
decision_id = int(cursor.lastrowid)
@@ -1142,6 +1152,8 @@ def persist_route_decision(
"flex_preference": flex_preference,
"flex_swapped": int(bool(flex_swapped)),
"flex_forced": int(bool(flex_forced)),
"pinch_original_tokens": pinch_original_tokens,
"pinch_final_tokens": pinch_final_tokens,
}
)
except Exception as e: # noqa: BLE001 - best-effort must never raise
@@ -1462,6 +1474,7 @@ def metrics_endpoint():
"verdict_mix": verdict_mix(conn),
"top_proficiency": top_proficiency(conn, "coding_general"),
"local_energy": local_energy_summary(conn, cfg),
"pinch": pinch_summary(conn, cfg),
"generated_at": datetime.now(timezone.utc).isoformat(),
}
finally:
@@ -1933,12 +1946,21 @@ def _embed_for_relevance(
return order_by_relevance(embeddings[0], embeddings[1:])
def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optional[list[int]]:
def _relevance_order_for(
messages: list[dict],
cfg,
budget_tokens: int,
extra_fixed_tokens: int = 0,
) -> Optional[list[int]]:
"""Compute the relevance_order for prune_context, or None (uniform).
When pinch.relevance is off, or candidate count is below min_candidates,
returns None so prune_context uses today's uniform pass. Otherwise makes
one batched embedding call and returns the least-relevant-first index list.
``extra_fixed_tokens`` accounts for overhead that ``prune_context``'s
budget guard also adds to ``orig_tokens`` (e.g. tool-definition size),
so the skip-budget check in here matches the same total.
"""
rel = cfg.pinch.relevance
if not (cfg.pinch.enabled and rel.enabled):
@@ -1947,8 +1969,10 @@ def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optio
if len(candidates) < rel.min_candidates:
return None
# Skip the embed when the conversation is under budget — prune_context would
# be a no-op, so ranking is pointless.
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
# be a no-op, so ranking is pointless. Same total as prune_context includes
# extra_fixed_tokens (e.g. tool definitions) so relevance doesn't silently
# decline on tool-heavy requests.
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
if orig_tokens <= budget_tokens:
return None
query = _last_user_text(messages)
@@ -2355,6 +2379,11 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
session_key = session_fingerprint(messages)
streamed = bool(body.get("stream"))
# Pinch stats are filled by the routed and passthrough prune paths (when
# pinch is enabled) and read by all persist sites. A single declaration
# keeps the variable in scope regardless of branch.
pinch_stats = None
# What this request needs, read from the body once and used by both the
# routed and pass-through branches. Stated by the caller, not guessed at:
# a tools array, an image_url part and a response_format each say exactly
@@ -2404,20 +2433,26 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
# messages, so the classification turn above (which used
# `_previous_context` from the full messages) is undisturbed.
if cfg.pinch.enabled:
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
send_messages, pinch_stats = prune_context(
list(messages),
budget_tokens=cfg.pinch.budget_tokens,
keep_last_turns=cfg.pinch.keep_last_turns,
max_summarize_chars=cfg.pinch.max_summarize_chars,
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
)
logs.debug(
"pinch",
pruned=pinch_stats["pruned"],
saved=pinch_stats["tokens_saved"],
orig=pinch_stats["original_tokens"],
final=pinch_stats["final_tokens"],
relevance_order=_relevance_order_for(
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
extra_fixed_tokens=_tools_overhead,
),
extra_fixed_tokens=_tools_overhead,
)
if pinch_stats.get("pruned"):
logs.info(
"pinch",
pruned=pinch_stats["pruned"],
saved=pinch_stats["tokens_saved"],
orig=pinch_stats["original_tokens"],
final=pinch_stats["final_tokens"],
)
else:
send_messages = messages
measured = estimate_prompt_tokens(send_messages, tools=body.get("tools"))
@@ -2559,6 +2594,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
images=int(caps.has_images),
json_mode=int(caps.require_json_mode),
streamed=streamed,
pinch_original_tokens=pinch_stats.get("original_tokens")
if pinch_stats is not None else None,
pinch_final_tokens=pinch_stats.get("final_tokens")
if pinch_stats is not None else None,
)
raise HTTPException(
422,
@@ -2583,6 +2622,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
streamed=streamed,
classification_source=classified_src,
classifier_ms=classifier_ms,
pinch_original_tokens=pinch_stats.get("original_tokens")
if pinch_stats is not None else None,
pinch_final_tokens=pinch_stats.get("final_tokens")
if pinch_stats is not None else None,
)
target = decision.selected.model_id
provider = decision.selected.provider
@@ -2592,6 +2635,31 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
# Not a routing decision at all, and worth saying so plainly: a client
# pinned to one model gets none of the filtering or ranking below.
logs.info("passthrough", model=target, stream=bool(body.get("stream")))
send_messages = messages
if cfg.pinch.enabled:
# Same pruning as the routed path, but it must run before persistence
# so a pruned passthrough request records real pinch tokens rather
# than NULL.
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
send_messages, pinch_stats = prune_context(
list(messages),
budget_tokens=cfg.pinch.budget_tokens,
keep_last_turns=cfg.pinch.keep_last_turns,
max_summarize_chars=cfg.pinch.max_summarize_chars,
relevance_order=_relevance_order_for(
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
extra_fixed_tokens=_tools_overhead,
),
extra_fixed_tokens=_tools_overhead,
)
if pinch_stats.get("pruned"):
logs.info(
"pinch",
pruned=pinch_stats["pruned"],
saved=pinch_stats["tokens_saved"],
orig=pinch_stats["original_tokens"],
final=pinch_stats["final_tokens"],
)
persist_route_decision(
"passthrough",
selected_model=target,
@@ -2605,6 +2673,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
flex_preference=cfg.routing.default_flex_preference.value,
flex_swapped=0,
flex_forced=0,
pinch_original_tokens=pinch_stats.get("original_tokens")
if pinch_stats is not None else None,
pinch_final_tokens=pinch_stats.get("final_tokens")
if pinch_stats is not None else None,
)
# A pinned model id is dispatched as asked, but a pin that cannot
@@ -2625,31 +2697,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
# top (needed by the passthrough/local-vision branches), only the working
# directory is derived here because only the observation path uses it.
session_dir = session_directory(messages)
if not wants_routing:
# The routed path pruned send_messages once, before its measured-context
# decision, and the pruned list is reused here. A passthrough request
# never went through that path, so it prunes now — same gating as
# before (only when pinch is enabled).
send_messages = messages
if cfg.pinch.enabled:
# Optional relevance-based context pruning: when the conversation
# exceeds budget_tokens, trim old tool results BEFORE any paid token
# is sent upstream. User/assistant/system messages are always kept;
# only the classifier input is exempt (it already clamps to head+tail).
send_messages, pinch_stats = prune_context(
list(messages),
budget_tokens=cfg.pinch.budget_tokens,
keep_last_turns=cfg.pinch.keep_last_turns,
max_summarize_chars=cfg.pinch.max_summarize_chars,
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
)
logs.debug(
"pinch",
pruned=pinch_stats["pruned"],
saved=pinch_stats["tokens_saved"],
orig=pinch_stats["original_tokens"],
final=pinch_stats["final_tokens"],
)
# send_messages is already set: by the routed path's prune block above, or
# by the passthrough path's prune block. Both branches default to messages
# when pinch is disabled.
upstream_body = {**body, "model": target, "messages": list(send_messages)}
streaming = bool(body.get("stream"))
if streaming:

View File

@@ -16,6 +16,7 @@ recent_decisions — last N rows from the route_decisions observability table
per_model — per-model aggregates over energy_observations (last 30 d)
verdict_mix — counts by verdict from verifications (last N days)
top_proficiency — top models by blended_score for a category
pinch_summary — context-pruning savings over the last 30 d
"""
from __future__ import annotations
@@ -422,7 +423,8 @@ def recent_decisions(
candidates_considered, selected_model, selected_provider,
runner_up_models, est_cost_usd, est_proficiency,
rejected_reason, session_key, tools, images, json_mode, streamed,
flex_preference, flex_swapped, flex_forced
flex_preference, flex_swapped, flex_forced,
pinch_original_tokens, pinch_final_tokens
FROM route_decisions
ORDER BY id DESC
LIMIT ?
@@ -518,7 +520,7 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]:
COALESCE(SUM(cost_usd), 0) cost
FROM local_energy_observations
WHERE julianday(observed_at) > julianday('now', '-30 days')
GROUP BY call_type
GROUP BY call_type
"""
).fetchall()
by_type = {
@@ -540,3 +542,99 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]:
if reset_day is not None:
result["next_reset_date"] = _next_reset_date(reset_day)
return result
def pinch_summary(conn: sqlite3.Connection, cfg: Any) -> Optional[dict]:
"""Aggregate context-pruning savings over the last 30 days.
Returns None when pinch is disabled so callers can omit the section.
Computes the number of pruned calls, share of pruned calls, total and
median tokens saved, and an estimated dollar saving priced at the blended
prompt rate the router uses for cost estimation.
"""
if not cfg.pinch.enabled:
return None
cache_rate = cfg.objective.assumed_cache_rate
totals = conn.execute(
"""
SELECT COUNT(*) AS calls_30d,
SUM(CASE
WHEN pinch_original_tokens IS NOT NULL
AND pinch_final_tokens IS NOT NULL
AND pinch_original_tokens > pinch_final_tokens
THEN 1 ELSE 0
END) AS pruned_calls_30d,
SUM(CASE
WHEN pinch_original_tokens IS NOT NULL
AND pinch_final_tokens IS NOT NULL
THEN pinch_original_tokens - pinch_final_tokens
ELSE 0
END) AS total_tokens_saved
FROM route_decisions
WHERE julianday(observed_at) > julianday('now', '-30 days')
"""
).fetchone()
calls_30d = totals["calls_30d"] or 0
pruned_calls_30d = totals["pruned_calls_30d"] or 0
share_pruned = pruned_calls_30d / calls_30d if calls_30d else 0.0
total_tokens_saved = totals["total_tokens_saved"] or 0
saved_rows = conn.execute(
"""
SELECT pinch_original_tokens - pinch_final_tokens AS tokens_saved
FROM route_decisions
WHERE julianday(observed_at) > julianday('now', '-30 days')
AND pinch_original_tokens IS NOT NULL
AND pinch_final_tokens IS NOT NULL
AND pinch_original_tokens > pinch_final_tokens
"""
).fetchall()
median_tokens_saved = _percentile(
[r["tokens_saved"] for r in saved_rows], 50
)
dollar_rows = conn.execute(
"""
SELECT rd.pinch_original_tokens - rd.pinch_final_tokens AS tokens_saved,
m.cost_per_1m_prompt,
m.cost_per_1m_prompt_cached
FROM route_decisions rd
JOIN models m
ON rd.selected_model = m.model_id
AND rd.selected_provider = m.provider
WHERE julianday(rd.observed_at) > julianday('now', '-30 days')
AND rd.pinch_original_tokens IS NOT NULL
AND rd.pinch_final_tokens IS NOT NULL
AND rd.pinch_original_tokens > rd.pinch_final_tokens
AND rd.selected_model IS NOT NULL
AND m.cost_per_1m_prompt IS NOT NULL
"""
).fetchall()
dollars_saved_usd_30d = 0.0
for r in dollar_rows:
saved = r["tokens_saved"]
prompt_price = r["cost_per_1m_prompt"]
cached_price = r["cost_per_1m_prompt_cached"]
if cached_price is None:
cached_price = prompt_price
blended = (1.0 - cache_rate) * prompt_price + cache_rate * cached_price
dollars_saved_usd_30d += saved * blended / 1_000_000
result = {
"calls_30d": calls_30d,
"pruned_calls_30d": pruned_calls_30d,
"share_pruned": round(share_pruned, 4),
"total_tokens_saved": total_tokens_saved,
"median_tokens_saved": median_tokens_saved,
"dollars_saved_usd_30d": round(dollars_saved_usd_30d, 6),
}
reset_day = getattr(cfg.objective, "billing_reset_day", None)
if reset_day is not None:
result["reset_date"] = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat()
result["next_reset_date"] = _next_reset_date(reset_day)
return result

View File

@@ -254,6 +254,13 @@ class DashboardApp(App):
def _on_interval(self) -> None:
if getattr(self, "_unmounted", False):
return
# Skip rendering while the dashboard screen is still composing
# (the DataTable isn't in the DOM yet). The next interval tick will
# catch it once the widget is mounted.
try:
self.query_one("#model-table", DataTable)
except Exception: # noqa: BLE001 — widget not yet available
return
self._refresh()
def action_refresh(self) -> None:

View File

@@ -59,6 +59,17 @@ def build_model(data: dict) -> dict:
else:
quota_rows = []
pinch = data.get("pinch")
if pinch:
pinch_rows = [
{"label": "share_pruned", "value": pinch.get("share_pruned")},
{"label": "total_tokens_saved", "value": pinch.get("total_tokens_saved")},
{"label": "median_tokens_saved", "value": pinch.get("median_tokens_saved")},
{"label": "dollars_saved_usd_30d", "value": pinch.get("dollars_saved_usd_30d")},
]
else:
pinch_rows = []
per_model = [
{
"model": r.get("model_id"),
@@ -85,6 +96,7 @@ def build_model(data: dict) -> dict:
return {
"quota": quota_rows,
"pinch": pinch_rows,
"per_model": per_model,
"verdict_mix": verdict_mix,
"recent_decisions": recent,

View File

@@ -155,7 +155,9 @@ def test_admin_snapshot_has_all_top_level_keys(seeded_client):
"per_model",
"verdict_mix",
"top_proficiency",
"local_energy",
"health",
"pinch",
"generated_at",
):
assert key in data, f"missing top-level key {key!r}"

View File

@@ -31,6 +31,7 @@ EXHAUSTIVE_KEYS = (
"top_proficiency",
"local_energy",
"health",
"pinch",
"generated_at",
)

View File

@@ -536,6 +536,89 @@ def test_relevance_order_overrides_recency():
assert "chars trimmed" in out[3]["content"]
def test_extra_fixed_tokens_tips_budget_to_prune():
# A conversation whose message tokens sit just under the budget, but tool
# overhead pushes the accounting total over. Without overhead it is a no-op;
# with overhead it prunes.
messages = [
_user("question"),
_tool("read", "x" * 540), # 540 chars -> 180 tokens
_assistant("answer"),
_user("follow up"),
]
msg_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
assert msg_tokens == 187
out, stats = prune_context(
messages,
budget_tokens=200,
keep_last_turns=0,
extra_fixed_tokens=0,
)
assert stats["pruned"] is False
assert stats["original_tokens"] == msg_tokens
out, stats = prune_context(
messages,
budget_tokens=200,
keep_last_turns=0,
extra_fixed_tokens=100,
)
assert stats["pruned"] is True
assert stats["original_tokens"] == msg_tokens + 100
# The tool result was trimmed to cover the deficit (187 + 100 - 200 = 87).
assert stats["tokens_saved"] >= 87
tool_out = [m for m in out if m.get("role") == "tool"][0]
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
def test_extra_fixed_tokens_zero_preserves_behavior():
# The default and explicit zero must be byte-for-byte identical.
messages = [
_user("question"),
_tool("read", "big result " * 500),
_assistant("answer"),
_user("follow up"),
_assistant("ok"),
]
default_out, default_stats = prune_context(
messages, budget_tokens=100, keep_last_turns=0
)
explicit_zero_out, explicit_zero_stats = prune_context(
messages,
budget_tokens=100,
keep_last_turns=0,
extra_fixed_tokens=0,
)
assert explicit_zero_out == default_out
assert explicit_zero_stats == default_stats
def test_tool_content_never_summarized():
# Even when extra_fixed_tokens forces pruning, user/assistant/system
# messages survive verbatim; only tool results are candidates.
messages = [
_user("what is the answer"),
_assistant("i'll check"),
_tool("search", "big tool result " * 500),
_assistant("the answer is 42"),
_user("thanks"),
]
out, stats = prune_context(
messages,
budget_tokens=100,
keep_last_turns=0,
extra_fixed_tokens=1000,
)
assert stats["pruned"] is True
assert len(out) == len(messages)
for orig, pruned in zip(messages, out):
if orig.get("role") in ("user", "assistant", "system"):
assert pruned["content"] == orig["content"]
tool_out = [m for m in out if m.get("role") == "tool"][0]
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
def test_relevance_order_none_matches_default():
# relevance_order=None must be byte-for-byte identical to not passing the
# parameter at all (the historical uniform pass / fallback path).

View File

@@ -12,6 +12,8 @@ point of testing it.
import pytest
import eval_proficiency
from eval_proficiency import (
judge_for,
parse_check_results,
@@ -49,7 +51,8 @@ def test_code_that_does_not_parse_scores_zero_and_says_so():
assert "did not run" in detail
def test_infinite_loop_is_bounded_not_hung():
def test_infinite_loop_is_bounded_not_hung(monkeypatch):
monkeypatch.setattr(eval_proficiency, "CODE_TIMEOUT_SECONDS", 1)
score, detail = score_code("def f(x):\n while True: pass", ["f(1)==1"])
assert (score, detail) == (0.0, "timeout")

View File

@@ -32,6 +32,7 @@ from metrics import (
_next_reset_date,
context_ceilings,
per_model,
pinch_summary,
quota_burn,
recent_decisions,
scoring_coverage,
@@ -626,6 +627,119 @@ def test_top_proficiency_empty_for_missing_category(tmp_path):
# --- /health endpoint compatibility -------------------------------------------
def _seed_pinch_models(conn: sqlite3.Connection) -> None:
"""Insert two cheap/dear models plus a cheap-no-cached-price variant."""
fresh = _now().isoformat()
rows = [
# (model_id, tier, context, cost_per_1m_prompt, cost_per_1m_prompt_cached)
("cheap", 2, 262128, 1.0, 0.5),
("dear", 2, 262128, 5.0, None),
("tiny", 1, 131072, 0.1, 0.1),
]
for model_id, tier, context, cost, cached in rows:
conn.execute(
"""
INSERT INTO models (
model_id, provider, base_model_id, tier, context_window,
effective_context_window, max_output_tokens,
cost_per_1m_prompt, cost_per_1m_completion,
cost_per_1m_prompt_cached,
supports_vision, supports_json_mode,
latency_class, reasoning_mode, context_variant,
access_level, availability, last_updated
) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?, ?,
1, 1, 'standard', 'default', 'full', 'public', 'active',
?)
""",
(model_id, model_id, tier, context, cost, cost / 3, cached, fresh),
)
conn.commit()
def _insert_pinch_decision(conn, *, model, provider, orig, final):
"""Insert a route_decisions row with observed_at = now and given pinch values."""
conn.execute(
"""
INSERT INTO route_decisions (
observed_at, kind, task_category, task_tier,
required_context_tokens, confidence, classifier_ms,
classification_source, latency_tolerance, candidates_considered,
selected_model, selected_provider, runner_up_models,
est_cost_usd, est_proficiency, rejected_reason, session_key,
tools, images, json_mode, streamed,
pinch_original_tokens, pinch_final_tokens
) VALUES (?, 'chat', 'coding_general', 2, 100, 0.9, 100,
'classifier', 'interactive', 3, ?, ?, NULL,
0.001, 0.9, NULL, 'sess', 0, 0, 0, 0, ?, ?)
""",
(_now().isoformat(), model, provider, orig, final),
)
conn.commit()
def test_pinch_summary_aggregates_correctly(tmp_path, monkeypatch):
"""Hand-computed aggregates over a seeded 30-day window."""
cfg = load_config(str(ROOT / "config" / "config.yaml"))
monkeypatch.setattr(cfg.pinch, "enabled", True)
cache_rate = cfg.objective.assumed_cache_rate
conn = _make_db(tmp_path)
_seed_pinch_models(conn)
# Two pruned cheap rows: saved tokens 50 and 30.
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=100, final=50)
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=80, final=50)
# One non-pruned cheap row (final == orig).
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=60, final=60)
# One pruned dear row with NULL cost_per_1m_prompt_cached -> falls back to prompt price.
_insert_pinch_decision(conn, model="dear", provider="neuralwatt", orig=200, final=100)
# One rejection row with NULL selected_model -> counted in share/tokens but excluded from dollars.
_insert_pinch_decision(conn, model=None, provider=None, orig=1000, final=500)
result = pinch_summary(conn, cfg)
assert result is not None
assert result["calls_30d"] == 5
assert result["pruned_calls_30d"] == 4
assert result["share_pruned"] == round(4 / 5, 4)
# saved tokens: 50 + 30 + 100 + 500 = 680
assert result["total_tokens_saved"] == 680
# median of [30, 50, 100, 500] = 75
assert result["median_tokens_saved"] == 75
# dollar math: cheap saved 80 tokens at blended rate over cached prompt.
cheap_blended = (1 - cache_rate) * 1.0 + cache_rate * 0.5
cheap_dollars = 80 * cheap_blended / 1_000_000
# dear saved 100 tokens; cached price NULL -> fallback to prompt price 5.0.
dear_dollars = 100 * 5.0 / 1_000_000
# rejection row with NULL selected_model is excluded from dollar math.
assert result["dollars_saved_usd_30d"] == round(cheap_dollars + dear_dollars, 6)
def test_pinch_summary_zero_rows(tmp_path, monkeypatch):
"""Enabled with no decisions in the window returns zeros and None median."""
cfg = load_config(str(ROOT / "config" / "config.yaml"))
monkeypatch.setattr(cfg.pinch, "enabled", True)
conn = _make_db(tmp_path)
result = pinch_summary(conn, cfg)
assert result is not None
assert result["calls_30d"] == 0
assert result["pruned_calls_30d"] == 0
assert result["share_pruned"] == 0.0
assert result["total_tokens_saved"] == 0
assert result["median_tokens_saved"] is None
assert result["dollars_saved_usd_30d"] == 0.0
def test_pinch_summary_disabled_returns_none(tmp_path, monkeypatch):
"""When cfg.pinch.enabled is False, None is returned."""
conn = _make_db(tmp_path)
cfg = SimpleNamespace(pinch=SimpleNamespace(enabled=False))
assert pinch_summary(conn, cfg) is None
# --- /health endpoint compatibility -------------------------------------------
def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch):
"""/health still returns the same SHAPE after moving functions to metrics."""
db_path = tmp_path / "test.db"

View File

@@ -161,6 +161,7 @@ def test_metrics_endpoint_has_all_top_level_keys(seeded_client):
"verdict_mix",
"top_proficiency",
"local_energy",
"pinch",
"generated_at",
):
assert key in data, f"missing top-level key {key!r}"

View File

@@ -58,6 +58,8 @@ ROUTE_DECISIONS_COLUMNS = [
"flex_preference",
"flex_swapped",
"flex_forced",
"pinch_original_tokens",
"pinch_final_tokens",
]
@@ -205,6 +207,54 @@ def test_ensure_route_decisions_allows_insert(tmp_path):
conn.close()
def test_persist_writes_pinch_columns(tmp_path, monkeypatch):
"""persist_route_decision writes pinch_original_tokens and pinch_final_tokens
when provided, and defaults them to NULL when omitted."""
db_path = tmp_path / "pinch.db"
conn = sqlite3.connect(db_path)
conn.executescript(_schema_minus_route_decisions())
conn.close()
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
monkeypatch.setattr(dispatcher.cfg.logging, "log_route_decisions", True)
dispatcher.persist_route_decision(
"route",
classification=Classification(
task_category="coding_general", task_tier=2,
required_context_tokens=100, confidence=0.9,
),
latency_tolerance="interactive",
selected_model="deepseek-v4-flash",
selected_provider="neuralwatt",
pinch_original_tokens=100,
pinch_final_tokens=50,
)
dispatcher.persist_route_decision(
"route",
classification=Classification(
task_category="coding_general", task_tier=2,
required_context_tokens=100, confidence=0.9,
),
latency_tolerance="interactive",
selected_model="deepseek-v4-flash",
selected_provider="neuralwatt",
)
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
"SELECT pinch_original_tokens, pinch_final_tokens FROM route_decisions ORDER BY id"
).fetchall()
assert len(rows) == 2
assert rows[0]["pinch_original_tokens"] == 100
assert rows[0]["pinch_final_tokens"] == 50
assert rows[1]["pinch_original_tokens"] is None
assert rows[1]["pinch_final_tokens"] is None
conn.close()
def test_persist_ensure_on_write_fixes_live_db_missing_table(tmp_path, monkeypatch):
"""A live router.db without route_decisions gets it on the WRITE path.
@@ -600,6 +650,53 @@ def test_passthrough_persists_one_row_with_no_nameerror(decision_router):
assert r["session_key"] is not None
def test_passthrough_records_pinch_columns_when_pruned(decision_router, monkeypatch):
"""Regression: passthrough pruning must run before persistence so pinch stats
are recorded rather than NULL."""
client, db_path = decision_router
monkeypatch.setattr(dispatcher.cfg.pinch, "enabled", True)
monkeypatch.setattr(dispatcher.cfg.pinch, "budget_tokens", 1)
monkeypatch.setattr(dispatcher.cfg.pinch, "keep_last_turns", 1)
# Long conversation plus tools to exceed the tiny budget.
long_text = "word " * 500
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": long_text},
{"role": "assistant", "content": "acknowledged"},
{"role": "user", "content": "now summarize"},
]
resp = client.post(
"/v1/chat/completions",
json={
"model": DEAR,
"messages": messages,
"stream": False,
"tools": [
{
"type": "function",
"function": {
"name": "noop",
"description": "A tool with a long description to inflate overhead",
"parameters": {"type": "object", "properties": {}},
},
}
],
},
)
# Routing succeeds because the fixture stubs the provider call; persistence
# happens before forwarding, so the row is already written.
assert resp.status_code == 200
rows = _rows(db_path)
passthrough_rows = [r for r in rows if r["kind"] == "passthrough"]
assert len(passthrough_rows) == 1
r = passthrough_rows[0]
assert r["pinch_original_tokens"] is not None
assert r["pinch_final_tokens"] is not None
assert r["pinch_original_tokens"] > r["pinch_final_tokens"]
def test_local_vision_success_persists_one_local_row(decision_router, monkeypatch):
client, db_path = decision_router
_drop_cheap(db_path)

View File

@@ -114,6 +114,14 @@ def _fixture() -> dict:
"blended_score": 1.0, "source": "self_eval_thin",
"self_eval_samples": 3}
],
"pinch": {
"calls_30d": 120,
"pruned_calls_30d": 30,
"share_pruned": 0.25,
"total_tokens_saved": 50000,
"median_tokens_saved": 1200,
"dollars_saved_usd_30d": 0.0125,
},
"generated_at": "2026-08-23T10:01:00+00:00",
}
@@ -134,6 +142,23 @@ def test_build_model_quota_panel():
assert "calls=18" in joined
def test_build_model_pinch_panel():
m = build_model(_fixture())
rows = m["pinch"]
by_label = {r["label"]: r["value"] for r in rows}
assert by_label["share_pruned"] == 0.25
assert by_label["total_tokens_saved"] == 50000
assert by_label["median_tokens_saved"] == 1200
assert by_label["dollars_saved_usd_30d"] == 0.0125
def test_build_model_pinch_panel_empty_when_none():
data = _fixture()
data["pinch"] = None
m = build_model(data)
assert m["pinch"] == []
def test_build_model_quota_panel_includes_reset_date():
m = build_model(_fixture())
rows = m["quota"]