feat(pinch): instrument savings and count tool overhead in the budget #18
@@ -311,7 +311,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
|||||||
<div class="container-xl">
|
<div class="container-xl">
|
||||||
<div class="row row-cards">
|
<div class="row row-cards">
|
||||||
|
|
||||||
<!-- Row 1: Local compute (own meter) + Cloud compute (NeuralWatt twin)
|
<!-- Row 1: Local compute (own meter) + Pinch savings
|
||||||
side-by-side above the wider model-usage section. Card bodies
|
side-by-side above the wider model-usage section. Card bodies
|
||||||
rebuild every poll; they share no chart lifecycle. -->
|
rebuild every poll; they share no chart lifecycle. -->
|
||||||
<div class="col-xl-6">
|
<div class="col-xl-6">
|
||||||
@@ -324,6 +324,18 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="col-xl-6">
|
||||||
|
<div class="card h-100">
|
||||||
|
<div class="card-header d-flex align-items-center justify-content-between">
|
||||||
|
<h3 class="card-title mb-0"><span class="me-2" data-icon="bar-chart"></span>Pinch savings</h3>
|
||||||
|
</div>
|
||||||
|
<div class="card-body p-3" id="pinch-content">
|
||||||
|
<div class="empty">Loading…</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Row 1.5: Cloud compute (NeuralWatt twin) -->
|
||||||
<div class="col-xl-6">
|
<div class="col-xl-6">
|
||||||
<div class="card h-100">
|
<div class="card h-100">
|
||||||
<div class="card-header d-flex align-items-center justify-content-between">
|
<div class="card-header d-flex align-items-center justify-content-between">
|
||||||
@@ -540,6 +552,7 @@ async function loadSnapshot() {
|
|||||||
renderQuotaChip(data.quota);
|
renderQuotaChip(data.quota);
|
||||||
renderDecisions(data.recent_decisions);
|
renderDecisions(data.recent_decisions);
|
||||||
renderLocalEnergy(data.local_energy);
|
renderLocalEnergy(data.local_energy);
|
||||||
|
renderPinch(data.pinch);
|
||||||
renderCloudEnergy(data.per_model);
|
renderCloudEnergy(data.per_model);
|
||||||
renderBars(data.per_model);
|
renderBars(data.per_model);
|
||||||
renderVerdict(data.verdict_mix);
|
renderVerdict(data.verdict_mix);
|
||||||
@@ -765,6 +778,27 @@ function renderLocalEnergy(summary) {
|
|||||||
${typeHtml}`;
|
${typeHtml}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderPinch(summary) {
|
||||||
|
const el = document.getElementById('pinch-content');
|
||||||
|
if (!summary) {
|
||||||
|
el.innerHTML = '<div class="empty">Pinch is disabled</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const share = Number(summary.share_pruned || 0);
|
||||||
|
const totalSaved = Number(summary.total_tokens_saved || 0).toLocaleString();
|
||||||
|
const medianSaved = summary.median_tokens_saved == null
|
||||||
|
? 'n/a'
|
||||||
|
: Number(summary.median_tokens_saved).toLocaleString();
|
||||||
|
const dollars = Number(summary.dollars_saved_usd_30d || 0).toFixed(6);
|
||||||
|
el.innerHTML = `
|
||||||
|
<div class="d-flex gap-4">
|
||||||
|
<div><div class="text-muted small">share pruned</div><div class="h4 m-0">${(share * 100).toFixed(1)}%</div></div>
|
||||||
|
<div><div class="text-muted small">tokens saved</div><div class="h4 m-0">${totalSaved}</div></div>
|
||||||
|
<div><div class="text-muted small">median saved</div><div class="h4 m-0">${medianSaved}</div></div>
|
||||||
|
<div><div class="text-muted small">30d dollars saved</div><div class="h4 m-0">$${dollars}</div></div>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
function renderCloudEnergy(perModel) {
|
function renderCloudEnergy(perModel) {
|
||||||
const el = document.getElementById('cloud-energy-content');
|
const el = document.getElementById('cloud-energy-content');
|
||||||
const data = perModel || [];
|
const data = perModel || [];
|
||||||
|
|||||||
@@ -234,7 +234,10 @@ CREATE TABLE IF NOT EXISTS route_decisions (
|
|||||||
flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex'
|
flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex'
|
||||||
-- |'force-flex' (resolved)
|
-- |'force-flex' (resolved)
|
||||||
flex_swapped INTEGER, -- 0/1 post-rank flex swap applied
|
flex_swapped INTEGER, -- 0/1 post-rank flex swap applied
|
||||||
flex_forced INTEGER -- 0/1 swap bypassed interactive
|
flex_forced INTEGER, -- 0/1 swap bypassed interactive
|
||||||
|
pinch_original_tokens INTEGER, -- estimated tokens before pruning
|
||||||
|
pinch_final_tokens INTEGER -- tokens sent after pruning (may
|
||||||
|
-- equal original when no pruning)
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider);
|
CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider);
|
||||||
|
|||||||
@@ -129,10 +129,28 @@ compare them to.
|
|||||||
it already adds `request_id` and `exploration` to this table.*
|
it already adds `request_id` and `exploration` to this table.*
|
||||||
3. **Surface it in `/metrics`.** A `pinch` block alongside the existing
|
3. **Surface it in `/metrics`.** A `pinch` block alongside the existing
|
||||||
aggregates: share of requests pruned, median and total `tokens_saved`, and
|
aggregates: share of requests pruned, median and total `tokens_saved`, and
|
||||||
estimated dollars saved (price the delta at the selected model's cached
|
estimated dollars saved. **Price the saved tokens at the same BLENDED rate
|
||||||
prompt rate — `metrics.py` already joins what it needs). This is the number
|
`routing.estimated_cost` uses**, not at the cached rate:
|
||||||
that says whether pinch earns its place, and there is currently no way to
|
|
||||||
ask for it.
|
```
|
||||||
|
saved_usd = saved_tokens
|
||||||
|
* ( (1 - cache_rate) * cost_per_1m_prompt
|
||||||
|
+ cache_rate * COALESCE(cost_per_1m_prompt_cached, cost_per_1m_prompt) )
|
||||||
|
/ 1_000_000
|
||||||
|
```
|
||||||
|
|
||||||
|
*(Corrected 2026-09-01. An earlier draft said "price the delta at the
|
||||||
|
cached prompt rate", which is wrong twice over: `cost_per_1m_prompt_cached`
|
||||||
|
is already the discounted per-1M price, so multiplying by `cache_rate`
|
||||||
|
applies the discount a second time, and it silently drops the
|
||||||
|
`(1 - cache_rate)` fraction billed at full price. Match
|
||||||
|
`routing.estimated_cost` — `src/routing.py:363-374` — or the dashboard will
|
||||||
|
report a savings figure that disagrees with the cost model the router
|
||||||
|
actually ranks on. Note the COALESCE: `cost_per_1m_prompt_cached` can be
|
||||||
|
NULL, and routing falls back to the full prompt price in that case.)*
|
||||||
|
|
||||||
|
This is the number that says whether pinch earns its place, and there is
|
||||||
|
currently no way to ask for it.
|
||||||
|
|
||||||
### Commit 2 — count what gets billed
|
### Commit 2 — count what gets billed
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ openai==3.0.0
|
|||||||
python-dotenv==1.2.2
|
python-dotenv==1.2.2
|
||||||
pytest==9.1.1
|
pytest==9.1.1
|
||||||
pytest-cov==7.1.0
|
pytest-cov==7.1.0
|
||||||
|
pytest-xdist==3.8.0
|
||||||
# Deliberate UI-bring-your-own-tool dependency: textual is the terminal
|
# Deliberate UI-bring-your-own-tool dependency: textual is the terminal
|
||||||
# dashboard framework for the monitoring TUI (tui.py), a SEPARATE entrypoint.
|
# dashboard framework for the monitoring TUI (tui.py), a SEPARATE entrypoint.
|
||||||
# It is only imported by tui.py — never by the service dispatch path, so the
|
# It is only imported by tui.py — never by the service dispatch path, so the
|
||||||
|
|||||||
@@ -653,6 +653,7 @@ def build_router(
|
|||||||
"verdict_mix": metrics.verdict_mix(conn),
|
"verdict_mix": metrics.verdict_mix(conn),
|
||||||
"top_proficiency": metrics.top_proficiency(conn, "coding_general"),
|
"top_proficiency": metrics.top_proficiency(conn, "coding_general"),
|
||||||
"local_energy": metrics.local_energy_summary(conn, cfg),
|
"local_energy": metrics.local_energy_summary(conn, cfg),
|
||||||
|
"pinch": metrics.pinch_summary(conn, cfg),
|
||||||
"health": _health_check(conn, cfg),
|
"health": _health_check(conn, cfg),
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -242,6 +242,7 @@ def prune_context(
|
|||||||
keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default,
|
keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default,
|
||||||
max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default,
|
max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default,
|
||||||
relevance_order: Optional[list[int]] = None,
|
relevance_order: Optional[list[int]] = None,
|
||||||
|
extra_fixed_tokens: int = 0,
|
||||||
) -> tuple[list[dict], dict]:
|
) -> tuple[list[dict], dict]:
|
||||||
"""Trim old tool results once a conversation exceeds ``budget_tokens``.
|
"""Trim old tool results once a conversation exceeds ``budget_tokens``.
|
||||||
|
|
||||||
@@ -260,7 +261,7 @@ def prune_context(
|
|||||||
Only runs (and only mutates anything) when the estimate actually exceeds
|
Only runs (and only mutates anything) when the estimate actually exceeds
|
||||||
the budget; otherwise the original list is returned untouched.
|
the budget; otherwise the original list is returned untouched.
|
||||||
"""
|
"""
|
||||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
|
||||||
if orig_tokens <= budget_tokens:
|
if orig_tokens <= budget_tokens:
|
||||||
return messages, {
|
return messages, {
|
||||||
"pruned": False,
|
"pruned": False,
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ from verification import (
|
|||||||
from metrics import (
|
from metrics import (
|
||||||
local_energy_summary,
|
local_energy_summary,
|
||||||
per_model,
|
per_model,
|
||||||
|
pinch_summary,
|
||||||
quota_burn,
|
quota_burn,
|
||||||
recent_decisions,
|
recent_decisions,
|
||||||
scoring_coverage,
|
scoring_coverage,
|
||||||
@@ -328,7 +329,9 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
|
|||||||
streamed INTEGER,
|
streamed INTEGER,
|
||||||
flex_preference TEXT,
|
flex_preference TEXT,
|
||||||
flex_swapped INTEGER,
|
flex_swapped INTEGER,
|
||||||
flex_forced INTEGER
|
flex_forced INTEGER,
|
||||||
|
pinch_original_tokens INTEGER,
|
||||||
|
pinch_final_tokens INTEGER
|
||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
@@ -346,6 +349,8 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
|
|||||||
("flex_preference", "TEXT"),
|
("flex_preference", "TEXT"),
|
||||||
("flex_swapped", "INTEGER"),
|
("flex_swapped", "INTEGER"),
|
||||||
("flex_forced", "INTEGER"),
|
("flex_forced", "INTEGER"),
|
||||||
|
("pinch_original_tokens", "INTEGER"),
|
||||||
|
("pinch_final_tokens", "INTEGER"),
|
||||||
):
|
):
|
||||||
if name not in existing:
|
if name not in existing:
|
||||||
conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}")
|
conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}")
|
||||||
@@ -989,6 +994,8 @@ def persist_route_decision(
|
|||||||
flex_preference=None,
|
flex_preference=None,
|
||||||
flex_swapped=0,
|
flex_swapped=0,
|
||||||
flex_forced=0,
|
flex_forced=0,
|
||||||
|
pinch_original_tokens=None,
|
||||||
|
pinch_final_tokens=None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Record one routing decision to route_decisions, best-effort and gated.
|
"""Record one routing decision to route_decisions, best-effort and gated.
|
||||||
|
|
||||||
@@ -1078,8 +1085,9 @@ def persist_route_decision(
|
|||||||
selected_model, selected_provider, runner_up_models,
|
selected_model, selected_provider, runner_up_models,
|
||||||
est_cost_usd, est_proficiency, rejected_reason, session_key,
|
est_cost_usd, est_proficiency, rejected_reason, session_key,
|
||||||
tools, images, json_mode, streamed,
|
tools, images, json_mode, streamed,
|
||||||
flex_preference, flex_swapped, flex_forced
|
flex_preference, flex_swapped, flex_forced,
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
pinch_original_tokens, pinch_final_tokens
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
observed_at,
|
observed_at,
|
||||||
@@ -1106,6 +1114,8 @@ def persist_route_decision(
|
|||||||
flex_preference,
|
flex_preference,
|
||||||
int(bool(flex_swapped)),
|
int(bool(flex_swapped)),
|
||||||
int(bool(flex_forced)),
|
int(bool(flex_forced)),
|
||||||
|
pinch_original_tokens,
|
||||||
|
pinch_final_tokens,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
decision_id = int(cursor.lastrowid)
|
decision_id = int(cursor.lastrowid)
|
||||||
@@ -1142,6 +1152,8 @@ def persist_route_decision(
|
|||||||
"flex_preference": flex_preference,
|
"flex_preference": flex_preference,
|
||||||
"flex_swapped": int(bool(flex_swapped)),
|
"flex_swapped": int(bool(flex_swapped)),
|
||||||
"flex_forced": int(bool(flex_forced)),
|
"flex_forced": int(bool(flex_forced)),
|
||||||
|
"pinch_original_tokens": pinch_original_tokens,
|
||||||
|
"pinch_final_tokens": pinch_final_tokens,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as e: # noqa: BLE001 - best-effort must never raise
|
except Exception as e: # noqa: BLE001 - best-effort must never raise
|
||||||
@@ -1462,6 +1474,7 @@ def metrics_endpoint():
|
|||||||
"verdict_mix": verdict_mix(conn),
|
"verdict_mix": verdict_mix(conn),
|
||||||
"top_proficiency": top_proficiency(conn, "coding_general"),
|
"top_proficiency": top_proficiency(conn, "coding_general"),
|
||||||
"local_energy": local_energy_summary(conn, cfg),
|
"local_energy": local_energy_summary(conn, cfg),
|
||||||
|
"pinch": pinch_summary(conn, cfg),
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
}
|
}
|
||||||
finally:
|
finally:
|
||||||
@@ -1933,12 +1946,21 @@ def _embed_for_relevance(
|
|||||||
return order_by_relevance(embeddings[0], embeddings[1:])
|
return order_by_relevance(embeddings[0], embeddings[1:])
|
||||||
|
|
||||||
|
|
||||||
def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optional[list[int]]:
|
def _relevance_order_for(
|
||||||
|
messages: list[dict],
|
||||||
|
cfg,
|
||||||
|
budget_tokens: int,
|
||||||
|
extra_fixed_tokens: int = 0,
|
||||||
|
) -> Optional[list[int]]:
|
||||||
"""Compute the relevance_order for prune_context, or None (uniform).
|
"""Compute the relevance_order for prune_context, or None (uniform).
|
||||||
|
|
||||||
When pinch.relevance is off, or candidate count is below min_candidates,
|
When pinch.relevance is off, or candidate count is below min_candidates,
|
||||||
returns None so prune_context uses today's uniform pass. Otherwise makes
|
returns None so prune_context uses today's uniform pass. Otherwise makes
|
||||||
one batched embedding call and returns the least-relevant-first index list.
|
one batched embedding call and returns the least-relevant-first index list.
|
||||||
|
|
||||||
|
``extra_fixed_tokens`` accounts for overhead that ``prune_context``'s
|
||||||
|
budget guard also adds to ``orig_tokens`` (e.g. tool-definition size),
|
||||||
|
so the skip-budget check in here matches the same total.
|
||||||
"""
|
"""
|
||||||
rel = cfg.pinch.relevance
|
rel = cfg.pinch.relevance
|
||||||
if not (cfg.pinch.enabled and rel.enabled):
|
if not (cfg.pinch.enabled and rel.enabled):
|
||||||
@@ -1947,8 +1969,10 @@ def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optio
|
|||||||
if len(candidates) < rel.min_candidates:
|
if len(candidates) < rel.min_candidates:
|
||||||
return None
|
return None
|
||||||
# Skip the embed when the conversation is under budget — prune_context would
|
# Skip the embed when the conversation is under budget — prune_context would
|
||||||
# be a no-op, so ranking is pointless.
|
# be a no-op, so ranking is pointless. Same total as prune_context includes
|
||||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
# extra_fixed_tokens (e.g. tool definitions) so relevance doesn't silently
|
||||||
|
# decline on tool-heavy requests.
|
||||||
|
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
|
||||||
if orig_tokens <= budget_tokens:
|
if orig_tokens <= budget_tokens:
|
||||||
return None
|
return None
|
||||||
query = _last_user_text(messages)
|
query = _last_user_text(messages)
|
||||||
@@ -2355,6 +2379,11 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
session_key = session_fingerprint(messages)
|
session_key = session_fingerprint(messages)
|
||||||
streamed = bool(body.get("stream"))
|
streamed = bool(body.get("stream"))
|
||||||
|
|
||||||
|
# Pinch stats are filled by the routed and passthrough prune paths (when
|
||||||
|
# pinch is enabled) and read by all persist sites. A single declaration
|
||||||
|
# keeps the variable in scope regardless of branch.
|
||||||
|
pinch_stats = None
|
||||||
|
|
||||||
# What this request needs, read from the body once and used by both the
|
# What this request needs, read from the body once and used by both the
|
||||||
# routed and pass-through branches. Stated by the caller, not guessed at:
|
# routed and pass-through branches. Stated by the caller, not guessed at:
|
||||||
# a tools array, an image_url part and a response_format each say exactly
|
# a tools array, an image_url part and a response_format each say exactly
|
||||||
@@ -2404,20 +2433,26 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
# messages, so the classification turn above (which used
|
# messages, so the classification turn above (which used
|
||||||
# `_previous_context` from the full messages) is undisturbed.
|
# `_previous_context` from the full messages) is undisturbed.
|
||||||
if cfg.pinch.enabled:
|
if cfg.pinch.enabled:
|
||||||
|
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
|
||||||
send_messages, pinch_stats = prune_context(
|
send_messages, pinch_stats = prune_context(
|
||||||
list(messages),
|
list(messages),
|
||||||
budget_tokens=cfg.pinch.budget_tokens,
|
budget_tokens=cfg.pinch.budget_tokens,
|
||||||
keep_last_turns=cfg.pinch.keep_last_turns,
|
keep_last_turns=cfg.pinch.keep_last_turns,
|
||||||
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
||||||
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
|
relevance_order=_relevance_order_for(
|
||||||
)
|
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
|
||||||
logs.debug(
|
extra_fixed_tokens=_tools_overhead,
|
||||||
"pinch",
|
),
|
||||||
pruned=pinch_stats["pruned"],
|
extra_fixed_tokens=_tools_overhead,
|
||||||
saved=pinch_stats["tokens_saved"],
|
|
||||||
orig=pinch_stats["original_tokens"],
|
|
||||||
final=pinch_stats["final_tokens"],
|
|
||||||
)
|
)
|
||||||
|
if pinch_stats.get("pruned"):
|
||||||
|
logs.info(
|
||||||
|
"pinch",
|
||||||
|
pruned=pinch_stats["pruned"],
|
||||||
|
saved=pinch_stats["tokens_saved"],
|
||||||
|
orig=pinch_stats["original_tokens"],
|
||||||
|
final=pinch_stats["final_tokens"],
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
send_messages = messages
|
send_messages = messages
|
||||||
measured = estimate_prompt_tokens(send_messages, tools=body.get("tools"))
|
measured = estimate_prompt_tokens(send_messages, tools=body.get("tools"))
|
||||||
@@ -2559,6 +2594,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
images=int(caps.has_images),
|
images=int(caps.has_images),
|
||||||
json_mode=int(caps.require_json_mode),
|
json_mode=int(caps.require_json_mode),
|
||||||
streamed=streamed,
|
streamed=streamed,
|
||||||
|
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
|
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
)
|
)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
422,
|
422,
|
||||||
@@ -2583,6 +2622,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
streamed=streamed,
|
streamed=streamed,
|
||||||
classification_source=classified_src,
|
classification_source=classified_src,
|
||||||
classifier_ms=classifier_ms,
|
classifier_ms=classifier_ms,
|
||||||
|
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
|
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
)
|
)
|
||||||
target = decision.selected.model_id
|
target = decision.selected.model_id
|
||||||
provider = decision.selected.provider
|
provider = decision.selected.provider
|
||||||
@@ -2592,6 +2635,31 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
# Not a routing decision at all, and worth saying so plainly: a client
|
# Not a routing decision at all, and worth saying so plainly: a client
|
||||||
# pinned to one model gets none of the filtering or ranking below.
|
# pinned to one model gets none of the filtering or ranking below.
|
||||||
logs.info("passthrough", model=target, stream=bool(body.get("stream")))
|
logs.info("passthrough", model=target, stream=bool(body.get("stream")))
|
||||||
|
send_messages = messages
|
||||||
|
if cfg.pinch.enabled:
|
||||||
|
# Same pruning as the routed path, but it must run before persistence
|
||||||
|
# so a pruned passthrough request records real pinch tokens rather
|
||||||
|
# than NULL.
|
||||||
|
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
|
||||||
|
send_messages, pinch_stats = prune_context(
|
||||||
|
list(messages),
|
||||||
|
budget_tokens=cfg.pinch.budget_tokens,
|
||||||
|
keep_last_turns=cfg.pinch.keep_last_turns,
|
||||||
|
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
||||||
|
relevance_order=_relevance_order_for(
|
||||||
|
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
|
||||||
|
extra_fixed_tokens=_tools_overhead,
|
||||||
|
),
|
||||||
|
extra_fixed_tokens=_tools_overhead,
|
||||||
|
)
|
||||||
|
if pinch_stats.get("pruned"):
|
||||||
|
logs.info(
|
||||||
|
"pinch",
|
||||||
|
pruned=pinch_stats["pruned"],
|
||||||
|
saved=pinch_stats["tokens_saved"],
|
||||||
|
orig=pinch_stats["original_tokens"],
|
||||||
|
final=pinch_stats["final_tokens"],
|
||||||
|
)
|
||||||
persist_route_decision(
|
persist_route_decision(
|
||||||
"passthrough",
|
"passthrough",
|
||||||
selected_model=target,
|
selected_model=target,
|
||||||
@@ -2605,6 +2673,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
flex_preference=cfg.routing.default_flex_preference.value,
|
flex_preference=cfg.routing.default_flex_preference.value,
|
||||||
flex_swapped=0,
|
flex_swapped=0,
|
||||||
flex_forced=0,
|
flex_forced=0,
|
||||||
|
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
|
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||||
|
if pinch_stats is not None else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
# A pinned model id is dispatched as asked, but a pin that cannot
|
# A pinned model id is dispatched as asked, but a pin that cannot
|
||||||
@@ -2625,31 +2697,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
|||||||
# top (needed by the passthrough/local-vision branches), only the working
|
# top (needed by the passthrough/local-vision branches), only the working
|
||||||
# directory is derived here because only the observation path uses it.
|
# directory is derived here because only the observation path uses it.
|
||||||
session_dir = session_directory(messages)
|
session_dir = session_directory(messages)
|
||||||
if not wants_routing:
|
# send_messages is already set: by the routed path's prune block above, or
|
||||||
# The routed path pruned send_messages once, before its measured-context
|
# by the passthrough path's prune block. Both branches default to messages
|
||||||
# decision, and the pruned list is reused here. A passthrough request
|
# when pinch is disabled.
|
||||||
# never went through that path, so it prunes now — same gating as
|
|
||||||
# before (only when pinch is enabled).
|
|
||||||
send_messages = messages
|
|
||||||
if cfg.pinch.enabled:
|
|
||||||
# Optional relevance-based context pruning: when the conversation
|
|
||||||
# exceeds budget_tokens, trim old tool results BEFORE any paid token
|
|
||||||
# is sent upstream. User/assistant/system messages are always kept;
|
|
||||||
# only the classifier input is exempt (it already clamps to head+tail).
|
|
||||||
send_messages, pinch_stats = prune_context(
|
|
||||||
list(messages),
|
|
||||||
budget_tokens=cfg.pinch.budget_tokens,
|
|
||||||
keep_last_turns=cfg.pinch.keep_last_turns,
|
|
||||||
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
|
||||||
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
|
|
||||||
)
|
|
||||||
logs.debug(
|
|
||||||
"pinch",
|
|
||||||
pruned=pinch_stats["pruned"],
|
|
||||||
saved=pinch_stats["tokens_saved"],
|
|
||||||
orig=pinch_stats["original_tokens"],
|
|
||||||
final=pinch_stats["final_tokens"],
|
|
||||||
)
|
|
||||||
upstream_body = {**body, "model": target, "messages": list(send_messages)}
|
upstream_body = {**body, "model": target, "messages": list(send_messages)}
|
||||||
streaming = bool(body.get("stream"))
|
streaming = bool(body.get("stream"))
|
||||||
if streaming:
|
if streaming:
|
||||||
|
|||||||
102
src/metrics.py
102
src/metrics.py
@@ -16,6 +16,7 @@ recent_decisions — last N rows from the route_decisions observability table
|
|||||||
per_model — per-model aggregates over energy_observations (last 30 d)
|
per_model — per-model aggregates over energy_observations (last 30 d)
|
||||||
verdict_mix — counts by verdict from verifications (last N days)
|
verdict_mix — counts by verdict from verifications (last N days)
|
||||||
top_proficiency — top models by blended_score for a category
|
top_proficiency — top models by blended_score for a category
|
||||||
|
pinch_summary — context-pruning savings over the last 30 d
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -422,7 +423,8 @@ def recent_decisions(
|
|||||||
candidates_considered, selected_model, selected_provider,
|
candidates_considered, selected_model, selected_provider,
|
||||||
runner_up_models, est_cost_usd, est_proficiency,
|
runner_up_models, est_cost_usd, est_proficiency,
|
||||||
rejected_reason, session_key, tools, images, json_mode, streamed,
|
rejected_reason, session_key, tools, images, json_mode, streamed,
|
||||||
flex_preference, flex_swapped, flex_forced
|
flex_preference, flex_swapped, flex_forced,
|
||||||
|
pinch_original_tokens, pinch_final_tokens
|
||||||
FROM route_decisions
|
FROM route_decisions
|
||||||
ORDER BY id DESC
|
ORDER BY id DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
@@ -518,7 +520,7 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]:
|
|||||||
COALESCE(SUM(cost_usd), 0) cost
|
COALESCE(SUM(cost_usd), 0) cost
|
||||||
FROM local_energy_observations
|
FROM local_energy_observations
|
||||||
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
||||||
GROUP BY call_type
|
GROUP BY call_type
|
||||||
"""
|
"""
|
||||||
).fetchall()
|
).fetchall()
|
||||||
by_type = {
|
by_type = {
|
||||||
@@ -540,3 +542,99 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]:
|
|||||||
if reset_day is not None:
|
if reset_day is not None:
|
||||||
result["next_reset_date"] = _next_reset_date(reset_day)
|
result["next_reset_date"] = _next_reset_date(reset_day)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def pinch_summary(conn: sqlite3.Connection, cfg: Any) -> Optional[dict]:
|
||||||
|
"""Aggregate context-pruning savings over the last 30 days.
|
||||||
|
|
||||||
|
Returns None when pinch is disabled so callers can omit the section.
|
||||||
|
Computes the number of pruned calls, share of pruned calls, total and
|
||||||
|
median tokens saved, and an estimated dollar saving priced at the blended
|
||||||
|
prompt rate the router uses for cost estimation.
|
||||||
|
"""
|
||||||
|
if not cfg.pinch.enabled:
|
||||||
|
return None
|
||||||
|
|
||||||
|
cache_rate = cfg.objective.assumed_cache_rate
|
||||||
|
|
||||||
|
totals = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) AS calls_30d,
|
||||||
|
SUM(CASE
|
||||||
|
WHEN pinch_original_tokens IS NOT NULL
|
||||||
|
AND pinch_final_tokens IS NOT NULL
|
||||||
|
AND pinch_original_tokens > pinch_final_tokens
|
||||||
|
THEN 1 ELSE 0
|
||||||
|
END) AS pruned_calls_30d,
|
||||||
|
SUM(CASE
|
||||||
|
WHEN pinch_original_tokens IS NOT NULL
|
||||||
|
AND pinch_final_tokens IS NOT NULL
|
||||||
|
THEN pinch_original_tokens - pinch_final_tokens
|
||||||
|
ELSE 0
|
||||||
|
END) AS total_tokens_saved
|
||||||
|
FROM route_decisions
|
||||||
|
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
calls_30d = totals["calls_30d"] or 0
|
||||||
|
pruned_calls_30d = totals["pruned_calls_30d"] or 0
|
||||||
|
share_pruned = pruned_calls_30d / calls_30d if calls_30d else 0.0
|
||||||
|
total_tokens_saved = totals["total_tokens_saved"] or 0
|
||||||
|
|
||||||
|
saved_rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT pinch_original_tokens - pinch_final_tokens AS tokens_saved
|
||||||
|
FROM route_decisions
|
||||||
|
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
||||||
|
AND pinch_original_tokens IS NOT NULL
|
||||||
|
AND pinch_final_tokens IS NOT NULL
|
||||||
|
AND pinch_original_tokens > pinch_final_tokens
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
median_tokens_saved = _percentile(
|
||||||
|
[r["tokens_saved"] for r in saved_rows], 50
|
||||||
|
)
|
||||||
|
|
||||||
|
dollar_rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT rd.pinch_original_tokens - rd.pinch_final_tokens AS tokens_saved,
|
||||||
|
m.cost_per_1m_prompt,
|
||||||
|
m.cost_per_1m_prompt_cached
|
||||||
|
FROM route_decisions rd
|
||||||
|
JOIN models m
|
||||||
|
ON rd.selected_model = m.model_id
|
||||||
|
AND rd.selected_provider = m.provider
|
||||||
|
WHERE julianday(rd.observed_at) > julianday('now', '-30 days')
|
||||||
|
AND rd.pinch_original_tokens IS NOT NULL
|
||||||
|
AND rd.pinch_final_tokens IS NOT NULL
|
||||||
|
AND rd.pinch_original_tokens > rd.pinch_final_tokens
|
||||||
|
AND rd.selected_model IS NOT NULL
|
||||||
|
AND m.cost_per_1m_prompt IS NOT NULL
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
dollars_saved_usd_30d = 0.0
|
||||||
|
for r in dollar_rows:
|
||||||
|
saved = r["tokens_saved"]
|
||||||
|
prompt_price = r["cost_per_1m_prompt"]
|
||||||
|
cached_price = r["cost_per_1m_prompt_cached"]
|
||||||
|
if cached_price is None:
|
||||||
|
cached_price = prompt_price
|
||||||
|
blended = (1.0 - cache_rate) * prompt_price + cache_rate * cached_price
|
||||||
|
dollars_saved_usd_30d += saved * blended / 1_000_000
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"calls_30d": calls_30d,
|
||||||
|
"pruned_calls_30d": pruned_calls_30d,
|
||||||
|
"share_pruned": round(share_pruned, 4),
|
||||||
|
"total_tokens_saved": total_tokens_saved,
|
||||||
|
"median_tokens_saved": median_tokens_saved,
|
||||||
|
"dollars_saved_usd_30d": round(dollars_saved_usd_30d, 6),
|
||||||
|
}
|
||||||
|
reset_day = getattr(cfg.objective, "billing_reset_day", None)
|
||||||
|
if reset_day is not None:
|
||||||
|
result["reset_date"] = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat()
|
||||||
|
result["next_reset_date"] = _next_reset_date(reset_day)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -254,6 +254,13 @@ class DashboardApp(App):
|
|||||||
def _on_interval(self) -> None:
|
def _on_interval(self) -> None:
|
||||||
if getattr(self, "_unmounted", False):
|
if getattr(self, "_unmounted", False):
|
||||||
return
|
return
|
||||||
|
# Skip rendering while the dashboard screen is still composing
|
||||||
|
# (the DataTable isn't in the DOM yet). The next interval tick will
|
||||||
|
# catch it once the widget is mounted.
|
||||||
|
try:
|
||||||
|
self.query_one("#model-table", DataTable)
|
||||||
|
except Exception: # noqa: BLE001 — widget not yet available
|
||||||
|
return
|
||||||
self._refresh()
|
self._refresh()
|
||||||
|
|
||||||
def action_refresh(self) -> None:
|
def action_refresh(self) -> None:
|
||||||
|
|||||||
@@ -59,6 +59,17 @@ def build_model(data: dict) -> dict:
|
|||||||
else:
|
else:
|
||||||
quota_rows = []
|
quota_rows = []
|
||||||
|
|
||||||
|
pinch = data.get("pinch")
|
||||||
|
if pinch:
|
||||||
|
pinch_rows = [
|
||||||
|
{"label": "share_pruned", "value": pinch.get("share_pruned")},
|
||||||
|
{"label": "total_tokens_saved", "value": pinch.get("total_tokens_saved")},
|
||||||
|
{"label": "median_tokens_saved", "value": pinch.get("median_tokens_saved")},
|
||||||
|
{"label": "dollars_saved_usd_30d", "value": pinch.get("dollars_saved_usd_30d")},
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
pinch_rows = []
|
||||||
|
|
||||||
per_model = [
|
per_model = [
|
||||||
{
|
{
|
||||||
"model": r.get("model_id"),
|
"model": r.get("model_id"),
|
||||||
@@ -85,6 +96,7 @@ def build_model(data: dict) -> dict:
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
"quota": quota_rows,
|
"quota": quota_rows,
|
||||||
|
"pinch": pinch_rows,
|
||||||
"per_model": per_model,
|
"per_model": per_model,
|
||||||
"verdict_mix": verdict_mix,
|
"verdict_mix": verdict_mix,
|
||||||
"recent_decisions": recent,
|
"recent_decisions": recent,
|
||||||
|
|||||||
@@ -155,7 +155,9 @@ def test_admin_snapshot_has_all_top_level_keys(seeded_client):
|
|||||||
"per_model",
|
"per_model",
|
||||||
"verdict_mix",
|
"verdict_mix",
|
||||||
"top_proficiency",
|
"top_proficiency",
|
||||||
|
"local_energy",
|
||||||
"health",
|
"health",
|
||||||
|
"pinch",
|
||||||
"generated_at",
|
"generated_at",
|
||||||
):
|
):
|
||||||
assert key in data, f"missing top-level key {key!r}"
|
assert key in data, f"missing top-level key {key!r}"
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ EXHAUSTIVE_KEYS = (
|
|||||||
"top_proficiency",
|
"top_proficiency",
|
||||||
"local_energy",
|
"local_energy",
|
||||||
"health",
|
"health",
|
||||||
|
"pinch",
|
||||||
"generated_at",
|
"generated_at",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -536,6 +536,89 @@ def test_relevance_order_overrides_recency():
|
|||||||
assert "chars trimmed" in out[3]["content"]
|
assert "chars trimmed" in out[3]["content"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_extra_fixed_tokens_tips_budget_to_prune():
|
||||||
|
# A conversation whose message tokens sit just under the budget, but tool
|
||||||
|
# overhead pushes the accounting total over. Without overhead it is a no-op;
|
||||||
|
# with overhead it prunes.
|
||||||
|
messages = [
|
||||||
|
_user("question"),
|
||||||
|
_tool("read", "x" * 540), # 540 chars -> 180 tokens
|
||||||
|
_assistant("answer"),
|
||||||
|
_user("follow up"),
|
||||||
|
]
|
||||||
|
msg_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
||||||
|
assert msg_tokens == 187
|
||||||
|
|
||||||
|
out, stats = prune_context(
|
||||||
|
messages,
|
||||||
|
budget_tokens=200,
|
||||||
|
keep_last_turns=0,
|
||||||
|
extra_fixed_tokens=0,
|
||||||
|
)
|
||||||
|
assert stats["pruned"] is False
|
||||||
|
assert stats["original_tokens"] == msg_tokens
|
||||||
|
|
||||||
|
out, stats = prune_context(
|
||||||
|
messages,
|
||||||
|
budget_tokens=200,
|
||||||
|
keep_last_turns=0,
|
||||||
|
extra_fixed_tokens=100,
|
||||||
|
)
|
||||||
|
assert stats["pruned"] is True
|
||||||
|
assert stats["original_tokens"] == msg_tokens + 100
|
||||||
|
# The tool result was trimmed to cover the deficit (187 + 100 - 200 = 87).
|
||||||
|
assert stats["tokens_saved"] >= 87
|
||||||
|
tool_out = [m for m in out if m.get("role") == "tool"][0]
|
||||||
|
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_extra_fixed_tokens_zero_preserves_behavior():
|
||||||
|
# The default and explicit zero must be byte-for-byte identical.
|
||||||
|
messages = [
|
||||||
|
_user("question"),
|
||||||
|
_tool("read", "big result " * 500),
|
||||||
|
_assistant("answer"),
|
||||||
|
_user("follow up"),
|
||||||
|
_assistant("ok"),
|
||||||
|
]
|
||||||
|
default_out, default_stats = prune_context(
|
||||||
|
messages, budget_tokens=100, keep_last_turns=0
|
||||||
|
)
|
||||||
|
explicit_zero_out, explicit_zero_stats = prune_context(
|
||||||
|
messages,
|
||||||
|
budget_tokens=100,
|
||||||
|
keep_last_turns=0,
|
||||||
|
extra_fixed_tokens=0,
|
||||||
|
)
|
||||||
|
assert explicit_zero_out == default_out
|
||||||
|
assert explicit_zero_stats == default_stats
|
||||||
|
|
||||||
|
|
||||||
|
def test_tool_content_never_summarized():
|
||||||
|
# Even when extra_fixed_tokens forces pruning, user/assistant/system
|
||||||
|
# messages survive verbatim; only tool results are candidates.
|
||||||
|
messages = [
|
||||||
|
_user("what is the answer"),
|
||||||
|
_assistant("i'll check"),
|
||||||
|
_tool("search", "big tool result " * 500),
|
||||||
|
_assistant("the answer is 42"),
|
||||||
|
_user("thanks"),
|
||||||
|
]
|
||||||
|
out, stats = prune_context(
|
||||||
|
messages,
|
||||||
|
budget_tokens=100,
|
||||||
|
keep_last_turns=0,
|
||||||
|
extra_fixed_tokens=1000,
|
||||||
|
)
|
||||||
|
assert stats["pruned"] is True
|
||||||
|
assert len(out) == len(messages)
|
||||||
|
for orig, pruned in zip(messages, out):
|
||||||
|
if orig.get("role") in ("user", "assistant", "system"):
|
||||||
|
assert pruned["content"] == orig["content"]
|
||||||
|
tool_out = [m for m in out if m.get("role") == "tool"][0]
|
||||||
|
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
|
||||||
|
|
||||||
|
|
||||||
def test_relevance_order_none_matches_default():
|
def test_relevance_order_none_matches_default():
|
||||||
# relevance_order=None must be byte-for-byte identical to not passing the
|
# relevance_order=None must be byte-for-byte identical to not passing the
|
||||||
# parameter at all (the historical uniform pass / fallback path).
|
# parameter at all (the historical uniform pass / fallback path).
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ point of testing it.
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import eval_proficiency
|
||||||
|
|
||||||
from eval_proficiency import (
|
from eval_proficiency import (
|
||||||
judge_for,
|
judge_for,
|
||||||
parse_check_results,
|
parse_check_results,
|
||||||
@@ -49,7 +51,8 @@ def test_code_that_does_not_parse_scores_zero_and_says_so():
|
|||||||
assert "did not run" in detail
|
assert "did not run" in detail
|
||||||
|
|
||||||
|
|
||||||
def test_infinite_loop_is_bounded_not_hung():
|
def test_infinite_loop_is_bounded_not_hung(monkeypatch):
|
||||||
|
monkeypatch.setattr(eval_proficiency, "CODE_TIMEOUT_SECONDS", 1)
|
||||||
score, detail = score_code("def f(x):\n while True: pass", ["f(1)==1"])
|
score, detail = score_code("def f(x):\n while True: pass", ["f(1)==1"])
|
||||||
assert (score, detail) == (0.0, "timeout")
|
assert (score, detail) == (0.0, "timeout")
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from metrics import (
|
|||||||
_next_reset_date,
|
_next_reset_date,
|
||||||
context_ceilings,
|
context_ceilings,
|
||||||
per_model,
|
per_model,
|
||||||
|
pinch_summary,
|
||||||
quota_burn,
|
quota_burn,
|
||||||
recent_decisions,
|
recent_decisions,
|
||||||
scoring_coverage,
|
scoring_coverage,
|
||||||
@@ -626,6 +627,119 @@ def test_top_proficiency_empty_for_missing_category(tmp_path):
|
|||||||
# --- /health endpoint compatibility -------------------------------------------
|
# --- /health endpoint compatibility -------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_pinch_models(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Insert two cheap/dear models plus a cheap-no-cached-price variant."""
|
||||||
|
fresh = _now().isoformat()
|
||||||
|
rows = [
|
||||||
|
# (model_id, tier, context, cost_per_1m_prompt, cost_per_1m_prompt_cached)
|
||||||
|
("cheap", 2, 262128, 1.0, 0.5),
|
||||||
|
("dear", 2, 262128, 5.0, None),
|
||||||
|
("tiny", 1, 131072, 0.1, 0.1),
|
||||||
|
]
|
||||||
|
for model_id, tier, context, cost, cached in rows:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO models (
|
||||||
|
model_id, provider, base_model_id, tier, context_window,
|
||||||
|
effective_context_window, max_output_tokens,
|
||||||
|
cost_per_1m_prompt, cost_per_1m_completion,
|
||||||
|
cost_per_1m_prompt_cached,
|
||||||
|
supports_vision, supports_json_mode,
|
||||||
|
latency_class, reasoning_mode, context_variant,
|
||||||
|
access_level, availability, last_updated
|
||||||
|
) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?, ?,
|
||||||
|
1, 1, 'standard', 'default', 'full', 'public', 'active',
|
||||||
|
?)
|
||||||
|
""",
|
||||||
|
(model_id, model_id, tier, context, cost, cost / 3, cached, fresh),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_pinch_decision(conn, *, model, provider, orig, final):
|
||||||
|
"""Insert a route_decisions row with observed_at = now and given pinch values."""
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO route_decisions (
|
||||||
|
observed_at, kind, task_category, task_tier,
|
||||||
|
required_context_tokens, confidence, classifier_ms,
|
||||||
|
classification_source, latency_tolerance, candidates_considered,
|
||||||
|
selected_model, selected_provider, runner_up_models,
|
||||||
|
est_cost_usd, est_proficiency, rejected_reason, session_key,
|
||||||
|
tools, images, json_mode, streamed,
|
||||||
|
pinch_original_tokens, pinch_final_tokens
|
||||||
|
) VALUES (?, 'chat', 'coding_general', 2, 100, 0.9, 100,
|
||||||
|
'classifier', 'interactive', 3, ?, ?, NULL,
|
||||||
|
0.001, 0.9, NULL, 'sess', 0, 0, 0, 0, ?, ?)
|
||||||
|
""",
|
||||||
|
(_now().isoformat(), model, provider, orig, final),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pinch_summary_aggregates_correctly(tmp_path, monkeypatch):
|
||||||
|
"""Hand-computed aggregates over a seeded 30-day window."""
|
||||||
|
cfg = load_config(str(ROOT / "config" / "config.yaml"))
|
||||||
|
monkeypatch.setattr(cfg.pinch, "enabled", True)
|
||||||
|
cache_rate = cfg.objective.assumed_cache_rate
|
||||||
|
|
||||||
|
conn = _make_db(tmp_path)
|
||||||
|
_seed_pinch_models(conn)
|
||||||
|
|
||||||
|
# Two pruned cheap rows: saved tokens 50 and 30.
|
||||||
|
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=100, final=50)
|
||||||
|
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=80, final=50)
|
||||||
|
# One non-pruned cheap row (final == orig).
|
||||||
|
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=60, final=60)
|
||||||
|
# One pruned dear row with NULL cost_per_1m_prompt_cached -> falls back to prompt price.
|
||||||
|
_insert_pinch_decision(conn, model="dear", provider="neuralwatt", orig=200, final=100)
|
||||||
|
# One rejection row with NULL selected_model -> counted in share/tokens but excluded from dollars.
|
||||||
|
_insert_pinch_decision(conn, model=None, provider=None, orig=1000, final=500)
|
||||||
|
|
||||||
|
result = pinch_summary(conn, cfg)
|
||||||
|
assert result is not None
|
||||||
|
assert result["calls_30d"] == 5
|
||||||
|
assert result["pruned_calls_30d"] == 4
|
||||||
|
assert result["share_pruned"] == round(4 / 5, 4)
|
||||||
|
# saved tokens: 50 + 30 + 100 + 500 = 680
|
||||||
|
assert result["total_tokens_saved"] == 680
|
||||||
|
# median of [30, 50, 100, 500] = 75
|
||||||
|
assert result["median_tokens_saved"] == 75
|
||||||
|
|
||||||
|
# dollar math: cheap saved 80 tokens at blended rate over cached prompt.
|
||||||
|
cheap_blended = (1 - cache_rate) * 1.0 + cache_rate * 0.5
|
||||||
|
cheap_dollars = 80 * cheap_blended / 1_000_000
|
||||||
|
# dear saved 100 tokens; cached price NULL -> fallback to prompt price 5.0.
|
||||||
|
dear_dollars = 100 * 5.0 / 1_000_000
|
||||||
|
# rejection row with NULL selected_model is excluded from dollar math.
|
||||||
|
assert result["dollars_saved_usd_30d"] == round(cheap_dollars + dear_dollars, 6)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pinch_summary_zero_rows(tmp_path, monkeypatch):
|
||||||
|
"""Enabled with no decisions in the window returns zeros and None median."""
|
||||||
|
cfg = load_config(str(ROOT / "config" / "config.yaml"))
|
||||||
|
monkeypatch.setattr(cfg.pinch, "enabled", True)
|
||||||
|
conn = _make_db(tmp_path)
|
||||||
|
result = pinch_summary(conn, cfg)
|
||||||
|
assert result is not None
|
||||||
|
assert result["calls_30d"] == 0
|
||||||
|
assert result["pruned_calls_30d"] == 0
|
||||||
|
assert result["share_pruned"] == 0.0
|
||||||
|
assert result["total_tokens_saved"] == 0
|
||||||
|
assert result["median_tokens_saved"] is None
|
||||||
|
assert result["dollars_saved_usd_30d"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pinch_summary_disabled_returns_none(tmp_path, monkeypatch):
|
||||||
|
"""When cfg.pinch.enabled is False, None is returned."""
|
||||||
|
conn = _make_db(tmp_path)
|
||||||
|
cfg = SimpleNamespace(pinch=SimpleNamespace(enabled=False))
|
||||||
|
assert pinch_summary(conn, cfg) is None
|
||||||
|
|
||||||
|
|
||||||
|
# --- /health endpoint compatibility -------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch):
|
def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch):
|
||||||
"""/health still returns the same SHAPE after moving functions to metrics."""
|
"""/health still returns the same SHAPE after moving functions to metrics."""
|
||||||
db_path = tmp_path / "test.db"
|
db_path = tmp_path / "test.db"
|
||||||
|
|||||||
@@ -161,6 +161,7 @@ def test_metrics_endpoint_has_all_top_level_keys(seeded_client):
|
|||||||
"verdict_mix",
|
"verdict_mix",
|
||||||
"top_proficiency",
|
"top_proficiency",
|
||||||
"local_energy",
|
"local_energy",
|
||||||
|
"pinch",
|
||||||
"generated_at",
|
"generated_at",
|
||||||
):
|
):
|
||||||
assert key in data, f"missing top-level key {key!r}"
|
assert key in data, f"missing top-level key {key!r}"
|
||||||
|
|||||||
@@ -58,6 +58,8 @@ ROUTE_DECISIONS_COLUMNS = [
|
|||||||
"flex_preference",
|
"flex_preference",
|
||||||
"flex_swapped",
|
"flex_swapped",
|
||||||
"flex_forced",
|
"flex_forced",
|
||||||
|
"pinch_original_tokens",
|
||||||
|
"pinch_final_tokens",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -205,6 +207,54 @@ def test_ensure_route_decisions_allows_insert(tmp_path):
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_persist_writes_pinch_columns(tmp_path, monkeypatch):
|
||||||
|
"""persist_route_decision writes pinch_original_tokens and pinch_final_tokens
|
||||||
|
when provided, and defaults them to NULL when omitted."""
|
||||||
|
db_path = tmp_path / "pinch.db"
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.executescript(_schema_minus_route_decisions())
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.logging, "log_route_decisions", True)
|
||||||
|
|
||||||
|
dispatcher.persist_route_decision(
|
||||||
|
"route",
|
||||||
|
classification=Classification(
|
||||||
|
task_category="coding_general", task_tier=2,
|
||||||
|
required_context_tokens=100, confidence=0.9,
|
||||||
|
),
|
||||||
|
latency_tolerance="interactive",
|
||||||
|
selected_model="deepseek-v4-flash",
|
||||||
|
selected_provider="neuralwatt",
|
||||||
|
pinch_original_tokens=100,
|
||||||
|
pinch_final_tokens=50,
|
||||||
|
)
|
||||||
|
|
||||||
|
dispatcher.persist_route_decision(
|
||||||
|
"route",
|
||||||
|
classification=Classification(
|
||||||
|
task_category="coding_general", task_tier=2,
|
||||||
|
required_context_tokens=100, confidence=0.9,
|
||||||
|
),
|
||||||
|
latency_tolerance="interactive",
|
||||||
|
selected_model="deepseek-v4-flash",
|
||||||
|
selected_provider="neuralwatt",
|
||||||
|
)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT pinch_original_tokens, pinch_final_tokens FROM route_decisions ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
assert len(rows) == 2
|
||||||
|
assert rows[0]["pinch_original_tokens"] == 100
|
||||||
|
assert rows[0]["pinch_final_tokens"] == 50
|
||||||
|
assert rows[1]["pinch_original_tokens"] is None
|
||||||
|
assert rows[1]["pinch_final_tokens"] is None
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
def test_persist_ensure_on_write_fixes_live_db_missing_table(tmp_path, monkeypatch):
|
def test_persist_ensure_on_write_fixes_live_db_missing_table(tmp_path, monkeypatch):
|
||||||
"""A live router.db without route_decisions gets it on the WRITE path.
|
"""A live router.db without route_decisions gets it on the WRITE path.
|
||||||
|
|
||||||
@@ -600,6 +650,53 @@ def test_passthrough_persists_one_row_with_no_nameerror(decision_router):
|
|||||||
assert r["session_key"] is not None
|
assert r["session_key"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_passthrough_records_pinch_columns_when_pruned(decision_router, monkeypatch):
|
||||||
|
"""Regression: passthrough pruning must run before persistence so pinch stats
|
||||||
|
are recorded rather than NULL."""
|
||||||
|
client, db_path = decision_router
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.pinch, "enabled", True)
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.pinch, "budget_tokens", 1)
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.pinch, "keep_last_turns", 1)
|
||||||
|
|
||||||
|
# Long conversation plus tools to exceed the tiny budget.
|
||||||
|
long_text = "word " * 500
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": "You are a helpful assistant."},
|
||||||
|
{"role": "user", "content": long_text},
|
||||||
|
{"role": "assistant", "content": "acknowledged"},
|
||||||
|
{"role": "user", "content": "now summarize"},
|
||||||
|
]
|
||||||
|
resp = client.post(
|
||||||
|
"/v1/chat/completions",
|
||||||
|
json={
|
||||||
|
"model": DEAR,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": False,
|
||||||
|
"tools": [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "noop",
|
||||||
|
"description": "A tool with a long description to inflate overhead",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
# Routing succeeds because the fixture stubs the provider call; persistence
|
||||||
|
# happens before forwarding, so the row is already written.
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
rows = _rows(db_path)
|
||||||
|
passthrough_rows = [r for r in rows if r["kind"] == "passthrough"]
|
||||||
|
assert len(passthrough_rows) == 1
|
||||||
|
r = passthrough_rows[0]
|
||||||
|
assert r["pinch_original_tokens"] is not None
|
||||||
|
assert r["pinch_final_tokens"] is not None
|
||||||
|
assert r["pinch_original_tokens"] > r["pinch_final_tokens"]
|
||||||
|
|
||||||
|
|
||||||
def test_local_vision_success_persists_one_local_row(decision_router, monkeypatch):
|
def test_local_vision_success_persists_one_local_row(decision_router, monkeypatch):
|
||||||
client, db_path = decision_router
|
client, db_path = decision_router
|
||||||
_drop_cheap(db_path)
|
_drop_cheap(db_path)
|
||||||
|
|||||||
@@ -114,6 +114,14 @@ def _fixture() -> dict:
|
|||||||
"blended_score": 1.0, "source": "self_eval_thin",
|
"blended_score": 1.0, "source": "self_eval_thin",
|
||||||
"self_eval_samples": 3}
|
"self_eval_samples": 3}
|
||||||
],
|
],
|
||||||
|
"pinch": {
|
||||||
|
"calls_30d": 120,
|
||||||
|
"pruned_calls_30d": 30,
|
||||||
|
"share_pruned": 0.25,
|
||||||
|
"total_tokens_saved": 50000,
|
||||||
|
"median_tokens_saved": 1200,
|
||||||
|
"dollars_saved_usd_30d": 0.0125,
|
||||||
|
},
|
||||||
"generated_at": "2026-08-23T10:01:00+00:00",
|
"generated_at": "2026-08-23T10:01:00+00:00",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,6 +142,23 @@ def test_build_model_quota_panel():
|
|||||||
assert "calls=18" in joined
|
assert "calls=18" in joined
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_model_pinch_panel():
|
||||||
|
m = build_model(_fixture())
|
||||||
|
rows = m["pinch"]
|
||||||
|
by_label = {r["label"]: r["value"] for r in rows}
|
||||||
|
assert by_label["share_pruned"] == 0.25
|
||||||
|
assert by_label["total_tokens_saved"] == 50000
|
||||||
|
assert by_label["median_tokens_saved"] == 1200
|
||||||
|
assert by_label["dollars_saved_usd_30d"] == 0.0125
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_model_pinch_panel_empty_when_none():
|
||||||
|
data = _fixture()
|
||||||
|
data["pinch"] = None
|
||||||
|
m = build_model(data)
|
||||||
|
assert m["pinch"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_build_model_quota_panel_includes_reset_date():
|
def test_build_model_quota_panel_includes_reset_date():
|
||||||
m = build_model(_fixture())
|
m = build_model(_fixture())
|
||||||
rows = m["quota"]
|
rows = m["quota"]
|
||||||
|
|||||||
Reference in New Issue
Block a user