feat(pinch): instrument savings and count tool overhead in the budget #18
@@ -311,7 +311,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
<div class="container-xl">
|
||||
<div class="row row-cards">
|
||||
|
||||
<!-- Row 1: Local compute (own meter) + Cloud compute (NeuralWatt twin)
|
||||
<!-- Row 1: Local compute (own meter) + Pinch savings
|
||||
side-by-side above the wider model-usage section. Card bodies
|
||||
rebuild every poll; they share no chart lifecycle. -->
|
||||
<div class="col-xl-6">
|
||||
@@ -324,6 +324,18 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="card h-100">
|
||||
<div class="card-header d-flex align-items-center justify-content-between">
|
||||
<h3 class="card-title mb-0"><span class="me-2" data-icon="bar-chart"></span>Pinch savings</h3>
|
||||
</div>
|
||||
<div class="card-body p-3" id="pinch-content">
|
||||
<div class="empty">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Row 1.5: Cloud compute (NeuralWatt twin) -->
|
||||
<div class="col-xl-6">
|
||||
<div class="card h-100">
|
||||
<div class="card-header d-flex align-items-center justify-content-between">
|
||||
@@ -540,6 +552,7 @@ async function loadSnapshot() {
|
||||
renderQuotaChip(data.quota);
|
||||
renderDecisions(data.recent_decisions);
|
||||
renderLocalEnergy(data.local_energy);
|
||||
renderPinch(data.pinch);
|
||||
renderCloudEnergy(data.per_model);
|
||||
renderBars(data.per_model);
|
||||
renderVerdict(data.verdict_mix);
|
||||
@@ -765,6 +778,27 @@ function renderLocalEnergy(summary) {
|
||||
${typeHtml}`;
|
||||
}
|
||||
|
||||
function renderPinch(summary) {
|
||||
const el = document.getElementById('pinch-content');
|
||||
if (!summary) {
|
||||
el.innerHTML = '<div class="empty">Pinch is disabled</div>';
|
||||
return;
|
||||
}
|
||||
const share = Number(summary.share_pruned || 0);
|
||||
const totalSaved = Number(summary.total_tokens_saved || 0).toLocaleString();
|
||||
const medianSaved = summary.median_tokens_saved == null
|
||||
? 'n/a'
|
||||
: Number(summary.median_tokens_saved).toLocaleString();
|
||||
const dollars = Number(summary.dollars_saved_usd_30d || 0).toFixed(6);
|
||||
el.innerHTML = `
|
||||
<div class="d-flex gap-4">
|
||||
<div><div class="text-muted small">share pruned</div><div class="h4 m-0">${(share * 100).toFixed(1)}%</div></div>
|
||||
<div><div class="text-muted small">tokens saved</div><div class="h4 m-0">${totalSaved}</div></div>
|
||||
<div><div class="text-muted small">median saved</div><div class="h4 m-0">${medianSaved}</div></div>
|
||||
<div><div class="text-muted small">30d dollars saved</div><div class="h4 m-0">$${dollars}</div></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderCloudEnergy(perModel) {
|
||||
const el = document.getElementById('cloud-energy-content');
|
||||
const data = perModel || [];
|
||||
|
||||
@@ -234,7 +234,10 @@ CREATE TABLE IF NOT EXISTS route_decisions (
|
||||
flex_preference TEXT, -- 'no-flex'|'auto'|'prefer-flex'
|
||||
-- |'force-flex' (resolved)
|
||||
flex_swapped INTEGER, -- 0/1 post-rank flex swap applied
|
||||
flex_forced INTEGER -- 0/1 swap bypassed interactive
|
||||
flex_forced INTEGER, -- 0/1 swap bypassed interactive
|
||||
pinch_original_tokens INTEGER, -- estimated tokens before pruning
|
||||
pinch_final_tokens INTEGER -- tokens sent after pruning (may
|
||||
-- equal original when no pruning)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_verifications_model ON verifications (model_id, provider);
|
||||
|
||||
@@ -129,10 +129,28 @@ compare them to.
|
||||
it already adds `request_id` and `exploration` to this table.*
|
||||
3. **Surface it in `/metrics`.** A `pinch` block alongside the existing
|
||||
aggregates: share of requests pruned, median and total `tokens_saved`, and
|
||||
estimated dollars saved (price the delta at the selected model's cached
|
||||
prompt rate — `metrics.py` already joins what it needs). This is the number
|
||||
that says whether pinch earns its place, and there is currently no way to
|
||||
ask for it.
|
||||
estimated dollars saved. **Price the saved tokens at the same BLENDED rate
|
||||
`routing.estimated_cost` uses**, not at the cached rate:
|
||||
|
||||
```
|
||||
saved_usd = saved_tokens
|
||||
* ( (1 - cache_rate) * cost_per_1m_prompt
|
||||
+ cache_rate * COALESCE(cost_per_1m_prompt_cached, cost_per_1m_prompt) )
|
||||
/ 1_000_000
|
||||
```
|
||||
|
||||
*(Corrected 2026-09-01. An earlier draft said "price the delta at the
|
||||
cached prompt rate", which is wrong twice over: `cost_per_1m_prompt_cached`
|
||||
is already the discounted per-1M price, so multiplying by `cache_rate`
|
||||
applies the discount a second time, and it silently drops the
|
||||
`(1 - cache_rate)` fraction billed at full price. Match
|
||||
`routing.estimated_cost` — `src/routing.py:363-374` — or the dashboard will
|
||||
report a savings figure that disagrees with the cost model the router
|
||||
actually ranks on. Note the COALESCE: `cost_per_1m_prompt_cached` can be
|
||||
NULL, and routing falls back to the full prompt price in that case.)*
|
||||
|
||||
This is the number that says whether pinch earns its place, and there is
|
||||
currently no way to ask for it.
|
||||
|
||||
### Commit 2 — count what gets billed
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ openai==3.0.0
|
||||
python-dotenv==1.2.2
|
||||
pytest==9.1.1
|
||||
pytest-cov==7.1.0
|
||||
pytest-xdist==3.8.0
|
||||
# Deliberate UI-bring-your-own-tool dependency: textual is the terminal
|
||||
# dashboard framework for the monitoring TUI (tui.py), a SEPARATE entrypoint.
|
||||
# It is only imported by tui.py — never by the service dispatch path, so the
|
||||
|
||||
@@ -653,6 +653,7 @@ def build_router(
|
||||
"verdict_mix": metrics.verdict_mix(conn),
|
||||
"top_proficiency": metrics.top_proficiency(conn, "coding_general"),
|
||||
"local_energy": metrics.local_energy_summary(conn, cfg),
|
||||
"pinch": metrics.pinch_summary(conn, cfg),
|
||||
"health": _health_check(conn, cfg),
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
@@ -242,6 +242,7 @@ def prune_context(
|
||||
keep_last_turns: int = PinchConfig.model_fields["keep_last_turns"].default,
|
||||
max_summarize_chars: int = PinchConfig.model_fields["max_summarize_chars"].default,
|
||||
relevance_order: Optional[list[int]] = None,
|
||||
extra_fixed_tokens: int = 0,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Trim old tool results once a conversation exceeds ``budget_tokens``.
|
||||
|
||||
@@ -260,7 +261,7 @@ def prune_context(
|
||||
Only runs (and only mutates anything) when the estimate actually exceeds
|
||||
the budget; otherwise the original list is returned untouched.
|
||||
"""
|
||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
|
||||
if orig_tokens <= budget_tokens:
|
||||
return messages, {
|
||||
"pruned": False,
|
||||
|
||||
@@ -91,6 +91,7 @@ from verification import (
|
||||
from metrics import (
|
||||
local_energy_summary,
|
||||
per_model,
|
||||
pinch_summary,
|
||||
quota_burn,
|
||||
recent_decisions,
|
||||
scoring_coverage,
|
||||
@@ -328,7 +329,9 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
|
||||
streamed INTEGER,
|
||||
flex_preference TEXT,
|
||||
flex_swapped INTEGER,
|
||||
flex_forced INTEGER
|
||||
flex_forced INTEGER,
|
||||
pinch_original_tokens INTEGER,
|
||||
pinch_final_tokens INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
@@ -346,6 +349,8 @@ def ensure_route_decisions(conn: sqlite3.Connection) -> None:
|
||||
("flex_preference", "TEXT"),
|
||||
("flex_swapped", "INTEGER"),
|
||||
("flex_forced", "INTEGER"),
|
||||
("pinch_original_tokens", "INTEGER"),
|
||||
("pinch_final_tokens", "INTEGER"),
|
||||
):
|
||||
if name not in existing:
|
||||
conn.execute(f"ALTER TABLE route_decisions ADD COLUMN {name} {decl}")
|
||||
@@ -989,6 +994,8 @@ def persist_route_decision(
|
||||
flex_preference=None,
|
||||
flex_swapped=0,
|
||||
flex_forced=0,
|
||||
pinch_original_tokens=None,
|
||||
pinch_final_tokens=None,
|
||||
) -> None:
|
||||
"""Record one routing decision to route_decisions, best-effort and gated.
|
||||
|
||||
@@ -1078,8 +1085,9 @@ def persist_route_decision(
|
||||
selected_model, selected_provider, runner_up_models,
|
||||
est_cost_usd, est_proficiency, rejected_reason, session_key,
|
||||
tools, images, json_mode, streamed,
|
||||
flex_preference, flex_swapped, flex_forced
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
flex_preference, flex_swapped, flex_forced,
|
||||
pinch_original_tokens, pinch_final_tokens
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
observed_at,
|
||||
@@ -1106,6 +1114,8 @@ def persist_route_decision(
|
||||
flex_preference,
|
||||
int(bool(flex_swapped)),
|
||||
int(bool(flex_forced)),
|
||||
pinch_original_tokens,
|
||||
pinch_final_tokens,
|
||||
),
|
||||
)
|
||||
decision_id = int(cursor.lastrowid)
|
||||
@@ -1142,6 +1152,8 @@ def persist_route_decision(
|
||||
"flex_preference": flex_preference,
|
||||
"flex_swapped": int(bool(flex_swapped)),
|
||||
"flex_forced": int(bool(flex_forced)),
|
||||
"pinch_original_tokens": pinch_original_tokens,
|
||||
"pinch_final_tokens": pinch_final_tokens,
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - best-effort must never raise
|
||||
@@ -1462,6 +1474,7 @@ def metrics_endpoint():
|
||||
"verdict_mix": verdict_mix(conn),
|
||||
"top_proficiency": top_proficiency(conn, "coding_general"),
|
||||
"local_energy": local_energy_summary(conn, cfg),
|
||||
"pinch": pinch_summary(conn, cfg),
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
finally:
|
||||
@@ -1933,12 +1946,21 @@ def _embed_for_relevance(
|
||||
return order_by_relevance(embeddings[0], embeddings[1:])
|
||||
|
||||
|
||||
def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optional[list[int]]:
|
||||
def _relevance_order_for(
|
||||
messages: list[dict],
|
||||
cfg,
|
||||
budget_tokens: int,
|
||||
extra_fixed_tokens: int = 0,
|
||||
) -> Optional[list[int]]:
|
||||
"""Compute the relevance_order for prune_context, or None (uniform).
|
||||
|
||||
When pinch.relevance is off, or candidate count is below min_candidates,
|
||||
returns None so prune_context uses today's uniform pass. Otherwise makes
|
||||
one batched embedding call and returns the least-relevant-first index list.
|
||||
|
||||
``extra_fixed_tokens`` accounts for overhead that ``prune_context``'s
|
||||
budget guard also adds to ``orig_tokens`` (e.g. tool-definition size),
|
||||
so the skip-budget check in here matches the same total.
|
||||
"""
|
||||
rel = cfg.pinch.relevance
|
||||
if not (cfg.pinch.enabled and rel.enabled):
|
||||
@@ -1947,8 +1969,10 @@ def _relevance_order_for(messages: list[dict], cfg, budget_tokens: int) -> Optio
|
||||
if len(candidates) < rel.min_candidates:
|
||||
return None
|
||||
# Skip the embed when the conversation is under budget — prune_context would
|
||||
# be a no-op, so ranking is pointless.
|
||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
||||
# be a no-op, so ranking is pointless. Same total as prune_context includes
|
||||
# extra_fixed_tokens (e.g. tool definitions) so relevance doesn't silently
|
||||
# decline on tool-heavy requests.
|
||||
orig_tokens = sum(estimate_tokens(extract_text(m)) for m in messages) + extra_fixed_tokens
|
||||
if orig_tokens <= budget_tokens:
|
||||
return None
|
||||
query = _last_user_text(messages)
|
||||
@@ -2355,6 +2379,11 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
session_key = session_fingerprint(messages)
|
||||
streamed = bool(body.get("stream"))
|
||||
|
||||
# Pinch stats are filled by the routed and passthrough prune paths (when
|
||||
# pinch is enabled) and read by all persist sites. A single declaration
|
||||
# keeps the variable in scope regardless of branch.
|
||||
pinch_stats = None
|
||||
|
||||
# What this request needs, read from the body once and used by both the
|
||||
# routed and pass-through branches. Stated by the caller, not guessed at:
|
||||
# a tools array, an image_url part and a response_format each say exactly
|
||||
@@ -2404,14 +2433,20 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
# messages, so the classification turn above (which used
|
||||
# `_previous_context` from the full messages) is undisturbed.
|
||||
if cfg.pinch.enabled:
|
||||
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
|
||||
send_messages, pinch_stats = prune_context(
|
||||
list(messages),
|
||||
budget_tokens=cfg.pinch.budget_tokens,
|
||||
keep_last_turns=cfg.pinch.keep_last_turns,
|
||||
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
||||
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
|
||||
relevance_order=_relevance_order_for(
|
||||
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
|
||||
extra_fixed_tokens=_tools_overhead,
|
||||
),
|
||||
extra_fixed_tokens=_tools_overhead,
|
||||
)
|
||||
logs.debug(
|
||||
if pinch_stats.get("pruned"):
|
||||
logs.info(
|
||||
"pinch",
|
||||
pruned=pinch_stats["pruned"],
|
||||
saved=pinch_stats["tokens_saved"],
|
||||
@@ -2559,6 +2594,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
images=int(caps.has_images),
|
||||
json_mode=int(caps.require_json_mode),
|
||||
streamed=streamed,
|
||||
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
)
|
||||
raise HTTPException(
|
||||
422,
|
||||
@@ -2583,6 +2622,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
streamed=streamed,
|
||||
classification_source=classified_src,
|
||||
classifier_ms=classifier_ms,
|
||||
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
)
|
||||
target = decision.selected.model_id
|
||||
provider = decision.selected.provider
|
||||
@@ -2592,6 +2635,31 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
# Not a routing decision at all, and worth saying so plainly: a client
|
||||
# pinned to one model gets none of the filtering or ranking below.
|
||||
logs.info("passthrough", model=target, stream=bool(body.get("stream")))
|
||||
send_messages = messages
|
||||
if cfg.pinch.enabled:
|
||||
# Same pruning as the routed path, but it must run before persistence
|
||||
# so a pruned passthrough request records real pinch tokens rather
|
||||
# than NULL.
|
||||
_tools_overhead = len(json.dumps(body.get("tools") or [])) // CHARS_PER_TOKEN
|
||||
send_messages, pinch_stats = prune_context(
|
||||
list(messages),
|
||||
budget_tokens=cfg.pinch.budget_tokens,
|
||||
keep_last_turns=cfg.pinch.keep_last_turns,
|
||||
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
||||
relevance_order=_relevance_order_for(
|
||||
messages, cfg, budget_tokens=cfg.pinch.budget_tokens,
|
||||
extra_fixed_tokens=_tools_overhead,
|
||||
),
|
||||
extra_fixed_tokens=_tools_overhead,
|
||||
)
|
||||
if pinch_stats.get("pruned"):
|
||||
logs.info(
|
||||
"pinch",
|
||||
pruned=pinch_stats["pruned"],
|
||||
saved=pinch_stats["tokens_saved"],
|
||||
orig=pinch_stats["original_tokens"],
|
||||
final=pinch_stats["final_tokens"],
|
||||
)
|
||||
persist_route_decision(
|
||||
"passthrough",
|
||||
selected_model=target,
|
||||
@@ -2605,6 +2673,10 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
flex_preference=cfg.routing.default_flex_preference.value,
|
||||
flex_swapped=0,
|
||||
flex_forced=0,
|
||||
pinch_original_tokens=pinch_stats.get("original_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
pinch_final_tokens=pinch_stats.get("final_tokens")
|
||||
if pinch_stats is not None else None,
|
||||
)
|
||||
|
||||
# A pinned model id is dispatched as asked, but a pin that cannot
|
||||
@@ -2625,31 +2697,9 @@ def chat_completions(body: dict[str, Any], background: BackgroundTasks):
|
||||
# top (needed by the passthrough/local-vision branches), only the working
|
||||
# directory is derived here because only the observation path uses it.
|
||||
session_dir = session_directory(messages)
|
||||
if not wants_routing:
|
||||
# The routed path pruned send_messages once, before its measured-context
|
||||
# decision, and the pruned list is reused here. A passthrough request
|
||||
# never went through that path, so it prunes now — same gating as
|
||||
# before (only when pinch is enabled).
|
||||
send_messages = messages
|
||||
if cfg.pinch.enabled:
|
||||
# Optional relevance-based context pruning: when the conversation
|
||||
# exceeds budget_tokens, trim old tool results BEFORE any paid token
|
||||
# is sent upstream. User/assistant/system messages are always kept;
|
||||
# only the classifier input is exempt (it already clamps to head+tail).
|
||||
send_messages, pinch_stats = prune_context(
|
||||
list(messages),
|
||||
budget_tokens=cfg.pinch.budget_tokens,
|
||||
keep_last_turns=cfg.pinch.keep_last_turns,
|
||||
max_summarize_chars=cfg.pinch.max_summarize_chars,
|
||||
relevance_order=_relevance_order_for(messages, cfg, budget_tokens=cfg.pinch.budget_tokens),
|
||||
)
|
||||
logs.debug(
|
||||
"pinch",
|
||||
pruned=pinch_stats["pruned"],
|
||||
saved=pinch_stats["tokens_saved"],
|
||||
orig=pinch_stats["original_tokens"],
|
||||
final=pinch_stats["final_tokens"],
|
||||
)
|
||||
# send_messages is already set: by the routed path's prune block above, or
|
||||
# by the passthrough path's prune block. Both branches default to messages
|
||||
# when pinch is disabled.
|
||||
upstream_body = {**body, "model": target, "messages": list(send_messages)}
|
||||
streaming = bool(body.get("stream"))
|
||||
if streaming:
|
||||
|
||||
100
src/metrics.py
100
src/metrics.py
@@ -16,6 +16,7 @@ recent_decisions — last N rows from the route_decisions observability table
|
||||
per_model — per-model aggregates over energy_observations (last 30 d)
|
||||
verdict_mix — counts by verdict from verifications (last N days)
|
||||
top_proficiency — top models by blended_score for a category
|
||||
pinch_summary — context-pruning savings over the last 30 d
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -422,7 +423,8 @@ def recent_decisions(
|
||||
candidates_considered, selected_model, selected_provider,
|
||||
runner_up_models, est_cost_usd, est_proficiency,
|
||||
rejected_reason, session_key, tools, images, json_mode, streamed,
|
||||
flex_preference, flex_swapped, flex_forced
|
||||
flex_preference, flex_swapped, flex_forced,
|
||||
pinch_original_tokens, pinch_final_tokens
|
||||
FROM route_decisions
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
@@ -540,3 +542,99 @@ def local_energy_summary(conn: sqlite3.Connection, cfg) -> Optional[dict]:
|
||||
if reset_day is not None:
|
||||
result["next_reset_date"] = _next_reset_date(reset_day)
|
||||
return result
|
||||
|
||||
|
||||
def pinch_summary(conn: sqlite3.Connection, cfg: Any) -> Optional[dict]:
|
||||
"""Aggregate context-pruning savings over the last 30 days.
|
||||
|
||||
Returns None when pinch is disabled so callers can omit the section.
|
||||
Computes the number of pruned calls, share of pruned calls, total and
|
||||
median tokens saved, and an estimated dollar saving priced at the blended
|
||||
prompt rate the router uses for cost estimation.
|
||||
"""
|
||||
if not cfg.pinch.enabled:
|
||||
return None
|
||||
|
||||
cache_rate = cfg.objective.assumed_cache_rate
|
||||
|
||||
totals = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS calls_30d,
|
||||
SUM(CASE
|
||||
WHEN pinch_original_tokens IS NOT NULL
|
||||
AND pinch_final_tokens IS NOT NULL
|
||||
AND pinch_original_tokens > pinch_final_tokens
|
||||
THEN 1 ELSE 0
|
||||
END) AS pruned_calls_30d,
|
||||
SUM(CASE
|
||||
WHEN pinch_original_tokens IS NOT NULL
|
||||
AND pinch_final_tokens IS NOT NULL
|
||||
THEN pinch_original_tokens - pinch_final_tokens
|
||||
ELSE 0
|
||||
END) AS total_tokens_saved
|
||||
FROM route_decisions
|
||||
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
||||
"""
|
||||
).fetchone()
|
||||
|
||||
calls_30d = totals["calls_30d"] or 0
|
||||
pruned_calls_30d = totals["pruned_calls_30d"] or 0
|
||||
share_pruned = pruned_calls_30d / calls_30d if calls_30d else 0.0
|
||||
total_tokens_saved = totals["total_tokens_saved"] or 0
|
||||
|
||||
saved_rows = conn.execute(
|
||||
"""
|
||||
SELECT pinch_original_tokens - pinch_final_tokens AS tokens_saved
|
||||
FROM route_decisions
|
||||
WHERE julianday(observed_at) > julianday('now', '-30 days')
|
||||
AND pinch_original_tokens IS NOT NULL
|
||||
AND pinch_final_tokens IS NOT NULL
|
||||
AND pinch_original_tokens > pinch_final_tokens
|
||||
"""
|
||||
).fetchall()
|
||||
median_tokens_saved = _percentile(
|
||||
[r["tokens_saved"] for r in saved_rows], 50
|
||||
)
|
||||
|
||||
dollar_rows = conn.execute(
|
||||
"""
|
||||
SELECT rd.pinch_original_tokens - rd.pinch_final_tokens AS tokens_saved,
|
||||
m.cost_per_1m_prompt,
|
||||
m.cost_per_1m_prompt_cached
|
||||
FROM route_decisions rd
|
||||
JOIN models m
|
||||
ON rd.selected_model = m.model_id
|
||||
AND rd.selected_provider = m.provider
|
||||
WHERE julianday(rd.observed_at) > julianday('now', '-30 days')
|
||||
AND rd.pinch_original_tokens IS NOT NULL
|
||||
AND rd.pinch_final_tokens IS NOT NULL
|
||||
AND rd.pinch_original_tokens > rd.pinch_final_tokens
|
||||
AND rd.selected_model IS NOT NULL
|
||||
AND m.cost_per_1m_prompt IS NOT NULL
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
dollars_saved_usd_30d = 0.0
|
||||
for r in dollar_rows:
|
||||
saved = r["tokens_saved"]
|
||||
prompt_price = r["cost_per_1m_prompt"]
|
||||
cached_price = r["cost_per_1m_prompt_cached"]
|
||||
if cached_price is None:
|
||||
cached_price = prompt_price
|
||||
blended = (1.0 - cache_rate) * prompt_price + cache_rate * cached_price
|
||||
dollars_saved_usd_30d += saved * blended / 1_000_000
|
||||
|
||||
result = {
|
||||
"calls_30d": calls_30d,
|
||||
"pruned_calls_30d": pruned_calls_30d,
|
||||
"share_pruned": round(share_pruned, 4),
|
||||
"total_tokens_saved": total_tokens_saved,
|
||||
"median_tokens_saved": median_tokens_saved,
|
||||
"dollars_saved_usd_30d": round(dollars_saved_usd_30d, 6),
|
||||
}
|
||||
reset_day = getattr(cfg.objective, "billing_reset_day", None)
|
||||
if reset_day is not None:
|
||||
result["reset_date"] = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat()
|
||||
result["next_reset_date"] = _next_reset_date(reset_day)
|
||||
return result
|
||||
|
||||
|
||||
@@ -254,6 +254,13 @@ class DashboardApp(App):
|
||||
def _on_interval(self) -> None:
|
||||
if getattr(self, "_unmounted", False):
|
||||
return
|
||||
# Skip rendering while the dashboard screen is still composing
|
||||
# (the DataTable isn't in the DOM yet). The next interval tick will
|
||||
# catch it once the widget is mounted.
|
||||
try:
|
||||
self.query_one("#model-table", DataTable)
|
||||
except Exception: # noqa: BLE001 — widget not yet available
|
||||
return
|
||||
self._refresh()
|
||||
|
||||
def action_refresh(self) -> None:
|
||||
|
||||
@@ -59,6 +59,17 @@ def build_model(data: dict) -> dict:
|
||||
else:
|
||||
quota_rows = []
|
||||
|
||||
pinch = data.get("pinch")
|
||||
if pinch:
|
||||
pinch_rows = [
|
||||
{"label": "share_pruned", "value": pinch.get("share_pruned")},
|
||||
{"label": "total_tokens_saved", "value": pinch.get("total_tokens_saved")},
|
||||
{"label": "median_tokens_saved", "value": pinch.get("median_tokens_saved")},
|
||||
{"label": "dollars_saved_usd_30d", "value": pinch.get("dollars_saved_usd_30d")},
|
||||
]
|
||||
else:
|
||||
pinch_rows = []
|
||||
|
||||
per_model = [
|
||||
{
|
||||
"model": r.get("model_id"),
|
||||
@@ -85,6 +96,7 @@ def build_model(data: dict) -> dict:
|
||||
|
||||
return {
|
||||
"quota": quota_rows,
|
||||
"pinch": pinch_rows,
|
||||
"per_model": per_model,
|
||||
"verdict_mix": verdict_mix,
|
||||
"recent_decisions": recent,
|
||||
|
||||
@@ -155,7 +155,9 @@ def test_admin_snapshot_has_all_top_level_keys(seeded_client):
|
||||
"per_model",
|
||||
"verdict_mix",
|
||||
"top_proficiency",
|
||||
"local_energy",
|
||||
"health",
|
||||
"pinch",
|
||||
"generated_at",
|
||||
):
|
||||
assert key in data, f"missing top-level key {key!r}"
|
||||
|
||||
@@ -31,6 +31,7 @@ EXHAUSTIVE_KEYS = (
|
||||
"top_proficiency",
|
||||
"local_energy",
|
||||
"health",
|
||||
"pinch",
|
||||
"generated_at",
|
||||
)
|
||||
|
||||
|
||||
@@ -536,6 +536,89 @@ def test_relevance_order_overrides_recency():
|
||||
assert "chars trimmed" in out[3]["content"]
|
||||
|
||||
|
||||
def test_extra_fixed_tokens_tips_budget_to_prune():
|
||||
# A conversation whose message tokens sit just under the budget, but tool
|
||||
# overhead pushes the accounting total over. Without overhead it is a no-op;
|
||||
# with overhead it prunes.
|
||||
messages = [
|
||||
_user("question"),
|
||||
_tool("read", "x" * 540), # 540 chars -> 180 tokens
|
||||
_assistant("answer"),
|
||||
_user("follow up"),
|
||||
]
|
||||
msg_tokens = sum(estimate_tokens(extract_text(m)) for m in messages)
|
||||
assert msg_tokens == 187
|
||||
|
||||
out, stats = prune_context(
|
||||
messages,
|
||||
budget_tokens=200,
|
||||
keep_last_turns=0,
|
||||
extra_fixed_tokens=0,
|
||||
)
|
||||
assert stats["pruned"] is False
|
||||
assert stats["original_tokens"] == msg_tokens
|
||||
|
||||
out, stats = prune_context(
|
||||
messages,
|
||||
budget_tokens=200,
|
||||
keep_last_turns=0,
|
||||
extra_fixed_tokens=100,
|
||||
)
|
||||
assert stats["pruned"] is True
|
||||
assert stats["original_tokens"] == msg_tokens + 100
|
||||
# The tool result was trimmed to cover the deficit (187 + 100 - 200 = 87).
|
||||
assert stats["tokens_saved"] >= 87
|
||||
tool_out = [m for m in out if m.get("role") == "tool"][0]
|
||||
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
|
||||
|
||||
|
||||
def test_extra_fixed_tokens_zero_preserves_behavior():
|
||||
# The default and explicit zero must be byte-for-byte identical.
|
||||
messages = [
|
||||
_user("question"),
|
||||
_tool("read", "big result " * 500),
|
||||
_assistant("answer"),
|
||||
_user("follow up"),
|
||||
_assistant("ok"),
|
||||
]
|
||||
default_out, default_stats = prune_context(
|
||||
messages, budget_tokens=100, keep_last_turns=0
|
||||
)
|
||||
explicit_zero_out, explicit_zero_stats = prune_context(
|
||||
messages,
|
||||
budget_tokens=100,
|
||||
keep_last_turns=0,
|
||||
extra_fixed_tokens=0,
|
||||
)
|
||||
assert explicit_zero_out == default_out
|
||||
assert explicit_zero_stats == default_stats
|
||||
|
||||
|
||||
def test_tool_content_never_summarized():
|
||||
# Even when extra_fixed_tokens forces pruning, user/assistant/system
|
||||
# messages survive verbatim; only tool results are candidates.
|
||||
messages = [
|
||||
_user("what is the answer"),
|
||||
_assistant("i'll check"),
|
||||
_tool("search", "big tool result " * 500),
|
||||
_assistant("the answer is 42"),
|
||||
_user("thanks"),
|
||||
]
|
||||
out, stats = prune_context(
|
||||
messages,
|
||||
budget_tokens=100,
|
||||
keep_last_turns=0,
|
||||
extra_fixed_tokens=1000,
|
||||
)
|
||||
assert stats["pruned"] is True
|
||||
assert len(out) == len(messages)
|
||||
for orig, pruned in zip(messages, out):
|
||||
if orig.get("role") in ("user", "assistant", "system"):
|
||||
assert pruned["content"] == orig["content"]
|
||||
tool_out = [m for m in out if m.get("role") == "tool"][0]
|
||||
assert "chars trimmed" in tool_out["content"] or "result omitted" in tool_out["content"]
|
||||
|
||||
|
||||
def test_relevance_order_none_matches_default():
|
||||
# relevance_order=None must be byte-for-byte identical to not passing the
|
||||
# parameter at all (the historical uniform pass / fallback path).
|
||||
|
||||
@@ -12,6 +12,8 @@ point of testing it.
|
||||
|
||||
import pytest
|
||||
|
||||
import eval_proficiency
|
||||
|
||||
from eval_proficiency import (
|
||||
judge_for,
|
||||
parse_check_results,
|
||||
@@ -49,7 +51,8 @@ def test_code_that_does_not_parse_scores_zero_and_says_so():
|
||||
assert "did not run" in detail
|
||||
|
||||
|
||||
def test_infinite_loop_is_bounded_not_hung():
|
||||
def test_infinite_loop_is_bounded_not_hung(monkeypatch):
|
||||
monkeypatch.setattr(eval_proficiency, "CODE_TIMEOUT_SECONDS", 1)
|
||||
score, detail = score_code("def f(x):\n while True: pass", ["f(1)==1"])
|
||||
assert (score, detail) == (0.0, "timeout")
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from metrics import (
|
||||
_next_reset_date,
|
||||
context_ceilings,
|
||||
per_model,
|
||||
pinch_summary,
|
||||
quota_burn,
|
||||
recent_decisions,
|
||||
scoring_coverage,
|
||||
@@ -626,6 +627,119 @@ def test_top_proficiency_empty_for_missing_category(tmp_path):
|
||||
# --- /health endpoint compatibility -------------------------------------------
|
||||
|
||||
|
||||
def _seed_pinch_models(conn: sqlite3.Connection) -> None:
|
||||
"""Insert two cheap/dear models plus a cheap-no-cached-price variant."""
|
||||
fresh = _now().isoformat()
|
||||
rows = [
|
||||
# (model_id, tier, context, cost_per_1m_prompt, cost_per_1m_prompt_cached)
|
||||
("cheap", 2, 262128, 1.0, 0.5),
|
||||
("dear", 2, 262128, 5.0, None),
|
||||
("tiny", 1, 131072, 0.1, 0.1),
|
||||
]
|
||||
for model_id, tier, context, cost, cached in rows:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO models (
|
||||
model_id, provider, base_model_id, tier, context_window,
|
||||
effective_context_window, max_output_tokens,
|
||||
cost_per_1m_prompt, cost_per_1m_completion,
|
||||
cost_per_1m_prompt_cached,
|
||||
supports_vision, supports_json_mode,
|
||||
latency_class, reasoning_mode, context_variant,
|
||||
access_level, availability, last_updated
|
||||
) VALUES (?, 'neuralwatt', ?, ?, ?, 192500, 16384, ?, ?, ?,
|
||||
1, 1, 'standard', 'default', 'full', 'public', 'active',
|
||||
?)
|
||||
""",
|
||||
(model_id, model_id, tier, context, cost, cost / 3, cached, fresh),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _insert_pinch_decision(conn, *, model, provider, orig, final):
|
||||
"""Insert a route_decisions row with observed_at = now and given pinch values."""
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO route_decisions (
|
||||
observed_at, kind, task_category, task_tier,
|
||||
required_context_tokens, confidence, classifier_ms,
|
||||
classification_source, latency_tolerance, candidates_considered,
|
||||
selected_model, selected_provider, runner_up_models,
|
||||
est_cost_usd, est_proficiency, rejected_reason, session_key,
|
||||
tools, images, json_mode, streamed,
|
||||
pinch_original_tokens, pinch_final_tokens
|
||||
) VALUES (?, 'chat', 'coding_general', 2, 100, 0.9, 100,
|
||||
'classifier', 'interactive', 3, ?, ?, NULL,
|
||||
0.001, 0.9, NULL, 'sess', 0, 0, 0, 0, ?, ?)
|
||||
""",
|
||||
(_now().isoformat(), model, provider, orig, final),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def test_pinch_summary_aggregates_correctly(tmp_path, monkeypatch):
|
||||
"""Hand-computed aggregates over a seeded 30-day window."""
|
||||
cfg = load_config(str(ROOT / "config" / "config.yaml"))
|
||||
monkeypatch.setattr(cfg.pinch, "enabled", True)
|
||||
cache_rate = cfg.objective.assumed_cache_rate
|
||||
|
||||
conn = _make_db(tmp_path)
|
||||
_seed_pinch_models(conn)
|
||||
|
||||
# Two pruned cheap rows: saved tokens 50 and 30.
|
||||
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=100, final=50)
|
||||
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=80, final=50)
|
||||
# One non-pruned cheap row (final == orig).
|
||||
_insert_pinch_decision(conn, model="cheap", provider="neuralwatt", orig=60, final=60)
|
||||
# One pruned dear row with NULL cost_per_1m_prompt_cached -> falls back to prompt price.
|
||||
_insert_pinch_decision(conn, model="dear", provider="neuralwatt", orig=200, final=100)
|
||||
# One rejection row with NULL selected_model -> counted in share/tokens but excluded from dollars.
|
||||
_insert_pinch_decision(conn, model=None, provider=None, orig=1000, final=500)
|
||||
|
||||
result = pinch_summary(conn, cfg)
|
||||
assert result is not None
|
||||
assert result["calls_30d"] == 5
|
||||
assert result["pruned_calls_30d"] == 4
|
||||
assert result["share_pruned"] == round(4 / 5, 4)
|
||||
# saved tokens: 50 + 30 + 100 + 500 = 680
|
||||
assert result["total_tokens_saved"] == 680
|
||||
# median of [30, 50, 100, 500] = 75
|
||||
assert result["median_tokens_saved"] == 75
|
||||
|
||||
# dollar math: cheap saved 80 tokens at blended rate over cached prompt.
|
||||
cheap_blended = (1 - cache_rate) * 1.0 + cache_rate * 0.5
|
||||
cheap_dollars = 80 * cheap_blended / 1_000_000
|
||||
# dear saved 100 tokens; cached price NULL -> fallback to prompt price 5.0.
|
||||
dear_dollars = 100 * 5.0 / 1_000_000
|
||||
# rejection row with NULL selected_model is excluded from dollar math.
|
||||
assert result["dollars_saved_usd_30d"] == round(cheap_dollars + dear_dollars, 6)
|
||||
|
||||
|
||||
def test_pinch_summary_zero_rows(tmp_path, monkeypatch):
|
||||
"""Enabled with no decisions in the window returns zeros and None median."""
|
||||
cfg = load_config(str(ROOT / "config" / "config.yaml"))
|
||||
monkeypatch.setattr(cfg.pinch, "enabled", True)
|
||||
conn = _make_db(tmp_path)
|
||||
result = pinch_summary(conn, cfg)
|
||||
assert result is not None
|
||||
assert result["calls_30d"] == 0
|
||||
assert result["pruned_calls_30d"] == 0
|
||||
assert result["share_pruned"] == 0.0
|
||||
assert result["total_tokens_saved"] == 0
|
||||
assert result["median_tokens_saved"] is None
|
||||
assert result["dollars_saved_usd_30d"] == 0.0
|
||||
|
||||
|
||||
def test_pinch_summary_disabled_returns_none(tmp_path, monkeypatch):
|
||||
"""When cfg.pinch.enabled is False, None is returned."""
|
||||
conn = _make_db(tmp_path)
|
||||
cfg = SimpleNamespace(pinch=SimpleNamespace(enabled=False))
|
||||
assert pinch_summary(conn, cfg) is None
|
||||
|
||||
|
||||
# --- /health endpoint compatibility -------------------------------------------
|
||||
|
||||
|
||||
def test_health_endpoint_returns_scoring_key(tmp_path, monkeypatch):
|
||||
"""/health still returns the same SHAPE after moving functions to metrics."""
|
||||
db_path = tmp_path / "test.db"
|
||||
|
||||
@@ -161,6 +161,7 @@ def test_metrics_endpoint_has_all_top_level_keys(seeded_client):
|
||||
"verdict_mix",
|
||||
"top_proficiency",
|
||||
"local_energy",
|
||||
"pinch",
|
||||
"generated_at",
|
||||
):
|
||||
assert key in data, f"missing top-level key {key!r}"
|
||||
|
||||
@@ -58,6 +58,8 @@ ROUTE_DECISIONS_COLUMNS = [
|
||||
"flex_preference",
|
||||
"flex_swapped",
|
||||
"flex_forced",
|
||||
"pinch_original_tokens",
|
||||
"pinch_final_tokens",
|
||||
]
|
||||
|
||||
|
||||
@@ -205,6 +207,54 @@ def test_ensure_route_decisions_allows_insert(tmp_path):
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_persist_writes_pinch_columns(tmp_path, monkeypatch):
|
||||
"""persist_route_decision writes pinch_original_tokens and pinch_final_tokens
|
||||
when provided, and defaults them to NULL when omitted."""
|
||||
db_path = tmp_path / "pinch.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(_schema_minus_route_decisions())
|
||||
conn.close()
|
||||
|
||||
monkeypatch.setattr(dispatcher.cfg.database, "path", str(db_path))
|
||||
monkeypatch.setattr(dispatcher.cfg.logging, "log_route_decisions", True)
|
||||
|
||||
dispatcher.persist_route_decision(
|
||||
"route",
|
||||
classification=Classification(
|
||||
task_category="coding_general", task_tier=2,
|
||||
required_context_tokens=100, confidence=0.9,
|
||||
),
|
||||
latency_tolerance="interactive",
|
||||
selected_model="deepseek-v4-flash",
|
||||
selected_provider="neuralwatt",
|
||||
pinch_original_tokens=100,
|
||||
pinch_final_tokens=50,
|
||||
)
|
||||
|
||||
dispatcher.persist_route_decision(
|
||||
"route",
|
||||
classification=Classification(
|
||||
task_category="coding_general", task_tier=2,
|
||||
required_context_tokens=100, confidence=0.9,
|
||||
),
|
||||
latency_tolerance="interactive",
|
||||
selected_model="deepseek-v4-flash",
|
||||
selected_provider="neuralwatt",
|
||||
)
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
rows = conn.execute(
|
||||
"SELECT pinch_original_tokens, pinch_final_tokens FROM route_decisions ORDER BY id"
|
||||
).fetchall()
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["pinch_original_tokens"] == 100
|
||||
assert rows[0]["pinch_final_tokens"] == 50
|
||||
assert rows[1]["pinch_original_tokens"] is None
|
||||
assert rows[1]["pinch_final_tokens"] is None
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_persist_ensure_on_write_fixes_live_db_missing_table(tmp_path, monkeypatch):
|
||||
"""A live router.db without route_decisions gets it on the WRITE path.
|
||||
|
||||
@@ -600,6 +650,53 @@ def test_passthrough_persists_one_row_with_no_nameerror(decision_router):
|
||||
assert r["session_key"] is not None
|
||||
|
||||
|
||||
def test_passthrough_records_pinch_columns_when_pruned(decision_router, monkeypatch):
|
||||
"""Regression: passthrough pruning must run before persistence so pinch stats
|
||||
are recorded rather than NULL."""
|
||||
client, db_path = decision_router
|
||||
monkeypatch.setattr(dispatcher.cfg.pinch, "enabled", True)
|
||||
monkeypatch.setattr(dispatcher.cfg.pinch, "budget_tokens", 1)
|
||||
monkeypatch.setattr(dispatcher.cfg.pinch, "keep_last_turns", 1)
|
||||
|
||||
# Long conversation plus tools to exceed the tiny budget.
|
||||
long_text = "word " * 500
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": long_text},
|
||||
{"role": "assistant", "content": "acknowledged"},
|
||||
{"role": "user", "content": "now summarize"},
|
||||
]
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": DEAR,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "noop",
|
||||
"description": "A tool with a long description to inflate overhead",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
# Routing succeeds because the fixture stubs the provider call; persistence
|
||||
# happens before forwarding, so the row is already written.
|
||||
assert resp.status_code == 200
|
||||
|
||||
rows = _rows(db_path)
|
||||
passthrough_rows = [r for r in rows if r["kind"] == "passthrough"]
|
||||
assert len(passthrough_rows) == 1
|
||||
r = passthrough_rows[0]
|
||||
assert r["pinch_original_tokens"] is not None
|
||||
assert r["pinch_final_tokens"] is not None
|
||||
assert r["pinch_original_tokens"] > r["pinch_final_tokens"]
|
||||
|
||||
|
||||
def test_local_vision_success_persists_one_local_row(decision_router, monkeypatch):
|
||||
client, db_path = decision_router
|
||||
_drop_cheap(db_path)
|
||||
|
||||
@@ -114,6 +114,14 @@ def _fixture() -> dict:
|
||||
"blended_score": 1.0, "source": "self_eval_thin",
|
||||
"self_eval_samples": 3}
|
||||
],
|
||||
"pinch": {
|
||||
"calls_30d": 120,
|
||||
"pruned_calls_30d": 30,
|
||||
"share_pruned": 0.25,
|
||||
"total_tokens_saved": 50000,
|
||||
"median_tokens_saved": 1200,
|
||||
"dollars_saved_usd_30d": 0.0125,
|
||||
},
|
||||
"generated_at": "2026-08-23T10:01:00+00:00",
|
||||
}
|
||||
|
||||
@@ -134,6 +142,23 @@ def test_build_model_quota_panel():
|
||||
assert "calls=18" in joined
|
||||
|
||||
|
||||
def test_build_model_pinch_panel():
|
||||
m = build_model(_fixture())
|
||||
rows = m["pinch"]
|
||||
by_label = {r["label"]: r["value"] for r in rows}
|
||||
assert by_label["share_pruned"] == 0.25
|
||||
assert by_label["total_tokens_saved"] == 50000
|
||||
assert by_label["median_tokens_saved"] == 1200
|
||||
assert by_label["dollars_saved_usd_30d"] == 0.0125
|
||||
|
||||
|
||||
def test_build_model_pinch_panel_empty_when_none():
|
||||
data = _fixture()
|
||||
data["pinch"] = None
|
||||
m = build_model(data)
|
||||
assert m["pinch"] == []
|
||||
|
||||
|
||||
def test_build_model_quota_panel_includes_reset_date():
|
||||
m = build_model(_fixture())
|
||||
rows = m["quota"]
|
||||
|
||||
Reference in New Issue
Block a user