feat(admin): classifier knob coverage and categorized Controls page #113

Merged
alee merged 15 commits from feat/admin-knob-taxonomy into main 2026-10-06 00:50:19 +00:00
10 changed files with 1092 additions and 69 deletions

View File

@@ -23,6 +23,24 @@ oversight nobody noticed.
absent, because enabling it must be a config edit plus a restart. That is a
recorded choice, not a gap.
The classifier section is now fully accounted for under the gate. Seven
classifier scalars (`context_framing`, `cooldown_seconds`, `fallback_tier`,
`fallback_category`, `max_input_chars`, `degraded_warn_min`,
`degraded_warn_threshold`) have a runtime control, a persisted control, or both
(`fallback_category` and `max_input_chars` are persisted-only). Every other
classifier scalar is either card-backed (`_CARD_BACKED_PATHS` in
`src/admin.py`: mode, cloud primary/fallback, encoder, and local-decision
fields) or excused in `DELIBERATELY_NOT_IN_ADMIN` (timeout, temperature,
`max_output_tokens`, `encoder.tier_from_features`).
Outside-gate knobs are still pending. Sections not yet reached by the portal
(database, dispatch providers/settings, local dispatch models, profiles,
local energy, tiers/tiering/proficiency/context, and deployment wiring inside
classifier/verification/local vision) are tracked in
`plans/deferred-knobs.md`. The first control added under any of those sections
will drag every scalar under it into scope at once, per the coverage test's
clause 1.
**Why:** Wave 2 shipped `incumbent_cache_pricing` and
`incumbent_challenger_cache_rate` with no control at all. Nobody decided that;
it just never came up. The dial's entire purpose is tuning from neutral to full

View File

@@ -135,10 +135,38 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
Scoped so any future div list keeps the grid grammar. */
.knobs-table .setting-row{display:table-row;min-height:0;padding:0;border-radius:0;border-left:2px solid transparent}
.knobs-table .setting-row.is-dirty{border-left-color:#fbbf24;background:rgba(245,158,11,.06)}
/* A config key wraps instead of truncating: the knob cell is the row's
identity, and a clipped path hides which scope the row belongs to. */
/* Persisted-cell hints: the note badge is white-space:nowrap by default, so a
long note sets the cell's min-content width and pushes the table past an
800px viewport. Let the hints wrap (the pill drops under the unit) and the
pill's own text wrap. d-inline-flex is !important, so this targets the
cell's direct .ms-2 child rather than overriding display. */
.knobs-table td > .ms-2{flex-wrap:wrap;max-width:100%}
.knobs-table td > .ms-2 .badge{white-space:normal;text-align:left}
.knobs-table .setting-key{white-space:normal;overflow:visible;text-overflow:clip}
.knobs-table .setting-key .scope{display:inline}
/* -- Category header rows: a subtle accent that segments the knob list
without adding visual weight. The chevron is the only interactive cue;
the label and icon supply context. -- */
.cat-header th{
background:rgba(255,255,255,.02);
border-bottom:1px solid rgba(255,255,255,.08);
cursor:pointer;user-select:none;
font-size:.85rem;font-weight:600;
padding:.35rem .6rem;
color:var(--tblr-secondary)
}
.cat-header th:hover{background:rgba(255,255,255,.045)}
.cat-header .cat-label{color:#e2e8f0}
.cat-chevron{display:inline-flex;align-items:center;color:var(--tblr-secondary);opacity:.7;transition:transform .18s ease}
.cat-chevron:not(.cat-chevron-open){transform:rotate(90deg)}
.cat-icon{opacity:.7;margin-right:.15rem}
.adv-subheader th{
cursor:pointer;user-select:none;
font-size:.78rem;font-weight:500;
padding:.2rem .6rem;
color:var(--tblr-secondary);opacity:.7
}
.adv-subheader th:hover{opacity:1;color:#e2e8f0}
.setting-key{
font-size:.82rem;line-height:1.25;
overflow:hidden;text-overflow:ellipsis;white-space:nowrap;
@@ -644,6 +672,9 @@ function keyHtml(key) {
const UNITS = {
'objective.max_energy_per_request': 'kWh',
'objective.plan_kwh_per_period': 'kWh',
'classifier.cooldown_seconds': 's',
'classifier.degraded_warn_min': 'decisions/24h',
'classifier.max_input_chars': 'chars',
};
/* A consequence the control itself cannot show. Keyed by both the runtime knob
@@ -692,6 +723,18 @@ const SESSION_CACHE_TTL_NOTE = {
cls: 'bg-info',
glyph: 'info',
};
const DEGRADED_WARN_MIN_NOTE = {
text: 'silent below this many decisions in 24h',
title: 'No warning is raised until the classifier completes at least this many decisions in a 24-hour window. Below the threshold the degraded-count warning is suppressed entirely.',
cls: 'bg-info',
glyph: 'info',
};
const DEGRADED_WARN_THRESHOLD_NOTE = {
text: 'share of degraded classifications that warns',
title: 'When the fraction of degraded (non-ok) classifier verdicts exceeds this ratio within the warn-min window, an amber badge appears on the admin navbar. 0.01 = 1% of classifications degraded.',
cls: 'bg-info',
glyph: 'info',
};
const KNOB_NOTES = {
pinch_prefix_probe: PROBE_NOTE,
'pinch.prefix_probe': PROBE_NOTE,
@@ -703,6 +746,10 @@ const KNOB_NOTES = {
'objective.incumbent_cache_pricing': INCUMBENT_GATE_NOTE,
incumbent_challenger_cache_rate: CHALLENGER_DIAL_NOTE,
'objective.incumbent_challenger_cache_rate': CHALLENGER_DIAL_NOTE,
classifier_degraded_warn_min: DEGRADED_WARN_MIN_NOTE,
'classifier.degraded_warn_min': DEGRADED_WARN_MIN_NOTE,
classifier_degraded_warn_threshold: DEGRADED_WARN_THRESHOLD_NOTE,
'classifier.degraded_warn_threshold': DEGRADED_WARN_THRESHOLD_NOTE,
};
function noteBadge(key) {
@@ -723,8 +770,52 @@ const NUMBER_BOUNDS = {
// operator was reaching for. min is 5 rather than 0 -- 0 is not "off", it
// still caches and the classifier-failure cascade still replays the entry.
session_cache_staleness_seconds: { min: 5, max: 7200, step: 1 },
classifier_degraded_warn_threshold: { min: 0.001, max: 1, step: 0.01 },
classifier_max_input_chars: { min: 0, step: 1000 },
};
// Category definitions for the knob table section headers. Order here matches
// _CONFIG_GET_ORDER in admin.py so sections render in the same sequence the
// YAML uses. `icon` picks a renderStaticIcons-compatible name.
const CATEGORIES = [
{ id: 'routing_quality_cost', label: 'Routing / Quality / Cost', icon: 'settings' },
{ id: 'classification', label: 'Classification', icon: 'info' },
{ id: 'caching_context', label: 'Caching / Context', icon: 'database' },
{ id: 'local_hardware', label: 'Local Hardware', icon: 'cpu' },
{ id: 'safety_nets', label: 'Safety Nets', icon: 'alert' },
{ id: 'watchdog', label: 'Watchdog', icon: 'alert' },
];
// Collapsible category and advanced-section state. Persisted across 30s
// re-renders: toggling mutates this Set, and renderKnobsTable reads it.
// Category keys are `cat.<id>.collapsed`; advanced keys are `cat.<id>.adv`.
let collapsedCategories = new Set();
// Categories whose Advanced accordion has been seeded collapsed. Per category,
// not global: see seedAdvancedCollapsed.
let _advSeededCategories = new Set();
/* SEED_ADV_COLLAPSED:BEGIN */
function seedAdvancedCollapsed(byCategory, rows, categoryIds, seeded, collapsed) {
// Seed a category's Advanced accordion as collapsed the FIRST time that
// category renders an advanced row, not on the first render overall. The
// runtime payload can arrive before the config payload, and persisted-only
// advanced knobs (watchdog.detector.*) are absent from that early render; a
// category marked seeded before its advanced rows exist would load expanded.
// Later renders leave whatever the user toggled alone.
for (const id of categoryIds) {
if (seeded.has(id)) continue;
const hasAdvanced = (byCategory[id] || []).some(key => {
const row = rows[key];
return (row.runtimeItem && row.runtimeItem.advanced) ||
(row.persistedItem && row.persistedItem.advanced);
});
if (!hasAdvanced) continue;
seeded.add(id);
collapsed.add(id + '.adv');
}
}
/* SEED_ADV_COLLAPSED:END */
/* One knob table replaces the old runtime-list/config-list div columns.
Runtime rows and persisted rows pair by the dotted config key the
/admin/api/runtime payload carries in `config_key` -- the join lives here,
@@ -858,7 +949,7 @@ function persistedControlHtml(key, entry, dirtyValue) {
return `${control}${hints ? `<span class="d-inline-flex align-items-center gap-1 ms-2">${hints}</span>` : ''}`;
}
function knobRowHtml(row) {
function knobRowHtml(row, catId, isAdv) {
const rt = row.runtimeItem;
const entry = row.persistedItem;
const keyCell = row.drift
@@ -869,7 +960,7 @@ function knobRowHtml(row) {
: '<span class="text-muted small">-</span>';
let persistedCell = '<span class="text-muted small">-</span>';
let layerCell = '<td class="setting-meta"></td>';
let dirtyAttrs = '';
let extraAttrs = '';
if (entry) {
persistedCell = persistedControlHtml(row.configKey, entry, row.dirtyValue);
// Provenance is the layer column: where the effective value came from.
@@ -880,9 +971,11 @@ function knobRowHtml(row) {
layer = '<span class="badge bg-secondary" title="from config.yaml">base</span>';
}
layerCell = `<td class="setting-meta">${layer}</td>`;
dirtyAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`;
extraAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`;
}
return `<tr class="setting-row"${dirtyAttrs}>
if (catId) extraAttrs += ` data-cat="cat-${catId}"`;
if (isAdv) extraAttrs += ' data-adv="true"';
return `<tr class="setting-row"${extraAttrs}>
<td class="setting-key">${keyCell}</td>
<td>${liveCell}</td>
<td>${persistedCell}</td>
@@ -917,13 +1010,155 @@ function renderKnobsTable() {
list.innerHTML = '<tr><td colspan="4" class="text-muted fst-italic small px-2 py-1">No knobs to show</td></tr>';
return;
}
list.innerHTML = keys.map(key => {
// Group rows by category using the category/advanced metadata from the API.
const byCategory = {};
const uncategorized = [];
for (const key of keys) {
const row = rows[key];
row.dirtyValue = row.persistedItem ? dirtyInputs[key] : undefined;
return knobRowHtml(row);
}).join('');
const cat = (row.runtimeItem && row.runtimeItem.category) ||
(row.persistedItem && row.persistedItem.category) || '';
if (!cat) { uncategorized.push(key); continue; }
if (!byCategory[cat]) byCategory[cat] = [];
byCategory[cat].push(key);
}
const html = [];
// Advanced subsections load collapsed, so the user sees the clean knob list
// before the Advanced expander. Seeded per category, once.
seedAdvancedCollapsed(byCategory, rows, CATEGORIES.map(c => c.id),
_advSeededCategories, collapsedCategories);
for (const cat of CATEGORIES) {
const catKeys = byCategory[cat.id] || [];
if (!catKeys.length) continue;
const isCatCollapsed = collapsedCategories.has(cat.id + '.collapsed');
// Category header with chevron toggle, rendered via data-icon for
// renderStaticIcons(). All 4 columns spanned so the click target
// is the full table width.
html.push(`<tr class="cat-header" data-cat="cat-${cat.id}">
<th colspan="4" id="cat-${cat.id}" class="cat-header-cell" onclick="toggleCategory('${cat.id}')">
<span class="d-inline-flex align-items-center gap-2">
<span data-icon="chevronDown" class="cat-chevron${isCatCollapsed ? '' : ' cat-chevron-open'}"></span>
<span data-icon="${cat.icon}" class="cat-icon"></span>
<span class="cat-label">${cat.label}</span>
</span>
</th>
</tr>`);
// Split into standard and advanced rows.
const standard = [];
const advanced = [];
for (const key of catKeys) {
const row = rows[key];
const isAdv = (row.runtimeItem && row.runtimeItem.advanced) ||
(row.persistedItem && row.persistedItem.advanced) || false;
(isAdv ? advanced : standard).push(key);
}
// Standard (non-advanced) rows
for (const key of standard) {
html.push(knobRowHtml(rows[key], cat.id, false));
}
// Advanced sub-header + rows (default collapsed on first render)
if (advanced.length) {
const advKey = cat.id + '.adv';
const isAdvCollapsed = collapsedCategories.has(advKey);
html.push(`<tr class="adv-subheader" data-cat="cat-${cat.id}" data-adv-subheader="${cat.id}">
<th colspan="4" class="adv-subheader-cell" onclick="toggleAdvanced('${cat.id}')">
<span class="d-inline-flex align-items-center gap-2 text-muted">
<span data-icon="chevronDown" class="cat-chevron${isAdvCollapsed ? '' : ' cat-chevron-open'}"></span>
<span class="small">Advanced</span>
</span>
</th>
</tr>`);
for (const key of advanced) {
html.push(knobRowHtml(rows[key], cat.id, true));
}
}
}
// Uncategorized rows at the end (fallback for knob metadata edge cases).
for (const key of uncategorized) {
html.push(knobRowHtml(rows[key], '', false));
}
list.innerHTML = html.join('');
// Apply collapsed state to category rows after rendering.
for (const cat of CATEGORIES) {
if (collapsedCategories.has(cat.id + '.collapsed')) {
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"]:not(.cat-header)`);
rows.forEach(r => { r.style.display = 'none'; });
}
if (collapsedCategories.has(cat.id + '.adv')) {
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"][data-adv="true"]`);
rows.forEach(r => { r.style.display = 'none'; });
}
}
// Re-render data-icon attributes that were just injected.
renderStaticIcons();
updateDefaultProfileHint();
markDirty();
// If the URL carries a category anchor, expand that section now.
if (window.location.hash && window.location.hash.startsWith('#cat-')) {
const hashCat = window.location.hash.replace('#cat-', '');
if (collapsedCategories.has(hashCat + '.collapsed')) {
collapsedCategories.delete(hashCat + '.collapsed');
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${hashCat}"]:not(.cat-header)`);
rows.forEach(r => { r.style.display = ''; });
const chevron = document.querySelector(`#config-list tr[data-cat="cat-${hashCat}"].cat-header .cat-chevron`);
if (chevron) chevron.classList.add('cat-chevron-open');
}
}
}
// Toggle a category section: collapse/expand all knob rows within it.
function toggleCategory(catId) {
const key = catId + '.collapsed';
if (collapsedCategories.has(key)) {
collapsedCategories.delete(key);
} else {
collapsedCategories.add(key);
}
const isCollapsed = collapsedCategories.has(key);
// All knob rows + advanced sub-header under this category.
const rows = document.querySelectorAll(
`#config-list tr[data-cat="cat-${catId}"]:not(.cat-header)`
);
rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; });
// Flip the chevron on the category header.
const chevron = document.querySelector(
`#config-list tr[data-cat="cat-${catId}"].cat-header .cat-chevron`
);
if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed);
}
// Toggle an advanced subsection within a category: collapse/expand only the
// rows flagged data-adv="true" under that category.
function toggleAdvanced(catId) {
const key = catId + '.adv';
if (collapsedCategories.has(key)) {
collapsedCategories.delete(key);
} else {
collapsedCategories.add(key);
}
const isCollapsed = collapsedCategories.has(key);
const rows = document.querySelectorAll(
`#config-list tr[data-cat="cat-${catId}"][data-adv="true"]`
);
rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; });
// Flip the chevron on the advanced sub-header.
const chevron = document.querySelector(
`#config-list tr[data-adv-subheader="${catId}"] .cat-chevron`
);
if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed);
}
function renderRuntime(state) {

View File

@@ -145,6 +145,39 @@ land in `config/config.local.yaml`; see
[config-local-overlay](config-local-overlay.md)). Changes are marked as dirty
and written only on save.
#### Knob organization
The generic knob table is grouped into **six categories**, each with a
collapsible section header:
1. **Routing / Quality / Cost** — the core dispatch objective: logging
levels, energy ceiling, plan quota, cache pricing, and default profile.
2. **Classification** — classifier runtime scalars such as cooldown,
fallback tier/category, degraded warning thresholds, and context framing.
3. **Local Hardware** — gates over local compute paths: the local-compute
master switch, local verification, and local vision fallback.
4. **Caching / Context** — session classification cache and context pruning
(pinch).
5. **Safety Nets** — circuit breaker.
6. **Watchdog** — response-loop detector thresholds.
Within each category, knobs marked **Advanced** hide behind a secondary
"Advanced" accordion. These are shape parameters internal to a covered master
switch (for example, pinch relevance options and watchdog detector
thresholds). The accordion keeps the first view short without removing access.
Knobs reach the page through **three coverage sources**:
- **Registry knobs** — runtime toggles from `_BOOL_KNOBS`, `_FLOAT_KNOBS`, and
`_INT_KNOBS` in `admin.py`. They take effect immediately in memory and
revert on restart.
- **Allowlist knobs** — the same scalars persisted through
`_CONFIG_ALLOWLIST`, written to `config/config.local.yaml` and surviving a
restart.
- **Card-backed paths** — knobs with their own dedicated admin card and
endpoint pair because they have cross-field structure a flat scalar input
cannot safely represent.
It carries three dedicated cards, none a row in the generic runtime-knob
list, because each has cross-field structure a flat scalar/boolean input
can't safely represent:

116
plans/deferred-knobs.md Normal file
View File

@@ -0,0 +1,116 @@
# Deferred admin portal knobs
Status: reference -- Tracker for knobs outside current admin gate
This file tracks config knobs that are outside the admin portal's current
gate. The coverage test in `tests/test_admin_knob_coverage.py` enforces that
every knob in a section the portal reaches is either covered or excused with a
reason. Sections the portal has not reached are out of scope for that test,
but they still need a recorded home so the first control added under a section
does not leave half its siblings orphaned.
Rows use four columns:
| Column | Meaning |
|---|---|
| Knob | Dotted config path in `config/config.yaml`. |
| Section | Top-level config section. |
| Disposition | `deferred` (future admin page) or `excused-with-reason` (deployment wiring or file path). |
| Future Home | The page or surface that should own this knob when the portal reaches the section. |
## Outside-gate knobs
These are `RouterConfig` scalars in sections the portal has not reached yet.
They fall into two buckets:
- Sections not yet reached: `context`, `database`, `dispatch_settings`,
`escalation`, `exploration`, `freshness`, `iteration`, `local_energy`,
`proficiency`, `tiering`.
- Deployment wiring inside those sections: endpoints, model ids, credentials,
filesystem paths, and device names.
| Knob | Section | Disposition | Future Home |
|---|---|---|---|
| `context.default_output_reserve_tokens` | context | deferred | Context page |
| `context.max_output_reserve_fraction` | context | deferred | Context page |
| `context.safety_factor` | context | deferred | Context page |
| `database.path` | database | excused-with-reason | N/A (file path) |
| `dispatch_settings.default_provider` | dispatch_settings | deferred | Providers page |
| `escalation.enabled` | escalation | deferred | Routing page |
| `escalation.max_tier` | escalation | deferred | Routing page |
| `escalation.min_confidence_before_bump` | escalation | deferred | Routing page |
| `escalation.preemptive_on_low_confidence` | escalation | deferred | Routing page |
| `exploration.enabled` | exploration | deferred | Routing page |
| `exploration.epsilon` | exploration | deferred | Routing page |
| `exploration.max_cost_ratio` | exploration | deferred | Routing page |
| `exploration.max_tier` | exploration | deferred | Routing page |
| `freshness.exclude_deprecated` | freshness | deferred | Catalog page |
| `freshness.exclude_stale` | freshness | deferred | Catalog page |
| `freshness.repoll_after_allowlist_change_seconds` | freshness | deferred | Catalog page |
| `freshness.stale_after_days` | freshness | deferred | Catalog page |
| `iteration.enabled` | iteration | deferred | Routing page |
| `iteration.max_attempts_interactive` | iteration | deferred | Routing page |
| `iteration.max_rebill_prompt_tokens` | iteration | deferred | Routing page |
| `local_energy.enabled` | local_energy | deferred | Local Hardware page |
| `local_energy.grid_intensity_g_per_kwh` | local_energy | deferred | Local Hardware page |
| `local_energy.meter` | local_energy | excused-with-reason | N/A (deployment wiring) |
| `local_energy.sample_interval_seconds` | local_energy | deferred | Local Hardware page |
| `local_energy.tariff_usd_per_kwh` | local_energy | deferred | Local Hardware page |
| `proficiency.leaderboard_weight` | proficiency | deferred | Proficiency page |
| `proficiency.outcome_prior_strength` | proficiency | deferred | Proficiency page |
| `proficiency.self_eval_min_samples` | proficiency | deferred | Proficiency page |
| `proficiency.self_eval_weight` | proficiency | deferred | Proficiency page |
| `tiering.cheap_completion_max` | tiering | deferred | Routing page |
| `tiering.tier1_context_max` | tiering | deferred | Routing page |
## Objective warning knobs
These 31 `objective.*` knobs are currently excused from the admin portal in
`tests/test_admin_knob_coverage.py`. They tune `/metrics` warnings and report
series rather than dispatch behavior. Their natural future home is a
**Warnings page** that collects all alarm and report thresholds in one place.
| Knob | Section | Disposition | Future Home |
|---|---|---|---|
| `objective.adoption_window_seconds` | objective | deferred | Warnings page |
| `objective.assumed_cache_rate` | objective | deferred | Warnings page |
| `objective.assumed_completion_tokens` | objective | deferred | Warnings page |
| `objective.billing_reset_day` | objective | deferred | Warnings page |
| `objective.cache_rate_warn_margin` | objective | deferred | Warnings page |
| `objective.cache_rate_warn_min_observations` | objective | deferred | Warnings page |
| `objective.cache_rate_window_hours` | objective | deferred | Warnings page |
| `objective.cost_calibration_min_observations` | objective | deferred | Warnings page |
| `objective.cost_calibration_window_hours` | objective | deferred | Warnings page |
| `objective.credit_attenuation.enabled` | objective | deferred | Warnings page |
| `objective.credit_attenuation.max_multiplier` | objective | deferred | Warnings page |
| `objective.credit_attenuation.refresh_seconds` | objective | deferred | Warnings page |
| `objective.credit_attenuation.soft_floor_usd` | objective | deferred | Warnings page |
| `objective.credit_attenuation.zero_floor_usd` | objective | deferred | Warnings page |
| `objective.cumulative_spend_warn_min_rows` | objective | deferred | Warnings page |
| `objective.cumulative_spend_warn_usd` | objective | deferred | Warnings page |
| `objective.incumbent_rate_min_observations` | objective | deferred | Warnings page |
| `objective.incumbent_rate_refresh_seconds` | objective | deferred | Warnings page |
| `objective.latency_min_observations` | objective | deferred | Warnings page |
| `objective.latency_window_hours` | objective | deferred | Warnings page |
| `objective.plan_pace_warn_ratio` | objective | deferred | Warnings page |
| `objective.proficiency_depth_warn_min_rows` | objective | deferred | Warnings page |
| `objective.proficiency_depth_warn_min_samples` | objective | deferred | Warnings page |
| `objective.quota_burn_min_segment_hours` | objective | deferred | Warnings page |
| `objective.quota_burn_min_segment_samples` | objective | deferred | Warnings page |
| `objective.quota_burn_window_hours` | objective | deferred | Warnings page |
| `objective.quota_runway_warning_hours` | objective | deferred | Warnings page |
| `objective.rejection_warning_baseline_hours` | objective | deferred | Warnings page |
| `objective.rejection_warning_min_count` | objective | deferred | Warnings page |
| `objective.rejection_warning_window_hours` | objective | deferred | Warnings page |
| `objective.selection_coverage_window_hours` | objective | deferred | Warnings page |
## Notes
- A section moves from "outside gate" to "in scope" the moment the portal adds
its first control there. At that point every scalar under it must be covered,
excused, or removed from this tracker into `DELIBERATELY_NOT_IN_ADMIN`.
- Deployment wiring (endpoints, model ids, credentials, paths, devices) is
deliberately off the admin allowlist. Those knobs should stay
`excused-with-reason` even after their section is reached.
- The Warnings page is not built yet. When it is, these `objective.*` knobs
should move from `DELIBERATELY_NOT_IN_ADMIN` into registry or allowlist
entries, and this table should shrink.

View File

@@ -46,6 +46,14 @@ import poller
import routing
from config import (
BUILTIN_PROFILES,
COOLDOWN_CEILING,
COOLDOWN_FLOOR,
DEGRADED_WARN_MIN_CEILING,
DEGRADED_WARN_MIN_FLOOR,
DEGRADED_WARN_THRESHOLD_FLOOR,
DEGRADED_WARN_THRESHOLD_MAX,
FALLBACK_TIER_MAX,
FALLBACK_TIER_MIN,
DispatchProvider,
FlexPreference,
RouterConfig,
@@ -356,6 +364,7 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = {
"pinch_enabled": ("pinch", "enabled"),
"pinch_prefix_probe": ("pinch", "prefix_probe"),
"pinch_relevance_enabled": ("pinch", "relevance", "enabled"),
"classifier_context_framing": ("classifier", "context_framing"),
"incumbent_cache_pricing": ("objective", "incumbent_cache_pricing"),
}
@@ -384,14 +393,20 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = {
# representations" would stop holding the moment this endpoint could write a
# None onto cfg.
#
# knob -> (cfg path, low, high, path whose value ``null`` resolves to)
_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, tuple[str, ...]]] = {
# knob -> (cfg path, low, high, path whose value ``null`` resolves to, if any)
_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, Optional[tuple[str, ...]]]] = {
"incumbent_challenger_cache_rate": (
("objective", "incumbent_challenger_cache_rate"),
0.0,
1.0,
("objective", "assumed_cache_rate"),
),
"classifier_degraded_warn_threshold": (
("classifier", "degraded_warn_threshold"),
DEGRADED_WARN_THRESHOLD_FLOOR,
DEGRADED_WARN_THRESHOLD_MAX,
None,
),
}
# Runtime knobs whose value is a whole number. A SEPARATE table from
@@ -427,6 +442,21 @@ _INT_KNOBS: dict[str, tuple[tuple[str, ...], int, int]] = {
STALENESS_SECONDS_MIN,
STALENESS_SECONDS_MAX,
),
"classifier_cooldown_seconds": (
("classifier", "cooldown_seconds"),
COOLDOWN_FLOOR,
COOLDOWN_CEILING,
),
"classifier_fallback_tier": (
("classifier", "fallback_tier"),
FALLBACK_TIER_MIN,
FALLBACK_TIER_MAX,
),
"classifier_degraded_warn_min": (
("classifier", "degraded_warn_min"),
DEGRADED_WARN_MIN_FLOOR,
DEGRADED_WARN_MIN_CEILING,
),
}
_LOCAL_COMPUTE_KNOB = "local_compute_enabled"
@@ -489,6 +519,87 @@ _RUNTIME_KNOB_PATHS: dict[str, tuple[str, ...]] = {
_PROFILE_KNOB: _PROFILE_PATH,
}
# --- knob category taxonomy ----------------------------------------------------
# Every covered knob, keyed by its dotted config path, with its category id and
# whether it belongs behind the Advanced accordion. Six categories:
# routing_quality_cost, classification, local_hardware,
# caching_context, safety_nets, watchdog.
# Used by the admin API to add category + advanced to knob metadata, and by the
# frontend to group knobs into sectioned accordions.
# 28 existing + 7 new classifier knobs = 35 entries total.
_KNOB_CATEGORY: dict[str, dict] = {
# -------- routing_quality_cost (10) -------------------------------------------
"logging.log_route_decisions": {"category": "routing_quality_cost", "advanced": False},
"logging.log_energy_observations": {"category": "routing_quality_cost", "advanced": False},
"logging.level": {"category": "routing_quality_cost", "advanced": False},
"objective.quality_tolerance": {"category": "routing_quality_cost", "advanced": False},
"objective.max_energy_per_request": {"category": "routing_quality_cost", "advanced": False},
"objective.plan_kwh_per_period": {"category": "routing_quality_cost", "advanced": False},
"objective.incumbent_cache_pricing": {"category": "routing_quality_cost", "advanced": False},
"objective.incumbent_challenger_cache_rate": {"category": "routing_quality_cost", "advanced": False},
"routing.default_flex_preference": {"category": "routing_quality_cost", "advanced": False},
"routing.default_profile": {"category": "routing_quality_cost", "advanced": False},
# -------- classification (7) --------------------------------------------------
"classifier.context_framing": {"category": "classification", "advanced": False},
"classifier.cooldown_seconds": {"category": "classification", "advanced": False},
"classifier.fallback_tier": {"category": "classification", "advanced": False},
"classifier.degraded_warn_min": {"category": "classification", "advanced": False},
"classifier.degraded_warn_threshold": {"category": "classification", "advanced": False},
"classifier.fallback_category": {"category": "classification", "advanced": False},
"classifier.max_input_chars": {"category": "classification", "advanced": False},
# -------- safety_nets (1) -----------------------------------------------------
"circuit_breaker.enabled": {"category": "safety_nets", "advanced": False},
# -------- caching_context (5) -------------------------------------------------
"session_cache.enabled": {"category": "caching_context", "advanced": False},
"session_cache.staleness_seconds": {"category": "caching_context", "advanced": False},
"pinch.enabled": {"category": "caching_context", "advanced": False},
"pinch.prefix_probe": {"category": "caching_context", "advanced": True},
"pinch.relevance.enabled": {"category": "caching_context", "advanced": True},
# -------- local_hardware (3) --------------------------------------------------
"local_compute.enabled": {"category": "local_hardware", "advanced": False},
"verification.local_llm_enabled": {"category": "local_hardware", "advanced": False},
"local_vision.enabled": {"category": "local_hardware", "advanced": False},
# -------- watchdog (9) --------------------------------------------------------
"watchdog.enabled": {"category": "watchdog", "advanced": False},
"watchdog.local_llm_enabled": {"category": "watchdog", "advanced": False},
"watchdog.detector.window": {"category": "watchdog", "advanced": True},
"watchdog.detector.dup_min": {"category": "watchdog", "advanced": True},
"watchdog.detector.top_min": {"category": "watchdog", "advanced": True},
"watchdog.detector.top_min_ro": {"category": "watchdog", "advanced": True},
"watchdog.detector.cum_min": {"category": "watchdog", "advanced": True},
"watchdog.detector.cover_min": {"category": "watchdog", "advanced": True},
"watchdog.detector.min_calls": {"category": "watchdog", "advanced": True},
}
# --- card-backed paths -------------------------------------------------------------
# Config keys whose controls live in their own dedicated admin card (classifier
# mode selector, cloud primary, cloud fallback, encoder, decision) rather than
# in the generic knob table. Each maps to the HTML element id the frontend
# uses for it, so the test can verify the id still exists in controls.html.
_CARD_BACKED_PATHS: dict[str, str] = {
"classifier.mode": "classifier-mode-select",
"classifier.cloud_primary_auto": "classifier-cloud-auto",
"classifier.cloud_primary.base_url": "classifier-cloud-base-url",
"classifier.cloud_primary.model": "classifier-cloud-model",
"classifier.cloud_primary.api_key_env": "classifier-cloud-api-key-env",
"classifier.cloud_primary.timeout_seconds": "classifier-cloud-timeout",
"classifier.cloud_fallback.base_url": "cf-base-url",
"classifier.cloud_fallback.model": "cf-model",
"classifier.cloud_fallback.api_key_env": "cf-api-key-env",
"classifier.cloud_fallback.timeout_seconds": "cf-timeout",
"classifier.cloud_fallback.max_output_tokens": "cf-max-tokens",
"classifier.encoder.model": "classifier-encoder-model",
"classifier.encoder.device": "classifier-encoder-device",
"classifier.encoder.confidence_min": "classifier-encoder-threshold",
"classifier.decision.base_url": "classifier-decision-base-url",
"classifier.decision.model": "classifier-decision-model",
"classifier.decision.num_ctx": "classifier-decision-num-ctx",
"classifier.decision.timeout_s": "classifier-decision-timeout-s",
"classifier.decision.confidence_min": "classifier-decision-confidence-min",
"classifier.decision.coverage_min": "classifier-decision-coverage-min",
"classifier.decision.tier_enabled": "classifier-decision-tier-enabled",
}
# --- persisted config allowlist -------------------------------------------------
# Dotted config.yaml paths an operator is allowed to edit. Everything else —
# classifier/verification/local_vision URLs and model names, api_key_env,
@@ -517,6 +628,13 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = {
"pinch.enabled": ("pinch", "enabled"),
"pinch.prefix_probe": ("pinch", "prefix_probe"),
"pinch.relevance.enabled": ("pinch", "relevance", "enabled"),
"classifier.context_framing": ("classifier", "context_framing"),
"classifier.cooldown_seconds": ("classifier", "cooldown_seconds"),
"classifier.fallback_tier": ("classifier", "fallback_tier"),
"classifier.degraded_warn_min": ("classifier", "degraded_warn_min"),
"classifier.degraded_warn_threshold": ("classifier", "degraded_warn_threshold"),
"classifier.fallback_category": ("classifier", "fallback_category"),
"classifier.max_input_chars": ("classifier", "max_input_chars"),
"routing.default_flex_preference": ("routing", "default_flex_preference"),
"routing.default_profile": ("routing", "default_profile"),
# Watchdog — persisted config knobs for the model-response watchdog.
@@ -533,23 +651,36 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = {
# Order preserves config.yaml layout for the GET response.
_CONFIG_GET_ORDER: list[str] = [
# routing_quality_cost
"logging.level",
"objective.quality_tolerance",
"objective.max_energy_per_request",
"objective.plan_kwh_per_period",
"objective.incumbent_cache_pricing",
"objective.incumbent_challenger_cache_rate",
"circuit_breaker.enabled",
"routing.default_flex_preference",
"routing.default_profile",
# classification
"classifier.context_framing",
"classifier.cooldown_seconds",
"classifier.fallback_tier",
"classifier.degraded_warn_min",
"classifier.degraded_warn_threshold",
"classifier.fallback_category",
"classifier.max_input_chars",
# caching_context
"session_cache.enabled",
"session_cache.staleness_seconds",
"local_compute.enabled",
"verification.local_llm_enabled",
"local_vision.enabled",
"pinch.enabled",
"pinch.prefix_probe",
"pinch.relevance.enabled",
"routing.default_flex_preference",
"routing.default_profile",
# local_hardware
"local_compute.enabled",
"verification.local_llm_enabled",
"local_vision.enabled",
# safety_nets
"circuit_breaker.enabled",
# watchdog
"watchdog.enabled",
"watchdog.local_llm_enabled",
"watchdog.detector.window",
@@ -992,6 +1123,22 @@ def _runtime_state(cfg: Any) -> dict:
"pinch_relevance_enabled": _get_at(
cfg, _BOOL_KNOBS["pinch_relevance_enabled"]
),
# --- classifier runtime toggles (Phase B) -------------------------------
"classifier_context_framing": _get_at(
cfg, _BOOL_KNOBS["classifier_context_framing"]
),
"classifier_cooldown_seconds": _get_at(
cfg, _INT_KNOBS["classifier_cooldown_seconds"][0]
),
"classifier_fallback_tier": _get_at(
cfg, _INT_KNOBS["classifier_fallback_tier"][0]
),
"classifier_degraded_warn_min": _get_at(
cfg, _INT_KNOBS["classifier_degraded_warn_min"][0]
),
"classifier_degraded_warn_threshold": _get_at(
cfg, _FLOAT_KNOBS["classifier_degraded_warn_threshold"][0]
),
# Kept adjacent and in this order on purpose: the gate reads above the
# dial it enables, which is what lets the UI note say so in four words
# instead of a paragraph.
@@ -2434,6 +2581,8 @@ def build_router(
"persisted": persisted[key],
"runtime": runtime[key],
"config_key": ".".join(_RUNTIME_KNOB_PATHS[key]),
"category": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("category"),
"advanced": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("advanced"),
}
for key in persisted
}
@@ -2455,30 +2604,36 @@ def build_router(
path, low, high, neutral_path = _FLOAT_KNOBS[knob]
value = body.value
if value is None:
# Blank means NEUTRAL, not zero. Resolved here rather than
# stored as None so cfg holds one representation of neutral,
# matching what load_config would have produced.
value = float(_get_at(cfg, neutral_path))
if neutral_path is not None:
# Blank means NEUTRAL, not zero. Resolved here rather than
# stored as None so cfg holds one representation of neutral,
# matching what load_config would have produced.
value = float(_get_at(cfg, neutral_path))
else:
# No neutral for this knob: clearing the field is not a
# setting, and _get_at(cfg, None) would 500 on the lookup.
raise HTTPException(
status_code=422,
detail=f"{knob} has no blank setting; send a number in [{low}, {high}]",
)
else:
# Only a knob with a neutral may advertise ``null``; telling
# the operator to send it for a knob that 422s it is a lie.
or_null = ", or null for neutral" if neutral_path is not None else ""
null_note = f" (null means neutral, not {low})" if neutral_path is not None else ""
# bool is a subclass of int in Python, so `True` would sail
# through an isinstance(value, (int, float)) check and land as
# 1.0 -- a real dial setting, silently, from a checkbox body.
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise HTTPException(
status_code=422,
detail=(
f"{knob} expects a number in [{low}, {high}], "
f"or null for neutral"
),
detail=f"{knob} expects a number in [{low}, {high}]{or_null}",
)
value = float(value)
if not (low <= value <= high):
raise HTTPException(
status_code=422,
detail=(
f"{knob} must be in [{low}, {high}], got {value} "
f"(null means neutral, not {low})"
),
detail=f"{knob} must be in [{low}, {high}], got {value}{null_note}",
)
_set_at(cfg, path, value)
return {"ok": True, knob: value}
@@ -2585,7 +2740,12 @@ def build_router(
source_map[key] = "base"
merged = _load_merged_config_store(config_path, config_local_path)
return {
key: {"value": _dict_get_at(merged, path), "source": source_map[key]}
key: {
"value": _dict_get_at(merged, path),
"source": source_map[key],
"category": _KNOB_CATEGORY.get(key, {}).get("category"),
"advanced": _KNOB_CATEGORY.get(key, {}).get("advanced"),
}
for key, path in _CONFIG_ALLOWLIST.items()
}

View File

@@ -1163,6 +1163,23 @@ class LocalDecisionConfig(StrictModel):
DEGRADED_WARN_MIN_FLOOR = 1
DEGRADED_WARN_THRESHOLD_MIN_EXCLUSIVE = 0.0
DEGRADED_WARN_THRESHOLD_MAX = 1.0
# Bounds for the classifier cooldown window. At least 1s, at most 1h —
# a cloud-classifier retry during a sustained local outage cannot burn
# more than one attempt per hour per process.
COOLDOWN_FLOOR = 1
COOLDOWN_CEILING = 3600
# Bounds for the fallback tier. Must correspond to a real tier in the
# routing table (tier 1 = cheap+small, tier 3 = frontier).
FALLBACK_TIER_MIN = 1
FALLBACK_TIER_MAX = 3
# Upper bound on the degradation-warning sample-size floor. A value past
# 10 000 classifications per window is a configuration mistake regardless
# of traffic volume.
DEGRADED_WARN_MIN_CEILING = 10000
# Minimum value for the degradation-warning threshold. A share below this
# fires on the very first degraded classification in the window, which is
# never useful — the operator already knows one failure happened.
DEGRADED_WARN_THRESHOLD_FLOOR = 0.001
class ClassifierConfig(StrictModel):
@@ -1265,7 +1282,35 @@ class ClassifierConfig(StrictModel):
@classmethod
def degraded_warn_min_positive(cls, v: int) -> int:
if v < DEGRADED_WARN_MIN_FLOOR:
raise ValueError("classifier.degraded_warn_min must be at least 1")
raise ValueError(
"classifier.degraded_warn_min must be at least "
f"{DEGRADED_WARN_MIN_FLOOR}, got {v}"
)
if v > DEGRADED_WARN_MIN_CEILING:
raise ValueError(
"classifier.degraded_warn_min must be at most "
f"{DEGRADED_WARN_MIN_CEILING}, got {v}"
)
return v
@field_validator("cooldown_seconds")
@classmethod
def cooldown_seconds_in_range(cls, v: int) -> int:
if not (COOLDOWN_FLOOR <= v <= COOLDOWN_CEILING):
raise ValueError(
"classifier.cooldown_seconds must be between "
f"{COOLDOWN_FLOOR} and {COOLDOWN_CEILING}, got {v}"
)
return v
@field_validator("fallback_tier")
@classmethod
def fallback_tier_in_range(cls, v: int) -> int:
if not (FALLBACK_TIER_MIN <= v <= FALLBACK_TIER_MAX):
raise ValueError(
"classifier.fallback_tier must be between "
f"{FALLBACK_TIER_MIN} and {FALLBACK_TIER_MAX}, got {v}"
)
return v

View File

@@ -106,11 +106,13 @@ def test_config_GET_returns_allowlisted_values(client):
"routing.default_flex_preference",
):
assert key in body
assert body["logging.level"] == {"value": "info", "source": "base"}
assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base"}
assert body["logging.level"] == {"value": "info", "source": "base", "category": "routing_quality_cost", "advanced": False}
assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base", "category": "routing_quality_cost", "advanced": False}
assert body["routing.default_flex_preference"] == {
"value": "auto",
"source": "base",
"category": "routing_quality_cost",
"advanced": False,
}
@@ -154,14 +156,14 @@ def test_config_pinch_prefix_probe_round_trips_through_the_real_validator(
local_yaml = config_yaml.with_name("config.local.yaml")
before = tc.get("/admin/api/config").json()
assert before["pinch.prefix_probe"] == {"value": True, "source": "base"}
assert before["pinch.prefix_probe"] == {"value": True, "source": "base", "category": "caching_context", "advanced": True}
resp = tc.post("/admin/api/config/pinch.prefix_probe", json={"value": False})
assert resp.status_code == 200
assert resp.json()["value"] is False
after = tc.get("/admin/api/config").json()
assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay"}
assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay", "category": "caching_context", "advanced": True}
# The base file never carries an admin write.
assert yaml.safe_load(config_yaml.read_text())["pinch"]["prefix_probe"] is True
@@ -200,14 +202,14 @@ def test_config_local_vision_enabled_round_trips_through_the_real_validator(clie
local_yaml = config_yaml.with_name("config.local.yaml")
before = tc.get("/admin/api/config").json()
assert before["local_vision.enabled"] == {"value": True, "source": "base"}
assert before["local_vision.enabled"] == {"value": True, "source": "base", "category": "local_hardware", "advanced": False}
resp = tc.post("/admin/api/config/local_vision.enabled", json={"value": False})
assert resp.status_code == 200
assert resp.json()["value"] is False
after = tc.get("/admin/api/config").json()
assert after["local_vision.enabled"] == {"value": False, "source": "overlay"}
assert after["local_vision.enabled"] == {"value": False, "source": "overlay", "category": "local_hardware", "advanced": False}
# The base file never carries an admin write.
assert yaml.safe_load(config_yaml.read_text())["local_vision"]["enabled"] is True
@@ -239,12 +241,16 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client):
assert before["objective.incumbent_cache_pricing"] == {
"value": False,
"source": "base",
"category": "routing_quality_cost",
"advanced": False,
}
# Shipped blank in config.yaml, which YAML reads as null.
assert before["objective.incumbent_challenger_cache_rate"] == {
"value": None,
"source": "base",
}
"value": None,
"source": "base",
"category": "routing_quality_cost",
"advanced": False,
}
assert (
tc.post(
@@ -265,11 +271,15 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client):
assert after["objective.incumbent_cache_pricing"] == {
"value": True,
"source": "overlay",
"category": "routing_quality_cost",
"advanced": False,
}
assert after["objective.incumbent_challenger_cache_rate"] == {
"value": 0.0,
"source": "overlay",
}
"value": 0.0,
"source": "overlay",
"category": "routing_quality_cost",
"advanced": False,
}
# The base file never carries an admin write.
base = yaml.safe_load(config_yaml.read_text())["objective"]
@@ -325,6 +335,8 @@ def test_config_blank_challenger_dial_persists_as_null_not_zero(client):
assert body["objective.incumbent_challenger_cache_rate"] == {
"value": None,
"source": "overlay",
"category": "routing_quality_cost",
"advanced": False,
}
loaded = load_config(str(config_yaml), include_overlay=True)
@@ -382,7 +394,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client):
tc, config_yaml = client
before = tc.get("/admin/api/config").json()
assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base"}
assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base", "category": "caching_context", "advanced": False}
assert (
tc.post(
@@ -393,7 +405,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client):
)
after = tc.get("/admin/api/config").json()
assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay"}
assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay", "category": "caching_context", "advanced": False}
# The shipped default never moves; the admin write lands in the overlay.
base = yaml.safe_load(config_yaml.read_text())["session_cache"]
@@ -476,6 +488,8 @@ def test_config_GET_includes_routing_default_profile(client):
assert body["routing.default_profile"] == {
"value": "default",
"source": "base",
"category": "routing_quality_cost",
"advanced": False,
}

View File

@@ -460,6 +460,95 @@ def test_merge_knob_rows_value_normalization(persisted, runtime, expect_drift):
)
SEED_ADV_COLLAPSED_BEGIN = "/* SEED_ADV_COLLAPSED:BEGIN */"
SEED_ADV_COLLAPSED_END = "/* SEED_ADV_COLLAPSED:END */"
def _assert_against_seed_units(assertion_js: str) -> subprocess.CompletedProcess[str]:
"""Assert ``assertion_js`` against the real seedAdvancedCollapsed source.
The function lives in controls.html between the SEED_ADV_COLLAPSED markers
and is self-contained: it takes the grouped rows and the two Sets as
arguments and touches no DOM. ``group`` below mirrors the grouping step in
renderKnobsTable so each test can drive it payload by payload.
"""
return _run_node(
"const assert = require('assert');\n"
+ _extract_between(CONTROLS_HTML, SEED_ADV_COLLAPSED_BEGIN, SEED_ADV_COLLAPSED_END)
+ "\n"
"const IDS = ['caching_context', 'watchdog'];\n"
"function group(rows) {\n"
" const byCategory = {};\n"
" for (const [key, row] of Object.entries(rows)) {\n"
" const item = row.runtimeItem || row.persistedItem;\n"
" (byCategory[item.category] = byCategory[item.category] || []).push(key);\n"
" }\n"
" return byCategory;\n"
"}\n"
"function render(rows, seeded, collapsed) {\n"
" seedAdvancedCollapsed(group(rows), rows, IDS, seeded, collapsed);\n"
"}\n"
# First render: runtime payload only. Watchdog has a plain runtime knob
# and NO advanced row (its advanced knobs are persisted-only);
# Caching has an advanced runtime knob.
"const runtimeOnly = {\n"
" 'session_cache.staleness_seconds': { runtimeItem: { category: 'caching_context', advanced: true }, persistedItem: null },\n"
" 'watchdog.enabled': { runtimeItem: { category: 'watchdog', advanced: false }, persistedItem: null },\n"
"};\n"
# Second render: the config payload lands and adds the persisted-only
# advanced watchdog knobs.
"const withConfig = {\n"
" ...runtimeOnly,\n"
" 'watchdog.detector.dup_min': { runtimeItem: null, persistedItem: { category: 'watchdog', advanced: true } },\n"
"};\n"
+ assertion_js
)
@skip_without_node
def test_seed_advanced_collapsed_seeds_each_category_when_its_advanced_rows_first_appear():
"""A config-only render after a runtime-only render must leave the Watchdog
Advanced section collapsed. Seeding once on the first render marked
Watchdog seeded while it had no advanced rows, so it loaded EXPANDED."""
_assert_against_seed_units(
"const seeded = new Set(), collapsed = new Set();\n"
"render(runtimeOnly, seeded, collapsed);\n"
"assert.deepStrictEqual([...collapsed], ['caching_context.adv']);\n"
"assert.ok(!seeded.has('watchdog'), 'no advanced rows yet: watchdog must stay unseeded');\n"
"render(withConfig, seeded, collapsed);\n"
"assert.deepStrictEqual([...collapsed].sort(), ['caching_context.adv', 'watchdog.adv']);"
)
@skip_without_node
def test_seed_advanced_collapsed_leaves_a_user_toggled_section_alone():
"""Once a category is seeded, a toggled-open section stays open across
every later re-render (the page re-renders on a 30s poll)."""
_assert_against_seed_units(
"const seeded = new Set(), collapsed = new Set();\n"
"render(runtimeOnly, seeded, collapsed);\n"
"render(withConfig, seeded, collapsed);\n"
"collapsed.delete('watchdog.adv');\n"
"collapsed.delete('caching_context.adv');\n"
"render(withConfig, seeded, collapsed);\n"
"render(withConfig, seeded, collapsed);\n"
"assert.strictEqual(collapsed.size, 0);"
)
@skip_without_node
def test_seed_advanced_collapsed_ignores_a_category_with_no_advanced_rows():
"""A category with only plain knobs gets no Advanced entry and is never
marked seeded, so an advanced row arriving later is still seeded."""
_assert_against_seed_units(
"const seeded = new Set(), collapsed = new Set();\n"
"const plain = { 'watchdog.enabled': runtimeOnly['watchdog.enabled'] };\n"
"render(plain, seeded, collapsed);\n"
"assert.strictEqual(collapsed.size, 0);\n"
"assert.strictEqual(seeded.size, 0);"
)
INDEX_HTML = ROOT / "admin" / "frontend" / "index.html"
LIFT_A2_HELPERS_BEGIN = "/* LIFT_A2_HELPERS:BEGIN */"
LIFT_A2_HELPERS_END = "/* LIFT_A2_HELPERS:END */"

View File

@@ -12,10 +12,10 @@ So this file is the forcing function, in the shape ``test_tui_schema_drift`` and
message that NAMES the thing, so whoever broke it learns what they broke without
reading the test.
- covered — ``admin._CONFIG_ALLOWLIST`` (persisted to the overlay) union the
runtime registries ``_BOOL_KNOBS``, ``_FLOAT_KNOBS`` and ``_INT_KNOBS``
(in-memory, reverts on restart). Derived, never hand-copied: a hand-copy is
one more thing to drift.
- covered — ``admin._CONFIG_ALLOWLIST`` (persisted), the runtime registries
``_BOOL_KNOBS``, ``_FLOAT_KNOBS``, ``_INT_KNOBS`` (in-memory), and
``_CARD_BACKED_PATHS`` (dedicated admin cards). Derived, never hand-copied:
a hand-copy is one more thing to drift.
- ``DELIBERATELY_NOT_IN_ADMIN`` — in-scope knobs with no control, each with a
REASON STRING. The escape clause is load-bearing rather than hedging:
``objective.credit_attenuation.enabled`` is deliberately off the allowlist
@@ -50,10 +50,19 @@ against. Two clauses cut it to the knobs an operator would plausibly turn:
own dedicated admin card and endpoint pair (``/admin/api/classifier-config``,
``/admin/api/cloud-fallback-config``), so measuring it against the GENERIC
allowlist would report every field as missing while the card covers it.
``local_vision`` is reached now that ``local_vision.enabled`` has a persisted
control; the rest of the section (``base_url``, ``api_key_env``, ``model``)
is deployment wiring out by clause 2, and the timeout/image limits are
excused in ``DELIBERATELY_NOT_IN_ADMIN``.
``local_vision`` is reached now that ``local_vision.enabled`` has a persisted
control; the rest of the section (``base_url``, ``api_key_env``, ``model``)
is deployment wiring out by clause 2, and the timeout/image limits are
excused in ``DELIBERATELY_NOT_IN_ADMIN``.
``classifier`` has three coverage sources. **Registry knobs** (``_BOOL_KNOBS``,
``_INT_KNOBS``, ``_FLOAT_KNOBS``): runtime toggles like context_framing,
cooldown_seconds and fallback_tier. **Allowlist knobs** (``_CONFIG_ALLOWLIST``):
the same seven persisted to the overlay for restart durability. **Card-backed
paths** (``_CARD_BACKED_PATHS``): knobs with their own dedicated admin card
and endpoint pair (classifier mode, cloud primary, cloud fallback, encoder
and decision fields). The remaining classifier scalars (timeout_seconds,
temperature, max_output_tokens and encoder.tier_from_features) are wiring or
reproducibility settings excused by name in ``DELIBERATELY_NOT_IN_ADMIN``.
The known limitation: a brand-new section with no control at all is out of
scope and unchecked. The first control added under it drags every one of its
knobs into scope at once, which is the intended moment to decide.
@@ -347,6 +356,34 @@ DELIBERATELY_NOT_IN_ADMIN: dict[str, str] = {
"lookback for the conversation adoption counter in /metrics; a "
"read-only window that shapes a report, not a routing dial."
),
# --- classifier: wiring with no control or reproducibility scalars -------
"classifier.timeout_seconds": (
"deployment tuning paired with model; the operator's decision is "
"which classifier fits the deployment, and the timeout follows it."
),
"classifier.temperature": (
"reproducibility, not tunable (user Q6); per-request temperature 0 "
"is what makes the classification deterministic, and deviating from "
"it in a form would produce nondeterministic routing decisions."
),
"classifier.max_output_tokens": (
"safety cap against reasoning cascade (user Q6); a classifier that "
"spends its generation budget on reasoning returns no parsible JSON, "
"and the cascade catches that silently -- widening the cap from the "
"UI would make the hidden failure more expensive, not fix it."
),
"classifier.cloud_primary.max_output_tokens": (
"not an input in the card UI (encoder-level setting); only meaningful "
"when mode is cloud_llm and cloud_primary_auto is off, and even then "
"it is a cap inherited from the card backend, not a knob the mode "
"selector exposes for independent tuning."
),
"classifier.encoder.tier_from_features": (
"design sketch only, not wired; the field and its companion "
"tier_feature_fields exist in the Pydantic model as a placeholder "
"for a future feature that is not implemented anywhere outside "
"config.py -- no admin control because it would control nothing real."
),
# --- deployment wiring: set in config, never in admin --------------------
"watchdog.dashboard_base_url": (
"deployment wiring, set in config, not admin"
@@ -412,12 +449,16 @@ def _covered_paths() -> set[str]:
reading only ``_BOOL_KNOBS`` would report a runtime-only numeric knob as
having no control at all — a false failure that invites exactly the wrong
fix, an excuse entry for a knob that is in fact covered.
Card-backed paths (``_CARD_BACKED_PATHS``) are also covered: each maps a
config key to an HTML element id on a dedicated admin card.
"""
return (
set(admin._CONFIG_ALLOWLIST)
| {".".join(path) for path in admin._BOOL_KNOBS.values()}
| {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()}
| {".".join(entry[0]) for entry in admin._INT_KNOBS.values()}
| set(admin._CARD_BACKED_PATHS)
)
@@ -470,22 +511,53 @@ def test_every_in_scope_knob_is_reachable_or_deliberately_not():
)
def test_excused_knobs_are_not_already_covered():
"""Exactness: the excuse list must not accumulate knobs that HAVE controls.
def test_card_backed_gate_is_real():
"""Prove the gate actually uses _CARD_BACKED_PATHS by removing one entry.
Without this the list silently rots into a rubber stamp as knobs quietly
gain controls, and the next reader cannot tell which entries still describe
reality. Expected to fire on the two PENDING entries the moment
feat/admin-incumbent-knobs merges — that is the intended cleanup signal.
Temporarily remove a card-backed path that is NOT in the excuse list,
run the coverage check, verify it fails naming that leaf, then restore it.
"""
import admin as admin_mod
original = dict(admin_mod._CARD_BACKED_PATHS)
# Pick a card-backed path not already excused (classifier.mode is excused
# but still card-backed — pick one we deleted from the excuse list).
test_key = None
for key in original:
if key not in DELIBERATELY_NOT_IN_ADMIN:
test_key = key
break
if test_key is None:
# Fallback: any card-backed path will do
test_key = next(iter(original))
try:
del admin_mod._CARD_BACKED_PATHS[test_key]
scalars_set = set(_scalars())
in_scope = _in_scope(sorted(scalars_set))
covered = _covered_paths()
undecided = [
p for p in in_scope
if p not in covered and p not in DELIBERATELY_NOT_IN_ADMIN
]
assert test_key in undecided, (
f"Removing {test_key!r} from _CARD_BACKED_PATHS should cause "
f"the gate to flag it, but it did not"
)
finally:
admin_mod._CARD_BACKED_PATHS = original
def test_excused_knobs_are_not_already_covered():
"""Excused knobs should NOT be covered, otherwise the excuse is false.
A card-backed leaf in DELIBERATELY_NOT_IN_ADMIN is a false excuse:
the card IS its control. The excuse should be removed, not kept.
"""
covered = _covered_paths()
stale = sorted(set(DELIBERATELY_NOT_IN_ADMIN) & covered)
assert not stale, (
"DELIBERATELY_NOT_IN_ADMIN excuses knob(s) that the admin portal now "
"covers:\n "
+ "\n ".join(f"{path}: {DELIBERATELY_NOT_IN_ADMIN[path]}" for path in stale)
+ "\n\nDelete those entries."
)
for path in DELIBERATELY_NOT_IN_ADMIN:
assert path not in covered, (
f"{path!r} is in DELIBERATELY_NOT_IN_ADMIN but is already "
f"covered by the admin portal. Remove the excuse."
)
def test_excused_knobs_still_exist_and_are_in_scope():
@@ -524,6 +596,120 @@ def test_admin_registries_point_at_real_config_knobs():
)
# --- Phase A Step 3: taxonomy coverage ----------------------------------------
# The test below checks two-way coverage between _KNOB_CATEGORY and the covered
# registries (_CONFIG_ALLOWLIST + runtime + card-backed). Card-backed knobs
# (with their own dedicated admin card/endpoint pair) are excluded from both
# directions — they are covered by their card but don't need a taxonomy entry.
# _CARD_BACKED_PATHS is used throughout this module: in _covered_paths(),
# test_knob_taxonomy_coverage(), and test_card_backed_gate_is_real().
def _cfg_paths_from_runtime() -> set[str]:
"""Dotted config paths from every runtime registry entry.
``_RUNTIME_KNOB_PATHS`` is the single source of truth (it is derived
from the same registries that drive the POST handlers) but we rebuild
the path set directly from the registries to avoid importing a
dict whose values are the same as ``_CONFIG_ALLOWLIST``'s and would
obscure the separate provenance this check needs.
Unlike ``_covered_paths()`` this function exists solely for the taxonomy
two-way check — it does not deduplicate against the allowlist, because
both directions start from a different origin.
"""
return (
{".".join(path) for path in admin._BOOL_KNOBS.values()}
| {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()}
| {".".join(entry[0]) for entry in admin._INT_KNOBS.values()}
)
def _card_backed_set() -> set[str]:
"""Card-backed config paths to exclude from two-way coverage check.
Reads directly from ``admin._CARD_BACKED_PATHS``, which maps classifier
config keys to the HTML element ids of their dedicated admin card controls.
"""
return set(admin._CARD_BACKED_PATHS)
def test_knob_taxonomy_coverage():
"""Two-way coverage between _KNOB_CATEGORY and the covered registries.
Direction 1 (allowlist + runtime + card-backed -> taxonomy): every knob
that appears in ``_CONFIG_ALLOWLIST``, any runtime registry, or
``_CARD_BACKED_PATHS`` must have an entry in ``_KNOB_CATEGORY``.
Direction 2 (taxonomy -> registries + card-backed): every key in
``_KNOB_CATEGORY`` must resolve to a path covered by ``_CONFIG_ALLOWLIST``,
a runtime registry, or ``_CARD_BACKED_PATHS``.
Both directions exclude card-backed paths from the taxonomy check — they
have dedicated admin cards and don't belong in the generic knob table.
"""
# --- prepare covered set --------------------------------------------------
allowlist_paths: set[str] = set(admin._CONFIG_ALLOWLIST)
runtime_paths: set[str] = _cfg_paths_from_runtime()
card_backed: set[str] = _card_backed_set()
covered = allowlist_paths | runtime_paths | card_backed
# --- card-backed exclusion ------------------------------------------------
card_backed = _card_backed_set()
# --- Direction 1 ----------------------------------------------------------
# Every covered knob (minus card-backed) needs a category assignment.
candidates_d1 = covered - card_backed
missing = sorted(candidates_d1 - set(admin._KNOB_CATEGORY))
assert not missing, (
"Covered knob(s) with no entry in _KNOB_CATEGORY:\n "
+ "\n ".join(missing)
+ "\n\nAdd an entry to _KNOB_CATEGORY in src/admin.py with the "
"appropriate category id and advanced flag."
)
# --- Direction 2 ----------------------------------------------------------
# Every taxonomy entry (minus card-backed) must be a real covered path.
candidates_d2 = set(admin._KNOB_CATEGORY) - card_backed
orphaned = sorted(candidates_d2 - covered)
assert not orphaned, (
"_KNOB_CATEGORY key(s) that are in neither _CONFIG_ALLOWLIST nor any "
"runtime registry:\n "
+ "\n ".join(orphaned)
+ "\n\nEither add the path to _CONFIG_ALLOWLIST or a runtime registry, "
"or --- if the knob has its own dedicated admin card --- add it to "
"_CARD_BACKED_PATHS in src/admin.py."
)
# --- well-formed entries --------------------------------------------------
valid_categories = {
"routing_quality_cost",
"classification",
"local_hardware",
"caching_context",
"safety_nets",
"watchdog",
}
for path, entry in admin._KNOB_CATEGORY.items():
assert isinstance(entry, dict), (
f"_KNOB_CATEGORY[{path!r}] is not a dict: {type(entry).__name__}"
)
assert "category" in entry, (
f"_KNOB_CATEGORY[{path!r}] has no 'category' key"
)
assert entry["category"] in valid_categories, (
f"_KNOB_CATEGORY[{path!r}].category={entry['category']!r} "
f"is not in {sorted(valid_categories)}"
)
assert "advanced" in entry, (
f"_KNOB_CATEGORY[{path!r}] has no 'advanced' key"
)
assert isinstance(entry["advanced"], bool), (
f"_KNOB_CATEGORY[{path!r}].advanced is not bool: "
f"{type(entry['advanced']).__name__}"
)
# --- Component 6: one knob table -----------------------------------------------
# The two tests below step outside the pure-reflection contract documented
# above: the first drives the real app (still offline -- a temp SQLite file,
@@ -562,7 +748,7 @@ def test_runtime_api_labels_every_knob_with_a_real_config_key(tmp_path, monkeypa
cfg = load_config(str(ROOT / "config" / "config.yaml"))
for knob, item in body.items():
assert set(item) == {"persisted", "runtime", "config_key"}, knob
assert set(item) == {"persisted", "runtime", "config_key", "category", "advanced"}, knob
# An AttributeError here names the broken label directly.
obj: typing.Any = cfg
for part in item["config_key"].split("."):
@@ -601,3 +787,25 @@ def test_controls_page_keeps_the_table_selectors_its_js_depends_on():
assert "data-config-input" in html
assert "data-profile-select" in html
assert '<tr><td colspan="4"' in html
def test_card_backed_paths_are_real_config_scalars_and_element_ids_exist():
"""Every card-backed path is a real scalar field and its element id exists.
(a) Walk ``RouterConfig`` via ``_walk`` to verify every key in
``admin._CARD_BACKED_PATHS`` resolves to a scalar leaf of the config model.
(b) Read ``controls.html`` and verify every element id value appears as an
``id="..."`` attribute somewhere in the source.
"""
scalars = set(_scalars())
html = CONTROLS_HTML.read_text(encoding="utf-8")
for path, element_id in admin._CARD_BACKED_PATHS.items():
assert path in scalars, (
f"_CARD_BACKED_PATHS key {path!r} is not a scalar field of "
f"RouterConfig"
)
assert element_id in html, (
f"_CARD_BACKED_PATHS value {element_id!r} (for {path!r}) not found "
f"in controls.html as an element id"
)

View File

@@ -19,7 +19,13 @@ from starlette.testclient import TestClient
import dispatcher
from admin import _INT_KNOBS
from config import STALENESS_SECONDS_MAX, STALENESS_SECONDS_MIN, load_config
from config import (
DEGRADED_WARN_THRESHOLD_FLOOR,
DEGRADED_WARN_THRESHOLD_MAX,
STALENESS_SECONDS_MAX,
STALENESS_SECONDS_MIN,
load_config,
)
ROOT = Path(__file__).resolve().parent.parent
SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text()
@@ -93,7 +99,7 @@ def test_runtime_GET_reports_every_knob(seeded_client):
"default_flex_preference",
):
assert knob in body
assert set(body[knob]) == {"persisted", "runtime", "config_key"}
assert set(body[knob]) == {"persisted", "runtime", "config_key", "category", "advanced"}
# The runtime value mirrors the live dispatcher.cfg for the boolean knobs;
# the persisted value mirrors config.yaml. circuit_breaker_enabled is a
@@ -106,11 +112,15 @@ def test_runtime_GET_reports_every_knob(seeded_client):
"persisted": CFG.circuit_breaker.enabled,
"runtime": dispatcher.cfg.circuit_breaker.enabled,
"config_key": "circuit_breaker.enabled",
"category": "safety_nets",
"advanced": False,
}
assert body["default_flex_preference"] == {
"persisted": CFG.routing.default_flex_preference.value,
"runtime": dispatcher.cfg.routing.default_flex_preference.value,
"config_key": "routing.default_flex_preference",
"category": "routing_quality_cost",
"advanced": False,
}
@@ -326,6 +336,99 @@ def test_challenger_dial_refuses_a_non_number(
assert dispatcher.cfg.objective.incumbent_challenger_cache_rate == before
def test_challenger_dial_422_still_advertises_null_as_neutral(
seeded_client, incumbent_cfg_guard
):
"""The dial HAS a neutral, so its 422s must keep telling the operator so.
These hints were dropped once when a second float knob without a neutral
shared the handler; they are conditional on the knob having one.
"""
url = "/admin/api/runtime/incumbent_challenger_cache_rate"
not_a_number = seeded_client.post(url, json={"value": "x"})
assert not_a_number.status_code == 422
assert "or null for neutral" in not_a_number.json()["detail"]
out_of_range = seeded_client.post(url, json={"value": 1.5})
assert out_of_range.status_code == 422
assert "(null means neutral, not 0.0)" in out_of_range.json()["detail"]
@pytest.fixture
def degraded_threshold_cfg_guard(monkeypatch):
"""Restore classifier.degraded_warn_threshold after a test writes it."""
classifier = dispatcher.cfg.classifier
monkeypatch.setattr(
classifier, "degraded_warn_threshold", classifier.degraded_warn_threshold
)
return classifier
DEGRADED_THRESHOLD_URL = "/admin/api/runtime/classifier_degraded_warn_threshold"
@pytest.mark.parametrize(
"bad",
[
None,
True,
"x",
0.0,
DEGRADED_WARN_THRESHOLD_FLOOR / 2,
DEGRADED_WARN_THRESHOLD_MAX + 0.5,
],
ids=["null", "bool", "string", "zero", "below-floor", "above-max"],
)
def test_degraded_warn_threshold_refuses_bad_values_with_422(
seeded_client, degraded_threshold_cfg_guard, bad
):
"""A runtime write skips Pydantic, so the registry bounds are the only guard.
``null`` is the sharp one: this knob has no neutral, and the float handler
used to resolve null through ``_get_at(cfg, neutral_path)``, which raised
TypeError (HTTP 500) when that path is None.
"""
before = degraded_threshold_cfg_guard.degraded_warn_threshold
resp = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": bad})
assert resp.status_code == 422
assert degraded_threshold_cfg_guard.degraded_warn_threshold == before
def test_degraded_warn_threshold_422_never_advertises_null(
seeded_client, degraded_threshold_cfg_guard
):
"""No neutral means no ``null`` hint: it would send the operator into a 422."""
for bad in (None, "x", 1.5):
detail = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": bad}).json()[
"detail"
]
assert "neutral" not in detail
assert "null" not in detail
@pytest.mark.parametrize(
"good",
[DEGRADED_WARN_THRESHOLD_FLOOR, 0.2, DEGRADED_WARN_THRESHOLD_MAX],
ids=["floor", "typical", "max"],
)
def test_degraded_warn_threshold_accepts_in_range_and_updates_cfg(
seeded_client, degraded_threshold_cfg_guard, good
):
resp = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": good})
assert resp.status_code == 200
assert resp.json() == {"ok": True, "classifier_degraded_warn_threshold": good}
stored = degraded_threshold_cfg_guard.degraded_warn_threshold
assert isinstance(stored, float) and not isinstance(stored, bool)
assert stored == good
assert (
seeded_client.get("/admin/api/runtime").json()[
"classifier_degraded_warn_threshold"
]["runtime"]
== good
)
@pytest.fixture
def staleness_cfg_guard(monkeypatch):
"""Restore the session-cache window on cfg after a test moves it."""
@@ -510,6 +613,8 @@ def test_active_profile_is_reported_with_persisted_and_runtime(seeded_client):
"persisted",
"runtime",
"config_key",
"category",
"advanced",
}