feat(admin): classifier knob coverage and categorized Controls page #113
18
CLAUDE.md
18
CLAUDE.md
@@ -23,6 +23,24 @@ oversight nobody noticed.
|
||||
absent, because enabling it must be a config edit plus a restart. That is a
|
||||
recorded choice, not a gap.
|
||||
|
||||
The classifier section is now fully accounted for under the gate. Seven
|
||||
classifier scalars (`context_framing`, `cooldown_seconds`, `fallback_tier`,
|
||||
`fallback_category`, `max_input_chars`, `degraded_warn_min`,
|
||||
`degraded_warn_threshold`) have a runtime control, a persisted control, or both
|
||||
(`fallback_category` and `max_input_chars` are persisted-only). Every other
|
||||
classifier scalar is either card-backed (`_CARD_BACKED_PATHS` in
|
||||
`src/admin.py`: mode, cloud primary/fallback, encoder, and local-decision
|
||||
fields) or excused in `DELIBERATELY_NOT_IN_ADMIN` (timeout, temperature,
|
||||
`max_output_tokens`, `encoder.tier_from_features`).
|
||||
|
||||
Outside-gate knobs are still pending. Sections not yet reached by the portal
|
||||
(database, dispatch providers/settings, local dispatch models, profiles,
|
||||
local energy, tiers/tiering/proficiency/context, and deployment wiring inside
|
||||
classifier/verification/local vision) are tracked in
|
||||
`plans/deferred-knobs.md`. The first control added under any of those sections
|
||||
will drag every scalar under it into scope at once, per the coverage test's
|
||||
clause 1.
|
||||
|
||||
**Why:** Wave 2 shipped `incumbent_cache_pricing` and
|
||||
`incumbent_challenger_cache_rate` with no control at all. Nobody decided that;
|
||||
it just never came up. The dial's entire purpose is tuning from neutral to full
|
||||
|
||||
@@ -135,10 +135,38 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
Scoped so any future div list keeps the grid grammar. */
|
||||
.knobs-table .setting-row{display:table-row;min-height:0;padding:0;border-radius:0;border-left:2px solid transparent}
|
||||
.knobs-table .setting-row.is-dirty{border-left-color:#fbbf24;background:rgba(245,158,11,.06)}
|
||||
/* A config key wraps instead of truncating: the knob cell is the row's
|
||||
identity, and a clipped path hides which scope the row belongs to. */
|
||||
/* Persisted-cell hints: the note badge is white-space:nowrap by default, so a
|
||||
long note sets the cell's min-content width and pushes the table past an
|
||||
800px viewport. Let the hints wrap (the pill drops under the unit) and the
|
||||
pill's own text wrap. d-inline-flex is !important, so this targets the
|
||||
cell's direct .ms-2 child rather than overriding display. */
|
||||
.knobs-table td > .ms-2{flex-wrap:wrap;max-width:100%}
|
||||
.knobs-table td > .ms-2 .badge{white-space:normal;text-align:left}
|
||||
.knobs-table .setting-key{white-space:normal;overflow:visible;text-overflow:clip}
|
||||
.knobs-table .setting-key .scope{display:inline}
|
||||
/* -- Category header rows: a subtle accent that segments the knob list
|
||||
without adding visual weight. The chevron is the only interactive cue;
|
||||
the label and icon supply context. -- */
|
||||
.cat-header th{
|
||||
background:rgba(255,255,255,.02);
|
||||
border-bottom:1px solid rgba(255,255,255,.08);
|
||||
cursor:pointer;user-select:none;
|
||||
font-size:.85rem;font-weight:600;
|
||||
padding:.35rem .6rem;
|
||||
color:var(--tblr-secondary)
|
||||
}
|
||||
.cat-header th:hover{background:rgba(255,255,255,.045)}
|
||||
.cat-header .cat-label{color:#e2e8f0}
|
||||
.cat-chevron{display:inline-flex;align-items:center;color:var(--tblr-secondary);opacity:.7;transition:transform .18s ease}
|
||||
.cat-chevron:not(.cat-chevron-open){transform:rotate(90deg)}
|
||||
.cat-icon{opacity:.7;margin-right:.15rem}
|
||||
.adv-subheader th{
|
||||
cursor:pointer;user-select:none;
|
||||
font-size:.78rem;font-weight:500;
|
||||
padding:.2rem .6rem;
|
||||
color:var(--tblr-secondary);opacity:.7
|
||||
}
|
||||
.adv-subheader th:hover{opacity:1;color:#e2e8f0}
|
||||
.setting-key{
|
||||
font-size:.82rem;line-height:1.25;
|
||||
overflow:hidden;text-overflow:ellipsis;white-space:nowrap;
|
||||
@@ -644,6 +672,9 @@ function keyHtml(key) {
|
||||
const UNITS = {
|
||||
'objective.max_energy_per_request': 'kWh',
|
||||
'objective.plan_kwh_per_period': 'kWh',
|
||||
'classifier.cooldown_seconds': 's',
|
||||
'classifier.degraded_warn_min': 'decisions/24h',
|
||||
'classifier.max_input_chars': 'chars',
|
||||
};
|
||||
|
||||
/* A consequence the control itself cannot show. Keyed by both the runtime knob
|
||||
@@ -692,6 +723,18 @@ const SESSION_CACHE_TTL_NOTE = {
|
||||
cls: 'bg-info',
|
||||
glyph: 'info',
|
||||
};
|
||||
const DEGRADED_WARN_MIN_NOTE = {
|
||||
text: 'silent below this many decisions in 24h',
|
||||
title: 'No warning is raised until the classifier completes at least this many decisions in a 24-hour window. Below the threshold the degraded-count warning is suppressed entirely.',
|
||||
cls: 'bg-info',
|
||||
glyph: 'info',
|
||||
};
|
||||
const DEGRADED_WARN_THRESHOLD_NOTE = {
|
||||
text: 'share of degraded classifications that warns',
|
||||
title: 'When the fraction of degraded (non-ok) classifier verdicts exceeds this ratio within the warn-min window, an amber badge appears on the admin navbar. 0.01 = 1% of classifications degraded.',
|
||||
cls: 'bg-info',
|
||||
glyph: 'info',
|
||||
};
|
||||
const KNOB_NOTES = {
|
||||
pinch_prefix_probe: PROBE_NOTE,
|
||||
'pinch.prefix_probe': PROBE_NOTE,
|
||||
@@ -703,6 +746,10 @@ const KNOB_NOTES = {
|
||||
'objective.incumbent_cache_pricing': INCUMBENT_GATE_NOTE,
|
||||
incumbent_challenger_cache_rate: CHALLENGER_DIAL_NOTE,
|
||||
'objective.incumbent_challenger_cache_rate': CHALLENGER_DIAL_NOTE,
|
||||
classifier_degraded_warn_min: DEGRADED_WARN_MIN_NOTE,
|
||||
'classifier.degraded_warn_min': DEGRADED_WARN_MIN_NOTE,
|
||||
classifier_degraded_warn_threshold: DEGRADED_WARN_THRESHOLD_NOTE,
|
||||
'classifier.degraded_warn_threshold': DEGRADED_WARN_THRESHOLD_NOTE,
|
||||
};
|
||||
|
||||
function noteBadge(key) {
|
||||
@@ -723,8 +770,52 @@ const NUMBER_BOUNDS = {
|
||||
// operator was reaching for. min is 5 rather than 0 -- 0 is not "off", it
|
||||
// still caches and the classifier-failure cascade still replays the entry.
|
||||
session_cache_staleness_seconds: { min: 5, max: 7200, step: 1 },
|
||||
classifier_degraded_warn_threshold: { min: 0.001, max: 1, step: 0.01 },
|
||||
classifier_max_input_chars: { min: 0, step: 1000 },
|
||||
};
|
||||
|
||||
// Category definitions for the knob table section headers. Order here matches
|
||||
// _CONFIG_GET_ORDER in admin.py so sections render in the same sequence the
|
||||
// YAML uses. `icon` picks a renderStaticIcons-compatible name.
|
||||
const CATEGORIES = [
|
||||
{ id: 'routing_quality_cost', label: 'Routing / Quality / Cost', icon: 'settings' },
|
||||
{ id: 'classification', label: 'Classification', icon: 'info' },
|
||||
{ id: 'caching_context', label: 'Caching / Context', icon: 'database' },
|
||||
{ id: 'local_hardware', label: 'Local Hardware', icon: 'cpu' },
|
||||
{ id: 'safety_nets', label: 'Safety Nets', icon: 'alert' },
|
||||
{ id: 'watchdog', label: 'Watchdog', icon: 'alert' },
|
||||
];
|
||||
|
||||
// Collapsible category and advanced-section state. Persisted across 30s
|
||||
// re-renders: toggling mutates this Set, and renderKnobsTable reads it.
|
||||
// Category keys are `cat.<id>.collapsed`; advanced keys are `cat.<id>.adv`.
|
||||
let collapsedCategories = new Set();
|
||||
// Categories whose Advanced accordion has been seeded collapsed. Per category,
|
||||
// not global: see seedAdvancedCollapsed.
|
||||
let _advSeededCategories = new Set();
|
||||
|
||||
/* SEED_ADV_COLLAPSED:BEGIN */
|
||||
function seedAdvancedCollapsed(byCategory, rows, categoryIds, seeded, collapsed) {
|
||||
// Seed a category's Advanced accordion as collapsed the FIRST time that
|
||||
// category renders an advanced row, not on the first render overall. The
|
||||
// runtime payload can arrive before the config payload, and persisted-only
|
||||
// advanced knobs (watchdog.detector.*) are absent from that early render; a
|
||||
// category marked seeded before its advanced rows exist would load expanded.
|
||||
// Later renders leave whatever the user toggled alone.
|
||||
for (const id of categoryIds) {
|
||||
if (seeded.has(id)) continue;
|
||||
const hasAdvanced = (byCategory[id] || []).some(key => {
|
||||
const row = rows[key];
|
||||
return (row.runtimeItem && row.runtimeItem.advanced) ||
|
||||
(row.persistedItem && row.persistedItem.advanced);
|
||||
});
|
||||
if (!hasAdvanced) continue;
|
||||
seeded.add(id);
|
||||
collapsed.add(id + '.adv');
|
||||
}
|
||||
}
|
||||
/* SEED_ADV_COLLAPSED:END */
|
||||
|
||||
/* One knob table replaces the old runtime-list/config-list div columns.
|
||||
Runtime rows and persisted rows pair by the dotted config key the
|
||||
/admin/api/runtime payload carries in `config_key` -- the join lives here,
|
||||
@@ -858,7 +949,7 @@ function persistedControlHtml(key, entry, dirtyValue) {
|
||||
return `${control}${hints ? `<span class="d-inline-flex align-items-center gap-1 ms-2">${hints}</span>` : ''}`;
|
||||
}
|
||||
|
||||
function knobRowHtml(row) {
|
||||
function knobRowHtml(row, catId, isAdv) {
|
||||
const rt = row.runtimeItem;
|
||||
const entry = row.persistedItem;
|
||||
const keyCell = row.drift
|
||||
@@ -869,7 +960,7 @@ function knobRowHtml(row) {
|
||||
: '<span class="text-muted small">-</span>';
|
||||
let persistedCell = '<span class="text-muted small">-</span>';
|
||||
let layerCell = '<td class="setting-meta"></td>';
|
||||
let dirtyAttrs = '';
|
||||
let extraAttrs = '';
|
||||
if (entry) {
|
||||
persistedCell = persistedControlHtml(row.configKey, entry, row.dirtyValue);
|
||||
// Provenance is the layer column: where the effective value came from.
|
||||
@@ -880,9 +971,11 @@ function knobRowHtml(row) {
|
||||
layer = '<span class="badge bg-secondary" title="from config.yaml">base</span>';
|
||||
}
|
||||
layerCell = `<td class="setting-meta">${layer}</td>`;
|
||||
dirtyAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`;
|
||||
extraAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`;
|
||||
}
|
||||
return `<tr class="setting-row"${dirtyAttrs}>
|
||||
if (catId) extraAttrs += ` data-cat="cat-${catId}"`;
|
||||
if (isAdv) extraAttrs += ' data-adv="true"';
|
||||
return `<tr class="setting-row"${extraAttrs}>
|
||||
<td class="setting-key">${keyCell}</td>
|
||||
<td>${liveCell}</td>
|
||||
<td>${persistedCell}</td>
|
||||
@@ -917,13 +1010,155 @@ function renderKnobsTable() {
|
||||
list.innerHTML = '<tr><td colspan="4" class="text-muted fst-italic small px-2 py-1">No knobs to show</td></tr>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = keys.map(key => {
|
||||
|
||||
// Group rows by category using the category/advanced metadata from the API.
|
||||
const byCategory = {};
|
||||
const uncategorized = [];
|
||||
for (const key of keys) {
|
||||
const row = rows[key];
|
||||
row.dirtyValue = row.persistedItem ? dirtyInputs[key] : undefined;
|
||||
return knobRowHtml(row);
|
||||
}).join('');
|
||||
const cat = (row.runtimeItem && row.runtimeItem.category) ||
|
||||
(row.persistedItem && row.persistedItem.category) || '';
|
||||
if (!cat) { uncategorized.push(key); continue; }
|
||||
if (!byCategory[cat]) byCategory[cat] = [];
|
||||
byCategory[cat].push(key);
|
||||
}
|
||||
|
||||
const html = [];
|
||||
|
||||
// Advanced subsections load collapsed, so the user sees the clean knob list
|
||||
// before the Advanced expander. Seeded per category, once.
|
||||
seedAdvancedCollapsed(byCategory, rows, CATEGORIES.map(c => c.id),
|
||||
_advSeededCategories, collapsedCategories);
|
||||
|
||||
for (const cat of CATEGORIES) {
|
||||
const catKeys = byCategory[cat.id] || [];
|
||||
if (!catKeys.length) continue;
|
||||
const isCatCollapsed = collapsedCategories.has(cat.id + '.collapsed');
|
||||
|
||||
// Category header with chevron toggle, rendered via data-icon for
|
||||
// renderStaticIcons(). All 4 columns spanned so the click target
|
||||
// is the full table width.
|
||||
html.push(`<tr class="cat-header" data-cat="cat-${cat.id}">
|
||||
<th colspan="4" id="cat-${cat.id}" class="cat-header-cell" onclick="toggleCategory('${cat.id}')">
|
||||
<span class="d-inline-flex align-items-center gap-2">
|
||||
<span data-icon="chevronDown" class="cat-chevron${isCatCollapsed ? '' : ' cat-chevron-open'}"></span>
|
||||
<span data-icon="${cat.icon}" class="cat-icon"></span>
|
||||
<span class="cat-label">${cat.label}</span>
|
||||
</span>
|
||||
</th>
|
||||
</tr>`);
|
||||
|
||||
// Split into standard and advanced rows.
|
||||
const standard = [];
|
||||
const advanced = [];
|
||||
for (const key of catKeys) {
|
||||
const row = rows[key];
|
||||
const isAdv = (row.runtimeItem && row.runtimeItem.advanced) ||
|
||||
(row.persistedItem && row.persistedItem.advanced) || false;
|
||||
(isAdv ? advanced : standard).push(key);
|
||||
}
|
||||
|
||||
// Standard (non-advanced) rows
|
||||
for (const key of standard) {
|
||||
html.push(knobRowHtml(rows[key], cat.id, false));
|
||||
}
|
||||
|
||||
// Advanced sub-header + rows (default collapsed on first render)
|
||||
if (advanced.length) {
|
||||
const advKey = cat.id + '.adv';
|
||||
const isAdvCollapsed = collapsedCategories.has(advKey);
|
||||
html.push(`<tr class="adv-subheader" data-cat="cat-${cat.id}" data-adv-subheader="${cat.id}">
|
||||
<th colspan="4" class="adv-subheader-cell" onclick="toggleAdvanced('${cat.id}')">
|
||||
<span class="d-inline-flex align-items-center gap-2 text-muted">
|
||||
<span data-icon="chevronDown" class="cat-chevron${isAdvCollapsed ? '' : ' cat-chevron-open'}"></span>
|
||||
<span class="small">Advanced</span>
|
||||
</span>
|
||||
</th>
|
||||
</tr>`);
|
||||
for (const key of advanced) {
|
||||
html.push(knobRowHtml(rows[key], cat.id, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Uncategorized rows at the end (fallback for knob metadata edge cases).
|
||||
for (const key of uncategorized) {
|
||||
html.push(knobRowHtml(rows[key], '', false));
|
||||
}
|
||||
|
||||
list.innerHTML = html.join('');
|
||||
|
||||
// Apply collapsed state to category rows after rendering.
|
||||
for (const cat of CATEGORIES) {
|
||||
if (collapsedCategories.has(cat.id + '.collapsed')) {
|
||||
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"]:not(.cat-header)`);
|
||||
rows.forEach(r => { r.style.display = 'none'; });
|
||||
}
|
||||
if (collapsedCategories.has(cat.id + '.adv')) {
|
||||
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"][data-adv="true"]`);
|
||||
rows.forEach(r => { r.style.display = 'none'; });
|
||||
}
|
||||
}
|
||||
|
||||
// Re-render data-icon attributes that were just injected.
|
||||
renderStaticIcons();
|
||||
updateDefaultProfileHint();
|
||||
markDirty();
|
||||
|
||||
// If the URL carries a category anchor, expand that section now.
|
||||
if (window.location.hash && window.location.hash.startsWith('#cat-')) {
|
||||
const hashCat = window.location.hash.replace('#cat-', '');
|
||||
if (collapsedCategories.has(hashCat + '.collapsed')) {
|
||||
collapsedCategories.delete(hashCat + '.collapsed');
|
||||
const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${hashCat}"]:not(.cat-header)`);
|
||||
rows.forEach(r => { r.style.display = ''; });
|
||||
const chevron = document.querySelector(`#config-list tr[data-cat="cat-${hashCat}"].cat-header .cat-chevron`);
|
||||
if (chevron) chevron.classList.add('cat-chevron-open');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Toggle a category section: collapse/expand all knob rows within it.
|
||||
function toggleCategory(catId) {
|
||||
const key = catId + '.collapsed';
|
||||
if (collapsedCategories.has(key)) {
|
||||
collapsedCategories.delete(key);
|
||||
} else {
|
||||
collapsedCategories.add(key);
|
||||
}
|
||||
const isCollapsed = collapsedCategories.has(key);
|
||||
// All knob rows + advanced sub-header under this category.
|
||||
const rows = document.querySelectorAll(
|
||||
`#config-list tr[data-cat="cat-${catId}"]:not(.cat-header)`
|
||||
);
|
||||
rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; });
|
||||
// Flip the chevron on the category header.
|
||||
const chevron = document.querySelector(
|
||||
`#config-list tr[data-cat="cat-${catId}"].cat-header .cat-chevron`
|
||||
);
|
||||
if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed);
|
||||
}
|
||||
|
||||
// Toggle an advanced subsection within a category: collapse/expand only the
|
||||
// rows flagged data-adv="true" under that category.
|
||||
function toggleAdvanced(catId) {
|
||||
const key = catId + '.adv';
|
||||
if (collapsedCategories.has(key)) {
|
||||
collapsedCategories.delete(key);
|
||||
} else {
|
||||
collapsedCategories.add(key);
|
||||
}
|
||||
const isCollapsed = collapsedCategories.has(key);
|
||||
const rows = document.querySelectorAll(
|
||||
`#config-list tr[data-cat="cat-${catId}"][data-adv="true"]`
|
||||
);
|
||||
rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; });
|
||||
// Flip the chevron on the advanced sub-header.
|
||||
const chevron = document.querySelector(
|
||||
`#config-list tr[data-adv-subheader="${catId}"] .cat-chevron`
|
||||
);
|
||||
if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed);
|
||||
}
|
||||
|
||||
function renderRuntime(state) {
|
||||
|
||||
@@ -145,6 +145,39 @@ land in `config/config.local.yaml`; see
|
||||
[config-local-overlay](config-local-overlay.md)). Changes are marked as dirty
|
||||
and written only on save.
|
||||
|
||||
#### Knob organization
|
||||
|
||||
The generic knob table is grouped into **six categories**, each with a
|
||||
collapsible section header:
|
||||
|
||||
1. **Routing / Quality / Cost** — the core dispatch objective: logging
|
||||
levels, energy ceiling, plan quota, cache pricing, and default profile.
|
||||
2. **Classification** — classifier runtime scalars such as cooldown,
|
||||
fallback tier/category, degraded warning thresholds, and context framing.
|
||||
3. **Local Hardware** — gates over local compute paths: the local-compute
|
||||
master switch, local verification, and local vision fallback.
|
||||
4. **Caching / Context** — session classification cache and context pruning
|
||||
(pinch).
|
||||
5. **Safety Nets** — circuit breaker.
|
||||
6. **Watchdog** — response-loop detector thresholds.
|
||||
|
||||
Within each category, knobs marked **Advanced** hide behind a secondary
|
||||
"Advanced" accordion. These are shape parameters internal to a covered master
|
||||
switch (for example, pinch relevance options and watchdog detector
|
||||
thresholds). The accordion keeps the first view short without removing access.
|
||||
|
||||
Knobs reach the page through **three coverage sources**:
|
||||
|
||||
- **Registry knobs** — runtime toggles from `_BOOL_KNOBS`, `_FLOAT_KNOBS`, and
|
||||
`_INT_KNOBS` in `admin.py`. They take effect immediately in memory and
|
||||
revert on restart.
|
||||
- **Allowlist knobs** — the same scalars persisted through
|
||||
`_CONFIG_ALLOWLIST`, written to `config/config.local.yaml` and surviving a
|
||||
restart.
|
||||
- **Card-backed paths** — knobs with their own dedicated admin card and
|
||||
endpoint pair because they have cross-field structure a flat scalar input
|
||||
cannot safely represent.
|
||||
|
||||
It carries three dedicated cards, none a row in the generic runtime-knob
|
||||
list, because each has cross-field structure a flat scalar/boolean input
|
||||
can't safely represent:
|
||||
|
||||
116
plans/deferred-knobs.md
Normal file
116
plans/deferred-knobs.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# Deferred admin portal knobs
|
||||
Status: reference -- Tracker for knobs outside current admin gate
|
||||
|
||||
This file tracks config knobs that are outside the admin portal's current
|
||||
gate. The coverage test in `tests/test_admin_knob_coverage.py` enforces that
|
||||
every knob in a section the portal reaches is either covered or excused with a
|
||||
reason. Sections the portal has not reached are out of scope for that test,
|
||||
but they still need a recorded home so the first control added under a section
|
||||
does not leave half its siblings orphaned.
|
||||
|
||||
Rows use four columns:
|
||||
|
||||
| Column | Meaning |
|
||||
|---|---|
|
||||
| Knob | Dotted config path in `config/config.yaml`. |
|
||||
| Section | Top-level config section. |
|
||||
| Disposition | `deferred` (future admin page) or `excused-with-reason` (deployment wiring or file path). |
|
||||
| Future Home | The page or surface that should own this knob when the portal reaches the section. |
|
||||
|
||||
## Outside-gate knobs
|
||||
|
||||
These are `RouterConfig` scalars in sections the portal has not reached yet.
|
||||
They fall into two buckets:
|
||||
|
||||
- Sections not yet reached: `context`, `database`, `dispatch_settings`,
|
||||
`escalation`, `exploration`, `freshness`, `iteration`, `local_energy`,
|
||||
`proficiency`, `tiering`.
|
||||
- Deployment wiring inside those sections: endpoints, model ids, credentials,
|
||||
filesystem paths, and device names.
|
||||
|
||||
| Knob | Section | Disposition | Future Home |
|
||||
|---|---|---|---|
|
||||
| `context.default_output_reserve_tokens` | context | deferred | Context page |
|
||||
| `context.max_output_reserve_fraction` | context | deferred | Context page |
|
||||
| `context.safety_factor` | context | deferred | Context page |
|
||||
| `database.path` | database | excused-with-reason | N/A (file path) |
|
||||
| `dispatch_settings.default_provider` | dispatch_settings | deferred | Providers page |
|
||||
| `escalation.enabled` | escalation | deferred | Routing page |
|
||||
| `escalation.max_tier` | escalation | deferred | Routing page |
|
||||
| `escalation.min_confidence_before_bump` | escalation | deferred | Routing page |
|
||||
| `escalation.preemptive_on_low_confidence` | escalation | deferred | Routing page |
|
||||
| `exploration.enabled` | exploration | deferred | Routing page |
|
||||
| `exploration.epsilon` | exploration | deferred | Routing page |
|
||||
| `exploration.max_cost_ratio` | exploration | deferred | Routing page |
|
||||
| `exploration.max_tier` | exploration | deferred | Routing page |
|
||||
| `freshness.exclude_deprecated` | freshness | deferred | Catalog page |
|
||||
| `freshness.exclude_stale` | freshness | deferred | Catalog page |
|
||||
| `freshness.repoll_after_allowlist_change_seconds` | freshness | deferred | Catalog page |
|
||||
| `freshness.stale_after_days` | freshness | deferred | Catalog page |
|
||||
| `iteration.enabled` | iteration | deferred | Routing page |
|
||||
| `iteration.max_attempts_interactive` | iteration | deferred | Routing page |
|
||||
| `iteration.max_rebill_prompt_tokens` | iteration | deferred | Routing page |
|
||||
| `local_energy.enabled` | local_energy | deferred | Local Hardware page |
|
||||
| `local_energy.grid_intensity_g_per_kwh` | local_energy | deferred | Local Hardware page |
|
||||
| `local_energy.meter` | local_energy | excused-with-reason | N/A (deployment wiring) |
|
||||
| `local_energy.sample_interval_seconds` | local_energy | deferred | Local Hardware page |
|
||||
| `local_energy.tariff_usd_per_kwh` | local_energy | deferred | Local Hardware page |
|
||||
| `proficiency.leaderboard_weight` | proficiency | deferred | Proficiency page |
|
||||
| `proficiency.outcome_prior_strength` | proficiency | deferred | Proficiency page |
|
||||
| `proficiency.self_eval_min_samples` | proficiency | deferred | Proficiency page |
|
||||
| `proficiency.self_eval_weight` | proficiency | deferred | Proficiency page |
|
||||
| `tiering.cheap_completion_max` | tiering | deferred | Routing page |
|
||||
| `tiering.tier1_context_max` | tiering | deferred | Routing page |
|
||||
|
||||
## Objective warning knobs
|
||||
|
||||
These 31 `objective.*` knobs are currently excused from the admin portal in
|
||||
`tests/test_admin_knob_coverage.py`. They tune `/metrics` warnings and report
|
||||
series rather than dispatch behavior. Their natural future home is a
|
||||
**Warnings page** that collects all alarm and report thresholds in one place.
|
||||
|
||||
| Knob | Section | Disposition | Future Home |
|
||||
|---|---|---|---|
|
||||
| `objective.adoption_window_seconds` | objective | deferred | Warnings page |
|
||||
| `objective.assumed_cache_rate` | objective | deferred | Warnings page |
|
||||
| `objective.assumed_completion_tokens` | objective | deferred | Warnings page |
|
||||
| `objective.billing_reset_day` | objective | deferred | Warnings page |
|
||||
| `objective.cache_rate_warn_margin` | objective | deferred | Warnings page |
|
||||
| `objective.cache_rate_warn_min_observations` | objective | deferred | Warnings page |
|
||||
| `objective.cache_rate_window_hours` | objective | deferred | Warnings page |
|
||||
| `objective.cost_calibration_min_observations` | objective | deferred | Warnings page |
|
||||
| `objective.cost_calibration_window_hours` | objective | deferred | Warnings page |
|
||||
| `objective.credit_attenuation.enabled` | objective | deferred | Warnings page |
|
||||
| `objective.credit_attenuation.max_multiplier` | objective | deferred | Warnings page |
|
||||
| `objective.credit_attenuation.refresh_seconds` | objective | deferred | Warnings page |
|
||||
| `objective.credit_attenuation.soft_floor_usd` | objective | deferred | Warnings page |
|
||||
| `objective.credit_attenuation.zero_floor_usd` | objective | deferred | Warnings page |
|
||||
| `objective.cumulative_spend_warn_min_rows` | objective | deferred | Warnings page |
|
||||
| `objective.cumulative_spend_warn_usd` | objective | deferred | Warnings page |
|
||||
| `objective.incumbent_rate_min_observations` | objective | deferred | Warnings page |
|
||||
| `objective.incumbent_rate_refresh_seconds` | objective | deferred | Warnings page |
|
||||
| `objective.latency_min_observations` | objective | deferred | Warnings page |
|
||||
| `objective.latency_window_hours` | objective | deferred | Warnings page |
|
||||
| `objective.plan_pace_warn_ratio` | objective | deferred | Warnings page |
|
||||
| `objective.proficiency_depth_warn_min_rows` | objective | deferred | Warnings page |
|
||||
| `objective.proficiency_depth_warn_min_samples` | objective | deferred | Warnings page |
|
||||
| `objective.quota_burn_min_segment_hours` | objective | deferred | Warnings page |
|
||||
| `objective.quota_burn_min_segment_samples` | objective | deferred | Warnings page |
|
||||
| `objective.quota_burn_window_hours` | objective | deferred | Warnings page |
|
||||
| `objective.quota_runway_warning_hours` | objective | deferred | Warnings page |
|
||||
| `objective.rejection_warning_baseline_hours` | objective | deferred | Warnings page |
|
||||
| `objective.rejection_warning_min_count` | objective | deferred | Warnings page |
|
||||
| `objective.rejection_warning_window_hours` | objective | deferred | Warnings page |
|
||||
| `objective.selection_coverage_window_hours` | objective | deferred | Warnings page |
|
||||
|
||||
## Notes
|
||||
|
||||
- A section moves from "outside gate" to "in scope" the moment the portal adds
|
||||
its first control there. At that point every scalar under it must be covered,
|
||||
excused, or removed from this tracker into `DELIBERATELY_NOT_IN_ADMIN`.
|
||||
- Deployment wiring (endpoints, model ids, credentials, paths, devices) is
|
||||
deliberately off the admin allowlist. Those knobs should stay
|
||||
`excused-with-reason` even after their section is reached.
|
||||
- The Warnings page is not built yet. When it is, these `objective.*` knobs
|
||||
should move from `DELIBERATELY_NOT_IN_ADMIN` into registry or allowlist
|
||||
entries, and this table should shrink.
|
||||
202
src/admin.py
202
src/admin.py
@@ -46,6 +46,14 @@ import poller
|
||||
import routing
|
||||
from config import (
|
||||
BUILTIN_PROFILES,
|
||||
COOLDOWN_CEILING,
|
||||
COOLDOWN_FLOOR,
|
||||
DEGRADED_WARN_MIN_CEILING,
|
||||
DEGRADED_WARN_MIN_FLOOR,
|
||||
DEGRADED_WARN_THRESHOLD_FLOOR,
|
||||
DEGRADED_WARN_THRESHOLD_MAX,
|
||||
FALLBACK_TIER_MAX,
|
||||
FALLBACK_TIER_MIN,
|
||||
DispatchProvider,
|
||||
FlexPreference,
|
||||
RouterConfig,
|
||||
@@ -356,6 +364,7 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = {
|
||||
"pinch_enabled": ("pinch", "enabled"),
|
||||
"pinch_prefix_probe": ("pinch", "prefix_probe"),
|
||||
"pinch_relevance_enabled": ("pinch", "relevance", "enabled"),
|
||||
"classifier_context_framing": ("classifier", "context_framing"),
|
||||
"incumbent_cache_pricing": ("objective", "incumbent_cache_pricing"),
|
||||
}
|
||||
|
||||
@@ -384,14 +393,20 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = {
|
||||
# representations" would stop holding the moment this endpoint could write a
|
||||
# None onto cfg.
|
||||
#
|
||||
# knob -> (cfg path, low, high, path whose value ``null`` resolves to)
|
||||
_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, tuple[str, ...]]] = {
|
||||
# knob -> (cfg path, low, high, path whose value ``null`` resolves to, if any)
|
||||
_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, Optional[tuple[str, ...]]]] = {
|
||||
"incumbent_challenger_cache_rate": (
|
||||
("objective", "incumbent_challenger_cache_rate"),
|
||||
0.0,
|
||||
1.0,
|
||||
("objective", "assumed_cache_rate"),
|
||||
),
|
||||
"classifier_degraded_warn_threshold": (
|
||||
("classifier", "degraded_warn_threshold"),
|
||||
DEGRADED_WARN_THRESHOLD_FLOOR,
|
||||
DEGRADED_WARN_THRESHOLD_MAX,
|
||||
None,
|
||||
),
|
||||
}
|
||||
|
||||
# Runtime knobs whose value is a whole number. A SEPARATE table from
|
||||
@@ -427,6 +442,21 @@ _INT_KNOBS: dict[str, tuple[tuple[str, ...], int, int]] = {
|
||||
STALENESS_SECONDS_MIN,
|
||||
STALENESS_SECONDS_MAX,
|
||||
),
|
||||
"classifier_cooldown_seconds": (
|
||||
("classifier", "cooldown_seconds"),
|
||||
COOLDOWN_FLOOR,
|
||||
COOLDOWN_CEILING,
|
||||
),
|
||||
"classifier_fallback_tier": (
|
||||
("classifier", "fallback_tier"),
|
||||
FALLBACK_TIER_MIN,
|
||||
FALLBACK_TIER_MAX,
|
||||
),
|
||||
"classifier_degraded_warn_min": (
|
||||
("classifier", "degraded_warn_min"),
|
||||
DEGRADED_WARN_MIN_FLOOR,
|
||||
DEGRADED_WARN_MIN_CEILING,
|
||||
),
|
||||
}
|
||||
|
||||
_LOCAL_COMPUTE_KNOB = "local_compute_enabled"
|
||||
@@ -489,6 +519,87 @@ _RUNTIME_KNOB_PATHS: dict[str, tuple[str, ...]] = {
|
||||
_PROFILE_KNOB: _PROFILE_PATH,
|
||||
}
|
||||
|
||||
# --- knob category taxonomy ----------------------------------------------------
|
||||
# Every covered knob, keyed by its dotted config path, with its category id and
|
||||
# whether it belongs behind the Advanced accordion. Six categories:
|
||||
# routing_quality_cost, classification, local_hardware,
|
||||
# caching_context, safety_nets, watchdog.
|
||||
# Used by the admin API to add category + advanced to knob metadata, and by the
|
||||
# frontend to group knobs into sectioned accordions.
|
||||
# 28 existing + 7 new classifier knobs = 35 entries total.
|
||||
_KNOB_CATEGORY: dict[str, dict] = {
|
||||
# -------- routing_quality_cost (10) -------------------------------------------
|
||||
"logging.log_route_decisions": {"category": "routing_quality_cost", "advanced": False},
|
||||
"logging.log_energy_observations": {"category": "routing_quality_cost", "advanced": False},
|
||||
"logging.level": {"category": "routing_quality_cost", "advanced": False},
|
||||
"objective.quality_tolerance": {"category": "routing_quality_cost", "advanced": False},
|
||||
"objective.max_energy_per_request": {"category": "routing_quality_cost", "advanced": False},
|
||||
"objective.plan_kwh_per_period": {"category": "routing_quality_cost", "advanced": False},
|
||||
"objective.incumbent_cache_pricing": {"category": "routing_quality_cost", "advanced": False},
|
||||
"objective.incumbent_challenger_cache_rate": {"category": "routing_quality_cost", "advanced": False},
|
||||
"routing.default_flex_preference": {"category": "routing_quality_cost", "advanced": False},
|
||||
"routing.default_profile": {"category": "routing_quality_cost", "advanced": False},
|
||||
# -------- classification (7) --------------------------------------------------
|
||||
"classifier.context_framing": {"category": "classification", "advanced": False},
|
||||
"classifier.cooldown_seconds": {"category": "classification", "advanced": False},
|
||||
"classifier.fallback_tier": {"category": "classification", "advanced": False},
|
||||
"classifier.degraded_warn_min": {"category": "classification", "advanced": False},
|
||||
"classifier.degraded_warn_threshold": {"category": "classification", "advanced": False},
|
||||
"classifier.fallback_category": {"category": "classification", "advanced": False},
|
||||
"classifier.max_input_chars": {"category": "classification", "advanced": False},
|
||||
# -------- safety_nets (1) -----------------------------------------------------
|
||||
"circuit_breaker.enabled": {"category": "safety_nets", "advanced": False},
|
||||
# -------- caching_context (5) -------------------------------------------------
|
||||
"session_cache.enabled": {"category": "caching_context", "advanced": False},
|
||||
"session_cache.staleness_seconds": {"category": "caching_context", "advanced": False},
|
||||
"pinch.enabled": {"category": "caching_context", "advanced": False},
|
||||
"pinch.prefix_probe": {"category": "caching_context", "advanced": True},
|
||||
"pinch.relevance.enabled": {"category": "caching_context", "advanced": True},
|
||||
# -------- local_hardware (3) --------------------------------------------------
|
||||
"local_compute.enabled": {"category": "local_hardware", "advanced": False},
|
||||
"verification.local_llm_enabled": {"category": "local_hardware", "advanced": False},
|
||||
"local_vision.enabled": {"category": "local_hardware", "advanced": False},
|
||||
# -------- watchdog (9) --------------------------------------------------------
|
||||
"watchdog.enabled": {"category": "watchdog", "advanced": False},
|
||||
"watchdog.local_llm_enabled": {"category": "watchdog", "advanced": False},
|
||||
"watchdog.detector.window": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.dup_min": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.top_min": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.top_min_ro": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.cum_min": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.cover_min": {"category": "watchdog", "advanced": True},
|
||||
"watchdog.detector.min_calls": {"category": "watchdog", "advanced": True},
|
||||
}
|
||||
|
||||
# --- card-backed paths -------------------------------------------------------------
|
||||
# Config keys whose controls live in their own dedicated admin card (classifier
|
||||
# mode selector, cloud primary, cloud fallback, encoder, decision) rather than
|
||||
# in the generic knob table. Each maps to the HTML element id the frontend
|
||||
# uses for it, so the test can verify the id still exists in controls.html.
|
||||
_CARD_BACKED_PATHS: dict[str, str] = {
|
||||
"classifier.mode": "classifier-mode-select",
|
||||
"classifier.cloud_primary_auto": "classifier-cloud-auto",
|
||||
"classifier.cloud_primary.base_url": "classifier-cloud-base-url",
|
||||
"classifier.cloud_primary.model": "classifier-cloud-model",
|
||||
"classifier.cloud_primary.api_key_env": "classifier-cloud-api-key-env",
|
||||
"classifier.cloud_primary.timeout_seconds": "classifier-cloud-timeout",
|
||||
"classifier.cloud_fallback.base_url": "cf-base-url",
|
||||
"classifier.cloud_fallback.model": "cf-model",
|
||||
"classifier.cloud_fallback.api_key_env": "cf-api-key-env",
|
||||
"classifier.cloud_fallback.timeout_seconds": "cf-timeout",
|
||||
"classifier.cloud_fallback.max_output_tokens": "cf-max-tokens",
|
||||
"classifier.encoder.model": "classifier-encoder-model",
|
||||
"classifier.encoder.device": "classifier-encoder-device",
|
||||
"classifier.encoder.confidence_min": "classifier-encoder-threshold",
|
||||
"classifier.decision.base_url": "classifier-decision-base-url",
|
||||
"classifier.decision.model": "classifier-decision-model",
|
||||
"classifier.decision.num_ctx": "classifier-decision-num-ctx",
|
||||
"classifier.decision.timeout_s": "classifier-decision-timeout-s",
|
||||
"classifier.decision.confidence_min": "classifier-decision-confidence-min",
|
||||
"classifier.decision.coverage_min": "classifier-decision-coverage-min",
|
||||
"classifier.decision.tier_enabled": "classifier-decision-tier-enabled",
|
||||
}
|
||||
|
||||
# --- persisted config allowlist -------------------------------------------------
|
||||
# Dotted config.yaml paths an operator is allowed to edit. Everything else —
|
||||
# classifier/verification/local_vision URLs and model names, api_key_env,
|
||||
@@ -517,6 +628,13 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = {
|
||||
"pinch.enabled": ("pinch", "enabled"),
|
||||
"pinch.prefix_probe": ("pinch", "prefix_probe"),
|
||||
"pinch.relevance.enabled": ("pinch", "relevance", "enabled"),
|
||||
"classifier.context_framing": ("classifier", "context_framing"),
|
||||
"classifier.cooldown_seconds": ("classifier", "cooldown_seconds"),
|
||||
"classifier.fallback_tier": ("classifier", "fallback_tier"),
|
||||
"classifier.degraded_warn_min": ("classifier", "degraded_warn_min"),
|
||||
"classifier.degraded_warn_threshold": ("classifier", "degraded_warn_threshold"),
|
||||
"classifier.fallback_category": ("classifier", "fallback_category"),
|
||||
"classifier.max_input_chars": ("classifier", "max_input_chars"),
|
||||
"routing.default_flex_preference": ("routing", "default_flex_preference"),
|
||||
"routing.default_profile": ("routing", "default_profile"),
|
||||
# Watchdog — persisted config knobs for the model-response watchdog.
|
||||
@@ -533,23 +651,36 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = {
|
||||
|
||||
# Order preserves config.yaml layout for the GET response.
|
||||
_CONFIG_GET_ORDER: list[str] = [
|
||||
# routing_quality_cost
|
||||
"logging.level",
|
||||
"objective.quality_tolerance",
|
||||
"objective.max_energy_per_request",
|
||||
"objective.plan_kwh_per_period",
|
||||
"objective.incumbent_cache_pricing",
|
||||
"objective.incumbent_challenger_cache_rate",
|
||||
"circuit_breaker.enabled",
|
||||
"routing.default_flex_preference",
|
||||
"routing.default_profile",
|
||||
# classification
|
||||
"classifier.context_framing",
|
||||
"classifier.cooldown_seconds",
|
||||
"classifier.fallback_tier",
|
||||
"classifier.degraded_warn_min",
|
||||
"classifier.degraded_warn_threshold",
|
||||
"classifier.fallback_category",
|
||||
"classifier.max_input_chars",
|
||||
# caching_context
|
||||
"session_cache.enabled",
|
||||
"session_cache.staleness_seconds",
|
||||
"local_compute.enabled",
|
||||
"verification.local_llm_enabled",
|
||||
"local_vision.enabled",
|
||||
"pinch.enabled",
|
||||
"pinch.prefix_probe",
|
||||
"pinch.relevance.enabled",
|
||||
"routing.default_flex_preference",
|
||||
"routing.default_profile",
|
||||
# local_hardware
|
||||
"local_compute.enabled",
|
||||
"verification.local_llm_enabled",
|
||||
"local_vision.enabled",
|
||||
# safety_nets
|
||||
"circuit_breaker.enabled",
|
||||
# watchdog
|
||||
"watchdog.enabled",
|
||||
"watchdog.local_llm_enabled",
|
||||
"watchdog.detector.window",
|
||||
@@ -992,6 +1123,22 @@ def _runtime_state(cfg: Any) -> dict:
|
||||
"pinch_relevance_enabled": _get_at(
|
||||
cfg, _BOOL_KNOBS["pinch_relevance_enabled"]
|
||||
),
|
||||
# --- classifier runtime toggles (Phase B) -------------------------------
|
||||
"classifier_context_framing": _get_at(
|
||||
cfg, _BOOL_KNOBS["classifier_context_framing"]
|
||||
),
|
||||
"classifier_cooldown_seconds": _get_at(
|
||||
cfg, _INT_KNOBS["classifier_cooldown_seconds"][0]
|
||||
),
|
||||
"classifier_fallback_tier": _get_at(
|
||||
cfg, _INT_KNOBS["classifier_fallback_tier"][0]
|
||||
),
|
||||
"classifier_degraded_warn_min": _get_at(
|
||||
cfg, _INT_KNOBS["classifier_degraded_warn_min"][0]
|
||||
),
|
||||
"classifier_degraded_warn_threshold": _get_at(
|
||||
cfg, _FLOAT_KNOBS["classifier_degraded_warn_threshold"][0]
|
||||
),
|
||||
# Kept adjacent and in this order on purpose: the gate reads above the
|
||||
# dial it enables, which is what lets the UI note say so in four words
|
||||
# instead of a paragraph.
|
||||
@@ -2434,6 +2581,8 @@ def build_router(
|
||||
"persisted": persisted[key],
|
||||
"runtime": runtime[key],
|
||||
"config_key": ".".join(_RUNTIME_KNOB_PATHS[key]),
|
||||
"category": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("category"),
|
||||
"advanced": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("advanced"),
|
||||
}
|
||||
for key in persisted
|
||||
}
|
||||
@@ -2455,30 +2604,36 @@ def build_router(
|
||||
path, low, high, neutral_path = _FLOAT_KNOBS[knob]
|
||||
value = body.value
|
||||
if value is None:
|
||||
# Blank means NEUTRAL, not zero. Resolved here rather than
|
||||
# stored as None so cfg holds one representation of neutral,
|
||||
# matching what load_config would have produced.
|
||||
value = float(_get_at(cfg, neutral_path))
|
||||
if neutral_path is not None:
|
||||
# Blank means NEUTRAL, not zero. Resolved here rather than
|
||||
# stored as None so cfg holds one representation of neutral,
|
||||
# matching what load_config would have produced.
|
||||
value = float(_get_at(cfg, neutral_path))
|
||||
else:
|
||||
# No neutral for this knob: clearing the field is not a
|
||||
# setting, and _get_at(cfg, None) would 500 on the lookup.
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"{knob} has no blank setting; send a number in [{low}, {high}]",
|
||||
)
|
||||
else:
|
||||
# Only a knob with a neutral may advertise ``null``; telling
|
||||
# the operator to send it for a knob that 422s it is a lie.
|
||||
or_null = ", or null for neutral" if neutral_path is not None else ""
|
||||
null_note = f" (null means neutral, not {low})" if neutral_path is not None else ""
|
||||
# bool is a subclass of int in Python, so `True` would sail
|
||||
# through an isinstance(value, (int, float)) check and land as
|
||||
# 1.0 -- a real dial setting, silently, from a checkbox body.
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
f"{knob} expects a number in [{low}, {high}], "
|
||||
f"or null for neutral"
|
||||
),
|
||||
detail=f"{knob} expects a number in [{low}, {high}]{or_null}",
|
||||
)
|
||||
value = float(value)
|
||||
if not (low <= value <= high):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
f"{knob} must be in [{low}, {high}], got {value} "
|
||||
f"(null means neutral, not {low})"
|
||||
),
|
||||
detail=f"{knob} must be in [{low}, {high}], got {value}{null_note}",
|
||||
)
|
||||
_set_at(cfg, path, value)
|
||||
return {"ok": True, knob: value}
|
||||
@@ -2585,7 +2740,12 @@ def build_router(
|
||||
source_map[key] = "base"
|
||||
merged = _load_merged_config_store(config_path, config_local_path)
|
||||
return {
|
||||
key: {"value": _dict_get_at(merged, path), "source": source_map[key]}
|
||||
key: {
|
||||
"value": _dict_get_at(merged, path),
|
||||
"source": source_map[key],
|
||||
"category": _KNOB_CATEGORY.get(key, {}).get("category"),
|
||||
"advanced": _KNOB_CATEGORY.get(key, {}).get("advanced"),
|
||||
}
|
||||
for key, path in _CONFIG_ALLOWLIST.items()
|
||||
}
|
||||
|
||||
|
||||
@@ -1163,6 +1163,23 @@ class LocalDecisionConfig(StrictModel):
|
||||
DEGRADED_WARN_MIN_FLOOR = 1
|
||||
DEGRADED_WARN_THRESHOLD_MIN_EXCLUSIVE = 0.0
|
||||
DEGRADED_WARN_THRESHOLD_MAX = 1.0
|
||||
# Bounds for the classifier cooldown window. At least 1s, at most 1h —
|
||||
# a cloud-classifier retry during a sustained local outage cannot burn
|
||||
# more than one attempt per hour per process.
|
||||
COOLDOWN_FLOOR = 1
|
||||
COOLDOWN_CEILING = 3600
|
||||
# Bounds for the fallback tier. Must correspond to a real tier in the
|
||||
# routing table (tier 1 = cheap+small, tier 3 = frontier).
|
||||
FALLBACK_TIER_MIN = 1
|
||||
FALLBACK_TIER_MAX = 3
|
||||
# Upper bound on the degradation-warning sample-size floor. A value past
|
||||
# 10 000 classifications per window is a configuration mistake regardless
|
||||
# of traffic volume.
|
||||
DEGRADED_WARN_MIN_CEILING = 10000
|
||||
# Minimum value for the degradation-warning threshold. A share below this
|
||||
# fires on the very first degraded classification in the window, which is
|
||||
# never useful — the operator already knows one failure happened.
|
||||
DEGRADED_WARN_THRESHOLD_FLOOR = 0.001
|
||||
|
||||
|
||||
class ClassifierConfig(StrictModel):
|
||||
@@ -1265,7 +1282,35 @@ class ClassifierConfig(StrictModel):
|
||||
@classmethod
|
||||
def degraded_warn_min_positive(cls, v: int) -> int:
|
||||
if v < DEGRADED_WARN_MIN_FLOOR:
|
||||
raise ValueError("classifier.degraded_warn_min must be at least 1")
|
||||
raise ValueError(
|
||||
"classifier.degraded_warn_min must be at least "
|
||||
f"{DEGRADED_WARN_MIN_FLOOR}, got {v}"
|
||||
)
|
||||
if v > DEGRADED_WARN_MIN_CEILING:
|
||||
raise ValueError(
|
||||
"classifier.degraded_warn_min must be at most "
|
||||
f"{DEGRADED_WARN_MIN_CEILING}, got {v}"
|
||||
)
|
||||
return v
|
||||
|
||||
@field_validator("cooldown_seconds")
|
||||
@classmethod
|
||||
def cooldown_seconds_in_range(cls, v: int) -> int:
|
||||
if not (COOLDOWN_FLOOR <= v <= COOLDOWN_CEILING):
|
||||
raise ValueError(
|
||||
"classifier.cooldown_seconds must be between "
|
||||
f"{COOLDOWN_FLOOR} and {COOLDOWN_CEILING}, got {v}"
|
||||
)
|
||||
return v
|
||||
|
||||
@field_validator("fallback_tier")
|
||||
@classmethod
|
||||
def fallback_tier_in_range(cls, v: int) -> int:
|
||||
if not (FALLBACK_TIER_MIN <= v <= FALLBACK_TIER_MAX):
|
||||
raise ValueError(
|
||||
"classifier.fallback_tier must be between "
|
||||
f"{FALLBACK_TIER_MIN} and {FALLBACK_TIER_MAX}, got {v}"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
|
||||
@@ -106,11 +106,13 @@ def test_config_GET_returns_allowlisted_values(client):
|
||||
"routing.default_flex_preference",
|
||||
):
|
||||
assert key in body
|
||||
assert body["logging.level"] == {"value": "info", "source": "base"}
|
||||
assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base"}
|
||||
assert body["logging.level"] == {"value": "info", "source": "base", "category": "routing_quality_cost", "advanced": False}
|
||||
assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base", "category": "routing_quality_cost", "advanced": False}
|
||||
assert body["routing.default_flex_preference"] == {
|
||||
"value": "auto",
|
||||
"source": "base",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
|
||||
@@ -154,14 +156,14 @@ def test_config_pinch_prefix_probe_round_trips_through_the_real_validator(
|
||||
local_yaml = config_yaml.with_name("config.local.yaml")
|
||||
|
||||
before = tc.get("/admin/api/config").json()
|
||||
assert before["pinch.prefix_probe"] == {"value": True, "source": "base"}
|
||||
assert before["pinch.prefix_probe"] == {"value": True, "source": "base", "category": "caching_context", "advanced": True}
|
||||
|
||||
resp = tc.post("/admin/api/config/pinch.prefix_probe", json={"value": False})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["value"] is False
|
||||
|
||||
after = tc.get("/admin/api/config").json()
|
||||
assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay"}
|
||||
assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay", "category": "caching_context", "advanced": True}
|
||||
|
||||
# The base file never carries an admin write.
|
||||
assert yaml.safe_load(config_yaml.read_text())["pinch"]["prefix_probe"] is True
|
||||
@@ -200,14 +202,14 @@ def test_config_local_vision_enabled_round_trips_through_the_real_validator(clie
|
||||
local_yaml = config_yaml.with_name("config.local.yaml")
|
||||
|
||||
before = tc.get("/admin/api/config").json()
|
||||
assert before["local_vision.enabled"] == {"value": True, "source": "base"}
|
||||
assert before["local_vision.enabled"] == {"value": True, "source": "base", "category": "local_hardware", "advanced": False}
|
||||
|
||||
resp = tc.post("/admin/api/config/local_vision.enabled", json={"value": False})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["value"] is False
|
||||
|
||||
after = tc.get("/admin/api/config").json()
|
||||
assert after["local_vision.enabled"] == {"value": False, "source": "overlay"}
|
||||
assert after["local_vision.enabled"] == {"value": False, "source": "overlay", "category": "local_hardware", "advanced": False}
|
||||
|
||||
# The base file never carries an admin write.
|
||||
assert yaml.safe_load(config_yaml.read_text())["local_vision"]["enabled"] is True
|
||||
@@ -239,12 +241,16 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client):
|
||||
assert before["objective.incumbent_cache_pricing"] == {
|
||||
"value": False,
|
||||
"source": "base",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
# Shipped blank in config.yaml, which YAML reads as null.
|
||||
assert before["objective.incumbent_challenger_cache_rate"] == {
|
||||
"value": None,
|
||||
"source": "base",
|
||||
}
|
||||
"value": None,
|
||||
"source": "base",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
assert (
|
||||
tc.post(
|
||||
@@ -265,11 +271,15 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client):
|
||||
assert after["objective.incumbent_cache_pricing"] == {
|
||||
"value": True,
|
||||
"source": "overlay",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
assert after["objective.incumbent_challenger_cache_rate"] == {
|
||||
"value": 0.0,
|
||||
"source": "overlay",
|
||||
}
|
||||
"value": 0.0,
|
||||
"source": "overlay",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
# The base file never carries an admin write.
|
||||
base = yaml.safe_load(config_yaml.read_text())["objective"]
|
||||
@@ -325,6 +335,8 @@ def test_config_blank_challenger_dial_persists_as_null_not_zero(client):
|
||||
assert body["objective.incumbent_challenger_cache_rate"] == {
|
||||
"value": None,
|
||||
"source": "overlay",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
loaded = load_config(str(config_yaml), include_overlay=True)
|
||||
@@ -382,7 +394,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client):
|
||||
tc, config_yaml = client
|
||||
|
||||
before = tc.get("/admin/api/config").json()
|
||||
assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base"}
|
||||
assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base", "category": "caching_context", "advanced": False}
|
||||
|
||||
assert (
|
||||
tc.post(
|
||||
@@ -393,7 +405,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client):
|
||||
)
|
||||
|
||||
after = tc.get("/admin/api/config").json()
|
||||
assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay"}
|
||||
assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay", "category": "caching_context", "advanced": False}
|
||||
|
||||
# The shipped default never moves; the admin write lands in the overlay.
|
||||
base = yaml.safe_load(config_yaml.read_text())["session_cache"]
|
||||
@@ -476,6 +488,8 @@ def test_config_GET_includes_routing_default_profile(client):
|
||||
assert body["routing.default_profile"] == {
|
||||
"value": "default",
|
||||
"source": "base",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -460,6 +460,95 @@ def test_merge_knob_rows_value_normalization(persisted, runtime, expect_drift):
|
||||
)
|
||||
|
||||
|
||||
SEED_ADV_COLLAPSED_BEGIN = "/* SEED_ADV_COLLAPSED:BEGIN */"
|
||||
SEED_ADV_COLLAPSED_END = "/* SEED_ADV_COLLAPSED:END */"
|
||||
|
||||
|
||||
def _assert_against_seed_units(assertion_js: str) -> subprocess.CompletedProcess[str]:
|
||||
"""Assert ``assertion_js`` against the real seedAdvancedCollapsed source.
|
||||
|
||||
The function lives in controls.html between the SEED_ADV_COLLAPSED markers
|
||||
and is self-contained: it takes the grouped rows and the two Sets as
|
||||
arguments and touches no DOM. ``group`` below mirrors the grouping step in
|
||||
renderKnobsTable so each test can drive it payload by payload.
|
||||
"""
|
||||
return _run_node(
|
||||
"const assert = require('assert');\n"
|
||||
+ _extract_between(CONTROLS_HTML, SEED_ADV_COLLAPSED_BEGIN, SEED_ADV_COLLAPSED_END)
|
||||
+ "\n"
|
||||
"const IDS = ['caching_context', 'watchdog'];\n"
|
||||
"function group(rows) {\n"
|
||||
" const byCategory = {};\n"
|
||||
" for (const [key, row] of Object.entries(rows)) {\n"
|
||||
" const item = row.runtimeItem || row.persistedItem;\n"
|
||||
" (byCategory[item.category] = byCategory[item.category] || []).push(key);\n"
|
||||
" }\n"
|
||||
" return byCategory;\n"
|
||||
"}\n"
|
||||
"function render(rows, seeded, collapsed) {\n"
|
||||
" seedAdvancedCollapsed(group(rows), rows, IDS, seeded, collapsed);\n"
|
||||
"}\n"
|
||||
# First render: runtime payload only. Watchdog has a plain runtime knob
|
||||
# and NO advanced row (its advanced knobs are persisted-only);
|
||||
# Caching has an advanced runtime knob.
|
||||
"const runtimeOnly = {\n"
|
||||
" 'session_cache.staleness_seconds': { runtimeItem: { category: 'caching_context', advanced: true }, persistedItem: null },\n"
|
||||
" 'watchdog.enabled': { runtimeItem: { category: 'watchdog', advanced: false }, persistedItem: null },\n"
|
||||
"};\n"
|
||||
# Second render: the config payload lands and adds the persisted-only
|
||||
# advanced watchdog knobs.
|
||||
"const withConfig = {\n"
|
||||
" ...runtimeOnly,\n"
|
||||
" 'watchdog.detector.dup_min': { runtimeItem: null, persistedItem: { category: 'watchdog', advanced: true } },\n"
|
||||
"};\n"
|
||||
+ assertion_js
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_seed_advanced_collapsed_seeds_each_category_when_its_advanced_rows_first_appear():
|
||||
"""A config-only render after a runtime-only render must leave the Watchdog
|
||||
Advanced section collapsed. Seeding once on the first render marked
|
||||
Watchdog seeded while it had no advanced rows, so it loaded EXPANDED."""
|
||||
_assert_against_seed_units(
|
||||
"const seeded = new Set(), collapsed = new Set();\n"
|
||||
"render(runtimeOnly, seeded, collapsed);\n"
|
||||
"assert.deepStrictEqual([...collapsed], ['caching_context.adv']);\n"
|
||||
"assert.ok(!seeded.has('watchdog'), 'no advanced rows yet: watchdog must stay unseeded');\n"
|
||||
"render(withConfig, seeded, collapsed);\n"
|
||||
"assert.deepStrictEqual([...collapsed].sort(), ['caching_context.adv', 'watchdog.adv']);"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_seed_advanced_collapsed_leaves_a_user_toggled_section_alone():
|
||||
"""Once a category is seeded, a toggled-open section stays open across
|
||||
every later re-render (the page re-renders on a 30s poll)."""
|
||||
_assert_against_seed_units(
|
||||
"const seeded = new Set(), collapsed = new Set();\n"
|
||||
"render(runtimeOnly, seeded, collapsed);\n"
|
||||
"render(withConfig, seeded, collapsed);\n"
|
||||
"collapsed.delete('watchdog.adv');\n"
|
||||
"collapsed.delete('caching_context.adv');\n"
|
||||
"render(withConfig, seeded, collapsed);\n"
|
||||
"render(withConfig, seeded, collapsed);\n"
|
||||
"assert.strictEqual(collapsed.size, 0);"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_seed_advanced_collapsed_ignores_a_category_with_no_advanced_rows():
|
||||
"""A category with only plain knobs gets no Advanced entry and is never
|
||||
marked seeded, so an advanced row arriving later is still seeded."""
|
||||
_assert_against_seed_units(
|
||||
"const seeded = new Set(), collapsed = new Set();\n"
|
||||
"const plain = { 'watchdog.enabled': runtimeOnly['watchdog.enabled'] };\n"
|
||||
"render(plain, seeded, collapsed);\n"
|
||||
"assert.strictEqual(collapsed.size, 0);\n"
|
||||
"assert.strictEqual(seeded.size, 0);"
|
||||
)
|
||||
|
||||
|
||||
INDEX_HTML = ROOT / "admin" / "frontend" / "index.html"
|
||||
LIFT_A2_HELPERS_BEGIN = "/* LIFT_A2_HELPERS:BEGIN */"
|
||||
LIFT_A2_HELPERS_END = "/* LIFT_A2_HELPERS:END */"
|
||||
|
||||
@@ -12,10 +12,10 @@ So this file is the forcing function, in the shape ``test_tui_schema_drift`` and
|
||||
message that NAMES the thing, so whoever broke it learns what they broke without
|
||||
reading the test.
|
||||
|
||||
- covered — ``admin._CONFIG_ALLOWLIST`` (persisted to the overlay) union the
|
||||
runtime registries ``_BOOL_KNOBS``, ``_FLOAT_KNOBS`` and ``_INT_KNOBS``
|
||||
(in-memory, reverts on restart). Derived, never hand-copied: a hand-copy is
|
||||
one more thing to drift.
|
||||
- covered — ``admin._CONFIG_ALLOWLIST`` (persisted), the runtime registries
|
||||
``_BOOL_KNOBS``, ``_FLOAT_KNOBS``, ``_INT_KNOBS`` (in-memory), and
|
||||
``_CARD_BACKED_PATHS`` (dedicated admin cards). Derived, never hand-copied:
|
||||
a hand-copy is one more thing to drift.
|
||||
- ``DELIBERATELY_NOT_IN_ADMIN`` — in-scope knobs with no control, each with a
|
||||
REASON STRING. The escape clause is load-bearing rather than hedging:
|
||||
``objective.credit_attenuation.enabled`` is deliberately off the allowlist
|
||||
@@ -50,10 +50,19 @@ against. Two clauses cut it to the knobs an operator would plausibly turn:
|
||||
own dedicated admin card and endpoint pair (``/admin/api/classifier-config``,
|
||||
``/admin/api/cloud-fallback-config``), so measuring it against the GENERIC
|
||||
allowlist would report every field as missing while the card covers it.
|
||||
``local_vision`` is reached now that ``local_vision.enabled`` has a persisted
|
||||
control; the rest of the section (``base_url``, ``api_key_env``, ``model``)
|
||||
is deployment wiring out by clause 2, and the timeout/image limits are
|
||||
excused in ``DELIBERATELY_NOT_IN_ADMIN``.
|
||||
``local_vision`` is reached now that ``local_vision.enabled`` has a persisted
|
||||
control; the rest of the section (``base_url``, ``api_key_env``, ``model``)
|
||||
is deployment wiring out by clause 2, and the timeout/image limits are
|
||||
excused in ``DELIBERATELY_NOT_IN_ADMIN``.
|
||||
``classifier`` has three coverage sources. **Registry knobs** (``_BOOL_KNOBS``,
|
||||
``_INT_KNOBS``, ``_FLOAT_KNOBS``): runtime toggles like context_framing,
|
||||
cooldown_seconds and fallback_tier. **Allowlist knobs** (``_CONFIG_ALLOWLIST``):
|
||||
the same seven persisted to the overlay for restart durability. **Card-backed
|
||||
paths** (``_CARD_BACKED_PATHS``): knobs with their own dedicated admin card
|
||||
and endpoint pair (classifier mode, cloud primary, cloud fallback, encoder
|
||||
and decision fields). The remaining classifier scalars (timeout_seconds,
|
||||
temperature, max_output_tokens and encoder.tier_from_features) are wiring or
|
||||
reproducibility settings excused by name in ``DELIBERATELY_NOT_IN_ADMIN``.
|
||||
The known limitation: a brand-new section with no control at all is out of
|
||||
scope and unchecked. The first control added under it drags every one of its
|
||||
knobs into scope at once, which is the intended moment to decide.
|
||||
@@ -347,6 +356,34 @@ DELIBERATELY_NOT_IN_ADMIN: dict[str, str] = {
|
||||
"lookback for the conversation adoption counter in /metrics; a "
|
||||
"read-only window that shapes a report, not a routing dial."
|
||||
),
|
||||
# --- classifier: wiring with no control or reproducibility scalars -------
|
||||
"classifier.timeout_seconds": (
|
||||
"deployment tuning paired with model; the operator's decision is "
|
||||
"which classifier fits the deployment, and the timeout follows it."
|
||||
),
|
||||
"classifier.temperature": (
|
||||
"reproducibility, not tunable (user Q6); per-request temperature 0 "
|
||||
"is what makes the classification deterministic, and deviating from "
|
||||
"it in a form would produce nondeterministic routing decisions."
|
||||
),
|
||||
"classifier.max_output_tokens": (
|
||||
"safety cap against reasoning cascade (user Q6); a classifier that "
|
||||
"spends its generation budget on reasoning returns no parsible JSON, "
|
||||
"and the cascade catches that silently -- widening the cap from the "
|
||||
"UI would make the hidden failure more expensive, not fix it."
|
||||
),
|
||||
"classifier.cloud_primary.max_output_tokens": (
|
||||
"not an input in the card UI (encoder-level setting); only meaningful "
|
||||
"when mode is cloud_llm and cloud_primary_auto is off, and even then "
|
||||
"it is a cap inherited from the card backend, not a knob the mode "
|
||||
"selector exposes for independent tuning."
|
||||
),
|
||||
"classifier.encoder.tier_from_features": (
|
||||
"design sketch only, not wired; the field and its companion "
|
||||
"tier_feature_fields exist in the Pydantic model as a placeholder "
|
||||
"for a future feature that is not implemented anywhere outside "
|
||||
"config.py -- no admin control because it would control nothing real."
|
||||
),
|
||||
# --- deployment wiring: set in config, never in admin --------------------
|
||||
"watchdog.dashboard_base_url": (
|
||||
"deployment wiring, set in config, not admin"
|
||||
@@ -412,12 +449,16 @@ def _covered_paths() -> set[str]:
|
||||
reading only ``_BOOL_KNOBS`` would report a runtime-only numeric knob as
|
||||
having no control at all — a false failure that invites exactly the wrong
|
||||
fix, an excuse entry for a knob that is in fact covered.
|
||||
|
||||
Card-backed paths (``_CARD_BACKED_PATHS``) are also covered: each maps a
|
||||
config key to an HTML element id on a dedicated admin card.
|
||||
"""
|
||||
return (
|
||||
set(admin._CONFIG_ALLOWLIST)
|
||||
| {".".join(path) for path in admin._BOOL_KNOBS.values()}
|
||||
| {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()}
|
||||
| {".".join(entry[0]) for entry in admin._INT_KNOBS.values()}
|
||||
| set(admin._CARD_BACKED_PATHS)
|
||||
)
|
||||
|
||||
|
||||
@@ -470,22 +511,53 @@ def test_every_in_scope_knob_is_reachable_or_deliberately_not():
|
||||
)
|
||||
|
||||
|
||||
def test_excused_knobs_are_not_already_covered():
|
||||
"""Exactness: the excuse list must not accumulate knobs that HAVE controls.
|
||||
def test_card_backed_gate_is_real():
|
||||
"""Prove the gate actually uses _CARD_BACKED_PATHS by removing one entry.
|
||||
|
||||
Without this the list silently rots into a rubber stamp as knobs quietly
|
||||
gain controls, and the next reader cannot tell which entries still describe
|
||||
reality. Expected to fire on the two PENDING entries the moment
|
||||
feat/admin-incumbent-knobs merges — that is the intended cleanup signal.
|
||||
Temporarily remove a card-backed path that is NOT in the excuse list,
|
||||
run the coverage check, verify it fails naming that leaf, then restore it.
|
||||
"""
|
||||
import admin as admin_mod
|
||||
original = dict(admin_mod._CARD_BACKED_PATHS)
|
||||
# Pick a card-backed path not already excused (classifier.mode is excused
|
||||
# but still card-backed — pick one we deleted from the excuse list).
|
||||
test_key = None
|
||||
for key in original:
|
||||
if key not in DELIBERATELY_NOT_IN_ADMIN:
|
||||
test_key = key
|
||||
break
|
||||
if test_key is None:
|
||||
# Fallback: any card-backed path will do
|
||||
test_key = next(iter(original))
|
||||
try:
|
||||
del admin_mod._CARD_BACKED_PATHS[test_key]
|
||||
scalars_set = set(_scalars())
|
||||
in_scope = _in_scope(sorted(scalars_set))
|
||||
covered = _covered_paths()
|
||||
undecided = [
|
||||
p for p in in_scope
|
||||
if p not in covered and p not in DELIBERATELY_NOT_IN_ADMIN
|
||||
]
|
||||
assert test_key in undecided, (
|
||||
f"Removing {test_key!r} from _CARD_BACKED_PATHS should cause "
|
||||
f"the gate to flag it, but it did not"
|
||||
)
|
||||
finally:
|
||||
admin_mod._CARD_BACKED_PATHS = original
|
||||
|
||||
|
||||
def test_excused_knobs_are_not_already_covered():
|
||||
"""Excused knobs should NOT be covered, otherwise the excuse is false.
|
||||
|
||||
A card-backed leaf in DELIBERATELY_NOT_IN_ADMIN is a false excuse:
|
||||
the card IS its control. The excuse should be removed, not kept.
|
||||
"""
|
||||
covered = _covered_paths()
|
||||
stale = sorted(set(DELIBERATELY_NOT_IN_ADMIN) & covered)
|
||||
assert not stale, (
|
||||
"DELIBERATELY_NOT_IN_ADMIN excuses knob(s) that the admin portal now "
|
||||
"covers:\n "
|
||||
+ "\n ".join(f"{path}: {DELIBERATELY_NOT_IN_ADMIN[path]}" for path in stale)
|
||||
+ "\n\nDelete those entries."
|
||||
)
|
||||
for path in DELIBERATELY_NOT_IN_ADMIN:
|
||||
assert path not in covered, (
|
||||
f"{path!r} is in DELIBERATELY_NOT_IN_ADMIN but is already "
|
||||
f"covered by the admin portal. Remove the excuse."
|
||||
)
|
||||
|
||||
|
||||
def test_excused_knobs_still_exist_and_are_in_scope():
|
||||
@@ -524,6 +596,120 @@ def test_admin_registries_point_at_real_config_knobs():
|
||||
)
|
||||
|
||||
|
||||
# --- Phase A Step 3: taxonomy coverage ----------------------------------------
|
||||
# The test below checks two-way coverage between _KNOB_CATEGORY and the covered
|
||||
# registries (_CONFIG_ALLOWLIST + runtime + card-backed). Card-backed knobs
|
||||
# (with their own dedicated admin card/endpoint pair) are excluded from both
|
||||
# directions — they are covered by their card but don't need a taxonomy entry.
|
||||
# _CARD_BACKED_PATHS is used throughout this module: in _covered_paths(),
|
||||
# test_knob_taxonomy_coverage(), and test_card_backed_gate_is_real().
|
||||
|
||||
|
||||
def _cfg_paths_from_runtime() -> set[str]:
|
||||
"""Dotted config paths from every runtime registry entry.
|
||||
|
||||
``_RUNTIME_KNOB_PATHS`` is the single source of truth (it is derived
|
||||
from the same registries that drive the POST handlers) but we rebuild
|
||||
the path set directly from the registries to avoid importing a
|
||||
dict whose values are the same as ``_CONFIG_ALLOWLIST``'s and would
|
||||
obscure the separate provenance this check needs.
|
||||
|
||||
Unlike ``_covered_paths()`` this function exists solely for the taxonomy
|
||||
two-way check — it does not deduplicate against the allowlist, because
|
||||
both directions start from a different origin.
|
||||
"""
|
||||
return (
|
||||
{".".join(path) for path in admin._BOOL_KNOBS.values()}
|
||||
| {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()}
|
||||
| {".".join(entry[0]) for entry in admin._INT_KNOBS.values()}
|
||||
)
|
||||
|
||||
|
||||
def _card_backed_set() -> set[str]:
|
||||
"""Card-backed config paths to exclude from two-way coverage check.
|
||||
|
||||
Reads directly from ``admin._CARD_BACKED_PATHS``, which maps classifier
|
||||
config keys to the HTML element ids of their dedicated admin card controls.
|
||||
"""
|
||||
return set(admin._CARD_BACKED_PATHS)
|
||||
|
||||
|
||||
def test_knob_taxonomy_coverage():
|
||||
"""Two-way coverage between _KNOB_CATEGORY and the covered registries.
|
||||
|
||||
Direction 1 (allowlist + runtime + card-backed -> taxonomy): every knob
|
||||
that appears in ``_CONFIG_ALLOWLIST``, any runtime registry, or
|
||||
``_CARD_BACKED_PATHS`` must have an entry in ``_KNOB_CATEGORY``.
|
||||
|
||||
Direction 2 (taxonomy -> registries + card-backed): every key in
|
||||
``_KNOB_CATEGORY`` must resolve to a path covered by ``_CONFIG_ALLOWLIST``,
|
||||
a runtime registry, or ``_CARD_BACKED_PATHS``.
|
||||
|
||||
Both directions exclude card-backed paths from the taxonomy check — they
|
||||
have dedicated admin cards and don't belong in the generic knob table.
|
||||
"""
|
||||
# --- prepare covered set --------------------------------------------------
|
||||
allowlist_paths: set[str] = set(admin._CONFIG_ALLOWLIST)
|
||||
runtime_paths: set[str] = _cfg_paths_from_runtime()
|
||||
card_backed: set[str] = _card_backed_set()
|
||||
covered = allowlist_paths | runtime_paths | card_backed
|
||||
|
||||
# --- card-backed exclusion ------------------------------------------------
|
||||
card_backed = _card_backed_set()
|
||||
|
||||
# --- Direction 1 ----------------------------------------------------------
|
||||
# Every covered knob (minus card-backed) needs a category assignment.
|
||||
candidates_d1 = covered - card_backed
|
||||
missing = sorted(candidates_d1 - set(admin._KNOB_CATEGORY))
|
||||
assert not missing, (
|
||||
"Covered knob(s) with no entry in _KNOB_CATEGORY:\n "
|
||||
+ "\n ".join(missing)
|
||||
+ "\n\nAdd an entry to _KNOB_CATEGORY in src/admin.py with the "
|
||||
"appropriate category id and advanced flag."
|
||||
)
|
||||
|
||||
# --- Direction 2 ----------------------------------------------------------
|
||||
# Every taxonomy entry (minus card-backed) must be a real covered path.
|
||||
candidates_d2 = set(admin._KNOB_CATEGORY) - card_backed
|
||||
orphaned = sorted(candidates_d2 - covered)
|
||||
assert not orphaned, (
|
||||
"_KNOB_CATEGORY key(s) that are in neither _CONFIG_ALLOWLIST nor any "
|
||||
"runtime registry:\n "
|
||||
+ "\n ".join(orphaned)
|
||||
+ "\n\nEither add the path to _CONFIG_ALLOWLIST or a runtime registry, "
|
||||
"or --- if the knob has its own dedicated admin card --- add it to "
|
||||
"_CARD_BACKED_PATHS in src/admin.py."
|
||||
)
|
||||
|
||||
# --- well-formed entries --------------------------------------------------
|
||||
valid_categories = {
|
||||
"routing_quality_cost",
|
||||
"classification",
|
||||
"local_hardware",
|
||||
"caching_context",
|
||||
"safety_nets",
|
||||
"watchdog",
|
||||
}
|
||||
for path, entry in admin._KNOB_CATEGORY.items():
|
||||
assert isinstance(entry, dict), (
|
||||
f"_KNOB_CATEGORY[{path!r}] is not a dict: {type(entry).__name__}"
|
||||
)
|
||||
assert "category" in entry, (
|
||||
f"_KNOB_CATEGORY[{path!r}] has no 'category' key"
|
||||
)
|
||||
assert entry["category"] in valid_categories, (
|
||||
f"_KNOB_CATEGORY[{path!r}].category={entry['category']!r} "
|
||||
f"is not in {sorted(valid_categories)}"
|
||||
)
|
||||
assert "advanced" in entry, (
|
||||
f"_KNOB_CATEGORY[{path!r}] has no 'advanced' key"
|
||||
)
|
||||
assert isinstance(entry["advanced"], bool), (
|
||||
f"_KNOB_CATEGORY[{path!r}].advanced is not bool: "
|
||||
f"{type(entry['advanced']).__name__}"
|
||||
)
|
||||
|
||||
|
||||
# --- Component 6: one knob table -----------------------------------------------
|
||||
# The two tests below step outside the pure-reflection contract documented
|
||||
# above: the first drives the real app (still offline -- a temp SQLite file,
|
||||
@@ -562,7 +748,7 @@ def test_runtime_api_labels_every_knob_with_a_real_config_key(tmp_path, monkeypa
|
||||
|
||||
cfg = load_config(str(ROOT / "config" / "config.yaml"))
|
||||
for knob, item in body.items():
|
||||
assert set(item) == {"persisted", "runtime", "config_key"}, knob
|
||||
assert set(item) == {"persisted", "runtime", "config_key", "category", "advanced"}, knob
|
||||
# An AttributeError here names the broken label directly.
|
||||
obj: typing.Any = cfg
|
||||
for part in item["config_key"].split("."):
|
||||
@@ -601,3 +787,25 @@ def test_controls_page_keeps_the_table_selectors_its_js_depends_on():
|
||||
assert "data-config-input" in html
|
||||
assert "data-profile-select" in html
|
||||
assert '<tr><td colspan="4"' in html
|
||||
|
||||
|
||||
def test_card_backed_paths_are_real_config_scalars_and_element_ids_exist():
|
||||
"""Every card-backed path is a real scalar field and its element id exists.
|
||||
|
||||
(a) Walk ``RouterConfig`` via ``_walk`` to verify every key in
|
||||
``admin._CARD_BACKED_PATHS`` resolves to a scalar leaf of the config model.
|
||||
(b) Read ``controls.html`` and verify every element id value appears as an
|
||||
``id="..."`` attribute somewhere in the source.
|
||||
"""
|
||||
scalars = set(_scalars())
|
||||
html = CONTROLS_HTML.read_text(encoding="utf-8")
|
||||
|
||||
for path, element_id in admin._CARD_BACKED_PATHS.items():
|
||||
assert path in scalars, (
|
||||
f"_CARD_BACKED_PATHS key {path!r} is not a scalar field of "
|
||||
f"RouterConfig"
|
||||
)
|
||||
assert element_id in html, (
|
||||
f"_CARD_BACKED_PATHS value {element_id!r} (for {path!r}) not found "
|
||||
f"in controls.html as an element id"
|
||||
)
|
||||
|
||||
@@ -19,7 +19,13 @@ from starlette.testclient import TestClient
|
||||
|
||||
import dispatcher
|
||||
from admin import _INT_KNOBS
|
||||
from config import STALENESS_SECONDS_MAX, STALENESS_SECONDS_MIN, load_config
|
||||
from config import (
|
||||
DEGRADED_WARN_THRESHOLD_FLOOR,
|
||||
DEGRADED_WARN_THRESHOLD_MAX,
|
||||
STALENESS_SECONDS_MAX,
|
||||
STALENESS_SECONDS_MIN,
|
||||
load_config,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text()
|
||||
@@ -93,7 +99,7 @@ def test_runtime_GET_reports_every_knob(seeded_client):
|
||||
"default_flex_preference",
|
||||
):
|
||||
assert knob in body
|
||||
assert set(body[knob]) == {"persisted", "runtime", "config_key"}
|
||||
assert set(body[knob]) == {"persisted", "runtime", "config_key", "category", "advanced"}
|
||||
|
||||
# The runtime value mirrors the live dispatcher.cfg for the boolean knobs;
|
||||
# the persisted value mirrors config.yaml. circuit_breaker_enabled is a
|
||||
@@ -106,11 +112,15 @@ def test_runtime_GET_reports_every_knob(seeded_client):
|
||||
"persisted": CFG.circuit_breaker.enabled,
|
||||
"runtime": dispatcher.cfg.circuit_breaker.enabled,
|
||||
"config_key": "circuit_breaker.enabled",
|
||||
"category": "safety_nets",
|
||||
"advanced": False,
|
||||
}
|
||||
assert body["default_flex_preference"] == {
|
||||
"persisted": CFG.routing.default_flex_preference.value,
|
||||
"runtime": dispatcher.cfg.routing.default_flex_preference.value,
|
||||
"config_key": "routing.default_flex_preference",
|
||||
"category": "routing_quality_cost",
|
||||
"advanced": False,
|
||||
}
|
||||
|
||||
|
||||
@@ -326,6 +336,99 @@ def test_challenger_dial_refuses_a_non_number(
|
||||
assert dispatcher.cfg.objective.incumbent_challenger_cache_rate == before
|
||||
|
||||
|
||||
def test_challenger_dial_422_still_advertises_null_as_neutral(
|
||||
seeded_client, incumbent_cfg_guard
|
||||
):
|
||||
"""The dial HAS a neutral, so its 422s must keep telling the operator so.
|
||||
|
||||
These hints were dropped once when a second float knob without a neutral
|
||||
shared the handler; they are conditional on the knob having one.
|
||||
"""
|
||||
url = "/admin/api/runtime/incumbent_challenger_cache_rate"
|
||||
not_a_number = seeded_client.post(url, json={"value": "x"})
|
||||
assert not_a_number.status_code == 422
|
||||
assert "or null for neutral" in not_a_number.json()["detail"]
|
||||
|
||||
out_of_range = seeded_client.post(url, json={"value": 1.5})
|
||||
assert out_of_range.status_code == 422
|
||||
assert "(null means neutral, not 0.0)" in out_of_range.json()["detail"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def degraded_threshold_cfg_guard(monkeypatch):
|
||||
"""Restore classifier.degraded_warn_threshold after a test writes it."""
|
||||
classifier = dispatcher.cfg.classifier
|
||||
monkeypatch.setattr(
|
||||
classifier, "degraded_warn_threshold", classifier.degraded_warn_threshold
|
||||
)
|
||||
return classifier
|
||||
|
||||
|
||||
DEGRADED_THRESHOLD_URL = "/admin/api/runtime/classifier_degraded_warn_threshold"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad",
|
||||
[
|
||||
None,
|
||||
True,
|
||||
"x",
|
||||
0.0,
|
||||
DEGRADED_WARN_THRESHOLD_FLOOR / 2,
|
||||
DEGRADED_WARN_THRESHOLD_MAX + 0.5,
|
||||
],
|
||||
ids=["null", "bool", "string", "zero", "below-floor", "above-max"],
|
||||
)
|
||||
def test_degraded_warn_threshold_refuses_bad_values_with_422(
|
||||
seeded_client, degraded_threshold_cfg_guard, bad
|
||||
):
|
||||
"""A runtime write skips Pydantic, so the registry bounds are the only guard.
|
||||
|
||||
``null`` is the sharp one: this knob has no neutral, and the float handler
|
||||
used to resolve null through ``_get_at(cfg, neutral_path)``, which raised
|
||||
TypeError (HTTP 500) when that path is None.
|
||||
"""
|
||||
before = degraded_threshold_cfg_guard.degraded_warn_threshold
|
||||
resp = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": bad})
|
||||
assert resp.status_code == 422
|
||||
assert degraded_threshold_cfg_guard.degraded_warn_threshold == before
|
||||
|
||||
|
||||
def test_degraded_warn_threshold_422_never_advertises_null(
|
||||
seeded_client, degraded_threshold_cfg_guard
|
||||
):
|
||||
"""No neutral means no ``null`` hint: it would send the operator into a 422."""
|
||||
for bad in (None, "x", 1.5):
|
||||
detail = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": bad}).json()[
|
||||
"detail"
|
||||
]
|
||||
assert "neutral" not in detail
|
||||
assert "null" not in detail
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"good",
|
||||
[DEGRADED_WARN_THRESHOLD_FLOOR, 0.2, DEGRADED_WARN_THRESHOLD_MAX],
|
||||
ids=["floor", "typical", "max"],
|
||||
)
|
||||
def test_degraded_warn_threshold_accepts_in_range_and_updates_cfg(
|
||||
seeded_client, degraded_threshold_cfg_guard, good
|
||||
):
|
||||
resp = seeded_client.post(DEGRADED_THRESHOLD_URL, json={"value": good})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"ok": True, "classifier_degraded_warn_threshold": good}
|
||||
|
||||
stored = degraded_threshold_cfg_guard.degraded_warn_threshold
|
||||
assert isinstance(stored, float) and not isinstance(stored, bool)
|
||||
assert stored == good
|
||||
assert (
|
||||
seeded_client.get("/admin/api/runtime").json()[
|
||||
"classifier_degraded_warn_threshold"
|
||||
]["runtime"]
|
||||
== good
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def staleness_cfg_guard(monkeypatch):
|
||||
"""Restore the session-cache window on cfg after a test moves it."""
|
||||
@@ -510,6 +613,8 @@ def test_active_profile_is_reported_with_persisted_and_runtime(seeded_client):
|
||||
"persisted",
|
||||
"runtime",
|
||||
"config_key",
|
||||
"category",
|
||||
"advanced",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user