diff --git a/CLAUDE.md b/CLAUDE.md index aa5b4cd..ddea845 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,24 @@ oversight nobody noticed. absent, because enabling it must be a config edit plus a restart. That is a recorded choice, not a gap. +The classifier section is now fully accounted for under the gate. Seven +classifier scalars (`context_framing`, `cooldown_seconds`, `fallback_tier`, +`fallback_category`, `max_input_chars`, `degraded_warn_min`, +`degraded_warn_threshold`) have a runtime control, a persisted control, or both +(`fallback_category` and `max_input_chars` are persisted-only). Every other +classifier scalar is either card-backed (`_CARD_BACKED_PATHS` in +`src/admin.py`: mode, cloud primary/fallback, encoder, and local-decision +fields) or excused in `DELIBERATELY_NOT_IN_ADMIN` (timeout, temperature, +`max_output_tokens`, `encoder.tier_from_features`). + +Outside-gate knobs are still pending. Sections not yet reached by the portal +(database, dispatch providers/settings, local dispatch models, profiles, +local energy, tiers/tiering/proficiency/context, and deployment wiring inside +classifier/verification/local vision) are tracked in +`plans/deferred-knobs.md`. The first control added under any of those sections +will drag every scalar under it into scope at once, per the coverage test's +clause 1. + **Why:** Wave 2 shipped `incumbent_cache_pricing` and `incumbent_challenger_cache_rate` with no control at all. Nobody decided that; it just never came up. The dial's entire purpose is tuning from neutral to full diff --git a/admin/frontend/controls.html b/admin/frontend/controls.html index 6e412f4..2acf44e 100644 --- a/admin/frontend/controls.html +++ b/admin/frontend/controls.html @@ -135,10 +135,38 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important} Scoped so any future div list keeps the grid grammar. */ .knobs-table .setting-row{display:table-row;min-height:0;padding:0;border-radius:0;border-left:2px solid transparent} .knobs-table .setting-row.is-dirty{border-left-color:#fbbf24;background:rgba(245,158,11,.06)} -/* A config key wraps instead of truncating: the knob cell is the row's - identity, and a clipped path hides which scope the row belongs to. */ +/* Persisted-cell hints: the note badge is white-space:nowrap by default, so a + long note sets the cell's min-content width and pushes the table past an + 800px viewport. Let the hints wrap (the pill drops under the unit) and the + pill's own text wrap. d-inline-flex is !important, so this targets the + cell's direct .ms-2 child rather than overriding display. */ +.knobs-table td > .ms-2{flex-wrap:wrap;max-width:100%} +.knobs-table td > .ms-2 .badge{white-space:normal;text-align:left} .knobs-table .setting-key{white-space:normal;overflow:visible;text-overflow:clip} .knobs-table .setting-key .scope{display:inline} +/* -- Category header rows: a subtle accent that segments the knob list + without adding visual weight. The chevron is the only interactive cue; + the label and icon supply context. -- */ +.cat-header th{ + background:rgba(255,255,255,.02); + border-bottom:1px solid rgba(255,255,255,.08); + cursor:pointer;user-select:none; + font-size:.85rem;font-weight:600; + padding:.35rem .6rem; + color:var(--tblr-secondary) +} +.cat-header th:hover{background:rgba(255,255,255,.045)} +.cat-header .cat-label{color:#e2e8f0} +.cat-chevron{display:inline-flex;align-items:center;color:var(--tblr-secondary);opacity:.7;transition:transform .18s ease} +.cat-chevron:not(.cat-chevron-open){transform:rotate(90deg)} +.cat-icon{opacity:.7;margin-right:.15rem} +.adv-subheader th{ + cursor:pointer;user-select:none; + font-size:.78rem;font-weight:500; + padding:.2rem .6rem; + color:var(--tblr-secondary);opacity:.7 +} +.adv-subheader th:hover{opacity:1;color:#e2e8f0} .setting-key{ font-size:.82rem;line-height:1.25; overflow:hidden;text-overflow:ellipsis;white-space:nowrap; @@ -644,6 +672,9 @@ function keyHtml(key) { const UNITS = { 'objective.max_energy_per_request': 'kWh', 'objective.plan_kwh_per_period': 'kWh', + 'classifier.cooldown_seconds': 's', + 'classifier.degraded_warn_min': 'decisions/24h', + 'classifier.max_input_chars': 'chars', }; /* A consequence the control itself cannot show. Keyed by both the runtime knob @@ -692,6 +723,18 @@ const SESSION_CACHE_TTL_NOTE = { cls: 'bg-info', glyph: 'info', }; +const DEGRADED_WARN_MIN_NOTE = { + text: 'silent below this many decisions in 24h', + title: 'No warning is raised until the classifier completes at least this many decisions in a 24-hour window. Below the threshold the degraded-count warning is suppressed entirely.', + cls: 'bg-info', + glyph: 'info', +}; +const DEGRADED_WARN_THRESHOLD_NOTE = { + text: 'share of degraded classifications that warns', + title: 'When the fraction of degraded (non-ok) classifier verdicts exceeds this ratio within the warn-min window, an amber badge appears on the admin navbar. 0.01 = 1% of classifications degraded.', + cls: 'bg-info', + glyph: 'info', +}; const KNOB_NOTES = { pinch_prefix_probe: PROBE_NOTE, 'pinch.prefix_probe': PROBE_NOTE, @@ -703,6 +746,10 @@ const KNOB_NOTES = { 'objective.incumbent_cache_pricing': INCUMBENT_GATE_NOTE, incumbent_challenger_cache_rate: CHALLENGER_DIAL_NOTE, 'objective.incumbent_challenger_cache_rate': CHALLENGER_DIAL_NOTE, + classifier_degraded_warn_min: DEGRADED_WARN_MIN_NOTE, + 'classifier.degraded_warn_min': DEGRADED_WARN_MIN_NOTE, + classifier_degraded_warn_threshold: DEGRADED_WARN_THRESHOLD_NOTE, + 'classifier.degraded_warn_threshold': DEGRADED_WARN_THRESHOLD_NOTE, }; function noteBadge(key) { @@ -723,8 +770,52 @@ const NUMBER_BOUNDS = { // operator was reaching for. min is 5 rather than 0 -- 0 is not "off", it // still caches and the classifier-failure cascade still replays the entry. session_cache_staleness_seconds: { min: 5, max: 7200, step: 1 }, + classifier_degraded_warn_threshold: { min: 0.001, max: 1, step: 0.01 }, + classifier_max_input_chars: { min: 0, step: 1000 }, }; +// Category definitions for the knob table section headers. Order here matches +// _CONFIG_GET_ORDER in admin.py so sections render in the same sequence the +// YAML uses. `icon` picks a renderStaticIcons-compatible name. +const CATEGORIES = [ + { id: 'routing_quality_cost', label: 'Routing / Quality / Cost', icon: 'settings' }, + { id: 'classification', label: 'Classification', icon: 'info' }, + { id: 'caching_context', label: 'Caching / Context', icon: 'database' }, + { id: 'local_hardware', label: 'Local Hardware', icon: 'cpu' }, + { id: 'safety_nets', label: 'Safety Nets', icon: 'alert' }, + { id: 'watchdog', label: 'Watchdog', icon: 'alert' }, +]; + +// Collapsible category and advanced-section state. Persisted across 30s +// re-renders: toggling mutates this Set, and renderKnobsTable reads it. +// Category keys are `cat..collapsed`; advanced keys are `cat..adv`. +let collapsedCategories = new Set(); +// Categories whose Advanced accordion has been seeded collapsed. Per category, +// not global: see seedAdvancedCollapsed. +let _advSeededCategories = new Set(); + +/* SEED_ADV_COLLAPSED:BEGIN */ +function seedAdvancedCollapsed(byCategory, rows, categoryIds, seeded, collapsed) { + // Seed a category's Advanced accordion as collapsed the FIRST time that + // category renders an advanced row, not on the first render overall. The + // runtime payload can arrive before the config payload, and persisted-only + // advanced knobs (watchdog.detector.*) are absent from that early render; a + // category marked seeded before its advanced rows exist would load expanded. + // Later renders leave whatever the user toggled alone. + for (const id of categoryIds) { + if (seeded.has(id)) continue; + const hasAdvanced = (byCategory[id] || []).some(key => { + const row = rows[key]; + return (row.runtimeItem && row.runtimeItem.advanced) || + (row.persistedItem && row.persistedItem.advanced); + }); + if (!hasAdvanced) continue; + seeded.add(id); + collapsed.add(id + '.adv'); + } +} +/* SEED_ADV_COLLAPSED:END */ + /* One knob table replaces the old runtime-list/config-list div columns. Runtime rows and persisted rows pair by the dotted config key the /admin/api/runtime payload carries in `config_key` -- the join lives here, @@ -858,7 +949,7 @@ function persistedControlHtml(key, entry, dirtyValue) { return `${control}${hints ? `${hints}` : ''}`; } -function knobRowHtml(row) { +function knobRowHtml(row, catId, isAdv) { const rt = row.runtimeItem; const entry = row.persistedItem; const keyCell = row.drift @@ -869,7 +960,7 @@ function knobRowHtml(row) { : '-'; let persistedCell = '-'; let layerCell = ''; - let dirtyAttrs = ''; + let extraAttrs = ''; if (entry) { persistedCell = persistedControlHtml(row.configKey, entry, row.dirtyValue); // Provenance is the layer column: where the effective value came from. @@ -880,9 +971,11 @@ function knobRowHtml(row) { layer = 'base'; } layerCell = `${layer}`; - dirtyAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`; + extraAttrs = ` data-key="${escapeHtml(row.configKey)}" data-orig="${escapeHtml(String(entry.value === null ? '' : entry.value))}"`; } - return ` + if (catId) extraAttrs += ` data-cat="cat-${catId}"`; + if (isAdv) extraAttrs += ' data-adv="true"'; + return ` ${keyCell} ${liveCell} ${persistedCell} @@ -917,13 +1010,155 @@ function renderKnobsTable() { list.innerHTML = 'No knobs to show'; return; } - list.innerHTML = keys.map(key => { + + // Group rows by category using the category/advanced metadata from the API. + const byCategory = {}; + const uncategorized = []; + for (const key of keys) { const row = rows[key]; row.dirtyValue = row.persistedItem ? dirtyInputs[key] : undefined; - return knobRowHtml(row); - }).join(''); + const cat = (row.runtimeItem && row.runtimeItem.category) || + (row.persistedItem && row.persistedItem.category) || ''; + if (!cat) { uncategorized.push(key); continue; } + if (!byCategory[cat]) byCategory[cat] = []; + byCategory[cat].push(key); + } + + const html = []; + + // Advanced subsections load collapsed, so the user sees the clean knob list + // before the Advanced expander. Seeded per category, once. + seedAdvancedCollapsed(byCategory, rows, CATEGORIES.map(c => c.id), + _advSeededCategories, collapsedCategories); + + for (const cat of CATEGORIES) { + const catKeys = byCategory[cat.id] || []; + if (!catKeys.length) continue; + const isCatCollapsed = collapsedCategories.has(cat.id + '.collapsed'); + + // Category header with chevron toggle, rendered via data-icon for + // renderStaticIcons(). All 4 columns spanned so the click target + // is the full table width. + html.push(` + + + + + ${cat.label} + + + `); + + // Split into standard and advanced rows. + const standard = []; + const advanced = []; + for (const key of catKeys) { + const row = rows[key]; + const isAdv = (row.runtimeItem && row.runtimeItem.advanced) || + (row.persistedItem && row.persistedItem.advanced) || false; + (isAdv ? advanced : standard).push(key); + } + + // Standard (non-advanced) rows + for (const key of standard) { + html.push(knobRowHtml(rows[key], cat.id, false)); + } + + // Advanced sub-header + rows (default collapsed on first render) + if (advanced.length) { + const advKey = cat.id + '.adv'; + const isAdvCollapsed = collapsedCategories.has(advKey); + html.push(` + + + + Advanced + + + `); + for (const key of advanced) { + html.push(knobRowHtml(rows[key], cat.id, true)); + } + } + } + + // Uncategorized rows at the end (fallback for knob metadata edge cases). + for (const key of uncategorized) { + html.push(knobRowHtml(rows[key], '', false)); + } + + list.innerHTML = html.join(''); + + // Apply collapsed state to category rows after rendering. + for (const cat of CATEGORIES) { + if (collapsedCategories.has(cat.id + '.collapsed')) { + const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"]:not(.cat-header)`); + rows.forEach(r => { r.style.display = 'none'; }); + } + if (collapsedCategories.has(cat.id + '.adv')) { + const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${cat.id}"][data-adv="true"]`); + rows.forEach(r => { r.style.display = 'none'; }); + } + } + + // Re-render data-icon attributes that were just injected. + renderStaticIcons(); updateDefaultProfileHint(); markDirty(); + + // If the URL carries a category anchor, expand that section now. + if (window.location.hash && window.location.hash.startsWith('#cat-')) { + const hashCat = window.location.hash.replace('#cat-', ''); + if (collapsedCategories.has(hashCat + '.collapsed')) { + collapsedCategories.delete(hashCat + '.collapsed'); + const rows = document.querySelectorAll(`#config-list tr[data-cat="cat-${hashCat}"]:not(.cat-header)`); + rows.forEach(r => { r.style.display = ''; }); + const chevron = document.querySelector(`#config-list tr[data-cat="cat-${hashCat}"].cat-header .cat-chevron`); + if (chevron) chevron.classList.add('cat-chevron-open'); + } + } +} + +// Toggle a category section: collapse/expand all knob rows within it. +function toggleCategory(catId) { + const key = catId + '.collapsed'; + if (collapsedCategories.has(key)) { + collapsedCategories.delete(key); + } else { + collapsedCategories.add(key); + } + const isCollapsed = collapsedCategories.has(key); + // All knob rows + advanced sub-header under this category. + const rows = document.querySelectorAll( + `#config-list tr[data-cat="cat-${catId}"]:not(.cat-header)` + ); + rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; }); + // Flip the chevron on the category header. + const chevron = document.querySelector( + `#config-list tr[data-cat="cat-${catId}"].cat-header .cat-chevron` + ); + if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed); +} + +// Toggle an advanced subsection within a category: collapse/expand only the +// rows flagged data-adv="true" under that category. +function toggleAdvanced(catId) { + const key = catId + '.adv'; + if (collapsedCategories.has(key)) { + collapsedCategories.delete(key); + } else { + collapsedCategories.add(key); + } + const isCollapsed = collapsedCategories.has(key); + const rows = document.querySelectorAll( + `#config-list tr[data-cat="cat-${catId}"][data-adv="true"]` + ); + rows.forEach(r => { r.style.display = isCollapsed ? 'none' : ''; }); + // Flip the chevron on the advanced sub-header. + const chevron = document.querySelector( + `#config-list tr[data-adv-subheader="${catId}"] .cat-chevron` + ); + if (chevron) chevron.classList.toggle('cat-chevron-open', !isCollapsed); } function renderRuntime(state) { diff --git a/docs/admin-portal.md b/docs/admin-portal.md index 0b4ed7f..cf9d441 100644 --- a/docs/admin-portal.md +++ b/docs/admin-portal.md @@ -145,6 +145,39 @@ land in `config/config.local.yaml`; see [config-local-overlay](config-local-overlay.md)). Changes are marked as dirty and written only on save. +#### Knob organization + +The generic knob table is grouped into **six categories**, each with a +collapsible section header: + +1. **Routing / Quality / Cost** — the core dispatch objective: logging + levels, energy ceiling, plan quota, cache pricing, and default profile. +2. **Classification** — classifier runtime scalars such as cooldown, + fallback tier/category, degraded warning thresholds, and context framing. +3. **Local Hardware** — gates over local compute paths: the local-compute + master switch, local verification, and local vision fallback. +4. **Caching / Context** — session classification cache and context pruning + (pinch). +5. **Safety Nets** — circuit breaker. +6. **Watchdog** — response-loop detector thresholds. + +Within each category, knobs marked **Advanced** hide behind a secondary +"Advanced" accordion. These are shape parameters internal to a covered master +switch (for example, pinch relevance options and watchdog detector +thresholds). The accordion keeps the first view short without removing access. + +Knobs reach the page through **three coverage sources**: + +- **Registry knobs** — runtime toggles from `_BOOL_KNOBS`, `_FLOAT_KNOBS`, and + `_INT_KNOBS` in `admin.py`. They take effect immediately in memory and + revert on restart. +- **Allowlist knobs** — the same scalars persisted through + `_CONFIG_ALLOWLIST`, written to `config/config.local.yaml` and surviving a + restart. +- **Card-backed paths** — knobs with their own dedicated admin card and + endpoint pair because they have cross-field structure a flat scalar input + cannot safely represent. + It carries three dedicated cards, none a row in the generic runtime-knob list, because each has cross-field structure a flat scalar/boolean input can't safely represent: diff --git a/plans/deferred-knobs.md b/plans/deferred-knobs.md new file mode 100644 index 0000000..82d37d5 --- /dev/null +++ b/plans/deferred-knobs.md @@ -0,0 +1,116 @@ +# Deferred admin portal knobs +Status: reference -- Tracker for knobs outside current admin gate + +This file tracks config knobs that are outside the admin portal's current +gate. The coverage test in `tests/test_admin_knob_coverage.py` enforces that +every knob in a section the portal reaches is either covered or excused with a +reason. Sections the portal has not reached are out of scope for that test, +but they still need a recorded home so the first control added under a section +does not leave half its siblings orphaned. + +Rows use four columns: + +| Column | Meaning | +|---|---| +| Knob | Dotted config path in `config/config.yaml`. | +| Section | Top-level config section. | +| Disposition | `deferred` (future admin page) or `excused-with-reason` (deployment wiring or file path). | +| Future Home | The page or surface that should own this knob when the portal reaches the section. | + +## Outside-gate knobs + +These are `RouterConfig` scalars in sections the portal has not reached yet. +They fall into two buckets: + +- Sections not yet reached: `context`, `database`, `dispatch_settings`, + `escalation`, `exploration`, `freshness`, `iteration`, `local_energy`, + `proficiency`, `tiering`. +- Deployment wiring inside those sections: endpoints, model ids, credentials, + filesystem paths, and device names. + +| Knob | Section | Disposition | Future Home | +|---|---|---|---| +| `context.default_output_reserve_tokens` | context | deferred | Context page | +| `context.max_output_reserve_fraction` | context | deferred | Context page | +| `context.safety_factor` | context | deferred | Context page | +| `database.path` | database | excused-with-reason | N/A (file path) | +| `dispatch_settings.default_provider` | dispatch_settings | deferred | Providers page | +| `escalation.enabled` | escalation | deferred | Routing page | +| `escalation.max_tier` | escalation | deferred | Routing page | +| `escalation.min_confidence_before_bump` | escalation | deferred | Routing page | +| `escalation.preemptive_on_low_confidence` | escalation | deferred | Routing page | +| `exploration.enabled` | exploration | deferred | Routing page | +| `exploration.epsilon` | exploration | deferred | Routing page | +| `exploration.max_cost_ratio` | exploration | deferred | Routing page | +| `exploration.max_tier` | exploration | deferred | Routing page | +| `freshness.exclude_deprecated` | freshness | deferred | Catalog page | +| `freshness.exclude_stale` | freshness | deferred | Catalog page | +| `freshness.repoll_after_allowlist_change_seconds` | freshness | deferred | Catalog page | +| `freshness.stale_after_days` | freshness | deferred | Catalog page | +| `iteration.enabled` | iteration | deferred | Routing page | +| `iteration.max_attempts_interactive` | iteration | deferred | Routing page | +| `iteration.max_rebill_prompt_tokens` | iteration | deferred | Routing page | +| `local_energy.enabled` | local_energy | deferred | Local Hardware page | +| `local_energy.grid_intensity_g_per_kwh` | local_energy | deferred | Local Hardware page | +| `local_energy.meter` | local_energy | excused-with-reason | N/A (deployment wiring) | +| `local_energy.sample_interval_seconds` | local_energy | deferred | Local Hardware page | +| `local_energy.tariff_usd_per_kwh` | local_energy | deferred | Local Hardware page | +| `proficiency.leaderboard_weight` | proficiency | deferred | Proficiency page | +| `proficiency.outcome_prior_strength` | proficiency | deferred | Proficiency page | +| `proficiency.self_eval_min_samples` | proficiency | deferred | Proficiency page | +| `proficiency.self_eval_weight` | proficiency | deferred | Proficiency page | +| `tiering.cheap_completion_max` | tiering | deferred | Routing page | +| `tiering.tier1_context_max` | tiering | deferred | Routing page | + +## Objective warning knobs + +These 31 `objective.*` knobs are currently excused from the admin portal in +`tests/test_admin_knob_coverage.py`. They tune `/metrics` warnings and report +series rather than dispatch behavior. Their natural future home is a +**Warnings page** that collects all alarm and report thresholds in one place. + +| Knob | Section | Disposition | Future Home | +|---|---|---|---| +| `objective.adoption_window_seconds` | objective | deferred | Warnings page | +| `objective.assumed_cache_rate` | objective | deferred | Warnings page | +| `objective.assumed_completion_tokens` | objective | deferred | Warnings page | +| `objective.billing_reset_day` | objective | deferred | Warnings page | +| `objective.cache_rate_warn_margin` | objective | deferred | Warnings page | +| `objective.cache_rate_warn_min_observations` | objective | deferred | Warnings page | +| `objective.cache_rate_window_hours` | objective | deferred | Warnings page | +| `objective.cost_calibration_min_observations` | objective | deferred | Warnings page | +| `objective.cost_calibration_window_hours` | objective | deferred | Warnings page | +| `objective.credit_attenuation.enabled` | objective | deferred | Warnings page | +| `objective.credit_attenuation.max_multiplier` | objective | deferred | Warnings page | +| `objective.credit_attenuation.refresh_seconds` | objective | deferred | Warnings page | +| `objective.credit_attenuation.soft_floor_usd` | objective | deferred | Warnings page | +| `objective.credit_attenuation.zero_floor_usd` | objective | deferred | Warnings page | +| `objective.cumulative_spend_warn_min_rows` | objective | deferred | Warnings page | +| `objective.cumulative_spend_warn_usd` | objective | deferred | Warnings page | +| `objective.incumbent_rate_min_observations` | objective | deferred | Warnings page | +| `objective.incumbent_rate_refresh_seconds` | objective | deferred | Warnings page | +| `objective.latency_min_observations` | objective | deferred | Warnings page | +| `objective.latency_window_hours` | objective | deferred | Warnings page | +| `objective.plan_pace_warn_ratio` | objective | deferred | Warnings page | +| `objective.proficiency_depth_warn_min_rows` | objective | deferred | Warnings page | +| `objective.proficiency_depth_warn_min_samples` | objective | deferred | Warnings page | +| `objective.quota_burn_min_segment_hours` | objective | deferred | Warnings page | +| `objective.quota_burn_min_segment_samples` | objective | deferred | Warnings page | +| `objective.quota_burn_window_hours` | objective | deferred | Warnings page | +| `objective.quota_runway_warning_hours` | objective | deferred | Warnings page | +| `objective.rejection_warning_baseline_hours` | objective | deferred | Warnings page | +| `objective.rejection_warning_min_count` | objective | deferred | Warnings page | +| `objective.rejection_warning_window_hours` | objective | deferred | Warnings page | +| `objective.selection_coverage_window_hours` | objective | deferred | Warnings page | + +## Notes + +- A section moves from "outside gate" to "in scope" the moment the portal adds + its first control there. At that point every scalar under it must be covered, + excused, or removed from this tracker into `DELIBERATELY_NOT_IN_ADMIN`. +- Deployment wiring (endpoints, model ids, credentials, paths, devices) is + deliberately off the admin allowlist. Those knobs should stay + `excused-with-reason` even after their section is reached. +- The Warnings page is not built yet. When it is, these `objective.*` knobs + should move from `DELIBERATELY_NOT_IN_ADMIN` into registry or allowlist + entries, and this table should shrink. diff --git a/src/admin.py b/src/admin.py index fbd1305..2286b20 100644 --- a/src/admin.py +++ b/src/admin.py @@ -46,6 +46,14 @@ import poller import routing from config import ( BUILTIN_PROFILES, + COOLDOWN_CEILING, + COOLDOWN_FLOOR, + DEGRADED_WARN_MIN_CEILING, + DEGRADED_WARN_MIN_FLOOR, + DEGRADED_WARN_THRESHOLD_FLOOR, + DEGRADED_WARN_THRESHOLD_MAX, + FALLBACK_TIER_MAX, + FALLBACK_TIER_MIN, DispatchProvider, FlexPreference, RouterConfig, @@ -356,6 +364,7 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = { "pinch_enabled": ("pinch", "enabled"), "pinch_prefix_probe": ("pinch", "prefix_probe"), "pinch_relevance_enabled": ("pinch", "relevance", "enabled"), + "classifier_context_framing": ("classifier", "context_framing"), "incumbent_cache_pricing": ("objective", "incumbent_cache_pricing"), } @@ -384,14 +393,20 @@ _BOOL_KNOBS: dict[str, tuple[str, ...]] = { # representations" would stop holding the moment this endpoint could write a # None onto cfg. # -# knob -> (cfg path, low, high, path whose value ``null`` resolves to) -_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, tuple[str, ...]]] = { +# knob -> (cfg path, low, high, path whose value ``null`` resolves to, if any) +_FLOAT_KNOBS: dict[str, tuple[tuple[str, ...], float, float, Optional[tuple[str, ...]]]] = { "incumbent_challenger_cache_rate": ( ("objective", "incumbent_challenger_cache_rate"), 0.0, 1.0, ("objective", "assumed_cache_rate"), ), + "classifier_degraded_warn_threshold": ( + ("classifier", "degraded_warn_threshold"), + DEGRADED_WARN_THRESHOLD_FLOOR, + DEGRADED_WARN_THRESHOLD_MAX, + None, + ), } # Runtime knobs whose value is a whole number. A SEPARATE table from @@ -427,6 +442,21 @@ _INT_KNOBS: dict[str, tuple[tuple[str, ...], int, int]] = { STALENESS_SECONDS_MIN, STALENESS_SECONDS_MAX, ), + "classifier_cooldown_seconds": ( + ("classifier", "cooldown_seconds"), + COOLDOWN_FLOOR, + COOLDOWN_CEILING, + ), + "classifier_fallback_tier": ( + ("classifier", "fallback_tier"), + FALLBACK_TIER_MIN, + FALLBACK_TIER_MAX, + ), + "classifier_degraded_warn_min": ( + ("classifier", "degraded_warn_min"), + DEGRADED_WARN_MIN_FLOOR, + DEGRADED_WARN_MIN_CEILING, + ), } _LOCAL_COMPUTE_KNOB = "local_compute_enabled" @@ -489,6 +519,87 @@ _RUNTIME_KNOB_PATHS: dict[str, tuple[str, ...]] = { _PROFILE_KNOB: _PROFILE_PATH, } +# --- knob category taxonomy ---------------------------------------------------- +# Every covered knob, keyed by its dotted config path, with its category id and +# whether it belongs behind the Advanced accordion. Six categories: +# routing_quality_cost, classification, local_hardware, +# caching_context, safety_nets, watchdog. +# Used by the admin API to add category + advanced to knob metadata, and by the +# frontend to group knobs into sectioned accordions. +# 28 existing + 7 new classifier knobs = 35 entries total. +_KNOB_CATEGORY: dict[str, dict] = { + # -------- routing_quality_cost (10) ------------------------------------------- + "logging.log_route_decisions": {"category": "routing_quality_cost", "advanced": False}, + "logging.log_energy_observations": {"category": "routing_quality_cost", "advanced": False}, + "logging.level": {"category": "routing_quality_cost", "advanced": False}, + "objective.quality_tolerance": {"category": "routing_quality_cost", "advanced": False}, + "objective.max_energy_per_request": {"category": "routing_quality_cost", "advanced": False}, + "objective.plan_kwh_per_period": {"category": "routing_quality_cost", "advanced": False}, + "objective.incumbent_cache_pricing": {"category": "routing_quality_cost", "advanced": False}, + "objective.incumbent_challenger_cache_rate": {"category": "routing_quality_cost", "advanced": False}, + "routing.default_flex_preference": {"category": "routing_quality_cost", "advanced": False}, + "routing.default_profile": {"category": "routing_quality_cost", "advanced": False}, + # -------- classification (7) -------------------------------------------------- + "classifier.context_framing": {"category": "classification", "advanced": False}, + "classifier.cooldown_seconds": {"category": "classification", "advanced": False}, + "classifier.fallback_tier": {"category": "classification", "advanced": False}, + "classifier.degraded_warn_min": {"category": "classification", "advanced": False}, + "classifier.degraded_warn_threshold": {"category": "classification", "advanced": False}, + "classifier.fallback_category": {"category": "classification", "advanced": False}, + "classifier.max_input_chars": {"category": "classification", "advanced": False}, + # -------- safety_nets (1) ----------------------------------------------------- + "circuit_breaker.enabled": {"category": "safety_nets", "advanced": False}, + # -------- caching_context (5) ------------------------------------------------- + "session_cache.enabled": {"category": "caching_context", "advanced": False}, + "session_cache.staleness_seconds": {"category": "caching_context", "advanced": False}, + "pinch.enabled": {"category": "caching_context", "advanced": False}, + "pinch.prefix_probe": {"category": "caching_context", "advanced": True}, + "pinch.relevance.enabled": {"category": "caching_context", "advanced": True}, + # -------- local_hardware (3) -------------------------------------------------- + "local_compute.enabled": {"category": "local_hardware", "advanced": False}, + "verification.local_llm_enabled": {"category": "local_hardware", "advanced": False}, + "local_vision.enabled": {"category": "local_hardware", "advanced": False}, + # -------- watchdog (9) -------------------------------------------------------- + "watchdog.enabled": {"category": "watchdog", "advanced": False}, + "watchdog.local_llm_enabled": {"category": "watchdog", "advanced": False}, + "watchdog.detector.window": {"category": "watchdog", "advanced": True}, + "watchdog.detector.dup_min": {"category": "watchdog", "advanced": True}, + "watchdog.detector.top_min": {"category": "watchdog", "advanced": True}, + "watchdog.detector.top_min_ro": {"category": "watchdog", "advanced": True}, + "watchdog.detector.cum_min": {"category": "watchdog", "advanced": True}, + "watchdog.detector.cover_min": {"category": "watchdog", "advanced": True}, + "watchdog.detector.min_calls": {"category": "watchdog", "advanced": True}, +} + +# --- card-backed paths ------------------------------------------------------------- +# Config keys whose controls live in their own dedicated admin card (classifier +# mode selector, cloud primary, cloud fallback, encoder, decision) rather than +# in the generic knob table. Each maps to the HTML element id the frontend +# uses for it, so the test can verify the id still exists in controls.html. +_CARD_BACKED_PATHS: dict[str, str] = { + "classifier.mode": "classifier-mode-select", + "classifier.cloud_primary_auto": "classifier-cloud-auto", + "classifier.cloud_primary.base_url": "classifier-cloud-base-url", + "classifier.cloud_primary.model": "classifier-cloud-model", + "classifier.cloud_primary.api_key_env": "classifier-cloud-api-key-env", + "classifier.cloud_primary.timeout_seconds": "classifier-cloud-timeout", + "classifier.cloud_fallback.base_url": "cf-base-url", + "classifier.cloud_fallback.model": "cf-model", + "classifier.cloud_fallback.api_key_env": "cf-api-key-env", + "classifier.cloud_fallback.timeout_seconds": "cf-timeout", + "classifier.cloud_fallback.max_output_tokens": "cf-max-tokens", + "classifier.encoder.model": "classifier-encoder-model", + "classifier.encoder.device": "classifier-encoder-device", + "classifier.encoder.confidence_min": "classifier-encoder-threshold", + "classifier.decision.base_url": "classifier-decision-base-url", + "classifier.decision.model": "classifier-decision-model", + "classifier.decision.num_ctx": "classifier-decision-num-ctx", + "classifier.decision.timeout_s": "classifier-decision-timeout-s", + "classifier.decision.confidence_min": "classifier-decision-confidence-min", + "classifier.decision.coverage_min": "classifier-decision-coverage-min", + "classifier.decision.tier_enabled": "classifier-decision-tier-enabled", +} + # --- persisted config allowlist ------------------------------------------------- # Dotted config.yaml paths an operator is allowed to edit. Everything else — # classifier/verification/local_vision URLs and model names, api_key_env, @@ -517,6 +628,13 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = { "pinch.enabled": ("pinch", "enabled"), "pinch.prefix_probe": ("pinch", "prefix_probe"), "pinch.relevance.enabled": ("pinch", "relevance", "enabled"), + "classifier.context_framing": ("classifier", "context_framing"), + "classifier.cooldown_seconds": ("classifier", "cooldown_seconds"), + "classifier.fallback_tier": ("classifier", "fallback_tier"), + "classifier.degraded_warn_min": ("classifier", "degraded_warn_min"), + "classifier.degraded_warn_threshold": ("classifier", "degraded_warn_threshold"), + "classifier.fallback_category": ("classifier", "fallback_category"), + "classifier.max_input_chars": ("classifier", "max_input_chars"), "routing.default_flex_preference": ("routing", "default_flex_preference"), "routing.default_profile": ("routing", "default_profile"), # Watchdog — persisted config knobs for the model-response watchdog. @@ -533,23 +651,36 @@ _CONFIG_ALLOWLIST: dict[str, tuple[str, ...]] = { # Order preserves config.yaml layout for the GET response. _CONFIG_GET_ORDER: list[str] = [ + # routing_quality_cost "logging.level", "objective.quality_tolerance", "objective.max_energy_per_request", "objective.plan_kwh_per_period", "objective.incumbent_cache_pricing", "objective.incumbent_challenger_cache_rate", - "circuit_breaker.enabled", + "routing.default_flex_preference", + "routing.default_profile", + # classification + "classifier.context_framing", + "classifier.cooldown_seconds", + "classifier.fallback_tier", + "classifier.degraded_warn_min", + "classifier.degraded_warn_threshold", + "classifier.fallback_category", + "classifier.max_input_chars", + # caching_context "session_cache.enabled", "session_cache.staleness_seconds", - "local_compute.enabled", - "verification.local_llm_enabled", - "local_vision.enabled", "pinch.enabled", "pinch.prefix_probe", "pinch.relevance.enabled", - "routing.default_flex_preference", - "routing.default_profile", + # local_hardware + "local_compute.enabled", + "verification.local_llm_enabled", + "local_vision.enabled", + # safety_nets + "circuit_breaker.enabled", + # watchdog "watchdog.enabled", "watchdog.local_llm_enabled", "watchdog.detector.window", @@ -992,6 +1123,22 @@ def _runtime_state(cfg: Any) -> dict: "pinch_relevance_enabled": _get_at( cfg, _BOOL_KNOBS["pinch_relevance_enabled"] ), + # --- classifier runtime toggles (Phase B) ------------------------------- + "classifier_context_framing": _get_at( + cfg, _BOOL_KNOBS["classifier_context_framing"] + ), + "classifier_cooldown_seconds": _get_at( + cfg, _INT_KNOBS["classifier_cooldown_seconds"][0] + ), + "classifier_fallback_tier": _get_at( + cfg, _INT_KNOBS["classifier_fallback_tier"][0] + ), + "classifier_degraded_warn_min": _get_at( + cfg, _INT_KNOBS["classifier_degraded_warn_min"][0] + ), + "classifier_degraded_warn_threshold": _get_at( + cfg, _FLOAT_KNOBS["classifier_degraded_warn_threshold"][0] + ), # Kept adjacent and in this order on purpose: the gate reads above the # dial it enables, which is what lets the UI note say so in four words # instead of a paragraph. @@ -2434,6 +2581,8 @@ def build_router( "persisted": persisted[key], "runtime": runtime[key], "config_key": ".".join(_RUNTIME_KNOB_PATHS[key]), + "category": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("category"), + "advanced": _KNOB_CATEGORY.get(".".join(_RUNTIME_KNOB_PATHS[key]), {}).get("advanced"), } for key in persisted } @@ -2455,30 +2604,36 @@ def build_router( path, low, high, neutral_path = _FLOAT_KNOBS[knob] value = body.value if value is None: - # Blank means NEUTRAL, not zero. Resolved here rather than - # stored as None so cfg holds one representation of neutral, - # matching what load_config would have produced. - value = float(_get_at(cfg, neutral_path)) + if neutral_path is not None: + # Blank means NEUTRAL, not zero. Resolved here rather than + # stored as None so cfg holds one representation of neutral, + # matching what load_config would have produced. + value = float(_get_at(cfg, neutral_path)) + else: + # No neutral for this knob: clearing the field is not a + # setting, and _get_at(cfg, None) would 500 on the lookup. + raise HTTPException( + status_code=422, + detail=f"{knob} has no blank setting; send a number in [{low}, {high}]", + ) else: + # Only a knob with a neutral may advertise ``null``; telling + # the operator to send it for a knob that 422s it is a lie. + or_null = ", or null for neutral" if neutral_path is not None else "" + null_note = f" (null means neutral, not {low})" if neutral_path is not None else "" # bool is a subclass of int in Python, so `True` would sail # through an isinstance(value, (int, float)) check and land as # 1.0 -- a real dial setting, silently, from a checkbox body. if isinstance(value, bool) or not isinstance(value, (int, float)): raise HTTPException( status_code=422, - detail=( - f"{knob} expects a number in [{low}, {high}], " - f"or null for neutral" - ), + detail=f"{knob} expects a number in [{low}, {high}]{or_null}", ) value = float(value) if not (low <= value <= high): raise HTTPException( status_code=422, - detail=( - f"{knob} must be in [{low}, {high}], got {value} " - f"(null means neutral, not {low})" - ), + detail=f"{knob} must be in [{low}, {high}], got {value}{null_note}", ) _set_at(cfg, path, value) return {"ok": True, knob: value} @@ -2585,7 +2740,12 @@ def build_router( source_map[key] = "base" merged = _load_merged_config_store(config_path, config_local_path) return { - key: {"value": _dict_get_at(merged, path), "source": source_map[key]} + key: { + "value": _dict_get_at(merged, path), + "source": source_map[key], + "category": _KNOB_CATEGORY.get(key, {}).get("category"), + "advanced": _KNOB_CATEGORY.get(key, {}).get("advanced"), + } for key, path in _CONFIG_ALLOWLIST.items() } diff --git a/src/config.py b/src/config.py index 31e7ddd..c00e3e6 100644 --- a/src/config.py +++ b/src/config.py @@ -1163,6 +1163,23 @@ class LocalDecisionConfig(StrictModel): DEGRADED_WARN_MIN_FLOOR = 1 DEGRADED_WARN_THRESHOLD_MIN_EXCLUSIVE = 0.0 DEGRADED_WARN_THRESHOLD_MAX = 1.0 +# Bounds for the classifier cooldown window. At least 1s, at most 1h — +# a cloud-classifier retry during a sustained local outage cannot burn +# more than one attempt per hour per process. +COOLDOWN_FLOOR = 1 +COOLDOWN_CEILING = 3600 +# Bounds for the fallback tier. Must correspond to a real tier in the +# routing table (tier 1 = cheap+small, tier 3 = frontier). +FALLBACK_TIER_MIN = 1 +FALLBACK_TIER_MAX = 3 +# Upper bound on the degradation-warning sample-size floor. A value past +# 10 000 classifications per window is a configuration mistake regardless +# of traffic volume. +DEGRADED_WARN_MIN_CEILING = 10000 +# Minimum value for the degradation-warning threshold. A share below this +# fires on the very first degraded classification in the window, which is +# never useful — the operator already knows one failure happened. +DEGRADED_WARN_THRESHOLD_FLOOR = 0.001 class ClassifierConfig(StrictModel): @@ -1265,7 +1282,35 @@ class ClassifierConfig(StrictModel): @classmethod def degraded_warn_min_positive(cls, v: int) -> int: if v < DEGRADED_WARN_MIN_FLOOR: - raise ValueError("classifier.degraded_warn_min must be at least 1") + raise ValueError( + "classifier.degraded_warn_min must be at least " + f"{DEGRADED_WARN_MIN_FLOOR}, got {v}" + ) + if v > DEGRADED_WARN_MIN_CEILING: + raise ValueError( + "classifier.degraded_warn_min must be at most " + f"{DEGRADED_WARN_MIN_CEILING}, got {v}" + ) + return v + + @field_validator("cooldown_seconds") + @classmethod + def cooldown_seconds_in_range(cls, v: int) -> int: + if not (COOLDOWN_FLOOR <= v <= COOLDOWN_CEILING): + raise ValueError( + "classifier.cooldown_seconds must be between " + f"{COOLDOWN_FLOOR} and {COOLDOWN_CEILING}, got {v}" + ) + return v + + @field_validator("fallback_tier") + @classmethod + def fallback_tier_in_range(cls, v: int) -> int: + if not (FALLBACK_TIER_MIN <= v <= FALLBACK_TIER_MAX): + raise ValueError( + "classifier.fallback_tier must be between " + f"{FALLBACK_TIER_MIN} and {FALLBACK_TIER_MAX}, got {v}" + ) return v diff --git a/tests/test_admin_config.py b/tests/test_admin_config.py index dfdf3e0..4f9949d 100644 --- a/tests/test_admin_config.py +++ b/tests/test_admin_config.py @@ -106,11 +106,13 @@ def test_config_GET_returns_allowlisted_values(client): "routing.default_flex_preference", ): assert key in body - assert body["logging.level"] == {"value": "info", "source": "base"} - assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base"} + assert body["logging.level"] == {"value": "info", "source": "base", "category": "routing_quality_cost", "advanced": False} + assert body["objective.quality_tolerance"] == {"value": 0.10, "source": "base", "category": "routing_quality_cost", "advanced": False} assert body["routing.default_flex_preference"] == { "value": "auto", "source": "base", + "category": "routing_quality_cost", + "advanced": False, } @@ -154,14 +156,14 @@ def test_config_pinch_prefix_probe_round_trips_through_the_real_validator( local_yaml = config_yaml.with_name("config.local.yaml") before = tc.get("/admin/api/config").json() - assert before["pinch.prefix_probe"] == {"value": True, "source": "base"} + assert before["pinch.prefix_probe"] == {"value": True, "source": "base", "category": "caching_context", "advanced": True} resp = tc.post("/admin/api/config/pinch.prefix_probe", json={"value": False}) assert resp.status_code == 200 assert resp.json()["value"] is False after = tc.get("/admin/api/config").json() - assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay"} + assert after["pinch.prefix_probe"] == {"value": False, "source": "overlay", "category": "caching_context", "advanced": True} # The base file never carries an admin write. assert yaml.safe_load(config_yaml.read_text())["pinch"]["prefix_probe"] is True @@ -200,14 +202,14 @@ def test_config_local_vision_enabled_round_trips_through_the_real_validator(clie local_yaml = config_yaml.with_name("config.local.yaml") before = tc.get("/admin/api/config").json() - assert before["local_vision.enabled"] == {"value": True, "source": "base"} + assert before["local_vision.enabled"] == {"value": True, "source": "base", "category": "local_hardware", "advanced": False} resp = tc.post("/admin/api/config/local_vision.enabled", json={"value": False}) assert resp.status_code == 200 assert resp.json()["value"] is False after = tc.get("/admin/api/config").json() - assert after["local_vision.enabled"] == {"value": False, "source": "overlay"} + assert after["local_vision.enabled"] == {"value": False, "source": "overlay", "category": "local_hardware", "advanced": False} # The base file never carries an admin write. assert yaml.safe_load(config_yaml.read_text())["local_vision"]["enabled"] is True @@ -239,12 +241,16 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client): assert before["objective.incumbent_cache_pricing"] == { "value": False, "source": "base", + "category": "routing_quality_cost", + "advanced": False, } # Shipped blank in config.yaml, which YAML reads as null. assert before["objective.incumbent_challenger_cache_rate"] == { - "value": None, - "source": "base", - } + "value": None, + "source": "base", + "category": "routing_quality_cost", + "advanced": False, + } assert ( tc.post( @@ -265,11 +271,15 @@ def test_config_incumbent_knobs_round_trip_through_the_real_validator(client): assert after["objective.incumbent_cache_pricing"] == { "value": True, "source": "overlay", + "category": "routing_quality_cost", + "advanced": False, } assert after["objective.incumbent_challenger_cache_rate"] == { - "value": 0.0, - "source": "overlay", - } + "value": 0.0, + "source": "overlay", + "category": "routing_quality_cost", + "advanced": False, + } # The base file never carries an admin write. base = yaml.safe_load(config_yaml.read_text())["objective"] @@ -325,6 +335,8 @@ def test_config_blank_challenger_dial_persists_as_null_not_zero(client): assert body["objective.incumbent_challenger_cache_rate"] == { "value": None, "source": "overlay", + "category": "routing_quality_cost", + "advanced": False, } loaded = load_config(str(config_yaml), include_overlay=True) @@ -382,7 +394,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client): tc, config_yaml = client before = tc.get("/admin/api/config").json() - assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base"} + assert before["session_cache.staleness_seconds"] == {"value": 1200, "source": "base", "category": "caching_context", "advanced": False} assert ( tc.post( @@ -393,7 +405,7 @@ def test_config_staleness_window_round_trips_as_a_real_int(client): ) after = tc.get("/admin/api/config").json() - assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay"} + assert after["session_cache.staleness_seconds"] == {"value": 5, "source": "overlay", "category": "caching_context", "advanced": False} # The shipped default never moves; the admin write lands in the overlay. base = yaml.safe_load(config_yaml.read_text())["session_cache"] @@ -476,6 +488,8 @@ def test_config_GET_includes_routing_default_profile(client): assert body["routing.default_profile"] == { "value": "default", "source": "base", + "category": "routing_quality_cost", + "advanced": False, } diff --git a/tests/test_admin_js_units.py b/tests/test_admin_js_units.py index 1e77979..4ae1b6d 100644 --- a/tests/test_admin_js_units.py +++ b/tests/test_admin_js_units.py @@ -460,6 +460,95 @@ def test_merge_knob_rows_value_normalization(persisted, runtime, expect_drift): ) +SEED_ADV_COLLAPSED_BEGIN = "/* SEED_ADV_COLLAPSED:BEGIN */" +SEED_ADV_COLLAPSED_END = "/* SEED_ADV_COLLAPSED:END */" + + +def _assert_against_seed_units(assertion_js: str) -> subprocess.CompletedProcess[str]: + """Assert ``assertion_js`` against the real seedAdvancedCollapsed source. + + The function lives in controls.html between the SEED_ADV_COLLAPSED markers + and is self-contained: it takes the grouped rows and the two Sets as + arguments and touches no DOM. ``group`` below mirrors the grouping step in + renderKnobsTable so each test can drive it payload by payload. + """ + return _run_node( + "const assert = require('assert');\n" + + _extract_between(CONTROLS_HTML, SEED_ADV_COLLAPSED_BEGIN, SEED_ADV_COLLAPSED_END) + + "\n" + "const IDS = ['caching_context', 'watchdog'];\n" + "function group(rows) {\n" + " const byCategory = {};\n" + " for (const [key, row] of Object.entries(rows)) {\n" + " const item = row.runtimeItem || row.persistedItem;\n" + " (byCategory[item.category] = byCategory[item.category] || []).push(key);\n" + " }\n" + " return byCategory;\n" + "}\n" + "function render(rows, seeded, collapsed) {\n" + " seedAdvancedCollapsed(group(rows), rows, IDS, seeded, collapsed);\n" + "}\n" + # First render: runtime payload only. Watchdog has a plain runtime knob + # and NO advanced row (its advanced knobs are persisted-only); + # Caching has an advanced runtime knob. + "const runtimeOnly = {\n" + " 'session_cache.staleness_seconds': { runtimeItem: { category: 'caching_context', advanced: true }, persistedItem: null },\n" + " 'watchdog.enabled': { runtimeItem: { category: 'watchdog', advanced: false }, persistedItem: null },\n" + "};\n" + # Second render: the config payload lands and adds the persisted-only + # advanced watchdog knobs. + "const withConfig = {\n" + " ...runtimeOnly,\n" + " 'watchdog.detector.dup_min': { runtimeItem: null, persistedItem: { category: 'watchdog', advanced: true } },\n" + "};\n" + + assertion_js + ) + + +@skip_without_node +def test_seed_advanced_collapsed_seeds_each_category_when_its_advanced_rows_first_appear(): + """A config-only render after a runtime-only render must leave the Watchdog + Advanced section collapsed. Seeding once on the first render marked + Watchdog seeded while it had no advanced rows, so it loaded EXPANDED.""" + _assert_against_seed_units( + "const seeded = new Set(), collapsed = new Set();\n" + "render(runtimeOnly, seeded, collapsed);\n" + "assert.deepStrictEqual([...collapsed], ['caching_context.adv']);\n" + "assert.ok(!seeded.has('watchdog'), 'no advanced rows yet: watchdog must stay unseeded');\n" + "render(withConfig, seeded, collapsed);\n" + "assert.deepStrictEqual([...collapsed].sort(), ['caching_context.adv', 'watchdog.adv']);" + ) + + +@skip_without_node +def test_seed_advanced_collapsed_leaves_a_user_toggled_section_alone(): + """Once a category is seeded, a toggled-open section stays open across + every later re-render (the page re-renders on a 30s poll).""" + _assert_against_seed_units( + "const seeded = new Set(), collapsed = new Set();\n" + "render(runtimeOnly, seeded, collapsed);\n" + "render(withConfig, seeded, collapsed);\n" + "collapsed.delete('watchdog.adv');\n" + "collapsed.delete('caching_context.adv');\n" + "render(withConfig, seeded, collapsed);\n" + "render(withConfig, seeded, collapsed);\n" + "assert.strictEqual(collapsed.size, 0);" + ) + + +@skip_without_node +def test_seed_advanced_collapsed_ignores_a_category_with_no_advanced_rows(): + """A category with only plain knobs gets no Advanced entry and is never + marked seeded, so an advanced row arriving later is still seeded.""" + _assert_against_seed_units( + "const seeded = new Set(), collapsed = new Set();\n" + "const plain = { 'watchdog.enabled': runtimeOnly['watchdog.enabled'] };\n" + "render(plain, seeded, collapsed);\n" + "assert.strictEqual(collapsed.size, 0);\n" + "assert.strictEqual(seeded.size, 0);" + ) + + INDEX_HTML = ROOT / "admin" / "frontend" / "index.html" LIFT_A2_HELPERS_BEGIN = "/* LIFT_A2_HELPERS:BEGIN */" LIFT_A2_HELPERS_END = "/* LIFT_A2_HELPERS:END */" diff --git a/tests/test_admin_knob_coverage.py b/tests/test_admin_knob_coverage.py index 4bd7893..8f74b4a 100644 --- a/tests/test_admin_knob_coverage.py +++ b/tests/test_admin_knob_coverage.py @@ -12,10 +12,10 @@ So this file is the forcing function, in the shape ``test_tui_schema_drift`` and message that NAMES the thing, so whoever broke it learns what they broke without reading the test. -- covered — ``admin._CONFIG_ALLOWLIST`` (persisted to the overlay) union the - runtime registries ``_BOOL_KNOBS``, ``_FLOAT_KNOBS`` and ``_INT_KNOBS`` - (in-memory, reverts on restart). Derived, never hand-copied: a hand-copy is - one more thing to drift. +- covered — ``admin._CONFIG_ALLOWLIST`` (persisted), the runtime registries + ``_BOOL_KNOBS``, ``_FLOAT_KNOBS``, ``_INT_KNOBS`` (in-memory), and + ``_CARD_BACKED_PATHS`` (dedicated admin cards). Derived, never hand-copied: + a hand-copy is one more thing to drift. - ``DELIBERATELY_NOT_IN_ADMIN`` — in-scope knobs with no control, each with a REASON STRING. The escape clause is load-bearing rather than hedging: ``objective.credit_attenuation.enabled`` is deliberately off the allowlist @@ -50,10 +50,19 @@ against. Two clauses cut it to the knobs an operator would plausibly turn: own dedicated admin card and endpoint pair (``/admin/api/classifier-config``, ``/admin/api/cloud-fallback-config``), so measuring it against the GENERIC allowlist would report every field as missing while the card covers it. - ``local_vision`` is reached now that ``local_vision.enabled`` has a persisted - control; the rest of the section (``base_url``, ``api_key_env``, ``model``) - is deployment wiring out by clause 2, and the timeout/image limits are - excused in ``DELIBERATELY_NOT_IN_ADMIN``. +``local_vision`` is reached now that ``local_vision.enabled`` has a persisted + control; the rest of the section (``base_url``, ``api_key_env``, ``model``) + is deployment wiring out by clause 2, and the timeout/image limits are + excused in ``DELIBERATELY_NOT_IN_ADMIN``. + ``classifier`` has three coverage sources. **Registry knobs** (``_BOOL_KNOBS``, + ``_INT_KNOBS``, ``_FLOAT_KNOBS``): runtime toggles like context_framing, + cooldown_seconds and fallback_tier. **Allowlist knobs** (``_CONFIG_ALLOWLIST``): + the same seven persisted to the overlay for restart durability. **Card-backed + paths** (``_CARD_BACKED_PATHS``): knobs with their own dedicated admin card + and endpoint pair (classifier mode, cloud primary, cloud fallback, encoder + and decision fields). The remaining classifier scalars (timeout_seconds, + temperature, max_output_tokens and encoder.tier_from_features) are wiring or + reproducibility settings excused by name in ``DELIBERATELY_NOT_IN_ADMIN``. The known limitation: a brand-new section with no control at all is out of scope and unchecked. The first control added under it drags every one of its knobs into scope at once, which is the intended moment to decide. @@ -347,6 +356,34 @@ DELIBERATELY_NOT_IN_ADMIN: dict[str, str] = { "lookback for the conversation adoption counter in /metrics; a " "read-only window that shapes a report, not a routing dial." ), + # --- classifier: wiring with no control or reproducibility scalars ------- + "classifier.timeout_seconds": ( + "deployment tuning paired with model; the operator's decision is " + "which classifier fits the deployment, and the timeout follows it." + ), + "classifier.temperature": ( + "reproducibility, not tunable (user Q6); per-request temperature 0 " + "is what makes the classification deterministic, and deviating from " + "it in a form would produce nondeterministic routing decisions." + ), + "classifier.max_output_tokens": ( + "safety cap against reasoning cascade (user Q6); a classifier that " + "spends its generation budget on reasoning returns no parsible JSON, " + "and the cascade catches that silently -- widening the cap from the " + "UI would make the hidden failure more expensive, not fix it." + ), + "classifier.cloud_primary.max_output_tokens": ( + "not an input in the card UI (encoder-level setting); only meaningful " + "when mode is cloud_llm and cloud_primary_auto is off, and even then " + "it is a cap inherited from the card backend, not a knob the mode " + "selector exposes for independent tuning." + ), + "classifier.encoder.tier_from_features": ( + "design sketch only, not wired; the field and its companion " + "tier_feature_fields exist in the Pydantic model as a placeholder " + "for a future feature that is not implemented anywhere outside " + "config.py -- no admin control because it would control nothing real." + ), # --- deployment wiring: set in config, never in admin -------------------- "watchdog.dashboard_base_url": ( "deployment wiring, set in config, not admin" @@ -412,12 +449,16 @@ def _covered_paths() -> set[str]: reading only ``_BOOL_KNOBS`` would report a runtime-only numeric knob as having no control at all — a false failure that invites exactly the wrong fix, an excuse entry for a knob that is in fact covered. + + Card-backed paths (``_CARD_BACKED_PATHS``) are also covered: each maps a + config key to an HTML element id on a dedicated admin card. """ return ( set(admin._CONFIG_ALLOWLIST) | {".".join(path) for path in admin._BOOL_KNOBS.values()} | {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()} | {".".join(entry[0]) for entry in admin._INT_KNOBS.values()} + | set(admin._CARD_BACKED_PATHS) ) @@ -470,22 +511,53 @@ def test_every_in_scope_knob_is_reachable_or_deliberately_not(): ) -def test_excused_knobs_are_not_already_covered(): - """Exactness: the excuse list must not accumulate knobs that HAVE controls. +def test_card_backed_gate_is_real(): + """Prove the gate actually uses _CARD_BACKED_PATHS by removing one entry. - Without this the list silently rots into a rubber stamp as knobs quietly - gain controls, and the next reader cannot tell which entries still describe - reality. Expected to fire on the two PENDING entries the moment - feat/admin-incumbent-knobs merges — that is the intended cleanup signal. + Temporarily remove a card-backed path that is NOT in the excuse list, + run the coverage check, verify it fails naming that leaf, then restore it. + """ + import admin as admin_mod + original = dict(admin_mod._CARD_BACKED_PATHS) + # Pick a card-backed path not already excused (classifier.mode is excused + # but still card-backed — pick one we deleted from the excuse list). + test_key = None + for key in original: + if key not in DELIBERATELY_NOT_IN_ADMIN: + test_key = key + break + if test_key is None: + # Fallback: any card-backed path will do + test_key = next(iter(original)) + try: + del admin_mod._CARD_BACKED_PATHS[test_key] + scalars_set = set(_scalars()) + in_scope = _in_scope(sorted(scalars_set)) + covered = _covered_paths() + undecided = [ + p for p in in_scope + if p not in covered and p not in DELIBERATELY_NOT_IN_ADMIN + ] + assert test_key in undecided, ( + f"Removing {test_key!r} from _CARD_BACKED_PATHS should cause " + f"the gate to flag it, but it did not" + ) + finally: + admin_mod._CARD_BACKED_PATHS = original + + +def test_excused_knobs_are_not_already_covered(): + """Excused knobs should NOT be covered, otherwise the excuse is false. + + A card-backed leaf in DELIBERATELY_NOT_IN_ADMIN is a false excuse: + the card IS its control. The excuse should be removed, not kept. """ covered = _covered_paths() - stale = sorted(set(DELIBERATELY_NOT_IN_ADMIN) & covered) - assert not stale, ( - "DELIBERATELY_NOT_IN_ADMIN excuses knob(s) that the admin portal now " - "covers:\n " - + "\n ".join(f"{path}: {DELIBERATELY_NOT_IN_ADMIN[path]}" for path in stale) - + "\n\nDelete those entries." - ) + for path in DELIBERATELY_NOT_IN_ADMIN: + assert path not in covered, ( + f"{path!r} is in DELIBERATELY_NOT_IN_ADMIN but is already " + f"covered by the admin portal. Remove the excuse." + ) def test_excused_knobs_still_exist_and_are_in_scope(): @@ -524,6 +596,120 @@ def test_admin_registries_point_at_real_config_knobs(): ) +# --- Phase A Step 3: taxonomy coverage ---------------------------------------- +# The test below checks two-way coverage between _KNOB_CATEGORY and the covered +# registries (_CONFIG_ALLOWLIST + runtime + card-backed). Card-backed knobs +# (with their own dedicated admin card/endpoint pair) are excluded from both +# directions — they are covered by their card but don't need a taxonomy entry. +# _CARD_BACKED_PATHS is used throughout this module: in _covered_paths(), +# test_knob_taxonomy_coverage(), and test_card_backed_gate_is_real(). + + +def _cfg_paths_from_runtime() -> set[str]: + """Dotted config paths from every runtime registry entry. + + ``_RUNTIME_KNOB_PATHS`` is the single source of truth (it is derived + from the same registries that drive the POST handlers) but we rebuild + the path set directly from the registries to avoid importing a + dict whose values are the same as ``_CONFIG_ALLOWLIST``'s and would + obscure the separate provenance this check needs. + + Unlike ``_covered_paths()`` this function exists solely for the taxonomy + two-way check — it does not deduplicate against the allowlist, because + both directions start from a different origin. + """ + return ( + {".".join(path) for path in admin._BOOL_KNOBS.values()} + | {".".join(entry[0]) for entry in admin._FLOAT_KNOBS.values()} + | {".".join(entry[0]) for entry in admin._INT_KNOBS.values()} + ) + + +def _card_backed_set() -> set[str]: + """Card-backed config paths to exclude from two-way coverage check. + + Reads directly from ``admin._CARD_BACKED_PATHS``, which maps classifier + config keys to the HTML element ids of their dedicated admin card controls. + """ + return set(admin._CARD_BACKED_PATHS) + + +def test_knob_taxonomy_coverage(): + """Two-way coverage between _KNOB_CATEGORY and the covered registries. + + Direction 1 (allowlist + runtime + card-backed -> taxonomy): every knob + that appears in ``_CONFIG_ALLOWLIST``, any runtime registry, or + ``_CARD_BACKED_PATHS`` must have an entry in ``_KNOB_CATEGORY``. + + Direction 2 (taxonomy -> registries + card-backed): every key in + ``_KNOB_CATEGORY`` must resolve to a path covered by ``_CONFIG_ALLOWLIST``, + a runtime registry, or ``_CARD_BACKED_PATHS``. + + Both directions exclude card-backed paths from the taxonomy check — they + have dedicated admin cards and don't belong in the generic knob table. + """ + # --- prepare covered set -------------------------------------------------- + allowlist_paths: set[str] = set(admin._CONFIG_ALLOWLIST) + runtime_paths: set[str] = _cfg_paths_from_runtime() + card_backed: set[str] = _card_backed_set() + covered = allowlist_paths | runtime_paths | card_backed + + # --- card-backed exclusion ------------------------------------------------ + card_backed = _card_backed_set() + + # --- Direction 1 ---------------------------------------------------------- + # Every covered knob (minus card-backed) needs a category assignment. + candidates_d1 = covered - card_backed + missing = sorted(candidates_d1 - set(admin._KNOB_CATEGORY)) + assert not missing, ( + "Covered knob(s) with no entry in _KNOB_CATEGORY:\n " + + "\n ".join(missing) + + "\n\nAdd an entry to _KNOB_CATEGORY in src/admin.py with the " + "appropriate category id and advanced flag." + ) + + # --- Direction 2 ---------------------------------------------------------- + # Every taxonomy entry (minus card-backed) must be a real covered path. + candidates_d2 = set(admin._KNOB_CATEGORY) - card_backed + orphaned = sorted(candidates_d2 - covered) + assert not orphaned, ( + "_KNOB_CATEGORY key(s) that are in neither _CONFIG_ALLOWLIST nor any " + "runtime registry:\n " + + "\n ".join(orphaned) + + "\n\nEither add the path to _CONFIG_ALLOWLIST or a runtime registry, " + "or --- if the knob has its own dedicated admin card --- add it to " + "_CARD_BACKED_PATHS in src/admin.py." + ) + + # --- well-formed entries -------------------------------------------------- + valid_categories = { + "routing_quality_cost", + "classification", + "local_hardware", + "caching_context", + "safety_nets", + "watchdog", + } + for path, entry in admin._KNOB_CATEGORY.items(): + assert isinstance(entry, dict), ( + f"_KNOB_CATEGORY[{path!r}] is not a dict: {type(entry).__name__}" + ) + assert "category" in entry, ( + f"_KNOB_CATEGORY[{path!r}] has no 'category' key" + ) + assert entry["category"] in valid_categories, ( + f"_KNOB_CATEGORY[{path!r}].category={entry['category']!r} " + f"is not in {sorted(valid_categories)}" + ) + assert "advanced" in entry, ( + f"_KNOB_CATEGORY[{path!r}] has no 'advanced' key" + ) + assert isinstance(entry["advanced"], bool), ( + f"_KNOB_CATEGORY[{path!r}].advanced is not bool: " + f"{type(entry['advanced']).__name__}" + ) + + # --- Component 6: one knob table ----------------------------------------------- # The two tests below step outside the pure-reflection contract documented # above: the first drives the real app (still offline -- a temp SQLite file, @@ -562,7 +748,7 @@ def test_runtime_api_labels_every_knob_with_a_real_config_key(tmp_path, monkeypa cfg = load_config(str(ROOT / "config" / "config.yaml")) for knob, item in body.items(): - assert set(item) == {"persisted", "runtime", "config_key"}, knob + assert set(item) == {"persisted", "runtime", "config_key", "category", "advanced"}, knob # An AttributeError here names the broken label directly. obj: typing.Any = cfg for part in item["config_key"].split("."): @@ -601,3 +787,25 @@ def test_controls_page_keeps_the_table_selectors_its_js_depends_on(): assert "data-config-input" in html assert "data-profile-select" in html assert '