feat(admin): Lift A.2, surface loops and stalling models with one-click block #104

Merged
alee merged 17 commits from feat/no-progress-lift-a2 into main 2026-09-28 23:56:34 +00:00
9 changed files with 1419 additions and 28 deletions

View File

@@ -1336,28 +1336,206 @@ function hashCode(str) {
LOOPS (open watchdog alerts)
═══════════════════════════════════════ */
/* LIFT_A2_HELPERS:BEGIN */
function formatMoney(usd) {
if (usd == null) return 'n/a';
return '$' + Number(usd).toFixed(2);
}
function formatRelativeTime(iso8601) {
if (!iso8601) return '';
const now = new Date();
const date = new Date(iso8601);
const diffMs = now - date;
const diffSeconds = Math.floor(diffMs / 1000);
if (diffSeconds < 10) return 'just now';
if (diffSeconds < 60) return Math.floor(diffSeconds) + 's ago';
const diffMinutes = Math.floor(diffSeconds / 60);
if (diffMinutes < 60) return diffMinutes + 'm ago';
const diffHours = Math.floor(diffMinutes / 60);
if (diffHours < 24) return diffHours + 'h ago';
const diffDays = Math.floor(diffHours / 24);
if (diffDays < 30) return diffDays + 'd ago';
return diffDays + 'd ago';
}
function truncate(text, maxLen) {
if (!text) return '';
if (text.length <= maxLen) return text;
return text.substring(0, maxLen) + '...';
}
/* LIFT_A2_HELPERS:END */
async function loadLoops() {
const rows = await apiFetch(`${API}api/watchdog/loops`);
const data = await apiFetch(`${API}api/watchdog/loops`);
const body = document.getElementById('loops-body');
const countEl = document.getElementById('loops-count');
if (!rows || !rows.length) {
const alerts = data && data.alerts ? data.alerts : [];
const resolved = data && data.resolved ? data.resolved : [];
countEl.textContent = `${alerts.length} open`;
if (alerts.length === 0 && resolved.length === 0) {
body.innerHTML = '<div class="text-muted small">No open alerts</div>';
countEl.textContent = '0 open';
return;
}
countEl.textContent = `${rows.length} open`;
body.innerHTML = rows.map(r => {
let html = '<div class="table-responsive"><table class="table table-vcenter card-table"><thead><tr>'
+ '<th>Severity</th><th>Session</th><th>Model</th><th>Top target</th>'
+ '<th class="num">Calls since landed</th><th class="num">$ since landed</th><th class="num">Open for</th><th></th>'
+ '</tr></thead><tbody>';
alerts.forEach(r => {
const sev = r.severity === 'critical' ? 'bg-danger'
: r.severity === 'warning' ? 'bg-warning'
: 'bg-info';
return `<div class="d-flex align-items-center gap-2 py-1" style="border-bottom:1px solid rgba(255,255,255,.05)">
<span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span>
<span class="small" style="flex:1">${escapeHtml(r.dedup_key)}</span>
<span class="text-muted" style="font-size:.66rem">${escapeHtml(r.opened_at || '')}</span>
</div>`;
}).join('');
const title = escapeHtml(truncate(r.title || r.dedup_key || '', 40));
const fullTitle = escapeHtml(r.title || r.dedup_key || '');
const model = r.model_id ? escapeHtml(r.model_id) : '<span class="text-muted">unknown</span>';
const topWhat = r.top_what ? escapeHtml(r.top_what) : '<span class="text-muted">&mdash;</span>';
const calls = r.calls_since_landed != null ? escapeHtml(String(r.calls_since_landed)) : '?';
const cost = formatMoney(r.cost_since_landed_usd);
const relTime = formatRelativeTime(r.opened_at);
const dataModel = r.model_id || '';
const dataProvider = r.provider || '';
const dataReason = 'looping session ' + (r.title || r.dedup_key || '');
const blockBtn = `<button class="btn btn-sm btn-outline-danger" style="font-size:.62rem;padding:1px 6px" data-model="${escapeHtml(dataModel)}" data-provider="${escapeHtml(dataProvider)}" data-reason="${escapeHtml(dataReason)}" onclick="blockModelConfirm(event)">Block</button>`;
html += '<tr>'
+ `<td><span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span></td>`
+ `<td><span class="small text-truncate" style="max-width:200px" title="${fullTitle}">${title}</span></td>`
+ `<td>${model}</td>`
+ `<td class="text-muted" style="font-size:.65rem">${topWhat}</td>`
+ `<td class="num">${calls}</td>`
+ `<td class="num">${cost}</td>`
+ `<td class="num">${relTime}</td>`
+ `<td class="text-end">${blockBtn}</td>`
+ '</tr>';
});
if (resolved.length > 0) {
resolved.forEach(r => {
const sev = r.severity === 'critical' ? 'bg-danger'
: r.severity === 'warning' ? 'bg-warning'
: 'bg-info';
const title = escapeHtml(truncate(r.title || r.dedup_key || '', 40));
const fullTitle = escapeHtml(r.title || r.dedup_key || '');
const model = r.model_id ? escapeHtml(r.model_id) : '<span class="text-muted">unknown</span>';
const topWhat = r.top_what ? escapeHtml(r.top_what) : '<span class="text-muted">&mdash;</span>';
const calls = r.calls_since_landed != null ? escapeHtml(String(r.calls_since_landed)) : '?';
const cost = formatMoney(r.cost_since_landed_usd);
const relTime = formatRelativeTime(r.opened_at);
html += '<tr class="opacity-50">'
+ `<td><span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span></td>`
+ `<td><span class="small text-truncate" style="max-width:200px" title="${fullTitle}">${title}</span></td>`
+ `<td>${model}</td>`
+ `<td class="text-muted" style="font-size:.65rem">${topWhat}</td>`
+ `<td class="num">${calls}</td>`
+ `<td class="num">${cost}</td>`
+ `<td class="num">${relTime}</td>`
+ '<td></td>'
+ '</tr>';
});
}
html += '</tbody></table></div>';
body.innerHTML = html;
}
/* BLOCK_CONFIRM:BEGIN */
function blockModelConfirm(event) {
const btn = event.currentTarget;
const model = btn.dataset.model;
const prov = btn.dataset.provider;
const reason = btn.dataset.reason;
if (!model) return;
const row = btn.closest('div.d-flex') || btn.closest('tr');
if (!row) return;
const existing = row.nextElementSibling;
if (existing && existing.classList.contains('inline-confirm-row')) {
existing.remove();
return;
}
const confirmRow = document.createElement('div');
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
confirmRow.innerHTML = '<input type="text" class="form-control form-control-sm" style="font-size:.72rem;flex:1;min-width:120px" value="' + escapeHtml(reason) + '">'
+ '<button class="btn btn-sm btn-danger" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
const cancelBtn = confirmRow.querySelector('button:last-child');
cancelBtn.onclick = () => confirmRow.remove();
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
const input = confirmRow.querySelector('input');
confirmBtn.onclick = async () => {
const reason = input.value;
confirmBtn.disabled = true;
const resp = await apiFetch(
`${API}api/models/${encodeURIComponent(model)}/${encodeURIComponent(prov)}/availability`,
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: 'blocked', reason: reason }) }
);
if (resp) {
toast('Model blocked', 'success');
confirmRow.remove();
// Refresh dependent views
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
if (typeof loadBlockedModels === 'function') loadBlockedModels();
if (typeof loadLoops === 'function') loadLoops();
if (typeof loadModels === 'function') loadModels();
} else {
confirmBtn.disabled = false;
confirmRow.querySelector('.inline-error')?.remove();
const err = document.createElement('div');
err.className = 'text-danger small inline-error';
err.textContent = 'Block failed';
confirmRow.appendChild(err);
}
};
row.after(confirmRow);
}
function unblockModelConfirm(modelId, provider) {
const btn = event.target;
const row = btn.closest('div.d-flex') || btn.closest('tr');
if (!row) return;
const existing = row.nextElementSibling;
if (existing && existing.classList.contains('inline-confirm-row')) {
existing.remove();
return;
}
const confirmRow = document.createElement('div');
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
confirmRow.innerHTML = '<span class="small text-danger">Unblock this model?</span>'
+ '<button class="btn btn-sm btn-warning" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
const cancelBtn = confirmRow.querySelector('button:last-child');
cancelBtn.onclick = () => confirmRow.remove();
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
confirmBtn.onclick = async () => {
confirmBtn.disabled = true;
const resp = await apiFetch(
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
{ method:'DELETE' }
);
if (resp) {
toast('Model unblocked', 'success');
confirmRow.remove();
if (typeof loadBlockedModels === 'function') loadBlockedModels();
if (typeof loadModels === 'function') loadModels();
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
} else {
confirmBtn.disabled = false;
confirmRow.querySelector('.inline-error')?.remove();
const err = document.createElement('div');
err.className = 'text-danger small inline-error';
err.textContent = 'Unblock failed';
confirmRow.appendChild(err);
}
};
row.after(confirmRow);
}
/* BLOCK_CONFIRM:END */
/* ═══════════════════════════════════════
INIT
═══════════════════════════════════════ */

View File

@@ -124,7 +124,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
#model-tbody .model-override{height:24px;padding:.05rem .35rem;font-size:.7rem;line-height:1.2}
/* Numeric columns align right and share the tabular figures the rest of the
portal uses, so magnitudes line up down the column. */
#model-tbody td.num, thead th.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}
tbody td.num, thead th.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}
.prof-n{color:var(--tblr-secondary);opacity:.6;font-size:.62rem;margin-left:1px}
.cost-free{color:#4ade80}
/* Capability chips reuse the decisions page's flag-chip language so the two
@@ -221,6 +221,48 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
</div>
<div class="page-body">
<div class="container-xl">
<div class="row row-cards mb-3" id="view-stalls">
<div class="col-12">
<div class="card">
<div class="card-header">
<div class="d-flex align-items-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><line x1="3" y1="9" x2="21" y2="9"/><line x1="9" y1="21" x2="9" y2="9"/></svg>
<h3 class="card-title mb-0">Stalls by model</h3>
</div>
<div class="card-subtitle text-muted ms-2">Flagged looping sessions per model</div>
<div class="ms-auto d-flex align-items-center gap-2">
<div class="btn-seg" id="stall-window">
<button type="button" data-hours="1">1h</button>
<button type="button" data-hours="24" class="active">24h</button>
<button type="button" data-hours="168">7d</button>
</div>
</div>
</div>
<div class="card-body" style="padding:.6rem .8rem">
<div id="stalls-body">
<div class="text-muted small">Loading&hellip;</div>
</div>
</div>
</div>
</div>
</div>
<div class="row row-cards mb-3" id="view-blocked">
<div class="col-12">
<div class="card">
<div class="card-header">
<div class="d-flex align-items-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="4.93" y1="4.93" x2="19.07" y2="19.07"/></svg>
<h3 class="card-title mb-0">Blocked models</h3>
</div>
</div>
<div class="card-body" style="padding:.6rem .8rem">
<div id="blocked-models-body">
<div class="text-muted small">Loading&hellip;</div>
</div>
</div>
</div>
</div>
</div>
<div class="row row-cards" id="view-available">
<div class="col-12">
<div class="card">
@@ -354,7 +396,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
</div>
</div>
<div id="toast class="toast align-items-center text-bg-info border-0 position-fixed bottom-0 end-0 p-3" role="alert" aria-live="assertive" aria-atomic="true"></div>
<div id="toast" class="toast align-items-center text-bg-info border-0 position-fixed bottom-0 end-0 p-3" role="alert" aria-live="assertive" aria-atomic="true"></div>
<script>
/* ═══════════════════════════════════════════════════
@@ -619,9 +661,13 @@ function renderModels(models) {
async function updateAvailability(modelId, provider, avail) {
// Don't post if already that value via dropdown
const body = { availability: avail };
if (avail === 'blocked') {
body.reason = 'set from Models dropdown';
}
const resp = await apiFetch(
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: avail }) }
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(body) }
);
const tr = document.querySelector(`tr[data-model="${modelId}"][data-provider="${provider}"]`);
const warningDiv = tr ? tr.querySelector('.inline-warning') : null;
@@ -823,17 +869,216 @@ function escapeHtml(s) {
return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
}
/* LOAD_STALL_ROLLUP:BEGIN */
async function loadStallRollup(windowHours) {
const body = document.getElementById('stalls-body');
if (!body) return;
const data = await apiFetch(`${API}api/watchdog/rollup?hours=${windowHours}`);
if (!data || !data.length) {
body.innerHTML = '<div class="text-muted small">No stalled sessions in this window</div>';
return;
}
body.innerHTML = '<div class="table-responsive"><table class="table table-vcenter table-hover card-table"><thead><tr>'
+ '<th>Model</th><th>Provider</th><th class="num">Stalled</th><th class="num">$ Stalled</th><th class="num">Traffic</th><th>Availability</th><th></th>'
+ '</tr></thead><tbody>'
+ data.map(r => {
const model = escapeHtml(r.model_id || '');
const provider = escapeHtml(r.provider || '');
const stalled = r.stalled_sessions != null ? r.stalled_sessions : 0;
const cost = r.stalled_usd != null ? '$' + Number(r.stalled_usd).toFixed(2) : 'n/a';
const share = r.traffic_share != null ? (Number(r.traffic_share) * 100).toFixed(1) + '%' : '&mdash;';
const avail = escapeHtml(r.effective_availability || 'active');
const blocked = r.effective_availability === 'blocked';
const blockBtn = blocked ? '' : `<button class="btn btn-sm btn-outline-danger" style="font-size:.62rem;padding:1px 6px" data-model="${model}" data-provider="${provider}" data-reason="stalled ${stalled} sessions in ${windowHours}h" onclick="blockModelConfirm(event)">Block</button>`;
return '<tr>'
+ `<td class="text-nowrap">${model}</td>`
+ `<td>${provider}</td>`
+ `<td class="num">${stalled}</td>`
+ `<td class="num">${cost}</td>`
+ `<td class="num">${share}</td>`
+ `<td>${avail}</td>`
+ `<td class="text-end">${blockBtn}</td>`
+ '</tr>';
}).join('')
+ '</tbody></table></div>';
}
/* LOAD_STALL_ROLLUP:END */
/* BLOCK_CONFIRM:BEGIN */
function blockModelConfirm(event) {
const btn = event.currentTarget;
const model = btn.dataset.model;
const prov = btn.dataset.provider;
const reason = btn.dataset.reason;
if (!model) return;
const row = btn.closest('div.d-flex') || btn.closest('tr');
if (!row) return;
const existing = row.nextElementSibling;
if (existing && existing.classList.contains('inline-confirm-row')) {
existing.remove();
return;
}
const confirmRow = document.createElement('div');
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
confirmRow.innerHTML = '<input type="text" class="form-control form-control-sm" style="font-size:.72rem;flex:1;min-width:120px" value="' + escapeHtml(reason) + '">'
+ '<button class="btn btn-sm btn-danger" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
const cancelBtn = confirmRow.querySelector('button:last-child');
cancelBtn.onclick = () => confirmRow.remove();
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
const input = confirmRow.querySelector('input');
confirmBtn.onclick = async () => {
const reason = input.value;
confirmBtn.disabled = true;
const resp = await apiFetch(
`${API}api/models/${encodeURIComponent(model)}/${encodeURIComponent(prov)}/availability`,
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: 'blocked', reason: reason }) }
);
if (resp) {
toast('Model blocked', 'success');
confirmRow.remove();
// Refresh dependent views
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
if (typeof loadBlockedModels === 'function') loadBlockedModels();
if (typeof loadLoops === 'function') loadLoops();
if (typeof loadModels === 'function') loadModels();
} else {
confirmBtn.disabled = false;
confirmRow.querySelector('.inline-error')?.remove();
const err = document.createElement('div');
err.className = 'text-danger small inline-error';
err.textContent = 'Block failed';
confirmRow.appendChild(err);
}
};
row.after(confirmRow);
}
function unblockModelConfirm(modelId, provider) {
const btn = event.target;
const row = btn.closest('div.d-flex') || btn.closest('tr');
if (!row) return;
const existing = row.nextElementSibling;
if (existing && existing.classList.contains('inline-confirm-row')) {
existing.remove();
return;
}
const confirmRow = document.createElement('div');
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
confirmRow.innerHTML = '<span class="small text-danger">Unblock this model?</span>'
+ '<button class="btn btn-sm btn-warning" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
const cancelBtn = confirmRow.querySelector('button:last-child');
cancelBtn.onclick = () => confirmRow.remove();
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
confirmBtn.onclick = async () => {
confirmBtn.disabled = true;
const resp = await apiFetch(
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
{ method:'DELETE' }
);
if (resp) {
toast('Model unblocked', 'success');
confirmRow.remove();
if (typeof loadBlockedModels === 'function') loadBlockedModels();
if (typeof loadModels === 'function') loadModels();
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
} else {
confirmBtn.disabled = false;
confirmRow.querySelector('.inline-error')?.remove();
const err = document.createElement('div');
err.className = 'text-danger small inline-error';
err.textContent = 'Unblock failed';
confirmRow.appendChild(err);
}
};
row.after(confirmRow);
}
/* BLOCK_CONFIRM:END */
/* ═══════════════════════════════════════
BLOCKED MODELS
═══════════════════════════════════════ */
/* BLOCKED_HELPERS:BEGIN */
function formatRelativeTime(iso8601) {
if (!iso8601) return '';
const now = new Date();
const date = new Date(iso8601);
const diffMs = now - date;
const diffSeconds = Math.floor(diffMs / 1000);
if (diffSeconds < 10) return 'just now';
if (diffSeconds < 60) return Math.floor(diffSeconds) + 's ago';
const diffMinutes = Math.floor(diffSeconds / 60);
if (diffMinutes < 60) return diffMinutes + 'm ago';
const diffHours = Math.floor(diffMinutes / 60);
if (diffHours < 24) return diffHours + 'h ago';
const diffDays = Math.floor(diffHours / 24);
if (diffDays < 30) return diffDays + 'd ago';
return diffDays + 'd ago';
}
/* BLOCKED_HELPERS:END */
async function loadBlockedModels() {
const body = document.getElementById('blocked-models-body');
if (!body) return;
const data = await apiFetch(`${API}api/models/blocked`);
if (!data || !data.length) {
body.innerHTML = '<div class="text-muted small">No blocked models</div>';
return;
}
body.innerHTML = '<div class="table-responsive"><table class="table table-vcenter table-hover card-table"><thead><tr>'
+ '<th>Model</th><th>Provider</th><th>Reason</th><th>Blocked Since</th><th></th>'
+ '</tr></thead><tbody>'
+ data.map(r => {
const model = escapeHtml(r.model_id || '');
const provider = escapeHtml(r.provider || '');
const reason = escapeHtml(r.reason || '');
const since = r.blocked_since ? formatRelativeTime(r.blocked_since) : '&mdash;';
return '<tr>'
+ `<td class="text-nowrap">${model}</td>`
+ `<td>${provider}</td>`
+ `<td class="text-muted">${reason}</td>`
+ `<td class="text-muted">${since}</td>`
+ `<td class="text-end"><button class="btn btn-sm btn-outline-warning" style="font-size:.62rem;padding:1px 6px" onclick="unblockModelConfirm('${model}', '${provider}')">Unblock</button></td>`
+ '</tr>';
}).join('')
+ '</tbody></table></div>';
}
/* ═══════════════════════════════════════
INIT
═══════════════════════════════════════ */
function bindStallWindow() {
const seg = document.getElementById('stall-window');
if (!seg) return;
seg.addEventListener('click', ev => {
const btn = ev.target.closest('button');
if (!btn || !btn.dataset.hours) return;
seg.querySelectorAll('button').forEach(b => b.classList.toggle('active', b === btn));
loadStallRollup(btn.dataset.hours);
});
}
function init() {
// Bound once, before the first load: the controls are static markup, so
// they never get rebuilt and never need re-binding. (Rebuilding a control
// you are typing into is how the providers page ended up reversing text.)
bindModelFilters();
bindModelsViews();
bindStallWindow();
loadModels();
loadStallRollup(24);
loadBlockedModels();
connectSSE();
}

View File

@@ -0,0 +1,165 @@
# Lift A.2: loops and stalling models, surfaced with the lever beside them
Status: planned -- brief ready; worktree created from origin/main 5aaf1e6 (after #103)
Date: 2026-09-26
This brief is the contract. Lift A (PR #102) is the base; do NOT read
`plans/no-progress-detection.md` (reference only).
## Goal
Meet the surfacing half of North Star #4 (`CLAUDE.md`): "Waste is surfaced in
the admin portal, and stopping it is one click." Lift A built the detection,
the alerts and the data. The portal does not yet show the evidence or put the
lever beside it:
- **Loops panel** (`admin/frontend/index.html#loops`, `loadLoops`) shows a
severity badge, the raw key `opencode-loop:<root id>` and `opened_at`.
Nothing an operator can act on.
- **Models page** has no per-model stall rollup, no Block button beside any
evidence, and no Blocked list. `blocked` exists only as a fourth value in
the Model Availability override dropdown.
The data is already there. On the live DB, 18 of 19 `watchdog_verdicts` rows
carry `model_id`, and all carry `calls_since_landed` and
`cost_since_landed_usd`, now that conversation identity arrives (fixed in
#102).
**Out of scope:** detection changes, recovery modes, alert channels other than
desktop, Lift B, and any change to how `blocked` is enforced (it already
excludes routing and pinned requests via `_admin_excluded_models`).
## Facts to build on (checked on origin/main 2026-09-26)
- `watchdog_verdicts`: `tick_id, session_id, session_root, agent, model_id,
provider, flagged, dup, top, top_what, landed, slow, coverage,
calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at`.
One row per judged session per tick. **`agent` holds the opencode session
TITLE**, not the agent slug. `cost_since_landed_usd` is cumulative, so it
grows tick by tick for the same session.
- `watchdog_alerts`: `dedup_key` (`opencode-loop:<root id>`), `state`,
`severity`, `flagged_ticks`, `opened_at`, `last_fired_at`, `resolved_at`.
- `admin_model_overrides`: `model_id, provider, availability, reason,
updated_at`. `POST /admin/api/models/{model_id:path}/{provider}/availability`
takes `{availability, reason}` (`_AvailabilityBody`, `src/admin.py` ~885);
`DELETE` on the same path clears the override (~2116).
- Existing endpoints: `GET /admin/api/watchdog/status`, `/loops` (~2834),
`/channels` GET/POST, `POST /test-alert`.
- `route_decisions.session_key` is `"c:" + opencode session id` and
`route_decisions.agent` is the agent slug when identity arrives.
- Models page cards: "Model Availability" (`models.html` ~226) and
"Per-model usage" (~310). JS unit harness: `tests/test_admin_js_units.py`
(marker-delimited pure functions, run under node).
## Components
### 1. Loops panel shows the evidence (`index.html#loops`)
- Extend `GET /admin/api/watchdog/loops` to return each OPEN alert joined to
its root's most recent FLAGGED verdict (any session in that root). Fields:
`dedup_key, root_id, session_id, title, agent_slug, model_id, provider,
severity, flagged_ticks, opened_at, top_what, dup, top, coverage,
calls_since_landed, cost_since_landed_usd, llm_second_opinion`.
`agent_slug` comes from the latest `route_decisions.agent` for
`"c:" + session_id` (NULL if none). Also return `resolved` (alerts resolved
in the last 24h, same fields, capped at 20).
- One row per open alert: severity badge, title (truncate ~40ch, full title in
`title=`), model, top repeated target, calls since last landed, $ since last
landed (`$0.00`, `n/a` when NULL), open for (relative time), and a **Block
model** button (component 3 flow) when `model_id` is known.
- Resolved rows below, dimmed. Session ids truncated with a tooltip.
- Empty state stays "No open alerts".
### 2. Per-model stall rollup (`models.html`, new card "Stalls by model")
- `GET /admin/api/watchdog/rollup?hours=N`, N in {1, 24, 168}. Per
`(model_id, provider)` with at least one flagged verdict in the window:
- `stalled_sessions` = `COUNT(DISTINCT session_root)` over flagged rows. Not
row counts: one row is written per tick.
- `stalled_usd` = per flagged `session_id`, take the MAX
`cost_since_landed_usd` in the window (it is cumulative), then sum per
model.
- `traffic_share` = the model's share of `route_decisions` in the same
window, its own column; never divide sessions by requests.
- `effective_availability` (override if any, else catalog).
- Window picker is radio-style (1h / 24h / 7d), default 24h.
- Each row: model, provider, stalled sessions, $ in stalled sessions, traffic
share, availability, and **Block model** (hidden when already blocked).
- Card sits ABOVE "Model Availability". Empty state: "No stalled sessions in
this window".
### 3. Block from the evidence, with a reason
- Block (from 1 or 2) opens an INLINE confirm row, not a browser dialog:
reason input prefilled `stalled <N> sessions in <window>` (from the Loops
panel: `looping session <title>`), editable, then Confirm / Cancel.
- Confirm posts `{availability: "blocked", reason}` to the existing endpoint,
then refreshes the rollup, the Blocked list and the availability table.
- The existing dropdown path keeps working; when it sets `blocked` it sends
reason `set from Models dropdown`.
### 4. Blocked list (`models.html`, card "Blocked models")
- Rows from `admin_model_overrides WHERE availability = 'blocked'`: model,
provider, reason, blocked since (`updated_at`), **Unblock**
(`DELETE .../availability`, inline confirm).
- Empty state: "No blocked models". Endpoint: reuse an existing models/overrides
endpoint if one returns `reason` and `updated_at`; otherwise add
`GET /admin/api/models/blocked`.
### 5. Alert text carries the evidence
- In `src/watchdog.py`, the trigger and escalate `AlertEvent.summary` becomes
`<title>: <calls> calls, $<cost> since last landed change on <model>.
<dashboard_base_url>#loops` (omit the `on <model>` / `$` parts when NULL).
Keep it one line; `notify-send` shows title and summary only.
## Todos (small; one commit each, by explicit path)
1. Loops endpoint join + `resolved` + endpoint tests (seeded tables: two
ticks for one session so the latest flagged verdict wins; a root with two
sessions; NULL model/cost).
2. Loops panel UI + pure formatting helpers (money, relative time, truncation)
in marker-delimited functions with node cases in `test_admin_js_units.py`.
3. Rollup endpoint + tests (distinct roots across ticks; MAX-then-SUM cost;
traffic share; window filter).
4. Stalls-by-model card + window radio + node cases for its helpers.
5. Inline Block confirm (shared by Loops and rollup) + reason on the dropdown
path + tests that the POST carries the reason.
6. Blocked list card + Unblock + tests.
7. Alert summary text in `watchdog.py` + a test asserting the summary.
8. Visual QA (below) and the done report.
## Visual QA (required; HTTP 200 is not QA)
- On 8081 (`scripts/sandbox.sh`) against a sqlite-backup copy of the live DB.
Seed the COPY with one open alert whose verdicts carry a model, calls and
cost, one resolved alert, and one blocked override with a reason. Never
write to the live DB.
- Screenshots of Home (Loops) and Models at 1400px and 800px wide, plus zoomed
crops of each new card. Check truncation, alignment, empty states, and that
Block -> Confirm -> Blocked list -> Unblock round-trips on screen.
- Save screenshots under the scratch/evidence dir, not the repo.
## Ground rules (lessons from Lift A)
- Worktree already exists: `/home/alee/Sources/6krrt-worktrees/no-progress-lift-a2`,
branch `feat/no-progress-lift-a2` from `origin/main` `5aaf1e6` (includes #103).
`/start-work` runs with `--worktree` set to it. Nothing in the main checkout.
- Dispatch to Sisyphus-Junior categories or `general`. NEVER to
`oh-my-claudecode:*` agents: they are pinned to Anthropic models this
opencode cannot reach, and return "completed" with zero work. A subagent
session with 0 tool calls did nothing.
- One commit per todo; no squashed "full implementation" commits.
- No test exemptions or skips for labelled cases. If a test cannot pass, stop
and report why.
- Lint: pinned `uvx ruff@0.16.9`, no NEW findings vs `origin/main` in touched
files (recipe in `plans/no-progress-lift-a.md` Ground rules). Paste the
gate's empty output in the done report.
- Full pytest: record the `origin/main` baseline first; the only known failure
is `test_incumbent_routing::test_debug_log_emits_incumbent_identity`.
- Port 8080 is production: no requests to it, not even GETs.
- Do not push, do not open a PR, do not remove the worktree. The done report
lists commits, tests, screenshots and operator steps (router restart).
- ASCII only; no middle dots. No prose paragraphs in the admin UI; mutually
exclusive choices are radio-style.
- Stop a worker whose context passes ~150k tokens and start a fresh one.

View File

@@ -33,7 +33,7 @@ from pathlib import Path
from typing import Any, List, Literal, Optional
from urllib.parse import urlparse
from fastapi import APIRouter, BackgroundTasks, HTTPException, Response
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Response
from fastapi.responses import FileResponse
from openai import OpenAI, OpenAIError
from pydantic import BaseModel, ConfigDict, ValidationError
@@ -1999,6 +1999,25 @@ def build_router(
return rows
return [r for r in rows if r.get("effective_availability") == "active"]
@router.get("/api/models/blocked")
def _admin_blocked_models() -> list:
"""List all models that have an admin override set to 'blocked'.
Returns model_id, provider, reason, and blocked_since (aliased from
updated_at). Empty list when no models are blocked.
"""
conn = _db_callable()
try:
rows = conn.execute(
"SELECT model_id, provider, reason, updated_at AS blocked_since "
"FROM admin_model_overrides "
"WHERE availability = 'blocked' "
"ORDER BY updated_at DESC"
).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
@router.get("/api/models/{model_id:path}/{provider}")
def admin_model_detail(model_id: str, provider: str) -> dict:
"""A single model's detail (same shape as one /api/models element)."""
@@ -2835,11 +2854,142 @@ def build_router(
def _watchdog_loops():
conn = _db_callable()
try:
rows = conn.execute(
"SELECT * FROM watchdog_alerts WHERE resolved_at IS NULL "
"ORDER BY opened_at DESC"
).fetchall()
return [dict(r) for r in rows]
rows = conn.execute("""
SELECT a.dedup_key, a.severity, a.flagged_ticks, a.opened_at,
a.last_fired_at,
v.session_id, v.session_root, v.agent AS title,
v.model_id, v.provider, v.top_what, v.dup, v.top,
v.landed, v.coverage, v.calls_since_landed,
v.cost_since_landed_usd, v.llm_second_opinion,
rd.agent AS agent_slug
FROM watchdog_alerts a
LEFT JOIN watchdog_verdicts v
ON a.dedup_key = 'opencode-loop:' || v.session_root
AND v.id = (
SELECT v2.id FROM watchdog_verdicts v2
WHERE a.dedup_key = 'opencode-loop:' || v2.session_root
AND v2.flagged = 1
ORDER BY v2.id DESC LIMIT 1
)
LEFT JOIN route_decisions rd
ON rd.session_key = 'c:' || v.session_id
AND rd.id = (
SELECT MAX(rd2.id) FROM route_decisions rd2
WHERE rd2.session_key = 'c:' || v.session_id
)
WHERE a.resolved_at IS NULL
ORDER BY a.opened_at DESC
""").fetchall()
alerts = [dict(r) for r in rows]
resolved_rows = conn.execute("""
SELECT a.dedup_key, a.severity, a.flagged_ticks, a.opened_at,
a.last_fired_at, a.resolved_at,
v.session_id, v.session_root, v.agent AS title,
v.model_id, v.provider, v.top_what, v.dup, v.top,
v.landed, v.coverage, v.calls_since_landed,
v.cost_since_landed_usd, v.llm_second_opinion,
rd.agent AS agent_slug
FROM watchdog_alerts a
LEFT JOIN watchdog_verdicts v
ON a.dedup_key = 'opencode-loop:' || v.session_root
AND v.id = (
SELECT v2.id FROM watchdog_verdicts v2
WHERE a.dedup_key = 'opencode-loop:' || v2.session_root
AND v2.flagged = 1
ORDER BY v2.id DESC LIMIT 1
)
LEFT JOIN route_decisions rd
ON rd.session_key = 'c:' || v.session_id
AND rd.id = (
SELECT MAX(rd2.id) FROM route_decisions rd2
WHERE rd2.session_key = 'c:' || v.session_id
)
WHERE a.resolved_at IS NOT NULL
AND julianday(a.resolved_at) > julianday('now', '-1 day')
ORDER BY a.resolved_at DESC
LIMIT 20
""").fetchall()
resolved = [dict(r) for r in resolved_rows]
return {"alerts": alerts, "resolved": resolved}
finally:
conn.close()
@router.get("/api/watchdog/rollup")
def _watchdog_rollup(hours: int = Query(24, ge=1)):
"""Per-model stall aggregates for the given window.
Returns stalled_sessions (COUNT(DISTINCT session_root)), stalled_usd
(per-session MAX of cumulative cost_since_landed_usd, then SUM per model),
traffic_share (route_decisions in same window), and effective_availability
(admin override first, else catalog default).
"""
if hours not in (1, 24, 168):
raise HTTPException(status_code=422, detail="hours must be 1, 24, or 168")
conn = _db_callable()
try:
# Stalled sessions: aggregate flagged verdicts to one row per
# session (taking MAX cumulative cost), then group by model.
# COUNT(DISTINCT session_root) counts stalled sessions; SUM(max_cost)
# sums each session's peak cost once.
stalled = conn.execute("""
SELECT v.model_id, v.provider,
COUNT(DISTINCT v.session_root) AS stalled_sessions,
COALESCE(SUM(v.max_cost), 0) AS stalled_usd
FROM (
SELECT session_id, session_root, model_id, provider,
MAX(cost_since_landed_usd) AS max_cost
FROM watchdog_verdicts
WHERE flagged = 1
AND julianday(created_at) > julianday('now', '-' || ? || ' hours')
AND model_id IS NOT NULL
GROUP BY session_id
) v
GROUP BY v.model_id, v.provider
ORDER BY stalled_sessions DESC
""", (hours,)).fetchall()
result = []
for row in stalled:
row = dict(row)
model_id = row["model_id"]
provider = row["provider"]
# Traffic share: count route_decisions for this model in same window
traffic_row = conn.execute("""
SELECT CAST(COUNT(*) AS REAL) / NULLIF(
(SELECT CAST(COUNT(*) AS REAL) FROM route_decisions
WHERE julianday(observed_at) > julianday('now', '-' || ? || ' hours')
AND selected_model IS NOT NULL), 0
) AS traffic_share
FROM route_decisions
WHERE selected_model = ?
AND selected_provider = ?
AND julianday(observed_at) > julianday('now', '-' || ? || ' hours')
""", (hours, model_id, provider, hours)).fetchone()
row["traffic_share"] = traffic_row["traffic_share"] if traffic_row else 0.0
# Effective availability: admin_model_overrides first, else catalog
override = conn.execute(
"SELECT availability FROM admin_model_overrides "
"WHERE model_id = ? AND provider = ?",
(model_id, provider),
).fetchone()
if override:
row["effective_availability"] = override["availability"]
else:
catalog = conn.execute(
"SELECT availability FROM models "
"WHERE model_id = ? AND provider = ?",
(model_id, provider),
).fetchone()
row["effective_availability"] = catalog["availability"] if catalog else "active"
result.append(row)
return result
finally:
conn.close()

View File

@@ -546,7 +546,7 @@ def tick(
tree_last_landed[root] = latest
calls_since_landed_map: dict[str, int] = {}
cost_since_landed_map: dict[str, float] = {}
cost_since_landed_map: dict[str, float | None] = {}
for sid, cs in all_calls.items():
root = session_root.get(sid, sid)
last_landed = tree_last_landed.get(root, 0.0)
@@ -566,7 +566,7 @@ def tick(
+ ",".join("?" for _ in fps) + ") AND observed_at > ?",
fps + [since_iso],
).fetchone()
cost_since_landed_map[sid] = row[0] or 0.0 if row else 0.0
cost_since_landed_map[sid] = row[0] if row and row[0] is not None else None
else:
cost_since_landed_map[sid] = 0.0
@@ -651,9 +651,15 @@ def tick(
_fire_alert(conn, dedup, "trigger", severity,
f"opencode loop: {agent}")
if notifier:
calls = calls_since_landed_map.get(sid, 0)
cost = cost_since_landed_map.get(sid)
model_id = session_attr.get(sid, (None, None, 0.0, 0))[0]
summary = (
f"Session {sid} flagged for looping. "
f"See {cfg.watchdog.dashboard_base_url}#loops"
f"opencode loop: {agent}: {calls} calls"
+ (f", ${cost:.2f}" if cost is not None else "")
+ " since last landed change"
+ (f" on {model_id}" if model_id else "")
+ f". {cfg.watchdog.dashboard_base_url}#loops"
)
event = AlertEvent(
dedup_key=dedup, severity=severity, state="trigger",
@@ -680,10 +686,16 @@ def tick(
_fire_alert(conn, dedup, "escalate", "critical",
f"opencode loop: {agent}")
if notifier:
calls = calls_since_landed_map.get(sid, 0)
cost = cost_since_landed_map.get(sid)
model_id = session_attr.get(sid, (None, None, 0.0, 0))[0]
summary = (
f"Session {sid} persisting after {flagged_ticks} "
f"ticks ({flagged_ticks * 5} min). "
f"See {cfg.watchdog.dashboard_base_url}#loops"
f"opencode loop: {agent}: {calls} calls"
+ (f", ${cost:.2f}" if cost is not None else "")
+ " since last landed change"
+ (f" on {model_id}" if model_id else "")
+ f" (persisting {flagged_ticks * 5} min). "
+ f"{cfg.watchdog.dashboard_base_url}#loops"
)
event = AlertEvent(
dedup_key=dedup, severity="critical",

View File

@@ -458,3 +458,143 @@ def test_merge_knob_rows_value_normalization(persisted, runtime, expect_drift):
");\n"
f"assert.strictEqual(rows['a.knob'].drift, {str(expect_drift).lower()});"
)
INDEX_HTML = ROOT / "admin" / "frontend" / "index.html"
LIFT_A2_HELPERS_BEGIN = "/* LIFT_A2_HELPERS:BEGIN */"
LIFT_A2_HELPERS_END = "/* LIFT_A2_HELPERS:END */"
def _assert_against_lift_a2_helpers(assertion_js: str) -> subprocess.CompletedProcess[str]:
"""Assert ``assertion_js`` against the real lift-a2 formatting helpers source."""
return _run_node(
"const assert = require('assert');\n"
+ _extract_between(INDEX_HTML, LIFT_A2_HELPERS_BEGIN, LIFT_A2_HELPERS_END)
+ "\n"
+ assertion_js
)
@skip_without_node
def test_format_money():
_assert_against_lift_a2_helpers(
"assert.strictEqual(formatMoney(1.5), '$1.50');\n"
"assert.strictEqual(formatMoney(0), '$0.00');\n"
"assert.strictEqual(formatMoney(null), 'n/a');\n"
"assert.strictEqual(formatMoney(undefined), 'n/a');"
)
@skip_without_node
def test_format_relative_time():
_assert_against_lift_a2_helpers(
"// just now (within 10s)\n"
"assert.strictEqual(formatRelativeTime(new Date().toISOString()), 'just now');\n"
"// a few minutes ago\n"
"const fiveMinAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();\n"
"assert.strictEqual(formatRelativeTime(fiveMinAgo), '5m ago');\n"
"// null/empty\n"
"assert.strictEqual(formatRelativeTime(null), '');\n"
"assert.strictEqual(formatRelativeTime(''), '');"
)
@skip_without_node
def test_truncate():
_assert_against_lift_a2_helpers(
"assert.strictEqual(truncate('hello world', 40), 'hello world');\n"
"assert.strictEqual(truncate('hello world', 5), 'hello...');\n"
"assert.strictEqual(truncate(null, 5), '');\n"
"assert.strictEqual(truncate(undefined, 5), '');"
)
MODELS_HTML = ROOT / "admin" / "frontend" / "models.html"
BLOCK_MODEL_BEGIN = "/* BLOCK_CONFIRM:BEGIN */"
BLOCK_MODEL_END = "/* BLOCK_CONFIRM:END */"
def _assert_against_block_model(assertion_js: str) -> subprocess.CompletedProcess[str]:
return _run_node(
"const assert = require('assert');\n"
+ _extract_between(MODELS_HTML, BLOCK_MODEL_BEGIN, BLOCK_MODEL_END)
+ "\n"
+ assertion_js
)
@skip_without_node
def test_block_model_exists():
_assert_against_block_model(
"assert.strictEqual(typeof blockModelConfirm, 'function');"
)
@skip_without_node
def test_block_button_uses_data_attrs_not_onclick_interpolation():
"""The Block button in both pages must carry data-model/data-provider/
data-reason attributes and wire via onclick=\"blockModelConfirm(event)\"
rather than interpolating runtime values into JS — interpolation broke
on titles containing a double quote."""
for path in (INDEX_HTML, MODELS_HTML):
text = path.read_text(encoding="utf-8")
assert 'onclick="blockModelConfirm(event)"' in text, (
f"{path.name}: Block button missing onclick handler"
)
assert "data-model=" in text, f"{path.name}: missing data-model attr"
assert "data-provider=" in text, f"{path.name}: missing data-provider attr"
assert "data-reason=" in text, f"{path.name}: missing data-reason attr"
@skip_without_node
def test_block_data_reason_escapes_double_quotes():
"""A looping-session title containing both a double and a single quote is
safe inside data-reason=\"...\": escapeHtml turns the double quote into
&quot; so it cannot terminate the attribute, and a bare single quote is
harmless inside a double-quoted HTML attribute."""
escape = re.search(r"function escapeHtml\(s\) \{.*?\n\}", INDEX_HTML.read_text(encoding="utf-8"), re.DOTALL)
assert escape, "index.html: escapeHtml source not found"
js = (
"const assert = require('assert');\n"
+ escape.group(0)
+ "\n"
"const title = 'he said \"hello\\'s\"';\n"
"const dataReason = 'looping session ' + title;\n"
"const html = 'data-reason=\"' + escapeHtml(dataReason) + '\"';\n"
"assert.ok(html.includes('&quot;'));\n"
"assert.ok(!html.includes('\"hello'));\n"
"assert.strictEqual(escapeHtml(dataReason), "
" 'looping session he said &quot;hello\\'s&quot;');\n"
)
_run_node(js)
@skip_without_node
def test_block_button_onclick_wires_to_blockModelConfirm():
"""The Block button carries onclick=\"blockModelConfirm(event)\" so the
click actually invokes the handler. Without this the data-* attributes
are inert and clicking Block does nothing."""
for path in (INDEX_HTML, MODELS_HTML):
text = path.read_text(encoding="utf-8")
assert 'onclick="blockModelConfirm(event)"' in text, (
f"{path.name}: Block button missing onclick handler"
)
@skip_without_node
def test_blockModelConfirm_reads_from_event_currentTarget_dataset():
"""blockModelConfirm(event) must read model/provider/reason from
event.currentTarget.dataset — the old hybrid signature that expected
positional args is broken because onclick passes a single Event."""
js = (
"const assert = require('assert');\n"
+ _extract_between(MODELS_HTML, BLOCK_MODEL_BEGIN, BLOCK_MODEL_END)
+ "\n"
"assert.strictEqual(typeof blockModelConfirm, 'function');\n"
"const src = blockModelConfirm.toString();\n"
"assert.ok(src.includes('event.currentTarget'), 'must use event.currentTarget');\n"
"assert.ok(src.includes('dataset.model'), 'must read dataset.model');\n"
"assert.ok(src.includes('dataset.provider'), 'must read dataset.provider');\n"
"assert.ok(src.includes('dataset.reason'), 'must read dataset.reason');\n"
)
_run_node(js)

View File

@@ -492,3 +492,80 @@ def test_set_availability_returns_warning_when_override_drops_ceiling(
for w in data["warnings"]
)
# --- blocked list ------------------------------------------------------------
def test_blocked_list_empty(seeded_client):
"""GET /admin/api/models/blocked with no blocked overrides -> 200 + []."""
resp = seeded_client.get("/admin/api/models/blocked")
assert resp.status_code == 200
assert resp.json() == []
def test_blocked_list_does_not_shadow_model_detail(seeded_client):
"""The literal /blocked segment must not be consumed by the
{model_id:path}/{provider} converter: a real model id still resolves."""
resp = seeded_client.get("/admin/api/models/cheap/neuralwatt")
assert resp.status_code == 200
assert resp.json()["model_id"] == "cheap"
def test_blocked_list_with_data(tmp_path, monkeypatch):
"""Block a model via an admin override, then the blocked list shows it."""
conn = _make_db(tmp_path)
_seed_models(conn, ("test-model",))
conn.execute(
"INSERT INTO admin_model_overrides (model_id, provider, availability, "
"reason, updated_at) "
"VALUES ('test-model', 'neuralwatt', 'blocked', 'manual block', "
" datetime('now'))"
)
conn.commit()
conn.close()
monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db"))
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
with TestClient(dispatcher.app) as client:
resp = client.get("/admin/api/models/blocked")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 1
assert data[0]["model_id"] == "test-model"
assert data[0]["provider"] == "neuralwatt"
assert data[0]["reason"] == "manual block"
assert "blocked_since" in data[0]
def test_block_reason_stored(seeded_client):
"""POST to availability with a reason stores the reason on the block."""
resp = seeded_client.post(
"/admin/api/models/cheap/neuralwatt/availability",
json={"availability": "blocked", "reason": "looping session test-agent"},
)
assert resp.status_code == 200
blocked = seeded_client.get("/admin/api/models/blocked").json()
assert len(blocked) == 1
assert blocked[0]["model_id"] == "cheap"
assert blocked[0]["provider"] == "neuralwatt"
assert blocked[0]["reason"] == "looping session test-agent"
def test_block_reason_set_from_dropdown(seeded_client):
"""Block via the dropdown carries the 'set from Models dropdown' reason."""
resp = seeded_client.post(
"/admin/api/models/cheap/neuralwatt/availability",
json={"availability": "blocked", "reason": "set from Models dropdown"},
)
assert resp.status_code == 200
blocked = seeded_client.get("/admin/api/models/blocked").json()
assert len(blocked) == 1
assert blocked[0]["model_id"] == "cheap"
assert blocked[0]["reason"] == "set from Models dropdown"

View File

@@ -1,12 +1,14 @@
from __future__ import annotations
import sqlite3
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
from starlette.testclient import TestClient
import dispatcher
from admin import ensure_admin_tables
from watchdog_store import ensure_watchdog_tables
ROOT = Path(__file__).resolve().parent.parent
@@ -39,7 +41,149 @@ def test_watchdog_status_empty_db(client):
def test_watchdog_loops_empty_db(client):
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
assert resp.json() == []
data = resp.json()
assert "alerts" in data
assert "resolved" in data
assert data["alerts"] == []
assert data["resolved"] == []
def _seed_verdict(conn, tick_id, session_id, session_root, agent, model_id,
flagged, calls, cost, created_at):
conn.execute(
"""INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent,
model_id, provider, flagged, dup, top, top_what, landed, slow, coverage,
calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at)
VALUES (?, ?, ?, ?, ?, 'neuralwatt', ?, 0.9, 1, 'code', 1, 0, 0.8,
?, ?, 'no', ?)""",
(tick_id, session_id, session_root, agent, model_id,
flagged, calls, cost, created_at),
)
def test_watchdog_loops_with_verdict_evidence(client):
"""Alert joined with latest FLAGGED verdict evidence."""
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA_SQL)
conn.execute(
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
)
conn.execute(
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
opened_at, last_fired_at, resolved_at)
VALUES ('opencode-loop:root-abc', 'open', 'warning', 1,
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
)
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-1", "model-a",
0, 5, 0.5, "2026-09-26T10:00:00+00:00")
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-2", "model-b",
1, 10, 1.5, "2026-09-26T10:10:00+00:00")
conn.commit()
conn.close()
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
data = resp.json()
assert len(data["alerts"]) == 1
alert = data["alerts"][0]
assert alert["title"] == "agent-2"
assert alert["model_id"] == "model-b"
assert alert["provider"] == "neuralwatt"
assert alert["session_root"] == "root-abc"
assert alert["calls_since_landed"] == 10
assert alert["cost_since_landed_usd"] == 1.5
assert alert["agent_slug"] is None
def test_watchdog_loops_alert_no_matching_verdict(client):
"""Alert with no matching verdict returns NULL evidence (no error)."""
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
data = resp.json()
assert "alerts" in data
assert "resolved" in data
def test_watchdog_loops_resolved_last_24h(client):
"""Resolved alert within 24h appears in resolved list."""
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA_SQL)
resolved_time = (datetime.now(timezone.utc) - timedelta(hours=6)).isoformat()
conn.execute(
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
opened_at, last_fired_at, resolved_at)
VALUES ('opencode-loop:root-xyz', 'resolved', 'critical', 3,
'2026-09-26T04:00:00+00:00', '2026-09-26T08:00:00+00:00', ?)""",
(resolved_time,),
)
conn.commit()
conn.close()
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
data = resp.json()
assert len(data["resolved"]) == 1
assert data["resolved"][0]["dedup_key"] == "opencode-loop:root-xyz"
def test_watchdog_loops_resolved_older_than_24h(client):
"""Resolved alert older than 24h does NOT appear in resolved list."""
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA_SQL)
resolved_time = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat()
conn.execute(
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
opened_at, last_fired_at, resolved_at)
VALUES ('opencode-loop:root-old', 'resolved', 'critical', 3,
'2026-09-23T04:00:00+00:00', '2026-09-23T08:00:00+00:00', ?)""",
(resolved_time,),
)
conn.commit()
conn.close()
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
data = resp.json()
assert len(data["resolved"]) == 0
def test_watchdog_loops_agent_slug_from_route_decisions(client):
"""Agent slug comes from route_decisions.agent, not watchdog_verdicts."""
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA_SQL)
conn.execute(
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
)
conn.execute(
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
opened_at, last_fired_at, resolved_at)
VALUES ('opencode-loop:root-ads', 'open', 'warning', 1,
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
)
_seed_verdict(conn, 1, "sess-ads", "root-ads", "test-agent", "model-c",
1, 3, 0.25, "2026-09-26T10:00:00+00:00")
conn.execute(
"""INSERT INTO route_decisions (observed_at, kind, task_category, task_tier,
selected_model, selected_provider, session_key, agent, streamed)
VALUES ('2026-09-26T10:01:00+00:00', 'dispatch', 'coding_refactor', 2,
'model-c', 'neuralwatt', 'c:sess-ads', 'opencode-agent', 0)"""
)
conn.commit()
conn.close()
resp = client.get("/admin/api/watchdog/loops")
assert resp.status_code == 200
data = resp.json()
assert len(data["alerts"]) == 1
assert data["alerts"][0]["agent_slug"] == "opencode-agent"
assert data["alerts"][0]["title"] == "test-agent"
def test_watchdog_channels_empty_db(client):
@@ -129,3 +273,95 @@ def test_watchdog_status_after_tick(tmp_path, monkeypatch):
data = resp.json()
assert data["last_tick"] is not None
assert data["last_tick"]["sessions_seen"] == 5
def test_watchdog_rollup_empty(client):
"""Empty watchdog_verdicts returns []."""
resp = client.get("/admin/api/watchdog/rollup?hours=24")
assert resp.status_code == 200
assert resp.json() == []
def test_watchdog_rollup_invalid_hours(client):
"""hours=99 returns 422."""
resp = client.get("/admin/api/watchdog/rollup?hours=99")
assert resp.status_code == 422
def test_watchdog_rollup_with_data(client):
"""2 flagged verdicts for same session (costs 1.0 and 3.0), 1 flagged for
another session same model. Expect stalled_sessions=2, stalled_usd=4.0
(MAX 3.0 + MAX 1.0)."""
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA_SQL)
ensure_admin_tables(conn)
# Seed a model row
conn.execute(
"INSERT INTO models (model_id, provider, base_model_id, display_name, tier, "
"context_window, effective_context_window, max_output_tokens, "
"cost_per_1m_prompt, cost_per_1m_completion, access_level, latency_class, "
"availability, last_updated) "
"VALUES ('model-a', 'neuralwatt', 'model-a', 'Model A', 2, "
"128000, 128000, 8192, 0.5, 1.5, 'public', 'interactive', "
"'active', datetime('now'))"
)
# Need a tick for the FK
conn.execute(
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
"VALUES (datetime('now'), 5, 'flagged')"
)
# First session (root-1): two verdicts, costs 1.0 and 3.0
now = datetime.now(timezone.utc)
conn.execute(
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
"1, 0.9, 1, 'code', 1, 0, 0.8, 5, 1.0, 'no', ?)",
(now.isoformat(),)
)
conn.execute(
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
"1, 0.9, 1, 'code', 1, 0, 0.8, 10, 3.0, 'no', ?)",
(now.isoformat(),)
)
# Second session (root-2): one verdict, cost 1.0
conn.execute(
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
"VALUES (1, 'sess-2', 'root-2', 'agent', 'model-a', 'neuralwatt', "
"1, 0.9, 1, 'code', 1, 0, 0.8, 3, 1.0, 'no', ?)",
(now.isoformat(),)
)
# Seed a route_decision for traffic_share
conn.execute(
"INSERT INTO route_decisions (observed_at, kind, task_category, task_tier, "
"selected_model, selected_provider, session_key, agent, streamed) "
"VALUES (?, 'dispatch', 'coding_refactor', 2, "
"'model-a', 'neuralwatt', 'c:sess-1a', 'opencode-agent', 0)",
(now.isoformat(),)
)
conn.commit()
conn.close()
resp = client.get("/admin/api/watchdog/rollup?hours=24")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 1
assert data[0]["model_id"] == "model-a"
assert data[0]["provider"] == "neuralwatt"
assert data[0]["stalled_sessions"] == 2 # root-1 and root-2
assert data[0]["stalled_usd"] == 4.0 # MAX(3.0, 1.0) + 1.0
assert data[0]["traffic_share"] == 1.0 # 1/1 route_decisions
assert data[0]["effective_availability"] == "active"

View File

@@ -1198,3 +1198,191 @@ class TestMainExitCode:
rc = watchdog.main(["--once", "--config", "unused.yaml"])
assert fake_tick.called
assert rc == 0
class TestAlertSummaryEvidence:
"""Trigger/escalate alert summaries carry calls, cost, and model evidence.
The session loops (60 duplicate reads after a landed change), so the
summary must report calls since the landed change, its cost, and the
attributed model — and omit the 'on <model>' segment when no model is
attributed.
"""
def _run_flagged_tick(self, conn, path, attribution, cost_after_landed):
"""Run one tick against a loop-flagged worker session; return events."""
cfg = _make_cfg()
cfg.database = MagicMock()
cfg.database.path = path
cfg.database.base_url = "http://localhost:4096"
cfg.watchdog.dashboard_base_url = "http://dash/"
conn.execute("""
CREATE TABLE IF NOT EXISTS route_decisions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_key TEXT,
selected_model TEXT,
selected_provider TEXT,
est_cost_usd REAL DEFAULT 0,
observed_at TEXT
)
""")
conn.commit()
now = datetime.now(timezone.utc)
landed_ts = now.timestamp() * 1000 # ms
# Worker: 60 identical reads (no landed) -> flags; each is after the
# tree's last landed time so calls_since_landed is 60.
worker_msgs = [{
"id": f"wm{i}", "parts": [{
"type": "tool", "tool": "read",
"state": {
"input": {"filePath": "/x.py"},
"metadata": {},
"time": {"start": landed_ts + 1 + i},
},
}],
"info": {"time": {"created": landed_ts + 1 + i}},
} for i in range(60)]
# Sibling: one landed edit at landed_ts -> tree_last_landed is set.
sibling_msgs = [{
"id": "sm0", "parts": [{
"type": "tool", "tool": "edit",
"state": {
"input": {"filePath": "/y.py"},
"metadata": {"diff": "patch"},
"time": {"start": landed_ts},
},
}],
"info": {"time": {"created": landed_ts}},
}]
# Parent: one benign call so it shows as active (not flagged).
parent_msgs = [{
"id": "pm0", "parts": [{
"type": "tool", "tool": "read",
"state": {
"input": {"filePath": "/z.py"},
"metadata": {},
"time": {"start": landed_ts + 200},
},
}],
"info": {"time": {"created": landed_ts + 200}},
}]
sessions = [
{"id": "ses_p", "title": "parent",
"parentID": None,
"time": {"created": landed_ts},
"info": {"time": {"created": landed_ts}}},
{"id": "ses_worker", "title": "worker",
"parentID": "ses_p",
"time": {"created": landed_ts},
"info": {"time": {"created": landed_ts}}},
{"id": "ses_sibling", "title": "sibling",
"parentID": "ses_p",
"time": {"created": landed_ts},
"info": {"time": {"created": landed_ts}}},
]
if cost_after_landed is not None:
conn.execute(
"INSERT INTO route_decisions (session_key, selected_model, "
"selected_provider, est_cost_usd, observed_at) "
"VALUES (?,?,?,?,?)",
("c:ses_worker", "gpt-4o", "neuralwatt", cost_after_landed,
datetime.fromtimestamp((landed_ts + 1) / 1000,
tz=timezone.utc).isoformat()),
)
conn.commit()
events = []
class _Recorder:
def deliver(self, event):
events.append(event)
def fake_http(url, **kw):
if "message" in url:
if "ses_worker" in url:
return worker_msgs
if "ses_sibling" in url:
return sibling_msgs
if "ses_p" in url:
return parent_msgs
return []
return sessions
with patch("watchdog._read_rc_servers",
return_value=["http://localhost:4096"]), \
patch("watchdog._fetch_sessions", return_value=sessions), \
patch("watchdog._attribution", return_value=attribution):
tick(cfg, conn, notifier=_Recorder(), http_get=fake_http)
return events
def _worker_event(self, events, state):
"""Return the event for the worker session of the given state."""
for e in events:
if e.state == state and e.title == "opencode loop: worker":
return e
return None
def test_alert_summary_trigger_includes_calls_cost_model(self):
"""Trigger summary lists calls, cost, and model since last landed change."""
conn, path = _make_conns()
try:
events = self._run_flagged_tick(
conn, path,
attribution=("gpt-4o", "neuralwatt", 0.5, 10),
cost_after_landed=1.25,
)
trigger = self._worker_event(events, "trigger")
assert trigger, "Expected a worker trigger event"
summary = trigger.summary
assert "60 calls" in summary, f"calls missing: {summary}"
assert "$1.25" in summary, f"cost missing: {summary}"
assert "on gpt-4o" in summary, f"model missing: {summary}"
assert "#loops" in summary, f"loops link missing: {summary}"
finally:
_close_conn(conn, path)
def test_alert_summary_trigger_omits_model_when_null(self):
"""No 'on <model>' segment when model_id is NULL."""
conn, path = _make_conns()
try:
events = self._run_flagged_tick(
conn, path,
attribution=(None, None, 0.0, 0),
cost_after_landed=1.25,
)
trigger = self._worker_event(events, "trigger")
assert trigger, "Expected a worker trigger event"
summary = trigger.summary
assert "60 calls" in summary, f"calls missing: {summary}"
assert "on " not in summary, f"'on <model>' should be omitted: {summary}"
finally:
_close_conn(conn, path)
def test_alert_summary_escalate_includes_calls_cost_model(self):
"""Escalate summary (3rd tick) carries the same calls/cost/model evidence."""
conn, path = _make_conns()
try:
events = []
for _ in range(3):
events += self._run_flagged_tick(
conn, path,
attribution=("gpt-4o", "neuralwatt", 0.5, 10),
cost_after_landed=1.25,
)
escalate = self._worker_event(events, "escalate")
assert escalate, "Expected a worker escalate event"
summary = escalate.summary
assert "60 calls" in summary, f"calls missing: {summary}"
assert "$1.25" in summary, f"cost missing: {summary}"
assert "on gpt-4o" in summary, f"model missing: {summary}"
assert "#loops" in summary, f"loops link missing: {summary}"
finally:
_close_conn(conn, path)