feat(admin): Lift A.2, surface loops and stalling models with one-click block #104
@@ -1336,28 +1336,206 @@ function hashCode(str) {
|
||||
LOOPS (open watchdog alerts)
|
||||
═══════════════════════════════════════ */
|
||||
|
||||
/* LIFT_A2_HELPERS:BEGIN */
|
||||
function formatMoney(usd) {
|
||||
if (usd == null) return 'n/a';
|
||||
return '$' + Number(usd).toFixed(2);
|
||||
}
|
||||
|
||||
function formatRelativeTime(iso8601) {
|
||||
if (!iso8601) return '';
|
||||
const now = new Date();
|
||||
const date = new Date(iso8601);
|
||||
const diffMs = now - date;
|
||||
const diffSeconds = Math.floor(diffMs / 1000);
|
||||
if (diffSeconds < 10) return 'just now';
|
||||
if (diffSeconds < 60) return Math.floor(diffSeconds) + 's ago';
|
||||
const diffMinutes = Math.floor(diffSeconds / 60);
|
||||
if (diffMinutes < 60) return diffMinutes + 'm ago';
|
||||
const diffHours = Math.floor(diffMinutes / 60);
|
||||
if (diffHours < 24) return diffHours + 'h ago';
|
||||
const diffDays = Math.floor(diffHours / 24);
|
||||
if (diffDays < 30) return diffDays + 'd ago';
|
||||
return diffDays + 'd ago';
|
||||
}
|
||||
|
||||
function truncate(text, maxLen) {
|
||||
if (!text) return '';
|
||||
if (text.length <= maxLen) return text;
|
||||
return text.substring(0, maxLen) + '...';
|
||||
}
|
||||
/* LIFT_A2_HELPERS:END */
|
||||
|
||||
async function loadLoops() {
|
||||
const rows = await apiFetch(`${API}api/watchdog/loops`);
|
||||
const data = await apiFetch(`${API}api/watchdog/loops`);
|
||||
const body = document.getElementById('loops-body');
|
||||
const countEl = document.getElementById('loops-count');
|
||||
if (!rows || !rows.length) {
|
||||
const alerts = data && data.alerts ? data.alerts : [];
|
||||
const resolved = data && data.resolved ? data.resolved : [];
|
||||
countEl.textContent = `${alerts.length} open`;
|
||||
|
||||
if (alerts.length === 0 && resolved.length === 0) {
|
||||
body.innerHTML = '<div class="text-muted small">No open alerts</div>';
|
||||
countEl.textContent = '0 open';
|
||||
return;
|
||||
}
|
||||
countEl.textContent = `${rows.length} open`;
|
||||
body.innerHTML = rows.map(r => {
|
||||
let html = '<div class="table-responsive"><table class="table table-vcenter card-table"><thead><tr>'
|
||||
+ '<th>Severity</th><th>Session</th><th>Model</th><th>Top target</th>'
|
||||
+ '<th class="num">Calls since landed</th><th class="num">$ since landed</th><th class="num">Open for</th><th></th>'
|
||||
+ '</tr></thead><tbody>';
|
||||
alerts.forEach(r => {
|
||||
const sev = r.severity === 'critical' ? 'bg-danger'
|
||||
: r.severity === 'warning' ? 'bg-warning'
|
||||
: 'bg-info';
|
||||
return `<div class="d-flex align-items-center gap-2 py-1" style="border-bottom:1px solid rgba(255,255,255,.05)">
|
||||
<span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span>
|
||||
<span class="small" style="flex:1">${escapeHtml(r.dedup_key)}</span>
|
||||
<span class="text-muted" style="font-size:.66rem">${escapeHtml(r.opened_at || '')}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
const title = escapeHtml(truncate(r.title || r.dedup_key || '', 40));
|
||||
const fullTitle = escapeHtml(r.title || r.dedup_key || '');
|
||||
const model = r.model_id ? escapeHtml(r.model_id) : '<span class="text-muted">unknown</span>';
|
||||
const topWhat = r.top_what ? escapeHtml(r.top_what) : '<span class="text-muted">—</span>';
|
||||
const calls = r.calls_since_landed != null ? escapeHtml(String(r.calls_since_landed)) : '?';
|
||||
const cost = formatMoney(r.cost_since_landed_usd);
|
||||
const relTime = formatRelativeTime(r.opened_at);
|
||||
const dataModel = r.model_id || '';
|
||||
const dataProvider = r.provider || '';
|
||||
const dataReason = 'looping session ' + (r.title || r.dedup_key || '');
|
||||
const blockBtn = `<button class="btn btn-sm btn-outline-danger" style="font-size:.62rem;padding:1px 6px" data-model="${escapeHtml(dataModel)}" data-provider="${escapeHtml(dataProvider)}" data-reason="${escapeHtml(dataReason)}" onclick="blockModelConfirm(event)">Block</button>`;
|
||||
html += '<tr>'
|
||||
+ `<td><span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span></td>`
|
||||
+ `<td><span class="small text-truncate" style="max-width:200px" title="${fullTitle}">${title}</span></td>`
|
||||
+ `<td>${model}</td>`
|
||||
+ `<td class="text-muted" style="font-size:.65rem">${topWhat}</td>`
|
||||
+ `<td class="num">${calls}</td>`
|
||||
+ `<td class="num">${cost}</td>`
|
||||
+ `<td class="num">${relTime}</td>`
|
||||
+ `<td class="text-end">${blockBtn}</td>`
|
||||
+ '</tr>';
|
||||
});
|
||||
if (resolved.length > 0) {
|
||||
resolved.forEach(r => {
|
||||
const sev = r.severity === 'critical' ? 'bg-danger'
|
||||
: r.severity === 'warning' ? 'bg-warning'
|
||||
: 'bg-info';
|
||||
const title = escapeHtml(truncate(r.title || r.dedup_key || '', 40));
|
||||
const fullTitle = escapeHtml(r.title || r.dedup_key || '');
|
||||
const model = r.model_id ? escapeHtml(r.model_id) : '<span class="text-muted">unknown</span>';
|
||||
const topWhat = r.top_what ? escapeHtml(r.top_what) : '<span class="text-muted">—</span>';
|
||||
const calls = r.calls_since_landed != null ? escapeHtml(String(r.calls_since_landed)) : '?';
|
||||
const cost = formatMoney(r.cost_since_landed_usd);
|
||||
const relTime = formatRelativeTime(r.opened_at);
|
||||
html += '<tr class="opacity-50">'
|
||||
+ `<td><span class="badge ${sev}" style="font-size:.62rem">${escapeHtml(r.severity)}</span></td>`
|
||||
+ `<td><span class="small text-truncate" style="max-width:200px" title="${fullTitle}">${title}</span></td>`
|
||||
+ `<td>${model}</td>`
|
||||
+ `<td class="text-muted" style="font-size:.65rem">${topWhat}</td>`
|
||||
+ `<td class="num">${calls}</td>`
|
||||
+ `<td class="num">${cost}</td>`
|
||||
+ `<td class="num">${relTime}</td>`
|
||||
+ '<td></td>'
|
||||
+ '</tr>';
|
||||
});
|
||||
}
|
||||
html += '</tbody></table></div>';
|
||||
body.innerHTML = html;
|
||||
}
|
||||
|
||||
/* BLOCK_CONFIRM:BEGIN */
|
||||
function blockModelConfirm(event) {
|
||||
const btn = event.currentTarget;
|
||||
const model = btn.dataset.model;
|
||||
const prov = btn.dataset.provider;
|
||||
const reason = btn.dataset.reason;
|
||||
if (!model) return;
|
||||
const row = btn.closest('div.d-flex') || btn.closest('tr');
|
||||
if (!row) return;
|
||||
const existing = row.nextElementSibling;
|
||||
if (existing && existing.classList.contains('inline-confirm-row')) {
|
||||
existing.remove();
|
||||
return;
|
||||
}
|
||||
const confirmRow = document.createElement('div');
|
||||
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
|
||||
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
|
||||
confirmRow.innerHTML = '<input type="text" class="form-control form-control-sm" style="font-size:.72rem;flex:1;min-width:120px" value="' + escapeHtml(reason) + '">'
|
||||
+ '<button class="btn btn-sm btn-danger" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
|
||||
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
|
||||
|
||||
const cancelBtn = confirmRow.querySelector('button:last-child');
|
||||
cancelBtn.onclick = () => confirmRow.remove();
|
||||
|
||||
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
|
||||
const input = confirmRow.querySelector('input');
|
||||
confirmBtn.onclick = async () => {
|
||||
const reason = input.value;
|
||||
confirmBtn.disabled = true;
|
||||
const resp = await apiFetch(
|
||||
`${API}api/models/${encodeURIComponent(model)}/${encodeURIComponent(prov)}/availability`,
|
||||
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: 'blocked', reason: reason }) }
|
||||
);
|
||||
if (resp) {
|
||||
toast('Model blocked', 'success');
|
||||
confirmRow.remove();
|
||||
// Refresh dependent views
|
||||
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
|
||||
if (typeof loadBlockedModels === 'function') loadBlockedModels();
|
||||
if (typeof loadLoops === 'function') loadLoops();
|
||||
if (typeof loadModels === 'function') loadModels();
|
||||
} else {
|
||||
confirmBtn.disabled = false;
|
||||
confirmRow.querySelector('.inline-error')?.remove();
|
||||
const err = document.createElement('div');
|
||||
err.className = 'text-danger small inline-error';
|
||||
err.textContent = 'Block failed';
|
||||
confirmRow.appendChild(err);
|
||||
}
|
||||
};
|
||||
|
||||
row.after(confirmRow);
|
||||
}
|
||||
|
||||
function unblockModelConfirm(modelId, provider) {
|
||||
const btn = event.target;
|
||||
const row = btn.closest('div.d-flex') || btn.closest('tr');
|
||||
if (!row) return;
|
||||
const existing = row.nextElementSibling;
|
||||
if (existing && existing.classList.contains('inline-confirm-row')) {
|
||||
existing.remove();
|
||||
return;
|
||||
}
|
||||
const confirmRow = document.createElement('div');
|
||||
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
|
||||
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
|
||||
confirmRow.innerHTML = '<span class="small text-danger">Unblock this model?</span>'
|
||||
+ '<button class="btn btn-sm btn-warning" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
|
||||
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
|
||||
|
||||
const cancelBtn = confirmRow.querySelector('button:last-child');
|
||||
cancelBtn.onclick = () => confirmRow.remove();
|
||||
|
||||
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
|
||||
confirmBtn.onclick = async () => {
|
||||
confirmBtn.disabled = true;
|
||||
const resp = await apiFetch(
|
||||
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
|
||||
{ method:'DELETE' }
|
||||
);
|
||||
if (resp) {
|
||||
toast('Model unblocked', 'success');
|
||||
confirmRow.remove();
|
||||
if (typeof loadBlockedModels === 'function') loadBlockedModels();
|
||||
if (typeof loadModels === 'function') loadModels();
|
||||
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
|
||||
} else {
|
||||
confirmBtn.disabled = false;
|
||||
confirmRow.querySelector('.inline-error')?.remove();
|
||||
const err = document.createElement('div');
|
||||
err.className = 'text-danger small inline-error';
|
||||
err.textContent = 'Unblock failed';
|
||||
confirmRow.appendChild(err);
|
||||
}
|
||||
};
|
||||
|
||||
row.after(confirmRow);
|
||||
}
|
||||
/* BLOCK_CONFIRM:END */
|
||||
|
||||
/* ═══════════════════════════════════════
|
||||
INIT
|
||||
═══════════════════════════════════════ */
|
||||
|
||||
@@ -124,7 +124,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
#model-tbody .model-override{height:24px;padding:.05rem .35rem;font-size:.7rem;line-height:1.2}
|
||||
/* Numeric columns align right and share the tabular figures the rest of the
|
||||
portal uses, so magnitudes line up down the column. */
|
||||
#model-tbody td.num, thead th.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}
|
||||
tbody td.num, thead th.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}
|
||||
.prof-n{color:var(--tblr-secondary);opacity:.6;font-size:.62rem;margin-left:1px}
|
||||
.cost-free{color:#4ade80}
|
||||
/* Capability chips reuse the decisions page's flag-chip language so the two
|
||||
@@ -221,6 +221,48 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
</div>
|
||||
<div class="page-body">
|
||||
<div class="container-xl">
|
||||
<div class="row row-cards mb-3" id="view-stalls">
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><line x1="3" y1="9" x2="21" y2="9"/><line x1="9" y1="21" x2="9" y2="9"/></svg>
|
||||
<h3 class="card-title mb-0">Stalls by model</h3>
|
||||
</div>
|
||||
<div class="card-subtitle text-muted ms-2">Flagged looping sessions per model</div>
|
||||
<div class="ms-auto d-flex align-items-center gap-2">
|
||||
<div class="btn-seg" id="stall-window">
|
||||
<button type="button" data-hours="1">1h</button>
|
||||
<button type="button" data-hours="24" class="active">24h</button>
|
||||
<button type="button" data-hours="168">7d</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body" style="padding:.6rem .8rem">
|
||||
<div id="stalls-body">
|
||||
<div class="text-muted small">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row row-cards mb-3" id="view-blocked">
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="4.93" y1="4.93" x2="19.07" y2="19.07"/></svg>
|
||||
<h3 class="card-title mb-0">Blocked models</h3>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body" style="padding:.6rem .8rem">
|
||||
<div id="blocked-models-body">
|
||||
<div class="text-muted small">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row row-cards" id="view-available">
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
@@ -354,7 +396,7 @@ header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="toast class="toast align-items-center text-bg-info border-0 position-fixed bottom-0 end-0 p-3" role="alert" aria-live="assertive" aria-atomic="true"></div>
|
||||
<div id="toast" class="toast align-items-center text-bg-info border-0 position-fixed bottom-0 end-0 p-3" role="alert" aria-live="assertive" aria-atomic="true"></div>
|
||||
|
||||
<script>
|
||||
/* ═══════════════════════════════════════════════════
|
||||
@@ -619,9 +661,13 @@ function renderModels(models) {
|
||||
|
||||
async function updateAvailability(modelId, provider, avail) {
|
||||
// Don't post if already that value via dropdown
|
||||
const body = { availability: avail };
|
||||
if (avail === 'blocked') {
|
||||
body.reason = 'set from Models dropdown';
|
||||
}
|
||||
const resp = await apiFetch(
|
||||
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
|
||||
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: avail }) }
|
||||
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(body) }
|
||||
);
|
||||
const tr = document.querySelector(`tr[data-model="${modelId}"][data-provider="${provider}"]`);
|
||||
const warningDiv = tr ? tr.querySelector('.inline-warning') : null;
|
||||
@@ -823,17 +869,216 @@ function escapeHtml(s) {
|
||||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
||||
}
|
||||
|
||||
/* LOAD_STALL_ROLLUP:BEGIN */
|
||||
async function loadStallRollup(windowHours) {
|
||||
const body = document.getElementById('stalls-body');
|
||||
if (!body) return;
|
||||
const data = await apiFetch(`${API}api/watchdog/rollup?hours=${windowHours}`);
|
||||
if (!data || !data.length) {
|
||||
body.innerHTML = '<div class="text-muted small">No stalled sessions in this window</div>';
|
||||
return;
|
||||
}
|
||||
body.innerHTML = '<div class="table-responsive"><table class="table table-vcenter table-hover card-table"><thead><tr>'
|
||||
+ '<th>Model</th><th>Provider</th><th class="num">Stalled</th><th class="num">$ Stalled</th><th class="num">Traffic</th><th>Availability</th><th></th>'
|
||||
+ '</tr></thead><tbody>'
|
||||
+ data.map(r => {
|
||||
const model = escapeHtml(r.model_id || '');
|
||||
const provider = escapeHtml(r.provider || '');
|
||||
const stalled = r.stalled_sessions != null ? r.stalled_sessions : 0;
|
||||
const cost = r.stalled_usd != null ? '$' + Number(r.stalled_usd).toFixed(2) : 'n/a';
|
||||
const share = r.traffic_share != null ? (Number(r.traffic_share) * 100).toFixed(1) + '%' : '—';
|
||||
const avail = escapeHtml(r.effective_availability || 'active');
|
||||
const blocked = r.effective_availability === 'blocked';
|
||||
const blockBtn = blocked ? '' : `<button class="btn btn-sm btn-outline-danger" style="font-size:.62rem;padding:1px 6px" data-model="${model}" data-provider="${provider}" data-reason="stalled ${stalled} sessions in ${windowHours}h" onclick="blockModelConfirm(event)">Block</button>`;
|
||||
return '<tr>'
|
||||
+ `<td class="text-nowrap">${model}</td>`
|
||||
+ `<td>${provider}</td>`
|
||||
+ `<td class="num">${stalled}</td>`
|
||||
+ `<td class="num">${cost}</td>`
|
||||
+ `<td class="num">${share}</td>`
|
||||
+ `<td>${avail}</td>`
|
||||
+ `<td class="text-end">${blockBtn}</td>`
|
||||
+ '</tr>';
|
||||
}).join('')
|
||||
+ '</tbody></table></div>';
|
||||
}
|
||||
/* LOAD_STALL_ROLLUP:END */
|
||||
|
||||
/* BLOCK_CONFIRM:BEGIN */
|
||||
function blockModelConfirm(event) {
|
||||
const btn = event.currentTarget;
|
||||
const model = btn.dataset.model;
|
||||
const prov = btn.dataset.provider;
|
||||
const reason = btn.dataset.reason;
|
||||
if (!model) return;
|
||||
const row = btn.closest('div.d-flex') || btn.closest('tr');
|
||||
if (!row) return;
|
||||
const existing = row.nextElementSibling;
|
||||
if (existing && existing.classList.contains('inline-confirm-row')) {
|
||||
existing.remove();
|
||||
return;
|
||||
}
|
||||
const confirmRow = document.createElement('div');
|
||||
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
|
||||
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
|
||||
confirmRow.innerHTML = '<input type="text" class="form-control form-control-sm" style="font-size:.72rem;flex:1;min-width:120px" value="' + escapeHtml(reason) + '">'
|
||||
+ '<button class="btn btn-sm btn-danger" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
|
||||
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
|
||||
|
||||
const cancelBtn = confirmRow.querySelector('button:last-child');
|
||||
cancelBtn.onclick = () => confirmRow.remove();
|
||||
|
||||
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
|
||||
const input = confirmRow.querySelector('input');
|
||||
confirmBtn.onclick = async () => {
|
||||
const reason = input.value;
|
||||
confirmBtn.disabled = true;
|
||||
const resp = await apiFetch(
|
||||
`${API}api/models/${encodeURIComponent(model)}/${encodeURIComponent(prov)}/availability`,
|
||||
{ method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({ availability: 'blocked', reason: reason }) }
|
||||
);
|
||||
if (resp) {
|
||||
toast('Model blocked', 'success');
|
||||
confirmRow.remove();
|
||||
// Refresh dependent views
|
||||
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
|
||||
if (typeof loadBlockedModels === 'function') loadBlockedModels();
|
||||
if (typeof loadLoops === 'function') loadLoops();
|
||||
if (typeof loadModels === 'function') loadModels();
|
||||
} else {
|
||||
confirmBtn.disabled = false;
|
||||
confirmRow.querySelector('.inline-error')?.remove();
|
||||
const err = document.createElement('div');
|
||||
err.className = 'text-danger small inline-error';
|
||||
err.textContent = 'Block failed';
|
||||
confirmRow.appendChild(err);
|
||||
}
|
||||
};
|
||||
|
||||
row.after(confirmRow);
|
||||
}
|
||||
|
||||
function unblockModelConfirm(modelId, provider) {
|
||||
const btn = event.target;
|
||||
const row = btn.closest('div.d-flex') || btn.closest('tr');
|
||||
if (!row) return;
|
||||
const existing = row.nextElementSibling;
|
||||
if (existing && existing.classList.contains('inline-confirm-row')) {
|
||||
existing.remove();
|
||||
return;
|
||||
}
|
||||
const confirmRow = document.createElement('div');
|
||||
confirmRow.className = 'inline-confirm-row d-flex align-items-center gap-2 py-1';
|
||||
confirmRow.style.cssText = 'border-bottom:1px solid rgba(255,255,255,.05);background:rgba(220,38,38,.08);padding:4px 8px';
|
||||
confirmRow.innerHTML = '<span class="small text-danger">Unblock this model?</span>'
|
||||
+ '<button class="btn btn-sm btn-warning" style="font-size:.62rem;padding:1px 8px">Confirm</button>'
|
||||
+ '<button class="btn btn-sm btn-outline-secondary" style="font-size:.62rem;padding:1px 8px">Cancel</button>';
|
||||
|
||||
const cancelBtn = confirmRow.querySelector('button:last-child');
|
||||
cancelBtn.onclick = () => confirmRow.remove();
|
||||
|
||||
const confirmBtn = confirmRow.querySelector('button:nth-of-type(1)');
|
||||
confirmBtn.onclick = async () => {
|
||||
confirmBtn.disabled = true;
|
||||
const resp = await apiFetch(
|
||||
`${API}api/models/${encodeURIComponent(modelId)}/${encodeURIComponent(provider)}/availability`,
|
||||
{ method:'DELETE' }
|
||||
);
|
||||
if (resp) {
|
||||
toast('Model unblocked', 'success');
|
||||
confirmRow.remove();
|
||||
if (typeof loadBlockedModels === 'function') loadBlockedModels();
|
||||
if (typeof loadModels === 'function') loadModels();
|
||||
if (typeof loadStallRollup === 'function') loadStallRollup(document.querySelector('#stall-window .active')?.dataset?.hours || 24);
|
||||
} else {
|
||||
confirmBtn.disabled = false;
|
||||
confirmRow.querySelector('.inline-error')?.remove();
|
||||
const err = document.createElement('div');
|
||||
err.className = 'text-danger small inline-error';
|
||||
err.textContent = 'Unblock failed';
|
||||
confirmRow.appendChild(err);
|
||||
}
|
||||
};
|
||||
|
||||
row.after(confirmRow);
|
||||
}
|
||||
/* BLOCK_CONFIRM:END */
|
||||
|
||||
/* ═══════════════════════════════════════
|
||||
BLOCKED MODELS
|
||||
═══════════════════════════════════════ */
|
||||
|
||||
/* BLOCKED_HELPERS:BEGIN */
|
||||
function formatRelativeTime(iso8601) {
|
||||
if (!iso8601) return '';
|
||||
const now = new Date();
|
||||
const date = new Date(iso8601);
|
||||
const diffMs = now - date;
|
||||
const diffSeconds = Math.floor(diffMs / 1000);
|
||||
if (diffSeconds < 10) return 'just now';
|
||||
if (diffSeconds < 60) return Math.floor(diffSeconds) + 's ago';
|
||||
const diffMinutes = Math.floor(diffSeconds / 60);
|
||||
if (diffMinutes < 60) return diffMinutes + 'm ago';
|
||||
const diffHours = Math.floor(diffMinutes / 60);
|
||||
if (diffHours < 24) return diffHours + 'h ago';
|
||||
const diffDays = Math.floor(diffHours / 24);
|
||||
if (diffDays < 30) return diffDays + 'd ago';
|
||||
return diffDays + 'd ago';
|
||||
}
|
||||
/* BLOCKED_HELPERS:END */
|
||||
|
||||
async function loadBlockedModels() {
|
||||
const body = document.getElementById('blocked-models-body');
|
||||
if (!body) return;
|
||||
const data = await apiFetch(`${API}api/models/blocked`);
|
||||
if (!data || !data.length) {
|
||||
body.innerHTML = '<div class="text-muted small">No blocked models</div>';
|
||||
return;
|
||||
}
|
||||
body.innerHTML = '<div class="table-responsive"><table class="table table-vcenter table-hover card-table"><thead><tr>'
|
||||
+ '<th>Model</th><th>Provider</th><th>Reason</th><th>Blocked Since</th><th></th>'
|
||||
+ '</tr></thead><tbody>'
|
||||
+ data.map(r => {
|
||||
const model = escapeHtml(r.model_id || '');
|
||||
const provider = escapeHtml(r.provider || '');
|
||||
const reason = escapeHtml(r.reason || '');
|
||||
const since = r.blocked_since ? formatRelativeTime(r.blocked_since) : '—';
|
||||
return '<tr>'
|
||||
+ `<td class="text-nowrap">${model}</td>`
|
||||
+ `<td>${provider}</td>`
|
||||
+ `<td class="text-muted">${reason}</td>`
|
||||
+ `<td class="text-muted">${since}</td>`
|
||||
+ `<td class="text-end"><button class="btn btn-sm btn-outline-warning" style="font-size:.62rem;padding:1px 6px" onclick="unblockModelConfirm('${model}', '${provider}')">Unblock</button></td>`
|
||||
+ '</tr>';
|
||||
}).join('')
|
||||
+ '</tbody></table></div>';
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════
|
||||
INIT
|
||||
═══════════════════════════════════════ */
|
||||
|
||||
function bindStallWindow() {
|
||||
const seg = document.getElementById('stall-window');
|
||||
if (!seg) return;
|
||||
seg.addEventListener('click', ev => {
|
||||
const btn = ev.target.closest('button');
|
||||
if (!btn || !btn.dataset.hours) return;
|
||||
seg.querySelectorAll('button').forEach(b => b.classList.toggle('active', b === btn));
|
||||
loadStallRollup(btn.dataset.hours);
|
||||
});
|
||||
}
|
||||
|
||||
function init() {
|
||||
// Bound once, before the first load: the controls are static markup, so
|
||||
// they never get rebuilt and never need re-binding. (Rebuilding a control
|
||||
// you are typing into is how the providers page ended up reversing text.)
|
||||
bindModelFilters();
|
||||
bindModelsViews();
|
||||
bindStallWindow();
|
||||
loadModels();
|
||||
loadStallRollup(24);
|
||||
loadBlockedModels();
|
||||
connectSSE();
|
||||
}
|
||||
|
||||
|
||||
165
plans/no-progress-lift-a2.md
Normal file
165
plans/no-progress-lift-a2.md
Normal file
@@ -0,0 +1,165 @@
|
||||
# Lift A.2: loops and stalling models, surfaced with the lever beside them
|
||||
|
||||
Status: planned -- brief ready; worktree created from origin/main 5aaf1e6 (after #103)
|
||||
Date: 2026-09-26
|
||||
This brief is the contract. Lift A (PR #102) is the base; do NOT read
|
||||
`plans/no-progress-detection.md` (reference only).
|
||||
|
||||
## Goal
|
||||
|
||||
Meet the surfacing half of North Star #4 (`CLAUDE.md`): "Waste is surfaced in
|
||||
the admin portal, and stopping it is one click." Lift A built the detection,
|
||||
the alerts and the data. The portal does not yet show the evidence or put the
|
||||
lever beside it:
|
||||
|
||||
- **Loops panel** (`admin/frontend/index.html#loops`, `loadLoops`) shows a
|
||||
severity badge, the raw key `opencode-loop:<root id>` and `opened_at`.
|
||||
Nothing an operator can act on.
|
||||
- **Models page** has no per-model stall rollup, no Block button beside any
|
||||
evidence, and no Blocked list. `blocked` exists only as a fourth value in
|
||||
the Model Availability override dropdown.
|
||||
|
||||
The data is already there. On the live DB, 18 of 19 `watchdog_verdicts` rows
|
||||
carry `model_id`, and all carry `calls_since_landed` and
|
||||
`cost_since_landed_usd`, now that conversation identity arrives (fixed in
|
||||
#102).
|
||||
|
||||
**Out of scope:** detection changes, recovery modes, alert channels other than
|
||||
desktop, Lift B, and any change to how `blocked` is enforced (it already
|
||||
excludes routing and pinned requests via `_admin_excluded_models`).
|
||||
|
||||
## Facts to build on (checked on origin/main 2026-09-26)
|
||||
|
||||
- `watchdog_verdicts`: `tick_id, session_id, session_root, agent, model_id,
|
||||
provider, flagged, dup, top, top_what, landed, slow, coverage,
|
||||
calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at`.
|
||||
One row per judged session per tick. **`agent` holds the opencode session
|
||||
TITLE**, not the agent slug. `cost_since_landed_usd` is cumulative, so it
|
||||
grows tick by tick for the same session.
|
||||
- `watchdog_alerts`: `dedup_key` (`opencode-loop:<root id>`), `state`,
|
||||
`severity`, `flagged_ticks`, `opened_at`, `last_fired_at`, `resolved_at`.
|
||||
- `admin_model_overrides`: `model_id, provider, availability, reason,
|
||||
updated_at`. `POST /admin/api/models/{model_id:path}/{provider}/availability`
|
||||
takes `{availability, reason}` (`_AvailabilityBody`, `src/admin.py` ~885);
|
||||
`DELETE` on the same path clears the override (~2116).
|
||||
- Existing endpoints: `GET /admin/api/watchdog/status`, `/loops` (~2834),
|
||||
`/channels` GET/POST, `POST /test-alert`.
|
||||
- `route_decisions.session_key` is `"c:" + opencode session id` and
|
||||
`route_decisions.agent` is the agent slug when identity arrives.
|
||||
- Models page cards: "Model Availability" (`models.html` ~226) and
|
||||
"Per-model usage" (~310). JS unit harness: `tests/test_admin_js_units.py`
|
||||
(marker-delimited pure functions, run under node).
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Loops panel shows the evidence (`index.html#loops`)
|
||||
|
||||
- Extend `GET /admin/api/watchdog/loops` to return each OPEN alert joined to
|
||||
its root's most recent FLAGGED verdict (any session in that root). Fields:
|
||||
`dedup_key, root_id, session_id, title, agent_slug, model_id, provider,
|
||||
severity, flagged_ticks, opened_at, top_what, dup, top, coverage,
|
||||
calls_since_landed, cost_since_landed_usd, llm_second_opinion`.
|
||||
`agent_slug` comes from the latest `route_decisions.agent` for
|
||||
`"c:" + session_id` (NULL if none). Also return `resolved` (alerts resolved
|
||||
in the last 24h, same fields, capped at 20).
|
||||
- One row per open alert: severity badge, title (truncate ~40ch, full title in
|
||||
`title=`), model, top repeated target, calls since last landed, $ since last
|
||||
landed (`$0.00`, `n/a` when NULL), open for (relative time), and a **Block
|
||||
model** button (component 3 flow) when `model_id` is known.
|
||||
- Resolved rows below, dimmed. Session ids truncated with a tooltip.
|
||||
- Empty state stays "No open alerts".
|
||||
|
||||
### 2. Per-model stall rollup (`models.html`, new card "Stalls by model")
|
||||
|
||||
- `GET /admin/api/watchdog/rollup?hours=N`, N in {1, 24, 168}. Per
|
||||
`(model_id, provider)` with at least one flagged verdict in the window:
|
||||
- `stalled_sessions` = `COUNT(DISTINCT session_root)` over flagged rows. Not
|
||||
row counts: one row is written per tick.
|
||||
- `stalled_usd` = per flagged `session_id`, take the MAX
|
||||
`cost_since_landed_usd` in the window (it is cumulative), then sum per
|
||||
model.
|
||||
- `traffic_share` = the model's share of `route_decisions` in the same
|
||||
window, its own column; never divide sessions by requests.
|
||||
- `effective_availability` (override if any, else catalog).
|
||||
- Window picker is radio-style (1h / 24h / 7d), default 24h.
|
||||
- Each row: model, provider, stalled sessions, $ in stalled sessions, traffic
|
||||
share, availability, and **Block model** (hidden when already blocked).
|
||||
- Card sits ABOVE "Model Availability". Empty state: "No stalled sessions in
|
||||
this window".
|
||||
|
||||
### 3. Block from the evidence, with a reason
|
||||
|
||||
- Block (from 1 or 2) opens an INLINE confirm row, not a browser dialog:
|
||||
reason input prefilled `stalled <N> sessions in <window>` (from the Loops
|
||||
panel: `looping session <title>`), editable, then Confirm / Cancel.
|
||||
- Confirm posts `{availability: "blocked", reason}` to the existing endpoint,
|
||||
then refreshes the rollup, the Blocked list and the availability table.
|
||||
- The existing dropdown path keeps working; when it sets `blocked` it sends
|
||||
reason `set from Models dropdown`.
|
||||
|
||||
### 4. Blocked list (`models.html`, card "Blocked models")
|
||||
|
||||
- Rows from `admin_model_overrides WHERE availability = 'blocked'`: model,
|
||||
provider, reason, blocked since (`updated_at`), **Unblock**
|
||||
(`DELETE .../availability`, inline confirm).
|
||||
- Empty state: "No blocked models". Endpoint: reuse an existing models/overrides
|
||||
endpoint if one returns `reason` and `updated_at`; otherwise add
|
||||
`GET /admin/api/models/blocked`.
|
||||
|
||||
### 5. Alert text carries the evidence
|
||||
|
||||
- In `src/watchdog.py`, the trigger and escalate `AlertEvent.summary` becomes
|
||||
`<title>: <calls> calls, $<cost> since last landed change on <model>.
|
||||
<dashboard_base_url>#loops` (omit the `on <model>` / `$` parts when NULL).
|
||||
Keep it one line; `notify-send` shows title and summary only.
|
||||
|
||||
## Todos (small; one commit each, by explicit path)
|
||||
|
||||
1. Loops endpoint join + `resolved` + endpoint tests (seeded tables: two
|
||||
ticks for one session so the latest flagged verdict wins; a root with two
|
||||
sessions; NULL model/cost).
|
||||
2. Loops panel UI + pure formatting helpers (money, relative time, truncation)
|
||||
in marker-delimited functions with node cases in `test_admin_js_units.py`.
|
||||
3. Rollup endpoint + tests (distinct roots across ticks; MAX-then-SUM cost;
|
||||
traffic share; window filter).
|
||||
4. Stalls-by-model card + window radio + node cases for its helpers.
|
||||
5. Inline Block confirm (shared by Loops and rollup) + reason on the dropdown
|
||||
path + tests that the POST carries the reason.
|
||||
6. Blocked list card + Unblock + tests.
|
||||
7. Alert summary text in `watchdog.py` + a test asserting the summary.
|
||||
8. Visual QA (below) and the done report.
|
||||
|
||||
## Visual QA (required; HTTP 200 is not QA)
|
||||
|
||||
- On 8081 (`scripts/sandbox.sh`) against a sqlite-backup copy of the live DB.
|
||||
Seed the COPY with one open alert whose verdicts carry a model, calls and
|
||||
cost, one resolved alert, and one blocked override with a reason. Never
|
||||
write to the live DB.
|
||||
- Screenshots of Home (Loops) and Models at 1400px and 800px wide, plus zoomed
|
||||
crops of each new card. Check truncation, alignment, empty states, and that
|
||||
Block -> Confirm -> Blocked list -> Unblock round-trips on screen.
|
||||
- Save screenshots under the scratch/evidence dir, not the repo.
|
||||
|
||||
## Ground rules (lessons from Lift A)
|
||||
|
||||
- Worktree already exists: `/home/alee/Sources/6krrt-worktrees/no-progress-lift-a2`,
|
||||
branch `feat/no-progress-lift-a2` from `origin/main` `5aaf1e6` (includes #103).
|
||||
`/start-work` runs with `--worktree` set to it. Nothing in the main checkout.
|
||||
- Dispatch to Sisyphus-Junior categories or `general`. NEVER to
|
||||
`oh-my-claudecode:*` agents: they are pinned to Anthropic models this
|
||||
opencode cannot reach, and return "completed" with zero work. A subagent
|
||||
session with 0 tool calls did nothing.
|
||||
- One commit per todo; no squashed "full implementation" commits.
|
||||
- No test exemptions or skips for labelled cases. If a test cannot pass, stop
|
||||
and report why.
|
||||
- Lint: pinned `uvx ruff@0.16.9`, no NEW findings vs `origin/main` in touched
|
||||
files (recipe in `plans/no-progress-lift-a.md` Ground rules). Paste the
|
||||
gate's empty output in the done report.
|
||||
- Full pytest: record the `origin/main` baseline first; the only known failure
|
||||
is `test_incumbent_routing::test_debug_log_emits_incumbent_identity`.
|
||||
- Port 8080 is production: no requests to it, not even GETs.
|
||||
- Do not push, do not open a PR, do not remove the worktree. The done report
|
||||
lists commits, tests, screenshots and operator steps (router restart).
|
||||
- ASCII only; no middle dots. No prose paragraphs in the admin UI; mutually
|
||||
exclusive choices are radio-style.
|
||||
- Stop a worker whose context passes ~150k tokens and start a fresh one.
|
||||
162
src/admin.py
162
src/admin.py
@@ -33,7 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, List, Literal, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Response
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Response
|
||||
from fastapi.responses import FileResponse
|
||||
from openai import OpenAI, OpenAIError
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
@@ -1999,6 +1999,25 @@ def build_router(
|
||||
return rows
|
||||
return [r for r in rows if r.get("effective_availability") == "active"]
|
||||
|
||||
@router.get("/api/models/blocked")
|
||||
def _admin_blocked_models() -> list:
|
||||
"""List all models that have an admin override set to 'blocked'.
|
||||
|
||||
Returns model_id, provider, reason, and blocked_since (aliased from
|
||||
updated_at). Empty list when no models are blocked.
|
||||
"""
|
||||
conn = _db_callable()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT model_id, provider, reason, updated_at AS blocked_since "
|
||||
"FROM admin_model_overrides "
|
||||
"WHERE availability = 'blocked' "
|
||||
"ORDER BY updated_at DESC"
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@router.get("/api/models/{model_id:path}/{provider}")
|
||||
def admin_model_detail(model_id: str, provider: str) -> dict:
|
||||
"""A single model's detail (same shape as one /api/models element)."""
|
||||
@@ -2835,11 +2854,142 @@ def build_router(
|
||||
def _watchdog_loops():
|
||||
conn = _db_callable()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM watchdog_alerts WHERE resolved_at IS NULL "
|
||||
"ORDER BY opened_at DESC"
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
rows = conn.execute("""
|
||||
SELECT a.dedup_key, a.severity, a.flagged_ticks, a.opened_at,
|
||||
a.last_fired_at,
|
||||
v.session_id, v.session_root, v.agent AS title,
|
||||
v.model_id, v.provider, v.top_what, v.dup, v.top,
|
||||
v.landed, v.coverage, v.calls_since_landed,
|
||||
v.cost_since_landed_usd, v.llm_second_opinion,
|
||||
rd.agent AS agent_slug
|
||||
FROM watchdog_alerts a
|
||||
LEFT JOIN watchdog_verdicts v
|
||||
ON a.dedup_key = 'opencode-loop:' || v.session_root
|
||||
AND v.id = (
|
||||
SELECT v2.id FROM watchdog_verdicts v2
|
||||
WHERE a.dedup_key = 'opencode-loop:' || v2.session_root
|
||||
AND v2.flagged = 1
|
||||
ORDER BY v2.id DESC LIMIT 1
|
||||
)
|
||||
LEFT JOIN route_decisions rd
|
||||
ON rd.session_key = 'c:' || v.session_id
|
||||
AND rd.id = (
|
||||
SELECT MAX(rd2.id) FROM route_decisions rd2
|
||||
WHERE rd2.session_key = 'c:' || v.session_id
|
||||
)
|
||||
WHERE a.resolved_at IS NULL
|
||||
ORDER BY a.opened_at DESC
|
||||
""").fetchall()
|
||||
alerts = [dict(r) for r in rows]
|
||||
|
||||
resolved_rows = conn.execute("""
|
||||
SELECT a.dedup_key, a.severity, a.flagged_ticks, a.opened_at,
|
||||
a.last_fired_at, a.resolved_at,
|
||||
v.session_id, v.session_root, v.agent AS title,
|
||||
v.model_id, v.provider, v.top_what, v.dup, v.top,
|
||||
v.landed, v.coverage, v.calls_since_landed,
|
||||
v.cost_since_landed_usd, v.llm_second_opinion,
|
||||
rd.agent AS agent_slug
|
||||
FROM watchdog_alerts a
|
||||
LEFT JOIN watchdog_verdicts v
|
||||
ON a.dedup_key = 'opencode-loop:' || v.session_root
|
||||
AND v.id = (
|
||||
SELECT v2.id FROM watchdog_verdicts v2
|
||||
WHERE a.dedup_key = 'opencode-loop:' || v2.session_root
|
||||
AND v2.flagged = 1
|
||||
ORDER BY v2.id DESC LIMIT 1
|
||||
)
|
||||
LEFT JOIN route_decisions rd
|
||||
ON rd.session_key = 'c:' || v.session_id
|
||||
AND rd.id = (
|
||||
SELECT MAX(rd2.id) FROM route_decisions rd2
|
||||
WHERE rd2.session_key = 'c:' || v.session_id
|
||||
)
|
||||
WHERE a.resolved_at IS NOT NULL
|
||||
AND julianday(a.resolved_at) > julianday('now', '-1 day')
|
||||
ORDER BY a.resolved_at DESC
|
||||
LIMIT 20
|
||||
""").fetchall()
|
||||
resolved = [dict(r) for r in resolved_rows]
|
||||
|
||||
return {"alerts": alerts, "resolved": resolved}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@router.get("/api/watchdog/rollup")
|
||||
def _watchdog_rollup(hours: int = Query(24, ge=1)):
|
||||
"""Per-model stall aggregates for the given window.
|
||||
|
||||
Returns stalled_sessions (COUNT(DISTINCT session_root)), stalled_usd
|
||||
(per-session MAX of cumulative cost_since_landed_usd, then SUM per model),
|
||||
traffic_share (route_decisions in same window), and effective_availability
|
||||
(admin override first, else catalog default).
|
||||
"""
|
||||
if hours not in (1, 24, 168):
|
||||
raise HTTPException(status_code=422, detail="hours must be 1, 24, or 168")
|
||||
|
||||
conn = _db_callable()
|
||||
try:
|
||||
# Stalled sessions: aggregate flagged verdicts to one row per
|
||||
# session (taking MAX cumulative cost), then group by model.
|
||||
# COUNT(DISTINCT session_root) counts stalled sessions; SUM(max_cost)
|
||||
# sums each session's peak cost once.
|
||||
stalled = conn.execute("""
|
||||
SELECT v.model_id, v.provider,
|
||||
COUNT(DISTINCT v.session_root) AS stalled_sessions,
|
||||
COALESCE(SUM(v.max_cost), 0) AS stalled_usd
|
||||
FROM (
|
||||
SELECT session_id, session_root, model_id, provider,
|
||||
MAX(cost_since_landed_usd) AS max_cost
|
||||
FROM watchdog_verdicts
|
||||
WHERE flagged = 1
|
||||
AND julianday(created_at) > julianday('now', '-' || ? || ' hours')
|
||||
AND model_id IS NOT NULL
|
||||
GROUP BY session_id
|
||||
) v
|
||||
GROUP BY v.model_id, v.provider
|
||||
ORDER BY stalled_sessions DESC
|
||||
""", (hours,)).fetchall()
|
||||
|
||||
result = []
|
||||
for row in stalled:
|
||||
row = dict(row)
|
||||
model_id = row["model_id"]
|
||||
provider = row["provider"]
|
||||
|
||||
# Traffic share: count route_decisions for this model in same window
|
||||
traffic_row = conn.execute("""
|
||||
SELECT CAST(COUNT(*) AS REAL) / NULLIF(
|
||||
(SELECT CAST(COUNT(*) AS REAL) FROM route_decisions
|
||||
WHERE julianday(observed_at) > julianday('now', '-' || ? || ' hours')
|
||||
AND selected_model IS NOT NULL), 0
|
||||
) AS traffic_share
|
||||
FROM route_decisions
|
||||
WHERE selected_model = ?
|
||||
AND selected_provider = ?
|
||||
AND julianday(observed_at) > julianday('now', '-' || ? || ' hours')
|
||||
""", (hours, model_id, provider, hours)).fetchone()
|
||||
row["traffic_share"] = traffic_row["traffic_share"] if traffic_row else 0.0
|
||||
|
||||
# Effective availability: admin_model_overrides first, else catalog
|
||||
override = conn.execute(
|
||||
"SELECT availability FROM admin_model_overrides "
|
||||
"WHERE model_id = ? AND provider = ?",
|
||||
(model_id, provider),
|
||||
).fetchone()
|
||||
if override:
|
||||
row["effective_availability"] = override["availability"]
|
||||
else:
|
||||
catalog = conn.execute(
|
||||
"SELECT availability FROM models "
|
||||
"WHERE model_id = ? AND provider = ?",
|
||||
(model_id, provider),
|
||||
).fetchone()
|
||||
row["effective_availability"] = catalog["availability"] if catalog else "active"
|
||||
|
||||
result.append(row)
|
||||
|
||||
return result
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@@ -546,7 +546,7 @@ def tick(
|
||||
tree_last_landed[root] = latest
|
||||
|
||||
calls_since_landed_map: dict[str, int] = {}
|
||||
cost_since_landed_map: dict[str, float] = {}
|
||||
cost_since_landed_map: dict[str, float | None] = {}
|
||||
for sid, cs in all_calls.items():
|
||||
root = session_root.get(sid, sid)
|
||||
last_landed = tree_last_landed.get(root, 0.0)
|
||||
@@ -566,7 +566,7 @@ def tick(
|
||||
+ ",".join("?" for _ in fps) + ") AND observed_at > ?",
|
||||
fps + [since_iso],
|
||||
).fetchone()
|
||||
cost_since_landed_map[sid] = row[0] or 0.0 if row else 0.0
|
||||
cost_since_landed_map[sid] = row[0] if row and row[0] is not None else None
|
||||
else:
|
||||
cost_since_landed_map[sid] = 0.0
|
||||
|
||||
@@ -651,9 +651,15 @@ def tick(
|
||||
_fire_alert(conn, dedup, "trigger", severity,
|
||||
f"opencode loop: {agent}")
|
||||
if notifier:
|
||||
calls = calls_since_landed_map.get(sid, 0)
|
||||
cost = cost_since_landed_map.get(sid)
|
||||
model_id = session_attr.get(sid, (None, None, 0.0, 0))[0]
|
||||
summary = (
|
||||
f"Session {sid} flagged for looping. "
|
||||
f"See {cfg.watchdog.dashboard_base_url}#loops"
|
||||
f"opencode loop: {agent}: {calls} calls"
|
||||
+ (f", ${cost:.2f}" if cost is not None else "")
|
||||
+ " since last landed change"
|
||||
+ (f" on {model_id}" if model_id else "")
|
||||
+ f". {cfg.watchdog.dashboard_base_url}#loops"
|
||||
)
|
||||
event = AlertEvent(
|
||||
dedup_key=dedup, severity=severity, state="trigger",
|
||||
@@ -680,10 +686,16 @@ def tick(
|
||||
_fire_alert(conn, dedup, "escalate", "critical",
|
||||
f"opencode loop: {agent}")
|
||||
if notifier:
|
||||
calls = calls_since_landed_map.get(sid, 0)
|
||||
cost = cost_since_landed_map.get(sid)
|
||||
model_id = session_attr.get(sid, (None, None, 0.0, 0))[0]
|
||||
summary = (
|
||||
f"Session {sid} persisting after {flagged_ticks} "
|
||||
f"ticks ({flagged_ticks * 5} min). "
|
||||
f"See {cfg.watchdog.dashboard_base_url}#loops"
|
||||
f"opencode loop: {agent}: {calls} calls"
|
||||
+ (f", ${cost:.2f}" if cost is not None else "")
|
||||
+ " since last landed change"
|
||||
+ (f" on {model_id}" if model_id else "")
|
||||
+ f" (persisting {flagged_ticks * 5} min). "
|
||||
+ f"{cfg.watchdog.dashboard_base_url}#loops"
|
||||
)
|
||||
event = AlertEvent(
|
||||
dedup_key=dedup, severity="critical",
|
||||
|
||||
@@ -458,3 +458,143 @@ def test_merge_knob_rows_value_normalization(persisted, runtime, expect_drift):
|
||||
");\n"
|
||||
f"assert.strictEqual(rows['a.knob'].drift, {str(expect_drift).lower()});"
|
||||
)
|
||||
|
||||
|
||||
INDEX_HTML = ROOT / "admin" / "frontend" / "index.html"
|
||||
LIFT_A2_HELPERS_BEGIN = "/* LIFT_A2_HELPERS:BEGIN */"
|
||||
LIFT_A2_HELPERS_END = "/* LIFT_A2_HELPERS:END */"
|
||||
|
||||
|
||||
def _assert_against_lift_a2_helpers(assertion_js: str) -> subprocess.CompletedProcess[str]:
|
||||
"""Assert ``assertion_js`` against the real lift-a2 formatting helpers source."""
|
||||
return _run_node(
|
||||
"const assert = require('assert');\n"
|
||||
+ _extract_between(INDEX_HTML, LIFT_A2_HELPERS_BEGIN, LIFT_A2_HELPERS_END)
|
||||
+ "\n"
|
||||
+ assertion_js
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_format_money():
|
||||
_assert_against_lift_a2_helpers(
|
||||
"assert.strictEqual(formatMoney(1.5), '$1.50');\n"
|
||||
"assert.strictEqual(formatMoney(0), '$0.00');\n"
|
||||
"assert.strictEqual(formatMoney(null), 'n/a');\n"
|
||||
"assert.strictEqual(formatMoney(undefined), 'n/a');"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_format_relative_time():
|
||||
_assert_against_lift_a2_helpers(
|
||||
"// just now (within 10s)\n"
|
||||
"assert.strictEqual(formatRelativeTime(new Date().toISOString()), 'just now');\n"
|
||||
"// a few minutes ago\n"
|
||||
"const fiveMinAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();\n"
|
||||
"assert.strictEqual(formatRelativeTime(fiveMinAgo), '5m ago');\n"
|
||||
"// null/empty\n"
|
||||
"assert.strictEqual(formatRelativeTime(null), '');\n"
|
||||
"assert.strictEqual(formatRelativeTime(''), '');"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_truncate():
|
||||
_assert_against_lift_a2_helpers(
|
||||
"assert.strictEqual(truncate('hello world', 40), 'hello world');\n"
|
||||
"assert.strictEqual(truncate('hello world', 5), 'hello...');\n"
|
||||
"assert.strictEqual(truncate(null, 5), '');\n"
|
||||
"assert.strictEqual(truncate(undefined, 5), '');"
|
||||
)
|
||||
|
||||
|
||||
MODELS_HTML = ROOT / "admin" / "frontend" / "models.html"
|
||||
BLOCK_MODEL_BEGIN = "/* BLOCK_CONFIRM:BEGIN */"
|
||||
BLOCK_MODEL_END = "/* BLOCK_CONFIRM:END */"
|
||||
|
||||
|
||||
def _assert_against_block_model(assertion_js: str) -> subprocess.CompletedProcess[str]:
|
||||
return _run_node(
|
||||
"const assert = require('assert');\n"
|
||||
+ _extract_between(MODELS_HTML, BLOCK_MODEL_BEGIN, BLOCK_MODEL_END)
|
||||
+ "\n"
|
||||
+ assertion_js
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_block_model_exists():
|
||||
_assert_against_block_model(
|
||||
"assert.strictEqual(typeof blockModelConfirm, 'function');"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_block_button_uses_data_attrs_not_onclick_interpolation():
|
||||
"""The Block button in both pages must carry data-model/data-provider/
|
||||
data-reason attributes and wire via onclick=\"blockModelConfirm(event)\"
|
||||
rather than interpolating runtime values into JS — interpolation broke
|
||||
on titles containing a double quote."""
|
||||
for path in (INDEX_HTML, MODELS_HTML):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert 'onclick="blockModelConfirm(event)"' in text, (
|
||||
f"{path.name}: Block button missing onclick handler"
|
||||
)
|
||||
assert "data-model=" in text, f"{path.name}: missing data-model attr"
|
||||
assert "data-provider=" in text, f"{path.name}: missing data-provider attr"
|
||||
assert "data-reason=" in text, f"{path.name}: missing data-reason attr"
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_block_data_reason_escapes_double_quotes():
|
||||
"""A looping-session title containing both a double and a single quote is
|
||||
safe inside data-reason=\"...\": escapeHtml turns the double quote into
|
||||
" so it cannot terminate the attribute, and a bare single quote is
|
||||
harmless inside a double-quoted HTML attribute."""
|
||||
escape = re.search(r"function escapeHtml\(s\) \{.*?\n\}", INDEX_HTML.read_text(encoding="utf-8"), re.DOTALL)
|
||||
assert escape, "index.html: escapeHtml source not found"
|
||||
js = (
|
||||
"const assert = require('assert');\n"
|
||||
+ escape.group(0)
|
||||
+ "\n"
|
||||
"const title = 'he said \"hello\\'s\"';\n"
|
||||
"const dataReason = 'looping session ' + title;\n"
|
||||
"const html = 'data-reason=\"' + escapeHtml(dataReason) + '\"';\n"
|
||||
"assert.ok(html.includes('"'));\n"
|
||||
"assert.ok(!html.includes('\"hello'));\n"
|
||||
"assert.strictEqual(escapeHtml(dataReason), "
|
||||
" 'looping session he said "hello\\'s"');\n"
|
||||
)
|
||||
_run_node(js)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_block_button_onclick_wires_to_blockModelConfirm():
|
||||
"""The Block button carries onclick=\"blockModelConfirm(event)\" so the
|
||||
click actually invokes the handler. Without this the data-* attributes
|
||||
are inert and clicking Block does nothing."""
|
||||
for path in (INDEX_HTML, MODELS_HTML):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert 'onclick="blockModelConfirm(event)"' in text, (
|
||||
f"{path.name}: Block button missing onclick handler"
|
||||
)
|
||||
|
||||
|
||||
@skip_without_node
|
||||
def test_blockModelConfirm_reads_from_event_currentTarget_dataset():
|
||||
"""blockModelConfirm(event) must read model/provider/reason from
|
||||
event.currentTarget.dataset — the old hybrid signature that expected
|
||||
positional args is broken because onclick passes a single Event."""
|
||||
js = (
|
||||
"const assert = require('assert');\n"
|
||||
+ _extract_between(MODELS_HTML, BLOCK_MODEL_BEGIN, BLOCK_MODEL_END)
|
||||
+ "\n"
|
||||
"assert.strictEqual(typeof blockModelConfirm, 'function');\n"
|
||||
"const src = blockModelConfirm.toString();\n"
|
||||
"assert.ok(src.includes('event.currentTarget'), 'must use event.currentTarget');\n"
|
||||
"assert.ok(src.includes('dataset.model'), 'must read dataset.model');\n"
|
||||
"assert.ok(src.includes('dataset.provider'), 'must read dataset.provider');\n"
|
||||
"assert.ok(src.includes('dataset.reason'), 'must read dataset.reason');\n"
|
||||
)
|
||||
_run_node(js)
|
||||
|
||||
@@ -492,3 +492,80 @@ def test_set_availability_returns_warning_when_override_drops_ceiling(
|
||||
for w in data["warnings"]
|
||||
)
|
||||
|
||||
|
||||
# --- blocked list ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_blocked_list_empty(seeded_client):
|
||||
"""GET /admin/api/models/blocked with no blocked overrides -> 200 + []."""
|
||||
resp = seeded_client.get("/admin/api/models/blocked")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == []
|
||||
|
||||
|
||||
def test_blocked_list_does_not_shadow_model_detail(seeded_client):
|
||||
"""The literal /blocked segment must not be consumed by the
|
||||
{model_id:path}/{provider} converter: a real model id still resolves."""
|
||||
resp = seeded_client.get("/admin/api/models/cheap/neuralwatt")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["model_id"] == "cheap"
|
||||
|
||||
|
||||
def test_blocked_list_with_data(tmp_path, monkeypatch):
|
||||
"""Block a model via an admin override, then the blocked list shows it."""
|
||||
conn = _make_db(tmp_path)
|
||||
_seed_models(conn, ("test-model",))
|
||||
conn.execute(
|
||||
"INSERT INTO admin_model_overrides (model_id, provider, availability, "
|
||||
"reason, updated_at) "
|
||||
"VALUES ('test-model', 'neuralwatt', 'blocked', 'manual block', "
|
||||
" datetime('now'))"
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
monkeypatch.setattr(dispatcher.cfg.database, "path", str(tmp_path / "test.db"))
|
||||
monkeypatch.setattr(dispatcher.cfg.verification, "local_llm_enabled", False)
|
||||
monkeypatch.setattr(dispatcher.cfg.routing, "require_vision", False)
|
||||
monkeypatch.setenv("NEURALWATT_API_KEY", "test-key")
|
||||
|
||||
with TestClient(dispatcher.app) as client:
|
||||
resp = client.get("/admin/api/models/blocked")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data) == 1
|
||||
assert data[0]["model_id"] == "test-model"
|
||||
assert data[0]["provider"] == "neuralwatt"
|
||||
assert data[0]["reason"] == "manual block"
|
||||
assert "blocked_since" in data[0]
|
||||
|
||||
|
||||
def test_block_reason_stored(seeded_client):
|
||||
"""POST to availability with a reason stores the reason on the block."""
|
||||
resp = seeded_client.post(
|
||||
"/admin/api/models/cheap/neuralwatt/availability",
|
||||
json={"availability": "blocked", "reason": "looping session test-agent"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
blocked = seeded_client.get("/admin/api/models/blocked").json()
|
||||
assert len(blocked) == 1
|
||||
assert blocked[0]["model_id"] == "cheap"
|
||||
assert blocked[0]["provider"] == "neuralwatt"
|
||||
assert blocked[0]["reason"] == "looping session test-agent"
|
||||
|
||||
|
||||
def test_block_reason_set_from_dropdown(seeded_client):
|
||||
"""Block via the dropdown carries the 'set from Models dropdown' reason."""
|
||||
resp = seeded_client.post(
|
||||
"/admin/api/models/cheap/neuralwatt/availability",
|
||||
json={"availability": "blocked", "reason": "set from Models dropdown"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
blocked = seeded_client.get("/admin/api/models/blocked").json()
|
||||
assert len(blocked) == 1
|
||||
assert blocked[0]["model_id"] == "cheap"
|
||||
assert blocked[0]["reason"] == "set from Models dropdown"
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
import dispatcher
|
||||
from admin import ensure_admin_tables
|
||||
from watchdog_store import ensure_watchdog_tables
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -39,7 +41,149 @@ def test_watchdog_status_empty_db(client):
|
||||
def test_watchdog_loops_empty_db(client):
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == []
|
||||
data = resp.json()
|
||||
assert "alerts" in data
|
||||
assert "resolved" in data
|
||||
assert data["alerts"] == []
|
||||
assert data["resolved"] == []
|
||||
|
||||
|
||||
def _seed_verdict(conn, tick_id, session_id, session_root, agent, model_id,
|
||||
flagged, calls, cost, created_at):
|
||||
conn.execute(
|
||||
"""INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent,
|
||||
model_id, provider, flagged, dup, top, top_what, landed, slow, coverage,
|
||||
calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, 'neuralwatt', ?, 0.9, 1, 'code', 1, 0, 0.8,
|
||||
?, ?, 'no', ?)""",
|
||||
(tick_id, session_id, session_root, agent, model_id,
|
||||
flagged, calls, cost, created_at),
|
||||
)
|
||||
|
||||
|
||||
def test_watchdog_loops_with_verdict_evidence(client):
|
||||
"""Alert joined with latest FLAGGED verdict evidence."""
|
||||
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
||||
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
||||
opened_at, last_fired_at, resolved_at)
|
||||
VALUES ('opencode-loop:root-abc', 'open', 'warning', 1,
|
||||
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
|
||||
)
|
||||
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-1", "model-a",
|
||||
0, 5, 0.5, "2026-09-26T10:00:00+00:00")
|
||||
_seed_verdict(conn, 1, "sess-1", "root-abc", "agent-2", "model-b",
|
||||
1, 10, 1.5, "2026-09-26T10:10:00+00:00")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["alerts"]) == 1
|
||||
|
||||
alert = data["alerts"][0]
|
||||
assert alert["title"] == "agent-2"
|
||||
assert alert["model_id"] == "model-b"
|
||||
assert alert["provider"] == "neuralwatt"
|
||||
assert alert["session_root"] == "root-abc"
|
||||
assert alert["calls_since_landed"] == 10
|
||||
assert alert["cost_since_landed_usd"] == 1.5
|
||||
assert alert["agent_slug"] is None
|
||||
|
||||
|
||||
def test_watchdog_loops_alert_no_matching_verdict(client):
|
||||
"""Alert with no matching verdict returns NULL evidence (no error)."""
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "alerts" in data
|
||||
assert "resolved" in data
|
||||
|
||||
|
||||
def test_watchdog_loops_resolved_last_24h(client):
|
||||
"""Resolved alert within 24h appears in resolved list."""
|
||||
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
resolved_time = (datetime.now(timezone.utc) - timedelta(hours=6)).isoformat()
|
||||
conn.execute(
|
||||
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
||||
opened_at, last_fired_at, resolved_at)
|
||||
VALUES ('opencode-loop:root-xyz', 'resolved', 'critical', 3,
|
||||
'2026-09-26T04:00:00+00:00', '2026-09-26T08:00:00+00:00', ?)""",
|
||||
(resolved_time,),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["resolved"]) == 1
|
||||
assert data["resolved"][0]["dedup_key"] == "opencode-loop:root-xyz"
|
||||
|
||||
|
||||
def test_watchdog_loops_resolved_older_than_24h(client):
|
||||
"""Resolved alert older than 24h does NOT appear in resolved list."""
|
||||
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
resolved_time = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat()
|
||||
conn.execute(
|
||||
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
||||
opened_at, last_fired_at, resolved_at)
|
||||
VALUES ('opencode-loop:root-old', 'resolved', 'critical', 3,
|
||||
'2026-09-23T04:00:00+00:00', '2026-09-23T08:00:00+00:00', ?)""",
|
||||
(resolved_time,),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["resolved"]) == 0
|
||||
|
||||
|
||||
def test_watchdog_loops_agent_slug_from_route_decisions(client):
|
||||
"""Agent slug comes from route_decisions.agent, not watchdog_verdicts."""
|
||||
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
||||
"VALUES ('2026-09-26T10:00:00', 5, 'ok')"
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO watchdog_alerts (dedup_key, state, severity, flagged_ticks,
|
||||
opened_at, last_fired_at, resolved_at)
|
||||
VALUES ('opencode-loop:root-ads', 'open', 'warning', 1,
|
||||
'2026-09-26T10:00:00+00:00', '2026-09-26T10:05:00+00:00', NULL)"""
|
||||
)
|
||||
_seed_verdict(conn, 1, "sess-ads", "root-ads", "test-agent", "model-c",
|
||||
1, 3, 0.25, "2026-09-26T10:00:00+00:00")
|
||||
conn.execute(
|
||||
"""INSERT INTO route_decisions (observed_at, kind, task_category, task_tier,
|
||||
selected_model, selected_provider, session_key, agent, streamed)
|
||||
VALUES ('2026-09-26T10:01:00+00:00', 'dispatch', 'coding_refactor', 2,
|
||||
'model-c', 'neuralwatt', 'c:sess-ads', 'opencode-agent', 0)"""
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp = client.get("/admin/api/watchdog/loops")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["alerts"]) == 1
|
||||
assert data["alerts"][0]["agent_slug"] == "opencode-agent"
|
||||
assert data["alerts"][0]["title"] == "test-agent"
|
||||
|
||||
|
||||
def test_watchdog_channels_empty_db(client):
|
||||
@@ -129,3 +273,95 @@ def test_watchdog_status_after_tick(tmp_path, monkeypatch):
|
||||
data = resp.json()
|
||||
assert data["last_tick"] is not None
|
||||
assert data["last_tick"]["sessions_seen"] == 5
|
||||
|
||||
|
||||
def test_watchdog_rollup_empty(client):
|
||||
"""Empty watchdog_verdicts returns []."""
|
||||
resp = client.get("/admin/api/watchdog/rollup?hours=24")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == []
|
||||
|
||||
|
||||
def test_watchdog_rollup_invalid_hours(client):
|
||||
"""hours=99 returns 422."""
|
||||
resp = client.get("/admin/api/watchdog/rollup?hours=99")
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
def test_watchdog_rollup_with_data(client):
|
||||
"""2 flagged verdicts for same session (costs 1.0 and 3.0), 1 flagged for
|
||||
another session same model. Expect stalled_sessions=2, stalled_usd=4.0
|
||||
(MAX 3.0 + MAX 1.0)."""
|
||||
conn = sqlite3.connect(str(dispatcher.cfg.database.path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
ensure_admin_tables(conn)
|
||||
|
||||
# Seed a model row
|
||||
conn.execute(
|
||||
"INSERT INTO models (model_id, provider, base_model_id, display_name, tier, "
|
||||
"context_window, effective_context_window, max_output_tokens, "
|
||||
"cost_per_1m_prompt, cost_per_1m_completion, access_level, latency_class, "
|
||||
"availability, last_updated) "
|
||||
"VALUES ('model-a', 'neuralwatt', 'model-a', 'Model A', 2, "
|
||||
"128000, 128000, 8192, 0.5, 1.5, 'public', 'interactive', "
|
||||
"'active', datetime('now'))"
|
||||
)
|
||||
|
||||
# Need a tick for the FK
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_ticks (ticked_at, sessions_seen, outcome) "
|
||||
"VALUES (datetime('now'), 5, 'flagged')"
|
||||
)
|
||||
|
||||
# First session (root-1): two verdicts, costs 1.0 and 3.0
|
||||
now = datetime.now(timezone.utc)
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
||||
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
||||
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
||||
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
|
||||
"1, 0.9, 1, 'code', 1, 0, 0.8, 5, 1.0, 'no', ?)",
|
||||
(now.isoformat(),)
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
||||
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
||||
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
||||
"VALUES (1, 'sess-1a', 'root-1', 'agent', 'model-a', 'neuralwatt', "
|
||||
"1, 0.9, 1, 'code', 1, 0, 0.8, 10, 3.0, 'no', ?)",
|
||||
(now.isoformat(),)
|
||||
)
|
||||
|
||||
# Second session (root-2): one verdict, cost 1.0
|
||||
conn.execute(
|
||||
"INSERT INTO watchdog_verdicts (tick_id, session_id, session_root, agent, "
|
||||
"model_id, provider, flagged, dup, top, top_what, landed, slow, coverage, "
|
||||
"calls_since_landed, cost_since_landed_usd, llm_second_opinion, created_at) "
|
||||
"VALUES (1, 'sess-2', 'root-2', 'agent', 'model-a', 'neuralwatt', "
|
||||
"1, 0.9, 1, 'code', 1, 0, 0.8, 3, 1.0, 'no', ?)",
|
||||
(now.isoformat(),)
|
||||
)
|
||||
|
||||
# Seed a route_decision for traffic_share
|
||||
conn.execute(
|
||||
"INSERT INTO route_decisions (observed_at, kind, task_category, task_tier, "
|
||||
"selected_model, selected_provider, session_key, agent, streamed) "
|
||||
"VALUES (?, 'dispatch', 'coding_refactor', 2, "
|
||||
"'model-a', 'neuralwatt', 'c:sess-1a', 'opencode-agent', 0)",
|
||||
(now.isoformat(),)
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp = client.get("/admin/api/watchdog/rollup?hours=24")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data) == 1
|
||||
assert data[0]["model_id"] == "model-a"
|
||||
assert data[0]["provider"] == "neuralwatt"
|
||||
assert data[0]["stalled_sessions"] == 2 # root-1 and root-2
|
||||
assert data[0]["stalled_usd"] == 4.0 # MAX(3.0, 1.0) + 1.0
|
||||
assert data[0]["traffic_share"] == 1.0 # 1/1 route_decisions
|
||||
assert data[0]["effective_availability"] == "active"
|
||||
|
||||
@@ -1198,3 +1198,191 @@ class TestMainExitCode:
|
||||
rc = watchdog.main(["--once", "--config", "unused.yaml"])
|
||||
assert fake_tick.called
|
||||
assert rc == 0
|
||||
|
||||
|
||||
class TestAlertSummaryEvidence:
|
||||
"""Trigger/escalate alert summaries carry calls, cost, and model evidence.
|
||||
|
||||
The session loops (60 duplicate reads after a landed change), so the
|
||||
summary must report calls since the landed change, its cost, and the
|
||||
attributed model — and omit the 'on <model>' segment when no model is
|
||||
attributed.
|
||||
"""
|
||||
|
||||
def _run_flagged_tick(self, conn, path, attribution, cost_after_landed):
|
||||
"""Run one tick against a loop-flagged worker session; return events."""
|
||||
cfg = _make_cfg()
|
||||
cfg.database = MagicMock()
|
||||
cfg.database.path = path
|
||||
cfg.database.base_url = "http://localhost:4096"
|
||||
cfg.watchdog.dashboard_base_url = "http://dash/"
|
||||
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS route_decisions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_key TEXT,
|
||||
selected_model TEXT,
|
||||
selected_provider TEXT,
|
||||
est_cost_usd REAL DEFAULT 0,
|
||||
observed_at TEXT
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
landed_ts = now.timestamp() * 1000 # ms
|
||||
|
||||
# Worker: 60 identical reads (no landed) -> flags; each is after the
|
||||
# tree's last landed time so calls_since_landed is 60.
|
||||
worker_msgs = [{
|
||||
"id": f"wm{i}", "parts": [{
|
||||
"type": "tool", "tool": "read",
|
||||
"state": {
|
||||
"input": {"filePath": "/x.py"},
|
||||
"metadata": {},
|
||||
"time": {"start": landed_ts + 1 + i},
|
||||
},
|
||||
}],
|
||||
"info": {"time": {"created": landed_ts + 1 + i}},
|
||||
} for i in range(60)]
|
||||
|
||||
# Sibling: one landed edit at landed_ts -> tree_last_landed is set.
|
||||
sibling_msgs = [{
|
||||
"id": "sm0", "parts": [{
|
||||
"type": "tool", "tool": "edit",
|
||||
"state": {
|
||||
"input": {"filePath": "/y.py"},
|
||||
"metadata": {"diff": "patch"},
|
||||
"time": {"start": landed_ts},
|
||||
},
|
||||
}],
|
||||
"info": {"time": {"created": landed_ts}},
|
||||
}]
|
||||
|
||||
# Parent: one benign call so it shows as active (not flagged).
|
||||
parent_msgs = [{
|
||||
"id": "pm0", "parts": [{
|
||||
"type": "tool", "tool": "read",
|
||||
"state": {
|
||||
"input": {"filePath": "/z.py"},
|
||||
"metadata": {},
|
||||
"time": {"start": landed_ts + 200},
|
||||
},
|
||||
}],
|
||||
"info": {"time": {"created": landed_ts + 200}},
|
||||
}]
|
||||
|
||||
sessions = [
|
||||
{"id": "ses_p", "title": "parent",
|
||||
"parentID": None,
|
||||
"time": {"created": landed_ts},
|
||||
"info": {"time": {"created": landed_ts}}},
|
||||
{"id": "ses_worker", "title": "worker",
|
||||
"parentID": "ses_p",
|
||||
"time": {"created": landed_ts},
|
||||
"info": {"time": {"created": landed_ts}}},
|
||||
{"id": "ses_sibling", "title": "sibling",
|
||||
"parentID": "ses_p",
|
||||
"time": {"created": landed_ts},
|
||||
"info": {"time": {"created": landed_ts}}},
|
||||
]
|
||||
|
||||
if cost_after_landed is not None:
|
||||
conn.execute(
|
||||
"INSERT INTO route_decisions (session_key, selected_model, "
|
||||
"selected_provider, est_cost_usd, observed_at) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
("c:ses_worker", "gpt-4o", "neuralwatt", cost_after_landed,
|
||||
datetime.fromtimestamp((landed_ts + 1) / 1000,
|
||||
tz=timezone.utc).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
events = []
|
||||
|
||||
class _Recorder:
|
||||
def deliver(self, event):
|
||||
events.append(event)
|
||||
|
||||
def fake_http(url, **kw):
|
||||
if "message" in url:
|
||||
if "ses_worker" in url:
|
||||
return worker_msgs
|
||||
if "ses_sibling" in url:
|
||||
return sibling_msgs
|
||||
if "ses_p" in url:
|
||||
return parent_msgs
|
||||
return []
|
||||
return sessions
|
||||
|
||||
with patch("watchdog._read_rc_servers",
|
||||
return_value=["http://localhost:4096"]), \
|
||||
patch("watchdog._fetch_sessions", return_value=sessions), \
|
||||
patch("watchdog._attribution", return_value=attribution):
|
||||
tick(cfg, conn, notifier=_Recorder(), http_get=fake_http)
|
||||
|
||||
return events
|
||||
|
||||
def _worker_event(self, events, state):
|
||||
"""Return the event for the worker session of the given state."""
|
||||
for e in events:
|
||||
if e.state == state and e.title == "opencode loop: worker":
|
||||
return e
|
||||
return None
|
||||
|
||||
def test_alert_summary_trigger_includes_calls_cost_model(self):
|
||||
"""Trigger summary lists calls, cost, and model since last landed change."""
|
||||
conn, path = _make_conns()
|
||||
try:
|
||||
events = self._run_flagged_tick(
|
||||
conn, path,
|
||||
attribution=("gpt-4o", "neuralwatt", 0.5, 10),
|
||||
cost_after_landed=1.25,
|
||||
)
|
||||
trigger = self._worker_event(events, "trigger")
|
||||
assert trigger, "Expected a worker trigger event"
|
||||
summary = trigger.summary
|
||||
assert "60 calls" in summary, f"calls missing: {summary}"
|
||||
assert "$1.25" in summary, f"cost missing: {summary}"
|
||||
assert "on gpt-4o" in summary, f"model missing: {summary}"
|
||||
assert "#loops" in summary, f"loops link missing: {summary}"
|
||||
finally:
|
||||
_close_conn(conn, path)
|
||||
|
||||
def test_alert_summary_trigger_omits_model_when_null(self):
|
||||
"""No 'on <model>' segment when model_id is NULL."""
|
||||
conn, path = _make_conns()
|
||||
try:
|
||||
events = self._run_flagged_tick(
|
||||
conn, path,
|
||||
attribution=(None, None, 0.0, 0),
|
||||
cost_after_landed=1.25,
|
||||
)
|
||||
trigger = self._worker_event(events, "trigger")
|
||||
assert trigger, "Expected a worker trigger event"
|
||||
summary = trigger.summary
|
||||
assert "60 calls" in summary, f"calls missing: {summary}"
|
||||
assert "on " not in summary, f"'on <model>' should be omitted: {summary}"
|
||||
finally:
|
||||
_close_conn(conn, path)
|
||||
|
||||
def test_alert_summary_escalate_includes_calls_cost_model(self):
|
||||
"""Escalate summary (3rd tick) carries the same calls/cost/model evidence."""
|
||||
conn, path = _make_conns()
|
||||
try:
|
||||
events = []
|
||||
for _ in range(3):
|
||||
events += self._run_flagged_tick(
|
||||
conn, path,
|
||||
attribution=("gpt-4o", "neuralwatt", 0.5, 10),
|
||||
cost_after_landed=1.25,
|
||||
)
|
||||
escalate = self._worker_event(events, "escalate")
|
||||
assert escalate, "Expected a worker escalate event"
|
||||
summary = escalate.summary
|
||||
assert "60 calls" in summary, f"calls missing: {summary}"
|
||||
assert "$1.25" in summary, f"cost missing: {summary}"
|
||||
assert "on gpt-4o" in summary, f"model missing: {summary}"
|
||||
assert "#loops" in summary, f"loops link missing: {summary}"
|
||||
finally:
|
||||
_close_conn(conn, path)
|
||||
|
||||
Reference in New Issue
Block a user