640 lines
35 KiB
HTML
640 lines
35 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en" data-bs-theme="dark">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Proficiency — LLM Router Admin</title>
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js"></script>
|
|
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns@3.0.0/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
|
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@tabler/core@1.4.0/dist/css/tabler.min.css">
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Quicksand:wght@600;700&display=swap">
|
|
<link rel="icon" href="data:image/svg+xml,%3Csvg%20xmlns%3D%22http%3A//www.w3.org/2000/svg%22%20viewBox%3D%220%200%20256%20256%22%3E%3Ccircle%20cx%3D%22146%22%20cy%3D%2284%22%20r%3D%2245.6%22%20fill%3D%22%23fff%22/%3E%3Cpath%20d%3D%22M124%20110%20Q90%20140%2090%20176%20Q90%20212%20122%20212%20Q156%20212%20172%20186%22%20fill%3D%22none%22%20stroke%3D%22%23fff%22%20stroke-width%3D%2239.2%22%20stroke-linecap%3D%22round%22/%3E%3Ccircle%20cx%3D%22146%22%20cy%3D%2284%22%20r%3D%2242%22%20fill%3D%22%23f59e0b%22/%3E%3Cpath%20d%3D%22M124%20110%20Q90%20140%2090%20176%20Q90%20212%20122%20212%20Q156%20212%20172%20186%22%20fill%3D%22none%22%20stroke%3D%22%23f59e0b%22%20stroke-width%3D%2232%22%20stroke-linecap%3D%22round%22/%3E%3Ccircle%20cx%3D%22128%22%20cy%3D%2278%22%20r%3D%229%22%20fill%3D%22%23fff%22/%3E%3Ccircle%20cx%3D%22164%22%20cy%3D%2278%22%20r%3D%229%22%20fill%3D%22%23fff%22/%3E%3Ccircle%20cx%3D%22128%22%20cy%3D%2278%22%20r%3D%226%22%20fill%3D%22%231e293b%22/%3E%3Ccircle%20cx%3D%22164%22%20cy%3D%2278%22%20r%3D%226%22%20fill%3D%22%231e293b%22/%3E%3Ccircle%20cx%3D%22131%22%20cy%3D%2274%22%20r%3D%222.2%22%20fill%3D%22%23fff%22/%3E%3Ccircle%20cx%3D%22167%22%20cy%3D%2274%22%20r%3D%222.2%22%20fill%3D%22%23fff%22/%3E%3Cpath%20d%3D%22M134%20100%20Q146%20112%20158%20100%22%20fill%3D%22none%22%20stroke%3D%22%23fff%22%20stroke-width%3D%227%22%20stroke-linecap%3D%22round%22/%3E%3Cpath%20d%3D%22M134%20100%20Q146%20112%20158%20100%22%20fill%3D%22none%22%20stroke%3D%22%231e293b%22%20stroke-width%3D%223.5%22%20stroke-linecap%3D%22round%22/%3E%3Cline%20x1%3D%22150.47%22%20y1%3D%2247.59%22%20x2%3D%22156.01%22%20y2%3D%2227.13%22%20stroke%3D%22%23fff%22%20stroke-width%3D%2210.73%22%20stroke-linecap%3D%22round%22/%3E%3Cline%20x1%3D%22150.47%22%20y1%3D%2247.59%22%20x2%3D%22155.08%22%20y2%3D%2230.54%22%20stroke%3D%22%231e293b%22%20stroke-width%3D%226.26%22%20stroke-linecap%3D%22round%22/%3E%3Ccircle%20cx%3D%2295%22%20cy%3D%22226%22%20r%3D%2214%22%20fill%3D%22%23fff%22/%3E%3Ccircle%20cx%3D%22150%22%20cy%3D%22226%22%20r%3D%2214%22%20fill%3D%22%23fff%22/%3E%3Ccircle%20cx%3D%2295%22%20cy%3D%22226%22%20r%3D%229%22%20fill%3D%22%231e293b%22/%3E%3Ccircle%20cx%3D%22150%22%20cy%3D%22226%22%20r%3D%229%22%20fill%3D%22%231e293b%22/%3E%3C/svg%3E">
|
|
<script src="https://cdn.jsdelivr.net/npm/@tabler/core@1.4.0/dist/js/tabler.min.js"></script>
|
|
<link rel="stylesheet" href="navbar.css">
|
|
<script src="navbar.js"></script>
|
|
<style>
|
|
/* ── Liquid glass: soft gradient wash behind everything so the blur has
|
|
something to catch, then translucent+blurred navbar/cards on top. Kept
|
|
subtle — this is a working dashboard, not a marketing page. ── */
|
|
body{
|
|
background:
|
|
radial-gradient(900px circle at 6% -8%, rgba(245,158,11,.10), transparent 55%),
|
|
radial-gradient(800px circle at 94% -6%, rgba(59,130,246,.10), transparent 50%),
|
|
radial-gradient(760px circle at 50% 108%, rgba(139,92,246,.07), transparent 55%),
|
|
#111827;
|
|
background-attachment: fixed;
|
|
}
|
|
header.navbar{
|
|
background: rgba(31,41,55,.6)!important;
|
|
backdrop-filter: blur(16px) saturate(160%);
|
|
-webkit-backdrop-filter: blur(16px) saturate(160%);
|
|
border-bottom: 1px solid rgba(255,255,255,.06);
|
|
position: relative;
|
|
z-index: 1030;
|
|
}
|
|
.card{
|
|
background: rgba(31,41,55,.55);
|
|
backdrop-filter: blur(18px) saturate(140%);
|
|
-webkit-backdrop-filter: blur(18px) saturate(140%);
|
|
border: 1px solid rgba(255,255,255,.07);
|
|
box-shadow: 0 10px 30px -14px rgba(0,0,0,.55), inset 0 1px 0 rgba(255,255,255,.04);
|
|
}
|
|
.card-header{border-bottom-color: rgba(255,255,255,.06)}
|
|
/* ── Scroll context: body scrolls, html does not. ── */
|
|
html{overflow:hidden;height:100%;margin:0!important;padding:0!important}
|
|
body{overflow-y:auto;height:100%;margin:0!important;padding:0!important}
|
|
/* ── Buttons: glass pills instead of flat saturated fills. ── */
|
|
.btn-success{background:rgba(34,197,94,.16)!important;border-color:rgba(74,222,128,.45)!important;color:#4ade80!important}
|
|
.btn-success:hover{background:rgba(34,197,94,.28)!important;border-color:rgba(74,222,128,.7)!important;color:#86efac!important}
|
|
.btn-danger{background:rgba(239,68,68,.16)!important;border-color:rgba(248,113,113,.45)!important;color:#f87171!important}
|
|
.btn-danger:hover{background:rgba(239,68,68,.28)!important;border-color:rgba(248,113,113,.7)!important;color:#fca5a5!important}
|
|
.btn{backdrop-filter:blur(6px);-webkit-backdrop-filter:blur(6px)}
|
|
/* ── Badges: same glass treatment. ── */
|
|
.badge.bg-success{background:rgba(34,197,94,.20)!important;color:#4ade80!important}
|
|
.badge.bg-warning{background:rgba(245,158,11,.20)!important;color:#fbbf24!important}
|
|
.badge.bg-danger{background:rgba(239,68,68,.20)!important;color:#f87171!important}
|
|
.badge.bg-info{background:rgba(59,130,246,.20)!important;color:#93c5fd!important}
|
|
.badge.bg-secondary{background:rgba(148,163,184,.20)!important;color:#e2e8f0!important}
|
|
.badge.bg-purple{background:rgba(168,85,247,.20)!important;color:#d8b4fe!important}
|
|
header.navbar>.container-fluid{padding-left:20px!important;padding-right:20px!important}
|
|
.navbar-brand, .page-title{font-family:'Quicksand',var(--tblr-font-sans-serif,ui-sans-serif,system-ui,sans-serif);font-weight:700;letter-spacing:.01em}
|
|
.navbar-brand a{text-decoration:none!important;color:inherit}
|
|
/* ── Brand mascot sizing and filter guard. ── */
|
|
.navbar-brand-autodark .navbar-brand-image{filter:none!important;height:64px;width:64px}
|
|
.navbar-brand{padding-top:0!important;padding-bottom:0!important}
|
|
header.navbar{padding-top:2px!important;padding-bottom:2px!important}
|
|
/* ── Speed lines. ── */
|
|
.navbar-brand a{display:inline-flex;align-items:center;gap:.15rem}
|
|
.brand-speedlines{align-items:center;line-height:0;margin-right:-20px}
|
|
.brand-speedlines svg{filter:none!important}
|
|
@keyframes brand-dash{
|
|
0%{transform:translateX(-5px);opacity:.2}
|
|
55%{opacity:1}
|
|
100%{transform:translateX(4px);opacity:0}
|
|
}
|
|
.navbar-brand a:hover .brand-speedlines line{animation:brand-dash .75s ease-in infinite}
|
|
.navbar-brand a:hover .brand-speedlines line:nth-child(2){animation-delay:.12s}
|
|
.navbar-brand a:hover .brand-speedlines line:nth-child(3){animation-delay:.24s}
|
|
@media (prefers-reduced-motion: reduce){
|
|
.navbar-brand a:hover .brand-speedlines line{animation:none}
|
|
}
|
|
/* ── Proficiency matrix specifics ─────────────────────────────────────────
|
|
The one requirement this page exists to satisfy: a score that traffic
|
|
EARNED must not look like one that was copied in. Hue carries provenance
|
|
(green = measured from outcomes, amber = inherited prior, slate = thin
|
|
self-eval), and a corner wedge carries "inherited" independently, because
|
|
a row can be inherited without being thin and thin without being
|
|
inherited. Value is NOT encoded in colour — a heatmap of blended_score
|
|
would put the eye on the number and hide exactly the distinction that
|
|
matters. ────────────────────────────────────────────────────────────── */
|
|
.matrix-wrap{overflow-x:auto}
|
|
table.matrix{border-collapse:separate;border-spacing:3px;font-size:.78rem;min-width:100%}
|
|
table.matrix th{font-weight:600;color:var(--tblr-secondary);font-size:.7rem;text-align:center;padding:2px 6px;white-space:nowrap}
|
|
table.matrix th.model-head{text-align:left;position:sticky;left:0;background:rgba(31,41,55,.92);z-index:2;padding-right:12px}
|
|
table.matrix thead th{vertical-align:bottom}
|
|
table.matrix td{padding:0}
|
|
.cell{
|
|
position:relative;display:block;min-width:66px;padding:7px 6px;border-radius:6px;
|
|
text-align:center;font-variant-numeric:tabular-nums;line-height:1.15;
|
|
border:1px solid transparent;cursor:default;
|
|
}
|
|
.cell .score{font-weight:700;font-size:.86rem;display:block}
|
|
.cell .n{font-size:.63rem;opacity:.75;display:block;margin-top:1px}
|
|
.cell-measured{background:rgba(34,197,94,.16);border-color:rgba(74,222,128,.38);color:#86efac}
|
|
.cell-prior{background:rgba(245,158,11,.14);border-color:rgba(251,191,36,.34);color:#fcd34d}
|
|
.cell-selfeval{background:rgba(59,130,246,.13);border-color:rgba(147,197,253,.30);color:#bfdbfe}
|
|
.cell-thin{background:rgba(148,163,184,.10);border-color:rgba(148,163,184,.26);color:#cbd5e1}
|
|
.cell-missing{background:rgba(148,163,184,.05);border:1px dashed rgba(148,163,184,.18);color:rgba(148,163,184,.45)}
|
|
/* Inherited: a corner wedge, readable in the same glance as the hue and
|
|
independent of it. */
|
|
.cell-inherited::after{
|
|
content:"";position:absolute;top:0;right:0;width:0;height:0;
|
|
border-top:9px solid rgba(251,191,36,.85);border-left:9px solid transparent;
|
|
border-top-right-radius:6px;
|
|
}
|
|
.cell-dim{opacity:.24}
|
|
.prov-legend{display:flex;flex-wrap:wrap;gap:14px;font-size:.72rem;color:var(--tblr-secondary);align-items:center}
|
|
.prov-legend .prov-swatch{display:inline-block;width:13px;height:13px;border-radius:3px;vertical-align:-2px;margin-right:5px;border:1px solid rgba(255,255,255,.14)}
|
|
.prov-legend-item{white-space:nowrap}
|
|
.matrix-model{font-weight:600;color:var(--tblr-body-color)}
|
|
.matrix-provider{font-size:.66rem;color:var(--tblr-secondary);display:block}
|
|
.empty{color:var(--tblr-secondary);text-align:center;padding:2rem 1rem}
|
|
.filter-bar{display:flex;flex-wrap:wrap;gap:10px;align-items:center}
|
|
.filter-bar select,.filter-bar input{max-width:220px}
|
|
.outcome-table{font-size:.76rem}
|
|
.outcome-table td,.outcome-table th{padding:5px 8px;white-space:nowrap}
|
|
/* Details are agent shell lines — long, unbroken, and happy to push the table
|
|
past the viewport. Bound the column AND break inside words; max-width alone
|
|
does nothing to an unbreakable string. */
|
|
.outcome-detail{white-space:normal;max-width:420px;overflow-wrap:anywhere;color:var(--tblr-secondary)}
|
|
.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.72rem}
|
|
/* Footer parts are laid out, not joined by a glyph. See
|
|
plans/admin-design-standards.md, "No middot separators". */
|
|
.footer-bits{display:inline-flex;flex-wrap:wrap;gap:.25rem 1.4rem}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
|
|
|
|
<!-- ═══ Navbar ═══ -->
|
|
<header class="navbar navbar-expand-md navbar-dark" data-bs-theme="dark">
|
|
<div class="container-fluid">
|
|
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar-menu" aria-controls="navbar-menu" aria-expanded="false" aria-label="Toggle navigation"><span class="navbar-toggler-icon"></span></button>
|
|
<h1 class="navbar-brand navbar-brand-autodark d-none-navbar-horizontal pe-0 pe-md-3">
|
|
<a href="/admin/" style="display:inline-flex;align-items:center;gap:8px">
|
|
<span class="brand-speedlines d-none d-md-flex" aria-hidden="true">
|
|
<svg viewBox="0 0 40 64" width="40" height="64" xmlns="http://www.w3.org/2000/svg">
|
|
<defs>
|
|
<linearGradient id="speedfade" gradientUnits="userSpaceOnUse" x1="0" y1="0" x2="40" y2="0">
|
|
<stop offset="0" stop-color="#f59e0b" stop-opacity="0"/>
|
|
<stop offset="1" stop-color="#f59e0b" stop-opacity=".85"/>
|
|
</linearGradient>
|
|
</defs>
|
|
<g stroke="url(#speedfade)" stroke-linecap="round" fill="none">
|
|
<line x1="14" y1="20" x2="38" y2="20" stroke-width="5"/>
|
|
<line x1="2" y1="33" x2="36" y2="33" stroke-width="6"/>
|
|
<line x1="18" y1="46" x2="34" y2="46" stroke-width="4.5"/>
|
|
</g>
|
|
</svg>
|
|
</span>
|
|
<img class="navbar-brand-image" src="6krrt-logo.webp" width="64" height="64" style="width:64px;height:64px" alt="6krrt logo">
|
|
</a>
|
|
</h1>
|
|
<div class="navbar-nav flex-row order-md-last">
|
|
<div class="d-flex align-items-center gap-3">
|
|
<span id="nav-widgets" class="d-flex align-items-center"></span>
|
|
</div>
|
|
</div>
|
|
<div class="collapse navbar-collapse" id="navbar-menu">
|
|
<div class="d-flex flex-column flex-md-row flex-fill align-items-stretch align-items-md-center">
|
|
<ul class="navbar-nav" id="nav-links"></ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</header>
|
|
|
|
<!-- ═══ Page shell ═══ -->
|
|
<div class="page">
|
|
<div class="page-wrapper">
|
|
<div class="page-header d-print-none">
|
|
<div class="container-fluid">
|
|
<div class="row g-2 align-items-center">
|
|
<div class="col">
|
|
<h2 class="page-title">Proficiency</h2>
|
|
<div class="text-muted mt-1">Expected pass rate per model and task category — the only category-dependent term in the ranking</div>
|
|
<div class="text-muted mt-1" style="font-size:.75rem"><span data-icon="lock" style="display:inline-flex;vertical-align:middle;margin-right:4px"></span>Read-only. Proficiency is derived from the eval harness and from client outcomes; a hand-edited score would be a fabricated measurement.</div>
|
|
</div>
|
|
<div class="col-auto">
|
|
<span id="row-count" class="badge bg-secondary">—</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="page-body">
|
|
<div class="container-fluid">
|
|
|
|
<div class="row row-cards">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-block">
|
|
<div class="d-flex align-items-center justify-content-between flex-wrap gap-2">
|
|
<h3 class="card-title mb-0"><span class="me-2" data-icon="grid"></span>Model × category</h3>
|
|
<div class="filter-bar">
|
|
<select class="form-select form-select-sm" id="filter-category" onchange="render()">
|
|
<option value="">all categories</option>
|
|
</select>
|
|
<select class="form-select form-select-sm" id="filter-source" onchange="render()">
|
|
<option value="">all sources</option>
|
|
</select>
|
|
<input class="form-control form-control-sm" id="filter-model" type="search" placeholder="filter models…" oninput="render()" autocomplete="off">
|
|
<select class="form-select form-select-sm" id="sort-by" onchange="render()">
|
|
<option value="model">sort: model</option>
|
|
<option value="score">sort: mean score</option>
|
|
<option value="samples">sort: samples</option>
|
|
</select>
|
|
<div class="form-check form-switch mb-0">
|
|
<input class="form-check-input" type="checkbox" id="only-measured" onchange="render()">
|
|
<label class="form-check-label" for="only-measured" style="font-size:.74rem">measured only</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="prov-legend mt-2" id="legend"></div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="matrix-wrap" id="matrix"><div class="empty">Loading…</div></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-block">
|
|
<h3 class="card-title mb-0"><span class="me-2" data-icon="check"></span>Client outcomes</h3>
|
|
<div class="text-muted mt-1" style="font-size:.75rem">
|
|
<code>POST /outcome</code> is the only ground truth the router gets; everything else is a proxy.
|
|
A row marked <strong>excluded</strong> was recorded but is not folded into proficiency —
|
|
a degraded classification, or a failure the model did not cause. Silently dropping those is
|
|
the “recorded but never surfaced” pattern this portal exists to break.
|
|
</div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="matrix-wrap" id="outcomes"><div class="empty">Loading…</div></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div><!-- /container-fluid -->
|
|
</div><!-- /page-body -->
|
|
<footer class="footer footer-transparent d-print-none">
|
|
<div class="container-fluid">
|
|
<span class="footer-bits"><span>© 2026 adLee</span><span>6krrt, local LLM model router</span><span>admin</span></span>
|
|
</div>
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- ═══ Cell detail modal ═══ -->
|
|
<div class="modal modal-blur fade" id="cell-modal" tabindex="-1" role="dialog" aria-hidden="true">
|
|
<div class="modal-dialog modal-dialog-centered" role="document">
|
|
<div class="modal-content">
|
|
<div class="modal-header">
|
|
<h5 class="modal-title" id="cell-modal-title">—</h5>
|
|
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
</div>
|
|
<div class="modal-body" id="cell-modal-body"></div>
|
|
<div class="modal-footer">
|
|
<button type="button" class="btn btn-outline-secondary btn-sm" data-bs-dismiss="modal">Close</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div id="toast" class="toast align-items-center text-bg-info border-0 position-fixed bottom-0 end-0 p-3" role="alert" aria-live="assertive" aria-atomic="true"></div>
|
|
|
|
<script>
|
|
/* ═══════════════════════════════════════════════════
|
|
admin/frontend/proficiency.html — the routing table
|
|
API base: relative (works under /admin/)
|
|
═══════════════════════════════════════════════════ */
|
|
|
|
const API = '';
|
|
const SSE_URL = '/events/decisions';
|
|
|
|
function icon(name, size) {
|
|
const s = size || 18;
|
|
const svgs = {
|
|
grid: `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="7" height="7"/><rect x="14" y="3" width="7" height="7"/><rect x="14" y="14" width="7" height="7"/><rect x="3" y="14" width="7" height="7"/></svg>`,
|
|
check: `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>`,
|
|
lock: `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="11" width="18" height="11" rx="2" ry="2"/><path d="M7 11V7a5 5 0 0 1 10 0v4"/></svg>`,
|
|
default:`<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="1"/></svg>`,
|
|
};
|
|
return svgs[name] || svgs.default;
|
|
}
|
|
|
|
function renderStaticIcons() {
|
|
document.querySelectorAll('[data-icon]').forEach(el => { el.innerHTML = icon(el.dataset.icon); });
|
|
}
|
|
|
|
function escapeHtml(s) {
|
|
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
|
}
|
|
|
|
/* Detail strings are whole agent shell invocations. The full text stays in the
|
|
title attribute; the cell shows enough to recognise the report. */
|
|
function truncate(s, n) {
|
|
const t = String(s == null ? '' : s);
|
|
return t.length <= n ? t : `${t.slice(0, n - 1)}…`;
|
|
}
|
|
|
|
let _toastEl = null;
|
|
function toast(msg, type = 'info') {
|
|
const bgClass = { success:'text-bg-success', error:'text-bg-danger', warning:'text-bg-warning', info:'text-bg-info' }[type] || 'text-bg-info';
|
|
if (!_toastEl) {
|
|
_toastEl = document.getElementById('toast');
|
|
_toastEl.innerHTML = `<div class="d-flex"><div class="toast-body"></div>
|
|
<button type="button" class="btn-close btn-close-white me-2 m-auto" data-bs-dismiss="toast" aria-label="Close"></button></div>`;
|
|
_toastEl._bt = new tabler.Toast(_toastEl, { delay: 4000 });
|
|
}
|
|
_toastEl.querySelector('.toast-body').textContent = msg;
|
|
_toastEl.className = `toast align-items-center ${bgClass} border-0 position-fixed bottom-0 end-0 p-3`;
|
|
_toastEl._bt.show();
|
|
}
|
|
|
|
async function apiFetch(url, opts = {}) {
|
|
try {
|
|
const resp = await fetch(url, opts);
|
|
if (!resp.ok) throw new Error(`${resp.status} ${resp.statusText}`);
|
|
return await resp.json();
|
|
} catch (e) {
|
|
console.warn('API call failed:', url, e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
SSE LIVE STREAM (status chip only)
|
|
═══════════════════════════════════════ */
|
|
let sseConn = null;
|
|
function connectSSE() {
|
|
try {
|
|
if (sseConn) sseConn.close();
|
|
sseConn = new EventSource(SSE_URL);
|
|
navbarSse('connecting');
|
|
sseConn.onopen = () => { navbarSse('live'); };
|
|
sseConn.onerror = () => { navbarSse('offline', 'reconnecting'); };
|
|
} catch (e) {
|
|
navbarSse('offline', String(e));
|
|
}
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
PROVENANCE — the whole point of the page
|
|
═══════════════════════════════════════
|
|
|
|
An inherited prior and a measured score are the same number in the same
|
|
column, and routing weights them identically. The operator's job is to
|
|
know which is which, so provenance decides the cell's class and nothing
|
|
else does. Value is left to the digits.
|
|
|
|
The source strings come from proficiency_store / proficiency_outcome:
|
|
outcome_blended fresh per-model outcome evidence — measured here
|
|
outcome_prior a trafficked sibling's peer rate — inherited
|
|
self_eval / self_eval_thin the 43-task benchmark — no traffic yet
|
|
*/
|
|
const SOURCE_CLASS = {
|
|
outcome_blended: 'cell-measured',
|
|
outcome_prior: 'cell-prior',
|
|
self_eval: 'cell-selfeval',
|
|
self_eval_thin: 'cell-thin',
|
|
leaderboard: 'cell-thin',
|
|
blended: 'cell-measured',
|
|
};
|
|
|
|
const SOURCE_LABEL = {
|
|
outcome_blended: 'measured from client outcomes',
|
|
outcome_prior: 'inherited prior from a trafficked sibling',
|
|
self_eval: 'benchmark, past the sample floor',
|
|
self_eval_thin: 'benchmark, below the sample floor',
|
|
leaderboard: 'external leaderboard prior',
|
|
blended: 'blended',
|
|
};
|
|
|
|
function cellClass(row) {
|
|
// Inheritance OUTRANKS source, and that ordering is the point of the page.
|
|
// A -flex row carries its family's number verbatim, including source =
|
|
// 'outcome_blended' — the outcomes were real, they were just somebody
|
|
// else's. Colouring that green because the source column says so would
|
|
// paint a copy as a measurement, which is exactly the failure this page
|
|
// exists to make visible. The wedge stays as well: two marks for the claim
|
|
// that matters most beats one.
|
|
const base = row.inherited ? 'cell-prior' : (SOURCE_CLASS[row.source] || 'cell-thin');
|
|
const marks = [base];
|
|
if (row.inherited) marks.push('cell-inherited');
|
|
return marks.join(' ');
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
STATE
|
|
═══════════════════════════════════════ */
|
|
let DATA = null;
|
|
let INDEX = new Map(); // "model|provider|category" -> row
|
|
|
|
function keyOf(model, provider, category) { return `${model}|${provider}|${category}`; }
|
|
|
|
function identities() {
|
|
const seen = new Map();
|
|
for (const r of DATA.rows) {
|
|
const k = `${r.model_id}|${r.provider}`;
|
|
if (!seen.has(k)) seen.set(k, { model_id: r.model_id, provider: r.provider, rows: [] });
|
|
seen.get(k).rows.push(r);
|
|
}
|
|
return Array.from(seen.values());
|
|
}
|
|
|
|
function meanScore(rows) {
|
|
const vals = rows.map(r => r.blended_score).filter(v => v != null);
|
|
if (!vals.length) return null;
|
|
return vals.reduce((a, b) => a + b, 0) / vals.length;
|
|
}
|
|
|
|
function totalSamples(rows) {
|
|
return rows.reduce((a, r) => a + (r.total_samples || 0), 0);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
RENDER
|
|
═══════════════════════════════════════ */
|
|
|
|
function renderLegend() {
|
|
const min = DATA.self_eval_min_samples;
|
|
const counts = DATA.source_counts || {};
|
|
// Counted off cellClass rather than off source, so the legend can never
|
|
// disagree with the grid it describes — an inherited outcome_blended row
|
|
// is drawn as a copy, and has to be counted as one.
|
|
const bucket = {};
|
|
for (const r of DATA.rows) {
|
|
const cls = cellClass(r).split(' ')[0];
|
|
bucket[cls] = (bucket[cls] || 0) + 1;
|
|
}
|
|
const inheritedCount = DATA.rows.filter(r => r.inherited).length;
|
|
const items = [
|
|
['cell-measured', `measured here — this row's own client outcomes (${bucket['cell-measured'] || 0})`],
|
|
['cell-prior', `not measured here — inherited prior or copy (${bucket['cell-prior'] || 0})`],
|
|
['cell-selfeval', `benchmark, n ≥ ${min} (${bucket['cell-selfeval'] || 0})`],
|
|
['cell-thin', `benchmark, thin n < ${min} (${bucket['cell-thin'] || 0})`],
|
|
].map(([cls, label]) =>
|
|
`<span class="prov-legend-item"><span class="prov-swatch ${cls}"></span>${label}</span>`
|
|
);
|
|
items.push(`<span class="prov-legend-item"><span class="prov-swatch cell-prior cell-inherited" style="position:relative"></span>corner wedge = copied from a named row (${inheritedCount})</span>`);
|
|
document.getElementById('legend').innerHTML = items.join('');
|
|
}
|
|
|
|
function activeCategories() {
|
|
const chosen = document.getElementById('filter-category').value;
|
|
if (chosen) return [chosen];
|
|
return DATA.categories;
|
|
}
|
|
|
|
function render() {
|
|
if (!DATA) return;
|
|
const cats = activeCategories();
|
|
const sourceFilter = document.getElementById('filter-source').value;
|
|
const modelFilter = document.getElementById('filter-model').value.trim().toLowerCase();
|
|
const onlyMeasured = document.getElementById('only-measured').checked;
|
|
const sortBy = document.getElementById('sort-by').value;
|
|
|
|
let ids = identities();
|
|
if (modelFilter) ids = ids.filter(i => i.model_id.toLowerCase().includes(modelFilter));
|
|
if (onlyMeasured) {
|
|
ids = ids.filter(i => i.rows.some(r => !r.inherited && r.source === 'outcome_blended'));
|
|
}
|
|
if (sortBy === 'score') {
|
|
ids.sort((a, b) => (meanScore(b.rows) ?? -1) - (meanScore(a.rows) ?? -1));
|
|
} else if (sortBy === 'samples') {
|
|
ids.sort((a, b) => totalSamples(b.rows) - totalSamples(a.rows));
|
|
} else {
|
|
ids.sort((a, b) => a.model_id.localeCompare(b.model_id));
|
|
}
|
|
|
|
const el = document.getElementById('matrix');
|
|
if (!ids.length) {
|
|
el.innerHTML = '<div class="empty">No models match this filter.</div>';
|
|
document.getElementById('row-count').textContent = '0 rows';
|
|
return;
|
|
}
|
|
|
|
const head = `<tr><th class="model-head">model</th>${
|
|
cats.map(c => `<th>${escapeHtml(c)}</th>`).join('')
|
|
}</tr>`;
|
|
|
|
let shown = 0;
|
|
const body = ids.map(id => {
|
|
const cells = cats.map(cat => {
|
|
const row = INDEX.get(keyOf(id.model_id, id.provider, cat));
|
|
if (!row) {
|
|
return `<td><span class="cell cell-missing" title="no proficiency row">—</span></td>`;
|
|
}
|
|
shown += 1;
|
|
// The source filter DIMS rather than hides: the matrix is a shape, and
|
|
// punching holes in it loses the comparison the page is for.
|
|
const dim = sourceFilter && row.source !== sourceFilter ? ' cell-dim' : '';
|
|
const score = row.blended_score == null ? '—' : row.blended_score.toFixed(2);
|
|
const n = row.total_samples || 0;
|
|
const title = [
|
|
`${row.model_id}: ${cat}`,
|
|
`source: ${row.source || 'unknown'} — ${SOURCE_LABEL[row.source] || 'unknown provenance'}`,
|
|
row.inherited ? `inherited from: ${row.inherited_from}` : 'measured on this row',
|
|
`samples: self_eval ${row.self_eval_samples || 0}, outcome ${row.outcome_samples || 0}`,
|
|
row.thin ? `THIN — below self_eval_min_samples (${DATA.self_eval_min_samples})` : '',
|
|
`updated: ${row.last_updated || '—'}`,
|
|
].filter(Boolean).join('\n');
|
|
return `<td><span class="cell ${cellClass(row)}${dim}" title="${escapeHtml(title)}"
|
|
data-cell="${escapeHtml(keyOf(id.model_id, id.provider, cat))}">
|
|
<span class="score">${score}</span><span class="n">n=${n}${row.thin ? ' thin' : ''}</span>
|
|
</span></td>`;
|
|
}).join('');
|
|
return `<tr><th class="model-head"><span class="matrix-model">${escapeHtml(id.model_id)}</span>
|
|
<span class="matrix-provider">${escapeHtml(id.provider)}</span></th>${cells}</tr>`;
|
|
}).join('');
|
|
|
|
el.innerHTML = `<table class="matrix"><thead>${head}</thead><tbody>${body}</tbody></table>`;
|
|
document.getElementById('row-count').textContent =
|
|
`${shown} of ${DATA.rows.length} rows`;
|
|
}
|
|
|
|
function renderOutcomes(outcomes) {
|
|
const el = document.getElementById('outcomes');
|
|
if (!outcomes || !outcomes.length) {
|
|
el.innerHTML = '<div class="empty">No client outcomes reported yet. POST /outcome to record one.</div>';
|
|
return;
|
|
}
|
|
const rows = outcomes.map(o => {
|
|
const attributable = o.model_attributable === 1 || o.model_attributable === true;
|
|
const verdictBadge = o.verdict === 'succeeded'
|
|
? '<span class="badge bg-success">succeeded</span>'
|
|
: o.verdict === 'failed'
|
|
? '<span class="badge bg-danger">failed</span>'
|
|
: `<span class="badge bg-secondary">${escapeHtml(String(o.verdict))}</span>`;
|
|
const attrBadge = attributable
|
|
? '<span class="badge bg-info" title="Folded into proficiency">counted</span>'
|
|
: '<span class="badge bg-warning" title="Recorded but excluded from proficiency: a degraded classification, or a failure the model did not cause">excluded</span>';
|
|
const applied = o.applied_at
|
|
? `<span class="text-muted mono">${escapeHtml(String(o.applied_at))}</span>`
|
|
: '<span class="text-muted">pending</span>';
|
|
return `<tr>
|
|
<td class="text-muted mono">${escapeHtml(String(o.observed_at || '—'))}</td>
|
|
<td>${escapeHtml(String(o.model_id || '—'))}</td>
|
|
<td class="text-muted">${escapeHtml(String(o.task_category || '—'))}</td>
|
|
<td>${verdictBadge}</td>
|
|
<td>${attrBadge}</td>
|
|
<td>${applied}</td>
|
|
<td class="outcome-detail" title="${escapeHtml(String(o.detail || ''))}">${escapeHtml(truncate(o.detail, 160))}</td>
|
|
</tr>`;
|
|
}).join('');
|
|
el.innerHTML = `<table class="table table-sm outcome-table">
|
|
<thead><tr><th>observed</th><th>model</th><th>category</th><th>verdict</th>
|
|
<th>attribution</th><th>folded</th><th>detail</th></tr></thead>
|
|
<tbody>${rows}</tbody></table>`;
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
CELL DETAIL
|
|
═══════════════════════════════════════ */
|
|
let _cellModal = null;
|
|
function getCellModal() {
|
|
if (!_cellModal) _cellModal = new tabler.Modal(document.getElementById('cell-modal'));
|
|
return _cellModal;
|
|
}
|
|
|
|
function openCell(key) {
|
|
const row = INDEX.get(key);
|
|
if (!row) return;
|
|
document.getElementById('cell-modal-title').textContent = `${row.model_id}: ${row.category}`;
|
|
const line = (k, v) => `<div class="row g-0 mb-1"><div class="col-5 text-muted" style="font-size:.75rem">${k}</div>
|
|
<div class="col-7" style="font-size:.8rem">${v}</div></div>`;
|
|
const fmt = v => (v == null ? '<span class="text-muted">—</span>' : escapeHtml(String(v)));
|
|
const num = v => (v == null ? '<span class="text-muted">—</span>' : Number(v).toFixed(3));
|
|
document.getElementById('cell-modal-body').innerHTML = [
|
|
line('provider', fmt(row.provider)),
|
|
line('blended_score', num(row.blended_score)),
|
|
line('source', `${fmt(row.source)} <div class="text-muted" style="font-size:.72rem">${escapeHtml(SOURCE_LABEL[row.source] || 'unknown provenance')}</div>`),
|
|
line('inherited_from', row.inherited_from
|
|
? `${escapeHtml(row.inherited_from)} <span class="badge bg-warning">copied, not measured here</span>`
|
|
: '<span class="text-muted">— measured on this row</span>'),
|
|
line('leaderboard_score', num(row.leaderboard_score)),
|
|
line('self_eval_score', `${num(row.self_eval_score)} <span class="text-muted">n=${row.self_eval_samples || 0}</span>`),
|
|
line('outcome_score', `${num(row.outcome_score)} <span class="text-muted">n=${row.outcome_samples || 0}</span>`),
|
|
line('sample floor', row.thin
|
|
? `<span class="badge bg-warning">thin</span> <span class="text-muted">below self_eval_min_samples = ${DATA.self_eval_min_samples}</span>`
|
|
: `<span class="text-muted">n=${row.total_samples} ≥ ${DATA.self_eval_min_samples}</span>`),
|
|
line('last_updated', fmt(row.last_updated)),
|
|
].join('');
|
|
getCellModal().show();
|
|
}
|
|
|
|
/* ═══════════════════════════════════════
|
|
LOAD
|
|
═══════════════════════════════════════ */
|
|
|
|
async function load() {
|
|
const data = await apiFetch(`${API}api/proficiency`);
|
|
if (!data) {
|
|
document.getElementById('matrix').innerHTML =
|
|
'<div class="empty">Could not load proficiency. Check the API.</div>';
|
|
return;
|
|
}
|
|
DATA = data;
|
|
INDEX = new Map(data.rows.map(r => [keyOf(r.model_id, r.provider, r.category), r]));
|
|
|
|
const catSel = document.getElementById('filter-category');
|
|
catSel.innerHTML = '<option value="">all categories</option>' +
|
|
data.categories.map(c => `<option value="${escapeHtml(c)}">${escapeHtml(c)}</option>`).join('');
|
|
const srcSel = document.getElementById('filter-source');
|
|
srcSel.innerHTML = '<option value="">all sources</option>' +
|
|
Object.keys(data.source_counts || {}).sort().map(s =>
|
|
`<option value="${escapeHtml(s)}">${escapeHtml(s)} (${data.source_counts[s]})</option>`).join('');
|
|
|
|
renderLegend();
|
|
render();
|
|
renderOutcomes(data.outcomes);
|
|
navbarUpdated(new Date().toLocaleTimeString());
|
|
}
|
|
|
|
function init() {
|
|
renderStaticIcons();
|
|
document.getElementById('matrix').addEventListener('click', (e) => {
|
|
const cell = e.target.closest('[data-cell]');
|
|
if (cell) openCell(cell.dataset.cell);
|
|
});
|
|
load();
|
|
connectSSE();
|
|
}
|
|
|
|
init();
|
|
</script>
|
|
</body>
|
|
</html>
|