1c11861902
propagate_to_variants skipped any row with self_eval_samples > 0, meaning "this one was measured directly, do not overwrite it with the family's". But inheritance COPIES self_eval_samples, so an inherited row also has samples > 0 and the test could not tell the two apart. A variant could be inherited once and never again. Found by re-measuring docs_writing. kimi-k3 moved 0.85 (n=2) -> 0.957 (n=7) while kimi-k3-flex sat at 0.85 with a timestamp from 2026-08-16, and the run reported "propagated 0 inherited rows". Flex rows are what `auto:batch` admits, so overnight work was ranking on scores its family had left behind by a week -- silently, since a stale score looks exactly like a fresh one. Provenance is the missing fact, so the table now records it: proficiency.inherited_from names the model a row's scores were copied from, NULL when they were measured on that row. propagate_to_variants refreshes where it is set and still refuses where it is not. The migration ships the repair, not just the fix. ADD COLUMN gives every existing row NULL, which reads as "measured here" -- so on its own it would have left exactly the rows this change exists for frozen forever. Provenance that was never recorded cannot be recovered in general, but it can for the rows that matter, and not by guessing: eval_identities selects standard rows plus flex rows with no standard equivalent, so a flex row WITH one was never a candidate for direct evaluation whatever its sample count says. That is the harness's own selection rule read backwards. Everything else keeps NULL, which is the safe direction -- it means "do not overwrite", so a real measurement is never lost to the backfill. schema.sql is CREATE TABLE IF NOT EXISTS, which defines a new database and does nothing to an existing one, so ensure_columns() carries the ALTER for databases that predate the column. Applied to the live router.db. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WSkcSD2Jzkxo1Kw27ecfXJ
352 lines
14 KiB
Python
352 lines
14 KiB
Python
"""Tests for proficiency.py — blending leaderboard priors with self-eval.
|
|
|
|
The interesting cases are the ones the design doc's one-line rule does not
|
|
cover: what happens when a model has no leaderboard prior (common, because
|
|
NeuralWatt ships models faster than benchmarks cover them) and when a run
|
|
produces no usable scores at all.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from proficiency import accumulate, blend
|
|
|
|
BLEND_KW = {"leaderboard_weight": 0.3, "self_eval_weight": 0.7, "min_samples": 10}
|
|
|
|
|
|
def _blend(lb, se, n, **over):
|
|
return blend(lb, se, n, **{**BLEND_KW, **over})
|
|
|
|
|
|
# --- the documented rule --------------------------------------------------
|
|
|
|
def test_blends_both_sources_once_samples_suffice():
|
|
score, source = _blend(0.4, 0.9, 10)
|
|
assert score == pytest.approx(0.3 * 0.4 + 0.7 * 0.9)
|
|
assert source == "blended"
|
|
|
|
|
|
def test_leaderboard_alone_below_the_sample_threshold():
|
|
# Thin self-eval must not dominate a published benchmark early
|
|
score, source = _blend(0.4, 0.9, 9)
|
|
assert (score, source) == (0.4, "leaderboard")
|
|
|
|
|
|
def test_threshold_is_inclusive():
|
|
assert _blend(0.4, 0.9, 10)[1] == "blended"
|
|
assert _blend(0.4, 0.9, 9)[1] == "leaderboard"
|
|
|
|
|
|
# --- no leaderboard prior, which is the common case -----------------------
|
|
|
|
def test_self_eval_alone_when_no_prior_exists():
|
|
# Given: a model NeuralWatt added that no public benchmark covers yet.
|
|
# Read literally, the design doc's rule would fall back to a leaderboard
|
|
# score that does not exist and yield nothing — discarding real evidence.
|
|
score, source = _blend(None, 0.8, 10)
|
|
assert (score, source) == (0.8, "self_eval")
|
|
|
|
|
|
def test_thin_self_eval_still_beats_nothing():
|
|
# Nine real samples are worse than twelve, but far better than the
|
|
# neutral 0.5 a model gets for being unmeasured. Flagged so a caller can
|
|
# tell it apart from a score that cleared the threshold.
|
|
score, source = _blend(None, 0.8, 9)
|
|
assert (score, source) == (0.8, "self_eval_thin")
|
|
|
|
|
|
def test_unmeasured_model_returns_none_not_zero():
|
|
# None leaves the candidate on the neutral 0.5 downstream. Zero would
|
|
# rank it below every measured model for the crime of being new.
|
|
assert _blend(None, None, 0) == (None, None)
|
|
|
|
|
|
def test_zero_samples_is_not_evidence():
|
|
# A score with no samples behind it is a leftover, not a measurement
|
|
assert _blend(None, 0.9, 0) == (None, None)
|
|
assert _blend(0.4, 0.9, 0) == (0.4, "leaderboard")
|
|
|
|
|
|
def test_a_genuine_zero_score_is_kept():
|
|
# 0.0 means "measured, and it failed everything" — distinct from unmeasured
|
|
score, source = _blend(None, 0.0, 12)
|
|
assert (score, source) == (0.0, "self_eval")
|
|
|
|
|
|
# --- accumulation ---------------------------------------------------------
|
|
|
|
def test_first_run_sets_the_mean():
|
|
assert accumulate(None, 0, [1.0, 0.5, 0.0]) == (0.5, 3)
|
|
|
|
|
|
def test_later_runs_tighten_rather_than_replace():
|
|
# Given: 0.8 over 10 samples, then a run of 2 perfect scores
|
|
score, samples = accumulate(0.8, 10, [1.0, 1.0])
|
|
assert samples == 12
|
|
assert score == pytest.approx((0.8 * 10 + 2.0) / 12)
|
|
|
|
|
|
def test_an_all_errors_run_changes_nothing():
|
|
# Given: every task errored, so there are no scores. Resetting a model's
|
|
# history to zero on a bad run would silently erase months of evidence.
|
|
assert accumulate(0.8, 10, []) == (0.8, 10)
|
|
|
|
|
|
def test_no_history_and_no_scores_stays_empty():
|
|
assert accumulate(None, 0, []) == (None, 0)
|
|
|
|
|
|
def test_stale_score_with_zero_samples_is_overwritten():
|
|
# A score with no samples behind it carries no weight in the average
|
|
assert accumulate(0.9, 0, [0.1, 0.3]) == (pytest.approx(0.2), 2)
|
|
|
|
|
|
# --- variant inheritance --------------------------------------------------
|
|
|
|
def _models_db(tmp_path, rows):
|
|
import sqlite3
|
|
from pathlib import Path
|
|
schema = (Path(__file__).resolve().parent.parent / "schema.sql").read_text()
|
|
conn = sqlite3.connect(tmp_path / "t.db")
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(schema)
|
|
for model_id, base, latency, reasoning, ctx in rows:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO models (model_id, provider, base_model_id, latency_class,
|
|
reasoning_mode, context_variant, availability, last_updated)
|
|
VALUES (?, 'nw', ?, ?, ?, ?, 'active', '2026-08-17T00:00:00+00:00')
|
|
""",
|
|
(model_id, base, latency, reasoning, ctx),
|
|
)
|
|
conn.commit()
|
|
return conn
|
|
|
|
|
|
def test_flex_inherits_from_its_standard_equivalent(tmp_path):
|
|
from config import load_config
|
|
from pathlib import Path
|
|
from proficiency_store import add_self_eval, propagate_to_variants
|
|
|
|
cfg = load_config(Path(__file__).resolve().parent.parent / "config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-flex", "kimi-k3", "flex", "default", "full"),
|
|
])
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "coding_general", [1.0, 1.0])
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 1
|
|
got = conn.execute(
|
|
"SELECT self_eval_score FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()
|
|
assert got["self_eval_score"] == 1.0
|
|
|
|
|
|
def test_fast_does_not_inherit_reasoning_on_quality(tmp_path):
|
|
# A '-fast' row runs with reasoning off or capped, so it is NOT the same
|
|
# model for quality purposes. Inheriting across that would credit it with
|
|
# its reasoning-enabled sibling's answers.
|
|
from config import load_config
|
|
from pathlib import Path
|
|
from proficiency_store import add_self_eval, propagate_to_variants
|
|
|
|
cfg = load_config(Path(__file__).resolve().parent.parent / "config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-fast", "kimi-k3", "standard", "reduced", "full"),
|
|
])
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "reasoning_math", [1.0])
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 0
|
|
assert conn.execute(
|
|
"SELECT COUNT(*) c FROM proficiency WHERE model_id='kimi-k3-fast'"
|
|
).fetchone()["c"] == 0
|
|
|
|
|
|
def test_a_directly_measured_variant_is_never_overwritten(tmp_path):
|
|
from config import load_config
|
|
from pathlib import Path
|
|
from proficiency_store import add_self_eval, propagate_to_variants
|
|
|
|
cfg = load_config(Path(__file__).resolve().parent.parent / "config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-flex", "kimi-k3", "flex", "default", "full"),
|
|
])
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "coding_general", [1.0])
|
|
add_self_eval(conn, cfg, "kimi-k3-flex", "nw", "coding_general", [0.2])
|
|
propagate_to_variants(conn, cfg, "kimi-k3", "nw")
|
|
got = conn.execute(
|
|
"SELECT self_eval_score FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()
|
|
assert got["self_eval_score"] == 0.2
|
|
|
|
|
|
def test_a_score_above_one_never_reaches_the_table(tmp_path):
|
|
"""The single write path is where a bad scorer gets stopped.
|
|
|
|
A blended_score above 1.0 does not just misreport one row: it raises
|
|
`best` in rank_candidates and shifts every other candidate's quality band.
|
|
"""
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [("kimi-k3", "kimi-k3", "standard", "default", "full")])
|
|
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "coding_general", [1.5, 1.0])
|
|
|
|
row = conn.execute(
|
|
"SELECT self_eval_score, blended_score FROM proficiency WHERE model_id = 'kimi-k3'"
|
|
).fetchone()
|
|
assert row["self_eval_score"] == 1.0
|
|
assert row["blended_score"] <= 1.0
|
|
|
|
|
|
def test_a_negative_score_is_floored_at_zero(tmp_path):
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [("kimi-k3", "kimi-k3", "standard", "default", "full")])
|
|
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "coding_general", [-0.5, 0.5])
|
|
|
|
assert conn.execute(
|
|
"SELECT self_eval_score FROM proficiency WHERE model_id = 'kimi-k3'"
|
|
).fetchone()["self_eval_score"] == 0.25
|
|
|
|
|
|
def test_an_inherited_variant_refreshes_when_its_family_is_remeasured(tmp_path):
|
|
"""Inheritance was a one-shot: a variant froze at its first copy, forever.
|
|
|
|
The old guard skipped any row with self_eval_samples > 0, and an inherited
|
|
row has samples > 0 because inheritance copies them -- so it could never
|
|
tell "measured here" from "copied here" and never refreshed. Observed live:
|
|
kimi-k3 reached 0.957 while kimi-k3-flex sat at 0.85 with a week-old
|
|
timestamp, and the eval reported "propagated 0 inherited rows". Flex rows
|
|
serve auto:batch, so those requests ranked on stale scores.
|
|
"""
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval, propagate_to_variants
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-flex", "kimi-k3", "flex", "default", "full"),
|
|
])
|
|
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [0.85, 0.85])
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 1
|
|
inherited = conn.execute(
|
|
"SELECT blended_score, inherited_from FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()
|
|
assert inherited["blended_score"] == pytest.approx(0.85)
|
|
assert inherited["inherited_from"] == "kimi-k3"
|
|
|
|
# The family learns more; the variant must follow it.
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [1.0, 1.0, 1.0, 1.0])
|
|
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 1
|
|
refreshed = conn.execute(
|
|
"SELECT blended_score, self_eval_samples FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()
|
|
assert refreshed["blended_score"] == pytest.approx(0.95)
|
|
assert refreshed["self_eval_samples"] == 6
|
|
|
|
|
|
def test_a_measured_variant_still_outranks_its_family(tmp_path):
|
|
"""The protection the old guard was reaching for, kept intact."""
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval, propagate_to_variants
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-flex", "kimi-k3", "flex", "default", "full"),
|
|
])
|
|
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [1.0])
|
|
add_self_eval(conn, cfg, "kimi-k3-flex", "nw", "docs_writing", [0.2])
|
|
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 0
|
|
row = conn.execute(
|
|
"SELECT blended_score, inherited_from FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()
|
|
assert row["blended_score"] == pytest.approx(0.2)
|
|
assert row["inherited_from"] is None
|
|
|
|
|
|
def test_a_database_without_the_column_is_migrated(tmp_path):
|
|
"""schema.sql only defines a NEW database; existing ones need the ALTER."""
|
|
import sqlite3
|
|
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval, ensure_columns
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [("kimi-k3", "kimi-k3", "standard", "default", "full")])
|
|
conn.execute("ALTER TABLE proficiency DROP COLUMN inherited_from")
|
|
assert "inherited_from" not in {
|
|
r[1] for r in conn.execute("PRAGMA table_info(proficiency)")
|
|
}
|
|
|
|
ensure_columns(conn)
|
|
ensure_columns(conn) # idempotent
|
|
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [1.0])
|
|
assert conn.execute(
|
|
"SELECT inherited_from FROM proficiency WHERE model_id='kimi-k3'"
|
|
).fetchone()["inherited_from"] is None
|
|
|
|
|
|
def test_migration_unfreezes_variants_that_predate_the_column(tmp_path):
|
|
"""The migration must ship the repair, not just the fix.
|
|
|
|
ADD COLUMN gives every existing row NULL, and NULL means "measured here" --
|
|
so without the backfill, the rows this whole change exists for would stay
|
|
frozen and the live flex rows would never catch up.
|
|
"""
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval, ensure_columns, propagate_to_variants
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-flex", "kimi-k3", "flex", "default", "full"),
|
|
])
|
|
# An old database: both rows carry samples, neither records provenance.
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [0.957])
|
|
add_self_eval(conn, cfg, "kimi-k3-flex", "nw", "docs_writing", [0.85])
|
|
conn.execute("ALTER TABLE proficiency DROP COLUMN inherited_from")
|
|
|
|
ensure_columns(conn)
|
|
|
|
assert conn.execute(
|
|
"SELECT inherited_from FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()["inherited_from"] == "kimi-k3"
|
|
# ...so the next run actually moves it.
|
|
assert propagate_to_variants(conn, cfg, "kimi-k3", "nw") == 1
|
|
assert conn.execute(
|
|
"SELECT blended_score FROM proficiency WHERE model_id='kimi-k3-flex'"
|
|
).fetchone()["blended_score"] == pytest.approx(0.957)
|
|
|
|
|
|
def test_the_backfill_never_touches_a_standard_row(tmp_path):
|
|
"""Only flex rows with a standard equivalent are provably inherited."""
|
|
from config import load_config
|
|
from proficiency_store import add_self_eval, ensure_columns
|
|
|
|
cfg = load_config("config.yaml")
|
|
conn = _models_db(tmp_path, [
|
|
("kimi-k3", "kimi-k3", "standard", "default", "full"),
|
|
("kimi-k3-fast", "kimi-k3", "standard", "reduced", "full"),
|
|
])
|
|
add_self_eval(conn, cfg, "kimi-k3", "nw", "docs_writing", [1.0])
|
|
add_self_eval(conn, cfg, "kimi-k3-fast", "nw", "docs_writing", [0.888])
|
|
conn.execute("ALTER TABLE proficiency DROP COLUMN inherited_from")
|
|
|
|
ensure_columns(conn)
|
|
|
|
for model_id in ("kimi-k3", "kimi-k3-fast"):
|
|
assert conn.execute(
|
|
"SELECT inherited_from FROM proficiency WHERE model_id=?", (model_id,)
|
|
).fetchone()["inherited_from"] is None
|