c3484f05e5
Drops OpenRouter entirely. The provider column and (model_id, provider) key stay so a second provider needs no migration. Verified against the live API — the poller's field mappings were previously unconfirmed guesses and turned out correct. Routing correctness: - Tier on metadata.reasoning.default_enabled, not capabilities.reasoning. The latter only means "the endpoint accepts a reasoning param" and is true for 17 of 19 rows, which put 17 models in tier 3 and left tier 1 empty. Cost is now checked before the reasoning rule so $0.28/1M models can reach tier 1. Distribution goes from 2/17 to 4/6/9. - Capture serving class. NeuralWatt ships ~6 base models as 19 rows whose id suffixes are three orthogonal dimensions (hence glm-5.2-short-fast-flex): -flex is discounted async held during peak, -fast is reasoning disabled or capped, -short is a 200K pool. They carry identical catalog pricing, so without these columns all 7 GLM rows tie exactly and an interactive request could land on a preemptible row. Latency tolerance is a hard filter, not a weight. Suffixes match whole segments so deepseek-v4-flash is not read as a -fast row. - Exclude access-gated models. 6 of 19 rows are grant-gated or canary, marked only in prose, and would 403 at dispatch. - Log the provider's real billed cost and carbon rather than a tokens x list-price estimate, and score eco on carbon per design doc §4. New dispatcher.py exposes /health, /route (dry run, no spend), /dispatch, plus an OpenAI-compatible /v1/models and /v1/chat/completions so any normal client can use it. Streaming is proxied chunk by chunk; NeuralWatt emits energy and cost as SSE comment lines, which clients ignore and the router reads on the way past — otherwise streamed calls would log no energy at all. Classifier now gets the allowed category list injected from config (it was returning invented labels that join against nothing) and runs at temperature 0, because the same prompt was classifying tier 2 then tier 1 and routing to different models. Ships systemd user units. The poller timer is load-bearing, not housekeeping: stale_after_days is 3 with exclude_stale true, so an unpolled catalog eventually marks every row stale and the router returns no candidates at all. Documents the finding that most affects this project: NeuralWatt bills a flat $8.00/kWh, not per token. List price ranks models backwards — on the same prompt kimi-k2.7-code-fast ($4/1M) cost 10x more than kimi-k3-fast ($15/1M). scoring.cost_score still reads list price; re-basing it is the open call. Tests 28 -> 74. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018xTPER7K8fNyKiuqNvTCTa
167 lines
5.2 KiB
Python
167 lines
5.2 KiB
Python
"""Tests for scoring.py — pure scoring functions.
|
|
|
|
Covers the required edge cases from the router-scoring-tiering plan:
|
|
cost min-max inversion (incl. $0-cheapest and None handling), eco
|
|
inversion with neutral default, thin proficiency lookup, and the
|
|
weighted composite.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from scoring import (
|
|
composite_score,
|
|
cost_score,
|
|
eco_score,
|
|
proficiency_score,
|
|
)
|
|
|
|
|
|
# --- cost_score -----------------------------------------------------------
|
|
|
|
def test_cost_score_single_candidate_is_1_0():
|
|
# Given: one candidate with a known cost
|
|
costs = [5.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: it is the cheapest and only candidate -> 1.0
|
|
assert scores == [1.0]
|
|
|
|
|
|
def test_cost_score_two_equal_is_1_0():
|
|
# Given: two candidates sharing the same cost (min == max)
|
|
costs = [5.0, 5.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: guard returns 1.0 for all instead of dividing by zero
|
|
assert scores == [1.0, 1.0]
|
|
|
|
|
|
def test_cost_score_all_equal_is_1_0():
|
|
# Given: all candidates share the same cost
|
|
costs = [5.0, 5.0, 5.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: min == max -> every candidate scores 1.0
|
|
assert scores == [1.0, 1.0, 1.0]
|
|
|
|
|
|
def test_cost_score_cheapest_is_zero():
|
|
# Given: a free ($0) model and two paid models
|
|
costs = [0.0, 10.0, 20.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: free model scores 1.0, paid models scale relative to max
|
|
assert scores[0] == pytest.approx(1.0)
|
|
assert scores[1] == pytest.approx(0.5)
|
|
assert scores[2] == pytest.approx(0.0)
|
|
|
|
|
|
def test_cost_score_tied_costs_get_equal_scores():
|
|
# Given: two candidates tied at the cheapest price
|
|
costs = [4.0, 4.0, 8.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: tied candidates get identical scores
|
|
assert scores[0] == scores[1]
|
|
assert scores[0] == pytest.approx(1.0)
|
|
assert scores[2] == pytest.approx(0.0)
|
|
|
|
|
|
def test_cost_score_none_costs_get_neutral_and_are_excluded_from_range():
|
|
# Given: one candidate with unknown cost, two with known costs
|
|
costs = [None, 10.0, 20.0]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: None candidate gets neutral 0.5; min/max computed over known only
|
|
assert scores[0] == pytest.approx(0.5)
|
|
assert scores[1] == pytest.approx(1.0)
|
|
assert scores[2] == pytest.approx(0.0)
|
|
|
|
|
|
def test_cost_score_all_none_is_all_neutral():
|
|
# Given: no candidate has a known cost
|
|
costs = [None, None]
|
|
# When: scoring
|
|
scores = cost_score(costs)
|
|
# Then: every candidate gets the neutral 0.5
|
|
assert scores == [0.5, 0.5]
|
|
|
|
|
|
# --- eco_score ------------------------------------------------------------
|
|
|
|
def test_eco_score_mixed_data_discriminates_and_missing_is_neutral():
|
|
# Given: two candidates with eco data, one without
|
|
eco = [10.0, 30.0, None]
|
|
# When: scoring
|
|
scores = eco_score(eco)
|
|
# Then: lower eco -> higher score; missing -> neutral 0.5
|
|
assert scores[0] == pytest.approx(1.0)
|
|
assert scores[1] == pytest.approx(0.0)
|
|
assert scores[2] == pytest.approx(0.5)
|
|
|
|
|
|
def test_eco_score_all_none_is_all_neutral():
|
|
# Given: no candidate has eco data
|
|
eco = [None, None, None]
|
|
# When: scoring
|
|
scores = eco_score(eco)
|
|
# Then: every candidate gets 0.5
|
|
assert scores == [0.5, 0.5, 0.5]
|
|
|
|
|
|
def test_eco_score_all_equal_is_1_0_for_present_data():
|
|
# Given: all candidates with eco data share the same value
|
|
eco = [5.0, 5.0]
|
|
# When: scoring
|
|
scores = eco_score(eco)
|
|
# Then: min == max -> 1.0 for present-data candidates
|
|
assert scores == [1.0, 1.0]
|
|
|
|
|
|
# --- proficiency_score ----------------------------------------------------
|
|
|
|
def test_proficiency_score_none_is_neutral():
|
|
# Given: no blended proficiency value
|
|
blended = None
|
|
# When: scoring
|
|
score = proficiency_score(blended)
|
|
# Then: neutral default 0.5
|
|
assert score == pytest.approx(0.5)
|
|
|
|
|
|
def test_proficiency_score_passes_through_value():
|
|
# Given: a blended proficiency value
|
|
blended = 0.8
|
|
# When: scoring
|
|
score = proficiency_score(blended)
|
|
# Then: the value is returned unchanged
|
|
assert score == pytest.approx(0.8)
|
|
|
|
|
|
# --- composite_score ------------------------------------------------------
|
|
|
|
def test_composite_score_all_none_proficiency_eco_equals_0_4_cost_plus_0_3():
|
|
# Given: cost scores and neutral proficiency/eco (0.5 each)
|
|
cost_s = [1.0, 0.5, 0.0]
|
|
# When: composite with default weights (0.4, 0.2, 0.4)
|
|
composites = [composite_score(c, 0.5, 0.5) for c in cost_s]
|
|
# Then: 0.4*cost + 0.2*0.5 + 0.4*0.5 = 0.4*cost + 0.3
|
|
assert composites == pytest.approx([0.7, 0.5, 0.3])
|
|
|
|
|
|
def test_composite_score_applies_config_weights():
|
|
# Given: config weights cost=0.4, eco=0.2, proficiency=0.4
|
|
weights = (0.4, 0.2, 0.4)
|
|
# When: composite with distinct sub-scores
|
|
composite = composite_score(1.0, 0.0, 0.5, weights=weights)
|
|
# Then: 0.4*1.0 + 0.2*0.0 + 0.4*0.5 = 0.6
|
|
assert composite == pytest.approx(0.6)
|
|
|
|
|
|
def test_composite_score_default_weights_match_config():
|
|
# Given: default weights are (0.4, 0.2, 0.4) per config.yaml
|
|
# When: composite with default weights and all-1.0 sub-scores
|
|
composite = composite_score(1.0, 1.0, 1.0)
|
|
# Then: 0.4 + 0.2 + 0.4 = 1.0
|
|
assert composite == pytest.approx(1.0)
|