"""Tests for tiering.py — pure tier resolver. Covers override-wins, effective-reasoning -> 3, cheap non-reasoning -> 1, and every fallback path -> 2 (missing/NULL cost, pricing_tbd, cost >= the threshold, including the strict-< boundary at exactly the threshold). Also covers the two signals that replaced ``supports_reasoning``: a ``-fast`` row (``reasoning_mode='reduced'``) does not earn tier 3, and ``supports_reasoning`` is consulted only when ``reasoning_default_enabled`` is absent. """ from tiering import reasoning_effectively_on, resolve_tier def _row( *, model_id: str = "m", reasoning_default_enabled: bool = False, reasoning_mode: str = "default", cost_per_1m_completion: float | None = 1.0, pricing_tbd: bool = False, supports_reasoning: bool = False, context_window: int | None = None, ) -> dict: """Build a minimal models-table row dict (ModelRow-shaped). ``context_window`` defaults to None — absent, not small — so the tier-1 context gate is inert unless a test opts in. """ return { "model_id": model_id, "reasoning_default_enabled": reasoning_default_enabled, "reasoning_mode": reasoning_mode, "cost_per_1m_completion": cost_per_1m_completion, "pricing_tbd": pricing_tbd, "supports_reasoning": supports_reasoning, "context_window": context_window, } # --- rule 1: override wins ------------------------------------------------ def test_override_wins_over_heuristic(): # Given: a reasoning model (heuristic would say 3) with an override to 1 row = _row(model_id="deep-reasoner", reasoning_default_enabled=True) override_map = {"deep-reasoner": 1} # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map=override_map) # Then: the override wins over the heuristic assert tier == 1 def test_override_applies_to_all_providers_for_same_model_id(): # Given: the same model_id on two providers, override keyed by model_id only override_map = {"shared-model": 2} row_a = _row(model_id="shared-model", reasoning_default_enabled=True) row_b = _row(model_id="shared-model", reasoning_default_enabled=True) # When: resolving both rows tier_a = resolve_tier(row_a, cheap_completion_max=1.00, override_map=override_map) tier_b = resolve_tier(row_b, cheap_completion_max=1.00, override_map=override_map) # Then: both providers get the same override (provider-independent) assert tier_a == 2 assert tier_b == 2 # --- rule 2: effective reasoning -> 3 ------------------------------------- def test_reasoning_default_enabled_tiers_to_3(): # Given: a model that reasons by default, with any cost row = _row( model_id="deep-reasoner", reasoning_default_enabled=True, cost_per_1m_completion=5.0, ) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: it is tier 3 (frontier / high-stakes) assert tier == 3 def test_supports_reasoning_alone_does_not_tier_to_3(): # Given: a model that merely ACCEPTS a reasoning param but does not use it # by default — true for all but two rows of the live catalog row = _row( model_id="accepts-param", supports_reasoning=True, reasoning_default_enabled=False, cost_per_1m_completion=5.0, ) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: it is not promoted to tier 3 on capability alone assert tier == 2 def test_fast_variant_does_not_inherit_tier_3(): # Given: a '-fast' row — same weights, thinking disabled or capped row = _row( model_id="glm-5.2-fast", reasoning_default_enabled=True, reasoning_mode="reduced", cost_per_1m_completion=4.5, ) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: reduced reasoning drops it out of tier 3 assert tier == 2 def test_missing_default_enabled_falls_back_to_supports_reasoning(): # Given: a model exposing no reasoning block (the kimi-k2.7-code family), # so reasoning_default_enabled is absent entirely row = { "model_id": "kimi-k2.7-code", "reasoning_mode": "default", "cost_per_1m_completion": 4.0, "pricing_tbd": False, "supports_reasoning": True, } # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: the capability flag is used as the fallback signal assert reasoning_effectively_on(row) is True assert tier == 3 # --- rule 3: cheap non-reasoning -> 1 ------------------------------------- def test_no_reasoning_cheap_cost_tiers_to_1(): # Given: a non-reasoning model with completion cost below the threshold row = _row(model_id="cheap-fast", cost_per_1m_completion=0.25) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: it is tier 1 (cheap / simple) assert tier == 1 def test_cheap_reasoning_capable_but_off_by_default_reaches_tier_1(): # Given: deepseek-v4-flash's real shape — accepts a reasoning param, does # not reason by default, and costs $0.28/1M. The previous heuristic put # this in tier 3 because it checked the capability flag before cost. row = _row( model_id="deepseek-v4-flash", supports_reasoning=True, reasoning_default_enabled=False, cost_per_1m_completion=0.28, ) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: it lands in tier 1 where it belongs assert tier == 1 # --- rule 4: fallback -> 2 ------------------------------------------------ def test_no_reasoning_cost_at_or_above_threshold_tiers_to_2(): # Given: a non-reasoning model with completion cost above the threshold row = _row(model_id="mid-model", cost_per_1m_completion=5.0) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: it is tier 2 (mid / general) assert tier == 2 def test_null_completion_cost_tiers_to_2(): # Given: a non-reasoning model with NULL completion cost row = _row(model_id="unknown-cost", cost_per_1m_completion=None) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: tier 1 requires a non-NULL cost, so it is tier 2 assert tier == 2 def test_pricing_tbd_tiers_to_2(): # Given: a non-reasoning model flagged pricing_tbd, even with a low cost row = _row(model_id="tbd-model", cost_per_1m_completion=0.10, pricing_tbd=True) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: pricing_tbd disqualifies tier 1 -> tier 2 assert tier == 2 def test_cost_exactly_at_threshold_tiers_to_2(): # Given: a non-reasoning model whose cost equals the threshold exactly row = _row(model_id="boundary-model", cost_per_1m_completion=1.00) # When: resolving the tier tier = resolve_tier(row, cheap_completion_max=1.00, override_map={}) # Then: tier 1 requires strictly-below, so it is tier 2 assert tier == 2 # --- cheapness is not a capability ceiling -------------------------------- def test_a_cheap_model_with_a_huge_window_is_not_tier_1(): # deepseek-v4-flash: $0.28/1M completion, 1M advertised window, 1.00 on all # three coding categories. Tiering on price alone capped it at tier 1, # which — because tier is a FLOOR — excluded it outright from every tier-2 # request. Being inexpensive is not evidence of being incapable. row = _row( model_id="deepseek-v4-flash", reasoning_default_enabled=False, cost_per_1m_completion=0.28, context_window=1_048_560, ) assert resolve_tier(row, 1.00, {}, 512_000) == 2 def test_a_cheap_model_with_a_small_window_stays_tier_1(): # gemma-4-31b: same rule, opposite side. 256K is genuinely a small model. row = _row( model_id="gemma-4-31b", reasoning_default_enabled=False, cost_per_1m_completion=0.42, context_window=262_128, ) assert resolve_tier(row, 1.00, {}, 512_000) == 1 def test_the_context_gate_is_inclusive_at_the_threshold(): row = _row(reasoning_default_enabled=False, cost_per_1m_completion=0.1, context_window=512_000) assert resolve_tier(row, 1.00, {}, 512_000) == 2 row["context_window"] = 511_999 assert resolve_tier(row, 1.00, {}, 512_000) == 1 def test_a_missing_window_does_not_block_tier_1(): # Absent capability evidence must not promote a model — a row with no # advertised window falls through to the cost rule as before. row = _row(reasoning_default_enabled=False, cost_per_1m_completion=0.1, context_window=None) assert resolve_tier(row, 1.00, {}, 512_000) == 1 def test_the_gate_defaults_to_off_for_callers_that_do_not_pass_it(): # Omitting the argument keeps the old cost-only behaviour rather than # silently re-tiering a caller that has not opted in. row = _row(reasoning_default_enabled=False, cost_per_1m_completion=0.28, context_window=1_048_560) assert resolve_tier(row, 1.00, {}) == 1 def test_a_huge_window_does_not_rescue_an_expensive_model_into_tier_1(): # The gate only ever demotes; it is not a second route into tier 1. row = _row(reasoning_default_enabled=False, cost_per_1m_completion=15.0, context_window=131_056) assert resolve_tier(row, 1.00, {}, 512_000) == 2 def test_override_still_beats_the_context_gate(): row = _row(model_id="pinned", reasoning_default_enabled=False, cost_per_1m_completion=0.28, context_window=1_048_560) assert resolve_tier(row, 1.00, {"pinned": 1}, 512_000) == 1