test(dispatcher): tighten the local_decision backoff tests #107

Merged
alee merged 1 commits from test/local-decision-tighten-backoff-tests into main 2026-10-03 05:42:16 +00:00

View File

@@ -924,14 +924,16 @@ def test_local_decision_connection_error_opens_circuit_unmetered(monkeypatch):
assert got1.source == "fallback"
assert dispatcher._last_classifier_failure > 0.0
assert dispatcher._classifier_backoff_active() is True
first_failure_at = dispatcher._last_classifier_failure
# Second call — should be SKIPPED (post called exactly once total)
got2 = dispatcher.classify("another task", None)
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
assert got2.source == "fallback"
# _last_classifier_failure unchanged by second call
assert dispatcher._last_classifier_failure > 0.0 # still set from first call
# A skip must never re-stamp the clock, or the circuit stays open forever
# while traffic flows. Equality, not just "still set".
assert dispatcher._last_classifier_failure == first_failure_at
def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
@@ -968,16 +970,19 @@ def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
assert got1.source == "fallback"
assert dispatcher._last_classifier_failure > 0.0
assert dispatcher._classifier_backoff_active() is True
first_failure_at = dispatcher._last_classifier_failure
# Second call — should be SKIPPED (post called exactly once total)
got2 = dispatcher.classify("another task", None)
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
assert got2.source == "fallback"
assert dispatcher._last_classifier_failure == first_failure_at
def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
"""A successful classify with low confidence is a real answer, not a
transport failure — the circuit must stay closed."""
def test_local_decision_below_confidence_min_does_not_open_circuit(monkeypatch):
"""A verdict below classifier.decision.confidence_min is the dispatcher's
own RuntimeError, raised after a healthy answer. The endpoint is fine, so
the circuit must stay closed. (The coverage-floor path is the next test.)"""
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
monkeypatch.setattr(
@@ -987,8 +992,46 @@ def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
def fake_post(*a, **k):
# Return a response where the winning token has very low logprob
# → confidence below 0.5 → below confidence_min → RuntimeError
# Four letters with equal mass: coverage is about 0.99 (well above
# coverage_min), but the winner holds only a quarter of it, so
# confidence is about 0.25, below confidence_min (0.5).
return SimpleNamespace(
json=lambda: {
"logprobs": [{
"top_logprobs": [
{"token": letter, "logprob": -1.4}
for letter in ("A", "B", "C", "D")
]
}],
},
raise_for_status=lambda: None,
)
monkeypatch.setattr(local_decision.requests, "post", fake_post)
got = dispatcher.classify("refactor this function", None)
assert got.source == "fallback" # cascaded on the confidence failure
assert dispatcher._last_classifier_failure == 0.0
assert dispatcher._classifier_backoff_active() is False
def test_local_decision_coverage_floor_does_not_open_circuit(monkeypatch):
"""parse_logprobs raising because the model put almost no mass on any
option letter (coverage below coverage_min) is also not a transport
failure — the circuit must stay closed."""
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
monkeypatch.setattr(
dispatcher.cfg.classifier, "decision",
_decision_conf(confidence_min=0.5, tier_enabled=False),
)
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
def fake_post(*a, **k):
# "Z" is not an option letter and A/B carry almost no mass, so total
# option mass (about 0.012) is below coverage_min (0.3) and
# parse_logprobs raises RuntimeError before any confidence is computed.
return SimpleNamespace(
json=lambda: {
"model": "qwen3.5:4b",