test(dispatcher): tighten the local_decision backoff tests #107
@@ -924,14 +924,16 @@ def test_local_decision_connection_error_opens_circuit_unmetered(monkeypatch):
|
||||
assert got1.source == "fallback"
|
||||
assert dispatcher._last_classifier_failure > 0.0
|
||||
assert dispatcher._classifier_backoff_active() is True
|
||||
first_failure_at = dispatcher._last_classifier_failure
|
||||
|
||||
# Second call — should be SKIPPED (post called exactly once total)
|
||||
got2 = dispatcher.classify("another task", None)
|
||||
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
||||
assert got2.source == "fallback"
|
||||
|
||||
# _last_classifier_failure unchanged by second call
|
||||
assert dispatcher._last_classifier_failure > 0.0 # still set from first call
|
||||
# A skip must never re-stamp the clock, or the circuit stays open forever
|
||||
# while traffic flows. Equality, not just "still set".
|
||||
assert dispatcher._last_classifier_failure == first_failure_at
|
||||
|
||||
|
||||
def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
|
||||
@@ -968,16 +970,19 @@ def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
|
||||
assert got1.source == "fallback"
|
||||
assert dispatcher._last_classifier_failure > 0.0
|
||||
assert dispatcher._classifier_backoff_active() is True
|
||||
first_failure_at = dispatcher._last_classifier_failure
|
||||
|
||||
# Second call — should be SKIPPED (post called exactly once total)
|
||||
got2 = dispatcher.classify("another task", None)
|
||||
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
||||
assert got2.source == "fallback"
|
||||
assert dispatcher._last_classifier_failure == first_failure_at
|
||||
|
||||
|
||||
def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
|
||||
"""A successful classify with low confidence is a real answer, not a
|
||||
transport failure — the circuit must stay closed."""
|
||||
def test_local_decision_below_confidence_min_does_not_open_circuit(monkeypatch):
|
||||
"""A verdict below classifier.decision.confidence_min is the dispatcher's
|
||||
own RuntimeError, raised after a healthy answer. The endpoint is fine, so
|
||||
the circuit must stay closed. (The coverage-floor path is the next test.)"""
|
||||
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
|
||||
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
|
||||
monkeypatch.setattr(
|
||||
@@ -987,8 +992,46 @@ def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
|
||||
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
||||
|
||||
def fake_post(*a, **k):
|
||||
# Return a response where the winning token has very low logprob
|
||||
# → confidence below 0.5 → below confidence_min → RuntimeError
|
||||
# Four letters with equal mass: coverage is about 0.99 (well above
|
||||
# coverage_min), but the winner holds only a quarter of it, so
|
||||
# confidence is about 0.25, below confidence_min (0.5).
|
||||
return SimpleNamespace(
|
||||
json=lambda: {
|
||||
"logprobs": [{
|
||||
"top_logprobs": [
|
||||
{"token": letter, "logprob": -1.4}
|
||||
for letter in ("A", "B", "C", "D")
|
||||
]
|
||||
}],
|
||||
},
|
||||
raise_for_status=lambda: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(local_decision.requests, "post", fake_post)
|
||||
|
||||
got = dispatcher.classify("refactor this function", None)
|
||||
assert got.source == "fallback" # cascaded on the confidence failure
|
||||
|
||||
assert dispatcher._last_classifier_failure == 0.0
|
||||
assert dispatcher._classifier_backoff_active() is False
|
||||
|
||||
|
||||
def test_local_decision_coverage_floor_does_not_open_circuit(monkeypatch):
|
||||
"""parse_logprobs raising because the model put almost no mass on any
|
||||
option letter (coverage below coverage_min) is also not a transport
|
||||
failure — the circuit must stay closed."""
|
||||
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
|
||||
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
|
||||
monkeypatch.setattr(
|
||||
dispatcher.cfg.classifier, "decision",
|
||||
_decision_conf(confidence_min=0.5, tier_enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
||||
|
||||
def fake_post(*a, **k):
|
||||
# "Z" is not an option letter and A/B carry almost no mass, so total
|
||||
# option mass (about 0.012) is below coverage_min (0.3) and
|
||||
# parse_logprobs raises RuntimeError before any confidence is computed.
|
||||
return SimpleNamespace(
|
||||
json=lambda: {
|
||||
"model": "qwen3.5:4b",
|
||||
|
||||
Reference in New Issue
Block a user