test(dispatcher): tighten the local_decision backoff tests #107
@@ -924,14 +924,16 @@ def test_local_decision_connection_error_opens_circuit_unmetered(monkeypatch):
|
|||||||
assert got1.source == "fallback"
|
assert got1.source == "fallback"
|
||||||
assert dispatcher._last_classifier_failure > 0.0
|
assert dispatcher._last_classifier_failure > 0.0
|
||||||
assert dispatcher._classifier_backoff_active() is True
|
assert dispatcher._classifier_backoff_active() is True
|
||||||
|
first_failure_at = dispatcher._last_classifier_failure
|
||||||
|
|
||||||
# Second call — should be SKIPPED (post called exactly once total)
|
# Second call — should be SKIPPED (post called exactly once total)
|
||||||
got2 = dispatcher.classify("another task", None)
|
got2 = dispatcher.classify("another task", None)
|
||||||
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
||||||
assert got2.source == "fallback"
|
assert got2.source == "fallback"
|
||||||
|
|
||||||
# _last_classifier_failure unchanged by second call
|
# A skip must never re-stamp the clock, or the circuit stays open forever
|
||||||
assert dispatcher._last_classifier_failure > 0.0 # still set from first call
|
# while traffic flows. Equality, not just "still set".
|
||||||
|
assert dispatcher._last_classifier_failure == first_failure_at
|
||||||
|
|
||||||
|
|
||||||
def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
|
def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
|
||||||
@@ -968,16 +970,19 @@ def test_local_decision_connection_error_opens_circuit_metered(monkeypatch):
|
|||||||
assert got1.source == "fallback"
|
assert got1.source == "fallback"
|
||||||
assert dispatcher._last_classifier_failure > 0.0
|
assert dispatcher._last_classifier_failure > 0.0
|
||||||
assert dispatcher._classifier_backoff_active() is True
|
assert dispatcher._classifier_backoff_active() is True
|
||||||
|
first_failure_at = dispatcher._last_classifier_failure
|
||||||
|
|
||||||
# Second call — should be SKIPPED (post called exactly once total)
|
# Second call — should be SKIPPED (post called exactly once total)
|
||||||
got2 = dispatcher.classify("another task", None)
|
got2 = dispatcher.classify("another task", None)
|
||||||
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
assert call_count[0] == 1, f"post called {call_count[0]} times, expected 1"
|
||||||
assert got2.source == "fallback"
|
assert got2.source == "fallback"
|
||||||
|
assert dispatcher._last_classifier_failure == first_failure_at
|
||||||
|
|
||||||
|
|
||||||
def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
|
def test_local_decision_below_confidence_min_does_not_open_circuit(monkeypatch):
|
||||||
"""A successful classify with low confidence is a real answer, not a
|
"""A verdict below classifier.decision.confidence_min is the dispatcher's
|
||||||
transport failure — the circuit must stay closed."""
|
own RuntimeError, raised after a healthy answer. The endpoint is fine, so
|
||||||
|
the circuit must stay closed. (The coverage-floor path is the next test.)"""
|
||||||
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
|
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
|
||||||
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
|
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@@ -987,8 +992,46 @@ def test_local_decision_low_confidence_does_not_open_circuit(monkeypatch):
|
|||||||
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
||||||
|
|
||||||
def fake_post(*a, **k):
|
def fake_post(*a, **k):
|
||||||
# Return a response where the winning token has very low logprob
|
# Four letters with equal mass: coverage is about 0.99 (well above
|
||||||
# → confidence below 0.5 → below confidence_min → RuntimeError
|
# coverage_min), but the winner holds only a quarter of it, so
|
||||||
|
# confidence is about 0.25, below confidence_min (0.5).
|
||||||
|
return SimpleNamespace(
|
||||||
|
json=lambda: {
|
||||||
|
"logprobs": [{
|
||||||
|
"top_logprobs": [
|
||||||
|
{"token": letter, "logprob": -1.4}
|
||||||
|
for letter in ("A", "B", "C", "D")
|
||||||
|
]
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
raise_for_status=lambda: None,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(local_decision.requests, "post", fake_post)
|
||||||
|
|
||||||
|
got = dispatcher.classify("refactor this function", None)
|
||||||
|
assert got.source == "fallback" # cascaded on the confidence failure
|
||||||
|
|
||||||
|
assert dispatcher._last_classifier_failure == 0.0
|
||||||
|
assert dispatcher._classifier_backoff_active() is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_decision_coverage_floor_does_not_open_circuit(monkeypatch):
|
||||||
|
"""parse_logprobs raising because the model put almost no mass on any
|
||||||
|
option letter (coverage below coverage_min) is also not a transport
|
||||||
|
failure — the circuit must stay closed."""
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.classifier, "mode", "local_decision")
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.classifier, "fallback_tier", 2)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dispatcher.cfg.classifier, "decision",
|
||||||
|
_decision_conf(confidence_min=0.5, tier_enabled=False),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
||||||
|
|
||||||
|
def fake_post(*a, **k):
|
||||||
|
# "Z" is not an option letter and A/B carry almost no mass, so total
|
||||||
|
# option mass (about 0.012) is below coverage_min (0.3) and
|
||||||
|
# parse_logprobs raises RuntimeError before any confidence is computed.
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
json=lambda: {
|
json=lambda: {
|
||||||
"model": "qwen3.5:4b",
|
"model": "qwen3.5:4b",
|
||||||
|
|||||||
Reference in New Issue
Block a user