f76ee5a9f6
BLOCKING A: A client disconnect (GeneratorExit) in the streaming generator runs the finally block, which verified the empty content as 'malformed' and opened a breaker against a healthy model. Added stream_completed flag (initialized False, set True after the iteration loop exits normally) and gated the breaker call on it. Disconnect leaves the flag False, recording nothing. BLOCKING B: content_fault_warnings was wired into scoring_coverage but never registered in the TUI warning tripwire. Added dual-window (alert+baseline) pattern detection matching rejection_warnings: 'new content fault:' for novel patterns (absent from baseline), 'content fault:' for known patterns (present in baseline, count >= threshold). Registered both classes in WARN_CLASS_MATCHERS with lookbehind on the rate matcher. Seeded chernobyl fixture with both novel and known malformed verification rows.
128 lines
4.9 KiB
Python
128 lines
4.9 KiB
Python
"""Unit tests for the passive circuit breaker (circuit_breaker.py).
|
|
|
|
Offline, no network. Drive the pure get/record/clear logic directly. time.time
|
|
is monkeypatched so cooldown expiry is deterministic (no wall-clock dependence).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
import circuit_breaker
|
|
from verification import verify_response
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_circuit():
|
|
circuit_breaker.clear()
|
|
yield
|
|
circuit_breaker.clear()
|
|
|
|
|
|
def _freeze(monkeypatch, t: float):
|
|
monkeypatch.setattr(circuit_breaker.time, "time", lambda: t)
|
|
|
|
|
|
def test_is_down_miss_returns_false():
|
|
assert circuit_breaker.is_down("m1", "p1", now=1000.0) is False
|
|
|
|
|
|
def test_first_failure_sets_initial_cooldown(monkeypatch):
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
# Within the cooldown window it's down.
|
|
assert circuit_breaker.is_down("m1", "p1", now=1000.0) is True
|
|
# Just past the window it is not down (passive recovery probe).
|
|
assert circuit_breaker.is_down("m1", "p1", now=1000.0 + 30.0) is False
|
|
|
|
|
|
def test_second_consecutive_failure_doubles_cooldown(monkeypatch):
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
_freeze(monkeypatch, 1000.0 + 100.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
# Second failure at t=1100: cooldown doubled to 60, down_until = 1160.
|
|
assert circuit_breaker.is_down("m1", "p1", now=1159.0) is True
|
|
assert circuit_breaker.is_down("m1", "p1", now=1160.0) is False
|
|
|
|
|
|
def test_cooldown_never_exceeds_max(monkeypatch):
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> 10
|
|
_freeze(monkeypatch, 2000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> 20
|
|
_freeze(monkeypatch, 3000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> capped 25
|
|
_freeze(monkeypatch, 4000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 10.0, 25.0, 2.0) # -> capped 25
|
|
assert circuit_breaker.is_down("m1", "p1", now=4024.0) is True
|
|
assert circuit_breaker.is_down("m1", "p1", now=4025.0) is False
|
|
|
|
|
|
def test_success_clears_and_next_failure_restarts_at_initial(monkeypatch):
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
_freeze(monkeypatch, 2000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0) # -> 60
|
|
circuit_breaker.record_success("m1", "p1")
|
|
assert circuit_breaker.is_down("m1", "p1", now=2000.0) is False
|
|
_freeze(monkeypatch, 3000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
# Restarts at the INITIAL cooldown, not wherever backoff had climbed.
|
|
assert circuit_breaker.is_down("m1", "p1", now=3029.0) is True
|
|
assert circuit_breaker.is_down("m1", "p1", now=3030.0) is False
|
|
|
|
|
|
def test_state_is_keyed_by_model_and_provider(monkeypatch):
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
# Same model, different provider is unaffected.
|
|
assert circuit_breaker.is_down("m1", "p2", now=1000.0) is False
|
|
assert circuit_breaker.is_down("m1", "p1", now=1000.0) is True
|
|
|
|
|
|
def test_malformed_verdict_trips_breaker(monkeypatch):
|
|
result = verify_response("", None, has_tool_calls=False)
|
|
assert result.verdict == "malformed"
|
|
_freeze(monkeypatch, 1000.0)
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
assert circuit_breaker.is_down("m1", "p1", now=1000.0) is True
|
|
|
|
|
|
def test_tool_call_empty_content_does_not_trip_breaker():
|
|
result = verify_response("", None, has_tool_calls=True)
|
|
assert result.verdict == "unverifiable"
|
|
assert result.verdict != "malformed"
|
|
|
|
|
|
def test_ok_verdict_does_not_trip_breaker():
|
|
result = verify_response(
|
|
"```python\nprint('hello')\n```", None, has_tool_calls=False
|
|
)
|
|
assert result.verdict == "ok"
|
|
assert result.verdict != "malformed"
|
|
|
|
|
|
def test_truncated_verdict_does_not_trip_breaker():
|
|
result = verify_response("some text", "length", has_tool_calls=False)
|
|
assert result.verdict == "truncated"
|
|
assert result.verdict != "malformed"
|
|
|
|
|
|
def test_stream_cancelled_does_not_trip_breaker():
|
|
"""A client disconnect (GeneratorExit) must NOT record_failure.
|
|
|
|
The streaming generator's finally block runs on disconnect. An interrupted
|
|
stream verifies as 'malformed' (empty content), but that is a pipeline
|
|
event, not a model fault. The stream_completed flag gates the breaker call.
|
|
"""
|
|
# Simulate: verify_response on empty content returns malformed
|
|
result = verify_response("", None, has_tool_calls=False)
|
|
assert result.verdict == "malformed"
|
|
|
|
# But stream_completed=False means the breaker should NOT be touched
|
|
stream_completed = False
|
|
if stream_completed:
|
|
circuit_breaker.record_failure("m1", "p1", 30.0, 600.0, 2.0)
|
|
assert circuit_breaker.is_down("m1", "p1", now=0.0) is False
|