xsl
93b88db2f5
Covers routing, failover, the circuit breaker, structured-output retries, the cache-prefix validator, cost computation including cached-token pricing, the daily budget breaker, the quota protocol, the task state machine and log redaction. HTTP is mocked -- no unit test spends money. The integration test runs the real slice against PostgreSQL and Redis with a faked provider.
86 lines
2.6 KiB
Python
Executable File
86 lines
2.6 KiB
Python
Executable File
"""Circuit breaker behaviour (架构文档 §8.1 故障转移)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from packages.llm.circuit import CircuitBreaker, CircuitBreakerRegistry, CircuitState
|
|
|
|
|
|
class FakeClock:
|
|
def __init__(self) -> None:
|
|
self.now = 0.0
|
|
|
|
def __call__(self) -> float:
|
|
return self.now
|
|
|
|
def advance(self, seconds: float) -> None:
|
|
self.now += seconds
|
|
|
|
|
|
def test_starts_closed_and_allows_traffic() -> None:
|
|
breaker = CircuitBreaker("p", failure_threshold=3, clock=FakeClock())
|
|
assert breaker.state is CircuitState.CLOSED
|
|
assert breaker.allow()
|
|
|
|
|
|
def test_opens_only_after_threshold_consecutive_failures() -> None:
|
|
breaker = CircuitBreaker("p", failure_threshold=3, clock=FakeClock())
|
|
breaker.record_failure()
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.CLOSED
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
assert not breaker.allow()
|
|
|
|
|
|
def test_success_resets_the_failure_streak() -> None:
|
|
breaker = CircuitBreaker("p", failure_threshold=3, clock=FakeClock())
|
|
breaker.record_failure()
|
|
breaker.record_failure()
|
|
breaker.record_success()
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.CLOSED
|
|
|
|
|
|
def test_half_open_probe_after_cooldown_then_closes_on_success() -> None:
|
|
clock = FakeClock()
|
|
breaker = CircuitBreaker("p", failure_threshold=1, cooldown_seconds=30.0, clock=clock)
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
|
|
clock.advance(29.0)
|
|
assert not breaker.allow()
|
|
|
|
clock.advance(2.0)
|
|
assert breaker.allow()
|
|
assert breaker.state is CircuitState.HALF_OPEN
|
|
# only one probe is admitted
|
|
assert not breaker.allow()
|
|
|
|
breaker.record_success()
|
|
assert breaker.state is CircuitState.CLOSED
|
|
assert breaker.allow()
|
|
|
|
|
|
def test_half_open_failure_reopens_and_restarts_cooldown() -> None:
|
|
clock = FakeClock()
|
|
breaker = CircuitBreaker("p", failure_threshold=1, cooldown_seconds=10.0, clock=clock)
|
|
breaker.record_failure()
|
|
clock.advance(11.0)
|
|
assert breaker.allow()
|
|
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
assert not breaker.allow()
|
|
|
|
clock.advance(11.0)
|
|
assert breaker.allow()
|
|
|
|
|
|
def test_registry_returns_one_breaker_per_provider() -> None:
|
|
registry = CircuitBreakerRegistry(failure_threshold=2, cooldown_seconds=5.0)
|
|
assert registry.get("deepseek") is registry.get("deepseek")
|
|
assert registry.get("deepseek") is not registry.get("zhipu")
|
|
registry.get("deepseek").record_failure()
|
|
registry.get("deepseek").record_failure()
|
|
assert registry.snapshot() == {"deepseek": "open", "zhipu": "closed"}
|