xsl
54dc741811
DeepSeek (OpenAI-compatible) as primary with a 智谱 GLM slot that is a real routing path but raises until the 算法备案 model list is confirmed. Adds scene routing, failover, a per-provider circuit breaker, structured output with retry-on-validation-failure, prompts loaded by id+version from prompts/, and cost attribution written to ob_ai_call for every attempt -- successes, retries and failures alike. Two controls exist because they are cheap now and expensive later: a cache-prefix validator that rejects timestamps, uuids and nicknames in the cached system prefix (a volatile prefix silently multiplies input cost), and a hard platform-wide daily budget that rejects non-critical scenes instead of merely alerting, since a retry storm outruns any human on-call.
137 lines
4.0 KiB
Python
Executable File
137 lines
4.0 KiB
Python
Executable File
"""Circuit breaker for provider failover (架构文档 §8.1 故障转移).
|
|
|
|
主供应商超时/5xx/限流 → 自动切备用；连续失败触发熔断，冷却后半开探测.
|
|
|
|
The clock is injectable so tests do not sleep.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from collections.abc import Callable
|
|
from enum import StrEnum
|
|
|
|
|
|
class CircuitState(StrEnum):
|
|
CLOSED = "closed"
|
|
OPEN = "open"
|
|
HALF_OPEN = "half_open"
|
|
|
|
|
|
class CircuitBreaker:
|
|
"""Per-provider breaker.
|
|
|
|
closed → normal traffic; ``failure_threshold`` consecutive failures open it.
|
|
open → all calls rejected until ``cooldown_seconds`` elapse.
|
|
half_open → a limited number of probe calls; one success closes it, one
|
|
failure re-opens it and restarts the cooldown.
|
|
"""
|
|
|
|
__slots__ = (
|
|
"_clock",
|
|
"_cooldown",
|
|
"_failures",
|
|
"_half_open_calls",
|
|
"_half_open_max",
|
|
"_opened_at",
|
|
"_state",
|
|
"_threshold",
|
|
"name",
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
name: str,
|
|
*,
|
|
failure_threshold: int = 3,
|
|
cooldown_seconds: float = 30.0,
|
|
half_open_max_calls: int = 1,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
) -> None:
|
|
if failure_threshold < 1:
|
|
msg = "failure_threshold must be >= 1"
|
|
raise ValueError(msg)
|
|
self.name = name
|
|
self._threshold = failure_threshold
|
|
self._cooldown = cooldown_seconds
|
|
self._half_open_max = half_open_max_calls
|
|
self._clock = clock
|
|
self._state = CircuitState.CLOSED
|
|
self._failures = 0
|
|
self._opened_at = 0.0
|
|
self._half_open_calls = 0
|
|
|
|
@property
|
|
def state(self) -> CircuitState:
|
|
return self._state
|
|
|
|
@property
|
|
def consecutive_failures(self) -> int:
|
|
return self._failures
|
|
|
|
def allow(self) -> bool:
|
|
"""Whether a call may be attempted right now. Mutates state on cooldown expiry."""
|
|
if self._state is CircuitState.CLOSED:
|
|
return True
|
|
if self._state is CircuitState.OPEN:
|
|
if self._clock() - self._opened_at >= self._cooldown:
|
|
self._state = CircuitState.HALF_OPEN
|
|
self._half_open_calls = 1
|
|
return True
|
|
return False
|
|
# HALF_OPEN
|
|
if self._half_open_calls < self._half_open_max:
|
|
self._half_open_calls += 1
|
|
return True
|
|
return False
|
|
|
|
def record_success(self) -> None:
|
|
self._state = CircuitState.CLOSED
|
|
self._failures = 0
|
|
self._half_open_calls = 0
|
|
|
|
def record_failure(self) -> None:
|
|
if self._state is CircuitState.HALF_OPEN:
|
|
self._trip()
|
|
return
|
|
self._failures += 1
|
|
if self._failures >= self._threshold:
|
|
self._trip()
|
|
|
|
def _trip(self) -> None:
|
|
self._state = CircuitState.OPEN
|
|
self._opened_at = self._clock()
|
|
self._half_open_calls = 0
|
|
self._failures = self._threshold
|
|
|
|
|
|
class CircuitBreakerRegistry:
|
|
"""One breaker per provider, created on first use."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
failure_threshold: int = 3,
|
|
cooldown_seconds: float = 30.0,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
) -> None:
|
|
self._breakers: dict[str, CircuitBreaker] = {}
|
|
self._failure_threshold = failure_threshold
|
|
self._cooldown_seconds = cooldown_seconds
|
|
self._clock = clock
|
|
|
|
def get(self, provider: str) -> CircuitBreaker:
|
|
breaker = self._breakers.get(provider)
|
|
if breaker is None:
|
|
breaker = CircuitBreaker(
|
|
provider,
|
|
failure_threshold=self._failure_threshold,
|
|
cooldown_seconds=self._cooldown_seconds,
|
|
clock=self._clock,
|
|
)
|
|
self._breakers[provider] = breaker
|
|
return breaker
|
|
|
|
def snapshot(self) -> dict[str, str]:
|
|
return {name: breaker.state.value for name, breaker in self._breakers.items()}
|