xsl/meida_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: T1 四档分级降级熔断（T-P1-25b，NFR-COST-02）

Browse Source
This commit is contained in:
xsl
2026-08-14 16:28:22 +08:00
parent fcd7e63ce5 a4d59aca30
commit ddec3973ad
8 changed files with 617 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/budget.py
+243 -18
View File
@@ -1,27 +1,114 @@
"""Global daily cost circuit breaker (架构文档 §13「必须设置的三道成本闸门」).
"""Global daily cost circuit breaker — four-tier graded degradation (架构文档 §13.3).
Gate 1 in §13 is described as "某功能超预算自动降级，并告警". Alerting alone is
not enough: the failure modes that blow up AI spend (retry storms, a cache-key
bug, someone scripting the API) all produce cost faster than a human can react
to a page. So this is an *enforcement* point, not a notification: once the
platform-wide daily spend crosses the threshold, non-critical scenes are
rejected outright.
to a page. So this is an *enforcement* point, and the degradation is graded so
the product winds down gracefully instead of dying all at once. Thresholds are
percentages of ``LLM_DAILY_BUDGET_CENTS``; higher tiers *add* rejected scenes on
top of the lower ones (架构原文「追加拒绝」), never un-block them::
Critical scenes (e.g. content moderation follow-ups, paid-path completions)
opt out via ``SceneConfig.critical`` so that a budget overrun degrades the
product instead of breaking it.
tier 1 warn ≥60% 只告警（指标 + 上升沿日志），不影响请求
tier 2 degrade_secondary ≥80% 拒绝 hd_export / report_regen
tier 3 degrade_generation ≥90% 追加拒绝 hairstyle / tryon
tier 4 block_all ≥100% 拒绝所有非 critical；allow 名单放行
（默认 agent_chat ——「仅保留在途任务与
纯文本对话」）
Critical scenes (``SceneConfig.critical``, e.g. INTENT_CLASSIFY) opt out of
every tier so that a budget overrun degrades the product instead of breaking
it — but their cost still counts towards the water level. 在途任务同理放行：
熔断只拦网关**新请求**，worker 侧不做二次检查。
手动干预（运营/演练，redis-cli 可直接操作）::
SET llm:budget:override 4 # 强制 tier 4（加严演练）
SET llm:budget:override 2 # 水位已到 tier 4 时手动放宽回 tier 2
DEL llm:budget:override # 解除，回落成本水位判定
键 ``llm:budget:override`` 的值为 "1"~"4"（档位序号），优先级高于成本水位；
读不到或值非法则忽略。程序化写入走 :meth:`DailyBudgetGuard.set_override`，
TTL 到当日 UTC+8 24 点 —— override 最多活一个预算日，第二天必然回落水位。
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta, timezone
from decimal import Decimal
from typing import Protocol, runtime_checkable
from typing import Literal, Protocol, runtime_checkable
from packages.llm.errors import DailyBudgetExceededError
from prometheus_client import Counter, Gauge
from pydantic import BaseModel, ConfigDict, Field, field_validator
from packages.llm.errors import DailyBudgetExceededError, SceneDegradedError
from packages.shared.logging import get_logger
from packages.shared.observability import REGISTRY
logger = get_logger(__name__)
_DAY_SECONDS = 60 * 60 * 24
#: 运营手动干预档位的 Redis 键（值为 "1"~"4"；TTL 到当日 UTC+8 24 点）。
OVERRIDE_KEY = "llm:budget:override"
#: blocked 名单中的通配符：拒绝所有非 critical 场景（仅 block_all 档配置 allow 名单）。
BLOCK_ALL: Literal["*"] = "*"
#: 内置默认四档，与 ``Settings.llm_budget_tiers`` 的默认值保持一致（架构 §13.3）。
_DEFAULT_TIERS: list[dict[str, object]] = [
{"threshold": 0.6, "name": "warn", "blocked": []},
{"threshold": 0.8, "name": "degrade_secondary", "blocked": ["hd_export", "report_regen"]},
{"threshold": 0.9, "name": "degrade_generation", "blocked": ["hairstyle", "tryon"]},
{"threshold": 1.0, "name": "block_all", "blocked": BLOCK_ALL, "allow": ["agent_chat"]},
]
# 档位是全局状态（整个平台只有一个水位），挂到共享 REGISTRY 上随 /metrics 暴露。
LLM_BUDGET_TIER = Gauge(
"llm_budget_tier",
"当前生效的日预算降级档位（0=未触发，1~4 递进；手动 override 优先）",
registry=REGISTRY,
)
LLM_BUDGET_BLOCKED = Counter(
"llm_budget_blocked_total",
"被预算降级档位拒绝的网关请求数",
labelnames=("scene",),
registry=REGISTRY,
)
class BudgetTier(BaseModel):
"""一档降级配置（``LLM_BUDGET_TIERS_JSON`` 的元素，架构 §13.3）。
``blocked`` 里的 ``"*"`` 表示该档拒绝全部非 critical 场景（配置允许写成
裸字符串，这里统一归一成列表）；``allow`` 仅 block_all 档有意义。
"""
model_config = ConfigDict(frozen=True)
threshold: float
name: str
blocked: list[str] = Field(default_factory=list)
allow: list[str] = Field(default_factory=list)
@field_validator("blocked", "allow", mode="before")
@classmethod
def _wrap_bare_string(cls, value: object) -> object:
# Settings 默认值里 "blocked": "*" 是裸字符串写法，包成列表让类型恒为 list[str]。
if isinstance(value, str):
return [value]
return value
@property
def blocked_set(self) -> frozenset[str]:
return frozenset(self.blocked)
@property
def allow_set(self) -> frozenset[str]:
return frozenset(self.allow)
def blocks_all(self) -> bool:
return BLOCK_ALL in self.blocked
@runtime_checkable
class CostCounter(Protocol):
@@ -31,20 +118,36 @@ class CostCounter(Protocol):
async def get(self, key: str) -> Decimal: ...
async def set(self, key: str, value: str, ttl_seconds: int) -> None: ...
async def delete(self, key: str) -> None: ...
class InMemoryCostCounter:
"""Process-local counter. Correct only for a single process — tests only."""
def __init__(self) -> None:
self._values: dict[str, Decimal] = {}
self._strings: dict[str, str] = {}
async def incr(self, key: str, amount: Decimal, ttl_seconds: int) -> Decimal:
self._values[key] = self._values.get(key, Decimal("0")) + amount
return self._values[key]
async def get(self, key: str) -> Decimal:
if key in self._strings:
# 非法字符串在这里抛 InvalidOperation，与 Redis 实现的行为一致，
# 调用方（override 读取）按「值非法则忽略」处理。
return Decimal(self._strings[key])
return self._values.get(key, Decimal("0"))
async def set(self, key: str, value: str, ttl_seconds: int) -> None:
self._strings[key] = value
async def delete(self, key: str) -> None:
self._strings.pop(key, None)
self._values.pop(key, None)
class RedisCostCounter:
"""Cross-process counter backed by ``INCRBYFLOAT`` + a rolling TTL."""
@@ -63,9 +166,15 @@ class RedisCostCounter:
value = await self._redis.get(key) # type: ignore[attr-defined]
return Decimal(str(value)) if value is not None else Decimal("0")
async def set(self, key: str, value: str, ttl_seconds: int) -> None:
await self._redis.set(key, value, ex=ttl_seconds) # type: ignore[attr-defined]
async def delete(self, key: str) -> None:
await self._redis.delete(key) # type: ignore[attr-defined]
class DailyBudgetGuard:
"""Hard platform-wide daily AI budget."""
"""Platform-wide daily AI budget with four-tier graded degradation (§13.3)."""
def __init__(
self,
@@ -84,9 +193,13 @@ class DailyBudgetGuard:
self._key_prefix = key_prefix
self._tz = timezone(timedelta(hours=tz_offset_hours))
self._clock = clock
# 四档分级降级的配置（架构 §13.3 / T-P1-25b）。T1 任务实现判定逻辑；
# 这里先接收配置，使 deps 装配层不必随后再改。None = 使用内置默认四档。
self._tiers_config = tiers
# 档位配置（LLM_BUDGET_TIERS_JSON）。None = 内置默认四档。解析失败在装配期
# 就炸出来：一份写坏的降级配置不应该等到平台烧超预算时才被发现。
self._tiers = [BudgetTier.model_validate(t) for t in (tiers if tiers else _DEFAULT_TIERS)]
self._tiers.sort(key=lambda t: t.threshold)
self._thresholds = [Decimal(str(t.threshold)) for t in self._tiers]
# 上升沿告警的实例内游标：避免每个请求都刷一遍 budget_tier_raised。
self._last_tier = 0
@property
def enabled(self) -> bool:
@@ -109,13 +222,125 @@ class DailyBudgetGuard:
async def spent_today(self) -> Decimal:
return await self._counter.get(self.current_key())
async def ensure_within_budget(self, *, scene: str, critical: bool = False) -> None:
"""Raise :class:`DailyBudgetExceededError` for non-critical scenes over budget."""
if not self._enabled or critical:
# -- tiered degradation ------------------------------------------------
async def current_tier(self) -> int:
"""当前生效档位（0=未触发，1~4 递进）。手动 override 优先于成本水位。"""
if not self._enabled:
return 0
override = await self._override_tier()
if override is not None:
return override
return self._tier_for_spend(await self.spent_today())
async def tier_name(self) -> str:
"""当前档位名（warn/degrade_secondary/degrade_generation/block_all）。"""
tier = await self.current_tier()
return self._tiers[tier - 1].name if tier else ""
async def blocked_scenes(self) -> frozenset[str] | Literal["*"]:
"""当前档位**累计**拒绝的场景名单；``"*"`` 表示拒绝全部非 critical。"""
return self._blocked_scenes_at(await self.current_tier())
async def ensure_scene_allowed(self, *, scene: str, critical: bool = False) -> None:
"""Raise when the current tier rejects ``scene``.
- critical 场景全档豁免（但照常计入成本水位）；
- tier 2/3 且 scene 命中累计 blocked 名单 → :class:`SceneDegradedError`；
- tier 4（blocked="*"）且 scene 不在 allow 名单 →
:class:`DailyBudgetExceededError`——保持 100% 档的既有语义与 60003
错误码不变（现有测试与告警文案都依赖它）。
"""
if not self._enabled:
return
override = await self._override_tier()
spent = await self.spent_today()
if spent >= self._budget:
raise DailyBudgetExceededError(str(spent), str(self._budget), scene)
tier = override if override is not None else self._tier_for_spend(spent)
self._observe_tier(tier, spent=spent)
if critical or tier == 0:
return
blocked = self._blocked_scenes_at(tier)
if blocked == BLOCK_ALL:
if scene not in self._tiers[tier - 1].allow_set:
LLM_BUDGET_BLOCKED.labels(scene).inc()
raise DailyBudgetExceededError(str(spent), str(self._budget), scene)
return
if scene in blocked:
LLM_BUDGET_BLOCKED.labels(scene).inc()
raise SceneDegradedError(scene, self._tiers[tier - 1].name)
async def ensure_within_budget(self, *, scene: str, critical: bool = False) -> None:
"""兼容入口：与 :meth:`ensure_scene_allowed` 等价（存量调用方在用）。"""
await self.ensure_scene_allowed(scene=scene, critical=critical)
# -- manual override ---------------------------------------------------
async def set_override(self, tier: int) -> None:
"""运营/演练：强制指定当前档位，TTL 到当日 UTC+8 24 点。"""
if not 1 <= tier <= len(self._tiers):
msg = f"override tier must be in 1..{len(self._tiers)}, got {tier}"
raise ValueError(msg)
await self._counter.set(OVERRIDE_KEY, str(tier), self._seconds_until_day_end())
async def clear_override(self) -> None:
"""解除 override，回落成本水位判定。"""
await self._counter.delete(OVERRIDE_KEY)
# -- internals ----------------------------------------------------------
async def _override_tier(self) -> int | None:
"""Read the manual override; unreadable/invalid values are ignored."""
try:
value = await self._counter.get(OVERRIDE_KEY)
tier = int(value)
except Exception:
# 键不存在时计数器返回 0（→ 下方范围校验拒绝）；这里兜底的是
# 值非法（如 "abc"）或存储故障：override 是运营工具，绝不能
# 因为它把请求路径打挂。
return None
if 1 <= tier <= len(self._tiers):
return tier
return None
def _tier_for_spend(self, spent: Decimal) -> int:
"""水位判定：取「最后一个被跨过阈值」的档位（阈值已升序）。"""
tier = 0
for index, threshold in enumerate(self._thresholds, start=1):
if spent >= threshold * self._budget:
tier = index
return tier
def _blocked_scenes_at(self, tier: int) -> frozenset[str] | Literal["*"]:
"""档位越高拒绝越多：取 1..tier 各档 blocked 的并集（「追加拒绝」）。"""
scenes: set[str] = set()
for spec in self._tiers[:tier]:
if spec.blocks_all():
return BLOCK_ALL
scenes.update(spec.blocked_set)
return frozenset(scenes)
def _observe_tier(self, tier: int, *, spent: Decimal) -> None:
LLM_BUDGET_TIER.set(tier)
if tier <= self._last_tier:
return
self._last_tier = tier
if tier == 0:
return
# 只在上升沿告警一次：水位在 90% 徘徊时每个请求都打日志等于没有告警。
logger.warning(
"budget_tier_raised",
tier=tier,
tier_name=self._tiers[tier - 1].name,
spent=str(spent),
budget=str(self._budget),
)
def _seconds_until_day_end(self) -> int:
"""到当日 UTC+8 24 点的秒数（预算日切换即失效）。"""
local = self._now().astimezone(self._tz)
day_end = local.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1)
return max(int((day_end - local).total_seconds()), 1)
async def add_cost(self, cost_cents: Decimal) -> Decimal:
"""Accumulate spend. Called after every provider attempt, success or not."""
packages/llm/errors.py
+16
View File
@@ -138,6 +138,22 @@ class DailyBudgetExceededError(LLMError):
)
class SceneDegradedError(LLMError):
"""A scene was rejected by a *graded* budget tier (架构文档 §13.3 tier 2/3).
与 :class:`DailyBudgetExceededError` 的区别：平台还没烧穿预算，只是按档位
关停部分高成本场景（hd_export/report_regen/hairstyle/tryon）。用户看到的
是「该功能今日暂时关闭」(LLM_SCENE_DEGRADED, 60006) 而非「平台已达上限」。
"""
code = "llm_scene_degraded"
def __init__(self, scene: str, tier_name: str) -> None:
self.scene = scene
self.tier_name = tier_name
super().__init__(f"scene '{scene}' blocked by budget tier '{tier_name}'")
class PromptNotFoundError(LLMError):
code = "llm_prompt_not_found"
packages/llm/gateway.py
+4 -3
View File
@@ -9,7 +9,7 @@ Built-in capabilities, all exercised on every call:
Prompt 版本 模板文件按 id+version 加载，版本号随每次调用落库
缓存前缀纪律 固定前缀含时间戳/随机 ID/昵称 → 拒绝或告警 (§7.2, TD-04)
成本计量 每次 provider 调用（含失败与重试）写一行 ``ob_ai_call``
预算保护 平台级日成本硬熔断：超限即拒绝非关键场景 (§13)
预算保护 平台级日成本四档分级降级：warn/secondary/generation/block_all (§13.3)
=================== =========================================================
Deliberately not here yet: streaming (SSE), 审核联动 (input/output moderation),
@@ -106,8 +106,9 @@ class LLMGateway:
) -> Completion:
config = self._router.resolve(scene)
# Gate 1: platform-wide daily cost breaker, checked before any spend.
await self._budget.ensure_within_budget(scene=scene.value, critical=config.critical)
# Gate 1: four-tier graded daily cost breaker (架构文档 §13.3),
# checked before any spend — 在途任务不受影响，这里只拦新请求。
await self._budget.ensure_scene_allowed(scene=scene.value, critical=config.critical)
template = self._prompts.get(config.prompt_id, config.prompt_version)
violations = validate_cache_prefix(