feat(llm): provider-abstract gateway with cost and failure controls
DeepSeek (OpenAI-compatible) as primary with a 智谱 GLM slot that is a real routing path but raises until the 算法备案 model list is confirmed. Adds scene routing, failover, a per-provider circuit breaker, structured output with retry-on-validation-failure, prompts loaded by id+version from prompts/, and cost attribution written to ob_ai_call for every attempt -- successes, retries and failures alike. Two controls exist because they are cheap now and expensive later: a cache-prefix validator that rejects timestamps, uuids and nicknames in the cached system prefix (a volatile prefix silently multiplies input cost), and a hard platform-wide daily budget that rejects non-critical scenes instead of merely alerting, since a retry storm outruns any human on-call.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""LLM 供应商适配器。DeepSeek 为主，智谱 GLM 为备。"""
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
"""Provider abstraction.
|
||||
|
||||
架构文档 §4.3 requires the gateway interface to be designed as if it were a
|
||||
remote call, so providers take a fully-specified request (explicit timeout,
|
||||
serialisable payload) and return a fully-specified response. No provider is
|
||||
allowed to reach back into gateway state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from packages.llm.types import ProviderRequest, ProviderResponse
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class LLMProvider(Protocol):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Stable identifier used in routing config and in ``ob_ai_call``."""
|
||||
...
|
||||
|
||||
@property
|
||||
def default_model(self) -> str:
|
||||
"""Model used when the scene config does not pin one."""
|
||||
...
|
||||
|
||||
async def complete(self, request: ProviderRequest) -> ProviderResponse:
|
||||
"""Perform one completion. Must raise ``packages.llm.errors.ProviderError``."""
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Release connections."""
|
||||
...
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
"""DeepSeek provider — primary LLM supplier (TD-04, 架构文档 §5).
|
||||
|
||||
DeepSeek exposes an OpenAI-compatible API at ``https://api.deepseek.com/v1``.
|
||||
Its usage block reports ``prompt_cache_hit_tokens`` / ``prompt_cache_miss_tokens``,
|
||||
which is what makes the cache-prefix discipline in :mod:`packages.llm.prompts`
|
||||
measurable (NFR-COST-03).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from packages.llm.errors import (
|
||||
ProviderAuthError,
|
||||
ProviderBadRequestError,
|
||||
ProviderError,
|
||||
ProviderRateLimitedError,
|
||||
ProviderServerError,
|
||||
ProviderTimeoutError,
|
||||
)
|
||||
from packages.llm.types import ProviderRequest, ProviderResponse, Usage
|
||||
|
||||
|
||||
class DeepSeekProvider:
|
||||
"""OpenAI-compatible chat-completions client."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
base_url: str = "https://api.deepseek.com/v1",
|
||||
default_model: str = "deepseek-v4-flash",
|
||||
client: httpx.AsyncClient | None = None,
|
||||
name: str = "deepseek",
|
||||
) -> None:
|
||||
self._api_key = api_key
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._default_model = default_model
|
||||
self._client = client or httpx.AsyncClient(timeout=httpx.Timeout(60.0))
|
||||
self._name = name
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def default_model(self) -> str:
|
||||
return self._default_model
|
||||
|
||||
def _payload(self, request: ProviderRequest) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"model": request.model,
|
||||
"messages": [{"role": m.role, "content": m.content} for m in request.messages],
|
||||
"temperature": request.temperature,
|
||||
"max_tokens": request.max_tokens,
|
||||
"stream": request.stream,
|
||||
}
|
||||
if request.json_mode:
|
||||
payload["response_format"] = {"type": "json_object"}
|
||||
return payload
|
||||
|
||||
async def complete(self, request: ProviderRequest) -> ProviderResponse:
|
||||
url = f"{self._base_url}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
try:
|
||||
response = await self._client.post(
|
||||
url,
|
||||
json=self._payload(request),
|
||||
headers=headers,
|
||||
timeout=request.timeout_seconds,
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise ProviderTimeoutError(
|
||||
self._name, f"request timed out after {request.timeout_seconds}s"
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise ProviderServerError(self._name, f"transport error: {exc}") from exc
|
||||
|
||||
self._raise_for_status(response)
|
||||
return self._parse(response.json(), fallback_model=request.model)
|
||||
|
||||
def _raise_for_status(self, response: httpx.Response) -> None:
|
||||
status = response.status_code
|
||||
if status < 400:
|
||||
return
|
||||
detail = _short_body(response)
|
||||
if status == 429:
|
||||
raise ProviderRateLimitedError(self._name, detail, status_code=status)
|
||||
if status in (401, 403):
|
||||
raise ProviderAuthError(self._name, detail, status_code=status)
|
||||
if status >= 500:
|
||||
raise ProviderServerError(self._name, detail, status_code=status)
|
||||
raise ProviderBadRequestError(self._name, detail, status_code=status)
|
||||
|
||||
def _parse(self, data: dict[str, Any], *, fallback_model: str) -> ProviderResponse:
|
||||
try:
|
||||
choice = data["choices"][0]
|
||||
text = choice["message"]["content"] or ""
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ProviderServerError(self._name, f"malformed response: {data!r}") from exc
|
||||
|
||||
usage_raw: dict[str, Any] = data.get("usage") or {}
|
||||
cached = int(
|
||||
usage_raw.get("prompt_cache_hit_tokens")
|
||||
or (usage_raw.get("prompt_tokens_details") or {}).get("cached_tokens")
|
||||
or 0
|
||||
)
|
||||
usage = Usage(
|
||||
tokens_in=int(usage_raw.get("prompt_tokens", 0)),
|
||||
tokens_out=int(usage_raw.get("completion_tokens", 0)),
|
||||
tokens_cached=cached,
|
||||
)
|
||||
return ProviderResponse(
|
||||
text=text,
|
||||
model=str(data.get("model") or fallback_model),
|
||||
usage=usage,
|
||||
finish_reason=choice.get("finish_reason"),
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
def _short_body(response: httpx.Response, limit: int = 300) -> str:
|
||||
try:
|
||||
body = response.text
|
||||
except (UnicodeDecodeError, httpx.ResponseNotRead): # pragma: no cover
|
||||
return f"HTTP {response.status_code}"
|
||||
return f"HTTP {response.status_code}: {body[:limit]}"
|
||||
|
||||
|
||||
__all__ = ["DeepSeekProvider", "ProviderError"]
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
"""智谱 GLM provider — backup supplier slot (TD-04, 架构文档 §2 降级表).
|
||||
|
||||
**Structure only.** The routing/failover code path is real: the gateway will
|
||||
route to this provider, the breaker will count its failures, and ``ob_ai_call``
|
||||
will record the attempt. Only the HTTP call is missing.
|
||||
|
||||
Before implementing, resolve the open compliance item in 需求规格说明 §5.3:
|
||||
「备案覆盖的模型清单核实 —— 若备案未覆盖 DeepSeek + GLM 两家，双路由属超范围」.
|
||||
Shipping a working second route ahead of that check is a compliance risk, not a
|
||||
resilience win.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from packages.llm.errors import ProviderNotImplementedError
|
||||
from packages.llm.types import ProviderRequest, ProviderResponse
|
||||
|
||||
_NOT_IMPLEMENTED_MESSAGE = (
|
||||
"智谱 GLM provider is a structural placeholder and has no implementation. "
|
||||
"To enable it: (1) confirm the 算法备案 model list covers GLM "
|
||||
"(需求规格说明 §5.3), (2) implement complete() against "
|
||||
"POST {base_url}/chat/completions, (3) map GLM usage fields onto "
|
||||
"packages.llm.types.Usage, (4) set ZHIPU_API_KEY. "
|
||||
"Until then the gateway treats this slot as a permanently failing provider."
|
||||
)
|
||||
|
||||
|
||||
class ZhipuGLMProvider:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str = "",
|
||||
base_url: str = "https://open.bigmodel.cn/api/paas/v4",
|
||||
default_model: str = "glm-4-flash",
|
||||
name: str = "zhipu",
|
||||
) -> None:
|
||||
self._api_key = api_key
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._default_model = default_model
|
||||
self._name = name
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def default_model(self) -> str:
|
||||
return self._default_model
|
||||
|
||||
async def complete(self, request: ProviderRequest) -> ProviderResponse:
|
||||
raise ProviderNotImplementedError(self._name, _NOT_IMPLEMENTED_MESSAGE)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["ZhipuGLMProvider"]
|
||||
Reference in New Issue
Block a user