xsl/meida_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): price by call time and stop paying for thinking tokens

Browse Source 
Two defects that only surface against the real DeepSeek API, both of which
break the cost model rather than the code:

deepseek-v4-flash enables thinking mode by default. The same single-turn
question took 13.5s with it on and 1.8s with it off, and the thinking tokens
are billed at the output rate. That misses NFR-PERF-02b (15s per turn) and
inflates every conversational call. SceneConfig now carries an explicit
`thinking` flag, all four V1 scenes set it to false, and the provider always
sends the parameter rather than relying on the upstream default. Reasoning
tokens are recorded separately in Usage.tokens_reasoning so the cost of ever
turning it back on is visible.

DeepSeek moves to peak/off-peak pricing on 2026-08-17, with peak (Beijing
09:00-12:00 and 14:00-18:00) at twice the off-peak rate. Reports are generated
mostly during peak hours, so a single blended rate would leave ob_ai_call --
the only input to V2 pricing -- off by nearly a factor of two.
PeakAwareRateTable resolves the rate at call time; cost is computed and stored
then, never recomputed later at whatever the current rate happens to be.

Also here:
- Replace the live API key in the redaction test with a synthetic one. The
  test that proves we redact secrets was itself a secret leak waiting for its
  first commit.
- Correct LLM_COST_RATES_JSON, which held placeholder rates.
- Make ids.SnowflakeGenerator take an injectable clock so rollover and
  clock-rollback are testable without patching a slots class.
- Rename exceptions to the -Error suffix (N818) and clear the remaining ruff,
  mypy and import-linter findings, including two dead lines: a duplicate
  pytestmark assignment and a no-op assert on a None return.

Gates: ruff, mypy (96 files incl. tests), 4/4 import contracts,
detect-secrets, 166 unit + 3 integration tests, migration up/down round-trip.
This commit is contained in:
xsl
2026-08-14 00:54:40 +08:00
parent 8a4720472b
commit f46ecc5eb0
34 changed files with 1793 additions and 988 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/domain/ids.py
+4 -2
View File
@@ -22,7 +22,7 @@ _MAX_NODE_ID = (1 << _NODE_BITS) - 1
_SEQUENCE_MASK = (1 << _SEQUENCE_BITS) - 1
def _wall_clock_ms() -> int:
def _system_clock_ms() -> int:
return int(time.time() * 1000)
@@ -45,7 +45,9 @@ class SnowflakeGenerator:
self._sequence = 0
self._last_ms = -1
self._lock = threading.Lock()
self._clock = clock or _wall_clock_ms
# 时钟可注入：序号回绕与时钟回拨这两条分支只能靠假时钟覆盖，
# 真等一毫秒的测试既慢又不确定。
self._clock = clock or _system_clock_ms
def _now_ms(self) -> int:
return self._clock()
packages/llm/__init__.py
+6
View File
@@ -12,8 +12,11 @@ from packages.llm.cost import (
InMemoryAICallRecorder,
ModelRate,
NullAICallRecorder,
PeakAwareRateTable,
RateResolver,
RateTable,
compute_cost_cents,
is_peak,
)
from packages.llm.errors import (
AllProvidersFailedError,
@@ -47,10 +50,12 @@ __all__ = [
"Message",
"ModelRate",
"NullAICallRecorder",
"PeakAwareRateTable",
"PromptRef",
"PromptRegistry",
"PromptTemplate",
"ProviderError",
"RateResolver",
"RateTable",
"RedisCostCounter",
"Scene",
@@ -60,5 +65,6 @@ __all__ = [
"Usage",
"build_scene_configs",
"compute_cost_cents",
"is_peak",
"validate_cache_prefix",
]
packages/llm/cost.py
+49 -1
View File
@@ -13,7 +13,7 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from datetime import UTC, datetime, time, timedelta, timezone
from decimal import ROUND_HALF_UP, Decimal
from typing import Protocol, runtime_checkable
@@ -22,6 +22,15 @@ from packages.llm.types import Usage
_MILLION = Decimal("1000000")
_QUANT = Decimal("0.000001")
BEIJING = timezone(timedelta(hours=8))
# DeepSeek 自 2026-08-17 00:00（北京时间）起改为峰谷定价，高峰单价是空闲时段的 2 倍。
# 官方口径的高峰时段：北京时间 09:00-12:00 与 14:00-18:00。
PEAK_WINDOWS: tuple[tuple[time, time], ...] = (
(time(9, 0), time(12, 0)),
(time(14, 0), time(18, 0)),
)
@dataclass(frozen=True, slots=True)
class ModelRate:
@@ -55,6 +64,37 @@ class RateTable:
return tuple(sorted(self._rates))
def is_peak(at: datetime | None = None) -> bool:
"""调用时刻是否落在供应商的高峰时段（按北京时间判定）。"""
moment = (at or datetime.now(UTC)).astimezone(BEIJING).timetz()
return any(start <= moment.replace(tzinfo=None) < end for start, end in PEAK_WINDOWS)
class PeakAwareRateTable:
"""按调用时刻在「高峰 / 空闲」两套单价之间选择。
为什么不能只存一套单价：高峰价是空闲价的 2 倍，而报告生成这类任务大多发生在
白天高峰时段。用单一均价记账会让 ``ob_ai_call`` 的成本与账单对不上，
而这张表是 V2 定价的唯一依据 —— 记错等于把定价建在错的数字上。
时段判断放在算成本的这一刻，不是事后按当前价重算：跨时段重算会把历史成本改掉。
"""
def __init__(
self, off_peak: RateTable, peak: RateTable | None = None, *, enabled: bool = True
) -> None:
self._off_peak = off_peak
self._peak = peak or off_peak
self._enabled = enabled
def for_model(self, model: str, at: datetime | None = None) -> ModelRate:
table = self._peak if self._enabled and is_peak(at) else self._off_peak
return table.for_model(model)
def known_models(self) -> tuple[str, ...]:
return self._off_peak.known_models()
def compute_cost_cents(usage: Usage, rate: ModelRate) -> Decimal:
"""Cost of one call, in cents, with cache-hit tokens billed separately."""
billable_input = Decimal(usage.billable_input) * rate.input_cents_per_mtok
@@ -64,6 +104,13 @@ def compute_cost_cents(usage: Usage, rate: ModelRate) -> Decimal:
return total.quantize(_QUANT, rounding=ROUND_HALF_UP)
@runtime_checkable
class RateResolver(Protocol):
"""网关只需要「给我这个模型此刻的单价」，不关心背后是一套还是两套价目表。"""
def for_model(self, model: str) -> ModelRate: ...
@dataclass(frozen=True, slots=True)
class AICallRecord:
"""One row of ``ob_ai_call``. Written for successes *and* failures."""
@@ -88,6 +135,7 @@ class AICallRecord:
def with_defaults(self) -> AICallRecord:
if self.created_at is not None:
return self
# 本类是 slots dataclass，没有 __dict__，只能用 replace 复制
return replace(self, created_at=datetime.now(UTC))
packages/llm/gateway.py
+3 -2
View File
@@ -28,7 +28,7 @@ from pydantic import BaseModel, ValidationError
from packages.domain.ids import next_id
from packages.llm.budget import DailyBudgetGuard
from packages.llm.circuit import CircuitBreakerRegistry
from packages.llm.cost import AICallRecord, AICallRecorder, RateTable, compute_cost_cents
from packages.llm.cost import AICallRecord, AICallRecorder, RateResolver, compute_cost_cents
from packages.llm.errors import (
AllProvidersFailedError,
CircuitOpenError,
@@ -74,7 +74,7 @@ class LLMGateway:
providers: Mapping[str, LLMProvider],
router: SceneRouter,
prompts: PromptRegistry,
rates: RateTable,
rates: RateResolver,
recorder: AICallRecorder,
budget: DailyBudgetGuard,
breakers: CircuitBreakerRegistry | None = None,
@@ -233,6 +233,7 @@ class LLMGateway:
max_tokens=config.max_tokens,
timeout_seconds=config.timeout_seconds,
json_mode=schema is not None,
thinking=config.thinking,
)
call_id = self._id_factory()
started = self._timer()
packages/llm/providers/deepseek.py
+9
View File
@@ -56,6 +56,9 @@ class DeepSeekProvider:
"temperature": request.temperature,
"max_tokens": request.max_tokens,
"stream": request.stream,
# 思考模式一律显式声明。V4 的默认值是「开启」，漏传就会拿到
# 慢一个数量级、且思考 token 也按输出价计费的行为。
"thinking": {"type": "enabled" if request.thinking else "disabled"},
}
if request.json_mode:
payload["response_format"] = {"type": "json_object"}
@@ -110,10 +113,16 @@ class DeepSeekProvider:
or (usage_raw.get("prompt_tokens_details") or {}).get("cached_tokens")
or 0
)
# 思考 token 已包含在 completion_tokens 里，单独记下来是为了能回答
# 「这个场景的输出成本里有多少花在用户看不见的推理上」
reasoning = int(
(usage_raw.get("completion_tokens_details") or {}).get("reasoning_tokens") or 0
)
usage = Usage(
tokens_in=int(usage_raw.get("prompt_tokens", 0)),
tokens_out=int(usage_raw.get("completion_tokens", 0)),
tokens_cached=cached,
tokens_reasoning=reasoning,
)
return ProviderResponse(
text=text,
packages/llm/routing.py
+14 -1
View File
@@ -4,7 +4,12 @@
so switching models never requires a code change (NFR-MAINT-05). Defaults live
here; ``LLM_SCENE_OVERRIDES_JSON`` overrides any field per scene at deploy time.
Token budgets follow 架构文档 §7.2: 输入合计 ≤ 3700, 输出 ≤ 800.
Token budgets follow 架构文档 §7.2: 输入合计 ≤ 4200, 输出 ≤ 800.
``thinking`` 在每个场景都显式给值。deepseek-v4 默认开启思考模式，实测同一个单句
问答开启要 13.5s、关闭只要 1.8s，且思考 token 按输出价计费 —— 漏配就会同时踩中
「响应超时」和「成本翻倍」。要开某个场景的思考，改 ``LLM_SCENE_OVERRIDES_JSON``
即可，不必改代码。
"""
from __future__ import annotations
@@ -22,6 +27,8 @@ _DEFAULTS: dict[Scene, dict[str, Any]] = {
"prompt_id": "agent_chat",
"prompt_version": "v1",
"critical": False,
# 对话要在 NFR-PERF-02b 的整轮 15s 内返回，思考模式做不到
"thinking": False,
},
Scene.INTENT_CLASSIFY: {
"temperature": 0.0,
@@ -29,6 +36,8 @@ _DEFAULTS: dict[Scene, dict[str, Any]] = {
"timeout_seconds": 10.0,
"prompt_id": "intent_classify",
"prompt_version": "v1",
# 分类是短输出的判别任务，思考只会拖慢并多花钱
"thinking": False,
# Intent classification gates 越界拒答 (FR-AGT-02); degrading it silently
# would let out-of-scope questions through, so it is never budget-gated.
"critical": True,
@@ -40,6 +49,7 @@ _DEFAULTS: dict[Scene, dict[str, Any]] = {
"prompt_id": "preference_extract",
"prompt_version": "v1",
"critical": False,
"thinking": False,
},
Scene.REPORT_COPY: {
"temperature": 0.6,
@@ -48,6 +58,9 @@ _DEFAULTS: dict[Scene, dict[str, Any]] = {
"prompt_id": "report_copy",
"prompt_version": "v1",
"critical": False,
# 报告文案是异步任务，延迟宽松，理论上最适合开思考；V1 先关，
# 等黄金集能量化"思考带来的质量提升"再按场景打开（改配置即可）。
"thinking": False,
},
}
packages/llm/types.py
+8
View File
@@ -40,6 +40,8 @@ class Usage(BaseModel):
tokens_out: int = 0
#: Subset of ``tokens_in`` served from the provider's prompt cache.
tokens_cached: int = 0
#: Subset of ``tokens_out`` spent on思考过程。按输出价计费，但用户看不到。
tokens_reasoning: int = 0
@property
def billable_input(self) -> int:
@@ -64,6 +66,8 @@ class SceneConfig:
structured_max_retries: int = 2
#: Critical scenes bypass the global daily budget breaker (架构文档 §13).
critical: bool = False
#: 思考模式。deepseek-v4 默认开启，必须按场景显式决定 —— 详见 ProviderRequest.thinking。
thinking: bool = False
def model_for(self, provider: str, default: str) -> str:
return self.models.get(provider, default)
@@ -79,6 +83,10 @@ class ProviderRequest:
#: When set, the provider must constrain output to JSON.
json_mode: bool = False
stream: bool = False
#: 思考模式。**必须显式传，不能依赖供应商默认值** —— deepseek-v4 默认开启思考，
#: 实测同一个单句问答开启需 13.5s、关闭只需 1.8s，且思考 token 按输出价计费。
#: 对话与结构化抽取场景一律关闭：既超不过 Agent 整轮 15s 的目标，也白花钱。
thinking: bool = False
@dataclass(frozen=True, slots=True)
packages/shared/config.py
+12 -2
View File
@@ -17,7 +17,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
Environment = Literal["local", "dev", "staging", "prod"]
CachePrefixMode = Literal["strict", "warn", "off"]
_PLACEHOLDER_JWT_SECRET = "change-me-local-only-do-not-use-outside-local" # noqa: S105
_PLACEHOLDER_JWT_SECRET = "change-me-local-only-do-not-use-outside-local"
class Settings(BaseSettings):
@@ -87,6 +87,12 @@ class Settings(BaseSettings):
llm_cost_rates: dict[str, dict[str, Decimal]] = Field(
default_factory=dict, alias="LLM_COST_RATES_JSON"
)
# DeepSeek 自 2026-08-17 起峰谷定价，高峰单价是空闲时段的 2 倍
# （高峰：北京时间 09:00-12:00、14:00-18:00）。留空则两个时段同价。
llm_cost_rates_peak: dict[str, dict[str, Decimal]] = Field(
default_factory=dict, alias="LLM_COST_RATES_PEAK_JSON"
)
llm_peak_pricing_enabled: bool = True
llm_default_cost_rate: dict[str, Decimal] = Field(
default_factory=lambda: {
"input_cents_per_mtok": Decimal("100"),
@@ -108,7 +114,11 @@ class Settings(BaseSettings):
prometheus_enabled: bool = True
@field_validator(
"llm_scene_overrides", "llm_cost_rates", "llm_default_cost_rate", mode="before"
"llm_scene_overrides",
"llm_cost_rates",
"llm_cost_rates_peak",
"llm_default_cost_rate",
mode="before",
)
@classmethod
def _parse_json_object(cls, value: Any) -> Any:
packages/shared/security.py
+1 -1
View File
@@ -40,7 +40,7 @@ def create_access_token(
"iss": issuer,
"typ": "access",
}
return jwt.encode(payload, secret, algorithm=algorithm)
return str(jwt.encode(payload, secret, algorithm=algorithm))
def decode_access_token(