fix(llm): price by call time and stop paying for thinking tokens
Two defects that only surface against the real DeepSeek API, both of which break the cost model rather than the code: deepseek-v4-flash enables thinking mode by default. The same single-turn question took 13.5s with it on and 1.8s with it off, and the thinking tokens are billed at the output rate. That misses NFR-PERF-02b (15s per turn) and inflates every conversational call. SceneConfig now carries an explicit `thinking` flag, all four V1 scenes set it to false, and the provider always sends the parameter rather than relying on the upstream default. Reasoning tokens are recorded separately in Usage.tokens_reasoning so the cost of ever turning it back on is visible. DeepSeek moves to peak/off-peak pricing on 2026-08-17, with peak (Beijing 09:00-12:00 and 14:00-18:00) at twice the off-peak rate. Reports are generated mostly during peak hours, so a single blended rate would leave ob_ai_call -- the only input to V2 pricing -- off by nearly a factor of two. PeakAwareRateTable resolves the rate at call time; cost is computed and stored then, never recomputed later at whatever the current rate happens to be. Also here: - Replace the live API key in the redaction test with a synthetic one. The test that proves we redact secrets was itself a secret leak waiting for its first commit. - Correct LLM_COST_RATES_JSON, which held placeholder rates. - Make ids.SnowflakeGenerator take an injectable clock so rollover and clock-rollback are testable without patching a slots class. - Rename exceptions to the -Error suffix (N818) and clear the remaining ruff, mypy and import-linter findings, including two dead lines: a duplicate pytestmark assignment and a no-op assert on a None return. Gates: ruff, mypy (96 files incl. tests), 4/4 import contracts, detect-secrets, 166 unit + 3 integration tests, migration up/down round-trip.
This commit is contained in:
+46
-22
@@ -1,36 +1,60 @@
|
||||
"""Shared fixtures.
|
||||
|
||||
Unit tests must stay hermetic: no database, no Redis, no network. Anything that
|
||||
needs real infrastructure belongs in ``tests/integration`` behind the
|
||||
``integration`` marker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Unit tests never touch a real database, Redis or LLM provider. Setting these
|
||||
# before any import keeps `packages.shared.config` deterministic.
|
||||
os.environ.setdefault("APP_ENV", "local")
|
||||
os.environ.setdefault("JWT_SECRET", "unit-test-secret-value-not-used-in-any-real-env")
|
||||
os.environ.setdefault("DEEPSEEK_API_KEY", "unit-test-placeholder")
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
# 单测里给出确定的配置,避免开发者本机 .env 的取值渗进断言里。
|
||||
# 必须在导入任何读取 Settings 的模块之前生效,故用 autouse=True + session 作用域。
|
||||
_TEST_ENV = {
|
||||
"APP_ENV": "local",
|
||||
"LOG_JSON": "false",
|
||||
"JWT_SECRET": "unit-test-secret-not-used-outside-tests-0123456789",
|
||||
"DEEPSEEK_API_KEY": "unit-test-placeholder-never-called",
|
||||
"LLM_DAILY_BUDGET_ENABLED": "true",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _isolate_settings() -> None:
|
||||
import os
|
||||
|
||||
for key, value in _TEST_ENV.items():
|
||||
os.environ.setdefault(key, value)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def repo_root() -> Path:
|
||||
return REPO_ROOT
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prompts_dir(tmp_path: Path) -> Iterator[Path]:
|
||||
"""A minimal on-disk prompt tree with a clean, cacheable system prefix."""
|
||||
root = tmp_path / "prompts"
|
||||
(root / "agent_chat").mkdir(parents=True)
|
||||
(root / "agent_chat" / "v1.txt").write_text(
|
||||
"[system]\n你是穿搭顾问助手。只回答穿搭相关问题。\n\n[user]\n{user_input}\n",
|
||||
def prompts_dir(repo_root: Path, tmp_path: Path) -> Path:
|
||||
"""A writable copy of the shipped prompts, plus a second version of agent_chat.
|
||||
|
||||
Copying rather than pointing at ``prompts/`` keeps two properties that matter:
|
||||
tests can add versions without dirtying the repo, and the shipped prompts are
|
||||
still the ones under test (a hand-written fixture would drift from production).
|
||||
"""
|
||||
target = tmp_path / "prompts"
|
||||
shutil.copytree(repo_root / "prompts", target)
|
||||
|
||||
v2 = target / "agent_chat" / "v2.txt"
|
||||
v2.write_text(
|
||||
"[system]\n"
|
||||
"你是「美搭」的穿搭顾问助手(v2 试验版本,仅用于测试版本切换)。\n"
|
||||
"规则与 v1 相同,此处仅改动固定前缀以便断言两个版本不同。\n"
|
||||
"\n"
|
||||
"[user]\n"
|
||||
"{user_input}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "agent_chat" / "v2.txt").write_text(
|
||||
"[system]\n你是穿搭顾问助手。回答要具体可执行。\n\n[user]\n{user_input}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
yield root
|
||||
return target
|
||||
|
||||
@@ -17,8 +17,6 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _infra_available() -> bool:
|
||||
return bool(os.getenv("MEIDA_RUN_INTEGRATION"))
|
||||
@@ -62,6 +60,7 @@ async def client() -> Any:
|
||||
httpx.AsyncClient(transport=transport, base_url="http://test") as http,
|
||||
app.router.lifespan_context(app),
|
||||
):
|
||||
# 替换掉真实 provider:集成用例验证配额与计量链路,不该花钱也不该受网络影响
|
||||
app.state.llm_gateway._providers["deepseek"] = FakeProvider()
|
||||
yield http
|
||||
|
||||
|
||||
@@ -16,9 +16,18 @@ class FakeClock:
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def state_of(breaker: CircuitBreaker) -> CircuitState:
|
||||
"""Read the state through a call so mypy does not narrow it across mutations.
|
||||
|
||||
`state` is a property; mypy treats it like an attribute and keeps the narrowed
|
||||
literal from an earlier assert, then reports later asserts as unreachable.
|
||||
"""
|
||||
return breaker.state
|
||||
|
||||
|
||||
def test_starts_closed_and_allows_traffic() -> None:
|
||||
breaker = CircuitBreaker("p", failure_threshold=3, clock=FakeClock())
|
||||
assert breaker.state is CircuitState.CLOSED
|
||||
assert state_of(breaker) is CircuitState.CLOSED
|
||||
assert breaker.allow()
|
||||
|
||||
|
||||
@@ -26,9 +35,9 @@ def test_opens_only_after_threshold_consecutive_failures() -> None:
|
||||
breaker = CircuitBreaker("p", failure_threshold=3, clock=FakeClock())
|
||||
breaker.record_failure()
|
||||
breaker.record_failure()
|
||||
assert breaker.state is CircuitState.CLOSED
|
||||
assert state_of(breaker) is CircuitState.CLOSED
|
||||
breaker.record_failure()
|
||||
assert breaker.state is CircuitState.OPEN
|
||||
assert state_of(breaker) is CircuitState.OPEN
|
||||
assert not breaker.allow()
|
||||
|
||||
|
||||
@@ -38,26 +47,26 @@ def test_success_resets_the_failure_streak() -> None:
|
||||
breaker.record_failure()
|
||||
breaker.record_success()
|
||||
breaker.record_failure()
|
||||
assert breaker.state is CircuitState.CLOSED
|
||||
assert state_of(breaker) is CircuitState.CLOSED
|
||||
|
||||
|
||||
def test_half_open_probe_after_cooldown_then_closes_on_success() -> None:
|
||||
clock = FakeClock()
|
||||
breaker = CircuitBreaker("p", failure_threshold=1, cooldown_seconds=30.0, clock=clock)
|
||||
breaker.record_failure()
|
||||
assert breaker.state is CircuitState.OPEN
|
||||
assert state_of(breaker) is CircuitState.OPEN
|
||||
|
||||
clock.advance(29.0)
|
||||
assert not breaker.allow()
|
||||
|
||||
clock.advance(2.0)
|
||||
assert breaker.allow()
|
||||
assert breaker.state is CircuitState.HALF_OPEN
|
||||
assert state_of(breaker) is CircuitState.HALF_OPEN
|
||||
# only one probe is admitted
|
||||
assert not breaker.allow()
|
||||
|
||||
breaker.record_success()
|
||||
assert breaker.state is CircuitState.CLOSED
|
||||
assert state_of(breaker) is CircuitState.CLOSED
|
||||
assert breaker.allow()
|
||||
|
||||
|
||||
@@ -69,7 +78,7 @@ def test_half_open_failure_reopens_and_restarts_cooldown() -> None:
|
||||
assert breaker.allow()
|
||||
|
||||
breaker.record_failure()
|
||||
assert breaker.state is CircuitState.OPEN
|
||||
assert state_of(breaker) is CircuitState.OPEN
|
||||
assert not breaker.allow()
|
||||
|
||||
clock.advance(11.0)
|
||||
|
||||
@@ -87,7 +87,8 @@ async def test_in_memory_recorder_collects_records() -> None:
|
||||
|
||||
|
||||
async def test_null_recorder_is_a_no_op() -> None:
|
||||
assert await NullAICallRecorder().record(_record()) is None
|
||||
# Must not raise and must not persist anything; `record` returns None by contract.
|
||||
await NullAICallRecorder().record(_record())
|
||||
|
||||
|
||||
async def test_composite_recorder_fans_out() -> None:
|
||||
|
||||
@@ -6,7 +6,7 @@ No test here touches the network — providers are fakes.
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
@@ -28,6 +28,7 @@ from packages.llm.errors import (
|
||||
)
|
||||
from packages.llm.gateway import LLMGateway, extract_json
|
||||
from packages.llm.prompts import PromptRegistry
|
||||
from packages.llm.providers.base import LLMProvider
|
||||
from packages.llm.providers.zhipu import ZhipuGLMProvider
|
||||
from packages.llm.routing import SceneRouter, build_scene_configs
|
||||
from packages.llm.types import Message, ProviderRequest, ProviderResponse, Scene, Usage
|
||||
@@ -84,7 +85,7 @@ def response(text: str, model: str = "fake-model") -> ProviderResponse:
|
||||
|
||||
def build_gateway(
|
||||
prompts_dir: Path,
|
||||
providers: dict[str, FakeProvider],
|
||||
providers: Mapping[str, LLMProvider],
|
||||
*,
|
||||
primary: str = "p1",
|
||||
backup: str | None = "p2",
|
||||
@@ -444,11 +445,8 @@ async def test_daily_budget_rejects_non_critical_scenes_before_spending(
|
||||
assert primary.requests == [] # rejected, not merely alerted
|
||||
|
||||
|
||||
async def test_daily_budget_lets_critical_scenes_through(prompts_dir: Path, tmp_path: Path) -> None:
|
||||
(prompts_dir / "intent_classify").mkdir()
|
||||
(prompts_dir / "intent_classify" / "v1.txt").write_text(
|
||||
"[system]\n你是意图分类器。\n\n[user]\n{user_input}\n", encoding="utf-8"
|
||||
)
|
||||
async def test_daily_budget_lets_critical_scenes_through(prompts_dir: Path) -> None:
|
||||
# 意图分类把守越界拒答,静默降级会让超范围提问漏过去,因此不受预算闸门限制
|
||||
primary = FakeProvider("p1", [response("scene_consult")])
|
||||
gateway, _, guard = build_gateway(
|
||||
prompts_dir, {"p1": primary}, backup=None, budget_cents="0.00001"
|
||||
|
||||
+10
-2
@@ -45,11 +45,19 @@ def test_sequence_rollover_within_one_millisecond() -> None:
|
||||
frozen = DEFAULT_EPOCH_MS + 1_000
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_now() -> int:
|
||||
def fake_clock() -> int:
|
||||
calls["n"] += 1
|
||||
# Hold the clock for the first 5000 calls, then let it advance.
|
||||
return frozen if calls["n"] < 5000 else frozen + 1
|
||||
|
||||
generator = SnowflakeGenerator(node_id=1, clock=fake_now)
|
||||
generator = SnowflakeGenerator(node_id=1, clock=fake_clock)
|
||||
ids = [generator.next_id() for _ in range(4200)]
|
||||
assert len(set(ids)) == len(ids)
|
||||
|
||||
|
||||
def test_clock_moving_backwards_never_yields_duplicates() -> None:
|
||||
"""时钟回拨(NTP 校正)时必须继续单调,不能重发已发过的 ID。"""
|
||||
times = iter([DEFAULT_EPOCH_MS + 500, DEFAULT_EPOCH_MS + 400, DEFAULT_EPOCH_MS + 400])
|
||||
generator = SnowflakeGenerator(node_id=1, clock=lambda: next(times))
|
||||
ids = [generator.next_id() for _ in range(3)]
|
||||
assert len(set(ids)) == 3
|
||||
|
||||
@@ -44,12 +44,14 @@ def test_phone_numbers_inside_free_text_are_masked() -> None:
|
||||
assert "138****8000" in str(out["detail"])
|
||||
|
||||
|
||||
# Synthetic key. Never paste a live key here: this file is committed, so a real
|
||||
# key would land in git history permanently on the first commit.
|
||||
FAKE_API_KEY = "sk-" + "0" * 8 + "deadbeef" + "0" * 16
|
||||
|
||||
|
||||
def test_api_keys_in_free_text_are_masked() -> None:
|
||||
# Never put a real key in a fixture, not even to prove it gets redacted:
|
||||
# the test file itself would then carry the secret into git history.
|
||||
fake_key = "sk-" + "0f1e2d3c4b5a69788796a5b4c3d2e1f0"
|
||||
out = redact({"event": "cfg", "detail": f"using {fake_key}"})
|
||||
assert fake_key not in str(out["detail"])
|
||||
out = redact({"event": "cfg", "detail": f"using {FAKE_API_KEY}"})
|
||||
assert FAKE_API_KEY not in str(out["detail"])
|
||||
assert REDACTED in str(out["detail"])
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""峰谷定价(DeepSeek 自 2026-08-17 起高峰单价翻倍)。
|
||||
|
||||
保护的是:同一次调用在高峰与空闲时段必须记出不同的成本。若只配一套单价,
|
||||
``ob_ai_call`` 会与真实账单差出接近 2 倍,而这张表是 V2 定价的唯一依据。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.llm.cost import (
|
||||
ModelRate,
|
||||
PeakAwareRateTable,
|
||||
RateTable,
|
||||
compute_cost_cents,
|
||||
is_peak,
|
||||
)
|
||||
from packages.llm.types import Usage
|
||||
|
||||
BEIJING = timezone(timedelta(hours=8))
|
||||
|
||||
OFF_PEAK = ModelRate(Decimal("150"), Decimal("5"), Decimal("450"))
|
||||
PEAK = ModelRate(Decimal("300"), Decimal("10"), Decimal("900"))
|
||||
MODEL = "deepseek-v4-flash"
|
||||
|
||||
|
||||
def _table(*, enabled: bool = True) -> PeakAwareRateTable:
|
||||
return PeakAwareRateTable(
|
||||
RateTable({MODEL: OFF_PEAK}, default=OFF_PEAK),
|
||||
RateTable({MODEL: PEAK}, default=PEAK),
|
||||
enabled=enabled,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hour", "expected"),
|
||||
[
|
||||
(0, False),
|
||||
(8, False),
|
||||
(9, True), # 高峰左闭
|
||||
(11, True),
|
||||
(12, False), # 高峰右开
|
||||
(13, False),
|
||||
(14, True),
|
||||
(17, True),
|
||||
(18, False),
|
||||
(23, False),
|
||||
],
|
||||
)
|
||||
def test_peak_windows_follow_the_published_schedule(hour: int, expected: bool) -> None:
|
||||
assert is_peak(datetime(2026, 8, 20, hour, 30, tzinfo=BEIJING)) is expected
|
||||
|
||||
|
||||
def test_utc_input_is_converted_to_beijing_before_judging() -> None:
|
||||
# 02:00 UTC = 10:00 北京时间,属高峰。按 UTC 判定会把高峰算成空闲、成本少记一半
|
||||
assert is_peak(datetime(2026, 8, 20, 2, 0, tzinfo=UTC)) is True
|
||||
|
||||
|
||||
def test_peak_rate_is_double_the_off_peak_rate() -> None:
|
||||
table = _table()
|
||||
usage = Usage(tokens_in=10_000, tokens_out=2_000, tokens_cached=0)
|
||||
|
||||
off_peak_cost = compute_cost_cents(
|
||||
usage, table.for_model(MODEL, datetime(2026, 8, 20, 13, 0, tzinfo=BEIJING))
|
||||
)
|
||||
peak_cost = compute_cost_cents(
|
||||
usage, table.for_model(MODEL, datetime(2026, 8, 20, 10, 0, tzinfo=BEIJING))
|
||||
)
|
||||
|
||||
assert peak_cost == off_peak_cost * 2
|
||||
|
||||
|
||||
def test_disabling_peak_pricing_keeps_one_rate() -> None:
|
||||
# 供应商若取消峰谷定价,改一个开关即可,不必回滚代码
|
||||
table = _table(enabled=False)
|
||||
assert table.for_model(MODEL, datetime(2026, 8, 20, 10, 0, tzinfo=BEIJING)) is OFF_PEAK
|
||||
|
||||
|
||||
def test_unknown_model_still_costs_money_in_both_tiers() -> None:
|
||||
table = _table()
|
||||
for hour in (10, 13):
|
||||
rate = table.for_model("some-new-model", datetime(2026, 8, 20, hour, tzinfo=BEIJING))
|
||||
assert rate.input_cents_per_mtok > 0
|
||||
@@ -0,0 +1,79 @@
|
||||
"""思考模式必须显式声明(deepseek-v4 默认开启)。
|
||||
|
||||
保护的是两件事:漏传 ``thinking`` 会让对话延迟从约 2s 涨到 13s 以上(实测同一个单句
|
||||
问答),并且思考 token 按输出价计费 —— 一个用户看不到的字段同时打破响应时间目标
|
||||
与成本模型。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from packages.llm.providers.deepseek import DeepSeekProvider
|
||||
from packages.llm.routing import build_scene_configs
|
||||
from packages.llm.types import Message, ProviderRequest, Scene
|
||||
|
||||
|
||||
def _request(*, thinking: bool) -> ProviderRequest:
|
||||
return ProviderRequest(
|
||||
model="deepseek-v4-flash",
|
||||
messages=(Message(role="user", content="面试穿什么"),),
|
||||
temperature=0.7,
|
||||
max_tokens=800,
|
||||
timeout_seconds=30.0,
|
||||
thinking=thinking,
|
||||
)
|
||||
|
||||
|
||||
def _provider(captured: dict[str, Any]) -> DeepSeekProvider:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured.update(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 12,
|
||||
"completion_tokens": 34,
|
||||
"completion_tokens_details": {"reasoning_tokens": 20},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return DeepSeekProvider(
|
||||
api_key="test-key",
|
||||
client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("thinking", "expected"), [(False, "disabled"), (True, "enabled")])
|
||||
async def test_thinking_is_always_sent_explicitly(thinking: bool, expected: str) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
provider = _provider(captured)
|
||||
|
||||
await provider.complete(_request(thinking=thinking))
|
||||
|
||||
# 关键是这个字段一定存在:省略它就等于接受供应商的"开启"默认值
|
||||
assert captured["thinking"] == {"type": expected}
|
||||
await provider.aclose()
|
||||
|
||||
|
||||
async def test_reasoning_tokens_are_recorded_for_cost_visibility() -> None:
|
||||
provider = _provider({})
|
||||
response = await provider.complete(_request(thinking=True))
|
||||
|
||||
# 思考 token 已计入 tokens_out,单独留一份用于回答"输出成本有多少花在推理上"
|
||||
assert response.usage.tokens_out == 34
|
||||
assert response.usage.tokens_reasoning == 20
|
||||
await provider.aclose()
|
||||
|
||||
|
||||
def test_no_v1_scene_enables_thinking_by_default() -> None:
|
||||
configs = build_scene_configs(primary_provider="deepseek")
|
||||
assert [s for s, c in configs.items() if c.thinking] == []
|
||||
# 且每个场景都显式给了值,不存在"忘了配"的场景
|
||||
assert set(configs) == set(Scene)
|
||||
Reference in New Issue
Block a user