Files
meida_backend/scripts/smoke.sh
T
xsl f46ecc5eb0 fix(llm): price by call time and stop paying for thinking tokens
Two defects that only surface against the real DeepSeek API, both of which
break the cost model rather than the code:

deepseek-v4-flash enables thinking mode by default. The same single-turn
question took 13.5s with it on and 1.8s with it off, and the thinking tokens
are billed at the output rate. That misses NFR-PERF-02b (15s per turn) and
inflates every conversational call. SceneConfig now carries an explicit
`thinking` flag, all four V1 scenes set it to false, and the provider always
sends the parameter rather than relying on the upstream default. Reasoning
tokens are recorded separately in Usage.tokens_reasoning so the cost of ever
turning it back on is visible.

DeepSeek moves to peak/off-peak pricing on 2026-08-17, with peak (Beijing
09:00-12:00 and 14:00-18:00) at twice the off-peak rate. Reports are generated
mostly during peak hours, so a single blended rate would leave ob_ai_call --
the only input to V2 pricing -- off by nearly a factor of two.
PeakAwareRateTable resolves the rate at call time; cost is computed and stored
then, never recomputed later at whatever the current rate happens to be.

Also here:
- Replace the live API key in the redaction test with a synthetic one. The
  test that proves we redact secrets was itself a secret leak waiting for its
  first commit.
- Correct LLM_COST_RATES_JSON, which held placeholder rates.
- Make ids.SnowflakeGenerator take an injectable clock so rollover and
  clock-rollback are testable without patching a slots class.
- Rename exceptions to the -Error suffix (N818) and clear the remaining ruff,
  mypy and import-linter findings, including two dead lines: a duplicate
  pytestmark assignment and a no-op assert on a None return.

Gates: ruff, mypy (96 files incl. tests), 4/4 import contracts,
detect-secrets, 166 unit + 3 integration tests, migration up/down round-trip.
2026-08-14 00:54:40 +08:00

143 lines
6.3 KiB
Bash
Executable File

#!/usr/bin/env bash
# End-to-end proof that the scaffold works against the REAL DeepSeek API.
#
# compose up → migrate → start API → dev token → POST /v1/agent/chat
# → print the ob_ai_call row and the bl_ledger rows
#
# This is the only script that spends real money. Everything in `make test` is
# hermetic.
set -Eeuo pipefail
cd "$(dirname "$0")/.."
COMPOSE="docker compose -f deploy/docker-compose.dev.yml"
API_PORT="${API_PORT:-8010}"
BASE="http://127.0.0.1:${API_PORT}"
LOG_DIR=".smoke"
API_LOG="${LOG_DIR}/api.log"
API_PID=""
bold() { printf '\n\033[1m== %s\033[0m\n' "$1"; }
fail() { printf '\033[31mFAIL: %s\033[0m\n' "$1" >&2; exit 1; }
cleanup() {
if [[ -n "${API_PID}" ]] && kill -0 "${API_PID}" 2>/dev/null; then
kill "${API_PID}" 2>/dev/null || true
wait "${API_PID}" 2>/dev/null || true
fi
}
trap cleanup EXIT
mkdir -p "${LOG_DIR}"
[[ -f .env ]] || fail ".env not found. Copy .env.example to .env and set DEEPSEEK_API_KEY."
# 只取需要的这一个键,不 `source .env`:那些 *_JSON 值含双引号,
# 被 shell 解析后引号会消失,应用侧再读就成了非法 JSON。
DEEPSEEK_API_KEY=$(sed -n 's/^[[:space:]]*DEEPSEEK_API_KEY[[:space:]]*=[[:space:]]*//p' .env | tail -1)
DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY%\"}; DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY#\"}
DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY%\'}; DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY#\'}
[[ -n "${DEEPSEEK_API_KEY}" ]] || fail "DEEPSEEK_API_KEY is empty in .env"
[[ "${DEEPSEEK_API_KEY}" == sk-* ]] || fail "DEEPSEEK_API_KEY does not look like a real key"
bold "1/7 Starting PostgreSQL (pgvector) and Redis"
${COMPOSE} up -d postgres redis
for _ in $(seq 1 60); do
if ${COMPOSE} exec -T postgres pg_isready -U meida -d meida >/dev/null 2>&1; then break; fi
sleep 1
done
${COMPOSE} exec -T postgres pg_isready -U meida -d meida >/dev/null || fail "PostgreSQL never became ready"
${COMPOSE} exec -T redis redis-cli ping >/dev/null || fail "Redis never became ready"
echo "postgres + redis ready"
bold "2/7 Applying migrations"
uv run alembic upgrade head
${COMPOSE} exec -T postgres psql -U meida -d meida -c "\dt"
${COMPOSE} exec -T postgres psql -U meida -d meida -tAc \
"SELECT extname FROM pg_extension WHERE extname='vector'" | grep -q vector \
|| fail "pgvector extension missing"
echo "pgvector extension present"
bold "3/7 Starting the API"
uv run uvicorn apps.api.main:app --host 127.0.0.1 --port "${API_PORT}" >"${API_LOG}" 2>&1 &
API_PID=$!
for _ in $(seq 1 60); do
if curl -fsS "${BASE}/healthz" >/dev/null 2>&1; then break; fi
sleep 0.5
done
curl -fsS "${BASE}/healthz" >/dev/null || { cat "${API_LOG}"; fail "API did not start"; }
curl -fsS "${BASE}/readyz" | tee /dev/null >/dev/null || { cat "${API_LOG}"; fail "API not ready"; }
echo "api up on ${BASE} (logs: ${API_LOG})"
bold "4/7 Issuing a dev token"
DEVICE_KEY="smoke-$(date +%s)"
TOKEN_JSON=$(curl -fsS -X POST "${BASE}/v1/auth/dev-token" \
-H 'Content-Type: application/json' \
-d "{\"device_key\":\"${DEVICE_KEY}\"}")
TOKEN=$(printf '%s' "${TOKEN_JSON}" | python3 -c 'import json,sys;print(json.load(sys.stdin)["data"]["access_token"])')
USER_ID=$(printf '%s' "${TOKEN_JSON}" | python3 -c 'import json,sys;print(json.load(sys.stdin)["data"]["user_id"])')
[[ -n "${TOKEN}" ]] || fail "no access token issued"
echo "user_id=${USER_ID}"
bold "5/7 Calling POST /v1/agent/chat against the REAL DeepSeek API"
CHAT_JSON=$(curl -fsS -X POST "${BASE}/v1/agent/chat" \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer ${TOKEN}" \
-d '{"message":"三十岁男生,第一次去互联网公司面试,穿什么比较稳妥?"}')
printf '%s\n' "${CHAT_JSON}" | python3 -m json.tool
# 走临时文件而不是管道:heredoc 会占用 stdin,管道里的 JSON 读不到
printf '%s' "${CHAT_JSON}" >"${LOG_DIR}/chat.json"
python3 - "${LOG_DIR}/chat.json" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as fh:
body = json.load(fh)
assert body["code"] == 0, body
data = body["data"]
assert data["reply"].strip(), "empty model reply"
usage = data["usage"]
print("\n--- MODEL REPLY -------------------------------------------------")
print(data["reply"])
print("-----------------------------------------------------------------")
print("provider={} model={} prompt={}@{}".format(
data["provider"], data["model"], data["prompt_id"], data["prompt_version"]))
print("latency={}ms tokens_in={} tokens_out={} cached={} cost_cents={}".format(
usage["latency_ms"], usage["tokens_in"], usage["tokens_out"],
usage["tokens_cached"], usage["cost_cents"]))
print("quota_remaining={}".format(data["quota_remaining"]))
# 对话必须在 NFR-PERF-02b 的整轮 15s 之内。这里只量单次 LLM 调用,
# 留出检索与后处理的余量,因此阈值取 10s。
assert usage["latency_ms"] < 10_000, f"LLM 调用耗时 {usage['latency_ms']}ms,超出预期"
PY
bold "6/7 ob_ai_call row (成本归因的唯一数据源, 架构文档 §13)"
${COMPOSE} exec -T postgres psql -U meida -d meida -x -c \
"SELECT call_id, user_id, scene, provider, model, prompt_id, prompt_version,
tokens_in, tokens_out, tokens_cached, latency_ms, cost_cents,
success, error_code, ref_type, created_at
FROM ob_ai_call
WHERE user_id = ${USER_ID}
ORDER BY created_at DESC"
ROWS=$(${COMPOSE} exec -T postgres psql -U meida -d meida -tAc \
"SELECT count(*) FROM ob_ai_call WHERE user_id = ${USER_ID} AND success")
[[ "${ROWS// /}" -ge 1 ]] || fail "no successful ob_ai_call row was written"
bold "7/7 bl_ledger rows (预扣三段式, 架构文档 §7.4)"
${COMPOSE} exec -T postgres psql -U meida -d meida -c \
"SELECT tx_id, delta, tx_type, status, quota_key, period_key, balance_after,
idempotency_key, created_at
FROM bl_ledger
WHERE user_id = ${USER_ID}
ORDER BY created_at"
COMMITTED=$(${COMPOSE} exec -T postgres psql -U meida -d meida -tAc \
"SELECT count(*) FROM bl_ledger
WHERE user_id = ${USER_ID} AND tx_type='consume' AND status='committed'")
[[ "${COMMITTED// /}" -ge 1 ]] || fail "quota was not committed after a successful call"
${COMPOSE} exec -T postgres psql -U meida -d meida -c \
"SELECT quota_key, period_key, used, limit_value FROM bl_quota WHERE user_id = ${USER_ID}"
printf '\n\033[32mSMOKE PASSED\033[0m — real DeepSeek reply received, ob_ai_call and bl_ledger populated.\n'