feat(analytics): POST /events/batch with fail-close validation

ob_event 落库通道:形状校验 + props 字节上限 + client_ts 合理性 +
PII 键剔除,整批原子(FR-SYS-06 服务端部分)。

实现:packages/analytics/schemas.py(EventIn/EventBatchRequest/
EventBatchResponse + 事件名形状正则常量);service.py 拆出纯函数
validate_batch(批量/props 字节/事件名形状/client_ts 窗口/PII 剔除,
任一不过整批 40020),AnalyticsService.record 组装 ObEvent 逐条
add + 一次 flush;routers/events.py 提供 POST /events/batch
(/api/v1 与 /v1 别名,认证必需)。

测试:单测 tests/unit/test_events_validation.py 8 例(合法批、超批量、
props 超字节/不可序列化、事件名形状含尾随换行、client_ts 双边界、
naive 时区按 UTC、PII 键含大小写变体剔除);集成
tests/integration/test_events.py 3 例(两前缀可用+落库断言、
非法批 40020 且库无半条、无 token 40100)。门禁全绿:ruff
check/format、mypy(T2 列表 + domain strict)、lint-imports 4 契约、
单测 177 通过、集成 6 通过。

偏差说明:event_name 正则未写成 Field(pattern=...),而是作为
EVENT_NAME_PATTERN 常量放 schemas.py、由服务层 validate_batch 强制——
若挂在请求体校验上,非法事件名会先被 RequestValidationError 拦截、
返回通用 40001/422 信封,无法满足任务契约 §4「大写 event_name →
40020」的集成行为。正则本体与文档一致。
This commit is contained in:
xsl
2026-08-14 16:16:26 +08:00
parent 9e7a886863
commit 64175a1ea1
5 changed files with 446 additions and 3 deletions
+137
View File
@@ -0,0 +1,137 @@
"""埋点上报集成用例(真实 PostgreSQL,FR-SYS-06)。
覆盖:/v1 与 /api/v1 两个前缀、落库行内容(含 PII 剔除)、
非法批 40020 且整批原子(库里没有半条)、无 token 40100。
"""
from __future__ import annotations
import os
import uuid
from datetime import UTC, datetime
from typing import Any
import pytest
def _infra_available() -> bool:
return bool(os.getenv("MEIDA_RUN_INTEGRATION"))
pytestmark = [
pytest.mark.integration,
pytest.mark.skipif(
not _infra_available(),
reason="set MEIDA_RUN_INTEGRATION=1 with PostgreSQL and Redis running",
),
]
@pytest.fixture
async def client() -> Any:
import httpx
from apps.api.main import create_app
from packages.shared.config import get_settings
# 埋点链路不碰 LLM,无需替换 provider;lifespan 照常起网关与连接池
app = create_app(get_settings())
transport = httpx.ASGITransport(app=app)
async with (
httpx.AsyncClient(transport=transport, base_url="http://test") as http,
app.router.lifespan_context(app),
):
yield http
@pytest.fixture
async def auth(client: Any) -> dict[str, Any]:
"""dev-token 换一个专属测试用户,避免与其它用例的 ob_event 行串号。"""
resp = await client.post(
"/api/v1/auth/dev-token", json={"device_key": f"t3-{uuid.uuid4().hex[:8]}"}
)
assert resp.json()["code"] == 0
data = resp.json()["data"]
headers = {"Authorization": f"Bearer {data['access_token']}"}
return {"headers": headers, "user_id": int(data["user_id"])}
async def _count_events(user_id: int) -> int:
from sqlalchemy import func, select
from packages.shared.db import get_session_factory
from packages.shared.models import ObEvent
async with get_session_factory()() as session:
stmt = select(func.count()).select_from(ObEvent).where(ObEvent.user_id == user_id)
return int(await session.scalar(stmt))
async def _fetch_events(user_id: int) -> list[Any]:
from sqlalchemy import select
from packages.shared.db import get_session_factory
from packages.shared.models import ObEvent
async with get_session_factory()() as session:
result = await session.execute(
select(ObEvent).where(ObEvent.user_id == user_id).order_by(ObEvent.event_id)
)
return list(result.scalars().all())
async def test_batch_lands_two_rows_with_pii_stripped(client: Any, auth: dict[str, Any]) -> None:
resp = await client.post(
"/v1/events/batch",
json={
"events": [
{
"event_name": "page_view",
"props": {"page": "home", "phone": "13800001234"},
"client_ts": datetime.now(UTC).isoformat(),
},
{"event_name": "outfit_card.tap", "props": {"card_id": 42}},
]
},
headers=auth["headers"],
)
assert resp.status_code == 200
body = resp.json()
assert body["code"] == 0
assert body["data"] == {"accepted": 2}
rows = await _fetch_events(auth["user_id"])
assert len(rows) == 2
assert rows[0].event_name == "page_view"
assert rows[0].props == {"page": "home"} # phone 已在服务端剔除
assert rows[0].client_ts is not None
assert rows[0].server_ts is not None
assert rows[1].event_name == "outfit_card.tap"
assert rows[1].props == {"card_id": 42}
assert rows[1].client_ts is None
async def test_invalid_batch_is_rejected_atomically(client: Any, auth: dict[str, Any]) -> None:
before = await _count_events(auth["user_id"])
resp = await client.post(
"/api/v1/events/batch",
json={
"events": [
{"event_name": "page_view"},
{"event_name": "PageView"}, # 大写 → 形状非法
]
},
headers=auth["headers"],
)
assert resp.status_code == 400
assert resp.json()["code"] == 40020
# 整批原子:合法的那条也不能半落库
assert await _count_events(auth["user_id"]) == before
async def test_events_require_authentication(client: Any) -> None:
resp = await client.post("/api/v1/events/batch", json={"events": [{"event_name": "page_view"}]})
assert resp.status_code == 401
assert resp.json()["code"] == 40100
+136
View File
@@ -0,0 +1,136 @@
"""埋点整批校验的纯逻辑用例(FR-SYS-06,不连库)。
校验入口是 packages.analytics.service.validate_batch —— 模块级纯函数,
时间与上限全部注入,便于钉死边界值逐条断言。
"""
from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta
from typing import Any
import pytest
from packages.analytics.schemas import EventIn
from packages.analytics.service import validate_batch
from packages.shared.errors import AppError, ErrorCode
#: 钉死的"服务端当前时间",所有 client_ts 边界都相对它构造。
NOW = datetime(2026, 8, 14, 12, 0, 0, tzinfo=UTC)
USER_ID = 7_100_000_000_000_000_001
MAX_BATCH = 50
MAX_BYTES = 8192
def _ev(
name: str = "page_view",
props: dict[str, Any] | None = None,
client_ts: datetime | None = None,
) -> EventIn:
return EventIn(event_name=name, props=props if props is not None else {}, client_ts=client_ts)
def _run(events: list[EventIn]) -> list[Any]:
return validate_batch(
events, user_id=USER_ID, max_batch=MAX_BATCH, max_bytes=MAX_BYTES, now=NOW
)
def _assert_rejected(events: list[EventIn]) -> None:
"""fail-close:任一规则不过 → 整批 40020。"""
with pytest.raises(AppError) as excinfo:
_run(events)
assert excinfo.value.code == ErrorCode.EVENTS_BATCH_REJECTED
assert int(excinfo.value.code) == 40020
def test_valid_batch_passes_and_assembles_rows() -> None:
rows = _run(
[
_ev("outfit_card.tap", {"card_id": 42}, NOW - timedelta(minutes=1)),
_ev("page_view"),
]
)
assert len(rows) == 2
assert all(row.user_id == USER_ID for row in rows)
# server_ts 用注入的 now,与 client_ts 窗口判定同一时刻
assert all(row.server_ts == NOW for row in rows)
assert rows[0].event_name == "outfit_card.tap"
assert rows[0].props == {"card_id": 42}
assert rows[0].client_ts == NOW - timedelta(minutes=1)
assert rows[1].client_ts is None # 缺省 client_ts 允许
# 雪花 id 逐条独立分配
assert len({row.event_id for row in rows}) == 2
def test_empty_and_oversized_batch_are_rejected() -> None:
_assert_rejected([]) # schema 的 min_length=1 之外,服务层同样兜底
_assert_rejected([_ev() for _ in range(MAX_BATCH + 1)])
# 恰好等于上限是合法的
assert len(_run([_ev() for _ in range(MAX_BATCH)])) == MAX_BATCH
def test_oversize_props_are_rejected() -> None:
# 默认上限 8192 字节;8KB 的值序列化后必然超限
big = {"blob": "x" * 8192}
assert len(json.dumps(big, ensure_ascii=False).encode()) > MAX_BYTES
_assert_rejected([_ev(props=big)])
# 与合法事件混在同一批也要整批拒绝(fail-close,不部分落库)
_assert_rejected([_ev(), _ev(props=big)])
def test_unserializable_props_are_rejected() -> None:
# set 不可 JSON 序列化 → fail-close 拒收,而不是落库时炸事务
_assert_rejected([_ev(props={"bad": {1, 2}})]) # type: ignore[dict-item]
def test_event_name_shape_is_enforced() -> None:
for bad_name in (
"PageView", # 大写
"1page", # 数字开头
"p", # 太短(正则要求 >= 2 位)
"page view", # 空格
"page_view!", # 非法字符
"a" * 65, # 超长(正则上限 64)
"page_view\n", # $ 可被尾随换行骗过,fullmatch 不行
):
_assert_rejected([_ev(bad_name)])
# 64 位以内、含点与下划线、数字开头之外的位置都是合法的
assert len(_run([_ev("a" + "b" * 63)])) == 1
assert len(_run([_ev("wardrobe.item_2.sync_ok")])) == 1
def test_client_ts_window_boundaries() -> None:
# 不晚于 now+5min:恰好 +5min 合法,再多 1 秒拒收
assert len(_run([_ev(client_ts=NOW + timedelta(minutes=5))])) == 1
_assert_rejected([_ev(client_ts=NOW + timedelta(minutes=5, seconds=1))])
# 不早于 7 天前:恰好 7 天前合法,再早 1 秒拒收
assert len(_run([_ev(client_ts=NOW - timedelta(days=7))])) == 1
_assert_rejected([_ev(client_ts=NOW - timedelta(days=7, seconds=1))])
# 同批一条合法 + 一条越界 → 整批拒
_assert_rejected([_ev(client_ts=NOW), _ev(client_ts=NOW + timedelta(hours=1))])
def test_naive_client_ts_is_interpreted_as_utc() -> None:
# 客户端漏发时区按 UTC 解释,窗口内仍可落库
rows = _run([_ev(client_ts=datetime(2026, 8, 14, 11, 0, 0))])
assert rows[0].client_ts == datetime(2026, 8, 14, 11, 0, 0, tzinfo=UTC)
def test_pii_keys_are_stripped_from_props() -> None:
rows = _run(
[
_ev(
"report.share",
{
"phone": "13800001234",
"Token": "Bearer abc", # 大小写变体也拦
"authorization": "Basic xyz",
"scene": "home",
},
)
]
)
assert rows[0].props == {"scene": "home"}