feat(report): orchestration skeleton with staged pipeline and refund-on-failure
POST /reports 骨架：资格校验→终身配额预扣→tk_task→cpu 队列；动态装配 流水线（dotted-path 注册、缺阶段即 CONFIG_ERROR）；中间态逐段落库、失败 退款、derived_features 跨重试/重生成复用（T-P3-07/09）。 实现：packages/report 的 pipeline.py（Stage 契约保持不变，新增 assemble_stages/ run_pipeline/StageFailureError，ReportContext 增可选 task_id 参数、向后兼容）、 repo.py、schemas.py、service.py（重放优先于资格校验、reserve+建行+建任务同事务、 提交后才投递、并发同键落败回滚）；apps/api/routers/reports.py 填桩（send_task 按 任务名投递，不 import apps.worker）；apps/worker/report_tasks.py（worker 侧自建 gateway 装配、状态归一 QUEUED→RUNNING、成功 commit 配额+成本聚合写 model_info、 基础设施异常重试一次、重试耗尽兜底失败退款）。测试：单测 6 个（装配缺失/空配置/ 协议违约/顺序执行/失败保留产物），集成 6 场景（预扣 lifetime/PENDING、成功 committed+两阶段键、同 request_key 幂等不重复扣、二次首生成 40920、阶段失败 refund+derived_features 保留、regenerate 走 m: 月配额 version+1 复用视觉中间态）。 偏差说明： 1) apps/worker/celery_app.py 的 include 只含 apps.worker.tasks（禁改文件），真 worker 进程注册 meida.report.generate 需协调者合并时把 include 补上 apps.worker.report_tasks；API 侧已按任务名 send_task，两侧只靠名字衔接，测试 不受影响。2) ReportContext 增加可选 task_id 关键字参数（失败结案要推动 tk_task），Stage 协议/DEFAULT_STAGE_PATHS/stage_names 未动，T16/T17 兼容。 3) JSONB 深变更用 flag_modified 显式标脏（重新赋相等新 dict 会被 flush 的按值 比较跳过，集成测试抓到过该坑）。4) POST 后 tk_task 保持 PENDING（任务文档 §4 场景 1 口径），QUEUED→RUNNING 由 worker 入口归一。
This commit is contained in:
@@ -0,0 +1,425 @@
|
||||
"""报告域端到端（T-P3-07/09）：触发→配额→流水线→失败退款→重生成。
|
||||
|
||||
Fake 阶段定义在本文件里，通过 ``REPORT_PIPELINE_STAGES_JSON`` 让
|
||||
assemble_stages 按 dotted path 动态装配——与 T16/T17 的真实阶段走完全相同
|
||||
的装配路径。不启真 celery：worker 内芯函数直调；API 投递钩子替换为 no-op，
|
||||
避免测试消息进真 Redis 队列被开发环境的 worker 消费掉。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _infra_available() -> bool:
|
||||
return bool(os.getenv("MEIDA_RUN_INTEGRATION"))
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not _infra_available(),
|
||||
reason="set MEIDA_RUN_INTEGRATION=1 with PostgreSQL and Redis running",
|
||||
),
|
||||
]
|
||||
|
||||
# Fake 阶段走 importlib 动态装配，dotted path 必须可导入（本文件自身）。
|
||||
_MODULE = "tests.integration.test_report_flow"
|
||||
_SUCCESS_STAGES = json.dumps([f"{_MODULE}:FakeVisionStage", f"{_MODULE}:FakeDiagnosisStage"])
|
||||
_FAILING_STAGES = json.dumps([f"{_MODULE}:FakeVisionStage", f"{_MODULE}:FakeBoomStage"])
|
||||
|
||||
|
||||
class FakeVisionStage:
|
||||
"""视觉阶段替身：计算次数写进 derived_features，用例据此断言「已存在即跳过」。"""
|
||||
|
||||
name = "vision"
|
||||
|
||||
async def run(self, ctx: Any) -> None:
|
||||
features = ctx.report_input.derived_features
|
||||
if not features:
|
||||
# 与 T16 真实视觉阶段同一契约（T-P3-09）：中间态非空则不重算。
|
||||
features["vision_runs"] = 1
|
||||
ctx.report.result["vision"] = {"features": sorted(features)}
|
||||
|
||||
|
||||
class FakeDiagnosisStage:
|
||||
name = "diagnosis"
|
||||
|
||||
async def run(self, ctx: Any) -> None:
|
||||
# 顺序契约：诊断能读到视觉阶段的产物。
|
||||
assert "vision" in ctx.report.result
|
||||
ctx.report.result["diagnosis"] = {"summary": "肩宽腰细", "source": "vision"}
|
||||
|
||||
|
||||
class FakeBoomStage:
|
||||
name = "boom"
|
||||
|
||||
async def run(self, ctx: Any) -> None:
|
||||
raise RuntimeError("爆了：模拟阶段失败")
|
||||
|
||||
|
||||
# --- fixtures ---------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_enqueue(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""不向真 Redis 投递：worker 由用例直调（celery 注册/消费不在本用例范围）。"""
|
||||
monkeypatch.setattr("apps.api.routers.reports._enqueue_report", lambda task_id: None)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _stage_config(monkeypatch: pytest.MonkeyPatch, payload: str) -> Iterator[None]:
|
||||
"""把阶段配置换成 Fake 阶段（setenv + cache_clear，结束时恢复）。
|
||||
|
||||
API 进程与 worker 内芯读的是同一个 get_settings 单例，cache_clear 之后
|
||||
两边都会按 env 重建。
|
||||
"""
|
||||
from packages.shared.config import get_settings
|
||||
|
||||
monkeypatch.setenv("REPORT_PIPELINE_STAGES_JSON", payload)
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_stages(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
with _stage_config(monkeypatch, _SUCCESS_STAGES):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def failing_stages(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
with _stage_config(monkeypatch, _FAILING_STAGES):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(no_enqueue: None) -> AsyncIterator[Any]:
|
||||
import httpx
|
||||
|
||||
from apps.api.main import create_app
|
||||
from packages.shared.config import get_settings
|
||||
|
||||
app = create_app(get_settings())
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with (
|
||||
httpx.AsyncClient(transport=transport, base_url="http://test") as http,
|
||||
app.router.lifespan_context(app),
|
||||
):
|
||||
yield http
|
||||
|
||||
|
||||
# --- helpers ----------------------------------------------------------------
|
||||
|
||||
|
||||
async def _login(client: Any) -> tuple[dict[str, str], int]:
|
||||
"""dev-token：每个用例独立 device_key，拿到的就是全新用户。"""
|
||||
resp = await client.post(
|
||||
"/api/v1/auth/dev-token", json={"device_key": f"report-it-{uuid.uuid4().hex[:8]}"}
|
||||
)
|
||||
assert resp.json()["code"] == 0
|
||||
data = resp.json()["data"]
|
||||
return {"Authorization": f"Bearer {data['access_token']}"}, int(data["user_id"])
|
||||
|
||||
|
||||
async def _create_report(
|
||||
client: Any, headers: dict[str, str], *, request_key: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
resp = await client.post(
|
||||
"/api/v1/reports",
|
||||
headers=headers,
|
||||
json={
|
||||
"questionnaire": {"style": "通勤", "height_cm": 168},
|
||||
"photo_object_key": "report_photo/u1.jpg",
|
||||
"request_key": request_key or f"rk-{uuid.uuid4().hex}",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()["data"]
|
||||
assert body["status"] == "pending"
|
||||
return body
|
||||
|
||||
|
||||
async def _fetch_one(statement: Any) -> Any:
|
||||
from packages.shared.db import session_scope
|
||||
|
||||
async with session_scope() as session:
|
||||
result = await session.execute(statement)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _fetch_report(report_id: int) -> Any:
|
||||
from sqlalchemy import select
|
||||
|
||||
from packages.shared.models import RpReport
|
||||
|
||||
return await _fetch_one(select(RpReport).where(RpReport.report_id == report_id))
|
||||
|
||||
|
||||
async def _fetch_input(report_id: int) -> Any:
|
||||
from sqlalchemy import select
|
||||
|
||||
from packages.shared.models import RpInput
|
||||
|
||||
return await _fetch_one(select(RpInput).where(RpInput.report_id == report_id))
|
||||
|
||||
|
||||
async def _fetch_task(task_id: int) -> Any:
|
||||
from packages.shared.db import session_scope
|
||||
from packages.task import repo as task_repo
|
||||
|
||||
async with session_scope() as session:
|
||||
task = await task_repo.get(session, task_id)
|
||||
assert task is not None
|
||||
# 脱离会话前把要断言的 JSONB 读出来（expire_on_commit=False，够用）。
|
||||
return task
|
||||
|
||||
|
||||
async def _ledger_rows(user_id: int, quota_key: str) -> list[Any]:
|
||||
from sqlalchemy import select
|
||||
|
||||
from packages.shared.db import session_scope
|
||||
from packages.shared.models import Ledger
|
||||
|
||||
async with session_scope() as session:
|
||||
result = await session.execute(
|
||||
select(Ledger)
|
||||
.where(Ledger.user_id == user_id, Ledger.quota_key == quota_key)
|
||||
.order_by(Ledger.created_at.asc(), Ledger.tx_id.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
# --- 场景 1：触发与预扣 -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_create_reserves_lifetime_quota_and_creates_pending_task(
|
||||
fake_stages: None, client: Any
|
||||
) -> None:
|
||||
headers, user_id = await _login(client)
|
||||
|
||||
data = await _create_report(client, headers)
|
||||
task_id, report_id = int(data["task_id"]), int(data["report_id"])
|
||||
assert data["version"] == 1
|
||||
|
||||
task = await _fetch_task(task_id)
|
||||
assert task.status == "pending"
|
||||
assert task.task_type == "report"
|
||||
assert task.payload["report_id"] == str(report_id)
|
||||
assert task.quota_reserved == 1
|
||||
assert task.reserve_tx_id is not None
|
||||
|
||||
report = await _fetch_report(report_id)
|
||||
assert report.status == "pending"
|
||||
rp_input = await _fetch_input(report_id)
|
||||
assert rp_input.photo_meta == {"object_key": "report_photo/u1.jpg"}
|
||||
assert rp_input.derived_features == {}
|
||||
|
||||
rows = await _ledger_rows(user_id, "report_basic")
|
||||
assert len(rows) == 1
|
||||
assert rows[0].tx_type == "consume"
|
||||
assert rows[0].status == "reserved"
|
||||
assert rows[0].period_key == "life" # report_basic 是终身配额（迁移 0003）
|
||||
|
||||
|
||||
# --- 场景 2：worker 成功 ------------------------------------------------------
|
||||
|
||||
|
||||
async def test_worker_success_commits_quota_and_writes_stage_keys(
|
||||
fake_stages: None, client: Any
|
||||
) -> None:
|
||||
from apps.worker.report_tasks import _generate_report
|
||||
|
||||
headers, user_id = await _login(client)
|
||||
data = await _create_report(client, headers)
|
||||
task_id, report_id = int(data["task_id"]), int(data["report_id"])
|
||||
|
||||
result = await _generate_report(task_id)
|
||||
assert result["status"] == "succeeded"
|
||||
|
||||
report = await _fetch_report(report_id)
|
||||
assert report.status == "succeeded"
|
||||
assert report.completed_at is not None
|
||||
assert set(report.result) == {"vision", "diagnosis"}
|
||||
assert report.result["vision"] == {"features": ["vision_runs"]}
|
||||
assert "stages" in report.model_info
|
||||
|
||||
task = await _fetch_task(task_id)
|
||||
assert task.status == "succeeded"
|
||||
assert task.progress == 100
|
||||
|
||||
rows = await _ledger_rows(user_id, "report_basic")
|
||||
assert [row.status for row in rows] == ["committed"]
|
||||
|
||||
# 列表与详情视图透出同样的产物。
|
||||
listing = (await client.get("/api/v1/reports", headers=headers)).json()["data"]
|
||||
assert [item["report_id"] for item in listing] == [str(report_id)]
|
||||
assert listing[0]["result"]["diagnosis"]["summary"] == "肩宽腰细"
|
||||
|
||||
|
||||
# --- 场景 3：幂等重提交 -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_duplicate_request_key_returns_same_task_without_second_reserve(
|
||||
fake_stages: None, client: Any
|
||||
) -> None:
|
||||
from apps.worker.report_tasks import _generate_report
|
||||
|
||||
headers, user_id = await _login(client)
|
||||
request_key = f"rk-{uuid.uuid4().hex}"
|
||||
first = await _create_report(client, headers, request_key=request_key)
|
||||
await _generate_report(int(first["task_id"])) # 先让首单成功
|
||||
|
||||
again = await client.post(
|
||||
"/api/v1/reports",
|
||||
headers=headers,
|
||||
json={
|
||||
"questionnaire": {"style": "通勤"},
|
||||
"photo_object_key": "report_photo/u1.jpg",
|
||||
"request_key": request_key,
|
||||
},
|
||||
)
|
||||
assert again.status_code == 200
|
||||
replay = again.json()["data"]
|
||||
assert replay["task_id"] == first["task_id"]
|
||||
assert replay["report_id"] == first["report_id"]
|
||||
assert replay["status"] == "succeeded"
|
||||
|
||||
# tk_task 幂等键 + reserve 幂等键双保险：只有一笔 reserve。
|
||||
rows = await _ledger_rows(user_id, "report_basic")
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status == "committed"
|
||||
assert await _count_user_reports(user_id) == 1 # 重放不建新报告行
|
||||
|
||||
|
||||
async def _count_user_reports(user_id: int) -> int:
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from packages.shared.db import session_scope
|
||||
from packages.shared.models import RpReport
|
||||
|
||||
async with session_scope() as session:
|
||||
result = await session.execute(
|
||||
select(func.count()).select_from(RpReport).where(RpReport.user_id == user_id)
|
||||
)
|
||||
return int(result.scalar_one())
|
||||
|
||||
|
||||
# --- 场景 4：首次资格 40920 ---------------------------------------------------
|
||||
|
||||
|
||||
async def test_second_first_report_is_rejected_with_40920(fake_stages: None, client: Any) -> None:
|
||||
from apps.worker.report_tasks import _generate_report
|
||||
|
||||
headers, _ = await _login(client)
|
||||
data = await _create_report(client, headers)
|
||||
await _generate_report(int(data["task_id"])) # 首单成功，终身资格已用掉
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/reports",
|
||||
headers=headers,
|
||||
json={
|
||||
"questionnaire": {},
|
||||
"photo_object_key": "report_photo/u2.jpg",
|
||||
"request_key": f"rk-{uuid.uuid4().hex}", # 新键：不是幂等重放
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
body = resp.json()
|
||||
assert body["code"] == 40920
|
||||
assert "30" in (body.get("hint") or "")
|
||||
|
||||
|
||||
# --- 场景 5：失败退款 + derived_features 保留 ----------------------------------
|
||||
|
||||
|
||||
async def test_stage_failure_refunds_and_preserves_derived_features(
|
||||
failing_stages: None, client: Any
|
||||
) -> None:
|
||||
from apps.worker.report_tasks import _generate_report
|
||||
from packages.report.pipeline import StageFailureError
|
||||
|
||||
headers, user_id = await _login(client)
|
||||
data = await _create_report(client, headers)
|
||||
task_id, report_id = int(data["task_id"]), int(data["report_id"])
|
||||
|
||||
with pytest.raises(StageFailureError):
|
||||
await _generate_report(task_id)
|
||||
|
||||
report = await _fetch_report(report_id)
|
||||
assert report.status == "failed"
|
||||
# 已完成阶段的产物仍在 + 错误摘要写进 result。
|
||||
assert report.result["vision"] == {"features": ["vision_runs"]}
|
||||
assert report.result["error"]["stage"] == "boom"
|
||||
|
||||
# T-P3-09 核心：视觉中间态保留——重试/重生成不需要重新上传照片。
|
||||
rp_input = await _fetch_input(report_id)
|
||||
assert rp_input.derived_features == {"vision_runs": 1}
|
||||
|
||||
task = await _fetch_task(task_id)
|
||||
assert task.status == "failed"
|
||||
|
||||
# 退款：consume 流水转 refunded，反向 refund 流水 committed。
|
||||
rows = await _ledger_rows(user_id, "report_basic")
|
||||
assert [(row.tx_type, row.status) for row in rows] == [
|
||||
("consume", "refunded"),
|
||||
("refund", "committed"),
|
||||
]
|
||||
|
||||
# failed 报告的详情只透错误摘要，不透半成品。
|
||||
detail = (await client.get(f"/api/v1/reports/{report_id}", headers=headers)).json()["data"]
|
||||
assert set(detail["result"]) == {"error"}
|
||||
assert detail["result"]["error"]["stage"] == "boom"
|
||||
|
||||
|
||||
# --- 场景 6：regenerate 走月配额并复用视觉中间态 -------------------------------
|
||||
|
||||
|
||||
async def test_regenerate_uses_month_quota_and_reuses_derived_features(
|
||||
fake_stages: None, client: Any
|
||||
) -> None:
|
||||
from apps.worker.report_tasks import _generate_report
|
||||
|
||||
headers, user_id = await _login(client)
|
||||
first = await _create_report(client, headers)
|
||||
await _generate_report(int(first["task_id"]))
|
||||
|
||||
regen = await client.post(
|
||||
f"/api/v1/reports/{first['report_id']}/regenerate",
|
||||
headers=headers,
|
||||
json={"request_key": f"rk-regen-{uuid.uuid4().hex}"},
|
||||
)
|
||||
assert regen.status_code == 200
|
||||
data = regen.json()["data"]
|
||||
assert data["version"] == 2
|
||||
new_report_id = int(data["report_id"])
|
||||
assert new_report_id != int(first["report_id"])
|
||||
|
||||
# 重新生成走 report_regen 月配额（m:YYYY-MM），不复用终身额度。
|
||||
rows = await _ledger_rows(user_id, "report_regen")
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status == "reserved"
|
||||
assert rows[0].period_key.startswith("m:")
|
||||
|
||||
# 新输入行直接带着上一版的视觉中间态（无需重新上传照片）。
|
||||
new_input = await _fetch_input(new_report_id)
|
||||
assert new_input.derived_features == {"vision_runs": 1}
|
||||
|
||||
await _generate_report(int(data["task_id"]))
|
||||
report_v2 = await _fetch_report(new_report_id)
|
||||
assert report_v2.status == "succeeded"
|
||||
assert report_v2.version == 2
|
||||
# 成功后视觉仍未重算：vision_runs 计数保持 1（Fake 阶段非空即跳过）。
|
||||
input_after = await _fetch_input(new_report_id)
|
||||
assert input_after.derived_features == {"vision_runs": 1}
|
||||
Reference in New Issue
Block a user