POST /reports 骨架:资格校验→终身配额预扣→tk_task→cpu 队列;动态装配 流水线(dotted-path 注册、缺阶段即 CONFIG_ERROR);中间态逐段落库、失败 退款、derived_features 跨重试/重生成复用(T-P3-07/09)。 实现:packages/report 的 pipeline.py(Stage 契约保持不变,新增 assemble_stages/ run_pipeline/StageFailureError,ReportContext 增可选 task_id 参数、向后兼容)、 repo.py、schemas.py、service.py(重放优先于资格校验、reserve+建行+建任务同事务、 提交后才投递、并发同键落败回滚);apps/api/routers/reports.py 填桩(send_task 按 任务名投递,不 import apps.worker);apps/worker/report_tasks.py(worker 侧自建 gateway 装配、状态归一 QUEUED→RUNNING、成功 commit 配额+成本聚合写 model_info、 基础设施异常重试一次、重试耗尽兜底失败退款)。测试:单测 6 个(装配缺失/空配置/ 协议违约/顺序执行/失败保留产物),集成 6 场景(预扣 lifetime/PENDING、成功 committed+两阶段键、同 request_key 幂等不重复扣、二次首生成 40920、阶段失败 refund+derived_features 保留、regenerate 走 m: 月配额 version+1 复用视觉中间态)。 偏差说明: 1) apps/worker/celery_app.py 的 include 只含 apps.worker.tasks(禁改文件),真 worker 进程注册 meida.report.generate 需协调者合并时把 include 补上 apps.worker.report_tasks;API 侧已按任务名 send_task,两侧只靠名字衔接,测试 不受影响。2) ReportContext 增加可选 task_id 关键字参数(失败结案要推动 tk_task),Stage 协议/DEFAULT_STAGE_PATHS/stage_names 未动,T16/T17 兼容。 3) JSONB 深变更用 flag_modified 显式标脏(重新赋相等新 dict 会被 flush 的按值 比较跳过,集成测试抓到过该坑)。4) POST 后 tk_task 保持 PENDING(任务文档 §4 场景 1 口径),QUEUED→RUNNING 由 worker 入口归一。
176 lines
5.4 KiB
Python
176 lines
5.4 KiB
Python
"""report 流水线编排的纯逻辑单测(不连库)。
|
||
|
||
覆盖任务文档 §4 的两条:assemble_stages 缺失即 CONFIG_ERROR 且列出缺失项;
|
||
run_pipeline 顺序执行、产物按键写入、第二阶段抛错 → failed 且已完成阶段
|
||
产物仍在。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
|
||
from packages.report.pipeline import (
|
||
ReportContext,
|
||
StageFailureError,
|
||
assemble_stages,
|
||
run_pipeline,
|
||
)
|
||
from packages.shared.config import Settings
|
||
from packages.shared.errors import AppError, ErrorCode
|
||
from packages.shared.models import RpInput, RpReport
|
||
|
||
|
||
class FakeSession:
|
||
"""只满足 flush/commit/rollback 的最小会话——单测不连库。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.flush_count = 0
|
||
self.commit_count = 0
|
||
|
||
async def flush(self) -> None:
|
||
self.flush_count += 1
|
||
|
||
async def commit(self) -> None:
|
||
self.commit_count += 1
|
||
|
||
async def rollback(self) -> None: ...
|
||
|
||
|
||
def _stage_settings(*paths: str) -> Settings:
|
||
"""按别名构造阶段配置——字段名带 alias 且未开 populate_by_name,
|
||
用字段名构造会被静默忽略、落回默认值(踩过坑,写下来防再踩)。"""
|
||
return Settings(REPORT_PIPELINE_STAGES_JSON=json.dumps(list(paths)))
|
||
|
||
|
||
def _make_ctx() -> tuple[ReportContext, FakeSession]:
|
||
# 瞬态 ORM 对象:default=dict 只在 INSERT 时生效,这里必须显式给 JSONB 列初值。
|
||
report = RpReport(
|
||
report_id=101,
|
||
user_id=1,
|
||
status="running",
|
||
result={},
|
||
model_info={},
|
||
)
|
||
report_input = RpInput(
|
||
input_id=11,
|
||
report_id=101,
|
||
questionnaire={},
|
||
photo_meta={},
|
||
derived_features={},
|
||
)
|
||
session = FakeSession()
|
||
ctx = ReportContext(
|
||
report=report,
|
||
report_input=report_input,
|
||
session=session, # type: ignore[arg-type]
|
||
settings=Settings(),
|
||
)
|
||
return ctx, session
|
||
|
||
|
||
class AlphaStage:
|
||
name = "alpha"
|
||
|
||
async def run(self, ctx: ReportContext) -> None:
|
||
ctx.report.result["alpha"] = {"value": 1}
|
||
ctx.report_input.derived_features["pose"] = {"kps": 3}
|
||
|
||
|
||
class BetaStage:
|
||
name = "beta"
|
||
|
||
async def run(self, ctx: ReportContext) -> None:
|
||
# 顺序契约:后一阶段能读到前一阶段的产物。
|
||
assert ctx.report.result.get("alpha") == {"value": 1}
|
||
ctx.report.result["beta"] = {"value": ctx.report.result["alpha"]["value"] + 1}
|
||
|
||
|
||
class BoomStage:
|
||
name = "boom"
|
||
|
||
async def run(self, ctx: ReportContext) -> None:
|
||
raise RuntimeError("爆了:模拟阶段失败")
|
||
|
||
|
||
async def test_run_pipeline_executes_stages_in_order_and_flushes_each() -> None:
|
||
ctx, session = _make_ctx()
|
||
|
||
await run_pipeline(ctx, [AlphaStage(), BetaStage()])
|
||
|
||
assert ctx.report.result == {"alpha": {"value": 1}, "beta": {"value": 2}}
|
||
assert ctx.report_input.derived_features == {"pose": {"kps": 3}}
|
||
# 每个阶段一次 flush:中间态逐段落库(T-P3-09)的结构保证。
|
||
assert session.flush_count == 2
|
||
|
||
|
||
async def test_run_pipeline_failure_keeps_completed_stage_and_marks_failed() -> None:
|
||
ctx, session = _make_ctx()
|
||
|
||
with pytest.raises(StageFailureError) as excinfo:
|
||
await run_pipeline(ctx, [AlphaStage(), BoomStage()])
|
||
|
||
assert excinfo.value.stage_name == "boom"
|
||
assert ctx.report.status == "failed"
|
||
# 已完成阶段的产物与视觉中间态都还在——重试/重生成不重跑的结构基础。
|
||
assert ctx.report.result["alpha"] == {"value": 1}
|
||
assert ctx.report_input.derived_features == {"pose": {"kps": 3}}
|
||
# 错误摘要写进 result,且失败结案在事务里先行提交(退款事务纪律)。
|
||
assert ctx.report.result["error"]["stage"] == "boom"
|
||
assert ctx.report.result["error"]["type"] == "RuntimeError"
|
||
assert session.commit_count == 1
|
||
|
||
|
||
def test_assemble_stages_lists_all_missing_paths() -> None:
|
||
settings = _stage_settings(
|
||
"packages.report.no_such_module:Nope",
|
||
"packages.report.pipeline:NotAClass",
|
||
)
|
||
|
||
with pytest.raises(AppError) as excinfo:
|
||
assemble_stages(settings)
|
||
|
||
assert excinfo.value.code == ErrorCode.CONFIG_ERROR
|
||
assert excinfo.value.data == {
|
||
"missing": [
|
||
"packages.report.no_such_module:Nope",
|
||
"packages.report.pipeline:NotAClass",
|
||
]
|
||
}
|
||
|
||
|
||
def test_assemble_stages_rejects_empty_config() -> None:
|
||
settings = _stage_settings()
|
||
|
||
with pytest.raises(AppError) as excinfo:
|
||
assemble_stages(settings)
|
||
|
||
assert excinfo.value.code == ErrorCode.CONFIG_ERROR
|
||
|
||
|
||
class _NotAStage:
|
||
"""存在但不满足 Stage 协议(没有 run)——必须被当成配置错误拦下。"""
|
||
|
||
name = "broken"
|
||
|
||
|
||
def test_assemble_stages_rejects_protocol_violations() -> None:
|
||
settings = _stage_settings(
|
||
"tests.unit.test_report_pipeline:AlphaStage",
|
||
"tests.unit.test_report_pipeline:_NotAStage",
|
||
)
|
||
|
||
with pytest.raises(AppError) as excinfo:
|
||
assemble_stages(settings)
|
||
|
||
assert excinfo.value.code == ErrorCode.CONFIG_ERROR
|
||
assert excinfo.value.data == {"missing": ["tests.unit.test_report_pipeline:_NotAStage"]}
|
||
|
||
|
||
def test_assemble_stages_imports_from_any_module_path() -> None:
|
||
"""dotted path 装配对本模块的 Fake 阶段同样生效(importlib 动态导入路径)。"""
|
||
settings = _stage_settings("tests.unit.test_report_pipeline:AlphaStage")
|
||
stages = assemble_stages(settings)
|
||
assert [s.name for s in stages] == ["alpha"]
|