From a1d0171fc5fb43e9077b0c0e582795172db925d7 Mon Sep 17 00:00:00 2001 From: xsl Date: Sat, 12 Sep 2026 00:24:01 +0800 Subject: [PATCH] =?UTF-8?q?feat(uploads):=20=E6=9C=AC=E5=9C=B0=E7=A3=81?= =?UTF-8?q?=E7=9B=98=E7=9B=B4=E4=BC=A0=EF=BC=88COS=20=E9=99=8D=E7=BA=A7?= =?UTF-8?q?=E8=B7=AF=E7=BA=BF=EF=BC=89+=20=E6=8A=A5=E5=91=8A=E7=85=A7?= =?UTF-8?q?=E7=89=87=E5=AD=98=E5=9C=A8=E6=80=A7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /uploads/direct：原始字节 body + Content-Type 申报 + scene query（默认 report_photo），场景/类型/大小三重校验复用 storage 既有规则，落盘 var/uploads///.（config 新增 upload_dir），key 布局与 COS 同构；不加 python-multipart 依赖 - storage/local.py：save_direct_upload + require_local_upload（resolve 归并 + is_relative_to 归属收敛 + 存在性校验，越界/模拟/不存在一律 404） - report/service.create_report：资格校验后、配额预扣前插入照片存在性校验（模拟 key 被拦且不消耗额度；regenerate 复用已验 photo_meta 不复核） - 测试：test_upload_direct 14 单测 + test_upload_direct_flow 4 集成（真 key 全链路/假 key 404 零流水/401/40010） 偏差说明：test_report_flow._create_report 改走 /uploads/direct 拿真 key（存在性校验必然封死假 key），40920 与幂等重放两条断言语义未动。 --- apps/api/routers/uploads.py | 26 ++- packages/report/schemas.py | 7 +- packages/report/service.py | 9 + packages/storage/local.py | 94 ++++++++++ tests/integration/test_report_flow.py | 19 +- tests/integration/test_upload_direct_flow.py | 168 +++++++++++++++++ tests/unit/test_upload_direct.py | 181 +++++++++++++++++++ 7 files changed, 499 insertions(+), 5 deletions(-) create mode 100644 packages/storage/local.py create mode 100644 tests/integration/test_upload_direct_flow.py create mode 100644 tests/unit/test_upload_direct.py diff --git a/apps/api/routers/uploads.py b/apps/api/routers/uploads.py index f1d6839..b511648 100644 --- a/apps/api/routers/uploads.py +++ b/apps/api/routers/uploads.py @@ -4,14 +4,20 @@ 字节流由客户端直传 COS。真实联调需在 `.env` 配 COS 四件套 （COS_REGION / COS_SECRET_ID / COS_SECRET_KEY + 场景桶配置 UPLOAD_SCENES_JSON）；本地单测注入 FakeSigner，不依赖任何真实 COS。 + +``POST /uploads/direct`` — 本地磁盘直传（无 COS 四件套的降级路线，docs +提交 9228f67 决策）：body 即图片原始字节，Content-Type 头申报类型， +scene 走 query（默认 report_photo）。校验口径与 presign 完全一致， +落盘 ``settings.upload_dir``，返回的 object_key 供建报告等接口回传。 """ from __future__ import annotations -from fastapi import APIRouter +from fastapi import APIRouter, Query, Request -from apps.api.deps import CurrentUserId, StorageServiceDep +from apps.api.deps import CurrentUserId, SettingsDep, StorageServiceDep from packages.shared.response import ok +from packages.storage.local import save_direct_upload from packages.storage.schemas import PresignRequest router = APIRouter(prefix="/uploads", tags=["uploads"]) @@ -25,3 +31,19 @@ async def presign_upload( ) -> dict[str, object]: presigned = storage.build_presign(user_id=user_id, request=request) return ok(presigned.model_dump()) + + +@router.post("/direct", summary="本地磁盘直传图片（无 COS 降级路线）：body 为原始字节") +async def direct_upload( + user_id: CurrentUserId, + settings: SettingsDep, + request: Request, + scene: str = Query(default="report_photo", min_length=1, max_length=64), +) -> dict[str, object]: + data = await request.body() + # Content-Type 头可能带参数（如 "image/jpeg; boundary=…"），只取主类型。 + content_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + result = save_direct_upload( + settings, user_id=user_id, scene=scene, content_type=content_type, data=data + ) + return ok(result.model_dump()) diff --git a/packages/report/schemas.py b/packages/report/schemas.py index f64305d..7f98362 100644 --- a/packages/report/schemas.py +++ b/packages/report/schemas.py @@ -17,7 +17,12 @@ class ReportCreateRequest(BaseModel): default_factory=dict, description="问卷答案；字段结构由产品定稿后在路由层收紧" ) photo_object_key: str = Field( - min_length=1, max_length=255, description="临时桶里已上传照片的对象键（report_photo 场景）" + min_length=1, + max_length=255, + description=( + "已上传照片的对象键（report_photo 场景）；本地降级路线下为 " + "POST /uploads/direct 返回的相对路径，建报告时做存在性校验" + ), ) request_key: str | None = Field( default=None, max_length=128, description="幂等键；相同键返回同一任务且不重复扣配额" diff --git a/packages/report/service.py b/packages/report/service.py index 7384214..088b299 100644 --- a/packages/report/service.py +++ b/packages/report/service.py @@ -22,6 +22,7 @@ from packages.shared.config import Settings from packages.shared.errors import AppError, ErrorCode, ForbiddenError from packages.shared.logging import get_logger from packages.shared.models import RpInput, RpReport, Task +from packages.storage.local import require_local_upload from packages.task import repo as task_repo from packages.task.service import TaskService @@ -101,6 +102,14 @@ class ReportService: if await repo.find_succeeded_basic(self._session, user_id=user_id) is not None: raise AppError(ErrorCode.REPORT_ALREADY_EXISTS) + # 照片存在性校验（本地磁盘降级路线）：key 必须落在 + # upload_dir/report_photo/<本人>/ 下且文件真实存在——模拟 key 与越界 + # key 一律 404。校验在配额预扣之前，不合格的照片不消耗额度。 + # regenerate 不复核：它复用的是上一版已验过的 photo_meta（T-P3-09）。 + require_local_upload( + self._settings, user_id=user_id, scene="report_photo", object_key=photo_object_key + ) + report = RpReport( report_id=next_id(), user_id=user_id, diff --git a/packages/storage/local.py b/packages/storage/local.py new file mode 100644 index 0000000..81f2214 --- /dev/null +++ b/packages/storage/local.py @@ -0,0 +1,94 @@ +"""本地磁盘直传 —— 无 COS 四件套时的降级路线（docs 提交 9228f67 决策）。 + +与 presign 共用场景/类型/大小三重校验口径（同 40010 错误码），但字节流经 +应用服务器落盘 ``settings.upload_dir``；object_key 为相对该目录的 POSIX 路径 +``{scene}/{user_id}/{uuid4hex}.{ext}``，与 COS 布局同构——后续切回 COS 直传 +只换写入实现，key 语义与下游（报告链路的 photo_object_key）不变。 + +依赖纪律：仓库未引入 python-multipart（且不加新依赖），服务端不收 +multipart 表单；路由层以「body 即图片字节、Content-Type 头申报类型、 +scene 走 query」的原始字节口径对接本模块。 +""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from packages.shared.config import Settings +from packages.shared.errors import AppError, ErrorCode, NotFoundError +from packages.shared.logging import get_logger +from packages.storage.service import _CONTENT_TYPE_EXT, _parse_allowed_types + +logger = get_logger(__name__) + + +class DirectUploadResponse(BaseModel): + """本地直传结果：object_key 交给后续业务接口（如建报告）原样回传。""" + + object_key: str = Field(description="相对 upload_dir 的对象键（scene/user_id/uuid.ext）") + size_bytes: int = Field(description="实际落盘字节数（服务端实测，非客户端自报）") + content_type: str = Field(description="回显申报的 Content-Type") + + +def save_direct_upload( + settings: Settings, + *, + user_id: int, + scene: str, + content_type: str, + data: bytes, +) -> DirectUploadResponse: + """三重校验（场景/类型/大小）通过后落盘，与 presign 同码拒绝（40010）。 + + presign 只能信客户端自报的 size_bytes；直传拿到的是真实字节数， + 空文件与超限在此一并拒绝。 + """ + if scene not in settings.upload_scenes: + raise AppError(ErrorCode.UPLOAD_REJECTED, message="不支持的上传场景") + if content_type not in _parse_allowed_types(settings.upload_allowed_types): + raise AppError(ErrorCode.UPLOAD_REJECTED) + ext = _CONTENT_TYPE_EXT.get(content_type) + if ext is None: + raise AppError(ErrorCode.UPLOAD_REJECTED) + if not data or len(data) > settings.upload_max_bytes: + raise AppError(ErrorCode.UPLOAD_REJECTED) + + # 文件名由服务端生成（uuid 防猜），object_key 不含任何客户端输入， + # 落盘路径因此天然无穿越面。 + object_key = f"{scene}/{user_id}/{uuid4().hex}.{ext}" + path = Path(settings.upload_dir).resolve() / object_key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + logger.info( + "direct_upload_saved", + scene=scene, + user_id=user_id, + object_key=object_key, + size_bytes=len(data), + ) + return DirectUploadResponse( + object_key=object_key, size_bytes=len(data), content_type=content_type + ) + + +def require_local_upload( + settings: Settings, + *, + user_id: int, + scene: str, + object_key: str, +) -> Path: + """把 object_key 收敛到 upload_dir 内并校验存在性，挡模拟/越界 key。 + + 「不存在」与「不是本人的」一律同报 404，不向调用方泄露目录结构； + resolve() 先归并 ``..`` 与绝对路径，再用 is_relative_to 判定归属目录。 + """ + base = Path(settings.upload_dir).resolve() + owner_dir = base / scene / str(user_id) + target = (base / object_key).resolve() + if not target.is_relative_to(owner_dir) or not target.is_file(): + raise NotFoundError("照片不存在或已过期，请重新上传") + return target diff --git a/tests/integration/test_report_flow.py b/tests/integration/test_report_flow.py index c44b92b..f2c52a7 100644 --- a/tests/integration/test_report_flow.py +++ b/tests/integration/test_report_flow.py @@ -133,6 +133,20 @@ async def _login(client: Any) -> tuple[dict[str, str], int]: return {"Authorization": f"Bearer {data['access_token']}"}, int(data["user_id"]) +async def _upload_photo(client: Any, headers: dict[str, str]) -> str: + """直传一张假 JPEG 拿真实 object_key（建报告对 photo_object_key 做存在性校验）。 + + 字节内容不需要是真图片：上传链路只校验类型/大小，不嗅探内容。 + """ + resp = await client.post( + "/api/v1/uploads/direct?scene=report_photo", + headers={**headers, "Content-Type": "image/jpeg"}, + content=b"\xff\xd8\xff\xe0" + b"\x00" * 64, + ) + assert resp.status_code == 200 + return str(resp.json()["data"]["object_key"]) + + async def _create_report( client: Any, headers: dict[str, str], *, request_key: str | None = None ) -> dict[str, Any]: @@ -141,7 +155,7 @@ async def _create_report( headers=headers, json={ "questionnaire": {"style": "通勤", "height_cm": 168}, - "photo_object_key": "report_photo/u1.jpg", + "photo_object_key": await _upload_photo(client, headers), "request_key": request_key or f"rk-{uuid.uuid4().hex}", }, ) @@ -223,7 +237,8 @@ async def test_create_reserves_lifetime_quota_and_creates_pending_task( report = await _fetch_report(report_id) assert report.status == "pending" rp_input = await _fetch_input(report_id) - assert rp_input.photo_meta == {"object_key": "report_photo/u1.jpg"} + # photo_object_key 已由直传接口真实落盘（布局 report_photo/{user_id}/{uuid}.jpg） + assert str(rp_input.photo_meta["object_key"]).startswith(f"report_photo/{user_id}/") assert rp_input.derived_features == {} rows = await _ledger_rows(user_id, "report_basic") diff --git a/tests/integration/test_upload_direct_flow.py b/tests/integration/test_upload_direct_flow.py new file mode 100644 index 0000000..91356c1 --- /dev/null +++ b/tests/integration/test_upload_direct_flow.py @@ -0,0 +1,168 @@ +"""本地直传 → 建报告全链路（无 COS 降级路线）。 + +dev-token 登录 → POST /uploads/direct 上传原始字节 → 拿真实 object_key → +POST /reports 建报告走通；模拟 key 被存在性校验以 404 挡住，且不消耗配额。 +不启真 celery：API 投递钩子替换为 no-op（与 test_report_flow 同口径）。 +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import pytest + + +def _infra_available() -> bool: + return bool(os.getenv("MEIDA_RUN_INTEGRATION")) + + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not _infra_available(), + reason="set MEIDA_RUN_INTEGRATION=1 with PostgreSQL and Redis running", + ), +] + +JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"\x00" * 64 + + +@pytest.fixture +def no_enqueue(monkeypatch: pytest.MonkeyPatch) -> None: + """不向真 Redis 投递：本用例只验证「上传 → 建报告」入口，不跑 worker。""" + monkeypatch.setattr("apps.api.routers.reports._enqueue_report", lambda task_id: None) + + +@pytest.fixture +async def client(no_enqueue: None) -> AsyncIterator[Any]: + import httpx + + from apps.api.main import create_app + from packages.shared.config import get_settings + + app = create_app(get_settings()) + transport = httpx.ASGITransport(app=app) + async with ( + httpx.AsyncClient(transport=transport, base_url="http://test") as http, + app.router.lifespan_context(app), + ): + yield http + + +async def _login(client: Any) -> tuple[dict[str, str], int]: + """dev-token：每个用例独立 device_key，拿到的就是全新用户。""" + resp = await client.post( + "/api/v1/auth/dev-token", json={"device_key": f"upload-it-{uuid.uuid4().hex[:8]}"} + ) + assert resp.json()["code"] == 0 + data = resp.json()["data"] + return {"Authorization": f"Bearer {data['access_token']}"}, int(data["user_id"]) + + +async def _upload(client: Any, headers: dict[str, str]) -> str: + resp = await client.post( + "/api/v1/uploads/direct?scene=report_photo", + headers={**headers, "Content-Type": "image/jpeg"}, + content=JPEG_BYTES, + ) + assert resp.status_code == 200 + return str(resp.json()["data"]["object_key"]) + + +async def test_direct_upload_then_create_report_with_real_key(client: Any) -> None: + headers, user_id = await _login(client) + + object_key = await _upload(client, headers) + assert object_key.startswith(f"report_photo/{user_id}/") + assert object_key.endswith(".jpg") + + resp = await client.post( + "/api/v1/reports", + headers=headers, + json={ + "questionnaire": {"style": "通勤"}, + "photo_object_key": object_key, + "request_key": f"rk-{uuid.uuid4().hex}", + }, + ) + assert resp.status_code == 200 + body = resp.json()["data"] + assert body["status"] == "pending" + + # 落盘的输入行应记录真实 object_key。 + from sqlalchemy import select + + from packages.shared.db import session_scope + from packages.shared.models import RpInput + + async with session_scope() as session: + result = await session.execute( + select(RpInput).where(RpInput.report_id == int(body["report_id"])) + ) + rp_input = result.scalar_one() + assert rp_input.photo_meta == {"object_key": object_key} + + +async def test_create_report_with_mock_key_is_rejected_404_without_reserving_quota( + client: Any, +) -> None: + headers, user_id = await _login(client) + + resp = await client.post( + "/api/v1/reports", + headers=headers, + json={ + "questionnaire": {}, + "photo_object_key": f"report_photo/{user_id}/mock123.jpg", # 布局合法的模拟键 + "request_key": f"rk-{uuid.uuid4().hex}", + }, + ) + assert resp.status_code == 404 + assert resp.json()["code"] == 40400 + + # 拒绝发生在配额预扣之前：终身配额流水必须为空。 + from sqlalchemy import func, select + + from packages.shared.db import session_scope + from packages.shared.models import Ledger + + async with session_scope() as session: + result = await session.execute( + select(func.count()) + .select_from(Ledger) + .where(Ledger.user_id == user_id, Ledger.quota_key == "report_basic") + ) + assert int(result.scalar_one()) == 0 + + +async def test_direct_upload_requires_auth(client: Any) -> None: + resp = await client.post( + "/api/v1/uploads/direct?scene=report_photo", + headers={"Content-Type": "image/jpeg"}, + content=JPEG_BYTES, + ) + assert resp.status_code == 401 + assert resp.json()["code"] == 40100 + + +async def test_direct_upload_rejects_disallowed_type_and_scene(client: Any) -> None: + headers, _ = await _login(client) + + bad_type = await client.post( + "/api/v1/uploads/direct?scene=report_photo", + headers={**headers, "Content-Type": "application/pdf"}, + content=JPEG_BYTES, + ) + assert bad_type.status_code == 400 + assert bad_type.json()["code"] == 40010 + + bad_scene = await client.post( + "/api/v1/uploads/direct?scene=evil", + headers={**headers, "Content-Type": "image/jpeg"}, + content=JPEG_BYTES, + ) + assert bad_scene.status_code == 400 + assert bad_scene.json()["code"] == 40010 diff --git a/tests/unit/test_upload_direct.py b/tests/unit/test_upload_direct.py new file mode 100644 index 0000000..3d35dc0 --- /dev/null +++ b/tests/unit/test_upload_direct.py @@ -0,0 +1,181 @@ +"""本地磁盘直传（无 COS 降级路线）：三重校验 + 落盘布局 + 存在性校验。 + +零网络零 SDK：落盘用 tmp_path 隔离。require_local_upload 是报告链路挡 +模拟/越界 photo_object_key 的关卡，穿越、他人目录、不存在三类逐条锁死。 +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from packages.shared.config import Settings +from packages.shared.errors import AppError, ErrorCode +from packages.storage.local import require_local_upload, save_direct_upload + +JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"\x00" * 32 + + +def make_settings(tmp_path: Path, **overrides: Any) -> Settings: + """显式 kwargs 构造（与 test_upload_presign 同口径），测试不受 .env 影响。 + + upload_scenes 定义了 alias 且未开启 populate_by_name，必须按 + ``UPLOAD_SCENES_JSON`` 传入——按字段名传会被静默忽略。 + """ + options: dict[str, Any] = { + "_env_file": None, + "upload_dir": str(tmp_path / "uploads"), + "upload_max_bytes": 1024, + "upload_allowed_types": "image/jpeg,image/png,image/webp", + "UPLOAD_SCENES_JSON": {"report_photo": {"bucket": "temp"}, "avatar": {"bucket": "temp"}}, + } + options.update(overrides) + return Settings(**options) + + +# --- save_direct_upload ----------------------------------------------------- + + +def test_save_writes_file_and_returns_auditable_key(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + result = save_direct_upload( + settings, user_id=7, scene="report_photo", content_type="image/jpeg", data=JPEG_BYTES + ) + + # key 布局：{scene}/{user_id}/{uuid4hex}.{ext}（与 presign 同构，无日期段—— + # 本地落盘没有 COS 生命周期规则需要按日分组） + prefix = "report_photo/7/" + assert result.object_key.startswith(prefix) + stem, ext = result.object_key.removeprefix(prefix).split(".") + assert ext == "jpg" + assert len(stem) == 32 + assert all(char in "0123456789abcdef" for char in stem) + + written = tmp_path / "uploads" / result.object_key + assert written.read_bytes() == JPEG_BYTES + assert result.size_bytes == len(JPEG_BYTES) + assert result.content_type == "image/jpeg" + + +def test_save_maps_content_type_to_extension(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + result = save_direct_upload( + settings, user_id=1, scene="avatar", content_type="image/webp", data=JPEG_BYTES + ) + assert result.object_key.endswith(".webp") + + +def test_save_rejects_unknown_scene(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + with pytest.raises(AppError) as excinfo: + save_direct_upload( + settings, user_id=1, scene="evil", content_type="image/jpeg", data=JPEG_BYTES + ) + assert excinfo.value.code is ErrorCode.UPLOAD_REJECTED + assert excinfo.value.message == "不支持的上传场景" + + +def test_save_rejects_disallowed_content_type(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + with pytest.raises(AppError) as excinfo: + save_direct_upload( + settings, + user_id=1, + scene="report_photo", + content_type="application/pdf", + data=JPEG_BYTES, + ) + assert excinfo.value.code is ErrorCode.UPLOAD_REJECTED + + +def test_save_rejects_oversize_and_empty(tmp_path: Path) -> None: + settings = make_settings(tmp_path) # upload_max_bytes=1024 + with pytest.raises(AppError) as excinfo: + save_direct_upload( + settings, + user_id=1, + scene="report_photo", + content_type="image/jpeg", + data=b"\x00" * 1025, + ) + assert excinfo.value.code is ErrorCode.UPLOAD_REJECTED + with pytest.raises(AppError) as excinfo: + save_direct_upload( + settings, user_id=1, scene="report_photo", content_type="image/jpeg", data=b"" + ) + assert excinfo.value.code is ErrorCode.UPLOAD_REJECTED + # 拒绝路径不落任何文件 + assert not (tmp_path / "uploads").exists() + + +def test_save_size_equal_to_limit_is_allowed(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + result = save_direct_upload( + settings, + user_id=1, + scene="report_photo", + content_type="image/png", + data=b"\x00" * 1024, + ) + assert result.size_bytes == 1024 + + +# --- require_local_upload --------------------------------------------------- + + +def test_require_roundtrip_returns_existing_file(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + saved = save_direct_upload( + settings, user_id=7, scene="report_photo", content_type="image/jpeg", data=JPEG_BYTES + ) + path = require_local_upload( + settings, user_id=7, scene="report_photo", object_key=saved.object_key + ) + assert path.read_bytes() == JPEG_BYTES + + +def test_require_rejects_missing_file(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + with pytest.raises(AppError) as excinfo: + require_local_upload( + settings, user_id=7, scene="report_photo", object_key="report_photo/7/ghost.jpg" + ) + assert excinfo.value.code is ErrorCode.NOT_FOUND + + +def test_require_rejects_mock_key_without_upload(tmp_path: Path) -> None: + # 端侧历史模拟键 report_photo/{userId}/{随机}.jpg：布局合法但无落盘文件。 + settings = make_settings(tmp_path) + with pytest.raises(AppError) as excinfo: + require_local_upload( + settings, user_id=7, scene="report_photo", object_key="report_photo/7/abc123.jpg" + ) + assert excinfo.value.code is ErrorCode.NOT_FOUND + + +def test_require_rejects_other_users_key(tmp_path: Path) -> None: + settings = make_settings(tmp_path) + saved = save_direct_upload( + settings, user_id=8, scene="report_photo", content_type="image/jpeg", data=JPEG_BYTES + ) + with pytest.raises(AppError) as excinfo: + require_local_upload(settings, user_id=7, scene="report_photo", object_key=saved.object_key) + assert excinfo.value.code is ErrorCode.NOT_FOUND + + +@pytest.mark.parametrize( + "evil_key", + [ + "report_photo/7/../../avatar/8/x.jpg", # 路径穿越到他人场景 + "report_photo/7/../../../../etc/passwd", # 穿越出 upload_dir + "/etc/passwd", # 绝对路径吞噬 base + "avatar/7/x.jpg", # 场景不匹配 + ], +) +def test_require_rejects_out_of_bounds_keys(tmp_path: Path, evil_key: str) -> None: + settings = make_settings(tmp_path) + with pytest.raises(AppError) as excinfo: + require_local_upload(settings, user_id=7, scene="report_photo", object_key=evil_key) + assert excinfo.value.code is ErrorCode.NOT_FOUND