save code

python/server/face_renderer.py

@@ -0,0 +1,312 @@
+"""
+人脸贴图渲染核心：接受一张图 + 一个效果 ID，输出 PNG 字节。
+
+流程（与 Android 端 vulkan/FaceApp.cpp 一致，CPU 实现）：
+  1. MediaPipe FaceLandmarker → 478 normalized landmarks per face
+  2. 取最大人脸（按 landmark bbox 面积）
+  3. 把 obj 的 468 顶点位置改写为 landmarks[INDEX_MAP[i]]，得到 dst 顶点
+  4. 对 mesh 的每个三角形：从 512×512 effect 贴图取 src 三角形，仿射 warp
+     到 dst 三角形位置，按贴图 alpha 通道 alpha-blend 到原图副本上
+  5. 输出 PNG（保持原图尺寸）
+"""
+
+from __future__ import annotations
+
+import io
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+
+import cv2
+import numpy as np
+from PIL import Image, ImageOps
+
+from .index_map import INDEX_MAP
+from .mesh import FaceMesh, load_face_mesh
+
+
+class NoFaceError(Exception):
+    """图里检测不到人脸。"""
+
+
+class EffectNotFoundError(Exception):
+    """指定 effect_id 没有对应贴图。"""
+
+
+@dataclass
+class RendererPaths:
+    obj_path: Path
+    effects_dir: Path
+
+
+class FaceRenderer:
+    """
+    单例式渲染器：mesh / mediapipe 模型加载一次复用。
+
+    线程安全：MediaPipe FaceLandmarker 的 detect() 不保证多线程并发，
+    用 _lock 串行化。FastAPI 单进程同步部署下，请求本身就串行，开销可忽略。
+    """
+
+    def __init__(self, paths: RendererPaths, max_image_bytes: int = 20 * 1024 * 1024):
+        self.paths = paths
+        self.max_image_bytes = max_image_bytes
+        self._mesh: FaceMesh = load_face_mesh(paths.obj_path)
+        if len(self._mesh.uvs) != len(INDEX_MAP):
+            raise RuntimeError(
+                f"mesh has {len(self._mesh.uvs)} vertices but INDEX_MAP has "
+                f"{len(INDEX_MAP)}; obj/index_map.py 不匹配"
+            )
+        # 预转 numpy 加速
+        self._index_map = np.asarray(INDEX_MAP, dtype=np.int32)
+
+        self._effect_cache: dict[int, np.ndarray] = {}
+        self._cache_lock = threading.Lock()
+
+        # MediaPipe 延迟加载（首次请求时加载，缩短启动时间，便于本地调试）
+        self._landmarker = None
+        self._landmarker_lock = threading.Lock()
+
+    # ---------- public ----------
+
+    def list_effect_ids(self) -> list[int]:
+        """枚举 effects/<n>.png 中的 n。"""
+        ids: list[int] = []
+        if not self.paths.effects_dir.is_dir():
+            return ids
+        for p in self.paths.effects_dir.iterdir():
+            if p.suffix.lower() != ".png":
+                continue
+            try:
+                n = int(p.stem)
+            except ValueError:
+                continue
+            if 1 <= n <= 99:
+                ids.append(n)
+        return sorted(ids)
+
+    def render(self, image_bytes: bytes, effect_id: int) -> bytes:
+        if len(image_bytes) > self.max_image_bytes:
+            raise ValueError("image too large")
+
+        effect_rgba = self._load_effect(effect_id)  # (H,W,4) uint8
+
+        # 1. 解码 + 处理 EXIF orientation → RGB ndarray
+        img_rgb = self._decode_input(image_bytes)
+        h, w = img_rgb.shape[:2]
+
+        # 2. MediaPipe 检测，挑最大人脸
+        landmarks_norm = self._detect_largest_face(img_rgb)  # (N,2) ∈ [0,1]² (N>=468)
+
+        # 3. obj 顶点 → 像素坐标
+        dst_xy = self._compute_dst_vertices(landmarks_norm, w, h)
+
+        # 4. warp + 混合
+        out = img_rgb.copy()
+        self._composite(out, effect_rgba, dst_xy)
+
+        # 5. 编码 PNG
+        return self._encode_png(out)
+
+    # ---------- internals ----------
+
+    def _load_effect(self, effect_id: int) -> np.ndarray:
+        if not (1 <= effect_id <= 99):
+            raise EffectNotFoundError(f"effect_id out of range: {effect_id}")
+        with self._cache_lock:
+            cached = self._effect_cache.get(effect_id)
+            if cached is not None:
+                return cached
+        png_path = self.paths.effects_dir / f"{effect_id}.png"
+        if not png_path.is_file():
+            raise EffectNotFoundError(f"effect {effect_id} not found")
+        # 用 PIL 解码以保证 RGBA 一致；OpenCV imread 对部分 PNG 会丢 alpha
+        img = Image.open(png_path).convert("RGBA")
+        arr = np.array(img, dtype=np.uint8)  # (H,W,4) RGBA
+        with self._cache_lock:
+            self._effect_cache[effect_id] = arr
+        return arr
+
+    def _decode_input(self, image_bytes: bytes) -> np.ndarray:
+        try:
+            pil = Image.open(io.BytesIO(image_bytes))
+            pil = ImageOps.exif_transpose(pil)  # 处理手机 EXIF 旋转
+            pil = pil.convert("RGB")
+        except Exception as e:
+            raise ValueError(f"invalid image: {e}") from e
+        return np.array(pil, dtype=np.uint8)
+
+    def _ensure_landmarker(self):
+        if self._landmarker is not None:
+            return
+        with self._landmarker_lock:
+            if self._landmarker is not None:
+                return
+            # 延迟 import，避免单元测试时强依赖 mediapipe
+            from mediapipe.tasks import python as mp_python
+            from mediapipe.tasks.python import vision as mp_vision
+
+            model_path = self._find_model_file()
+            base_options = mp_python.BaseOptions(model_asset_path=str(model_path))
+            options = mp_vision.FaceLandmarkerOptions(
+                base_options=base_options,
+                running_mode=mp_vision.RunningMode.IMAGE,
+                num_faces=5,  # 检测最多 5 张，挑最大那张
+                min_face_detection_confidence=0.5,
+                min_face_presence_confidence=0.5,
+                min_tracking_confidence=0.5,
+                output_face_blendshapes=False,
+                output_facial_transformation_matrixes=False,
+            )
+            self._landmarker = mp_vision.FaceLandmarker.create_from_options(options)
+
+    def _find_model_file(self) -> Path:
+        """优先用本服务 assets 下的，找不到再回退到仓库 app/src/main/assets。"""
+        local = self.paths.obj_path.parent / "face_landmarker.task"
+        if local.is_file():
+            return local
+        # 回退到仓库 app/src/main/assets/face_landmarker.task（Android SDK 用同一份）
+        repo_root = self.paths.obj_path.resolve().parents[3]
+        fallback = repo_root / "app" / "src" / "main" / "assets" / "face_landmarker.task"
+        if fallback.is_file():
+            return fallback
+        raise FileNotFoundError(
+            "face_landmarker.task not found; place it under "
+            f"{local} or {fallback}"
+        )
+
+    def _detect_largest_face(self, img_rgb: np.ndarray) -> np.ndarray:
+        self._ensure_landmarker()
+        import mediapipe as mp
+
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
+        with self._landmarker_lock:
+            result = self._landmarker.detect(mp_image)
+
+        faces = result.face_landmarks or []
+        if not faces:
+            raise NoFaceError("no face detected")
+
+        # 选 bbox 面积最大的那张
+        def bbox_area(landmarks) -> float:
+            xs = [lm.x for lm in landmarks]
+            ys = [lm.y for lm in landmarks]
+            return (max(xs) - min(xs)) * (max(ys) - min(ys))
+
+        biggest = max(faces, key=bbox_area)
+        # 转 ndarray，只取 xy
+        n = len(biggest)
+        arr = np.empty((n, 2), dtype=np.float32)
+        for i, lm in enumerate(biggest):
+            arr[i, 0] = lm.x
+            arr[i, 1] = lm.y
+        return arr
+
+    def _compute_dst_vertices(
+        self, landmarks_norm: np.ndarray, w: int, h: int
+    ) -> np.ndarray:
+        """
+        landmarks_norm: (N, 2) ∈ [0,1]²
+        返回 (468, 2) float32 像素坐标，对应 mesh 的 468 个顶点。
+        """
+        n_landmarks = landmarks_norm.shape[0]
+        if self._index_map.max() >= n_landmarks:
+            raise RuntimeError(
+                f"INDEX_MAP references landmark index {int(self._index_map.max())} "
+                f"but mediapipe returned only {n_landmarks} landmarks"
+            )
+        picked = landmarks_norm[self._index_map]  # (468, 2)
+        out = np.empty_like(picked)
+        out[:, 0] = picked[:, 0] * w
+        out[:, 1] = picked[:, 1] * h
+        return out.astype(np.float32)
+
+    def _composite(
+        self,
+        canvas_rgb: np.ndarray,
+        effect_rgba: np.ndarray,
+        dst_xy: np.ndarray,
+    ) -> None:
+        """
+        在 canvas_rgb（uint8 H×W×3，原地修改）上对 mesh 每个三角形：
+            从 effect_rgba 取对应 src 三角形 → warpAffine 到 dst → alpha 混合。
+        """
+        eh, ew = effect_rgba.shape[:2]
+        canvas_h, canvas_w = canvas_rgb.shape[:2]
+        uvs = self._mesh.uvs  # (468, 2)
+        tris = self._mesh.triangles  # (852, 3)
+
+        # 把 UV 一次性转成 effect 像素坐标
+        src_pts_all = np.empty_like(uvs)
+        src_pts_all[:, 0] = uvs[:, 0] * ew
+        src_pts_all[:, 1] = uvs[:, 1] * eh
+
+        effect_rgb = effect_rgba[..., :3]
+        effect_a = effect_rgba[..., 3]
+
+        for tri in tris:
+            i0, i1, i2 = int(tri[0]), int(tri[1]), int(tri[2])
+            src_tri = np.float32([src_pts_all[i0], src_pts_all[i1], src_pts_all[i2]])
+            dst_tri = np.float32([dst_xy[i0], dst_xy[i1], dst_xy[i2]])
+
+            # 用 dst bbox 限定计算区域，避免对全画布 warp
+            x_min = int(np.floor(dst_tri[:, 0].min()))
+            y_min = int(np.floor(dst_tri[:, 1].min()))
+            x_max = int(np.ceil(dst_tri[:, 0].max()))
+            y_max = int(np.ceil(dst_tri[:, 1].max()))
+
+            # clip 到画布
+            x_min_c = max(0, x_min)
+            y_min_c = max(0, y_min)
+            x_max_c = min(canvas_w, x_max)
+            y_max_c = min(canvas_h, y_max)
+            if x_max_c <= x_min_c or y_max_c <= y_min_c:
+                continue
+
+            box_w = x_max_c - x_min_c
+            box_h = y_max_c - y_min_c
+
+            # 构造从 src → 局部 bbox 坐标系（左上为原点）的仿射
+            dst_tri_local = dst_tri - np.array([x_min_c, y_min_c], dtype=np.float32)
+            M = cv2.getAffineTransform(src_tri, dst_tri_local)
+
+            # 给 RGB 和 alpha 分别 warp 到 bbox 大小
+            warped_rgb = cv2.warpAffine(
+                effect_rgb,
+                M,
+                (box_w, box_h),
+                flags=cv2.INTER_LINEAR,
+                borderMode=cv2.BORDER_REPLICATE,
+            )
+            warped_a = cv2.warpAffine(
+                effect_a,
+                M,
+                (box_w, box_h),
+                flags=cv2.INTER_LINEAR,
+                borderMode=cv2.BORDER_CONSTANT,
+                borderValue=0,
+            )
+
+            # 限制只在三角形内部混合（防止 bbox 多余像素污染）
+            tri_mask = np.zeros((box_h, box_w), dtype=np.uint8)
+            cv2.fillConvexPoly(
+                tri_mask,
+                dst_tri_local.astype(np.int32),
+                255,
+                lineType=cv2.LINE_AA,
+            )
+            alpha = (
+                warped_a.astype(np.float32) * (tri_mask.astype(np.float32) / 255.0)
+            ) / 255.0
+            alpha = alpha[..., None]  # (h,w,1)
+
+            roi = canvas_rgb[y_min_c:y_max_c, x_min_c:x_max_c].astype(np.float32)
+            blended = roi * (1.0 - alpha) + warped_rgb.astype(np.float32) * alpha
+            canvas_rgb[y_min_c:y_max_c, x_min_c:x_max_c] = np.clip(
+                blended, 0, 255
+            ).astype(np.uint8)
+
+    def _encode_png(self, img_rgb: np.ndarray) -> bytes:
+        pil = Image.fromarray(img_rgb)
+        buf = io.BytesIO()
+        pil.save(buf, format="PNG")
+        return buf.getvalue()

python/server/index_map.py

@@ -0,0 +1,61 @@
+"""
+OBJ 顶点索引 → MediaPipe FaceLandmarker landmark 索引的映射表。
+
+直接从 vulkan/hardcode_data.h::indexMap[468] 同步而来。意义：
+    obj 顶点 i 在画面中的位置 = mediapipe_landmarks[INDEX_MAP[i]]
+
+注意：face_picture_3dmax.obj 的顶点数量必须正好 468，与本表长度一致。
+若以后换 obj，需要重新生成本表。
+"""
+
+INDEX_MAP = [
+    127,  34, 139,  11,   0,  37, 232, 231, 120,  72,
+     39, 128, 121,  47, 104,  69,  67, 175, 171, 148,
+    118,  50, 101,  73,  40,   9, 151, 108,  48, 115,
+    131, 194, 204, 211,  74, 185,  80,  42, 183,  92,
+    186, 230, 229, 202, 212, 214,  83,  18,  17,  76,
+     61, 146, 160,  29,  30,  56, 157, 173, 106, 135,
+    192, 203, 165,  98,  21,  71,  68,  51,  45,   4,
+    144,  24,  23,  77,  91, 205, 187, 201, 200, 182,
+     90, 181,  85,  84, 206,  36, 140, 193, 189, 244,
+    159, 158,  28, 247, 246, 161, 236,   3, 196,  54,
+    168,   8, 117, 228,  31,  55,  97,  99, 126, 100,
+    166,  79, 218, 155, 154,  26, 209,  49, 136, 150,
+    217, 223,  52,  53, 134, 170,  43, 119, 226, 130,
+     63, 238,  20, 242,  46,  70, 156,  78,  62,  96,
+    143, 227, 123, 111,  44, 125,  19, 216, 153,  22,
+    167, 208, 142,  57,  60,  35, 113,  27, 210, 225,
+    137, 116,  41,  38, 129,  64, 240, 102, 207, 184,
+    169, 149, 176, 105,  66, 122,   6, 147,  65, 107,
+     89, 180,  93,  15,  86,  14,  87, 145,  88, 179,
+     95, 138, 172, 215,  58, 219,  81, 195, 199,  82,
+    163, 110, 234, 109, 235, 191, 222, 141, 221, 197,
+     25,   7,  33, 220, 237, 245, 162, 188, 174,   2,
+    241, 164,  12,  13, 198, 133, 112, 243, 239, 190,
+     32, 178, 132, 177,   1, 213,  59,  94,  75, 224,
+    233, 114, 124, 356, 389, 368, 302, 267, 452, 350,
+    349, 303, 269, 357, 343, 277, 453, 333, 332, 297,
+    152, 377, 347, 348, 330, 304, 270, 336, 337, 278,
+    279, 360, 418, 262, 431, 408, 409, 310, 415, 407,
+    410, 450, 422, 430, 434, 313, 314, 306, 307, 375,
+    387, 388, 260, 286, 414, 398, 335, 406, 364, 367,
+    416, 423, 358, 327, 251, 284, 298, 281,   5, 373,
+    374, 253, 320, 321, 425, 427, 411, 421, 405, 404,
+    315,  16, 426, 266, 400, 369, 322, 391, 417, 465,
+    464, 386, 257, 258, 466, 456, 399, 419, 285, 346,
+    340, 261, 413, 441, 460, 328, 355, 371, 329, 392,
+    439, 438, 382, 341, 256, 429, 420, 394, 379, 437,
+    443, 444, 283, 275, 440, 363, 338, 273, 451, 446,
+    342, 467, 293, 334, 282, 458, 461, 462, 276, 353,
+    383, 308, 324, 325, 300, 372, 345, 447, 352, 274,
+    248, 436, 381, 252, 393, 428, 287, 250, 384, 265,
+    259, 424, 292, 366, 271, 294, 455, 272, 432, 395,
+    299, 351, 280, 319, 295, 296, 403, 323, 454, 316,
+    380, 318, 402, 365, 435, 397, 344, 311, 291, 396,
+    268, 445, 254, 339, 449, 264,  10, 442, 370, 263,
+    255, 359, 412, 301, 378, 326, 457, 362, 459, 463,
+    354, 401, 361, 309, 376, 433, 289, 305, 448, 290,
+    288, 249, 103, 385, 331, 317, 312, 390,
+]
+
+assert len(INDEX_MAP) == 468, f"INDEX_MAP must have 468 entries, got {len(INDEX_MAP)}"

python/server/main.py

@@ -0,0 +1,62 @@
+"""
+FastAPI 入口。启动：
+
+    python -m uvicorn server.main:app --host 0.0.0.0 --port 8000
+
+或在 python/ 目录下：
+
+    uvicorn server.main:app --host 0.0.0.0 --port 8000
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import JSONResponse, Response
+
+from .face_renderer import (
+    EffectNotFoundError,
+    FaceRenderer,
+    NoFaceError,
+    RendererPaths,
+)
+
+# 路径相对本文件，避免 cwd 不同时找不到资源
+_SERVER_DIR = Path(__file__).resolve().parent
+_PATHS = RendererPaths(
+    obj_path=_SERVER_DIR / "assets" / "face_picture_3dmax.obj",
+    effects_dir=_SERVER_DIR / "effects",
+)
+
+app = FastAPI(title="Face SDK Web", version="0.1.0")
+renderer = FaceRenderer(_PATHS)
+
+
+@app.get("/health")
+def health() -> dict:
+    return {"ok": True}
+
+
+@app.get("/effects")
+def list_effects() -> dict:
+    return {"effects": renderer.list_effect_ids()}
+
+
+@app.post("/render")
+async def render(
+    image: UploadFile = File(...),
+    effect_id: int = Form(...),
+) -> Response:
+    image_bytes = await image.read()
+    if not image_bytes:
+        raise HTTPException(status_code=400, detail="empty image")
+    try:
+        png_bytes = renderer.render(image_bytes, effect_id)
+    except NoFaceError:
+        return JSONResponse(status_code=400, content={"error": "no_face"})
+    except EffectNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return Response(content=png_bytes, media_type="image/png")

python/server/mesh.py

@@ -0,0 +1,75 @@
+"""
+解析 face_picture_3dmax.obj，得到顶点 UV 和三角形索引。
+
+OBJ 约定：v 顶点 / vt UV / vn 法线，f 三元组 v/vt/vn 索引（1-based）。
+本服务只需要每个顶点的 UV 和三角形顶点索引——3D 顶点位置在 SDK 里也只是
+占位（运行时被 MediaPipe landmark 覆盖），这里同样无需读 v。
+
+★ 与 vulkan/FaceApp.cpp::LoadOBJ 一致，UV 的 V 分量取 (1 - v)，把 OBJ 的
+  bottom-up 约定翻成 top-down，与图像（OpenCV/Pillow）一致。
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+
+@dataclass
+class FaceMesh:
+    # (468, 2) float32, 每个顶点的 UV，V 已翻转，∈ [0,1]²
+    uvs: np.ndarray
+    # (852, 3) int32, 每个三角形的 3 个顶点索引（0-based，指向 uvs 第一维）
+    triangles: np.ndarray
+
+
+def load_face_mesh(obj_path: str | Path) -> FaceMesh:
+    obj_path = Path(obj_path)
+    text = obj_path.read_text(encoding="utf-8", errors="ignore")
+
+    pos_count = 0
+    uv_list: list[tuple[float, float]] = []
+    # OBJ 的 face 用 v/vt/vn，三个索引可能不同；但本工程的 obj 三者一一对齐
+    # （pos_index == uv_index == normal_index），所以我们只看 v 索引来索引 UV
+    # 列表也成立。这里仍然显式校验以防换 obj 时出错。
+    triangles: list[tuple[int, int, int]] = []
+
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split()
+        tag = parts[0]
+
+        if tag == "v":
+            pos_count += 1
+        elif tag == "vt":
+            u = float(parts[1])
+            v = float(parts[2])
+            uv_list.append((u, 1.0 - v))  # V 翻转
+        elif tag == "f":
+            if len(parts) != 4:
+                raise ValueError(f"only triangle faces supported, got: {line}")
+            tri: list[int] = []
+            for token in parts[1:]:
+                # 形如 "v/vt/vn" 或 "v//vn" 或 "v"
+                seg = token.split("/")
+                v_idx = int(seg[0]) - 1  # OBJ 1-based → 0-based
+                vt_idx = int(seg[1]) - 1 if len(seg) > 1 and seg[1] else v_idx
+                if v_idx != vt_idx:
+                    raise ValueError(
+                        f"this loader assumes pos_idx == uv_idx, "
+                        f"got v={v_idx + 1} vt={vt_idx + 1} in line: {line}"
+                    )
+                tri.append(v_idx)
+            triangles.append((tri[0], tri[1], tri[2]))
+
+    if pos_count != len(uv_list):
+        raise ValueError(
+            f"vertex count {pos_count} != uv count {len(uv_list)}; "
+            "obj loader assumes 1:1 correspondence"
+        )
+
+    uvs = np.asarray(uv_list, dtype=np.float32)
+    tris = np.asarray(triangles, dtype=np.int32)
+    return FaceMesh(uvs=uvs, triangles=tris)