feat(接口2): 移植head3d发际线管线 + 接口2实现方案文档
- 从head3d复制发际线检测管线到 hairline/ 包：MediaPipe Tasks + SegFormer分割 + 17锚点射线检测 + 502点mesh(face_ext.obj)+UV - 复制模型：face_landmarker.task(3.7MB)、SegFormer config/preprocessor (model.safetensors 340MB 单独下载中) - 新增 docs/接口2-C端生发-技术实现方案.md：第一步=发际线曲线叠加预览图， 新增gender必填参数，按性别贴图数量输出(female5/male4)，hairline_type英文key， 服务端cv2逐三角形warp渲染器(head3d只有浏览器端Three.js渲染) - 接口文档.md 接口2章节同步：gender参数、输出语义、错误码说明 - hairline_texture/ 9张发际线贴图入库
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
"""接口 2（C 端生发）发际线渲染管线。
|
||||
|
||||
从 head3d 项目移植：MediaPipe 468 点 + SegFormer 人脸分割 + 17 锚点射线检测发际线
|
||||
→ 502 点 3D mesh（face_ext.obj）+ UV → 把发际线类型贴图渲染到照片对应位置。
|
||||
|
||||
模块来源见 docs/接口2-C端生发-技术实现方案.md。
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Auto-extracted indexMap[468] from the SDK's hardcode_data.h.
|
||||
|
||||
Each entry: OBJ-vertex-index i -> MediaPipe canonical landmark index.
|
||||
Do not edit by hand; regenerate from the C++ source.
|
||||
"""
|
||||
|
||||
INDEX_MAP_468 = [
|
||||
127, 34, 139, 11, 0, 37, 232, 231, 120, 72,
|
||||
39, 128, 121, 47, 104, 69, 67, 175, 171, 148,
|
||||
118, 50, 101, 73, 40, 9, 151, 108, 48, 115,
|
||||
131, 194, 204, 211, 74, 185, 80, 42, 183, 92,
|
||||
186, 230, 229, 202, 212, 214, 83, 18, 17, 76,
|
||||
61, 146, 160, 29, 30, 56, 157, 173, 106, 135,
|
||||
192, 203, 165, 98, 21, 71, 68, 51, 45, 4,
|
||||
144, 24, 23, 77, 91, 205, 187, 201, 200, 182,
|
||||
90, 181, 85, 84, 206, 36, 140, 193, 189, 244,
|
||||
159, 158, 28, 247, 246, 161, 236, 3, 196, 54,
|
||||
168, 8, 117, 228, 31, 55, 97, 99, 126, 100,
|
||||
166, 79, 218, 155, 154, 26, 209, 49, 136, 150,
|
||||
217, 223, 52, 53, 134, 170, 43, 119, 226, 130,
|
||||
63, 238, 20, 242, 46, 70, 156, 78, 62, 96,
|
||||
143, 227, 123, 111, 44, 125, 19, 216, 153, 22,
|
||||
167, 208, 142, 57, 60, 35, 113, 27, 210, 225,
|
||||
137, 116, 41, 38, 129, 64, 240, 102, 207, 184,
|
||||
169, 149, 176, 105, 66, 122, 6, 147, 65, 107,
|
||||
89, 180, 93, 15, 86, 14, 87, 145, 88, 179,
|
||||
95, 138, 172, 215, 58, 219, 81, 195, 199, 82,
|
||||
163, 110, 234, 109, 235, 191, 222, 141, 221, 197,
|
||||
25, 7, 33, 220, 237, 245, 162, 188, 174, 2,
|
||||
241, 164, 12, 13, 198, 133, 112, 243, 239, 190,
|
||||
32, 178, 132, 177, 1, 213, 59, 94, 75, 224,
|
||||
233, 114, 124, 356, 389, 368, 302, 267, 452, 350,
|
||||
349, 303, 269, 357, 343, 277, 453, 333, 332, 297,
|
||||
152, 377, 347, 348, 330, 304, 270, 336, 337, 278,
|
||||
279, 360, 418, 262, 431, 408, 409, 310, 415, 407,
|
||||
410, 450, 422, 430, 434, 313, 314, 306, 307, 375,
|
||||
387, 388, 260, 286, 414, 398, 335, 406, 364, 367,
|
||||
416, 423, 358, 327, 251, 284, 298, 281, 5, 373,
|
||||
374, 253, 320, 321, 425, 427, 411, 421, 405, 404,
|
||||
315, 16, 426, 266, 400, 369, 322, 391, 417, 465,
|
||||
464, 386, 257, 258, 466, 456, 399, 419, 285, 346,
|
||||
340, 261, 413, 441, 460, 328, 355, 371, 329, 392,
|
||||
439, 438, 382, 341, 256, 429, 420, 394, 379, 437,
|
||||
443, 444, 283, 275, 440, 363, 338, 273, 451, 446,
|
||||
342, 467, 293, 334, 282, 458, 461, 462, 276, 353,
|
||||
383, 308, 324, 325, 300, 372, 345, 447, 352, 274,
|
||||
248, 436, 381, 252, 393, 428, 287, 250, 384, 265,
|
||||
259, 424, 292, 366, 271, 294, 455, 272, 432, 395,
|
||||
299, 351, 280, 319, 295, 296, 403, 323, 454, 316,
|
||||
380, 318, 402, 365, 435, 397, 344, 311, 291, 396,
|
||||
268, 445, 254, 339, 449, 264, 10, 442, 370, 263,
|
||||
255, 359, 412, 301, 378, 326, 457, 362, 459, 463,
|
||||
354, 401, 361, 309, 376, 433, 289, 305, 448, 290,
|
||||
288, 249, 103, 385, 331, 317, 312, 390,
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Standalone MediaPipe FaceLandmarker runner used by web_service.py.
|
||||
|
||||
Running MediaPipe Tasks in the same process as the Flask dev server can
|
||||
segfault under WSL (D3D12 EGL backend). Spawning a fresh subprocess per
|
||||
request keeps the web server alive and lets us inject WSL-friendly env
|
||||
vars before any mediapipe import.
|
||||
|
||||
Usage:
|
||||
python -m python._mediapipe_subprocess <image_path> <output_npy>
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
os.environ.setdefault("LIBGL_ALWAYS_SOFTWARE", "1")
|
||||
os.environ.setdefault("MESA_LOADER_DRIVER_OVERRIDE", "llvmpipe")
|
||||
os.environ.setdefault("GALLIUM_DRIVER", "llvmpipe")
|
||||
os.environ.setdefault("MEDIAPIPE_DISABLE_GPU", "1")
|
||||
os.environ.setdefault("EGL_PLATFORM", "surfaceless")
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
|
||||
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_DIR = os.path.dirname(THIS_DIR)
|
||||
if PROJECT_DIR not in sys.path:
|
||||
sys.path.insert(0, PROJECT_DIR)
|
||||
|
||||
from python.face_landmarks import FaceLandmarker # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
print(
|
||||
"usage: python -m python._mediapipe_subprocess <image_path> <output_npy>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
image_path, output_path = sys.argv[1], sys.argv[2]
|
||||
|
||||
import cv2
|
||||
|
||||
bgr = cv2.imread(image_path)
|
||||
if bgr is None:
|
||||
print(f"could not read image: {image_path}", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
rgb = np.ascontiguousarray(rgb, dtype=np.uint8)
|
||||
|
||||
landmarker = FaceLandmarker(static_image_mode=True)
|
||||
try:
|
||||
landmarks = landmarker.detect(rgb)
|
||||
finally:
|
||||
landmarker.close()
|
||||
|
||||
if landmarks is None:
|
||||
print("no face detected", file=sys.stderr)
|
||||
return 4
|
||||
|
||||
np.save(output_path, landmarks.astype(np.float32))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Shared constants for the hairline-extension pipeline.
|
||||
|
||||
These define the topology of the extended mesh and must stay in sync with
|
||||
the C++ side (see sdk/ExtensionConstants.h). If you change anything here,
|
||||
regenerate face_ext.obj and update the C++ header.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# Number of MediaPipe FaceMesh landmarks (no iris refinement).
|
||||
N_MP = 468
|
||||
|
||||
# Anchors along the upper boundary of the MediaPipe face mesh,
|
||||
# ordered left-to-right when viewing the face frontally.
|
||||
# Each anchor will get a paired hairline sample directly "above" it.
|
||||
#
|
||||
# Verify visually with scripts/show_anchors.py before locking these in.
|
||||
MP_TOP_ANCHORS: list[int] = [
|
||||
127, 234, 162, 21, 54, 103, 67, 109, 10,
|
||||
338, 297, 332, 284, 251, 389, 356, 454,
|
||||
]
|
||||
N_ANCHORS = len(MP_TOP_ANCHORS) # 17
|
||||
|
||||
# Vertex ID layout in the extended array of length 468 + 2*N_ANCHORS.
|
||||
# [0 .. 468) : MediaPipe canonical landmarks
|
||||
# [468 .. 468+N) : middle row (between MP boundary and hairline)
|
||||
# [468+N .. 468+2N) : hairline row
|
||||
N_EXT = 2 * N_ANCHORS # 34
|
||||
N_TOTAL = N_MP + N_EXT # 502
|
||||
|
||||
MIDDLE_START = N_MP # 468
|
||||
HAIRLINE_START = N_MP + N_ANCHORS # 485
|
||||
|
||||
# Saggital head-curvature radius (in MediaPipe normalized-Y units),
|
||||
# expressed as a fraction of face height. The head's mid-line cross
|
||||
# section is treated locally as a circular arc; for a hairline / middle
|
||||
# vertex located dy=(y_hair - y_anchor) above an MP top anchor (dy < 0
|
||||
# since hairline y < anchor y) we compute its Z by
|
||||
#
|
||||
# z_hair = z_anchor + dy² / (2 R), R = HEAD_ARC_RADIUS_FRAC × face_h
|
||||
#
|
||||
# This always pushes the added vertex BACKWARD (toward +z in MP / face.obj
|
||||
# convention, i.e. toward the back of the head), matching the actual
|
||||
# anatomy. See README for the derivation.
|
||||
#
|
||||
# Smaller fraction = more pronounced backward bulge. 0.30 produces a
|
||||
# moderate offset (~0.024 in normalized z for a 0.08-y hairline lift on
|
||||
# a typical face).
|
||||
HEAD_ARC_RADIUS_FRAC = 0.30
|
||||
|
||||
|
||||
# Extra lift applied to the detected hairline along the face-up direction,
|
||||
# expressed as a fraction of the MP face height. The 2D hairline detector
|
||||
# stops at the hair-skin boundary (start of the visible hair); the mesh
|
||||
# ribbon's top row should sit at the crown of the head instead, so the
|
||||
# texture-overlay band can cover the whole forehead → crown region.
|
||||
#
|
||||
# 0.06 ≈ moves the hairline row up by 6% of face height, which on the
|
||||
# reference photos lands the top row just above the visible hairline and
|
||||
# below the crown — empirically tuned with the /preview slider and locked
|
||||
# in as the project default. Bump to 0.10..0.15 for taller foreheads /
|
||||
# higher crowns; drop to 0.03 to keep the ribbon hugging the hair-skin
|
||||
# boundary.
|
||||
HAIRLINE_CROWN_LIFT_FRAC = 0.06
|
||||
|
||||
|
||||
# Pure-geometry hairline offset for the /preview 502-point pipeline.
|
||||
#
|
||||
# When placing the hairline row WITHOUT hair detection (works for bald /
|
||||
# with-hair / hat — all head types), each anchor is offset upward along
|
||||
# face-up by GEOMETRIC_HAIRLINE_OFFSET_FRAC × face_h in normalised
|
||||
# image space. The sagittal-arc model then derives Z.
|
||||
#
|
||||
# 0.25 × face_h ≈ 0.12 normalised on a typical face (face_h ≈ 0.48),
|
||||
# giving dz ≈ 0.05 — matches the real hairline distance observed on
|
||||
# reference photos where hair detection succeeds.
|
||||
GEOMETRIC_HAIRLINE_OFFSET_FRAC = 0.25
|
||||
|
||||
|
||||
# UV layout for the 34 forehead-extension vertices.
|
||||
#
|
||||
# The texture (imgs/texture0.png, 512×512) is laid out with the original
|
||||
# MediaPipe face skin in V_raw ≈ 0.00..0.77 (image y ≈ 117..511) and a
|
||||
# horizontal stack of 5 hairline-design arcs at the TOP of the image
|
||||
# (image y ≈ 30..170, i.e. V_raw ≈ 0.67..0.94). Those 5 arcs are the
|
||||
# content that the extension strip is supposed to display: hairline row
|
||||
# samples the topmost arc (blue), middle row samples the bottom arc
|
||||
# (orange), and the 3 arcs in between fall out automatically because the
|
||||
# ribbon triangle interpolates V linearly between the two rows.
|
||||
#
|
||||
# V conventions
|
||||
# -------------
|
||||
# uv_template.py / Three.js (with texture.flipY=true) treat V_raw=1 as
|
||||
# the TOP of the image (image y=0) and V_raw=0 as the BOTTOM.
|
||||
#
|
||||
# U conventions — IMPORTANT
|
||||
# -------------------------
|
||||
# The 17 MP_TOP_ANCHORS in face.obj have NON-uniform u (≈ 0.00 at the
|
||||
# temples, ≈ 0.50 at the forehead center, ≈ 1.00 at the other temple).
|
||||
# Each ribbon triangle (anchor[i] → middle[i] → anchor[i+1] etc.) is a
|
||||
# vertical column in UV space ONLY when the middle/hairline vertex
|
||||
# inherits its U from the corresponding anchor. If we used a uniform
|
||||
# 0.05..0.95 U for the strip the columns would slant relative to the
|
||||
# anchor U values, warping the texture's 5 horizontal arcs into
|
||||
# zig-zags. So `extension_uv_for` takes `anchor_u` and copies it.
|
||||
UV_MIDDLE_DV = 0.110 # middle 行 V_raw 相对该列 anchor V_raw 上移这么多
|
||||
UV_HAIRLINE_DV = 0.220 # hairline 行 V_raw 相对该列 anchor V_raw 上移这么多
|
||||
# 为什么是"相对 anchor 平行偏移"而不是固定常数:
|
||||
#
|
||||
# face.obj 中 17 个 MP_TOP_ANCHORS 的 V_raw 是**非均匀弧形** (额头中央 MP 10
|
||||
# = 0.7724, 太阳穴 MP 127 = 0.4668, 横跨 0.30 V 单位)。 如果 middle/hairline
|
||||
# 用固定常数 V_raw (例如 0.82 / 0.998), 那每个 ribbon quad 的 V 跨度
|
||||
# (= middle.V − anchor[i].V) 在 17 列之间差异巨大 (中央列 0.05, 两端列 0.35,
|
||||
# 差了 7 倍)。 贴图最底部的弧线 (V_raw ≈ 0.76) 正好落在 V 跨度大的列上 →
|
||||
# 被拉伸成粗大色块, 而顶部弧线 (V_raw ≈ 0.93) 落在 V 跨度小的列上 → 被压
|
||||
# 缩成细线。 这就是"最下面那条线特别粗、上面 4 根都细"的根因。
|
||||
#
|
||||
# 把 middle/hairline 的 V 也设成"anchor V + 固定 Δ", 每列的 V 跨度变成恒定
|
||||
# 的 Δm / (Δh − Δm), ribbon 在贴图上是上下都跟随 anchor 弧度的弯月形带,
|
||||
# 贴图 5 条弧线在 mesh 上粗细均匀。
|
||||
#
|
||||
# 硬约束:
|
||||
# 1. Δm > 0 且 Δm < Δh (顺序保持 anchor < middle < hairline, 防 V 反向)
|
||||
# 2. anchor.V_max + Δh ≤ 1.0 (即 Δh ≤ 1 − 0.7724 = 0.2276)
|
||||
# 否则中央列 hairline V 溢出, 采到贴图边缘的抗锯齿像素。
|
||||
# 当前 Δh = 0.220 留 ≈ 0.008 V 单位 buffer; Δm = Δh / 2 让上下两段等宽。
|
||||
|
||||
|
||||
def extension_uv_for(row: int, anchor_u: float, anchor_v: float) -> tuple[float, float]:
|
||||
"""Return (u, v_raw) UV for an extension vertex.
|
||||
|
||||
row: 0 = middle, 1 = hairline.
|
||||
anchor_u: U of the corresponding MP anchor (copy verbatim → ribbon column
|
||||
is vertical in UV space).
|
||||
anchor_v: V_raw of the corresponding MP anchor (we add a constant Δ to
|
||||
it → ribbon row stays parallel to anchor row in UV space, so
|
||||
each quad has the same V span and texture arcs render at the
|
||||
same thickness across all 17 columns).
|
||||
"""
|
||||
dv = UV_MIDDLE_DV if row == 0 else UV_HAIRLINE_DV
|
||||
return (anchor_u, anchor_v + dv)
|
||||
|
||||
|
||||
# Face-parsing class indices for the jonathandinu/face-parsing
|
||||
# SegFormer model (matches CelebAMask-HQ labels):
|
||||
PARSE_BG = 0
|
||||
PARSE_SKIN = 1
|
||||
PARSE_NOSE = 2
|
||||
PARSE_EYE_G = 3
|
||||
PARSE_L_EYE = 4
|
||||
PARSE_R_EYE = 5
|
||||
PARSE_L_BROW = 6
|
||||
PARSE_R_BROW = 7
|
||||
PARSE_L_EAR = 8
|
||||
PARSE_R_EAR = 9
|
||||
PARSE_MOUTH = 10
|
||||
PARSE_U_LIP = 11
|
||||
PARSE_L_LIP = 12
|
||||
PARSE_HAIR = 13
|
||||
PARSE_HAT = 14
|
||||
PARSE_EAR_R = 15
|
||||
PARSE_NECK_L = 16
|
||||
PARSE_NECK = 17
|
||||
PARSE_CLOTH = 18
|
||||
|
||||
HF_FACE_PARSER_MODEL = "jonathandinu/face-parsing"
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Main CLI: image -> JSON of 502 3D points consumable by the SDK.
|
||||
|
||||
Pipeline:
|
||||
1. MediaPipe FaceMesh -> 468 normalized landmarks
|
||||
2. Face parsing (HF SegFormer) -> per-pixel class map
|
||||
3. Hairline curve detection + per-anchor ray casting -> 17 hairline 2D points
|
||||
4. Lift 2D hairline points to 3D using anchor Z + curvature offset
|
||||
5. Interpolate 17 middle-row 3D points
|
||||
6. Concatenate into (502, 3) and emit JSON
|
||||
|
||||
Output JSON schema:
|
||||
{
|
||||
"image": {"width": W, "height": H, "path": "..."},
|
||||
"n_total": 502,
|
||||
"n_mp": 468,
|
||||
"n_extension": 34,
|
||||
"layout": ["mp[0..468)", "middle[468..485)", "hairline[485..502)"],
|
||||
"points": [[x_norm, y_norm, z_relative], ...] # length 502
|
||||
"valid_hairline": [true/false, ...] # length 17
|
||||
}
|
||||
|
||||
Usage:
|
||||
py -3 python/extract_hairline.py path/to/image.jpg
|
||||
py -3 python/extract_hairline.py path/to/image.jpg --out data/out.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
if __package__ is None or __package__ == "":
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from python import constants as C
|
||||
from python.face_landmarks import FaceLandmarker
|
||||
from python.face_parsing import FaceParser
|
||||
from python.hairline_2d import sample_hairline, smooth_hairline
|
||||
from python.lift_3d import lift_hairline_to_3d, build_middle_row, assemble_full
|
||||
else:
|
||||
from . import constants as C
|
||||
from .face_landmarks import FaceLandmarker
|
||||
from .face_parsing import FaceParser
|
||||
from .hairline_2d import sample_hairline, smooth_hairline
|
||||
from .lift_3d import lift_hairline_to_3d, build_middle_row, assemble_full
|
||||
|
||||
|
||||
def run(image_path: str, out_path: str | None = None, device: str | None = None) -> dict:
|
||||
import cv2
|
||||
bgr = cv2.imread(image_path)
|
||||
if bgr is None:
|
||||
raise FileNotFoundError(f"could not read image: {image_path}")
|
||||
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
H, W = rgb.shape[:2]
|
||||
|
||||
t0 = time.perf_counter()
|
||||
lmk = FaceLandmarker(static_image_mode=True)
|
||||
landmarks = lmk.detect(rgb)
|
||||
lmk.close()
|
||||
if landmarks is None:
|
||||
raise RuntimeError("no face detected")
|
||||
t1 = time.perf_counter()
|
||||
print(f" [time] mediapipe: {(t1 - t0)*1000:.0f} ms")
|
||||
|
||||
parser = FaceParser(device=device)
|
||||
parse_map = parser.parse(rgb)
|
||||
t2 = time.perf_counter()
|
||||
print(f" [time] parsing: {(t2 - t1)*1000:.0f} ms")
|
||||
|
||||
hairline_2d, valid = sample_hairline(landmarks, parse_map)
|
||||
hairline_2d = smooth_hairline(hairline_2d, valid)
|
||||
hairline_3d = lift_hairline_to_3d(landmarks, hairline_2d)
|
||||
middle_3d = build_middle_row(landmarks, hairline_3d)
|
||||
points_full = assemble_full(landmarks, middle_3d, hairline_3d)
|
||||
t3 = time.perf_counter()
|
||||
print(f" [time] hairline: {(t3 - t2)*1000:.0f} ms")
|
||||
|
||||
record = {
|
||||
"image": {"width": int(W), "height": int(H), "path": image_path},
|
||||
"n_total": C.N_TOTAL,
|
||||
"n_mp": C.N_MP,
|
||||
"n_extension": C.N_EXT,
|
||||
"layout": [
|
||||
f"mp[0..{C.N_MP})",
|
||||
f"middle[{C.MIDDLE_START}..{C.HAIRLINE_START})",
|
||||
f"hairline[{C.HAIRLINE_START}..{C.N_TOTAL})",
|
||||
],
|
||||
"points": points_full.tolist(),
|
||||
"valid_hairline": valid.tolist(),
|
||||
}
|
||||
|
||||
if out_path is None:
|
||||
base = os.path.splitext(os.path.basename(image_path))[0]
|
||||
out_path = os.path.join("data", base + ".json")
|
||||
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(record, f, ensure_ascii=False, indent=2)
|
||||
print(f" wrote {out_path}")
|
||||
return record
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Extract MediaPipe + hairline points from one image.")
|
||||
ap.add_argument("image", help="path to input image (jpg/png)")
|
||||
ap.add_argument("--out", default=None, help="output JSON path")
|
||||
ap.add_argument("--device", default=None, help="torch device (cpu/cuda); auto if omitted")
|
||||
args = ap.parse_args()
|
||||
run(args.image, args.out, args.device)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""MediaPipe FaceMesh wrappers returning 468 3D landmarks in [0,1] x/y space.
|
||||
|
||||
`FaceLandmarker` uses the modern mediapipe.tasks.vision API and requires
|
||||
models/face_landmarker.task.
|
||||
|
||||
`SolutionsFaceLandmarker` uses the older mediapipe.solutions.face_mesh API.
|
||||
It is useful as a CPU fallback in WSL environments where the Tasks API may
|
||||
segfault while initializing EGL/OpenGL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
DEFAULT_MODEL_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||||
"models", "face_landmarker.task",
|
||||
)
|
||||
|
||||
|
||||
class FaceLandmarker:
|
||||
def __init__(self, static_image_mode: bool = True, model_path: str | None = None):
|
||||
import mediapipe as mp
|
||||
from mediapipe.tasks import python as mp_python
|
||||
from mediapipe.tasks.python import vision
|
||||
|
||||
path = model_path or DEFAULT_MODEL_PATH
|
||||
if not os.path.isfile(path):
|
||||
raise FileNotFoundError(
|
||||
f"face_landmarker.task not found at {path}. "
|
||||
"Download it from "
|
||||
"https://storage.googleapis.com/mediapipe-models/face_landmarker/"
|
||||
"face_landmarker/float16/1/face_landmarker.task"
|
||||
)
|
||||
|
||||
running_mode = vision.RunningMode.IMAGE if static_image_mode else vision.RunningMode.VIDEO
|
||||
options = vision.FaceLandmarkerOptions(
|
||||
base_options=mp_python.BaseOptions(model_asset_path=path),
|
||||
running_mode=running_mode,
|
||||
num_faces=1,
|
||||
output_face_blendshapes=False,
|
||||
output_facial_transformation_matrixes=False,
|
||||
)
|
||||
self._detector = vision.FaceLandmarker.create_from_options(options)
|
||||
self._mp = mp
|
||||
|
||||
def detect(self, image_rgb: np.ndarray) -> np.ndarray | None:
|
||||
"""Returns (468, 3) float32 of normalized x, y and relative z, or None.
|
||||
|
||||
The task model produces 478 landmarks (468 face + 10 iris); we return
|
||||
only the first 468 to match the canonical FaceMesh topology used by
|
||||
the SDK's OBJ file.
|
||||
"""
|
||||
mp_image = self._mp.Image(image_format=self._mp.ImageFormat.SRGB, data=image_rgb)
|
||||
result = self._detector.detect(mp_image)
|
||||
if not result.face_landmarks:
|
||||
return None
|
||||
lm = result.face_landmarks[0][:468]
|
||||
arr = np.array([[p.x, p.y, p.z] for p in lm], dtype=np.float32)
|
||||
return arr
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
self._detector.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class SolutionsFaceLandmarker:
|
||||
"""CPU-oriented fallback using mediapipe.solutions.face_mesh."""
|
||||
|
||||
def __init__(self, static_image_mode: bool = True):
|
||||
import mediapipe as mp
|
||||
|
||||
try:
|
||||
face_mesh_module = mp.solutions.face_mesh
|
||||
except AttributeError as exc:
|
||||
raise RuntimeError(
|
||||
"当前 mediapipe 包不包含 mediapipe.solutions.face_mesh。"
|
||||
"请使用 web_service.py 的默认 parsing backend，或安装包含 solutions API 的 mediapipe 版本。"
|
||||
) from exc
|
||||
|
||||
self._face_mesh = face_mesh_module.FaceMesh(
|
||||
static_image_mode=static_image_mode,
|
||||
max_num_faces=1,
|
||||
refine_landmarks=False,
|
||||
min_detection_confidence=0.5,
|
||||
)
|
||||
|
||||
def detect(self, image_rgb: np.ndarray) -> np.ndarray | None:
|
||||
"""Returns (468, 3) float32 of normalized x, y and relative z, or None."""
|
||||
image_rgb.flags.writeable = False
|
||||
result = self._face_mesh.process(image_rgb)
|
||||
image_rgb.flags.writeable = True
|
||||
if not result.multi_face_landmarks:
|
||||
return None
|
||||
lm = result.multi_face_landmarks[0].landmark[:468]
|
||||
return np.array([[p.x, p.y, p.z] for p in lm], dtype=np.float32)
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
self._face_mesh.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Face parsing via HuggingFace SegFormer (jonathandinu/face-parsing).
|
||||
|
||||
First call downloads ~150 MB of weights into the HF cache.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import numpy as np
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from . import constants as C
|
||||
|
||||
if TYPE_CHECKING: # avoid hard import at module load
|
||||
pass
|
||||
|
||||
|
||||
class FaceParser:
|
||||
"""Wraps a SegFormer face-parsing model and returns a (H, W) int label map."""
|
||||
|
||||
def __init__(self, device: str | None = None):
|
||||
import torch
|
||||
from transformers import SegformerImageProcessor, SegformerForSemanticSegmentation
|
||||
|
||||
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
self.processor = SegformerImageProcessor.from_pretrained(C.HF_FACE_PARSER_MODEL)
|
||||
self.model = SegformerForSemanticSegmentation.from_pretrained(C.HF_FACE_PARSER_MODEL)
|
||||
self.model.to(self.device).eval()
|
||||
self._torch = torch
|
||||
|
||||
def parse(self, image_rgb: np.ndarray) -> np.ndarray:
|
||||
"""image_rgb: (H, W, 3) uint8. Returns (H, W) int64 of class indices."""
|
||||
from PIL import Image
|
||||
|
||||
H, W = image_rgb.shape[:2]
|
||||
pil = Image.fromarray(image_rgb)
|
||||
inputs = self.processor(images=pil, return_tensors="pt").to(self.device)
|
||||
with self._torch.no_grad():
|
||||
logits = self.model(**inputs).logits # (1, C, h, w)
|
||||
# Upsample to original resolution
|
||||
up = self._torch.nn.functional.interpolate(
|
||||
logits, size=(H, W), mode="bilinear", align_corners=False
|
||||
)
|
||||
labels = up.argmax(dim=1).squeeze(0).to("cpu").numpy().astype(np.int32)
|
||||
return labels
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,133 @@
|
||||
"""Lift 2D hairline samples to 3D in MediaPipe's normalized space.
|
||||
|
||||
Saggital-arc Z model
|
||||
--------------------
|
||||
MediaPipe / face.obj use ``-z = front of face`` (nose tip is the most
|
||||
negative z), ``+z = back of head``. The frontal head surface curves
|
||||
backward as you move up from the forehead to the crown, so any vertex
|
||||
**above** an MP top anchor (smaller y in image coords) must have a
|
||||
**larger** z than that anchor.
|
||||
|
||||
For each new hairline / middle vertex (x_h, y_h) attached to MP anchor a
|
||||
located at (x_a, y_a, z_a), we model the local sagittal cross-section
|
||||
of the head as a circular arc of radius ``R = HEAD_ARC_RADIUS_FRAC × face_h``
|
||||
and derive
|
||||
|
||||
z_new = z_a + (y_h - y_a)² / (2 R)
|
||||
|
||||
The squared dy term guarantees ``z_new ≥ z_a`` (the surface always
|
||||
bulges backward as you walk up the head, never forward). x is taken
|
||||
directly from the 2D hairline detection (we don't project x onto the
|
||||
arc — the parsing tells us exactly where the hairline sits in x).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import numpy as np
|
||||
|
||||
from . import constants as C
|
||||
from .hairline_2d import face_up_vector
|
||||
|
||||
|
||||
def _face_height(landmarks_norm: np.ndarray) -> float:
|
||||
"""Range of MP y for the visible face (used for the arc radius)."""
|
||||
y = landmarks_norm[:, 1]
|
||||
return float(y.max() - y.min())
|
||||
|
||||
|
||||
def _arc_dz(dy: float, R: float) -> float:
|
||||
"""Backward (positive) z offset along a circular arc of radius R for
|
||||
a vertical displacement dy. Always non-negative."""
|
||||
return (dy * dy) / (2.0 * max(R, 1e-6))
|
||||
|
||||
|
||||
def lift_hairline_to_3d(
|
||||
landmarks_norm: np.ndarray,
|
||||
hairline_norm_xy: np.ndarray,
|
||||
crown_lift_frac: float | None = None,
|
||||
) -> np.ndarray:
|
||||
"""Combine 2D hairline samples with a sagittal-arc Z, after lifting
|
||||
the (x, y) along the face-up direction by ``crown_lift_frac × face_h``
|
||||
so the ribbon's top row sits at the crown of the head rather than at
|
||||
the detected hair-skin boundary.
|
||||
|
||||
landmarks_norm: (468, 3) MediaPipe normalized landmarks.
|
||||
hairline_norm_xy: (N_ANCHORS, 2) 2D hairline samples in [0,1] image space.
|
||||
crown_lift_frac: how far above the detected hairline (in fractions of
|
||||
face height) the mesh row should sit. ``None`` uses
|
||||
``C.HAIRLINE_CROWN_LIFT_FRAC``.
|
||||
Returns: (N_ANCHORS, 3) — (x_norm_lifted, y_norm_lifted, z).
|
||||
"""
|
||||
face_h = _face_height(landmarks_norm)
|
||||
R = C.HEAD_ARC_RADIUS_FRAC * face_h
|
||||
if crown_lift_frac is None:
|
||||
crown_lift_frac = C.HAIRLINE_CROWN_LIFT_FRAC
|
||||
up = face_up_vector(landmarks_norm)
|
||||
lift = up * (crown_lift_frac * face_h) # 2D offset, face-up direction
|
||||
|
||||
out = np.zeros((C.N_ANCHORS, 3), dtype=np.float32)
|
||||
for i, mp_idx in enumerate(C.MP_TOP_ANCHORS):
|
||||
z_a = float(landmarks_norm[mp_idx, 2])
|
||||
y_a = float(landmarks_norm[mp_idx, 1])
|
||||
x_h = float(hairline_norm_xy[i, 0]) + float(lift[0])
|
||||
y_h = float(hairline_norm_xy[i, 1]) + float(lift[1])
|
||||
dy = y_h - y_a # < 0 (上移更多 → dz 更大)
|
||||
out[i, 0] = x_h
|
||||
out[i, 1] = y_h
|
||||
out[i, 2] = z_a + _arc_dz(dy, R)
|
||||
return out
|
||||
|
||||
|
||||
def build_middle_row(
|
||||
landmarks_norm: np.ndarray,
|
||||
hairline_3d: np.ndarray,
|
||||
bias: float = 0.5,
|
||||
) -> np.ndarray:
|
||||
"""Place the middle row at the arc-length midpoint between each MP
|
||||
anchor and its hairline point.
|
||||
|
||||
The old approach used bias=0.5 linear interpolation in XY and then
|
||||
independently recomputed Z via the arc formula. Because dz ∝ dy²,
|
||||
the middle row only received 25 % of the hairline Z offset, creating
|
||||
a visible dent where the extension met the face mesh.
|
||||
|
||||
New approach: for each anchor→hairline pair, parameterise the
|
||||
sagittal circular arc by the angle θ and place the middle vertex at
|
||||
θ_mid = bias × θ_hair. This distributes both the Y displacement
|
||||
**and** the Z displacement smoothly along the arc, so the surface
|
||||
transitions from the face mesh through middle to hairline without
|
||||
any concavity.
|
||||
"""
|
||||
R = C.HEAD_ARC_RADIUS_FRAC * _face_height(landmarks_norm)
|
||||
out = np.zeros((C.N_ANCHORS, 3), dtype=np.float32)
|
||||
for i, mp_idx in enumerate(C.MP_TOP_ANCHORS):
|
||||
a = landmarks_norm[mp_idx]
|
||||
h = hairline_3d[i]
|
||||
# X: simple linear interpolation (no arc model in the coronal plane)
|
||||
out[i, 0] = (1 - bias) * float(a[0]) + bias * float(h[0])
|
||||
# Y, Z: interpolate along the circular arc in the sagittal plane.
|
||||
# The arc angle for the hairline point:
|
||||
# θ_h = |dy_h| / R (small-angle: arc-length ≈ R θ)
|
||||
# Middle sits at θ_m = bias × θ_h, giving:
|
||||
# dy_m = R sin(θ_m) ≈ R θ_m for small θ
|
||||
# dz_m = R (1 − cos(θ_m))
|
||||
# For the parabolic regime (θ < 0.8 rad ≈ 46°) these simplify to
|
||||
# the same formula but we use the full trig for correctness.
|
||||
dy_h = float(h[1]) - float(a[1]) # negative (upward)
|
||||
sign = -1.0 if dy_h < 0 else 1.0
|
||||
theta_h = abs(dy_h) / max(R, 1e-9)
|
||||
theta_m = bias * theta_h
|
||||
dy_m = sign * R * np.sin(theta_m) # same sign as dy_h
|
||||
dz_m = R * (1.0 - np.cos(theta_m)) # always ≥ 0
|
||||
out[i, 1] = float(a[1]) + dy_m
|
||||
out[i, 2] = float(a[2]) + dz_m
|
||||
return out
|
||||
|
||||
|
||||
def assemble_full(landmarks_norm: np.ndarray,
|
||||
middle_3d: np.ndarray,
|
||||
hairline_3d: np.ndarray) -> np.ndarray:
|
||||
"""Concatenate to a (N_TOTAL, 3) array in the SDK's expected order:
|
||||
[0..468) MediaPipe / [468..485) middle / [485..502) hairline."""
|
||||
assert landmarks_norm.shape == (C.N_MP, 3)
|
||||
assert middle_3d.shape == (C.N_ANCHORS, 3)
|
||||
assert hairline_3d.shape == (C.N_ANCHORS, 3)
|
||||
return np.concatenate([landmarks_norm, middle_3d, hairline_3d], axis=0).astype(np.float32)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"_name_or_path": "jonathandinu/face-parsing",
|
||||
"architectures": [
|
||||
"SegformerForSemanticSegmentation"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.0,
|
||||
"classifier_dropout_prob": 0.1,
|
||||
"decoder_hidden_size": 768,
|
||||
"depths": [
|
||||
3,
|
||||
6,
|
||||
40,
|
||||
3
|
||||
],
|
||||
"downsampling_rates": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16
|
||||
],
|
||||
"drop_path_rate": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.0,
|
||||
"hidden_sizes": [
|
||||
64,
|
||||
128,
|
||||
320,
|
||||
512
|
||||
],
|
||||
"id2label": {
|
||||
"0": "background",
|
||||
"1": "skin",
|
||||
"2": "nose",
|
||||
"3": "eye_g",
|
||||
"4": "l_eye",
|
||||
"5": "r_eye",
|
||||
"6": "l_brow",
|
||||
"7": "r_brow",
|
||||
"8": "l_ear",
|
||||
"9": "r_ear",
|
||||
"10": "mouth",
|
||||
"11": "u_lip",
|
||||
"12": "l_lip",
|
||||
"13": "hair",
|
||||
"14": "hat",
|
||||
"15": "ear_r",
|
||||
"16": "neck_l",
|
||||
"17": "neck",
|
||||
"18": "cloth"
|
||||
},
|
||||
"image_size": 224,
|
||||
"initializer_range": 0.02,
|
||||
"label2id": {
|
||||
"background": 0,
|
||||
"skin": 1,
|
||||
"nose": 2,
|
||||
"eye_g": 3,
|
||||
"l_eye": 4,
|
||||
"r_eye": 5,
|
||||
"l_brow": 6,
|
||||
"r_brow": 7,
|
||||
"l_ear": 8,
|
||||
"r_ear": 9,
|
||||
"mouth": 10,
|
||||
"u_lip": 11,
|
||||
"l_lip": 12,
|
||||
"hair": 13,
|
||||
"hat": 14,
|
||||
"ear_r": 15,
|
||||
"neck_l": 16,
|
||||
"neck": 17,
|
||||
"cloth": 18
|
||||
},
|
||||
"layer_norm_eps": 1e-06,
|
||||
"mlp_ratios": [
|
||||
4,
|
||||
4,
|
||||
4,
|
||||
4
|
||||
],
|
||||
"model_type": "segformer",
|
||||
"num_attention_heads": [
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
8
|
||||
],
|
||||
"num_channels": 3,
|
||||
"num_encoder_blocks": 4,
|
||||
"patch_sizes": [
|
||||
7,
|
||||
3,
|
||||
3,
|
||||
3
|
||||
],
|
||||
"reshape_last_stage": true,
|
||||
"semantic_loss_ignore_index": 255,
|
||||
"sr_ratios": [
|
||||
8,
|
||||
4,
|
||||
2,
|
||||
1
|
||||
],
|
||||
"strides": [
|
||||
4,
|
||||
2,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"transformers_version": "4.37.0.dev0"
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"do_normalize": true,
|
||||
"do_reduce_labels": false,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.485,
|
||||
0.456,
|
||||
0.406
|
||||
],
|
||||
"image_processor_type": "SegformerFeatureExtractor",
|
||||
"image_std": [
|
||||
0.229,
|
||||
0.224,
|
||||
0.225
|
||||
],
|
||||
"resample": 2,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size": {
|
||||
"height": 512,
|
||||
"width": 512
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
"""Minimal Wavefront OBJ reader/writer.
|
||||
|
||||
Tailored to the project's face.obj:
|
||||
- Latin-1 / GB-encoded comments allowed (we tolerate undecodable bytes).
|
||||
- Vertices written with 4 decimal places (matches existing file).
|
||||
- Preserves comment/mtllib/group lines if requested.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
@dataclass
|
||||
class ObjMesh:
|
||||
"""1-indexed in OBJ files; stored 0-indexed internally."""
|
||||
positions: list[tuple[float, float, float]] = field(default_factory=list)
|
||||
texcoords: list[tuple[float, float]] = field(default_factory=list)
|
||||
normals: list[tuple[float, float, float]] = field(default_factory=list)
|
||||
# Each face is a list of 3 (pos_idx, uv_idx, normal_idx) tuples, 0-indexed.
|
||||
# -1 means "absent".
|
||||
faces: list[list[tuple[int, int, int]]] = field(default_factory=list)
|
||||
header_lines: list[str] = field(default_factory=list) # comments / mtllib
|
||||
|
||||
def n_v(self) -> int: return len(self.positions)
|
||||
def n_vt(self) -> int: return len(self.texcoords)
|
||||
def n_vn(self) -> int: return len(self.normals)
|
||||
def n_f(self) -> int: return len(self.faces)
|
||||
|
||||
|
||||
def read_obj(path: str) -> ObjMesh:
|
||||
"""Read an OBJ file, ignoring undecodable bytes in comments."""
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
text = raw.decode("latin-1", errors="replace")
|
||||
mesh = ObjMesh()
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#") or line.startswith("mtllib") or line.startswith("o ") or line.startswith("g ") or line.startswith("s "):
|
||||
mesh.header_lines.append(line)
|
||||
continue
|
||||
parts = line.split()
|
||||
kind = parts[0]
|
||||
if kind == "v":
|
||||
mesh.positions.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
elif kind == "vt":
|
||||
mesh.texcoords.append((float(parts[1]), float(parts[2])))
|
||||
elif kind == "vn":
|
||||
mesh.normals.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
elif kind == "f":
|
||||
verts = []
|
||||
for spec in parts[1:]:
|
||||
seg = spec.split("/")
|
||||
pi = int(seg[0]) - 1 if seg[0] else -1
|
||||
ti = int(seg[1]) - 1 if len(seg) > 1 and seg[1] else -1
|
||||
ni = int(seg[2]) - 1 if len(seg) > 2 and seg[2] else -1
|
||||
verts.append((pi, ti, ni))
|
||||
# Triangulate fan if quad/n-gon (shouldn't happen for our mesh)
|
||||
for k in range(1, len(verts) - 1):
|
||||
mesh.faces.append([verts[0], verts[k], verts[k + 1]])
|
||||
# ignore everything else
|
||||
return mesh
|
||||
|
||||
|
||||
def write_obj(path: str, mesh: ObjMesh, header: Sequence[str] | None = None) -> None:
|
||||
"""Write an OBJ file with the project's conventions (1-indexed, 4dp)."""
|
||||
out: list[str] = []
|
||||
if header is not None:
|
||||
out.extend(header)
|
||||
else:
|
||||
out.append("# head3d extended face mesh")
|
||||
out.append(f"# vertices={mesh.n_v()} texcoords={mesh.n_vt()} normals={mesh.n_vn()} faces={mesh.n_f()}")
|
||||
|
||||
out.append("")
|
||||
for x, y, z in mesh.positions:
|
||||
out.append(f"v {x:.4f} {y:.4f} {z:.4f}")
|
||||
out.append("")
|
||||
for u, v in mesh.texcoords:
|
||||
out.append(f"vt {u:.4f} {v:.4f} 0.0000")
|
||||
out.append("")
|
||||
for nx, ny, nz in mesh.normals:
|
||||
out.append(f"vn {nx:.4f} {ny:.4f} {nz:.4f}")
|
||||
out.append("")
|
||||
for face in mesh.faces:
|
||||
toks = []
|
||||
for pi, ti, ni in face:
|
||||
a = str(pi + 1)
|
||||
b = str(ti + 1) if ti >= 0 else ""
|
||||
c = str(ni + 1) if ni >= 0 else ""
|
||||
toks.append(f"{a}/{b}/{c}")
|
||||
out.append("f " + " ".join(toks))
|
||||
|
||||
with open(path, "w", encoding="utf-8", newline="\r\n") as f:
|
||||
f.write("\n".join(out))
|
||||
f.write("\n")
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 272 KiB |
Reference in New Issue
Block a user