Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 xsl
|
4e25641522 | ||
|
xsl
|
bd1b7e66c8 | ||
|
xsl
|
f8b1e567c1 | ||
|
xsl
|
b753fd36a2 | ||
|
xsl
|
c5ec7c7dc5 |
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(awk '/^vt / {print $2, $3}' D:/face_sdk/app/src/main/assets/face_picture_3dmax.obj)",
|
||||
"Bash(awk 'BEGIN{minu=99;maxu=-99;minv=99;maxv=-99} {if\\($1<minu\\)minu=$1; if\\($1>maxu\\)maxu=$1; if\\($2<minv\\)minv=$2; if\\($2>maxv\\)maxv=$2} END{print \"u:\",minu,maxu,\" v:\",minv,maxv}')",
|
||||
"Bash(python -c \"from PIL import Image; print\\(Image.open\\('D:/face_sdk/example/src/main/assets/pic/4ex.png'\\).size\\)\")",
|
||||
"Bash(python -c ' *)",
|
||||
"Bash(python -c \"import sys; print\\(sys.executable\\)\")"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
# Face SDK 使用说明
|
||||
|
||||
基于 Vulkan + MediaPipe FaceLandmarker 的实时人脸贴图渲染 SDK，主要面向**美妆 / AR 试妆**场景：摄像头预览 → 人脸 landmark 检测 → 多帧 PNG 贴图按 motion 序列贴在人脸上。
|
||||
|
||||
支持任意分辨率手机（采用"短边正方形画布 + 长边 letterbox"布局）、前置 / 后置摄像头自动切换、镜子镜像效果。
|
||||
|
||||
---
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
face_sdk/
|
||||
├── app/ ← SDK 主体（可作为 AAR 给业务方接入）
|
||||
│ ├── src/main/java/com/hmwl/face_sdk/
|
||||
│ │ ├── FaceActivity.java ← 业务方继承的核心 Activity
|
||||
│ │ ├── FaceLandmarkerHelper.kt ← MediaPipe 封装
|
||||
│ │ ├── DebugLog.java ← Java/C++ 统一日志（写文件）
|
||||
│ │ ├── InitArg.java / Motion.java / MotionList.java / Frame.java ← JSON 配置 POJO
|
||||
│ │ └── ...
|
||||
│ ├── src/main/cpp/main.cpp ← JNI + native 主循环
|
||||
│ └── src/main/assets/shaders/ ← Vulkan GLSL（bg/texture/...）
|
||||
│
|
||||
├── vulkan/ ← 跨平台 Vulkan 渲染核心
|
||||
│ ├── Application.{h,cpp} ← Vulkan 初始化 / 窗口生命周期
|
||||
│ └── FaceApp.{h,cpp} ← 业务渲染（face mesh + bg quad + motion 调度）
|
||||
│
|
||||
├── example/ ← 示例 app，演示完整接入
|
||||
│ ├── src/main/java/com/inewme/uvmirror/
|
||||
│ │ ├── MainActivity.kt ← 入口（Compose UI，前/后置选择）
|
||||
│ │ ├── MakeupActivity.kt ← 业务 Activity（继承 FaceActivity）
|
||||
│ │ └── MotionManager.kt ← motion 资源加载封装
|
||||
│ ├── src/main/AndroidManifest.xml
|
||||
│ └── src/main/assets/pic/
|
||||
│ ├── motion_data_ex.json ← motion 元数据（帧文件名 + 锚点 x/y）
|
||||
│ └── <motion_id>/000xxx.png ← motion 各帧贴图
|
||||
│
|
||||
├── compile_shader.bat ← 编译所有 .vert/.frag 为 .spv（必须在改 shader 后执行）
|
||||
└── third_party/ ← VMA / glm / glfw / mediapipe …
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 快速上手
|
||||
|
||||
### 1. 编译 + 运行 example
|
||||
|
||||
```powershell
|
||||
# Windows 上，确保 SDK 路径在 local.properties 里，以及 NDK 已安装
|
||||
.\gradlew.bat :example:installDebug
|
||||
```
|
||||
|
||||
启动后：
|
||||
- 主界面有"前置摄像头 / 后置摄像头"两个 RadioButton（默认前置）
|
||||
- 点击 "Go to Makeup" 进入 `MakeupActivity`
|
||||
- `MakeupActivity` 自动加载 `assets/pic/motion_data_ex.json` 配置的 motion，按 `update()` 里的脚本播放
|
||||
|
||||
### 2. 改 shader 后必须重编
|
||||
|
||||
```powershell
|
||||
.\compile_shader.bat
|
||||
.\gradlew.bat :example:installDebug
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 在自己的 App 里集成
|
||||
|
||||
### Step 1：依赖 SDK 模块
|
||||
|
||||
在你的 `settings.gradle` / `build.gradle` 引入 `:app` 模块（或打包成 AAR）。
|
||||
|
||||
### Step 2：在 `AndroidManifest.xml` 声明业务 Activity
|
||||
|
||||
业务 Activity 必须**竖屏**，且**屏蔽常见 configChanges 不要重建**：
|
||||
|
||||
```xml
|
||||
<activity
|
||||
android:name=".MakeupActivity"
|
||||
android:exported="false"
|
||||
android:screenOrientation="portrait"
|
||||
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
|
||||
```
|
||||
|
||||
### Step 3：继承 `FaceActivity` 写业务 Activity
|
||||
|
||||
```kotlin
|
||||
class MakeupActivity : FaceActivity() {
|
||||
lateinit var motionMgr: MotionManager
|
||||
private var isMotionLoadFinished = false
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
// 在 super.onCreate 之前构造 MotionManager（它会调 SetInitArg）
|
||||
motionMgr = MotionManager(
|
||||
this,
|
||||
/* fps */ 10,
|
||||
/* zoom */ 1.5f,
|
||||
/* r,g,b */ 0f, 0f, 1f, // letterbox / 圆外纯色（蓝）
|
||||
/* radius */ 240f, // 480 设计基线下的圆半径，SDK 自动按 canvas_size/480 缩放
|
||||
/* offset */ 200f, -200f // 480 设计基线下的中心偏移
|
||||
)
|
||||
|
||||
super.onCreate(savedInstanceState)
|
||||
|
||||
// 一次性加载所有要用到的 motion，加载完才能 PlayMotionList
|
||||
motionMgr.LoadMotionList(listOf("4", "56")) {
|
||||
isMotionLoadFinished = true
|
||||
}
|
||||
|
||||
// 自己的 update 循环（生命周期感知，避免泄漏）
|
||||
lifecycleScope.launch {
|
||||
while (true) {
|
||||
update()
|
||||
delay(10)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun update() {
|
||||
if (!isMotionLoadFinished) return
|
||||
// 想播什么就调，调 SDK API（详见下面"API 参考"）
|
||||
}
|
||||
|
||||
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
|
||||
if (keyCode == KeyEvent.KEYCODE_BACK) {
|
||||
Stop() // ← 退出 Activity 前必须先 Stop()，否则 native 线程仍在跑
|
||||
finish()
|
||||
return true
|
||||
}
|
||||
return super.onKeyDown(keyCode, event)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Step 4：从入口 Activity 启动业务 Activity，传入摄像头朝向
|
||||
|
||||
```kotlin
|
||||
import androidx.camera.core.CameraSelector
|
||||
import com.hmwl.face_sdk.FaceActivity
|
||||
|
||||
val intent = Intent(context, MakeupActivity::class.java).apply {
|
||||
// 前置：CameraSelector.LENS_FACING_FRONT；后置：LENS_FACING_BACK
|
||||
// 不传时 SDK 默认走后置（向后兼容）
|
||||
putExtra(FaceActivity.EXTRA_LENS_FACING, CameraSelector.LENS_FACING_FRONT)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
```
|
||||
|
||||
前置摄像头 SDK 会自动：
|
||||
- 在 GPU 端把 bg + face mesh 整体水平翻转 → 镜子效果
|
||||
- mediapipe 输入仍是 raw（未镜像）帧，所以 landmark 与 raw 帧 UV 严格对齐
|
||||
|
||||
---
|
||||
|
||||
## API 参考（`FaceActivity` 公开方法）
|
||||
|
||||
| 方法 | 说明 |
|
||||
| --- | --- |
|
||||
| `static final String EXTRA_LENS_FACING` | Intent extra key，值为 `CameraSelector.LENS_FACING_FRONT` 或 `LENS_FACING_BACK` |
|
||||
| `void SetInitArg(String json)` | 设置全局参数（zoom / r,g,b / radius / offset_x,offset_y / action_fps），JSON 格式见 `InitArg` |
|
||||
| `String PreLoadAction(String json, LoadMotionListFinishedCallback cb)` | 异步预加载一组 motion 资源；加载完毕回调 cb |
|
||||
| `void PlayMotionList(List<String> motionList, boolean loop, PlayMotionListFinishedCallback cb)` | 播放 motion 序列，`loop=true` 时无限循环，`cb` 为播完一轮的回调（loop 模式下不会触发，传 `null` 即可） |
|
||||
| `void StopMotion()` | 暂停当前 motion 播放（停在当前帧） |
|
||||
| `void ResumeMotion()` | 恢复 `StopMotion()` 之前的播放进度 |
|
||||
| `void Stop()` | **完全停止 native 渲染**，退出 Activity 前必须调 |
|
||||
|
||||
**注意：** `LoadMotionList` / `PreLoadAction` 是异步的，回调里设置 `isMotionLoadFinished=true` 之后才能 `PlayMotionList`，否则播放命令会被静默丢弃。
|
||||
|
||||
---
|
||||
|
||||
## 资源准备
|
||||
|
||||
### Motion 帧图
|
||||
|
||||
每个 motion 用一个独立目录，PNG 帧按文件名顺序播放：
|
||||
|
||||
```
|
||||
assets/pic/4/000000.png
|
||||
assets/pic/4/000001.png
|
||||
...
|
||||
assets/pic/4/000031.png
|
||||
```
|
||||
|
||||
PNG 必须是 **RGBA**（透明通道），SDK 直接采样后 alpha-blend 到 bg 上方。
|
||||
|
||||
### `motion_data_ex.json`
|
||||
|
||||
motion 元数据，描述每帧贴图的**锚点偏移**（贴在脸上的相对位置，画布坐标系，原点在屏幕中心）：
|
||||
|
||||
```json
|
||||
{
|
||||
"4": {
|
||||
"000000.png": { "x": 0.0, "y": 0.0 },
|
||||
"000001.png": { "x": 0.0, "y": 0.0 },
|
||||
...
|
||||
},
|
||||
"56": {
|
||||
"000000.png": { "x": 0.0, "y": 0.0 },
|
||||
...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 设计要点（接入方可不读，但要改 shader / 渲染时必读）
|
||||
|
||||
### 1. 画布与 letterbox
|
||||
|
||||
- **画布**：屏幕短边的正方形，居中。屏幕长边方向自然 letterbox（上下或左右黑边）。
|
||||
- 画布坐标系（"画布 NDC"）∈ [-1, +1]²，原点在画布中心。
|
||||
- mediapipe FaceLandmarker 输出的归一化 landmark ∈ [0, 1]² 直接对应画布 NDC（经过 `*2-1`）。
|
||||
- 业务参数 `zoom / offset_x / offset_y / radius` 都是**屏幕物理像素**（SDK 端按 `canvas_size / 480` 自动缩放，业务方用 480 设计基线即可）。
|
||||
|
||||
### 2. 摄像头帧 → 画布的几何流水线
|
||||
|
||||
```
|
||||
raw 摄像头帧 (W×H, 4:3 典型)
|
||||
└─① 中央裁切 min(W,H)×min(W,H) 正方形
|
||||
└─② Matrix.postRotate(rotationDegrees) 顺时针旋转 → 正立 face crop
|
||||
└─③ MediaPipe 输出 landmark ∈ [0,1]² ─────▶ 画布 NDC
|
||||
└─ shader 做反向变换贴回 raw 帧 UV (bg)
|
||||
└─ shader 直接当 face mesh 顶点 (face)
|
||||
```
|
||||
|
||||
### 3. 镜子效果（前置摄像头）
|
||||
|
||||
`PushConstants.mirror_x ∈ {0.0, 1.0}`：
|
||||
- 前置时 = 1.0，shader 里把最终 NDC.x 翻转 (`pos.x *= 1.0 - 2.0*mirror_x`)
|
||||
- bg + face mesh 都做同样翻转 → 整体水平镜像，对齐保持
|
||||
- mediapipe 仍接收**未镜像**的 raw 帧，避免双重镜像把 landmark 弄反
|
||||
|
||||
### 4. bg 与 face mesh 对齐不变量
|
||||
|
||||
bg 圆内显示摄像头视频，face mesh 把 motion png 贴到 landmark 位置——两者对齐依赖：
|
||||
|
||||
> **屏幕同一像素位置，bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe landmark 坐标**
|
||||
|
||||
实现方式：bg quad 与 face mesh **完全相同**地经过 `zoom × offset × letterbox × mirror` 几何链路；同时 bg 的 `outTexCoord` 基于"未变换的 quad 顶点 pos"算 raw UV——这样屏幕 fragment 反算的 pos 就等价于"对应 landmark 的画布 NDC"，UV 完美贴合。
|
||||
|
||||
bg quad 顶点取 `±10`（而非 `±1`），保证任何 zoom/offset 下 quad 在屏幕上 GPU clip 后整屏覆盖，letterbox 也被 bg.frag 的 r/g/b 蓝色填充。
|
||||
|
||||
### 5. 摄像头分辨率适配
|
||||
|
||||
- CameraX 用 `setTargetResolution(640, 480)` 强制拿低分辨率帧（SDK 不需要高分辨率）
|
||||
- shader 用 `camera_aspect = W/H` 和 `camera_rotation`（CameraX 给的 rotationDegrees）做 UV 反变换
|
||||
- 摄像头帧分辨率/朝向变化时，SDK 会在 `processCameraFrame` 里自动重建 bg 纹理
|
||||
|
||||
### 6. 窗口生命周期
|
||||
|
||||
- `APP_CMD_TERM_WINDOW` → `Application::cleanupForWindowLost()`：只销毁**窗口资源**（surface / swapchain / framebuffers / imageViews / commandBuffers / sync），保留 renderPass / pipelines / device / VMA / textures
|
||||
- `APP_CMD_INIT_WINDOW` → `Application::reinitForNewWindow()`：复用全局资源，只重建窗口资源
|
||||
- `android_main` 退出时**不再额外销毁** Vulkan 资源（保留给下次 NativeActivity 复用），避免 use-after-free 崩溃
|
||||
|
||||
---
|
||||
|
||||
## 调试
|
||||
|
||||
### 日志文件
|
||||
|
||||
SDK 把 Java/Kotlin 日志（通过 `com.hmwl.face_sdk.DebugLog.i / e`）和 C++ 日志（`DebugLog::log`）**统一写入**：
|
||||
|
||||
```
|
||||
/data/user/0/<your.package>/files/face_sdk_debug.log
|
||||
```
|
||||
|
||||
```powershell
|
||||
# 拉到本地分析
|
||||
adb pull /data/user/0/com.inewme.uvmirror.f20260113/files/face_sdk_debug.log .
|
||||
```
|
||||
|
||||
崩溃时还会自动追加一段含信号、寄存器、backtrace 的 dump（`========== FACE_SDK CRASH ==========`）。
|
||||
|
||||
### 常见问题
|
||||
|
||||
| 现象 | 排查方向 |
|
||||
| --- | --- |
|
||||
| 黑屏 / 没有摄像头预览 | 检查相机权限是否授予；查看日志是否有 `processImageNative` 调用 |
|
||||
| 人脸贴图不出现 | 检查 `motion_data_ex.json` 里的 motion id 是否在 `LoadMotionList` 里加载过；检查 `isMotionLoadFinished` 是否被回调置 true |
|
||||
| 贴图不贴合人脸 | bg 与 face mesh 的 zoom/offset/mirror 必须严格一致——见"设计要点 #4 对齐不变量" |
|
||||
| 切换摄像头后崩溃 | 见 `cleanupSecondInit` 注释；不要在 Android 路径调用它，仅 desktop 入口可用 |
|
||||
| 改了 shader 不生效 | 必须先跑 `compile_shader.bat` 重新生成 `.spv` |
|
||||
|
||||
---
|
||||
|
||||
## 兼容性
|
||||
|
||||
- minSdk 29，targetSdk 看 `app/build.gradle`
|
||||
- 设备需支持 Vulkan 1.0+ 和 MediaPipe TFLite delegate
|
||||
- 已在 MI 9（Android 10、Adreno 640、Vulkan vendor `qglinternal`）上验证通过
|
||||
+1
-2
@@ -13,7 +13,7 @@ android {
|
||||
compileSdk 36
|
||||
defaultConfig {
|
||||
//applicationId "com.hmwl.face_sdk"
|
||||
minSdk 30
|
||||
minSdk 29
|
||||
targetSdk 36
|
||||
versionCode 1
|
||||
versionName "1.0"
|
||||
@@ -117,7 +117,6 @@ dependencies {
|
||||
//implementation fileTree(dir: 'libs/tasks-vision', include: ['*.jar'])
|
||||
//implementation fileTree(dir: 'libs/tasks-core', include: ['*.jar'])
|
||||
|
||||
|
||||
def camerax_version = '1.4.2'
|
||||
implementation "androidx.camera:camera-core:$camerax_version"
|
||||
implementation "androidx.camera:camera-camera2:$camerax_version"
|
||||
|
||||
@@ -16,7 +16,13 @@
|
||||
<activity
|
||||
android:name=".FaceActivity"
|
||||
android:exported="true"
|
||||
tools:ignore="MissingClass"> <!-- 如果IDE报类找不到，可以添加这个 -->
|
||||
android:screenOrientation="portrait"
|
||||
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize"
|
||||
tools:ignore="MissingClass,LockedOrientationActivity">
|
||||
<!-- 锁定竖屏：SDK 渲染按"屏幕中央正方形画布 + 上下黑边 letterbox"
|
||||
的方式适配任意分辨率，目前只验证过竖屏。
|
||||
configChanges 全包：避免软键盘/导航栏变化触发 Activity 重建——SDK
|
||||
自己会通过 onWindowLost/onWindowInit 重建 swapchain，性能更好。 -->
|
||||
|
||||
<!-- 重要：移除 MAIN/LAUNCHER intent-filter -->
|
||||
<!-- 让使用库的应用来决定哪个是主Activity -->
|
||||
|
||||
@@ -1,41 +1,46 @@
|
||||
#version 450
|
||||
|
||||
// 输入变量（与顶点着色器输出对应）
|
||||
layout(location = 0) in vec2 inTexCoord;
|
||||
|
||||
// 输出颜色
|
||||
layout(location = 0) out vec4 outColor;
|
||||
|
||||
// 纹理采样器
|
||||
layout(binding = 0) uniform sampler2D texSampler;
|
||||
|
||||
// ★ PushConstants 必须和 bg.vert / FaceApp.h 完全一致（包括我们没用到的
|
||||
// 字段 zoom/ux/uy/offset_x/offset_y，因为 push constant 是按 offset 取的，
|
||||
// 缺字段会导致后面 radius/screen_w/screen_h 落到错误偏移）。
|
||||
layout(push_constant) uniform PushConstants {
|
||||
float zoom;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
} pushConstants;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float offset_x;
|
||||
float offset_y;
|
||||
float canvas_size;
|
||||
float screen_w;
|
||||
float screen_h;
|
||||
// 相机帧元信息（fragment 不用，仅为对齐 push constant layout）
|
||||
float camera_aspect;
|
||||
float camera_rotation;
|
||||
float mirror_x;
|
||||
} pc;
|
||||
|
||||
void main() {
|
||||
// outFragColor = color;
|
||||
// 获取当前片元的屏幕坐标（左下角为原点）
|
||||
// gl_FragCoord 是屏幕 framebuffer 里的像素坐标（左下原点）。圆形 mask 中心
|
||||
// 锚定在屏幕真实中心 = (screen_w, screen_h)/2。这里不再用画布中心，因为
|
||||
// letterbox 后画布中心和屏幕中心其实是同一个点（min 边居中），但用屏幕坐标
|
||||
// 写法更直观也对 cover 模式更友好。
|
||||
vec2 fragCoord = gl_FragCoord.xy;
|
||||
|
||||
// 屏幕中心（480x480 的中心是 (240, 240)）
|
||||
vec2 center = vec2(240.0, 240.0);
|
||||
|
||||
// 计算到中心的距离（欧氏距离，单位：像素）
|
||||
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
|
||||
float dist = length(fragCoord - center);
|
||||
|
||||
// 判断是否在半径之外
|
||||
if (dist > pushConstants.radius) {
|
||||
// 使用 push constant 中的 RGB 颜色（注意 alpha 设为 1.0）
|
||||
outColor = vec4(pushConstants.r, pushConstants.g, pushConstants.b, 1.0);
|
||||
if (dist > pc.radius) {
|
||||
// 画布外（含 letterbox 黑边）+ 圆外区域统一用业务给的纯色覆盖。
|
||||
// 业务一般给 r=g=0 / b=1 之类，与原版语义保持一致。
|
||||
outColor = vec4(pc.r, pc.g, pc.b, 1.0);
|
||||
} else {
|
||||
// 采样纹理
|
||||
outColor = texture(texSampler, inTexCoord);
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,68 +1,131 @@
|
||||
#version 450
|
||||
|
||||
// 硬编码的顶点数据 - 4个顶点组成三角形带覆盖整个屏幕
|
||||
// const vec2 positions[6] = vec2[6](
|
||||
// vec2( 1.0, 1.0), // 右上 - 三角形1
|
||||
// vec2(-1.0, 1.0), // 左上 - 三角形1
|
||||
// vec2(-1.0, -1.0), // 左下 - 三角形1
|
||||
|
||||
// vec2(-1.0, -1.0), // 左下 - 三角形2
|
||||
// vec2( 1.0, -1.0), // 右下 - 三角形2
|
||||
// vec2( 1.0, 1.0) // 右上 - 三角形2
|
||||
// );
|
||||
|
||||
const float offset = (640-480)/(480.0);
|
||||
|
||||
|
||||
// ★ bg quad 顶点放大到 ±10（而不是 ±1）的原因：
|
||||
//
|
||||
// bg 与 face mesh 的对齐依赖一个关键不变量——
|
||||
// "屏幕上同一像素位置，bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe
|
||||
// landmark 坐标"。要保住这个不变量，bg quad 必须经过和 face mesh 完全相
|
||||
// 同的 zoom × offset × letterbox × mirror 链路 (见下方 gl_Position 计算)，
|
||||
// 而 outTexCoord 必须基于"未变换的 quad 顶点 pos"算 raw UV。这样屏幕 fragment
|
||||
// 反算出的 quad pos 就等价于"对应的 landmark NDC"，UV 完美对齐。
|
||||
//
|
||||
// 但是 zoom (>1) + 非零 offset + screen_h > screen_w 的 letterbox 会让 quad
|
||||
// 在屏幕上呈非对称分布——之前 quad 顶点 ±1 时实测在 zoom=1.5 offset=(450,-450)
|
||||
// 屏幕 1080×2340 下，quad 在 NDC.y 只覆盖 [-0.885, 0.500]，屏幕底部
|
||||
// [0.500, 1.0] 大片 letterbox 没有被 bg 覆盖，露出 RenderPass 的 clearColor
|
||||
// 黑色——表现为"上面有蓝色填充，下面没有"。
|
||||
//
|
||||
// 解法：把 quad 顶点扩大到 ±10。GPU 会把超出 NDC [-1, 1] 的部分硬件 clip 掉，
|
||||
// 但 clip 之后留在屏幕内的 fragment 上，outTexCoord 是顶点的"线性插值"，**屏幕
|
||||
// 内 fragment 反算出的 pos 与 quad 顶点取值范围无关**——所以 face mesh 对齐
|
||||
// 完全不受影响。同时 ±10 在任何合理的 zoom/offset/letterbox 下都能保证 quad
|
||||
// 在屏幕上完全覆盖 [-1, 1]² 含 letterbox。
|
||||
//
|
||||
// Worst-case 验证：zoom=1.5, offset=(450,-450), canvas=1080, screen=1080×2340，
|
||||
// pos.y = -10 → screen_ndc.y = (-10*1.5 + (-450/1080)) * 1080/2340 = -7.12 ✓
|
||||
// pos.y = +10 → screen_ndc.y = (+10*1.5 + (-450/1080)) * 1080/2340 = +6.73 ✓
|
||||
// 远超屏幕范围，clip 后保证整屏 letterbox 都被 bg.frag 的 r/g/b 蓝色覆盖。
|
||||
const vec2 positions[6] = vec2[6](
|
||||
vec2( 1.0, -1.0 -offset), // 右下
|
||||
vec2(1.0, 1.0 + offset), // 右上 - 三角形1
|
||||
vec2(-1.0, 1.0 + offset), // 左上 - 三角形1
|
||||
vec2( 10.0, -10.0),
|
||||
vec2( 10.0, 10.0),
|
||||
vec2(-10.0, 10.0),
|
||||
|
||||
vec2(-1.0, 1.0 + offset), // 左上 - 三角形2
|
||||
vec2( -1.0, -1.0-offset), // 左下 - 三角形2
|
||||
vec2( 1.0, -1.0-offset) // 右下 - 三角形2
|
||||
vec2(-10.0, 10.0),
|
||||
vec2(-10.0, -10.0),
|
||||
vec2( 10.0, -10.0)
|
||||
);
|
||||
|
||||
|
||||
|
||||
const vec2 texCoords[6] = vec2[6](
|
||||
vec2(1.0, 1.0), // 右上
|
||||
vec2(0.0, 1.0), // 左上
|
||||
vec2(0.0, 0.0), // 左下
|
||||
|
||||
vec2(0.0, 0.0), // 左下
|
||||
vec2(1.0, 0.0), // 右下
|
||||
vec2(1.0, 1.0) // 右上
|
||||
);
|
||||
|
||||
// 定义 Push Constant，只有一个 float
|
||||
// ★ PushConstants 必须和 C++ FaceApp.h 严格一致。前 9 个 float 是业务原始值
|
||||
// （SDK 已按 canvas_size/480 缩放到屏幕物理像素），后 5 个 float 是当前帧
|
||||
// 的画布几何 + 相机帧元信息，由 FaceApp::render() 每帧写入。
|
||||
layout(push_constant) uniform PushConstants {
|
||||
float zoom;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float offset_x;
|
||||
float offset_y;
|
||||
} pushConstants;
|
||||
float offset_y;
|
||||
// 画布几何（每帧写入）
|
||||
float canvas_size;
|
||||
float screen_w;
|
||||
float screen_h;
|
||||
// 相机帧元信息（每次相机帧上传时写入）
|
||||
float camera_aspect; // raw width / height（典型 4:3 = 1.333…）
|
||||
float camera_rotation; // CameraX rotationDegrees: 0/90/180/270
|
||||
// 镜子效果开关：1.0 = 前置摄像头，最终 NDC.x 翻转一次。
|
||||
// 这与 FaceLandmarkerHelper 在前置时对 bitmap 做 postScale(-1,1) 的语义对偶：
|
||||
// ① mediapipe 输入是"用户视角"图像 → landmark 也是用户视角；
|
||||
// ② bg.vert 算出的 outTexCoord 还是 raw 帧 UV（旋转矩阵把 user 视角→raw）；
|
||||
// ③ 最后我们把 NDC.x 翻转，bg 在屏幕上呈现为水平镜像 = 镜子效果；
|
||||
// ④ texture.vert 同样翻转一次，face 贴图与 bg 完美对齐。
|
||||
float mirror_x;
|
||||
} pc;
|
||||
|
||||
|
||||
// 输出变量
|
||||
layout(location = 0) out vec2 outTexCoord;
|
||||
|
||||
void main() {
|
||||
// 获取顶点位置
|
||||
vec2 position = positions[gl_VertexIndex];
|
||||
position = position * pushConstants.zoom;
|
||||
position.x = position.x + (pushConstants.offset_x/480);
|
||||
position.y = position.y + (pushConstants.offset_y/480);
|
||||
vec2 pos = positions[gl_VertexIndex];
|
||||
|
||||
// 设置输出位置（Vulkan使用不同的坐标系）
|
||||
gl_Position = vec4(position, 0.0, 1.0);
|
||||
// ===== outTexCoord：基于"未变换的 quad pos" 算 raw UV =====
|
||||
//
|
||||
// 重点：这里必须用未做 zoom/offset/letterbox/mirror 变换的原始 pos 来计算
|
||||
// outTexCoord，而 gl_Position 在下面才做这些变换。这样：
|
||||
// - 屏幕 fragment X 处对应的 quad pos = (X 反过 mirror、letterbox、offset、zoom)
|
||||
// - face mesh 顶点的 mediapipe landmark 在屏幕上的位置 X' 经过相同链路
|
||||
// - 当 X = X' 时（face mesh 顶点重叠某个屏幕像素），fragment 反算的 pos
|
||||
// 刚好等于 face landmark 的画布 NDC，bg 该 fragment 显示 landmark 对应
|
||||
// raw UV → 与 face mesh 完美贴合。
|
||||
//
|
||||
// 公式链：画布 NDC ─[逆时针 rotation]→ 中央方形 NDC ─[scale & shift]→ raw UV
|
||||
//
|
||||
// 关于旋转方向：
|
||||
// Android Matrix.postRotate(deg) 在 Bitmap canvas（y 朝下）里是「视觉顺时针」。
|
||||
// Vulkan NDC 也是 y 朝下。所以画布 NDC → 中央方形 NDC 是「视觉逆时针 rotation」。
|
||||
// GLSL column-major mat2(c, -s, s, c) 对应矩阵 [c, s; -s, c]。
|
||||
// 验证 rotation=0 时矩阵 = identity ✓
|
||||
// 验证 rotation=90 时矩阵 = [0, 1; -1, 0]，apply 到画布右上 (1,-1)：
|
||||
// (0*1 + 1*(-1), -1*1 + 0*(-1)) = (-1, -1) → 中央方形左上 ✓
|
||||
// 物理：mediapipe 输入右上像素 = 中央方形顺时针 90° 后的右上 = 中央方形左上 ✓
|
||||
float ang = radians(pc.camera_rotation);
|
||||
float c = cos(ang);
|
||||
float s = sin(ang);
|
||||
mat2 R = mat2(c, -s, s, c); // 画布 NDC → 中央方形 NDC（视觉逆时针 rotation）
|
||||
vec2 cm = R * pos;
|
||||
|
||||
// 输出纹理坐标
|
||||
outTexCoord = texCoords[gl_VertexIndex];
|
||||
// 中央方形 NDC ∈ [-1, +1]² → raw 帧 UV。中央方形在 raw 帧 UV 上是中央
|
||||
// min(W, H) x min(W, H) 的正方形，对应的 UV 半径：
|
||||
// raw 横向长（W >= H）：halfU = H/(2W) = 1/(2 * aspect)，halfV = 0.5
|
||||
// raw 纵向长（H > W）：halfU = 0.5，halfV = W/(2H) = aspect/2
|
||||
float halfU, halfV;
|
||||
if (pc.camera_aspect >= 1.0) {
|
||||
halfU = 0.5 / pc.camera_aspect;
|
||||
halfV = 0.5;
|
||||
} else {
|
||||
halfU = 0.5;
|
||||
halfV = 0.5 * pc.camera_aspect;
|
||||
}
|
||||
outTexCoord = vec2(0.5 + halfU * cm.x, 0.5 + halfV * cm.y);
|
||||
|
||||
// ===== gl_Position：业务 zoom/offset + letterbox + mirror，与 texture.vert 严格一致 =====
|
||||
//
|
||||
// pos 仍按 quad 顶点 ±10 参与位置计算，pc.offset_x / pc.offset_y 已是屏幕物理
|
||||
// 像素（SDK 端做过 canvas_size/480 缩放），/canvas_size 后是画布 NDC 单位；
|
||||
// 最后 *canvas_size/screen_{w,h} 把画布缩到屏幕中央正方形，长边方向自然
|
||||
// letterbox。quad 顶点放大到 ±10 让 GPU clip 后整屏覆盖。
|
||||
pos *= pc.zoom;
|
||||
pos.x += pc.offset_x / pc.canvas_size;
|
||||
pos.y += pc.offset_y / pc.canvas_size;
|
||||
pos.x *= pc.canvas_size / pc.screen_w;
|
||||
pos.y *= pc.canvas_size / pc.screen_h;
|
||||
|
||||
// 镜子效果：前置时 mirror_x=1.0，把 NDC.x 翻转一次。无分支：
|
||||
// 后置：(1 - 2*0) = +1 → 不变；
|
||||
// 前置：(1 - 2*1) = -1 → 水平翻转。
|
||||
// texture.vert 也做同样翻转，bg + face 整体水平镜像，face 贴图与 bg 完美对齐。
|
||||
pos.x *= (1.0 - 2.0 * pc.mirror_x);
|
||||
|
||||
gl_Position = vec4(pos, 0.0, 1.0);
|
||||
}
|
||||
Binary file not shown.
@@ -5,40 +5,48 @@ layout (binding = 1) uniform sampler2D samplerColor;
|
||||
layout (location = 0) in vec2 inUV;
|
||||
layout (location = 1) in vec3 inNormal;
|
||||
|
||||
// ★ PushConstants 必须和 C++ FaceApp.h / texture.vert 严格一致。
|
||||
// 关键：原版这里只声明到 uy，缺了 offset_x 之后的字段，那时之所以没报错是因为
|
||||
// fragment 没读到那些字段。本次重构 fragment 要读 screen_w/screen_h，所以必须
|
||||
// 把整个块声明完整，否则 layout offset 会错位。
|
||||
layout(push_constant) uniform PushConstants {
|
||||
float zoom;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
} pushConstants;
|
||||
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float offset_x;
|
||||
float offset_y;
|
||||
float canvas_size;
|
||||
float screen_w;
|
||||
float screen_h;
|
||||
// 相机帧元信息（fragment 不用，仅为对齐 push constant layout）
|
||||
float camera_aspect;
|
||||
float camera_rotation;
|
||||
float mirror_x;
|
||||
} pc;
|
||||
|
||||
layout (location = 0) out vec4 outFragColor;
|
||||
|
||||
void main()
|
||||
{
|
||||
// outFragColor = color;
|
||||
// 获取当前片元的屏幕坐标（左下角为原点）
|
||||
// gl_FragCoord 是屏幕 framebuffer 像素坐标。圆形 mask 中心锚定在屏幕真实中心，
|
||||
// 跟随分辨率自动适配；不再写死 (240,240)。
|
||||
vec2 fragCoord = gl_FragCoord.xy;
|
||||
|
||||
// 屏幕中心（480x480 的中心是 (240, 240)）
|
||||
vec2 center = vec2(240.0, 240.0);
|
||||
|
||||
// 计算到中心的距离（欧氏距离，单位：像素）
|
||||
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
|
||||
float dist = length(fragCoord - center);
|
||||
|
||||
// 判断是否在半径之外
|
||||
if (dist > pushConstants.radius) {
|
||||
// 使用 push constant 中的 RGB 颜色（注意 alpha 设为 1.0）
|
||||
outFragColor = vec4(pushConstants.r, pushConstants.g, pushConstants.b, 1.0);
|
||||
if (dist > pc.radius) {
|
||||
// 画布外（含 letterbox 黑边）+ 圆外区域用业务纯色覆盖，与 bg.frag 一致。
|
||||
outFragColor = vec4(pc.r, pc.g, pc.b, 1.0);
|
||||
} else {
|
||||
vec2 pos = vec2(pushConstants.ux/4096.f, pushConstants.uy/2048.f);
|
||||
vec2 uv = pos + vec2(inUV.x/8.f, inUV.y/4.f);
|
||||
// out.png 是 4096x2048 的 8x4 子图图集；ux/uy 选定当前 motion 帧子图，
|
||||
// inUV 在子图内插值。这里和分辨率重构无关，保持原版逻辑不动。
|
||||
vec2 pos = vec2(pc.ux / 4096.0, pc.uy / 2048.0);
|
||||
vec2 uv = pos + vec2(inUV.x / 8.0, inUV.y / 4.0);
|
||||
vec4 color = texture(samplerColor, uv);
|
||||
outFragColor = color;
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@@ -1,6 +1,5 @@
|
||||
#version 450
|
||||
|
||||
|
||||
layout (location = 0) in vec3 inPos;
|
||||
layout (location = 1) in vec2 inUV;
|
||||
layout (location = 2) in vec3 inNormal;
|
||||
@@ -16,17 +15,31 @@ layout (binding = 0) uniform UBO
|
||||
layout (location = 0) out vec2 outUV;
|
||||
layout (location = 1) out vec3 outNormal;
|
||||
|
||||
// ★ PushConstants 必须和 C++ FaceApp.h / bg.vert / bg.frag 严格一致。
|
||||
// 前 9 个 float 是业务原始值（已被 SDK 在 render() 里按 canvas_size/480 缩放成
|
||||
// 屏幕物理像素）；后 3 个 float 是当前帧画布几何，由 SDK 每帧写入。
|
||||
layout(push_constant) uniform PushConstants {
|
||||
float zoom;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float offset_x;
|
||||
float offset_y;
|
||||
} pushConstants;
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
float radius;
|
||||
float ux;
|
||||
float uy;
|
||||
float offset_x;
|
||||
float offset_y;
|
||||
float canvas_size;
|
||||
float screen_w;
|
||||
float screen_h;
|
||||
// 相机帧元信息：camera_aspect / camera_rotation 在 face 渲染中不用——face
|
||||
// landmark 已经是"正立中央方形"的归一化坐标，mediapipe 端转过；保留只是
|
||||
// 为了对齐 push constant layout。
|
||||
float camera_aspect;
|
||||
float camera_rotation;
|
||||
// mirror_x：1.0 表示前置摄像头镜子效果——把最终 gl_Position.x 翻转一次。
|
||||
// bg.vert 也做同样翻转，所以 bg 与 face 同步水平镜像、互相对齐。
|
||||
float mirror_x;
|
||||
} pc;
|
||||
|
||||
out gl_PerVertex
|
||||
{
|
||||
@@ -37,13 +50,33 @@ void main()
|
||||
{
|
||||
outUV = inUV;
|
||||
|
||||
vec2 position = vec2(inPos.xy*2 -1);
|
||||
position = position * pushConstants.zoom;
|
||||
position.x = position.x + (pushConstants.offset_x/480);
|
||||
position.y = position.y + (pushConstants.offset_y/480);
|
||||
gl_Position = vec4(position, 0.5, 1.0);
|
||||
// 模型顶点坐标 inPos.xy 来自 OBJ，已经是 [0,1]² 归一化范围。先映射到
|
||||
// "画布 NDC ∈ [-1,+1]"。
|
||||
vec2 position = vec2(inPos.xy * 2.0 - 1.0);
|
||||
|
||||
vec4 pos = ubo.model * vec4(inPos, 1.0);
|
||||
// 画布坐标系：缩放 + 业务像素偏移。
|
||||
// pc.offset_x / pc.offset_y 是屏幕物理像素（SDK 已做过 canvas_size/480 缩放），
|
||||
// 除以 canvas_size 转成画布 NDC，跟原版 "/480" 在 480 屏上完全等价。
|
||||
position = position * pc.zoom;
|
||||
position.x = position.x + (pc.offset_x / pc.canvas_size);
|
||||
position.y = position.y + (pc.offset_y / pc.canvas_size);
|
||||
|
||||
// 画布 NDC → 屏幕 NDC：屏幕长边方向乘以 canvas_size/screen_long < 1.0，
|
||||
// 让模型只占据屏幕中央正方形画布区域，长边方向的画布外是 letterbox。
|
||||
position.x = position.x * (pc.canvas_size / pc.screen_w);
|
||||
position.y = position.y * (pc.canvas_size / pc.screen_h);
|
||||
|
||||
// 镜子效果：前置摄像头时 mirror_x=1.0，把 NDC.x 翻转一次。
|
||||
// 后置：(1 - 2*0) = +1 → 不变
|
||||
// 前置：(1 - 2*1) = -1 → 水平翻转
|
||||
// 无分支写法，配合 bg.vert 同样的翻转，让 bg + face 整体镜像、保持对齐。
|
||||
position.x *= (1.0 - 2.0 * pc.mirror_x);
|
||||
|
||||
gl_Position = vec4(position, 0.5, 1.0);
|
||||
|
||||
// 注意：ubo.projection / ubo.model 在当前管线里没有真正参与 gl_Position 的
|
||||
// 计算（顶点位置已直接给出 NDC）。这里仍按原版保留 normal 的世界变换，避免
|
||||
// 影响其它依赖 outNormal 的 fragment 逻辑（例如 thick 版本）。
|
||||
vec4 pos = ubo.model * vec4(inPos, 1.0);
|
||||
outNormal = mat3(inverse(transpose(ubo.model))) * inNormal;
|
||||
|
||||
}
|
||||
|
||||
Binary file not shown.
+59
-18
@@ -191,19 +191,37 @@ void android_main(struct android_app *pApp) {
|
||||
}
|
||||
} while (!pApp->destroyRequested);
|
||||
DebugLog::log("android_main: destroyRequested, running cleanup");
|
||||
|
||||
// 关键：这里只复位 FaceApp 的"second-init" 状态机标记，**不销毁**任何
|
||||
// Vulkan 资源。
|
||||
//
|
||||
// - 窗口相关资源（surface/swapchain/framebuffers/imageViews/
|
||||
// commandBuffers/sync）已经在 APP_CMD_TERM_WINDOW → onWindowLost()
|
||||
// 里被 cleanupForWindowLost() 干净销毁；
|
||||
// - renderPass / graphicsPipeline / pipelineLayout / VkDevice /
|
||||
// VkInstance / VMA / 所有 textures / FaceApp 自身 pipelines 是 g_Application
|
||||
// 单例的"全局资源"，跨 NativeActivity 实例复用。下次 Activity 启动
|
||||
// onWindowInit 会走 reinitForNewWindow() 重建窗口资源、复用全局资源。
|
||||
//
|
||||
// 历史教训：之前这里调过 g_Application->cleanupSecondInit()，那个函数会
|
||||
// 把 renderPass / graphicsPipeline / pipelineLayout 全部 destroy 掉但既
|
||||
// 不置 VK_NULL_HANDLE 也不复位 _applicationInited。结果用户从主界面切镜
|
||||
// 头再进 MakeupActivity 时，onWindowInit 看到 _applicationInited=1 走 reinit
|
||||
// 复用路径，引用悬空的 renderPass 调 vkCreateFramebuffer，validation
|
||||
// layer 检测出 use-after-free 直接 SEGV。
|
||||
//application.cleanup();
|
||||
g_Application->cleanupSecondInit();
|
||||
//g_Application->cleanupSecondInit();
|
||||
g_Application->clearnSecondFaceApp();
|
||||
DebugLog::log("android_main: exit");
|
||||
}
|
||||
}
|
||||
|
||||
extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize);
|
||||
extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX);
|
||||
extern void ReceiveFacePoint(float* pos, int count, int width, int height);
|
||||
|
||||
void processWithVulkan(uint8_t* data, int width, int height, int format,
|
||||
int rowStride, size_t dataSize) {
|
||||
TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize);
|
||||
int rowStride, size_t dataSize, int rotation, bool mirrorX) {
|
||||
TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize, rotation, mirrorX);
|
||||
}
|
||||
|
||||
|
||||
@@ -213,14 +231,13 @@ JNIEXPORT void JNICALL
|
||||
Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer,
|
||||
jint width, jint height, jint format,
|
||||
jint row_stride, jint pixel_stride,
|
||||
jint rotation) {
|
||||
jint rotation, jboolean mirror_x) {
|
||||
static std::atomic<uint64_t> s_imgCount{0};
|
||||
uint64_t n = s_imgCount.fetch_add(1) + 1;
|
||||
if (n == 1 || n % 300 == 0) {
|
||||
DebugLog::log("processImageNative #%llu tid=%d w=%d h=%d",
|
||||
(unsigned long long)n, dbg_tid(), width, height);
|
||||
DebugLog::log("processImageNative #%llu tid=%d w=%d h=%d mirror=%d",
|
||||
(unsigned long long)n, dbg_tid(), width, height, (int)mirror_x);
|
||||
}
|
||||
// TODO: implement processImageNative()
|
||||
uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer));
|
||||
jlong capacity = env->GetDirectBufferCapacity(buffer);
|
||||
|
||||
@@ -228,9 +245,12 @@ Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thi
|
||||
return;
|
||||
}
|
||||
|
||||
// 在这里将图像数据传递给 Vulkan
|
||||
// 创建 Vulkan 图像或更新现有图像
|
||||
processWithVulkan(imageData, width, height, format, row_stride, capacity);
|
||||
// rotation = ImageInfo.getRotationDegrees()，给 shader 用来做 UV 旋转，
|
||||
// 让不同 sensor orientation 的设备显示方向一致。
|
||||
// mirror_x = true 表示前置摄像头：所有顶点 shader 会把 gl_Position.x 翻转
|
||||
// 一次，达到"镜子效果"——这是化妆/美妆类 app 的标准体验。
|
||||
processWithVulkan(imageData, width, height, format, row_stride, capacity,
|
||||
(int)rotation, mirror_x == JNI_TRUE);
|
||||
}
|
||||
extern "C"
|
||||
JNIEXPORT void JNICALL
|
||||
@@ -238,19 +258,21 @@ Java_com_hmwl_face_1sdk_FaceActivity_passDataToNative(JNIEnv *env, jobject thiz,
|
||||
jint point_count, jint width, jint height) {
|
||||
static std::atomic<uint64_t> s_ptCount{0};
|
||||
uint64_t n = s_ptCount.fetch_add(1) + 1;
|
||||
if (n == 1 || n % 300 == 0) {
|
||||
DebugLog::log("passDataToNative #%llu tid=%d point_count=%d w=%d h=%d",
|
||||
(unsigned long long)n, dbg_tid(), point_count, width, height);
|
||||
}
|
||||
// TODO: implement passDataToNative()
|
||||
float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer));
|
||||
|
||||
if (pos == nullptr) {
|
||||
// 处理错误
|
||||
DebugLog::log_throttled("passDataToNative.nullBuffer",
|
||||
"passDataToNative #%llu got NULL DirectBufferAddress!",
|
||||
(unsigned long long)n);
|
||||
return;
|
||||
}
|
||||
// 节流：每 120 次实际数据通路打一行（约每 4 秒一次，足够确认通路活着即可）。
|
||||
if (n == 1 || n % 120 == 0) {
|
||||
DebugLog::log("passDataToNative #%llu point_count=%d w=%d h=%d p0=(%.3f,%.3f,%.3f)",
|
||||
(unsigned long long)n, point_count, width, height,
|
||||
pos[0], pos[1], pos[2]);
|
||||
}
|
||||
|
||||
// 或者直接将指针传递给Vulkan
|
||||
ReceiveFacePoint(pos, point_count, width, height);
|
||||
}
|
||||
|
||||
@@ -492,3 +514,22 @@ Java_com_hmwl_face_1sdk_FaceActivity_ResumeMotionNative(JNIEnv *env, jobject thi
|
||||
DebugLog::log("JNI ResumeMotionNative tid=%d", dbg_tid());
|
||||
g_Application->ResumeMotion();
|
||||
}
|
||||
|
||||
// 给 Java/Kotlin 侧用的日志桥：把 Java 端关键事件写进 native 的
|
||||
// face_sdk_debug.log，使 Java + native + Vulkan 渲染日志能在同一个
|
||||
// 文件里按时间戳排好序，一次 adb pull 就能拿到完整时间线。
|
||||
extern "C"
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_hmwl_face_1sdk_DebugLog_nativeLog(JNIEnv *env, jclass clazz,
|
||||
jstring jtag, jstring jmsg) {
|
||||
if (jmsg == nullptr) return;
|
||||
const char* tag = jtag ? env->GetStringUTFChars(jtag, nullptr) : nullptr;
|
||||
const char* msg = env->GetStringUTFChars(jmsg, nullptr);
|
||||
if (tag) {
|
||||
DebugLog::log("[J][%s] %s", tag, msg);
|
||||
} else {
|
||||
DebugLog::log("[J] %s", msg);
|
||||
}
|
||||
if (tag) env->ReleaseStringUTFChars(jtag, tag);
|
||||
env->ReleaseStringUTFChars(jmsg, msg);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.hmwl.face_sdk;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
/**
|
||||
* Java/Kotlin 端日志桥。所有调用最终通过 JNI 写到 native 端的
|
||||
* face_sdk_debug.log 文件（与 native DebugLog::log 共用一份），
|
||||
* 同时镜像到 logcat（tag = 调用方 tag），方便联机调试时一次 adb pull
|
||||
* 就能拿到完整时间线（Java + native + Vulkan 渲染）。
|
||||
*
|
||||
* 使用约定：
|
||||
* - 业务低频事件用 i(tag, msg)
|
||||
* - 错误用 e(tag, msg)
|
||||
* - 高频帧级日志请自行节流，本类不再做二次节流
|
||||
*/
|
||||
public final class DebugLog {
|
||||
private DebugLog() {}
|
||||
|
||||
public static void i(String tag, String msg) {
|
||||
Log.i(tag, msg);
|
||||
try {
|
||||
nativeLog(tag, msg);
|
||||
} catch (UnsatisfiedLinkError e) {
|
||||
// native 还没 attach 的极早期阶段，吞掉避免崩溃
|
||||
}
|
||||
}
|
||||
|
||||
public static void e(String tag, String msg) {
|
||||
Log.e(tag, msg);
|
||||
try {
|
||||
nativeLog("[E]" + tag, msg);
|
||||
} catch (UnsatisfiedLinkError e) {
|
||||
// 同上
|
||||
}
|
||||
}
|
||||
|
||||
private static native void nativeLog(String tag, String msg);
|
||||
}
|
||||
@@ -7,7 +7,8 @@ import android.util.Log;
|
||||
import android.view.View;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.camera.core.AspectRatio;
|
||||
import android.util.Size;
|
||||
|
||||
import androidx.camera.core.CameraSelector;
|
||||
import androidx.camera.core.ImageAnalysis;
|
||||
import androidx.camera.core.ImageProxy;
|
||||
@@ -38,6 +39,27 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
System.loadLibrary("face_sdk");
|
||||
}
|
||||
|
||||
/**
|
||||
* Intent extra key：调用方（example/MainActivity 等）通过它告诉 FaceActivity
|
||||
* 启动哪个朝向的摄像头。值类型 int，取自 androidx.camera.core.CameraSelector：
|
||||
* - CameraSelector.LENS_FACING_BACK (0)
|
||||
* - CameraSelector.LENS_FACING_FRONT (1)
|
||||
* 缺省（未提供 extra）时走 LENS_FACING_BACK，保留旧调用方的行为。
|
||||
*
|
||||
* 该值同时决定两件事：
|
||||
* ① CameraSelector 朝向；
|
||||
* ② mirrorX 镜像标志：前置摄像头时 sensor 帧是"用户右手在画面右侧"，
|
||||
* 为了显示成"镜子效果"（用户右手在屏幕左侧），所有顶点 shader 在
|
||||
* gl_Position 阶段对 NDC.x 做一次水平翻转。这件事通过 push constant
|
||||
* 的 mirror_x 字段透传到 GPU；对应地 FaceLandmarkerHelper 也会对
|
||||
* bitmap 做 postScale(-1, 1) 让 mediapipe 输入与显示画面方向一致，
|
||||
* 确保 face 贴图与 bg 完美对齐。
|
||||
*/
|
||||
public static final String EXTRA_LENS_FACING = "com.hmwl.face_sdk.LENS_FACING";
|
||||
|
||||
/** 当前 session 使用的摄像头朝向。在 onCreate 里从 Intent extra 读取并固定。 */
|
||||
private int lensFacing = CameraSelector.LENS_FACING_BACK;
|
||||
|
||||
private ProcessCameraProvider cameraProvider;
|
||||
private void initCamera(){
|
||||
backgroundExecutor = Executors.newSingleThreadExecutor();
|
||||
@@ -72,11 +94,28 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
cameraProvider = cameraProviderFuture.get();
|
||||
|
||||
CameraSelector cameraSelector = new CameraSelector.Builder()
|
||||
.requireLensFacing(CameraSelector.LENS_FACING_BACK)
|
||||
.requireLensFacing(lensFacing)
|
||||
.build();
|
||||
|
||||
// 显式锁定分析帧分辨率为 480x640（CameraX UI 坐标系，竖屏：短边 x 长边）。
|
||||
// 不同手机的相机硬件原生支持的分辨率差异很大（旗舰机随便给 1920x1440、
|
||||
// 入门机可能是 320x240），CameraX 默认会按"最贴近 setTargetAspectRatio
|
||||
// 的 supported size"挑一个，结果在不同设备上拿到的分析帧大小都不一样。
|
||||
// 这会带来三个麻烦：
|
||||
// ① bg 纹理 GPU 内存占用波动很大（高端机一帧 ~10MB staging buffer）；
|
||||
// ② mediapipe 推理耗时随分辨率非线性升高（Pixel 上 1920x1440 比
|
||||
// 640x480 慢 4-6 倍），帧率掉得明显；
|
||||
// ③ 我们的 SDK 用归一化 landmark + 中央方形 crop，本来就不需要更高
|
||||
// 分辨率，多出来的像素是纯浪费。
|
||||
//
|
||||
// 锁定 480x640 后：sensor 坐标系下分析帧固定 640x480 (rotation=90 时)，
|
||||
// FaceLandmarkerHelper 中央方形裁切固定 480x480，bg 上传纹理也固定，
|
||||
// 不同手机上行为一致。
|
||||
//
|
||||
// 实际尺寸由 CameraX 决定（会找最接近的硬件支持档位），但设备实测
|
||||
// 几乎都能精确给出 640x480。
|
||||
imageAnalyzer = new ImageAnalysis.Builder()
|
||||
.setTargetAspectRatio(AspectRatio.RATIO_4_3)
|
||||
.setTargetResolution(new Size(480, 640))
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
|
||||
.build();
|
||||
@@ -114,24 +153,24 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
}
|
||||
|
||||
private void processImageForVulkan(ImageProxy image) {
|
||||
// 获取图像信息
|
||||
int width = image.getWidth();
|
||||
int height = image.getHeight();
|
||||
int format = image.getFormat();
|
||||
|
||||
// 获取图像数据平面
|
||||
ImageProxy.PlaneProxy[] planes = image.getPlanes();
|
||||
|
||||
// 对于 RGBA_8888 格式，通常只有一个平面
|
||||
if (planes.length > 0) {
|
||||
ImageProxy.PlaneProxy plane = planes[0];
|
||||
ByteBuffer buffer = plane.getBuffer();
|
||||
int rowStride = plane.getRowStride();
|
||||
int pixelStride = plane.getPixelStride();
|
||||
|
||||
// 调用 Native 方法处理图像
|
||||
// mirrorX 为 true 时所有顶点 shader 会把 gl_Position.x 翻转一次，达到
|
||||
// "镜子效果"——前置摄像头唯一需要的额外处理。后置不开镜像。
|
||||
boolean mirrorX = (lensFacing == CameraSelector.LENS_FACING_FRONT);
|
||||
processImageNative(buffer, width, height, format,
|
||||
rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
|
||||
rowStride, pixelStride, image.getImageInfo().getRotationDegrees(),
|
||||
mirrorX);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,11 +193,16 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
// Native 方法
|
||||
private native void processImageNative(ByteBuffer buffer, int width, int height,
|
||||
int format, int rowStride, int pixelStride,
|
||||
int rotation);
|
||||
int rotation, boolean mirrorX);
|
||||
|
||||
private void detectFace(ImageProxy imageProxy) {
|
||||
// 前置摄像头：FaceLandmarkerHelper 内部会对 bitmap 做 postScale(-1, 1)，
|
||||
// 让 mediapipe 看到的是"用户视角"的图像。这样 landmark 输出就是用户视角
|
||||
// 归一化坐标，跟 shader 里 gl_Position.x 翻转一次后的 bg 画面方向一致，
|
||||
// face 贴图位置才能精确对齐。
|
||||
boolean isFrontCamera = (lensFacing == CameraSelector.LENS_FACING_FRONT);
|
||||
faceLandmarkerHelper.detectLiveStream(
|
||||
imageProxy,false
|
||||
imageProxy, isFrontCamera
|
||||
);
|
||||
}
|
||||
|
||||
@@ -181,6 +225,15 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState){
|
||||
super.onCreate(savedInstanceState);
|
||||
// 必须在 initCamera() 之前读，CameraSelector 的朝向就靠这个 lensFacing。
|
||||
// 兼容旧调用方：未提供 EXTRA_LENS_FACING 时默认后置。
|
||||
if (getIntent() != null) {
|
||||
lensFacing = getIntent().getIntExtra(
|
||||
EXTRA_LENS_FACING, CameraSelector.LENS_FACING_BACK);
|
||||
}
|
||||
Log.d(TAG, "onCreate lensFacing=" + lensFacing
|
||||
+ " (front=" + CameraSelector.LENS_FACING_FRONT
|
||||
+ " back=" + CameraSelector.LENS_FACING_BACK + ")");
|
||||
initCamera();
|
||||
|
||||
backgroundExecutor.execute(new Runnable() {
|
||||
@@ -280,11 +333,18 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
|
||||
FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
|
||||
floatBuffer.put(points);
|
||||
floatBuffer.position(0);
|
||||
long cur_time = System.currentTimeMillis();
|
||||
Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
|
||||
// 节流：每 30 次调用打一次到 native 文件，方便和 native 那边的
|
||||
// passDataToNative.tick 对账。
|
||||
onResultsCallCount++;
|
||||
if (onResultsCallCount % 30 == 1) {
|
||||
DebugLog.i("FaceActivity",
|
||||
"onResults call#" + onResultsCallCount
|
||||
+ " index=" + index + " input=" + width + "x" + height
|
||||
+ " p0=(" + points[0] + "," + points[1] + "," + points[2] + ")");
|
||||
}
|
||||
passDataToNative(nativeBuffer, index, width, height);
|
||||
Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
|
||||
}
|
||||
private long onResultsCallCount = 0;
|
||||
|
||||
public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
|
||||
|
||||
|
||||
@@ -173,6 +173,13 @@ class FaceLandmarkerHelper(
|
||||
" while not using RunningMode.LIVE_STREAM"
|
||||
)
|
||||
}
|
||||
// 节流日志：每 ~30 次调用打一次（detectFace 在 FaceActivity 里只在
|
||||
// frameCount % 3 == 0 时调用，所以 30 次大约对应 90 帧 ≈ 3 秒）。用来
|
||||
// 确认 mediapipe 入口活着、isFrontCamera/rotation/分辨率有没有跑偏。
|
||||
if ((frameId % 30L) == 0L) {
|
||||
Log.i(TAG, "detectLiveStream tick frameId=$frameId src=${imageProxy.width}x${imageProxy.height}" +
|
||||
" rot=${imageProxy.imageInfo.rotationDegrees} isFrontCamera=$isFrontCamera")
|
||||
}
|
||||
val frameTime = SystemClock.uptimeMillis()
|
||||
|
||||
val currentFrameId = frameId++
|
||||
@@ -194,22 +201,40 @@ class FaceLandmarkerHelper(
|
||||
imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) }
|
||||
//imageProxy.close()
|
||||
|
||||
// 重要：这里 *不* 对前置摄像头做 postScale(-1, 1)！理由如下：
|
||||
//
|
||||
// 我们的渲染策略是 —— bg 与 face 顶点 shader 末尾统一对 gl_Position.x
|
||||
// 乘 (1 - 2*mirror_x) 做镜子翻转。bg 是直接采样 sensor 帧、然后 shader
|
||||
// 末尾翻一次；face 顶点用的是 mediapipe 输出的归一化坐标，shader 末尾
|
||||
// 也翻一次。两者只有同时基于"sensor 视角"做输入、再一起在 shader 末尾
|
||||
// 翻一次，face 才能精确对齐 bg 上的脸。
|
||||
//
|
||||
// 如果在这里给 mediapipe 喂"已镜像"的 bitmap（旧实现），mediapipe 输出
|
||||
// 的 landmark 就是"用户视角"坐标，再被 shader 翻一次 → face 实际上被
|
||||
// 翻了两次，跟只翻一次的 bg 错位（通常会在屏幕另一半，看起来就是"face
|
||||
// 完全没渲染"）。这正是我们最近调试看到的现象。
|
||||
//
|
||||
// mediapipe 对左右镜像不敏感（训练集本身覆盖各角度），sensor 视角下也
|
||||
// 能稳定检测出 478 个 landmark，无需迎合"镜子视角"。因此 isFrontCamera
|
||||
// 参数保留以备扩展，但当前不再据此修改输入图。
|
||||
val matrix = Matrix().apply {
|
||||
// Rotate the frame received from the camera to be in the same direction as it'll be shown
|
||||
postRotate(imageProxy.imageInfo.rotationDegrees.toFloat())
|
||||
|
||||
// flip image if user use front camera
|
||||
if (isFrontCamera) {
|
||||
postScale(
|
||||
-1f,
|
||||
1f,
|
||||
imageProxy.width.toFloat(),
|
||||
imageProxy.height.toFloat()
|
||||
)
|
||||
}
|
||||
}
|
||||
// 旧代码硬编码了 (640-480)/2 = 80 和 bitmapBuffer.height（=480），假设
|
||||
// 相机帧是 640x480。在不同手机/不同 CameraX 配置下相机分辨率会变（比如
|
||||
// 480x640 portrait sensor、1280x720 等），硬编码会越界裁切或丢一半画面。
|
||||
//
|
||||
// 改成：根据 imageProxy 的实际尺寸取中央 min(W,H) x min(W,H) 正方形作为
|
||||
// mediapipe 输入。这与 bg.vert 的 UV 计算严格对齐 —— shader 那边也是从
|
||||
// raw 帧的中央 min(W,H) 正方形采样，再按 rotation 旋转。两端语义一致才能
|
||||
// 让 FaceLandmark 输出的归一化坐标 [0,1] 直接对应屏幕画布 NDC。
|
||||
val srcW = imageProxy.width
|
||||
val srcH = imageProxy.height
|
||||
val side = minOf(srcW, srcH)
|
||||
val cropX = (srcW - side) / 2
|
||||
val cropY = (srcH - side) / 2
|
||||
val rotatedBitmap = Bitmap.createBitmap(
|
||||
bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height,
|
||||
bitmapBuffer, cropX, cropY, side, side,
|
||||
matrix, true
|
||||
)
|
||||
|
||||
@@ -270,9 +295,14 @@ class FaceLandmarkerHelper(
|
||||
//{
|
||||
val finishTimeMs = SystemClock.uptimeMillis()
|
||||
val inferenceTime = finishTimeMs - result.timestampMs()
|
||||
Log.i("TimeLatency",
|
||||
"总延时: ${inferenceTime}ms | "
|
||||
)
|
||||
hitResultCount++
|
||||
if (hitResultCount % 30L == 1L) {
|
||||
// 节流：检测到人脸的频次。和 emptyResultCount 配合看就能知道
|
||||
// 命中率（hitResultCount / (hitResultCount + emptyResultCount)）。
|
||||
Log.i(TAG, "returnLivestreamResult: HIT count=$hitResultCount" +
|
||||
" landmarks=${result.faceLandmarks().firstOrNull()?.size}" +
|
||||
" input=${input.width}x${input.height} infer=${inferenceTime}ms")
|
||||
}
|
||||
faceLandmarkerHelperListener?.onResults(
|
||||
ResultBundle(
|
||||
result,
|
||||
@@ -284,9 +314,19 @@ class FaceLandmarkerHelper(
|
||||
//}
|
||||
}
|
||||
else {
|
||||
// 节流：mediapipe 拿到 frame 但没识别出人脸时，每 30 次空结果打一行。
|
||||
// 只要这条日志在刷，就说明 mediapipe pipeline 正常活着，断点在"喂进去
|
||||
// 的图本身没有可识别的脸"——常见于 isFrontCamera 路径下输入图被翻坏。
|
||||
emptyResultCount++
|
||||
if (emptyResultCount % 30L == 1L) {
|
||||
Log.w(TAG, "returnLivestreamResult: faceLandmarks empty (count=$emptyResultCount)" +
|
||||
" input=${input.width}x${input.height} ts=$frameId")
|
||||
}
|
||||
faceLandmarkerHelperListener?.onEmpty()
|
||||
}
|
||||
}
|
||||
private var emptyResultCount: Long = 0
|
||||
private var hitResultCount: Long = 0
|
||||
|
||||
// Return errors thrown during detection to this FaceLandmarkerHelper's
|
||||
// caller
|
||||
|
||||
@@ -9,8 +9,8 @@ android {
|
||||
compileSdk 36
|
||||
|
||||
defaultConfig {
|
||||
applicationId "com.inewme.uvmirror.f20260113"
|
||||
minSdk 30
|
||||
applicationId "com.inewme.uvmirror.f20260127"
|
||||
minSdk 29
|
||||
targetSdk 36
|
||||
versionCode 1
|
||||
versionName "1.0"
|
||||
|
||||
@@ -23,7 +23,11 @@
|
||||
|
||||
<activity
|
||||
android:name=".MakeupActivity"
|
||||
android:exported="false" />
|
||||
android:exported="false"
|
||||
android:screenOrientation="portrait"
|
||||
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
|
||||
<!-- MakeupActivity 继承 FaceActivity，face_sdk 渲染目前只支持竖屏。
|
||||
configChanges 与 FaceActivity 保持一致，避免软键盘/导航栏触发重建。 -->
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -5,16 +5,27 @@ import android.os.Bundle
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.selection.selectable
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.RadioButton
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.semantics.Role
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import androidx.camera.core.CameraSelector
|
||||
import com.hmwl.face_sdk.FaceActivity
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
@@ -36,17 +47,30 @@ fun MyApp(modifier: Modifier = Modifier) {
|
||||
|
||||
@Composable
|
||||
fun Greeting(name: String, modifier: Modifier = Modifier) {
|
||||
val context = LocalContext.current // 获取上下文用于启动 Activity
|
||||
val context = LocalContext.current
|
||||
// 默认前置：化妆/美妆类典型场景。
|
||||
// 用 CameraSelector.LENS_FACING_FRONT / LENS_FACING_BACK 直接当 state，省去
|
||||
// 中间枚举类型，最后通过 Intent extra 透传给 FaceActivity。
|
||||
var lensFacing by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) }
|
||||
|
||||
Column(
|
||||
modifier = modifier.padding(16.dp)
|
||||
) {
|
||||
Text(text = "Hello $name!")
|
||||
|
||||
LensFacingRadioGroup(
|
||||
selected = lensFacing,
|
||||
onSelected = { lensFacing = it }
|
||||
)
|
||||
|
||||
Button(
|
||||
onClick = {
|
||||
// 跳转到 MakeupActivity
|
||||
val intent = Intent(context, MakeupActivity::class.java)
|
||||
val intent = Intent(context, MakeupActivity::class.java).apply {
|
||||
// FaceActivity 在 onCreate 里读这个 extra 来决定 CameraSelector 朝向
|
||||
// 与 isFrontCamera 镜像逻辑；缺省时（旧调用方）走 LENS_FACING_BACK，
|
||||
// 向后兼容。
|
||||
putExtra(FaceActivity.EXTRA_LENS_FACING, lensFacing)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
}
|
||||
) {
|
||||
@@ -55,6 +79,39 @@ fun Greeting(name: String, modifier: Modifier = Modifier) {
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun LensFacingRadioGroup(
|
||||
selected: Int,
|
||||
onSelected: (Int) -> Unit,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
val options = listOf(
|
||||
CameraSelector.LENS_FACING_FRONT to "前置摄像头",
|
||||
CameraSelector.LENS_FACING_BACK to "后置摄像头"
|
||||
)
|
||||
Column(modifier = modifier.padding(vertical = 8.dp)) {
|
||||
Text(text = "摄像头朝向")
|
||||
options.forEach { (value, label) ->
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.selectable(
|
||||
selected = (value == selected),
|
||||
onClick = { onSelected(value) },
|
||||
role = Role.RadioButton
|
||||
)
|
||||
.padding(vertical = 4.dp),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
RadioButton(
|
||||
selected = (value == selected),
|
||||
onClick = null
|
||||
)
|
||||
Text(text = label, modifier = Modifier.padding(start = 8.dp))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun GreetingPreview() {
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
|
||||
package com.inewme.uvmirror
|
||||
|
||||
import android.graphics.Color
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import android.view.Gravity
|
||||
import android.view.KeyEvent
|
||||
import android.view.KeyEvent.KEYCODE_BACK
|
||||
import android.view.MotionEvent
|
||||
import android.view.View
|
||||
import android.view.ViewGroup
|
||||
import android.widget.FrameLayout
|
||||
import android.widget.LinearLayout
|
||||
import android.widget.ProgressBar
|
||||
import android.widget.TextView
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
|
||||
import com.hmwl.face_sdk.FaceActivity
|
||||
@@ -18,9 +26,12 @@ class MakeupActivity : FaceActivity()
|
||||
{
|
||||
lateinit var motionMgr: MotionManager
|
||||
private var updateJob: Job? = null
|
||||
private var loadingOverlay: View? = null
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
motionMgr = MotionManager(this, 10, 1.5f, 0f,0f,1f,240f, 200f, -200f)
|
||||
super.onCreate(savedInstanceState)
|
||||
showLoadingOverlay()
|
||||
// 启动一个生命周期感知的协程, 每10ms 调用一次，可以通过update 来处理业务的逻辑
|
||||
updateJob = lifecycleScope.launch {
|
||||
while (true) {
|
||||
@@ -29,90 +40,71 @@ class MakeupActivity : FaceActivity()
|
||||
}
|
||||
}
|
||||
|
||||
// 首先一次性加载当前业务逻辑所要求的所有动画资源， 之后就可以任意跳转，和播放，不用考虑状态
|
||||
motionMgr.LoadMotionList(sequenceAll){
|
||||
// 首先一次性加载当前业务逻辑所要求的所有动画资源， 之后就可以任意跳转，和播放，不用考虑状态
|
||||
motionMgr.LoadMotionList(sequenceString){
|
||||
//当加载完成了所有资源后会回调
|
||||
isMotionLoadFinished = true
|
||||
isMotionLoadFinished = true
|
||||
hideLoadingOverlay()
|
||||
}
|
||||
}
|
||||
|
||||
private fun showLoadingOverlay() {
|
||||
val overlay = FrameLayout(this).apply {
|
||||
setBackgroundColor(0x99000000.toInt())
|
||||
}
|
||||
val container = LinearLayout(this).apply {
|
||||
orientation = LinearLayout.VERTICAL
|
||||
gravity = Gravity.CENTER
|
||||
layoutParams = FrameLayout.LayoutParams(
|
||||
FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
FrameLayout.LayoutParams.MATCH_PARENT
|
||||
)
|
||||
}
|
||||
container.addView(ProgressBar(this).apply {
|
||||
layoutParams = LinearLayout.LayoutParams(
|
||||
LinearLayout.LayoutParams.WRAP_CONTENT,
|
||||
LinearLayout.LayoutParams.WRAP_CONTENT
|
||||
)
|
||||
})
|
||||
container.addView(TextView(this).apply {
|
||||
text = "加载中..."
|
||||
setTextColor(Color.WHITE)
|
||||
textSize = 16f
|
||||
layoutParams = LinearLayout.LayoutParams(
|
||||
LinearLayout.LayoutParams.WRAP_CONTENT,
|
||||
LinearLayout.LayoutParams.WRAP_CONTENT
|
||||
).apply { topMargin = 24 }
|
||||
})
|
||||
overlay.addView(container)
|
||||
addContentView(overlay, ViewGroup.LayoutParams(
|
||||
ViewGroup.LayoutParams.MATCH_PARENT,
|
||||
ViewGroup.LayoutParams.MATCH_PARENT
|
||||
))
|
||||
loadingOverlay = overlay
|
||||
}
|
||||
|
||||
private fun hideLoadingOverlay() {
|
||||
runOnUiThread { loadingOverlay?.visibility = View.GONE }
|
||||
}
|
||||
|
||||
//定义加载完成的变量，通过回调函数设置，加载完成后才可以进行动画播放的操作
|
||||
var isMotionLoadFinished: Boolean = false
|
||||
private var hasStartedPlaying: Boolean = false
|
||||
private var currentIndex: Int = 0
|
||||
|
||||
|
||||
|
||||
//这里是模拟业务逻辑的地方
|
||||
// 模拟问题1， 打断当前的动画，然后跳转到指定步骤
|
||||
// 首先播放动画序列 粉底(4、5、6、7)、腮红(55、59)、口红由(76、77、78) 一直循环播放 "4", "5", "6", "7", "55", "59", "76", "77", "78"
|
||||
// 然后 当播放到 10秒的时候，也可以是其他事件，需要暂停动画
|
||||
// 然后 暂停5秒，这个时候需要继续播放动画
|
||||
// 然后 又播放了8秒钟，这个时候要却换到 (11, 13, 21 动画序列了)， 同时设置序列播放完成回调函数
|
||||
// 然后 播放完这个序列又要切换为原来的序列播放，通过序列播放完成回调函数实现
|
||||
private var elapsedTime = 0 // 总经过时间（ms）
|
||||
private val sequenceA = listOf("4")
|
||||
private val sequenceB = listOf("56")
|
||||
private val sequenceAll = listOf("4", "56")
|
||||
private val sequence = listOf(4,5,6,7,11,12,13,14,21,56,59,62,65,68,71,78,81)
|
||||
private val sequenceString = sequence.map { it.toString() }
|
||||
|
||||
private fun update()
|
||||
{
|
||||
if(isMotionLoadFinished)
|
||||
if(isMotionLoadFinished && !hasStartedPlaying)
|
||||
{
|
||||
if(elapsedTime == 0){
|
||||
//这次播放是一直循环播放，我们不需要判断动画是否播放完成, 回调函数直接传入null
|
||||
PlayMotionList(sequenceA, true, null)
|
||||
}
|
||||
|
||||
elapsedTime += 10 // 动画加载完成后开始计时，每次调用增加 10ms
|
||||
|
||||
if(elapsedTime == 10*1000){
|
||||
//暂停动画
|
||||
StopMotion()
|
||||
}
|
||||
|
||||
if(elapsedTime == 10*1000 + 5*1000){
|
||||
//恢复播放动画
|
||||
ResumeMotion()
|
||||
}
|
||||
|
||||
if(elapsedTime == 10*1000 + 5*1000 + 8*1000){
|
||||
PlayMotionList(sequenceB, true){
|
||||
//这里是播放完动画后的回调，再次切换到原来的 sequenceA
|
||||
PlayMotionList(sequenceA, true, null)
|
||||
}
|
||||
}
|
||||
|
||||
hasStartedPlaying = true
|
||||
PlayMotionList(listOf(sequenceString[0]), true, null)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 模拟问题2， 当语音播报结束的时候，打断当前的动画， 跳转到另外一个动画
|
||||
// 首先播放动画序列 粉底(4、5、6、7)、
|
||||
// 然后 当播放到 10秒的时候，表示语音播报结束， 条装到 "55", "59" 动画
|
||||
// private var elapsedTime = 0 // 总经过时间（ms）
|
||||
// private val sequenceA = listOf("4", "5", "6")
|
||||
// private val sequenceB = listOf("7", "55", "59")
|
||||
// private val sequenceAll = listOf("4", "5", "6", "7", "55", "59")
|
||||
//
|
||||
// private fun update()
|
||||
// {
|
||||
// if(isMotionLoadFinished)
|
||||
// {
|
||||
// if(elapsedTime == 0){
|
||||
// //这次播放是一直循环播放，我们不需要判断动画是否播放完成, 回调函数直接传入null
|
||||
// PlayMotionList(sequenceA, true, null)
|
||||
// }
|
||||
//
|
||||
// elapsedTime += 10 // 动画加载完成后开始计时，每次调用增加 10ms
|
||||
//
|
||||
// if(elapsedTime == 10*1000){
|
||||
// //暂停动画
|
||||
// PlayMotionList(sequenceB, true, null)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
|
||||
if (keyCode == KEYCODE_BACK) {
|
||||
Log.d("MakeupActivity", "Back key pressed")
|
||||
@@ -137,7 +129,10 @@ class MakeupActivity : FaceActivity()
|
||||
override fun onTouchEvent(event: MotionEvent): Boolean {
|
||||
when (event.action) {
|
||||
MotionEvent.ACTION_UP -> {
|
||||
|
||||
if (isMotionLoadFinished) {
|
||||
currentIndex = (currentIndex + 1) % sequenceString.size
|
||||
PlayMotionList(listOf(sequenceString[currentIndex]), true, null)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
<resources>
|
||||
<string name="app_name">f20260113</string>
|
||||
<string name="app_name">f20260127</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,2 @@
|
||||
venv/
|
||||
server/__pycache__/
|
||||
@@ -0,0 +1,133 @@
|
||||
# Face SDK Web 部署说明
|
||||
|
||||
本文说明如何在服务器或本机长期运行 **Face SDK Web**（FastAPI + 静态前端）。开发与接口文字说明见根目录 [README.md](./README.md)。
|
||||
|
||||
---
|
||||
|
||||
## API 文档是否已经写好？
|
||||
|
||||
有两层：
|
||||
|
||||
| 类型 | 说明 |
|
||||
|------|------|
|
||||
| **交互式文档（推荐联调）** | 服务启动后由 FastAPI **自动生成**：浏览器打开 **`/docs`**（Swagger UI）、**`/redoc`**（ReDoc），可直接试请求。机器可读的模式在 **`/openapi.json`**。 |
|
||||
| **文字说明** | [README.md](./README.md) 的「API」一节描述了 `GET /health`、`GET /effects`、`POST /render` 的请求与响应；与代码保持一致即可。 |
|
||||
|
||||
生产环境若不希望对外暴露 Swagger，应在网关层禁止访问 `/docs`、`/redoc`、`/openapi.json`（见下文反向代理示例）。
|
||||
|
||||
---
|
||||
|
||||
## 运行环境
|
||||
|
||||
- **Python**：3.10～3.12（MediaPipe 当前不支持 3.13）。
|
||||
- **CPU**：无需 GPU；推理与渲染在 CPU 上完成。
|
||||
- **系统**：Windows / Linux / macOS 均可；生产常见为 Linux。
|
||||
|
||||
---
|
||||
|
||||
## 部署时需携带的文件
|
||||
|
||||
在 **`python/`** 目录下至少包含：
|
||||
|
||||
- `requirements.txt`
|
||||
- `server/` 整个包，尤其：
|
||||
- `server/main.py` 及依赖的 `.py`
|
||||
- `server/assets/`（含 `face_picture_3dmax.obj`；`face_landmarker.task` 按 README 可选）
|
||||
- `server/effects/`（`<id>.png` 效果贴图）
|
||||
- `server/static/`（前端 `index.html` 及静态资源）
|
||||
|
||||
工作目录应为 **`python/`**（即包含 `server` 包的那一层），以便 `uvicorn server.main:app` 能正确导入。
|
||||
|
||||
---
|
||||
|
||||
## 安装依赖
|
||||
|
||||
```bash
|
||||
cd /path/to/face_sdk/python
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # Windows: .\venv\Scripts\Activate.ps1
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 启动方式
|
||||
|
||||
### 开发 / 小规模使用
|
||||
|
||||
```bash
|
||||
cd /path/to/face_sdk/python
|
||||
source venv/bin/activate
|
||||
uvicorn server.main:app --host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
- 根路径 **`/`**：返回 `server/static/index.html`
|
||||
- **`/static/*`**：静态文件
|
||||
- 日志出现 **`Application startup complete.`** 即就绪
|
||||
|
||||
### 生产建议（Linux）
|
||||
|
||||
- 使用 **进程管理器**（systemd、supervisor）或容器，保证崩溃自动拉起。
|
||||
- 前面挂 **Nginx / Caddy / 云 LB**，统一 TLS 与限流。
|
||||
- **Worker 数量**：README「已知限制」中说明 MediaPipe 检测在应用内通过锁串行化；单进程多 worker **不会**线性提升同一实例内的 GPU 式并行，且可能重复加载模型占内存。更高并发可采用：**多实例（每实例单 worker）+ 负载均衡**，或前置队列削峰。
|
||||
|
||||
示例（单 worker，监听本机 127.0.0.1，由 Nginx 对外）：
|
||||
|
||||
```bash
|
||||
uvicorn server.main:app --host 127.0.0.1 --port 8000 --workers 1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 反向代理示例（Nginx）
|
||||
|
||||
将 HTTPS 流量转到本机 Uvicorn，并可选屏蔽文档路径：
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name your.domain.example;
|
||||
|
||||
# ssl_certificate / ssl_certificate_key ...
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
client_max_body_size 25m; # 略高于应用内图片上限（默认 20MB）
|
||||
}
|
||||
|
||||
# 生产可注释掉以下整块，禁止外网访问 API 探索页
|
||||
# location ~ ^/(docs|redoc|openapi\.json)$ {
|
||||
# deny all;
|
||||
# }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 健康检查
|
||||
|
||||
负载均衡或编排（如 K8s）可使用：
|
||||
|
||||
- **`GET /health`** — 成功返回 `200`，JSON：`{"ok": true}`
|
||||
|
||||
---
|
||||
|
||||
## 防火墙与安全
|
||||
|
||||
- 若不经反向代理，直接监听 `0.0.0.0`，需在主机防火墙与安全组中 **仅开放必要端口**。
|
||||
- **`POST /render`** 接收用户上传文件，务必通过 HTTPS、合理 **`client_max_body_size`** / 限流，防止滥用。
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
- **端口被占用**：更换 `--port` 或结束占用进程后再启动。
|
||||
- **找不到模型或 OBJ**：确认 `server/assets/` 与 `server/effects/` 已随代码一并部署，且工作目录为 `python/`。
|
||||
|
||||
更多行为与性能说明见 [README.md](./README.md)「已知限制」。
|
||||
@@ -0,0 +1,146 @@
|
||||
# Face SDK Web 服务
|
||||
|
||||
把 Android Face SDK 的人脸贴图渲染逻辑迁移到 Python Web 服务：上传一张人物图 + 效果 ID，
|
||||
返回叠加渲染后的 PNG。CPU 实现（OpenCV 仿射 warp + alpha 混合），不依赖 GPU。
|
||||
|
||||
---
|
||||
|
||||
## 渲染原理（与 Android SDK 一致）
|
||||
|
||||
```
|
||||
input image ──► MediaPipe FaceLandmarker ──► 478 normalized landmarks
|
||||
│
|
||||
▼
|
||||
face_picture_3dmax.obj (468 顶点+UV / 852 三角形)
|
||||
每个 obj 顶点 i 的画面位置 = landmarks[INDEX_MAP[i]]
|
||||
│
|
||||
▼
|
||||
每个三角形：
|
||||
src = obj UV × effect 贴图尺寸 (在 512×512 effect 里)
|
||||
dst = landmarks 像素位置 (在 input image 里)
|
||||
cv2.getAffineTransform → cv2.warpAffine → alpha 混合
|
||||
│
|
||||
▼
|
||||
output PNG（同原图尺寸）
|
||||
```
|
||||
|
||||
`INDEX_MAP[468]` 同步自 `vulkan/hardcode_data.h::indexMap`，UV 的 V 翻转与
|
||||
`vulkan/FaceApp.cpp::LoadOBJ` 行为一致。
|
||||
|
||||
---
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
python/
|
||||
├── server/
|
||||
│ ├── main.py FastAPI 入口（/health /effects /render）
|
||||
│ ├── face_renderer.py 渲染核心
|
||||
│ ├── mesh.py OBJ 解析
|
||||
│ ├── index_map.py OBJ vertex → MediaPipe landmark 映射
|
||||
│ ├── assets/
|
||||
│ │ ├── face_picture_3dmax.obj ← 来自 app/src/main/assets
|
||||
│ │ └── face_landmarker.task ← 可选；不放则自动回退到 app/src/main/assets/
|
||||
│ └── effects/
|
||||
│ └── 1.png 512×512 RGBA 效果贴图，文件名是效果 ID（1-99）
|
||||
└── requirements.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 一次性安装
|
||||
|
||||
需要 Python 3.10–3.12（mediapipe 当前不支持 3.13）。
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell（或 cmd）
|
||||
cd D:\face_sdk\python
|
||||
python -m venv venv
|
||||
.\venv\Scripts\Activate.ps1 # cmd 用 .\venv\Scripts\activate.bat
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
```bash
|
||||
# Linux / macOS
|
||||
cd /path/to/face_sdk/python
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 启动服务
|
||||
|
||||
```powershell
|
||||
cd D:\face_sdk\python
|
||||
.\venv\Scripts\Activate.ps1
|
||||
uvicorn server.main:app --host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
启动日志看到 `Application startup complete.` 即就绪。
|
||||
|
||||
---
|
||||
|
||||
## API
|
||||
|
||||
### `GET /health`
|
||||
```json
|
||||
{"ok": true}
|
||||
```
|
||||
|
||||
### `GET /effects`
|
||||
返回当前 `server/effects/` 下存在的效果 ID（按 `<id>.png` 文件名提取，1–99）：
|
||||
```json
|
||||
{"effects": [1]}
|
||||
```
|
||||
|
||||
### `POST /render`
|
||||
- form-data：
|
||||
- `image`: 文件（jpeg / png / webp，≤ 20MB）
|
||||
- `effect_id`: 整数（1–99，必须在 `/effects` 列表内）
|
||||
- 成功：`200`，`Content-Type: image/png`，body 是 PNG 字节
|
||||
- 没检测到人脸：`400 {"error":"no_face"}`
|
||||
- effect_id 不存在：`404`
|
||||
- 参数错误（图非法 / 太大 / id 越界）：`400`
|
||||
|
||||
#### curl 示例
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/render \
|
||||
-F image=@/path/to/photo.jpg \
|
||||
-F effect_id=1 \
|
||||
--output result.png
|
||||
```
|
||||
|
||||
#### Python 示例
|
||||
|
||||
```python
|
||||
import requests
|
||||
files = {"image": open("photo.jpg", "rb")}
|
||||
data = {"effect_id": 1}
|
||||
r = requests.post("http://localhost:8000/render", files=files, data=data)
|
||||
r.raise_for_status()
|
||||
open("result.png", "wb").write(r.content)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 添加新效果
|
||||
|
||||
把一张 512×512 的 RGBA PNG 放到 `python/server/effects/<id>.png`（`<id>` 是 1–99 的整数）。
|
||||
服务下次响应 `/effects` 时会自动列出；首次被请求时按需载入并缓存内存。
|
||||
|
||||
PNG 必须有 alpha 通道——透明区域不会盖到原图上。
|
||||
|
||||
---
|
||||
|
||||
## 已知限制
|
||||
|
||||
- 纯 2D 仿射 warp，与 Android SDK 行为一致：人脸侧面角度大时贴图会拉伸。
|
||||
- MediaPipe FaceLandmarker 每次 `detect()` 不保证多线程并发安全，服务用锁串行化。
|
||||
单进程同步部署够用；要更高并发请用多 worker 或前置队列。
|
||||
- 单张 4K 图渲染约 100–300ms（852 三角形 × `cv2.warpAffine`）；如需更快可改成
|
||||
`cv2.remap` 一次性贴整脸，但需要更多代码。
|
||||
@@ -0,0 +1,7 @@
|
||||
fastapi>=0.110
|
||||
uvicorn[standard]>=0.27
|
||||
python-multipart>=0.0.9
|
||||
mediapipe>=0.10.14
|
||||
opencv-python>=4.9
|
||||
pillow>=10.0
|
||||
numpy>=1.26,<2.0
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 7.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
@@ -0,0 +1,312 @@
|
||||
"""
|
||||
人脸贴图渲染核心：接受一张图 + 一个效果 ID，输出 PNG 字节。
|
||||
|
||||
流程（与 Android 端 vulkan/FaceApp.cpp 一致，CPU 实现）：
|
||||
1. MediaPipe FaceLandmarker → 478 normalized landmarks per face
|
||||
2. 取最大人脸（按 landmark bbox 面积）
|
||||
3. 把 obj 的 468 顶点位置改写为 landmarks[INDEX_MAP[i]]，得到 dst 顶点
|
||||
4. 对 mesh 的每个三角形：从 512×512 effect 贴图取 src 三角形，仿射 warp
|
||||
到 dst 三角形位置，按贴图 alpha 通道 alpha-blend 到原图副本上
|
||||
5. 输出 PNG（保持原图尺寸）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from .index_map import INDEX_MAP
|
||||
from .mesh import FaceMesh, load_face_mesh
|
||||
|
||||
|
||||
class NoFaceError(Exception):
|
||||
"""图里检测不到人脸。"""
|
||||
|
||||
|
||||
class EffectNotFoundError(Exception):
|
||||
"""指定 effect_id 没有对应贴图。"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RendererPaths:
|
||||
obj_path: Path
|
||||
effects_dir: Path
|
||||
|
||||
|
||||
class FaceRenderer:
|
||||
"""
|
||||
单例式渲染器：mesh / mediapipe 模型加载一次复用。
|
||||
|
||||
线程安全：MediaPipe FaceLandmarker 的 detect() 不保证多线程并发，
|
||||
用 _lock 串行化。FastAPI 单进程同步部署下，请求本身就串行，开销可忽略。
|
||||
"""
|
||||
|
||||
def __init__(self, paths: RendererPaths, max_image_bytes: int = 20 * 1024 * 1024):
|
||||
self.paths = paths
|
||||
self.max_image_bytes = max_image_bytes
|
||||
self._mesh: FaceMesh = load_face_mesh(paths.obj_path)
|
||||
if len(self._mesh.uvs) != len(INDEX_MAP):
|
||||
raise RuntimeError(
|
||||
f"mesh has {len(self._mesh.uvs)} vertices but INDEX_MAP has "
|
||||
f"{len(INDEX_MAP)}; obj/index_map.py 不匹配"
|
||||
)
|
||||
# 预转 numpy 加速
|
||||
self._index_map = np.asarray(INDEX_MAP, dtype=np.int32)
|
||||
|
||||
self._effect_cache: dict[int, np.ndarray] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
# MediaPipe 延迟加载（首次请求时加载，缩短启动时间，便于本地调试）
|
||||
self._landmarker = None
|
||||
self._landmarker_lock = threading.Lock()
|
||||
|
||||
# ---------- public ----------
|
||||
|
||||
def list_effect_ids(self) -> list[int]:
|
||||
"""枚举 effects/<n>.png 中的 n。"""
|
||||
ids: list[int] = []
|
||||
if not self.paths.effects_dir.is_dir():
|
||||
return ids
|
||||
for p in self.paths.effects_dir.iterdir():
|
||||
if p.suffix.lower() != ".png":
|
||||
continue
|
||||
try:
|
||||
n = int(p.stem)
|
||||
except ValueError:
|
||||
continue
|
||||
if 1 <= n <= 99:
|
||||
ids.append(n)
|
||||
return sorted(ids)
|
||||
|
||||
def render(self, image_bytes: bytes, effect_id: int) -> bytes:
|
||||
if len(image_bytes) > self.max_image_bytes:
|
||||
raise ValueError("image too large")
|
||||
|
||||
effect_rgba = self._load_effect(effect_id) # (H,W,4) uint8
|
||||
|
||||
# 1. 解码 + 处理 EXIF orientation → RGB ndarray
|
||||
img_rgb = self._decode_input(image_bytes)
|
||||
h, w = img_rgb.shape[:2]
|
||||
|
||||
# 2. MediaPipe 检测，挑最大人脸
|
||||
landmarks_norm = self._detect_largest_face(img_rgb) # (N,2) ∈ [0,1]² (N>=468)
|
||||
|
||||
# 3. obj 顶点 → 像素坐标
|
||||
dst_xy = self._compute_dst_vertices(landmarks_norm, w, h)
|
||||
|
||||
# 4. warp + 混合
|
||||
out = img_rgb.copy()
|
||||
self._composite(out, effect_rgba, dst_xy)
|
||||
|
||||
# 5. 编码 PNG
|
||||
return self._encode_png(out)
|
||||
|
||||
# ---------- internals ----------
|
||||
|
||||
def _load_effect(self, effect_id: int) -> np.ndarray:
|
||||
if not (1 <= effect_id <= 99):
|
||||
raise EffectNotFoundError(f"effect_id out of range: {effect_id}")
|
||||
with self._cache_lock:
|
||||
cached = self._effect_cache.get(effect_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
png_path = self.paths.effects_dir / f"{effect_id}.png"
|
||||
if not png_path.is_file():
|
||||
raise EffectNotFoundError(f"effect {effect_id} not found")
|
||||
# 用 PIL 解码以保证 RGBA 一致；OpenCV imread 对部分 PNG 会丢 alpha
|
||||
img = Image.open(png_path).convert("RGBA")
|
||||
arr = np.array(img, dtype=np.uint8) # (H,W,4) RGBA
|
||||
with self._cache_lock:
|
||||
self._effect_cache[effect_id] = arr
|
||||
return arr
|
||||
|
||||
def _decode_input(self, image_bytes: bytes) -> np.ndarray:
|
||||
try:
|
||||
pil = Image.open(io.BytesIO(image_bytes))
|
||||
pil = ImageOps.exif_transpose(pil) # 处理手机 EXIF 旋转
|
||||
pil = pil.convert("RGB")
|
||||
except Exception as e:
|
||||
raise ValueError(f"invalid image: {e}") from e
|
||||
return np.array(pil, dtype=np.uint8)
|
||||
|
||||
def _ensure_landmarker(self):
|
||||
if self._landmarker is not None:
|
||||
return
|
||||
with self._landmarker_lock:
|
||||
if self._landmarker is not None:
|
||||
return
|
||||
# 延迟 import，避免单元测试时强依赖 mediapipe
|
||||
from mediapipe.tasks import python as mp_python
|
||||
from mediapipe.tasks.python import vision as mp_vision
|
||||
|
||||
model_path = self._find_model_file()
|
||||
base_options = mp_python.BaseOptions(model_asset_path=str(model_path))
|
||||
options = mp_vision.FaceLandmarkerOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mp_vision.RunningMode.IMAGE,
|
||||
num_faces=5, # 检测最多 5 张，挑最大那张
|
||||
min_face_detection_confidence=0.5,
|
||||
min_face_presence_confidence=0.5,
|
||||
min_tracking_confidence=0.5,
|
||||
output_face_blendshapes=False,
|
||||
output_facial_transformation_matrixes=False,
|
||||
)
|
||||
self._landmarker = mp_vision.FaceLandmarker.create_from_options(options)
|
||||
|
||||
def _find_model_file(self) -> Path:
|
||||
"""优先用本服务 assets 下的，找不到再回退到仓库 app/src/main/assets。"""
|
||||
local = self.paths.obj_path.parent / "face_landmarker.task"
|
||||
if local.is_file():
|
||||
return local
|
||||
# 回退到仓库 app/src/main/assets/face_landmarker.task（Android SDK 用同一份）
|
||||
repo_root = self.paths.obj_path.resolve().parents[3]
|
||||
fallback = repo_root / "app" / "src" / "main" / "assets" / "face_landmarker.task"
|
||||
if fallback.is_file():
|
||||
return fallback
|
||||
raise FileNotFoundError(
|
||||
"face_landmarker.task not found; place it under "
|
||||
f"{local} or {fallback}"
|
||||
)
|
||||
|
||||
def _detect_largest_face(self, img_rgb: np.ndarray) -> np.ndarray:
|
||||
self._ensure_landmarker()
|
||||
import mediapipe as mp
|
||||
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
|
||||
with self._landmarker_lock:
|
||||
result = self._landmarker.detect(mp_image)
|
||||
|
||||
faces = result.face_landmarks or []
|
||||
if not faces:
|
||||
raise NoFaceError("no face detected")
|
||||
|
||||
# 选 bbox 面积最大的那张
|
||||
def bbox_area(landmarks) -> float:
|
||||
xs = [lm.x for lm in landmarks]
|
||||
ys = [lm.y for lm in landmarks]
|
||||
return (max(xs) - min(xs)) * (max(ys) - min(ys))
|
||||
|
||||
biggest = max(faces, key=bbox_area)
|
||||
# 转 ndarray，只取 xy
|
||||
n = len(biggest)
|
||||
arr = np.empty((n, 2), dtype=np.float32)
|
||||
for i, lm in enumerate(biggest):
|
||||
arr[i, 0] = lm.x
|
||||
arr[i, 1] = lm.y
|
||||
return arr
|
||||
|
||||
def _compute_dst_vertices(
|
||||
self, landmarks_norm: np.ndarray, w: int, h: int
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
landmarks_norm: (N, 2) ∈ [0,1]²
|
||||
返回 (468, 2) float32 像素坐标，对应 mesh 的 468 个顶点。
|
||||
"""
|
||||
n_landmarks = landmarks_norm.shape[0]
|
||||
if self._index_map.max() >= n_landmarks:
|
||||
raise RuntimeError(
|
||||
f"INDEX_MAP references landmark index {int(self._index_map.max())} "
|
||||
f"but mediapipe returned only {n_landmarks} landmarks"
|
||||
)
|
||||
picked = landmarks_norm[self._index_map] # (468, 2)
|
||||
out = np.empty_like(picked)
|
||||
out[:, 0] = picked[:, 0] * w
|
||||
out[:, 1] = picked[:, 1] * h
|
||||
return out.astype(np.float32)
|
||||
|
||||
def _composite(
|
||||
self,
|
||||
canvas_rgb: np.ndarray,
|
||||
effect_rgba: np.ndarray,
|
||||
dst_xy: np.ndarray,
|
||||
) -> None:
|
||||
"""
|
||||
在 canvas_rgb（uint8 H×W×3，原地修改）上对 mesh 每个三角形：
|
||||
从 effect_rgba 取对应 src 三角形 → warpAffine 到 dst → alpha 混合。
|
||||
"""
|
||||
eh, ew = effect_rgba.shape[:2]
|
||||
canvas_h, canvas_w = canvas_rgb.shape[:2]
|
||||
uvs = self._mesh.uvs # (468, 2)
|
||||
tris = self._mesh.triangles # (852, 3)
|
||||
|
||||
# 把 UV 一次性转成 effect 像素坐标
|
||||
src_pts_all = np.empty_like(uvs)
|
||||
src_pts_all[:, 0] = uvs[:, 0] * ew
|
||||
src_pts_all[:, 1] = uvs[:, 1] * eh
|
||||
|
||||
effect_rgb = effect_rgba[..., :3]
|
||||
effect_a = effect_rgba[..., 3]
|
||||
|
||||
for tri in tris:
|
||||
i0, i1, i2 = int(tri[0]), int(tri[1]), int(tri[2])
|
||||
src_tri = np.float32([src_pts_all[i0], src_pts_all[i1], src_pts_all[i2]])
|
||||
dst_tri = np.float32([dst_xy[i0], dst_xy[i1], dst_xy[i2]])
|
||||
|
||||
# 用 dst bbox 限定计算区域，避免对全画布 warp
|
||||
x_min = int(np.floor(dst_tri[:, 0].min()))
|
||||
y_min = int(np.floor(dst_tri[:, 1].min()))
|
||||
x_max = int(np.ceil(dst_tri[:, 0].max()))
|
||||
y_max = int(np.ceil(dst_tri[:, 1].max()))
|
||||
|
||||
# clip 到画布
|
||||
x_min_c = max(0, x_min)
|
||||
y_min_c = max(0, y_min)
|
||||
x_max_c = min(canvas_w, x_max)
|
||||
y_max_c = min(canvas_h, y_max)
|
||||
if x_max_c <= x_min_c or y_max_c <= y_min_c:
|
||||
continue
|
||||
|
||||
box_w = x_max_c - x_min_c
|
||||
box_h = y_max_c - y_min_c
|
||||
|
||||
# 构造从 src → 局部 bbox 坐标系（左上为原点）的仿射
|
||||
dst_tri_local = dst_tri - np.array([x_min_c, y_min_c], dtype=np.float32)
|
||||
M = cv2.getAffineTransform(src_tri, dst_tri_local)
|
||||
|
||||
# 给 RGB 和 alpha 分别 warp 到 bbox 大小
|
||||
warped_rgb = cv2.warpAffine(
|
||||
effect_rgb,
|
||||
M,
|
||||
(box_w, box_h),
|
||||
flags=cv2.INTER_LINEAR,
|
||||
borderMode=cv2.BORDER_REPLICATE,
|
||||
)
|
||||
warped_a = cv2.warpAffine(
|
||||
effect_a,
|
||||
M,
|
||||
(box_w, box_h),
|
||||
flags=cv2.INTER_LINEAR,
|
||||
borderMode=cv2.BORDER_CONSTANT,
|
||||
borderValue=0,
|
||||
)
|
||||
|
||||
# 限制只在三角形内部混合（防止 bbox 多余像素污染）
|
||||
tri_mask = np.zeros((box_h, box_w), dtype=np.uint8)
|
||||
cv2.fillConvexPoly(
|
||||
tri_mask,
|
||||
dst_tri_local.astype(np.int32),
|
||||
255,
|
||||
lineType=cv2.LINE_AA,
|
||||
)
|
||||
alpha = (
|
||||
warped_a.astype(np.float32) * (tri_mask.astype(np.float32) / 255.0)
|
||||
) / 255.0
|
||||
alpha = alpha[..., None] # (h,w,1)
|
||||
|
||||
roi = canvas_rgb[y_min_c:y_max_c, x_min_c:x_max_c].astype(np.float32)
|
||||
blended = roi * (1.0 - alpha) + warped_rgb.astype(np.float32) * alpha
|
||||
canvas_rgb[y_min_c:y_max_c, x_min_c:x_max_c] = np.clip(
|
||||
blended, 0, 255
|
||||
).astype(np.uint8)
|
||||
|
||||
def _encode_png(self, img_rgb: np.ndarray) -> bytes:
|
||||
pil = Image.fromarray(img_rgb)
|
||||
buf = io.BytesIO()
|
||||
pil.save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
OBJ 顶点索引 → MediaPipe FaceLandmarker landmark 索引的映射表。
|
||||
|
||||
直接从 vulkan/hardcode_data.h::indexMap[468] 同步而来。意义：
|
||||
obj 顶点 i 在画面中的位置 = mediapipe_landmarks[INDEX_MAP[i]]
|
||||
|
||||
注意：face_picture_3dmax.obj 的顶点数量必须正好 468，与本表长度一致。
|
||||
若以后换 obj，需要重新生成本表。
|
||||
"""
|
||||
|
||||
INDEX_MAP = [
|
||||
127, 34, 139, 11, 0, 37, 232, 231, 120, 72,
|
||||
39, 128, 121, 47, 104, 69, 67, 175, 171, 148,
|
||||
118, 50, 101, 73, 40, 9, 151, 108, 48, 115,
|
||||
131, 194, 204, 211, 74, 185, 80, 42, 183, 92,
|
||||
186, 230, 229, 202, 212, 214, 83, 18, 17, 76,
|
||||
61, 146, 160, 29, 30, 56, 157, 173, 106, 135,
|
||||
192, 203, 165, 98, 21, 71, 68, 51, 45, 4,
|
||||
144, 24, 23, 77, 91, 205, 187, 201, 200, 182,
|
||||
90, 181, 85, 84, 206, 36, 140, 193, 189, 244,
|
||||
159, 158, 28, 247, 246, 161, 236, 3, 196, 54,
|
||||
168, 8, 117, 228, 31, 55, 97, 99, 126, 100,
|
||||
166, 79, 218, 155, 154, 26, 209, 49, 136, 150,
|
||||
217, 223, 52, 53, 134, 170, 43, 119, 226, 130,
|
||||
63, 238, 20, 242, 46, 70, 156, 78, 62, 96,
|
||||
143, 227, 123, 111, 44, 125, 19, 216, 153, 22,
|
||||
167, 208, 142, 57, 60, 35, 113, 27, 210, 225,
|
||||
137, 116, 41, 38, 129, 64, 240, 102, 207, 184,
|
||||
169, 149, 176, 105, 66, 122, 6, 147, 65, 107,
|
||||
89, 180, 93, 15, 86, 14, 87, 145, 88, 179,
|
||||
95, 138, 172, 215, 58, 219, 81, 195, 199, 82,
|
||||
163, 110, 234, 109, 235, 191, 222, 141, 221, 197,
|
||||
25, 7, 33, 220, 237, 245, 162, 188, 174, 2,
|
||||
241, 164, 12, 13, 198, 133, 112, 243, 239, 190,
|
||||
32, 178, 132, 177, 1, 213, 59, 94, 75, 224,
|
||||
233, 114, 124, 356, 389, 368, 302, 267, 452, 350,
|
||||
349, 303, 269, 357, 343, 277, 453, 333, 332, 297,
|
||||
152, 377, 347, 348, 330, 304, 270, 336, 337, 278,
|
||||
279, 360, 418, 262, 431, 408, 409, 310, 415, 407,
|
||||
410, 450, 422, 430, 434, 313, 314, 306, 307, 375,
|
||||
387, 388, 260, 286, 414, 398, 335, 406, 364, 367,
|
||||
416, 423, 358, 327, 251, 284, 298, 281, 5, 373,
|
||||
374, 253, 320, 321, 425, 427, 411, 421, 405, 404,
|
||||
315, 16, 426, 266, 400, 369, 322, 391, 417, 465,
|
||||
464, 386, 257, 258, 466, 456, 399, 419, 285, 346,
|
||||
340, 261, 413, 441, 460, 328, 355, 371, 329, 392,
|
||||
439, 438, 382, 341, 256, 429, 420, 394, 379, 437,
|
||||
443, 444, 283, 275, 440, 363, 338, 273, 451, 446,
|
||||
342, 467, 293, 334, 282, 458, 461, 462, 276, 353,
|
||||
383, 308, 324, 325, 300, 372, 345, 447, 352, 274,
|
||||
248, 436, 381, 252, 393, 428, 287, 250, 384, 265,
|
||||
259, 424, 292, 366, 271, 294, 455, 272, 432, 395,
|
||||
299, 351, 280, 319, 295, 296, 403, 323, 454, 316,
|
||||
380, 318, 402, 365, 435, 397, 344, 311, 291, 396,
|
||||
268, 445, 254, 339, 449, 264, 10, 442, 370, 263,
|
||||
255, 359, 412, 301, 378, 326, 457, 362, 459, 463,
|
||||
354, 401, 361, 309, 376, 433, 289, 305, 448, 290,
|
||||
288, 249, 103, 385, 331, 317, 312, 390,
|
||||
]
|
||||
|
||||
assert len(INDEX_MAP) == 468, f"INDEX_MAP must have 468 entries, got {len(INDEX_MAP)}"
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
FastAPI 入口。启动：
|
||||
|
||||
python -m uvicorn server.main:app --host 0.0.0.0 --port 8000
|
||||
|
||||
或在 python/ 目录下：
|
||||
|
||||
uvicorn server.main:app --host 0.0.0.0 --port 8000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .face_renderer import (
|
||||
EffectNotFoundError,
|
||||
FaceRenderer,
|
||||
NoFaceError,
|
||||
RendererPaths,
|
||||
)
|
||||
|
||||
# 路径相对本文件，避免 cwd 不同时找不到资源
|
||||
_SERVER_DIR = Path(__file__).resolve().parent
|
||||
_PATHS = RendererPaths(
|
||||
obj_path=_SERVER_DIR / "assets" / "face_picture_3dmax.obj",
|
||||
effects_dir=_SERVER_DIR / "effects",
|
||||
)
|
||||
|
||||
app = FastAPI(title="Face SDK Web", version="0.1.0")
|
||||
renderer = FaceRenderer(_PATHS)
|
||||
|
||||
_STATIC_DIR = _SERVER_DIR / "static"
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def index() -> FileResponse:
|
||||
return FileResponse(_STATIC_DIR / "index.html")
|
||||
|
||||
|
||||
# /static/* 提供其它资源（CSS/JS 拆分时用得上；当前 index.html 内联）
|
||||
app.mount("/static", StaticFiles(directory=_STATIC_DIR), name="static")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/effects")
|
||||
def list_effects() -> dict:
|
||||
return {"effects": renderer.list_effect_ids()}
|
||||
|
||||
|
||||
@app.post("/render")
|
||||
async def render(
|
||||
image: UploadFile = File(...),
|
||||
effect_id: int = Form(...),
|
||||
) -> Response:
|
||||
image_bytes = await image.read()
|
||||
if not image_bytes:
|
||||
raise HTTPException(status_code=400, detail="empty image")
|
||||
try:
|
||||
png_bytes = renderer.render(image_bytes, effect_id)
|
||||
except NoFaceError:
|
||||
return JSONResponse(status_code=400, content={"error": "no_face"})
|
||||
except EffectNotFoundError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return Response(content=png_bytes, media_type="image/png")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
解析 face_picture_3dmax.obj，得到顶点 UV 和三角形索引。
|
||||
|
||||
OBJ 约定：v 顶点 / vt UV / vn 法线，f 三元组 v/vt/vn 索引（1-based）。
|
||||
本服务只需要每个顶点的 UV 和三角形顶点索引——3D 顶点位置在 SDK 里也只是
|
||||
占位（运行时被 MediaPipe landmark 覆盖），这里同样无需读 v。
|
||||
|
||||
★ 与 vulkan/FaceApp.cpp::LoadOBJ 一致，UV 的 V 分量取 (1 - v)，把 OBJ 的
|
||||
bottom-up 约定翻成 top-down，与图像（OpenCV/Pillow）一致。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class FaceMesh:
|
||||
# (468, 2) float32, 每个顶点的 UV，V 已翻转，∈ [0,1]²
|
||||
uvs: np.ndarray
|
||||
# (852, 3) int32, 每个三角形的 3 个顶点索引（0-based，指向 uvs 第一维）
|
||||
triangles: np.ndarray
|
||||
|
||||
|
||||
def load_face_mesh(obj_path: str | Path) -> FaceMesh:
|
||||
obj_path = Path(obj_path)
|
||||
text = obj_path.read_text(encoding="utf-8", errors="ignore")
|
||||
|
||||
pos_count = 0
|
||||
uv_list: list[tuple[float, float]] = []
|
||||
# OBJ 的 face 用 v/vt/vn，三个索引可能不同；但本工程的 obj 三者一一对齐
|
||||
# （pos_index == uv_index == normal_index），所以我们只看 v 索引来索引 UV
|
||||
# 列表也成立。这里仍然显式校验以防换 obj 时出错。
|
||||
triangles: list[tuple[int, int, int]] = []
|
||||
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
tag = parts[0]
|
||||
|
||||
if tag == "v":
|
||||
pos_count += 1
|
||||
elif tag == "vt":
|
||||
u = float(parts[1])
|
||||
v = float(parts[2])
|
||||
uv_list.append((u, 1.0 - v)) # V 翻转
|
||||
elif tag == "f":
|
||||
if len(parts) != 4:
|
||||
raise ValueError(f"only triangle faces supported, got: {line}")
|
||||
tri: list[int] = []
|
||||
for token in parts[1:]:
|
||||
# 形如 "v/vt/vn" 或 "v//vn" 或 "v"
|
||||
seg = token.split("/")
|
||||
v_idx = int(seg[0]) - 1 # OBJ 1-based → 0-based
|
||||
vt_idx = int(seg[1]) - 1 if len(seg) > 1 and seg[1] else v_idx
|
||||
if v_idx != vt_idx:
|
||||
raise ValueError(
|
||||
f"this loader assumes pos_idx == uv_idx, "
|
||||
f"got v={v_idx + 1} vt={vt_idx + 1} in line: {line}"
|
||||
)
|
||||
tri.append(v_idx)
|
||||
triangles.append((tri[0], tri[1], tri[2]))
|
||||
|
||||
if pos_count != len(uv_list):
|
||||
raise ValueError(
|
||||
f"vertex count {pos_count} != uv count {len(uv_list)}; "
|
||||
"obj loader assumes 1:1 correspondence"
|
||||
)
|
||||
|
||||
uvs = np.asarray(uv_list, dtype=np.float32)
|
||||
tris = np.asarray(triangles, dtype=np.int32)
|
||||
return FaceMesh(uvs=uvs, triangles=tris)
|
||||
@@ -0,0 +1,310 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="mobile-web-app-capable" content="yes" />
|
||||
<meta name="theme-color" content="#0f1115" />
|
||||
<title>Face SDK Web · 测试</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1115;
|
||||
--card: #1a1d24;
|
||||
--border: #2a2f3a;
|
||||
--text: #e6e8eb;
|
||||
--muted: #8a92a3;
|
||||
--accent: #4f8cff;
|
||||
--accent-hover: #3d78e8;
|
||||
--error: #ff6b6b;
|
||||
--ok: #4fd18b;
|
||||
}
|
||||
* { box-sizing: border-box; -webkit-tap-highlight-color: transparent; }
|
||||
html, body { overflow-x: hidden; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC",
|
||||
"Hiragino Sans GB", "Microsoft YaHei", sans-serif;
|
||||
padding: 24px;
|
||||
padding-left: max(24px, env(safe-area-inset-left));
|
||||
padding-right: max(24px, env(safe-area-inset-right));
|
||||
padding-bottom: max(24px, env(safe-area-inset-bottom));
|
||||
min-height: 100vh;
|
||||
min-height: 100dvh;
|
||||
-webkit-text-size-adjust: 100%;
|
||||
}
|
||||
h1 { margin: 0 0 8px; font-size: 22px; font-weight: 600; }
|
||||
.subtitle { color: var(--muted); margin-bottom: 24px; font-size: 13px; }
|
||||
.card {
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 20px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.row { display: flex; gap: 16px; flex-wrap: wrap; align-items: end; }
|
||||
.field { display: flex; flex-direction: column; gap: 6px; flex: 1 1 200px; min-width: 0; }
|
||||
label { font-size: 12px; color: var(--muted); }
|
||||
/* font-size: 16px 防止 iOS Safari 聚焦时自动放大页面 */
|
||||
input[type="file"], select {
|
||||
background: #0f1115;
|
||||
color: var(--text);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
padding: 10px 12px;
|
||||
font-size: 16px;
|
||||
width: 100%;
|
||||
min-height: 44px;
|
||||
-webkit-appearance: none;
|
||||
appearance: none;
|
||||
}
|
||||
select {
|
||||
/* 自定义下拉箭头（去掉默认样式后补回来） */
|
||||
background-image: url("data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='12' height='8' viewBox='0 0 12 8'><path fill='%238a92a3' d='M6 8L0 0h12z'/></svg>");
|
||||
background-repeat: no-repeat;
|
||||
background-position: right 12px center;
|
||||
padding-right: 32px;
|
||||
}
|
||||
button {
|
||||
background: var(--accent);
|
||||
color: white;
|
||||
border: 0;
|
||||
border-radius: 6px;
|
||||
padding: 12px 20px;
|
||||
font-size: 15px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, transform 0.1s;
|
||||
min-height: 44px;
|
||||
touch-action: manipulation;
|
||||
}
|
||||
button:hover:not(:disabled) { background: var(--accent-hover); }
|
||||
button:active:not(:disabled) { transform: scale(0.98); }
|
||||
button:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.status { margin-top: 12px; font-size: 13px; min-height: 18px; word-break: break-word; }
|
||||
.status.error { color: var(--error); }
|
||||
.status.ok { color: var(--ok); }
|
||||
.preview {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 16px;
|
||||
}
|
||||
.preview-cell {
|
||||
background: #0a0c10;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 12px;
|
||||
min-height: 200px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.preview-cell h3 {
|
||||
margin: 0 0 10px;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--muted);
|
||||
}
|
||||
.preview-cell img {
|
||||
max-width: 100%;
|
||||
max-height: 70vh;
|
||||
border-radius: 4px;
|
||||
object-fit: contain;
|
||||
align-self: center;
|
||||
}
|
||||
.placeholder {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
text-align: center;
|
||||
padding: 20px;
|
||||
}
|
||||
.download {
|
||||
margin-top: 10px;
|
||||
font-size: 13px;
|
||||
}
|
||||
.download a {
|
||||
color: var(--accent);
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
padding: 6px 0;
|
||||
}
|
||||
.download a:hover { text-decoration: underline; }
|
||||
|
||||
/* 平板 / 小桌面 */
|
||||
@media (max-width: 720px) {
|
||||
body { padding: 16px; padding-left: max(16px, env(safe-area-inset-left)); padding-right: max(16px, env(safe-area-inset-right)); }
|
||||
h1 { font-size: 20px; }
|
||||
.subtitle { margin-bottom: 16px; }
|
||||
.card { padding: 16px; border-radius: 8px; margin-bottom: 16px; }
|
||||
.row { gap: 12px; }
|
||||
.field { flex: 1 1 100%; }
|
||||
/* 移动端按钮整行，方便单手点 */
|
||||
#submit-btn { width: 100%; flex: 1 1 100%; padding: 14px 20px; font-size: 16px; }
|
||||
.preview { grid-template-columns: 1fr; gap: 12px; }
|
||||
.preview-cell { padding: 10px; min-height: 160px; }
|
||||
.preview-cell img { max-height: 60vh; max-height: 60dvh; }
|
||||
}
|
||||
|
||||
/* 窄屏手机（375px 及以下） */
|
||||
@media (max-width: 380px) {
|
||||
body { padding: 12px; }
|
||||
h1 { font-size: 18px; }
|
||||
.card { padding: 14px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Face SDK Web · 测试页</h1>
|
||||
<div class="subtitle">上传一张人脸图片，选择效果 ID，查看叠加渲染结果。</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="row">
|
||||
<div class="field">
|
||||
<label for="image-input">图片</label>
|
||||
<input type="file" id="image-input" accept="image/*" />
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="effect-select">效果 ID</label>
|
||||
<select id="effect-select">
|
||||
<option value="">加载中...</option>
|
||||
</select>
|
||||
</div>
|
||||
<button id="submit-btn">渲染</button>
|
||||
</div>
|
||||
<div id="status" class="status"></div>
|
||||
</div>
|
||||
|
||||
<div class="preview">
|
||||
<div class="preview-cell">
|
||||
<h3>原图</h3>
|
||||
<div id="orig-wrap" class="placeholder">未选择图片</div>
|
||||
</div>
|
||||
<div class="preview-cell">
|
||||
<h3>渲染结果</h3>
|
||||
<div id="result-wrap" class="placeholder">等待渲染</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(() => {
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const imageInput = $("image-input");
|
||||
const effectSelect = $("effect-select");
|
||||
const submitBtn = $("submit-btn");
|
||||
const statusEl = $("status");
|
||||
const origWrap = $("orig-wrap");
|
||||
const resultWrap = $("result-wrap");
|
||||
|
||||
let lastResultUrl = null;
|
||||
|
||||
function setStatus(msg, kind) {
|
||||
statusEl.textContent = msg || "";
|
||||
statusEl.className = "status" + (kind ? " " + kind : "");
|
||||
}
|
||||
|
||||
function showOriginal(file) {
|
||||
origWrap.className = "";
|
||||
origWrap.innerHTML = "";
|
||||
const img = document.createElement("img");
|
||||
img.src = URL.createObjectURL(file);
|
||||
origWrap.appendChild(img);
|
||||
}
|
||||
|
||||
function showResult(blob) {
|
||||
if (lastResultUrl) URL.revokeObjectURL(lastResultUrl);
|
||||
lastResultUrl = URL.createObjectURL(blob);
|
||||
resultWrap.className = "";
|
||||
resultWrap.innerHTML = "";
|
||||
const img = document.createElement("img");
|
||||
img.src = lastResultUrl;
|
||||
resultWrap.appendChild(img);
|
||||
const dl = document.createElement("div");
|
||||
dl.className = "download";
|
||||
const a = document.createElement("a");
|
||||
a.href = lastResultUrl;
|
||||
a.download = "result.png";
|
||||
a.textContent = "下载 PNG";
|
||||
dl.appendChild(a);
|
||||
resultWrap.appendChild(dl);
|
||||
}
|
||||
|
||||
async function loadEffects() {
|
||||
try {
|
||||
const res = await fetch("/effects");
|
||||
if (!res.ok) throw new Error("HTTP " + res.status);
|
||||
const data = await res.json();
|
||||
const ids = data.effects || [];
|
||||
effectSelect.innerHTML = "";
|
||||
if (ids.length === 0) {
|
||||
effectSelect.innerHTML = '<option value="">无可用效果</option>';
|
||||
setStatus("server/effects/ 下没有 PNG 文件", "error");
|
||||
return;
|
||||
}
|
||||
for (const id of ids) {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = String(id);
|
||||
opt.textContent = String(id);
|
||||
effectSelect.appendChild(opt);
|
||||
}
|
||||
} catch (e) {
|
||||
effectSelect.innerHTML = '<option value="">加载失败</option>';
|
||||
setStatus("加载效果列表失败：" + e.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
imageInput.addEventListener("change", () => {
|
||||
const f = imageInput.files && imageInput.files[0];
|
||||
if (f) {
|
||||
showOriginal(f);
|
||||
setStatus("");
|
||||
}
|
||||
});
|
||||
|
||||
submitBtn.addEventListener("click", async () => {
|
||||
const file = imageInput.files && imageInput.files[0];
|
||||
const effectId = effectSelect.value;
|
||||
if (!file) { setStatus("请先选择图片", "error"); return; }
|
||||
if (!effectId) { setStatus("请选择效果 ID", "error"); return; }
|
||||
|
||||
submitBtn.disabled = true;
|
||||
setStatus("渲染中...");
|
||||
const t0 = performance.now();
|
||||
|
||||
try {
|
||||
const fd = new FormData();
|
||||
fd.append("image", file);
|
||||
fd.append("effect_id", effectId);
|
||||
const res = await fetch("/render", { method: "POST", body: fd });
|
||||
|
||||
if (!res.ok) {
|
||||
let msg = "HTTP " + res.status;
|
||||
try {
|
||||
const j = await res.json();
|
||||
if (j.error === "no_face") msg = "未检测到人脸";
|
||||
else if (j.detail) msg = j.detail;
|
||||
else if (j.error) msg = j.error;
|
||||
} catch (_) { /* 非 JSON 响应 */ }
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const blob = await res.blob();
|
||||
showResult(blob);
|
||||
const dt = ((performance.now() - t0) / 1000).toFixed(2);
|
||||
setStatus(`完成（${dt}s）`, "ok");
|
||||
} catch (e) {
|
||||
setStatus("失败：" + e.message, "error");
|
||||
} finally {
|
||||
submitBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
loadEffects();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+64
-8
@@ -60,7 +60,10 @@ void Application::initWindow()
|
||||
glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API);
|
||||
glfwWindowHint(GLFW_RESIZABLE, GLFW_FALSE);
|
||||
|
||||
window = glfwCreateWindow(480, 480, "Vulkan", nullptr, nullptr);
|
||||
// Windows 端仅用于本机调试。改成接近手机的竖屏比例（540x960），
|
||||
// 这样能直观验证「画布居中正方形 + 上下黑边 letterbox」的效果。
|
||||
// Android 端走另一条分支，窗口尺寸由系统 surface 决定。
|
||||
window = glfwCreateWindow(540, 960, "Vulkan", nullptr, nullptr);
|
||||
#else
|
||||
|
||||
#endif
|
||||
@@ -134,16 +137,68 @@ void Application::initVulkan()
|
||||
}
|
||||
}
|
||||
|
||||
// ⚠️ Android 路径不要调这个！这个函数销毁的是「第二阶段 init」期间创建的所有
|
||||
// 资源（含 renderPass / graphicsPipeline / pipelineLayout），但**没有**重置
|
||||
// _applicationInited / _sceondInited，也没有把销毁的句柄置 VK_NULL_HANDLE。
|
||||
//
|
||||
// 之前 main.cpp android_main 退出时调过它，结果用户在主界面切换镜头再进入
|
||||
// MakeupActivity 时 onWindowInit 看到 _applicationInited=1 走 reinit 复用路径
|
||||
// 引用悬空 renderPass，validation layer 检测出 use-after-free 触发 SEGV。
|
||||
//
|
||||
// Android 上的窗口生命周期是 cleanupForWindowLost() + reinitForNewWindow()，
|
||||
// renderPass / pipeline 跨 NativeActivity 实例由 g_Application 单例持有。
|
||||
// 这个函数目前只保留给 vulkan/main.cpp 那个独立 desktop 入口的进程退出路径。
|
||||
void Application::cleanupSecondInit()
|
||||
{
|
||||
vkDestroySwapchainKHR(device, swapChain, nullptr);
|
||||
vkDestroySurfaceKHR(instance, surface, nullptr);
|
||||
vkDestroyPipeline(device, graphicsPipeline, nullptr);
|
||||
vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
|
||||
vkDestroyRenderPass(device, renderPass, nullptr);
|
||||
for (auto imageView : swapChainImageViews) {
|
||||
vkDestroyImageView(device, imageView, nullptr);
|
||||
// ★ 幂等：销毁后立刻把句柄置 VK_NULL_HANDLE / 清空 vector。
|
||||
//
|
||||
// 这个函数过去在 Android 路径 (android_main 退出) 上被错误地调用过；
|
||||
// 由于销毁后没有清零句柄，紧接着重新进入的 reinitForNewWindow() 在
|
||||
// extent/format 不变时会跳过重建，让 framebuffer / drawFrame 在
|
||||
// 已经销毁的 renderPass 上继续工作，最终在
|
||||
// vkCmdBeginRenderPass 内部 (libGLES_mali) 触发 SIGSEGV。
|
||||
//
|
||||
// 现在 android_main 退出路径已经不再调本函数；本函数只剩 Windows
|
||||
// 路径 (mainLoop → cleanupSecondInit → initVulkan 的"二次 session"
|
||||
// 复用) 在用。把它写成幂等的可以兜住将来任何对它的误用。
|
||||
FACE_DBG_LOG("cleanupSecondInit: begin (swapChain=%s surface=%s renderPass=%s pipelineLayout=%s graphicsPipeline=%s imageViews=%zu framebuffers=%zu)",
|
||||
swapChain == VK_NULL_HANDLE ? "null" : "live",
|
||||
surface == VK_NULL_HANDLE ? "null" : "live",
|
||||
renderPass == VK_NULL_HANDLE ? "null" : "live",
|
||||
pipelineLayout == VK_NULL_HANDLE ? "null" : "live",
|
||||
graphicsPipeline == VK_NULL_HANDLE ? "null" : "live",
|
||||
swapChainImageViews.size(), swapChainFramebuffers.size());
|
||||
|
||||
if (device != VK_NULL_HANDLE) {
|
||||
vkDeviceWaitIdle(device);
|
||||
}
|
||||
|
||||
if (swapChain != VK_NULL_HANDLE) {
|
||||
vkDestroySwapchainKHR(device, swapChain, nullptr);
|
||||
swapChain = VK_NULL_HANDLE;
|
||||
}
|
||||
if (surface != VK_NULL_HANDLE) {
|
||||
vkDestroySurfaceKHR(instance, surface, nullptr);
|
||||
surface = VK_NULL_HANDLE;
|
||||
}
|
||||
if (graphicsPipeline != VK_NULL_HANDLE) {
|
||||
vkDestroyPipeline(device, graphicsPipeline, nullptr);
|
||||
graphicsPipeline = VK_NULL_HANDLE;
|
||||
}
|
||||
if (pipelineLayout != VK_NULL_HANDLE) {
|
||||
vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
|
||||
pipelineLayout = VK_NULL_HANDLE;
|
||||
}
|
||||
if (renderPass != VK_NULL_HANDLE) {
|
||||
vkDestroyRenderPass(device, renderPass, nullptr);
|
||||
renderPass = VK_NULL_HANDLE;
|
||||
}
|
||||
for (auto imageView : swapChainImageViews) {
|
||||
if (imageView != VK_NULL_HANDLE) {
|
||||
vkDestroyImageView(device, imageView, nullptr);
|
||||
}
|
||||
}
|
||||
swapChainImageViews.clear();
|
||||
for (auto framebuffer : swapChainFramebuffers)
|
||||
{
|
||||
if (framebuffer != VK_NULL_HANDLE)
|
||||
@@ -152,6 +207,7 @@ void Application::cleanupSecondInit()
|
||||
}
|
||||
}
|
||||
swapChainFramebuffers.clear();
|
||||
swapChainImages.clear();
|
||||
|
||||
_sceondInited = false;
|
||||
}
|
||||
|
||||
+244
-40
@@ -1,4 +1,4 @@
|
||||
#include "FaceApp.h"
|
||||
#include "FaceApp.h"
|
||||
#include "hardcode_data.h"
|
||||
#include "lodepng.h"
|
||||
#include <sstream>
|
||||
@@ -47,10 +47,13 @@ void FaceApp::Stop()
|
||||
void ReceiveFacePoint(float* pos, int pointCount, int width, int height)
|
||||
{
|
||||
FaceApp* self = FaceApp::Get();
|
||||
if (self != nullptr)
|
||||
if (self == nullptr)
|
||||
{
|
||||
FaceApp::Get()->update_face_vertex_buffer(pos, pointCount);
|
||||
FACE_DBG_LOG_THROTTLED("ReceiveFacePoint.noFaceApp",
|
||||
"ReceiveFacePoint dropped: FaceApp::Get() == nullptr");
|
||||
return;
|
||||
}
|
||||
self->update_face_vertex_buffer(pos, pointCount);
|
||||
}
|
||||
|
||||
// 定义帧数据结构
|
||||
@@ -60,10 +63,12 @@ struct FrameData {
|
||||
int height;
|
||||
int rowStride;
|
||||
size_t dataSize;
|
||||
int rotation; // CameraX 给的 rotationDegrees，用于 shader 端 UV 旋转
|
||||
bool mirrorX; // 前置摄像头时为 true，shader 在 gl_Position.x 翻转一次
|
||||
|
||||
FrameData(uint8_t* d, int w, int h, int rs, size_t ds)
|
||||
: data(nullptr), width(w), height(h), rowStride(rs), dataSize(ds) {
|
||||
// 深拷贝数据
|
||||
FrameData(uint8_t* d, int w, int h, int rs, size_t ds, int rot, bool mx)
|
||||
: data(nullptr), width(w), height(h), rowStride(rs), dataSize(ds),
|
||||
rotation(rot), mirrorX(mx) {
|
||||
if (d && ds > 0) {
|
||||
data = new uint8_t[dataSize];
|
||||
memcpy(data, d, dataSize);
|
||||
@@ -77,18 +82,16 @@ struct FrameData {
|
||||
}
|
||||
}
|
||||
|
||||
// 禁止拷贝构造和赋值（使用移动语义）
|
||||
FrameData(const FrameData&) = delete;
|
||||
FrameData& operator=(const FrameData&) = delete;
|
||||
|
||||
// 移动构造
|
||||
FrameData(FrameData&& other) noexcept
|
||||
: data(other.data), width(other.width), height(other.height),
|
||||
rowStride(other.rowStride), dataSize(other.dataSize) {
|
||||
rowStride(other.rowStride), dataSize(other.dataSize),
|
||||
rotation(other.rotation), mirrorX(other.mirrorX) {
|
||||
other.data = nullptr;
|
||||
}
|
||||
|
||||
// 移动赋值
|
||||
FrameData& operator=(FrameData&& other) noexcept {
|
||||
if (this != &other) {
|
||||
if (data) delete[] data;
|
||||
@@ -97,6 +100,8 @@ struct FrameData {
|
||||
height = other.height;
|
||||
rowStride = other.rowStride;
|
||||
dataSize = other.dataSize;
|
||||
rotation = other.rotation;
|
||||
mirrorX = other.mirrorX;
|
||||
other.data = nullptr;
|
||||
}
|
||||
return *this;
|
||||
@@ -108,7 +113,7 @@ std::queue<FrameData> frameQueue;
|
||||
std::mutex queueMutex;
|
||||
const int MAX_QUEUE_SIZE = 4;
|
||||
|
||||
void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize)
|
||||
void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX)
|
||||
{
|
||||
if (!FaceApp::Get()->isInited()) {
|
||||
return;
|
||||
@@ -116,24 +121,25 @@ void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowS
|
||||
|
||||
std::lock_guard<std::mutex> lock(queueMutex);
|
||||
|
||||
// 添加新帧数据到队列
|
||||
frameQueue.emplace(data, width, height, rowStride, dataSize);
|
||||
frameQueue.emplace(data, width, height, rowStride, dataSize, rotation, mirrorX);
|
||||
|
||||
// 如果队列大小达到阈值，开始处理最旧的一帧
|
||||
if (frameQueue.size() >= MAX_QUEUE_SIZE) {
|
||||
Texture& tex_bg = FaceApp::Get()->tex_bg;
|
||||
FrameData& oldestFrame = frameQueue.front();
|
||||
|
||||
FaceApp::Get()->processWithVulkan(
|
||||
// 走 FaceApp::processCameraFrame：相比 base::processWithVulkan，它多做两件
|
||||
// 事 —— ① 检测帧尺寸变化时 destroy/recreate tex_bg（不同手机或 CameraX
|
||||
// 重建 session 后第一帧尺寸可能与之前不同）；② 缓存 raw_aspect / rotation
|
||||
// / mirrorX 到 m_cameraAspect / m_cameraRotation / m_mirrorX，由 render()
|
||||
// 推到 push constant。
|
||||
FaceApp::Get()->processCameraFrame(
|
||||
oldestFrame.data,
|
||||
oldestFrame.width,
|
||||
oldestFrame.height,
|
||||
oldestFrame.rowStride,
|
||||
oldestFrame.dataSize,
|
||||
tex_bg,
|
||||
false,
|
||||
FaceApp::Get()->commandPool,
|
||||
"data_from_camera"
|
||||
oldestFrame.rotation,
|
||||
oldestFrame.mirrorX
|
||||
);
|
||||
|
||||
frameQueue.pop();
|
||||
@@ -298,15 +304,21 @@ void FaceApp::create_face_pipelines()
|
||||
VkPipelineRasterizationStateCreateInfo rasterization_state{};
|
||||
rasterization_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
|
||||
rasterization_state.polygonMode = VK_POLYGON_MODE_FILL;
|
||||
// ★ 关键：face 必须 NONE。原因：
|
||||
// 1. face mesh 是 2D 平面网格（texture.vert 让所有顶点 z=0.5），不存在
|
||||
// 自遮挡，没必要 cull；
|
||||
// 2. 前置摄像头镜子效果是在 vertex shader 里通过 NDC.x 翻转实现的
|
||||
// （`position.x *= (1.0 - 2.0 * pc.mirror_x)`）。X 翻转会把所有三角
|
||||
// 形的卷绕方向从 CCW 变 CW。如果这里 cullMode=BACK_BIT + frontFace=
|
||||
// CCW，前置时整张脸会被 cull，face 完全不可见——这正是历史踩坑点。
|
||||
// 所以请勿改回 BACK_BIT。如果未来要再加 cull，必须配两套 pipeline（前置
|
||||
// 用 CW、后置用 CCW），或在前置时改成 frontFace=CW，否则一定会复现这个 bug。
|
||||
rasterization_state.cullMode = VK_CULL_MODE_NONE;
|
||||
rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
|
||||
rasterization_state.flags = 0;
|
||||
rasterization_state.depthClampEnable = VK_FALSE;
|
||||
rasterization_state.lineWidth = 1.0f;
|
||||
|
||||
rasterization_state.cullMode = VK_CULL_MODE_BACK_BIT;
|
||||
rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
|
||||
|
||||
|
||||
|
||||
VkPipelineColorBlendAttachmentState colorBlendAttachment{};
|
||||
@@ -333,13 +345,16 @@ void FaceApp::create_face_pipelines()
|
||||
colorBlending.attachmentCount = 1;
|
||||
colorBlending.pAttachments = &colorBlendAttachment;
|
||||
|
||||
// Note: Using reversed depth-buffer for increased precision, so Greater depth values are kept
|
||||
// face 是 2D 平面网格（texture.vert 让顶点 z=0.5 一致），完全不需要 depth
|
||||
// test/write。bg 管线本来也是关 depth 的；保持两边一致，让"先 bg 再 face"
|
||||
// 的覆盖顺序由 draw 顺序而非 depth 决定，避免 reversed-depth（GREATER）+
|
||||
// renderpass 没显式 clear depth attachment 时引发的"face fragment 全部
|
||||
// 被 depth test fail"的隐性 bug。
|
||||
VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {};
|
||||
|
||||
depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
|
||||
depth_stencil_state.depthTestEnable = VK_TRUE;
|
||||
depth_stencil_state.depthWriteEnable = VK_TRUE;
|
||||
depth_stencil_state.depthCompareOp = VK_COMPARE_OP_GREATER;
|
||||
depth_stencil_state.depthTestEnable = VK_FALSE;
|
||||
depth_stencil_state.depthWriteEnable = VK_FALSE;
|
||||
depth_stencil_state.depthCompareOp = VK_COMPARE_OP_ALWAYS;
|
||||
depth_stencil_state.front = depth_stencil_state.back;
|
||||
depth_stencil_state.back.compareOp = VK_COMPARE_OP_ALWAYS;
|
||||
|
||||
@@ -617,6 +632,8 @@ void FaceApp::setup_descriptor_set()
|
||||
|
||||
void FaceApp::update_descriptor_set(vector<Texture>& texs, vector<VkDescriptorSet>& descriptSet)
|
||||
{
|
||||
FACE_DBG_LOG("update_descriptor_set: texs.size=%zu descriptSet.size=%zu",
|
||||
texs.size(), descriptSet.size());
|
||||
for (int i = 0; i < texs.size(); ++i)
|
||||
{
|
||||
if (texs[i].image == VK_NULL_HANDLE)
|
||||
@@ -697,21 +714,84 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
|
||||
|
||||
//Application::render(commandBuffer);
|
||||
|
||||
// === 分辨率适配：每帧把"业务参数"缩放到当前屏幕物理像素 ===
|
||||
// 业务层（MotionManager / InitArg）传进来的 radius / offset_x / offset_y 一直
|
||||
// 都是按"480x480 设计画布"给的，比如 radius=240 表示在 480 屏上半径 240px
|
||||
// （即 50% 短边）。现在屏幕变成任意分辨率后，要保持视觉一致：
|
||||
// 实际像素 = 业务值 * (画布短边 / 480)
|
||||
// 同时把 canvas/screen 几何写进 push constant，shader 用它们：
|
||||
// 1) 把"画布 NDC ∈ [-1,+1]" 缩到屏幕中央正方形（letterbox）；
|
||||
// 2) 把 fragment 里的圆心从写死的 (240,240) 改成屏幕真实中心。
|
||||
//
|
||||
// 注意 pushConstants 本身保留业务原始值不动（SetInitArg 设进来的），
|
||||
// 这里只把缩放后的副本 pc 推给 GPU。下面 ux/uy 仍然写回 pushConstants，
|
||||
// 是因为它们和分辨率无关，是 motion 帧的纹理坐标。
|
||||
updateCanvasMetrics();
|
||||
PushConstants pc = pushConstants;
|
||||
const float k = m_canvasSize / 480.0f;
|
||||
pc.radius *= k;
|
||||
pc.offset_x *= k;
|
||||
pc.offset_y *= k;
|
||||
pc.canvas_size = m_canvasSize;
|
||||
pc.screen_w = m_screenW;
|
||||
pc.screen_h = m_screenH;
|
||||
// 相机帧元信息：由 processCameraFrame 在每帧上传纹理之前更新。第一帧之前
|
||||
// （还没收到相机数据时）m_cameraAspect/m_cameraRotation 是构造函数设的
|
||||
// 4:3 + 90° 默认值 —— 这两个值正是 CameraX RATIO_4_3 + 大部分 Android 后置
|
||||
// 主摄竖屏的典型组合，所以即使首帧 bg 用默认值渲染也不会出现明显错位。
|
||||
pc.camera_aspect = m_cameraAspect;
|
||||
pc.camera_rotation = m_cameraRotation;
|
||||
// mirror_x: 前置摄像头时 = 1.0，所有顶点 shader 在最后输出 gl_Position 前
|
||||
// 把 NDC.x 翻转一次，达到镜子效果（用户右手 → 屏幕左侧）。bg 与 face 都
|
||||
// 同步翻转，对齐保持。后置时 = 0.0，shader 公式 (1 - 2 * mirror_x) = 1 短路。
|
||||
pc.mirror_x = m_mirrorX;
|
||||
|
||||
vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout_bg, 0, 1, &m_descriptor_set_bg, 0, NULL);
|
||||
vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_graphicsPipeline_bg);
|
||||
vkCmdPushConstants(commandBuffer, m_pipelineLayout_bg, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(pushConstants), &pushConstants);
|
||||
// bg.frag 也读 push constant（r/g/b/radius/screen_w/screen_h）所以 stage 必须
|
||||
// 同时包含 fragment；老代码这里只写了 VERTEX_BIT，是个隐藏的 spec 违规。
|
||||
vkCmdPushConstants(commandBuffer, m_pipelineLayout_bg, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc);
|
||||
|
||||
|
||||
|
||||
vkCmdDraw(commandBuffer, 6, 1, 0, 0);
|
||||
|
||||
// 诊断：把 face 渲染所有"前置门"状态打成一行节流日志。一旦 face 没出现，
|
||||
// 直接看这条就能知道断在哪一关：
|
||||
// _isChangeMostion=0 → 业务还没调 PlayMotionList / changeMotionList
|
||||
// _curMotions=0 → motion list 是空的（getMotionByName 全 miss）
|
||||
// _curMotionIndex 越界 → 业务逻辑或 callback 状态机错了
|
||||
// delta_last_update >= 2000 → mediapipe / passDataToNative 链路 2s 没动静
|
||||
const long long now_ms = getCurrentTimeMillis();
|
||||
const long long delta_lu = now_ms - (long long)last_update_time;
|
||||
FACE_DBG_LOG_THROTTLED("render.face.gate",
|
||||
"render.face gate _isChangeMostion=%d _curMotions=%zu"
|
||||
" _curMotionIndex=%d _curFrameIndex=%d delta_lastUpdate=%lldms"
|
||||
" _playMotion=%d",
|
||||
(int)_isChangeMostion, _curMotions.size(),
|
||||
(int)_curMotionIndex, (int)_curFrameIndex,
|
||||
delta_lu, (int)_playMotion);
|
||||
|
||||
if (!_isChangeMostion)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_curMotions.empty())
|
||||
{
|
||||
FACE_DBG_LOG_THROTTLED("render.face.emptyMotions",
|
||||
"render.face dropped: _isChangeMostion=1 but _curMotions is empty");
|
||||
return;
|
||||
}
|
||||
|
||||
string curMotionName = _curMotions[_curMotionIndex].name;
|
||||
int loadMotionIndex = motion_list_map[curMotionName];
|
||||
|
||||
// face draw 前关键参数节流日志，保留少量"任何时候挂了能立刻判断断点"的字段。
|
||||
FACE_DBG_LOG_THROTTLED("render.face.preDraw",
|
||||
"render.face preDraw motion='%s' loadMotionIdx=%d obj_idx=%zu mirror=%.1f",
|
||||
curMotionName.c_str(), loadMotionIndex, obj_indices.size(), pc.mirror_x);
|
||||
|
||||
//if (cur_left)
|
||||
//{
|
||||
vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, &m_descriptor_sets_left[loadMotionIndex], 0, NULL);
|
||||
@@ -730,12 +810,14 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
|
||||
float uy = _curMotions[_curMotionIndex].frames[_curFrameIndex].y;
|
||||
pushConstants.ux = ux;
|
||||
pushConstants.uy = uy;
|
||||
pc.ux = ux;
|
||||
pc.uy = uy;
|
||||
}
|
||||
}
|
||||
//fsValues[1] = 0;
|
||||
//fsValues[2] = 360;
|
||||
|
||||
vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pushConstants), &pushConstants);
|
||||
vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc);
|
||||
//vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(fsValues), fsValues);
|
||||
|
||||
|
||||
@@ -751,7 +833,16 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
|
||||
#else
|
||||
if (getCurrentTimeMillis() - last_update_time < 2000)
|
||||
{
|
||||
vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0);
|
||||
FACE_DBG_LOG_THROTTLED("render.face.draw",
|
||||
"render.face vkCmdDrawIndexed indices=%zu motion=%s frame=%d",
|
||||
obj_indices.size(), curMotionName.c_str(), (int)_curFrameIndex);
|
||||
vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
FACE_DBG_LOG_THROTTLED("render.face.heartbeatStale",
|
||||
"render.face SKIP vkCmdDrawIndexed: delta_last_update=%lldms (>= 2000ms)",
|
||||
(long long)(getCurrentTimeMillis() - last_update_time));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -964,11 +1055,14 @@ void FaceApp::recreatePipelinesForSwapchain()
|
||||
|
||||
void FaceApp::update_uniform_buffers()
|
||||
{
|
||||
uint32_t width = 480;
|
||||
uint32_t height = 480;
|
||||
float zoom = 2;
|
||||
// 注意：当前 vertex shader (texture.vert) 实际并没有使用 ubo.projection 把模型
|
||||
// 投影到屏幕——它直接拿 inPos.xy*2-1 当 NDC 用。所以下面 aspect 取 1.0 只是
|
||||
// 为了让 ubo 不再依赖原本写死的 480x480 输入分辨率；要真正影响屏幕显示比例的
|
||||
// 是 push constant 里的 canvas_size / screen_w / screen_h，由 render() 写入。
|
||||
const float aspect = 1.0f;
|
||||
const float zoom = 2.0f;
|
||||
// Vertex shader
|
||||
ubo_vs.projection = glm::perspective(glm::radians(60.0f), static_cast<float>(width) / static_cast<float>(height), 0.001f, 256.0f);
|
||||
ubo_vs.projection = glm::perspective(glm::radians(60.0f), aspect, 0.001f, 256.0f);
|
||||
glm::mat4 view_matrix = glm::translate(glm::mat4(1.0f), glm::vec3(0.0f, 0.0f, zoom));
|
||||
|
||||
ubo_vs.model = view_matrix * glm::translate(glm::mat4(1.0f), camera_pos);
|
||||
@@ -981,6 +1075,26 @@ void FaceApp::update_uniform_buffers()
|
||||
memcpy(uniform_buffer_mapped, &ubo_vs, sizeof(ubo_vs));
|
||||
}
|
||||
|
||||
void FaceApp::updateCanvasMetrics()
|
||||
{
|
||||
// 把"屏幕中央 min(w,h) 的正方形"作为设计画布。画布外的部分会落在
|
||||
// shader 计算的 NDC ∈ [-1,+1] 之外，被 GPU 自动裁剪为黑色 letterbox。
|
||||
//
|
||||
// swapChainExtent 来自 Application::createSwapChain 里的
|
||||
// surface_capabilities.currentExtent，即当前窗口/Surface 的真实分辨率。
|
||||
// 任何 swapchain 重建（onWindowLost -> onWindowInit）都会刷新它，所以
|
||||
// 这里每帧重算是廉价且自洽的。
|
||||
m_screenW = (float)swapChainExtent.width;
|
||||
m_screenH = (float)swapChainExtent.height;
|
||||
m_canvasSize = (m_screenW < m_screenH) ? m_screenW : m_screenH;
|
||||
if (m_canvasSize <= 0.0f) {
|
||||
// swapchain 还没初始化 / 已销毁时给个安全值，避免 shader 除零。
|
||||
m_canvasSize = 480.0f;
|
||||
m_screenW = 480.0f;
|
||||
m_screenH = 480.0f;
|
||||
}
|
||||
}
|
||||
|
||||
void FaceApp::createVertexBuffer()
|
||||
{
|
||||
VkDeviceSize vertexBufferSize = sizeof(TextureLoadingVertexStructure) * obj_vertices.size();
|
||||
@@ -1029,17 +1143,85 @@ void FaceApp::uploadVertexData() {
|
||||
|
||||
|
||||
|
||||
void FaceApp::processCameraFrame(uint8_t* data, int width, int height,
|
||||
int rowStride, size_t dataSize, int rotation,
|
||||
bool mirrorX)
|
||||
{
|
||||
if (!isInited() || data == nullptr || width <= 0 || height <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 1) rotation 收敛到 0/90/180/270。CameraX 文档保证只会给这四个值，
|
||||
// 但理论上 ((rotation % 360) + 360) % 360 更鲁棒。
|
||||
int rotNorm = ((rotation % 360) + 360) % 360;
|
||||
if (rotNorm != 0 && rotNorm != 90 && rotNorm != 180 && rotNorm != 270) {
|
||||
// 非法值时保守用 90（与原 hardcode 一致）。
|
||||
rotNorm = 90;
|
||||
}
|
||||
|
||||
// 2) 元信息更新（无锁单写者：仅相机分析线程写，render 线程读）。
|
||||
// 放在 sizeChanged 判断前后都安全：render() 每帧重新拷到 push constant。
|
||||
m_cameraAspect = (float)width / (float)height;
|
||||
m_cameraRotation = (float)rotNorm;
|
||||
m_mirrorX = mirrorX ? 1.0f : 0.0f;
|
||||
|
||||
// 3) 检查帧尺寸是否与现有 tex_bg 一致。
|
||||
// 场景：CameraX session 重建（如 Activity onResume）后第一帧分辨率可能
|
||||
// 不同；或者 setTargetResolution 在不同设备上落地为不同尺寸。
|
||||
bool sizeChanged = (tex_bg.image != VK_NULL_HANDLE)
|
||||
&& (tex_bg.width != width || tex_bg.height != height);
|
||||
|
||||
if (!sizeChanged) {
|
||||
// 快路径（每帧）：直接走 base 标准上传路径。第一次 image == VK_NULL_HANDLE
|
||||
// base 会自动 createTexture(width, height, ...) 自适应分辨率；之后每帧
|
||||
// image 已存在，直接走 staging buffer 拷贝。
|
||||
processWithVulkan(data, width, height, rowStride, dataSize,
|
||||
tex_bg, false, commandPool, "data_from_camera");
|
||||
return;
|
||||
}
|
||||
|
||||
// 慢路径（罕见，整个 session 通常 1 次）：尺寸变化 → 重建 image + 刷新
|
||||
// descriptor set。整个 ceremony 必须严格串行：
|
||||
// ① createTextureMtx 与 drawFrame / 其它 processWithVulkan 互斥
|
||||
// （FaceApp::drawFrame 进 Application::drawFrame 之前会持此锁，所以
|
||||
// 我们持锁期间渲染线程被阻塞，不会有新的 cmdbuf 绑定 m_descriptor_set_bg
|
||||
// 并 submit）。
|
||||
// ② vkDeviceWaitIdle 等待所有"已 submit 但未完成"的 cmdbuf 跑完，确保
|
||||
// 销毁旧 image/view 时 GPU 已经不再采样它。
|
||||
// ③ destroy → create → updateTexture → refresh_descriptor_set_bg
|
||||
// 全部在锁内完成，期间 m_descriptor_set_bg 处于"指向无效 view"的中间
|
||||
// 状态，但因为渲染线程被锁阻塞，这个中间状态对 GPU 不可见。
|
||||
// 注意不能调 base::processWithVulkan，那会再次 lock createTextureMtx 死锁；
|
||||
// 直接调 createTexture + updateTexture，这两个 base 方法不持 createTextureMtx。
|
||||
#ifndef _WIN32
|
||||
DebugLog::log("processCameraFrame: bg size changed %dx%d -> %dx%d, recreate texture",
|
||||
tex_bg.width, tex_bg.height, width, height);
|
||||
#endif
|
||||
std::unique_lock<std::mutex> lk_tex(createTextureMtx);
|
||||
{
|
||||
std::lock_guard<std::mutex> lk_pool(poolQueueMtx);
|
||||
vkDeviceWaitIdle(device);
|
||||
}
|
||||
destroyTexture(device, tex_bg);
|
||||
createTexture(device, physicalDevice, width, height, tex_bg, false,
|
||||
"data_from_camera", dataSize);
|
||||
updateTexture(device, physicalDevice, commandPool, graphicsQueue,
|
||||
data, width, height, rowStride, dataSize, tex_bg);
|
||||
refresh_descriptor_set_bg();
|
||||
}
|
||||
|
||||
void FaceApp::update_face_vertex_buffer(float* pos, int pointCount)
|
||||
{
|
||||
if(!isInited())
|
||||
{
|
||||
FACE_DBG_LOG_THROTTLED("update_face_vertex_buffer.notInited",
|
||||
"update_face_vertex_buffer dropped: !isInited()");
|
||||
return;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(mtx_point);
|
||||
last_update_time = getCurrentTimeMillis();
|
||||
|
||||
|
||||
for (int i = 0; i < obj_vertices.size(); ++i)
|
||||
{
|
||||
|
||||
@@ -1251,8 +1433,12 @@ void FaceApp::setup_descriptor_set_layout_bg()
|
||||
pipeline_layout_create_info.pSetLayouts = &m_descriptorSetLayout_bg;
|
||||
|
||||
|
||||
// bg.frag 同样要读 push constant（r/g/b/radius，以及分辨率适配重构后的
|
||||
// screen_w/screen_h），所以 stage 必须包含 FRAGMENT_BIT。老代码这里只写了
|
||||
// VERTEX_BIT，按 Vulkan spec 是非法的（验证层会 warning，部分驱动会读到
|
||||
// 未定义内容），借这次重构修掉。
|
||||
VkPushConstantRange pushConstantRanges[1];
|
||||
pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
|
||||
pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT;
|
||||
pushConstantRanges[0].offset = 0;
|
||||
pushConstantRanges[0].size = sizeof(PushConstants);
|
||||
|
||||
@@ -1273,7 +1459,18 @@ void FaceApp::setup_descriptor_set_bg()
|
||||
|
||||
VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, &m_descriptor_set_bg));
|
||||
|
||||
refresh_descriptor_set_bg();
|
||||
}
|
||||
|
||||
void FaceApp::refresh_descriptor_set_bg()
|
||||
{
|
||||
if (m_descriptor_set_bg == VK_NULL_HANDLE) {
|
||||
return;
|
||||
}
|
||||
if (tex_bg.view == VK_NULL_HANDLE || tex_bg.sampler == VK_NULL_HANDLE) {
|
||||
// 还没 loadTexture / processCameraFrame 过，没法绑定。
|
||||
return;
|
||||
}
|
||||
|
||||
VkDescriptorImageInfo image_descriptor;
|
||||
image_descriptor.imageView = tex_bg.view;
|
||||
@@ -1288,9 +1485,7 @@ void FaceApp::setup_descriptor_set_bg()
|
||||
write_descriptor_set.pImageInfo = &image_descriptor;
|
||||
write_descriptor_set.descriptorCount = 1;
|
||||
|
||||
std::vector<VkWriteDescriptorSet> write_descriptor_sets = { write_descriptor_set };
|
||||
|
||||
vkUpdateDescriptorSets(device, static_cast<uint32_t>(write_descriptor_sets.size()), write_descriptor_sets.data(), 0, NULL);
|
||||
vkUpdateDescriptorSets(device, 1, &write_descriptor_set, 0, NULL);
|
||||
}
|
||||
|
||||
void FaceApp::destroyTexture(VkDevice device, Texture& texture) {
|
||||
@@ -1604,6 +1799,7 @@ void FaceApp::loadMotionThread()
|
||||
string name = m.name;
|
||||
if (motion_list_map.find(name) != motion_list_map.end())
|
||||
{
|
||||
FACE_DBG_LOG("loadMotionThread: skip already-loaded motion='%s'", name.c_str());
|
||||
continue;
|
||||
}
|
||||
m_texs_left.push_back(Texture());
|
||||
@@ -1620,6 +1816,10 @@ void FaceApp::loadMotionThread()
|
||||
#endif // _WIN32
|
||||
motion_list_map[name] = m_texs_left.size() - 1;
|
||||
_loadMotionMap[name] = m;
|
||||
FACE_DBG_LOG("loadMotionThread: loaded motion='%s' idx=%zu tex_image=%p tex_view=%p tex_sampler=%p frames=%zu",
|
||||
name.c_str(), m_texs_left.size() - 1,
|
||||
(void*)newTex.image, (void*)newTex.view, (void*)newTex.sampler,
|
||||
m.frames.size());
|
||||
}
|
||||
|
||||
// for (int i = 0; i < _loadMotions.size(); ++i)
|
||||
@@ -1680,7 +1880,11 @@ void FaceApp::changeMotionList(vector<string> motions, AnimationFinishedCallback
|
||||
vector<Motion> motion_list;
|
||||
for (auto ms : motions) {
|
||||
Motion m = getMotionByName(ms);
|
||||
motion_list.push_back(m);
|
||||
auto it = motion_list_map.find(ms);
|
||||
int idx = (it == motion_list_map.end()) ? -1 : (int)it->second;
|
||||
FACE_DBG_LOG("changeMotionList: requested='%s' idx=%d frames=%zu (-1=NOT FOUND)",
|
||||
ms.c_str(), idx, m.frames.size());
|
||||
motion_list.push_back(m);
|
||||
}
|
||||
|
||||
_curMotions = motion_list;
|
||||
|
||||
@@ -49,6 +49,15 @@ struct InitArg
|
||||
};
|
||||
|
||||
|
||||
// 推送给 shader 的常量。前 9 个 float 是「业务原始值」，单位以 480x480 设计画布
|
||||
// 为基准（向后兼容）：业务给 radius=240 / offset_x=200 / offset_y=-200 等都按这个画布
|
||||
// 思考。后 3 个 float 是 SDK 在每次 render() 时根据当前 swapchain 尺寸计算并写入的
|
||||
// 「画布几何」，shader 通过它把「画布坐标」映射到「屏幕坐标」并把圆形 mask 中心
|
||||
// 锚定在屏幕中心。
|
||||
//
|
||||
// ★ 字段顺序和命名必须和 GLSL 中的 layout(push_constant) 块严格一致，否则会读到
|
||||
// 错位数据。修改时四个 shader (bg.vert / bg.frag / texture.vert / texture.frag)
|
||||
// 都要同步更新。
|
||||
struct PushConstants {
|
||||
float zoom = 0.5f;
|
||||
float r = 0;
|
||||
@@ -59,6 +68,32 @@ struct PushConstants {
|
||||
float uy = 0;
|
||||
float offset_x = 0;
|
||||
float offset_y = 0;
|
||||
// 由 SDK 在 FaceApp::render() 内每帧写入，业务无感。
|
||||
// canvas_size = min(swapchain.w, swapchain.h)，单位：物理像素
|
||||
// screen_w / screen_h = swapchain 当前实际尺寸，单位：物理像素
|
||||
// 设计画布始终居中铺在屏幕中央正方形区域；外围是 letterbox 黑边。
|
||||
float canvas_size = 480.0f;
|
||||
float screen_w = 480.0f;
|
||||
float screen_h = 480.0f;
|
||||
|
||||
// 相机帧元信息，由 SDK 在收到第一帧 / 帧尺寸变化时更新。shader 用来：
|
||||
// 1) 把相机帧（任意 raw aspect）正确 cover 到 1:1 画布；
|
||||
// 2) 按 camera_rotation 旋转 UV，让画面在屏幕上正立——这是不同手机的
|
||||
// sensor orientation 不同导致画面侧着 / 倒着的根因。
|
||||
// camera_aspect = camera_raw_w / camera_raw_h（典型 4:3 = 1.333…）
|
||||
// camera_rotation = CameraX 的 ImageInfo.getRotationDegrees()，
|
||||
// 表示「图像顺时针旋转多少度才能正立」，离散 0/90/180/270。
|
||||
// 默认值 4/3 + 90° 是兼容老 hardcode 行为（大部分 Android 手机后置主摄竖屏）。
|
||||
float camera_aspect = 4.0f / 3.0f;
|
||||
float camera_rotation = 90.0f;
|
||||
|
||||
// 镜像标志：1.0 表示前置摄像头需要"镜子效果"，所有顶点 shader 会把
|
||||
// gl_Position.x 翻转一次（NDC 水平翻转）。配合 FaceLandmarkerHelper 在
|
||||
// bitmap 阶段做的 postScale(-1,1)，bg 与 face 同步翻转、互相对齐。
|
||||
// 后置摄像头此值为 0.0，shader 短路无开销。
|
||||
// 用 float 而不是 bool/int 是为了和 Vulkan push constant 4-byte 对齐 + GLSL
|
||||
// 端 layout 简单一致；shader 里 `1.0 - 2.0 * mirror_x` 实现无分支翻转。
|
||||
float mirror_x = 0.0f;
|
||||
};
|
||||
|
||||
using Callback = std::function<void()>;
|
||||
@@ -163,13 +198,64 @@ private:
|
||||
|
||||
|
||||
//float myFloatValue = 1.5f;
|
||||
// 「业务原始值」缓存：SetInitArg 把 InitArg 拷到这里，render() 每帧读出来再
|
||||
// 缩放 + 注入屏幕几何。注意 ux/uy 由 motion 帧每帧写入，也保留在这里。
|
||||
PushConstants pushConstants;
|
||||
|
||||
// ===== 屏幕分辨率适配 =====
|
||||
// 原 SDK 把屏幕写死 480x480，所有业务参数都按 480 像素画布给。现在改成支持
|
||||
// 任意分辨率：把"屏幕中央 min(w,h) 边长的正方形"作为设计画布，画布外是黑边。
|
||||
// 业务参数语义不变，SDK 在 render() 里按 (canvas_size / 480) 缩放成屏幕物理
|
||||
// 像素后再推给 shader。
|
||||
// m_canvasSize = min(swapChainExtent.w, swapChainExtent.h)
|
||||
// m_screenW / m_screenH = swapChainExtent.w / swapChainExtent.h
|
||||
// updateCanvasMetrics() 由 render() 在每次 vkCmdPushConstants 之前调用。
|
||||
// 因为 swapchain 重建会走 onWindowLost -> onWindowInit ->
|
||||
// recreatePipelinesForSwapchain，而 render() 永远在 drawFrame 里被调，所以
|
||||
// 这套数值天然会跟随 swapchain 变化。
|
||||
float m_canvasSize = 480.0f;
|
||||
float m_screenW = 480.0f;
|
||||
float m_screenH = 480.0f;
|
||||
void updateCanvasMetrics();
|
||||
|
||||
// ===== 相机帧元信息适配 =====
|
||||
// 不同手机摄像头物理分辨率和 sensor orientation 都不一样，CameraX 给的图像
|
||||
// 既不一定是精确 4:3，也不一定是「正立」朝向。SDK 在这里缓存最近一帧的
|
||||
// (raw_aspect, rotation)，render() 时塞进 push constant 让 shader 自己处理：
|
||||
// - bg.vert 按 m_cameraRotation 旋转 UV，让画面在屏幕上方向正确
|
||||
// - bg.vert 按 m_cameraAspect 自适应 cover 画布，避免拉扁/拉长
|
||||
// 由 processCameraFrame() 根据每次相机帧实际 width / height / rotation 更新。
|
||||
// 第一帧之前用安全默认值（4:3 + 90°，对应大部分 Android 后置主摄竖屏行为）。
|
||||
float m_cameraAspect = 4.0f / 3.0f;
|
||||
float m_cameraRotation = 90.0f;
|
||||
// 镜像标志缓存：与 m_cameraAspect 同写者（相机分析线程），由 render() 拷到
|
||||
// push constant 的 mirror_x 字段。前置=1.0，后置=0.0。
|
||||
float m_mirrorX = 0.0f;
|
||||
public:
|
||||
// JNI 入口（main.cpp）调用，用来在每帧上传相机纹理之前同步元信息。
|
||||
// 内部会：
|
||||
// 1) 检测 width × height 与已有 tex_bg 是否一致，不一致就 destroy + recreate；
|
||||
// 2) 更新 m_cameraAspect / m_cameraRotation / m_mirrorX；
|
||||
// 3) 调用 base 的 processWithVulkan 走 staging buffer 上传。
|
||||
// mirrorX 来自 Java 层 lensFacing == LENS_FACING_FRONT，表示是否需要镜子效果。
|
||||
// 必须由唯一的相机分析线程调用（FaceActivity backgroundExecutor），
|
||||
// 内部依赖 createTextureMtx 序列化与 drawFrame / cleanup 的并发。
|
||||
void processCameraFrame(uint8_t* data, int width, int height,
|
||||
int rowStride, size_t dataSize, int rotation,
|
||||
bool mirrorX);
|
||||
private:
|
||||
|
||||
|
||||
|
||||
void create_pipelines_bg();
|
||||
void setup_descriptor_set_layout_bg();
|
||||
void setup_descriptor_set_bg();
|
||||
// 把 m_descriptor_set_bg 重新绑定到当前的 tex_bg.view / sampler。
|
||||
// 当相机分辨率变化导致 tex_bg 被 destroy + recreate 后，新建的 image
|
||||
// 有新的 VkImageView 句柄，旧的 descriptor set 里缓存的 handle 失效，
|
||||
// 必须 vkUpdateDescriptorSets 让 descriptor 指向新 view。该函数不重新
|
||||
// 分配 descriptor set，仅刷新绑定。
|
||||
void refresh_descriptor_set_bg();
|
||||
VkPipeline m_graphicsPipeline_bg = VK_NULL_HANDLE;
|
||||
VkPipelineLayout m_pipelineLayout_bg = VK_NULL_HANDLE;
|
||||
VkDescriptorSetLayout m_descriptorSetLayout_bg = VK_NULL_HANDLE;
|
||||
|
||||
Reference in New Issue
Block a user