63 Commits
Author SHA1 Message Date
xsl 4e25641522 save code 2026-05-05 22:37:49 +08:00
xsl bd1b7e66c8 save code 2026-05-05 21:44:42 +08:00
xsl f8b1e567c1 save code 2026-04-27 22:15:09 +08:00
xsl b753fd36a2 修复切换出现的 clash 2026-04-26 22:42:10 +08:00
xsl c5ec7c7dc5 save code 2026-04-26 00:12:56 +08:00
xsl 50680ee854 修复clash的问题 2026-04-25 16:42:20 +08:00
xsl 2066b3b124 AI 修复横竖屏的问题 2026-04-23 23:43:48 +08:00
xsl 27b7075818 fix bug and add log 2026-04-23 22:18:22 +08:00
xsl fb434f09b8 添加调试日志 2026-04-23 11:29:18 +08:00
xsl 7ab4274e4f 整理卡死的问题 2026-04-21 00:03:27 +08:00
xsl 5d8e108306 fix crash bug 2026-02-05 21:06:02 +08:00
xsl 52c9da08ed save code 2026-01-13 08:12:46 +08:00
xsl 496d56f9c6 save code 2026-01-08 21:52:28 +08:00
xsl 60b40bfd5f 添加圆心遮罩 2026-01-08 21:19:32 +08:00
xsl b44162e6cb 修改背景图为透明 2025-12-26 16:23:07 +08:00
xsl 6ef6e012bd save code 2025-12-24 21:01:14 +08:00
xsl f5b2d490c0 save code 2025-12-24 10:55:30 +08:00
xsl 4c5a05ae21 asdf 2025-12-24 10:20:53 +08:00
xsl 1f9e4939b9 修改为高分辨率 2025-12-22 15:55:43 +08:00
xsl c83f36c976 修复 动画播放 错误的bug 2025-12-22 14:55:00 +08:00
xsl 29e0c1e9a8 完成动画序列播放 2025-12-22 00:40:45 +08:00
xsl 7b07e7fbc1 save code 2025-12-20 22:34:23 +08:00
xsl b6ac86a134 save code 2025-12-19 18:55:44 +08:00
xsl 6bbbb3c3ef save code 2025-12-19 13:47:16 +08:00
xsl e38da96e37 save code 2025-12-18 22:05:10 +08:00
xsl 9ee028248d save code 2025-12-17 22:58:30 +08:00
xsl 84cc693593 保存代码 2025-12-16 22:15:42 +08:00
xsl 4cf1990b16 save code 2025-12-12 10:27:49 +08:00
xsl 2130cf0e6b save 2025-12-12 09:58:03 +08:00
xsl 6717047d1d save code 2025-12-12 09:49:56 +08:00
xsl da2c97e412 修改检测参数,降低置信度,提高检测的延时 2025-12-06 11:59:00 +08:00
xsl db80a5dbbd asdf 2025-12-02 20:36:32 +08:00
xsl 634d22f3be 完成动画切换 2025-11-30 12:22:43 +08:00
xsl 835f768a4a 基本调通 回调 2025-11-29 23:44:22 +08:00
xsl 1859706412 优化内存读取 2025-11-29 11:35:56 +08:00
xsl 9c71e40c97 新版本f1118 更新图片 修复_running bug 2025-11-17 22:36:59 +08:00
xsl 84d567674f 化妆切换完成 2025-11-12 15:13:55 +08:00
xsl 43b00c8446 添加是否运行的标致 2025-11-12 15:05:31 +08:00
xsl e23ccd75cd 清理所有资源,有两处内存泄漏,暂时找不到了 2025-11-12 15:03:15 +08:00
xsl 8623b71ab7 总算是编译通过了 2025-11-12 10:53:41 +08:00
xsl 16efcbcff3 现在才真的能打包正确的 aar 2025-11-09 10:25:43 +08:00
xsl 4f62081239 生成aar文件成功 2025-11-08 20:32:31 +08:00
xsl 9f29667b73 更新编译版本为 甲方提供的版本 2025-11-08 20:10:49 +08:00
xsl 0c7455650d 更新 新版本贴图 2025-11-06 22:24:58 +08:00
xsl 00aabfc6d9 发布f20251107 版本 2025-11-06 21:28:34 +08:00
xsl 02a005f5b1 动画调整为30帧 2025-11-02 15:25:20 +08:00
xsl 6495a27658 完成动画在设备上播放和切换 2025-11-01 14:56:03 +08:00
xsl 263a061d27 处理图片的缩放 2025-10-31 15:16:27 +08:00
xsl 2555047190 动画编码完成 2025-10-31 14:22:08 +08:00
xsl dfbef7caf1 36张图片打包 2025-10-29 21:38:03 +08:00
xsl 13357ab256 android 平台跑通中文路径 2025-10-28 22:24:24 +08:00
xsl 1dfc63543f 添加中文解决的代码 2025-10-28 18:16:41 +08:00
xsl 10505c616f 修改为单纹理,在android上编译通过,apk修改为日期名字 2025-10-27 21:48:15 +08:00
xsl 300340bb19 延时优化完成 2025-10-26 22:42:05 +08:00
xsl 57b7c40835 添加了动画,资源也改为英文名了 2025-10-26 21:54:34 +08:00
xsl b2da84f903 save code 2025-10-26 00:13:08 +08:00
xsl 41a659bd6d 完成双贴图效果,修改闪退的bug,换成赴宴的贴图 2025-10-23 21:58:11 +08:00
xsl afd02550a8 添加非线性颜色过滤 2025-10-23 21:30:24 +08:00
xsl b73e94b7bf 修改了 2025-10-23 21:16:10 +08:00
xsl 2876599568 添加Example 模块,并且在设备上跑通双纹理效果 2025-10-22 22:11:01 +08:00
xsl f6dd3ab38e 添加双纹理采样 2025-10-22 18:16:03 +08:00
xsl ce2444bd07 输出aar 到新项目,运行成功 2025-10-22 00:07:53 +08:00
xsl 25fd6b4a32 终于打包成功aar 2025-10-21 18:56:09 +08:00
732 changed files with 37892 additions and 1172 deletions
+11
View File
@@ -0,0 +1,11 @@
{
"permissions": {
"allow": [
"Bash(awk '/^vt / {print $2, $3}' D:/face_sdk/app/src/main/assets/face_picture_3dmax.obj)",
"Bash(awk 'BEGIN{minu=99;maxu=-99;minv=99;maxv=-99} {if\\($1<minu\\)minu=$1; if\\($1>maxu\\)maxu=$1; if\\($2<minv\\)minv=$2; if\\($2>maxv\\)maxv=$2} END{print \"u:\",minu,maxu,\" v:\",minv,maxv}')",
"Bash(python -c \"from PIL import Image; print\\(Image.open\\('D:/face_sdk/example/src/main/assets/pic/4ex.png'\\).size\\)\")",
"Bash(python -c ' *)",
"Bash(python -c \"import sys; print\\(sys.executable\\)\")"
]
}
}
+1
View File
@@ -12,4 +12,5 @@
/captures
.externalNativeBuild
.cxx
.idea
local.properties
+16 -9
View File
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.22.1)
project("sample")
project("face_sdk")
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -23,9 +23,13 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android")
third_party/vma/include
)
add_library(sample SHARED
add_library(face_sdk SHARED
app/src/main/cpp/main.cpp
app/src/main/cpp/AndroidOut.cpp
app/src/main/cpp/DebugLog.h
app/src/main/cpp/DebugLog.cpp
app/src/main/cpp/CrashHandler.h
app/src/main/cpp/CrashHandler.cpp
vulkan/AppBase.h
vulkan/AppBase.cpp
vulkan/Application.h
@@ -45,13 +49,16 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android")
find_package(Vulkan REQUIRED)
target_link_libraries(sample
target_link_libraries(face_sdk
game-activity::game-activity_static
Vulkan::Vulkan
android
log)
log
# dl: needed by CrashHandler::dumpFrame -> dladdr() to map a
# PC back to "module + symbol + offset" in the crash log.
dl)
target_compile_definitions(sample PRIVATE
target_compile_definitions(face_sdk PRIVATE
VMA_STATIC_VULKAN_FUNCTIONS=0
VMA_DYNAMIC_VULKAN_FUNCTIONS=1
)
@@ -82,7 +89,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR WIN32)
)
# 将源代码添加到此项目的可执行文件。
add_executable (sample
add_executable (face_sdk
vulkan/AppBase.h
vulkan/AppBase.cpp
vulkan/Application.h
@@ -97,7 +104,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR WIN32)
endif()
# target_compile_definitions(sample PRIVATE VK_NO_PROTOTYPES) 使用volk 动态加载的时候需要定义,不自动生成vulkan接口
target_compile_definitions(sample PRIVATE GLM_ENABLE_EXPERIMENTAL)
# target_compile_definitions(sample PRIVATE VULKAN_HPP_NO_STRUCT_CONSTRUCTORS) 解决vs 编译c++ 20 不支持新构造函数的特性
# target_compile_definitions(face_sdk PRIVATE VK_NO_PROTOTYPES) 使用volk 动态加载的时候需要定义,不自动生成vulkan接口
target_compile_definitions(face_sdk PRIVATE GLM_ENABLE_EXPERIMENTAL)
# target_compile_definitions(face_sdk PRIVATE VULKAN_HPP_NO_STRUCT_CONSTRUCTORS) 解决vs 编译c++ 20 不支持新构造函数的特性
+191
View File
@@ -0,0 +1,191 @@
# Face SDK 崩溃调试指南
本文档说明如何配合 AI 定位 `libface_sdk.so` 里的崩溃 / 异常。核心思路:**把 Vulkan 每一次异常的上下文写进一个持久日志文件里,崩溃或画面卡住后把这份文件发给 AI 分析。**
---
## 一、已经加好的东西(不需要你做什么)
### 1. `DebugLog` 模块(`app/src/main/cpp/DebugLog.{h,cpp}`
- 线程安全;
- 每条日志带**时间戳 + 线程 ID(tid)**
- 写到 App **私有目录下的文件** —— 不会被 logcat 自动清掉;
- 每写一行 `fflush`,进程挂了也不丢最后几行;
- 文件大小超过 2MB 自动滚动到 `.old`
- 同步把日志镜像到 logcat,tag 是 `FACE_DBG`
### 2. 打点位置
| 位置 | 打点内容 | 用途 |
|---|---|---|
| `android_main` 启动 | pid、tid、日志文件绝对路径 | 每次启动一条,作为分段锚点 |
| 渲染循环心跳 | 每 600 帧一条:`frame=X exceptions_so_far=Y` | 判断启动多久后崩溃 / 异常持续性 |
| `try/catch` 兜底 | 首 10 次异常每次都打,之后每 120 次打一次 | **Vulkan 异常不会再让进程硬崩**,留住现场 |
| `handle_cmd` | 每个 `APP_CMD_*` 事件(尤其 `INIT_WINDOW`/`TERM_WINDOW`/`CONFIG_CHANGED`) | 旋转 / 切后台 / 锁屏场景的时间线 |
| `FaceApp::initVulkan` 入/出口 | 调用次数、`_applicationInited` / `_faceAppInited` / `_sceondInited` 状态 | 判断是否被多次重复调用 |
| `processImageNative` / `passDataToNative` | 每 300 次调用打一条 tid | 验证"Java 两个回调在不同线程并发调 Vulkan"假设 |
| `Application::drawFrame` 里的 4 次 Vulkan 调用 | 非 `VK_SUCCESS` 时打印数值 + 字符串名(例如 `-4 (VK_ERROR_DEVICE_LOST)` | **最关键的证据** |
### 3. 关键行为变化
**⚠️ 现在 `drawFrame` 里 Vulkan 抛出的异常会被 `try/catch` 吃掉 —— App 不会再因为这类问题硬崩溃。**
- 好处:你能继续跑、继续复现、日志一直在写;
- 代价:出问题时表现从"崩溃"变成"**画面卡住 / 黑屏 / 不再更新**"。
- 如果想暂时回到硬崩行为:把 `app/src/main/cpp/main.cpp` 里那段 `try { g_Application->drawFrame(...); } catch (...) { ... }` 改回直接 `g_Application->drawFrame(frameTime);` 即可。
---
## 二、怎么把日志文件取出来
### 方式 A`run-as`(推荐,不需要 root
```bash
adb shell "run-as com.inewme.uvmirror cat files/face_sdk_debug.log" > face_sdk_debug.log
adb shell "run-as com.inewme.uvmirror cat files/face_sdk_debug.log.old" > face_sdk_debug.log.old
```
> `.old` 只有文件滚动过一次才存在,没有可以忽略错误。
### 方式 B:直接看 logcat(实时)
```bash
adb logcat -s FACE_DBG
```
和文件内容基本一致。跑较久的话还是文件更可靠。
### 方式 C:从 App 里拿路径
启动后 `FACE_DBG` 第一条日志长这样:
```
[10:12:03.041][tid=12345] android_main start, pid=23456 tid=12345, logPath=/data/user/0/com.inewme.uvmirror/files/face_sdk_debug.log
```
照这个路径走就对了。
---
## 三、你现在要做的事(B 方案:两种场景都复现一次)
### 场景 1:旋转崩溃(已确认必现)
1. 打开 App,等初始化完成(画面能看到人脸渲染)。
2. **旋转一次屏幕**
3. 等 3~5 秒让日志写进去。
4.`face_sdk_debug.log`**不要清文件**,继续场景 2。
期望看到的关键行:
```
>> handle_cmd APP_CMD_TERM_WINDOW(...)
>> handle_cmd APP_CMD_INIT_WINDOW(...)
FaceApp::initVulkan enter, call#2 ...
drawFrame[NNN] vkAcquireNextImageKHR -> ??? (...)
!!! drawFrame std::exception (frame=NNN count=M): failed to ...
```
注意里面那个 `???` —— 这是整件事的核心证据。
### 场景 2:长时间运行崩溃(之前那个 10 分钟左右的)
**不要退出 App,接着跑**。现在即使旋转触发了异常,进程没死,我们希望看后续会不会再出另一种 VkResult:
1. 尽量不操作(不切后台、不旋转、不锁屏,让 App 持续渲染)。
2. 连续跑 **15~20 分钟**,观察是否复现之前的"无外部事件崩溃"。
3. 复现后(或稳定 20 分钟未复现也行),再取一次 `face_sdk_debug.log`(这次会覆盖场景 1 的那份,所以务必**先把场景 1 的单独保存好**)。
期望看到的关键行(如果真的是 DEVICE_LOST):
```
drawFrame[NNN] vkQueueSubmit -> -4 (VK_ERROR_DEVICE_LOST) ...
```
或者某个 Vulkan 调用返回其他错误码。
### 辅助信息(手边有就顺便记一下)
| 项 | 为什么要 |
|---|---|
| 机型 + Android 版本 | GPU 驱动差异;Adreno / Mali / Xclipse 表现差别大 |
| 复现时手机是不是很烫 | 排查 GPU 温控 / thermal throttling |
| 旋转时是"横屏 → 竖屏"还是"竖屏 → 横屏" / 多次连续旋转? | 某些机型一次 vs 多次触发行为不同 |
| 10 分钟崩溃时 App 在做什么(有在切换 motion 吗?) | 帮助排除 `changeMotionList` 相关路径 |
---
## 四、发给 AI 的时候包含什么
最小集:
1. `face_sdk_debug.log`(场景 1 旋转那份)
2. `face_sdk_debug.log`(场景 2 长跑那份)
3. 如果文件很大,**完整发**比截断发强(AI 主要关心最后几千行)
4. 一句话说明这份日志对应场景 1 还是场景 2,是否复现了
不需要的:
- logcat 完整 dump(体积太大且大多无关,除非 AI 主动问特定 tag)
- 录屏
- 源代码(AI 已经能看到仓库)
---
## 五、当前未解决的假设,等日志验证
| 假设 | 对应 VkResult | 修法 |
|---|---|---|
| 旋转导致 surface 失效未处理 | `VK_ERROR_OUT_OF_DATE_KHR` (-1000001004) 或 `VK_ERROR_SURFACE_LOST_KHR` (-1000000000) 或 `VK_SUBOPTIMAL_KHR` (1000001003) | 实现 `APP_CMD_TERM_WINDOW` 销毁流程 + swapchain 重建 |
| Java 多线程并发提交 graphicsQueue 导致 GPU 长时间后挂 | `VK_ERROR_DEVICE_LOST` (-4),且 `processImageNative` / `passDataToNative` 日志里 tid 不同 | 给所有 `vkQueueSubmit`/`vkQueuePresentKHR` 加全局 queue mutex |
| 驱动 bug / 温度导致 GPU hang | `VK_ERROR_DEVICE_LOST` (-4),单线程也出 | 捕获并尝试重建 device,或上报给机型厂商 |
| 资源累积泄漏 | `VK_ERROR_OUT_OF_DEVICE_MEMORY` (-2) 或 `VK_ERROR_TOO_MANY_OBJECTS` | 排查 `beginSingleTimeCommands`/`processWithVulkan` 路径 |
---
## 六、日志样例(便于你确认打印是否正常)
正常启动应该看到:
```
========== DebugLog opened @ 2026-04-23 14:05:12 (pid=12345) ==========
[14:05:12.019][tid=12345] android_main start, pid=12345 tid=12345, logPath=/data/user/0/com.inewme.uvmirror/files/face_sdk_debug.log
[14:05:12.155][tid=12345] >> handle_cmd APP_CMD_START(10) tid=12345
[14:05:12.155][tid=12345] << handle_cmd APP_CMD_START done
[14:05:12.210][tid=12345] >> handle_cmd APP_CMD_INIT_WINDOW(1) tid=12345
[14:05:12.210][tid=12345] handle_cmd APP_CMD_INIT_WINDOW: calling initVulkan()
[14:05:12.210][tid=12345] FaceApp::initVulkan enter, call#1 _applicationInited=0 _faceAppInited=0 _secondfaceAppInited=0
[14:05:13.420][tid=12345] FaceApp::initVulkan exit, call#1 _applicationInited=1 _faceAppInited=1 _secondfaceAppInited=1
[14:05:13.420][tid=12345] handle_cmd APP_CMD_INIT_WINDOW: initVulkan() returned, isInited=1
[14:05:13.421][tid=12345] << handle_cmd APP_CMD_INIT_WINDOW done
[14:05:14.005][tid=67890] processImageNative #1 tid=67890 w=480 h=480
[14:05:14.008][tid=67891] passDataToNative #1 tid=67891 point_count=468 w=480 h=480
[14:05:35.200][tid=12345] heartbeat: frame=600 exceptions_so_far=0
```
旋转时如果触发异常应该看到:
```
[14:06:10.112][tid=12345] >> handle_cmd APP_CMD_CONFIG_CHANGED(8) tid=12345
[14:06:10.113][tid=12345] >> handle_cmd APP_CMD_TERM_WINDOW(2) tid=12345
[14:06:10.113][tid=12345] handle_cmd APP_CMD_TERM_WINDOW: (no handler yet, ...)
[14:06:10.113][tid=12345] << handle_cmd APP_CMD_TERM_WINDOW done
[14:06:10.250][tid=12345] >> handle_cmd APP_CMD_INIT_WINDOW(1) tid=12345
[14:06:10.250][tid=12345] FaceApp::initVulkan enter, call#2 _applicationInited=1 _faceAppInited=1 _secondfaceAppInited=0
[14:06:10.250][tid=12345] FaceApp::initVulkan exit, call#2 _applicationInited=1 _faceAppInited=1 _secondfaceAppInited=1
[14:06:10.280][tid=12345] drawFrame[1234] vkAcquireNextImageKHR -> -1000001004 (VK_ERROR_OUT_OF_DATE_KHR) currentFrame=0
[14:06:10.280][tid=12345] !!! drawFrame std::exception (frame=1234 count=1): failed to acquire swap chain image!
```
**只要日志里能看到 `vkXxxKHR -> <数值> (<名字>)` 这一行,诊断就成立了。**
---
## 七、文件清单(仅供参考,日常不用动)
```
app/src/main/cpp/
DebugLog.h 新增
DebugLog.cpp 新增
main.cpp 改:日志初始化 / 心跳 / try-catch / cmd 日志 / JNI tid
vulkan/
Application.cpp 改:drawFrame 4 处 VkResult 日志 + VkResultStr
FaceApp.cpp 改:initVulkan 入/出口日志
CMakeLists.txt 改:加入 DebugLog.cpp 到 Android 构建
```
+289
View File
@@ -0,0 +1,289 @@
# Face SDK 使用说明
基于 Vulkan + MediaPipe FaceLandmarker 的实时人脸贴图渲染 SDK,主要面向**美妆 / AR 试妆**场景:摄像头预览 → 人脸 landmark 检测 → 多帧 PNG 贴图按 motion 序列贴在人脸上。
支持任意分辨率手机(采用"短边正方形画布 + 长边 letterbox"布局)、前置 / 后置摄像头自动切换、镜子镜像效果。
---
## 目录结构
```
face_sdk/
├── app/ ← SDK 主体(可作为 AAR 给业务方接入)
│ ├── src/main/java/com/hmwl/face_sdk/
│ │ ├── FaceActivity.java ← 业务方继承的核心 Activity
│ │ ├── FaceLandmarkerHelper.kt ← MediaPipe 封装
│ │ ├── DebugLog.java ← Java/C++ 统一日志(写文件)
│ │ ├── InitArg.java / Motion.java / MotionList.java / Frame.java ← JSON 配置 POJO
│ │ └── ...
│ ├── src/main/cpp/main.cpp ← JNI + native 主循环
│ └── src/main/assets/shaders/ ← Vulkan GLSLbg/texture/...
├── vulkan/ ← 跨平台 Vulkan 渲染核心
│ ├── Application.{h,cpp} ← Vulkan 初始化 / 窗口生命周期
│ └── FaceApp.{h,cpp} ← 业务渲染(face mesh + bg quad + motion 调度)
├── example/ ← 示例 app,演示完整接入
│ ├── src/main/java/com/inewme/uvmirror/
│ │ ├── MainActivity.kt ← 入口(Compose UI,前/后置选择)
│ │ ├── MakeupActivity.kt ← 业务 Activity(继承 FaceActivity
│ │ └── MotionManager.kt ← motion 资源加载封装
│ ├── src/main/AndroidManifest.xml
│ └── src/main/assets/pic/
│ ├── motion_data_ex.json ← motion 元数据(帧文件名 + 锚点 x/y)
│ └── <motion_id>/000xxx.png ← motion 各帧贴图
├── compile_shader.bat ← 编译所有 .vert/.frag 为 .spv(必须在改 shader 后执行)
└── third_party/ ← VMA / glm / glfw / mediapipe …
```
---
## 快速上手
### 1. 编译 + 运行 example
```powershell
# Windows 上,确保 SDK 路径在 local.properties 里,以及 NDK 已安装
.\gradlew.bat :example:installDebug
```
启动后:
- 主界面有"前置摄像头 / 后置摄像头"两个 RadioButton(默认前置)
- 点击 "Go to Makeup" 进入 `MakeupActivity`
- `MakeupActivity` 自动加载 `assets/pic/motion_data_ex.json` 配置的 motion,按 `update()` 里的脚本播放
### 2. 改 shader 后必须重编
```powershell
.\compile_shader.bat
.\gradlew.bat :example:installDebug
```
---
## 在自己的 App 里集成
### Step 1:依赖 SDK 模块
在你的 `settings.gradle` / `build.gradle` 引入 `:app` 模块(或打包成 AAR)。
### Step 2:在 `AndroidManifest.xml` 声明业务 Activity
业务 Activity 必须**竖屏**,且**屏蔽常见 configChanges 不要重建**
```xml
<activity
android:name=".MakeupActivity"
android:exported="false"
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
```
### Step 3:继承 `FaceActivity` 写业务 Activity
```kotlin
class MakeupActivity : FaceActivity() {
lateinit var motionMgr: MotionManager
private var isMotionLoadFinished = false
override fun onCreate(savedInstanceState: Bundle?) {
// 在 super.onCreate 之前构造 MotionManager(它会调 SetInitArg
motionMgr = MotionManager(
this,
/* fps */ 10,
/* zoom */ 1.5f,
/* r,g,b */ 0f, 0f, 1f, // letterbox / 圆外纯色(蓝)
/* radius */ 240f, // 480 设计基线下的圆半径,SDK 自动按 canvas_size/480 缩放
/* offset */ 200f, -200f // 480 设计基线下的中心偏移
)
super.onCreate(savedInstanceState)
// 一次性加载所有要用到的 motion,加载完才能 PlayMotionList
motionMgr.LoadMotionList(listOf("4", "56")) {
isMotionLoadFinished = true
}
// 自己的 update 循环(生命周期感知,避免泄漏)
lifecycleScope.launch {
while (true) {
update()
delay(10)
}
}
}
private fun update() {
if (!isMotionLoadFinished) return
// 想播什么就调,调 SDK API(详见下面"API 参考"
}
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
if (keyCode == KeyEvent.KEYCODE_BACK) {
Stop() // ← 退出 Activity 前必须先 Stop(),否则 native 线程仍在跑
finish()
return true
}
return super.onKeyDown(keyCode, event)
}
}
```
### Step 4:从入口 Activity 启动业务 Activity,传入摄像头朝向
```kotlin
import androidx.camera.core.CameraSelector
import com.hmwl.face_sdk.FaceActivity
val intent = Intent(context, MakeupActivity::class.java).apply {
// 前置:CameraSelector.LENS_FACING_FRONT;后置:LENS_FACING_BACK
// 不传时 SDK 默认走后置(向后兼容)
putExtra(FaceActivity.EXTRA_LENS_FACING, CameraSelector.LENS_FACING_FRONT)
}
context.startActivity(intent)
```
前置摄像头 SDK 会自动:
- 在 GPU 端把 bg + face mesh 整体水平翻转 → 镜子效果
- mediapipe 输入仍是 raw(未镜像)帧,所以 landmark 与 raw 帧 UV 严格对齐
---
## API 参考(`FaceActivity` 公开方法)
| 方法 | 说明 |
| --- | --- |
| `static final String EXTRA_LENS_FACING` | Intent extra key,值为 `CameraSelector.LENS_FACING_FRONT``LENS_FACING_BACK` |
| `void SetInitArg(String json)` | 设置全局参数(zoom / r,g,b / radius / offset_x,offset_y / action_fps),JSON 格式见 `InitArg` |
| `String PreLoadAction(String json, LoadMotionListFinishedCallback cb)` | 异步预加载一组 motion 资源;加载完毕回调 cb |
| `void PlayMotionList(List<String> motionList, boolean loop, PlayMotionListFinishedCallback cb)` | 播放 motion 序列,`loop=true` 时无限循环,`cb` 为播完一轮的回调(loop 模式下不会触发,传 `null` 即可) |
| `void StopMotion()` | 暂停当前 motion 播放(停在当前帧) |
| `void ResumeMotion()` | 恢复 `StopMotion()` 之前的播放进度 |
| `void Stop()` | **完全停止 native 渲染**,退出 Activity 前必须调 |
**注意:** `LoadMotionList` / `PreLoadAction` 是异步的,回调里设置 `isMotionLoadFinished=true` 之后才能 `PlayMotionList`,否则播放命令会被静默丢弃。
---
## 资源准备
### Motion 帧图
每个 motion 用一个独立目录,PNG 帧按文件名顺序播放:
```
assets/pic/4/000000.png
assets/pic/4/000001.png
...
assets/pic/4/000031.png
```
PNG 必须是 **RGBA**(透明通道),SDK 直接采样后 alpha-blend 到 bg 上方。
### `motion_data_ex.json`
motion 元数据,描述每帧贴图的**锚点偏移**(贴在脸上的相对位置,画布坐标系,原点在屏幕中心):
```json
{
"4": {
"000000.png": { "x": 0.0, "y": 0.0 },
"000001.png": { "x": 0.0, "y": 0.0 },
...
},
"56": {
"000000.png": { "x": 0.0, "y": 0.0 },
...
}
}
```
---
## 设计要点(接入方可不读,但要改 shader / 渲染时必读)
### 1. 画布与 letterbox
- **画布**:屏幕短边的正方形,居中。屏幕长边方向自然 letterbox(上下或左右黑边)。
- 画布坐标系("画布 NDC")∈ [-1, +1]²,原点在画布中心。
- mediapipe FaceLandmarker 输出的归一化 landmark ∈ [0, 1]² 直接对应画布 NDC(经过 `*2-1`)。
- 业务参数 `zoom / offset_x / offset_y / radius` 都是**屏幕物理像素**(SDK 端按 `canvas_size / 480` 自动缩放,业务方用 480 设计基线即可)。
### 2. 摄像头帧 → 画布的几何流水线
```
raw 摄像头帧 (W×H, 4:3 典型)
└─① 中央裁切 min(W,H)×min(W,H) 正方形
└─② Matrix.postRotate(rotationDegrees) 顺时针旋转 → 正立 face crop
└─③ MediaPipe 输出 landmark ∈ [0,1]² ─────▶ 画布 NDC
└─ shader 做反向变换贴回 raw 帧 UV (bg)
└─ shader 直接当 face mesh 顶点 (face)
```
### 3. 镜子效果(前置摄像头)
`PushConstants.mirror_x ∈ {0.0, 1.0}`
- 前置时 = 1.0shader 里把最终 NDC.x 翻转 (`pos.x *= 1.0 - 2.0*mirror_x`)
- bg + face mesh 都做同样翻转 → 整体水平镜像,对齐保持
- mediapipe 仍接收**未镜像**的 raw 帧,避免双重镜像把 landmark 弄反
### 4. bg 与 face mesh 对齐不变量
bg 圆内显示摄像头视频,face mesh 把 motion png 贴到 landmark 位置——两者对齐依赖:
> **屏幕同一像素位置,bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe landmark 坐标**
实现方式:bg quad 与 face mesh **完全相同**地经过 `zoom × offset × letterbox × mirror` 几何链路;同时 bg 的 `outTexCoord` 基于"未变换的 quad 顶点 pos"算 raw UV——这样屏幕 fragment 反算的 pos 就等价于"对应 landmark 的画布 NDC"UV 完美贴合。
bg quad 顶点取 `±10`(而非 `±1`),保证任何 zoom/offset 下 quad 在屏幕上 GPU clip 后整屏覆盖,letterbox 也被 bg.frag 的 r/g/b 蓝色填充。
### 5. 摄像头分辨率适配
- CameraX 用 `setTargetResolution(640, 480)` 强制拿低分辨率帧(SDK 不需要高分辨率)
- shader 用 `camera_aspect = W/H``camera_rotation`CameraX 给的 rotationDegrees)做 UV 反变换
- 摄像头帧分辨率/朝向变化时,SDK 会在 `processCameraFrame` 里自动重建 bg 纹理
### 6. 窗口生命周期
- `APP_CMD_TERM_WINDOW``Application::cleanupForWindowLost()`:只销毁**窗口资源**surface / swapchain / framebuffers / imageViews / commandBuffers / sync),保留 renderPass / pipelines / device / VMA / textures
- `APP_CMD_INIT_WINDOW``Application::reinitForNewWindow()`:复用全局资源,只重建窗口资源
- `android_main` 退出时**不再额外销毁** Vulkan 资源(保留给下次 NativeActivity 复用),避免 use-after-free 崩溃
---
## 调试
### 日志文件
SDK 把 Java/Kotlin 日志(通过 `com.hmwl.face_sdk.DebugLog.i / e`)和 C++ 日志(`DebugLog::log`**统一写入**
```
/data/user/0/<your.package>/files/face_sdk_debug.log
```
```powershell
# 拉到本地分析
adb pull /data/user/0/com.inewme.uvmirror.f20260113/files/face_sdk_debug.log .
```
崩溃时还会自动追加一段含信号、寄存器、backtrace 的 dump`========== FACE_SDK CRASH ==========`)。
### 常见问题
| 现象 | 排查方向 |
| --- | --- |
| 黑屏 / 没有摄像头预览 | 检查相机权限是否授予;查看日志是否有 `processImageNative` 调用 |
| 人脸贴图不出现 | 检查 `motion_data_ex.json` 里的 motion id 是否在 `LoadMotionList` 里加载过;检查 `isMotionLoadFinished` 是否被回调置 true |
| 贴图不贴合人脸 | bg 与 face mesh 的 zoom/offset/mirror 必须严格一致——见"设计要点 #4 对齐不变量" |
| 切换摄像头后崩溃 | 见 `cleanupSecondInit` 注释;不要在 Android 路径调用它,仅 desktop 入口可用 |
| 改了 shader 不生效 | 必须先跑 `compile_shader.bat` 重新生成 `.spv` |
---
## 兼容性
- minSdk 29targetSdk 看 `app/build.gradle`
- 设备需支持 Vulkan 1.0+ 和 MediaPipe TFLite delegate
- 已在 MI 9Android 10、Adreno 640、Vulkan vendor `qglinternal`)上验证通过
+66 -12
View File
@@ -1,19 +1,20 @@
plugins {
alias(libs.plugins.android.application)
id 'org.jetbrains.kotlin.android'
//alias(libs.plugins.android.application)
alias(libs.plugins.android.library)
alias(libs.plugins.kotlin.android)
//id 'org.jetbrains.kotlin.android'
}
ext.vvl_version='1.4.321.0'
apply from: "./download_vvl.gradle"
android {
namespace 'com.hmwl.sample'
compileSdk 35
namespace 'com.hmwl.face_sdk'
compileSdk 36
defaultConfig {
applicationId "com.hmwl.sample"
minSdk 30
targetSdk 35
//applicationId "com.hmwl.face_sdk"
minSdk 29
targetSdk 36
versionCode 1
versionName "1.0"
@@ -22,12 +23,25 @@ android {
}
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
consumerProguardFiles "consumer-rules.pro"
}
buildTypes {
debug {
debuggable true
jniDebuggable true
externalNativeBuild {
cmake {
arguments "-DCMAKE_BUILD_TYPE=Debug"
cppFlags "-DDEBUG -O0 -g"
}
}
ndk {
debugSymbolLevel 'FULL' // Debug 版本包含完整符号
}
}
release {
@@ -36,18 +50,54 @@ android {
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_21
targetCompatibility JavaVersion.VERSION_21
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
// 添加 Kotlin 编译选项
// kotlinOptions {
// jvmTarget = '17' // 与 Java 版本保持一致
// }
// kotlin {
// jvmToolchain(17)
// }
buildFeatures {
prefab true
}
sourceSets {
main {
jniLibs.srcDirs += [
'libs/tasks-vision/jni',
'libs/tasks-core/jni'
]
//java.srcDirs += [
// 'libs/tasks-vision/vision_classes',
// 'libs/tasks-core/core_classes'
//]
res.srcDirs += [
'libs/tasks-vision/res',
'libs/tasks-core/res'
]
}
}
externalNativeBuild {
cmake {
path file('../CMakeLists.txt')
version '3.22.1'
}
}
packagingOptions {
pickFirst '**/*.jar'
}
kotlinOptions {
jvmTarget = '17'
}
}
dependencies {
@@ -55,13 +105,17 @@ dependencies {
implementation libs.appcompat
implementation libs.material
implementation libs.games.activity
implementation libs.androidx.core.core.ktx
testImplementation libs.junit
androidTestImplementation libs.ext.junit
androidTestImplementation libs.espresso.core
implementation files('libs/tasks-vision-0.10.26.aar')
implementation files('libs/tasks-core-0.10.26.aar')
implementation 'com.google.protobuf:protobuf-javalite:3.21.12'
implementation files('libs/tasks-vision.jar', 'libs/tasks-core.jar')
//implementation fileTree(dir: 'libs/tasks-vision', include: ['*.jar'])
//implementation fileTree(dir: 'libs/tasks-core', include: ['*.jar'])
def camerax_version = '1.4.2'
implementation "androidx.camera:camera-core:$camerax_version"
+2 -1
View File
@@ -1,4 +1,5 @@
apply plugin: 'com.android.application'
//apply plugin: 'com.android.application'
apply plugin: 'com.android.library'
/*
* Copyright (c) 2022-2024, The Android Open Source Project.
Binary file not shown.
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.tasks.core" >
<uses-sdk
android:minSdkVersion="24"
android:targetSdkVersion="34" />
</manifest>
+255
View File
@@ -0,0 +1,255 @@
int anim fragment_fast_out_extra_slow_in 0x00000000
int animator fragment_close_enter 0x00000000
int animator fragment_close_exit 0x00000000
int animator fragment_fade_enter 0x00000000
int animator fragment_fade_exit 0x00000000
int animator fragment_open_enter 0x00000000
int animator fragment_open_exit 0x00000000
int attr alpha 0x00000000
int attr coordinatorLayoutStyle 0x00000000
int attr font 0x00000000
int attr fontProviderAuthority 0x00000000
int attr fontProviderCerts 0x00000000
int attr fontProviderFetchStrategy 0x00000000
int attr fontProviderFetchTimeout 0x00000000
int attr fontProviderPackage 0x00000000
int attr fontProviderQuery 0x00000000
int attr fontStyle 0x00000000
int attr fontVariationSettings 0x00000000
int attr fontWeight 0x00000000
int attr keylines 0x00000000
int attr layout_anchor 0x00000000
int attr layout_anchorGravity 0x00000000
int attr layout_behavior 0x00000000
int attr layout_dodgeInsetEdges 0x00000000
int attr layout_insetEdge 0x00000000
int attr layout_keyline 0x00000000
int attr statusBarBackground 0x00000000
int attr ttcIndex 0x00000000
int color notification_action_color_filter 0x00000000
int color notification_icon_bg_color 0x00000000
int color notification_material_background_media_default_color 0x00000000
int color primary_text_default_material_dark 0x00000000
int color ripple_material_light 0x00000000
int color secondary_text_default_material_dark 0x00000000
int color secondary_text_default_material_light 0x00000000
int dimen compat_button_inset_horizontal_material 0x00000000
int dimen compat_button_inset_vertical_material 0x00000000
int dimen compat_button_padding_horizontal_material 0x00000000
int dimen compat_button_padding_vertical_material 0x00000000
int dimen compat_control_corner_material 0x00000000
int dimen compat_notification_large_icon_max_height 0x00000000
int dimen compat_notification_large_icon_max_width 0x00000000
int dimen notification_action_icon_size 0x00000000
int dimen notification_action_text_size 0x00000000
int dimen notification_big_circle_margin 0x00000000
int dimen notification_content_margin_start 0x00000000
int dimen notification_large_icon_height 0x00000000
int dimen notification_large_icon_width 0x00000000
int dimen notification_main_column_padding_top 0x00000000
int dimen notification_media_narrow_margin 0x00000000
int dimen notification_right_icon_size 0x00000000
int dimen notification_right_side_padding_top 0x00000000
int dimen notification_small_icon_background_padding 0x00000000
int dimen notification_small_icon_size_as_large 0x00000000
int dimen notification_subtext_size 0x00000000
int dimen notification_top_pad 0x00000000
int dimen notification_top_pad_large_text 0x00000000
int dimen subtitle_corner_radius 0x00000000
int dimen subtitle_outline_width 0x00000000
int dimen subtitle_shadow_offset 0x00000000
int dimen subtitle_shadow_radius 0x00000000
int drawable notification_action_background 0x00000000
int drawable notification_bg 0x00000000
int drawable notification_bg_low 0x00000000
int drawable notification_bg_low_normal 0x00000000
int drawable notification_bg_low_pressed 0x00000000
int drawable notification_bg_normal 0x00000000
int drawable notification_bg_normal_pressed 0x00000000
int drawable notification_icon_background 0x00000000
int drawable notification_template_icon_bg 0x00000000
int drawable notification_template_icon_low_bg 0x00000000
int drawable notification_tile_bg 0x00000000
int drawable notify_panel_notification_icon_bg 0x00000000
int id accessibility_action_clickable_span 0x00000000
int id accessibility_custom_action_0 0x00000000
int id accessibility_custom_action_1 0x00000000
int id accessibility_custom_action_10 0x00000000
int id accessibility_custom_action_11 0x00000000
int id accessibility_custom_action_12 0x00000000
int id accessibility_custom_action_13 0x00000000
int id accessibility_custom_action_14 0x00000000
int id accessibility_custom_action_15 0x00000000
int id accessibility_custom_action_16 0x00000000
int id accessibility_custom_action_17 0x00000000
int id accessibility_custom_action_18 0x00000000
int id accessibility_custom_action_19 0x00000000
int id accessibility_custom_action_2 0x00000000
int id accessibility_custom_action_20 0x00000000
int id accessibility_custom_action_21 0x00000000
int id accessibility_custom_action_22 0x00000000
int id accessibility_custom_action_23 0x00000000
int id accessibility_custom_action_24 0x00000000
int id accessibility_custom_action_25 0x00000000
int id accessibility_custom_action_26 0x00000000
int id accessibility_custom_action_27 0x00000000
int id accessibility_custom_action_28 0x00000000
int id accessibility_custom_action_29 0x00000000
int id accessibility_custom_action_3 0x00000000
int id accessibility_custom_action_30 0x00000000
int id accessibility_custom_action_31 0x00000000
int id accessibility_custom_action_4 0x00000000
int id accessibility_custom_action_5 0x00000000
int id accessibility_custom_action_6 0x00000000
int id accessibility_custom_action_7 0x00000000
int id accessibility_custom_action_8 0x00000000
int id accessibility_custom_action_9 0x00000000
int id action0 0x00000000
int id action_container 0x00000000
int id action_divider 0x00000000
int id action_image 0x00000000
int id action_text 0x00000000
int id actions 0x00000000
int id all 0x00000000
int id async 0x00000000
int id blocking 0x00000000
int id bottom 0x00000000
int id cancel_action 0x00000000
int id center 0x00000000
int id center_horizontal 0x00000000
int id center_vertical 0x00000000
int id chronometer 0x00000000
int id clip_horizontal 0x00000000
int id clip_vertical 0x00000000
int id dialog_button 0x00000000
int id end 0x00000000
int id end_padder 0x00000000
int id fill 0x00000000
int id fill_horizontal 0x00000000
int id fill_vertical 0x00000000
int id forever 0x00000000
int id fragment_container_view_tag 0x00000000
int id icon 0x00000000
int id icon_group 0x00000000
int id info 0x00000000
int id italic 0x00000000
int id left 0x00000000
int id line1 0x00000000
int id line3 0x00000000
int id media_actions 0x00000000
int id none 0x00000000
int id normal 0x00000000
int id notification_background 0x00000000
int id notification_main_column 0x00000000
int id notification_main_column_container 0x00000000
int id right 0x00000000
int id right_icon 0x00000000
int id right_side 0x00000000
int id special_effects_controller_view_tag 0x00000000
int id start 0x00000000
int id status_bar_latest_event_content 0x00000000
int id tag_accessibility_actions 0x00000000
int id tag_accessibility_clickable_spans 0x00000000
int id tag_accessibility_heading 0x00000000
int id tag_accessibility_pane_title 0x00000000
int id tag_screen_reader_focusable 0x00000000
int id tag_transition_group 0x00000000
int id tag_unhandled_key_event_manager 0x00000000
int id tag_unhandled_key_listeners 0x00000000
int id text 0x00000000
int id text2 0x00000000
int id time 0x00000000
int id title 0x00000000
int id top 0x00000000
int id view_tree_lifecycle_owner 0x00000000
int id view_tree_saved_state_registry_owner 0x00000000
int id view_tree_view_model_store_owner 0x00000000
int id visible_removing_fragment_view_tag 0x00000000
int integer cancel_button_image_alpha 0x00000000
int integer status_bar_notification_info_maxnum 0x00000000
int layout custom_dialog 0x00000000
int layout notification_action 0x00000000
int layout notification_action_tombstone 0x00000000
int layout notification_media_action 0x00000000
int layout notification_media_cancel_action 0x00000000
int layout notification_template_big_media 0x00000000
int layout notification_template_big_media_custom 0x00000000
int layout notification_template_big_media_narrow 0x00000000
int layout notification_template_big_media_narrow_custom 0x00000000
int layout notification_template_custom_big 0x00000000
int layout notification_template_icon_group 0x00000000
int layout notification_template_lines_media 0x00000000
int layout notification_template_media 0x00000000
int layout notification_template_media_custom 0x00000000
int layout notification_template_part_chronometer 0x00000000
int layout notification_template_part_time 0x00000000
int string status_bar_notification_info_overflow 0x00000000
int style TextAppearance_Compat_Notification 0x00000000
int style TextAppearance_Compat_Notification_Info 0x00000000
int style TextAppearance_Compat_Notification_Info_Media 0x00000000
int style TextAppearance_Compat_Notification_Line2 0x00000000
int style TextAppearance_Compat_Notification_Line2_Media 0x00000000
int style TextAppearance_Compat_Notification_Media 0x00000000
int style TextAppearance_Compat_Notification_Time 0x00000000
int style TextAppearance_Compat_Notification_Time_Media 0x00000000
int style TextAppearance_Compat_Notification_Title 0x00000000
int style TextAppearance_Compat_Notification_Title_Media 0x00000000
int style Widget_Compat_NotificationActionContainer 0x00000000
int style Widget_Compat_NotificationActionText 0x00000000
int style Widget_Support_CoordinatorLayout 0x00000000
int[] styleable ColorStateListItem { 0x010101a5, 0x0101031f, 0x00000000 }
int styleable ColorStateListItem_android_color 0
int styleable ColorStateListItem_android_alpha 1
int styleable ColorStateListItem_alpha 2
int[] styleable CoordinatorLayout { 0x00000000, 0x00000000 }
int styleable CoordinatorLayout_keylines 0
int styleable CoordinatorLayout_statusBarBackground 1
int[] styleable CoordinatorLayout_Layout { 0x010100b3, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable CoordinatorLayout_Layout_android_layout_gravity 0
int styleable CoordinatorLayout_Layout_layout_anchor 1
int styleable CoordinatorLayout_Layout_layout_anchorGravity 2
int styleable CoordinatorLayout_Layout_layout_behavior 3
int styleable CoordinatorLayout_Layout_layout_dodgeInsetEdges 4
int styleable CoordinatorLayout_Layout_layout_insetEdge 5
int styleable CoordinatorLayout_Layout_layout_keyline 6
int[] styleable FontFamily { 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable FontFamily_fontProviderAuthority 0
int styleable FontFamily_fontProviderCerts 1
int styleable FontFamily_fontProviderFetchStrategy 2
int styleable FontFamily_fontProviderFetchTimeout 3
int styleable FontFamily_fontProviderPackage 4
int styleable FontFamily_fontProviderQuery 5
int[] styleable FontFamilyFont { 0x01010532, 0x01010533, 0x0101053f, 0x0101056f, 0x01010570, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable FontFamilyFont_android_font 0
int styleable FontFamilyFont_android_fontWeight 1
int styleable FontFamilyFont_android_fontStyle 2
int styleable FontFamilyFont_android_ttcIndex 3
int styleable FontFamilyFont_android_fontVariationSettings 4
int styleable FontFamilyFont_font 5
int styleable FontFamilyFont_fontStyle 6
int styleable FontFamilyFont_fontVariationSettings 7
int styleable FontFamilyFont_fontWeight 8
int styleable FontFamilyFont_ttcIndex 9
int[] styleable Fragment { 0x01010003, 0x010100d0, 0x010100d1 }
int styleable Fragment_android_name 0
int styleable Fragment_android_id 1
int styleable Fragment_android_tag 2
int[] styleable FragmentContainerView { 0x01010003, 0x010100d1 }
int styleable FragmentContainerView_android_name 0
int styleable FragmentContainerView_android_tag 1
int[] styleable GradientColor { 0x0101019d, 0x0101019e, 0x010101a1, 0x010101a2, 0x010101a3, 0x010101a4, 0x01010201, 0x0101020b, 0x01010510, 0x01010511, 0x01010512, 0x01010513 }
int styleable GradientColor_android_startColor 0
int styleable GradientColor_android_endColor 1
int styleable GradientColor_android_type 2
int styleable GradientColor_android_centerX 3
int styleable GradientColor_android_centerY 4
int styleable GradientColor_android_gradientRadius 5
int styleable GradientColor_android_tileMode 6
int styleable GradientColor_android_centerColor 7
int styleable GradientColor_android_startX 8
int styleable GradientColor_android_startY 9
int styleable GradientColor_android_endX 10
int styleable GradientColor_android_endY 11
int[] styleable GradientColorItem { 0x010101a5, 0x01010514 }
int styleable GradientColorItem_android_color 0
int styleable GradientColorItem_android_offset 1
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.tasks.vision" >
<uses-sdk
android:minSdkVersion="24"
android:targetSdkVersion="34" />
</manifest>
+255
View File
@@ -0,0 +1,255 @@
int anim fragment_fast_out_extra_slow_in 0x00000000
int animator fragment_close_enter 0x00000000
int animator fragment_close_exit 0x00000000
int animator fragment_fade_enter 0x00000000
int animator fragment_fade_exit 0x00000000
int animator fragment_open_enter 0x00000000
int animator fragment_open_exit 0x00000000
int attr alpha 0x00000000
int attr coordinatorLayoutStyle 0x00000000
int attr font 0x00000000
int attr fontProviderAuthority 0x00000000
int attr fontProviderCerts 0x00000000
int attr fontProviderFetchStrategy 0x00000000
int attr fontProviderFetchTimeout 0x00000000
int attr fontProviderPackage 0x00000000
int attr fontProviderQuery 0x00000000
int attr fontStyle 0x00000000
int attr fontVariationSettings 0x00000000
int attr fontWeight 0x00000000
int attr keylines 0x00000000
int attr layout_anchor 0x00000000
int attr layout_anchorGravity 0x00000000
int attr layout_behavior 0x00000000
int attr layout_dodgeInsetEdges 0x00000000
int attr layout_insetEdge 0x00000000
int attr layout_keyline 0x00000000
int attr statusBarBackground 0x00000000
int attr ttcIndex 0x00000000
int color notification_action_color_filter 0x00000000
int color notification_icon_bg_color 0x00000000
int color notification_material_background_media_default_color 0x00000000
int color primary_text_default_material_dark 0x00000000
int color ripple_material_light 0x00000000
int color secondary_text_default_material_dark 0x00000000
int color secondary_text_default_material_light 0x00000000
int dimen compat_button_inset_horizontal_material 0x00000000
int dimen compat_button_inset_vertical_material 0x00000000
int dimen compat_button_padding_horizontal_material 0x00000000
int dimen compat_button_padding_vertical_material 0x00000000
int dimen compat_control_corner_material 0x00000000
int dimen compat_notification_large_icon_max_height 0x00000000
int dimen compat_notification_large_icon_max_width 0x00000000
int dimen notification_action_icon_size 0x00000000
int dimen notification_action_text_size 0x00000000
int dimen notification_big_circle_margin 0x00000000
int dimen notification_content_margin_start 0x00000000
int dimen notification_large_icon_height 0x00000000
int dimen notification_large_icon_width 0x00000000
int dimen notification_main_column_padding_top 0x00000000
int dimen notification_media_narrow_margin 0x00000000
int dimen notification_right_icon_size 0x00000000
int dimen notification_right_side_padding_top 0x00000000
int dimen notification_small_icon_background_padding 0x00000000
int dimen notification_small_icon_size_as_large 0x00000000
int dimen notification_subtext_size 0x00000000
int dimen notification_top_pad 0x00000000
int dimen notification_top_pad_large_text 0x00000000
int dimen subtitle_corner_radius 0x00000000
int dimen subtitle_outline_width 0x00000000
int dimen subtitle_shadow_offset 0x00000000
int dimen subtitle_shadow_radius 0x00000000
int drawable notification_action_background 0x00000000
int drawable notification_bg 0x00000000
int drawable notification_bg_low 0x00000000
int drawable notification_bg_low_normal 0x00000000
int drawable notification_bg_low_pressed 0x00000000
int drawable notification_bg_normal 0x00000000
int drawable notification_bg_normal_pressed 0x00000000
int drawable notification_icon_background 0x00000000
int drawable notification_template_icon_bg 0x00000000
int drawable notification_template_icon_low_bg 0x00000000
int drawable notification_tile_bg 0x00000000
int drawable notify_panel_notification_icon_bg 0x00000000
int id accessibility_action_clickable_span 0x00000000
int id accessibility_custom_action_0 0x00000000
int id accessibility_custom_action_1 0x00000000
int id accessibility_custom_action_10 0x00000000
int id accessibility_custom_action_11 0x00000000
int id accessibility_custom_action_12 0x00000000
int id accessibility_custom_action_13 0x00000000
int id accessibility_custom_action_14 0x00000000
int id accessibility_custom_action_15 0x00000000
int id accessibility_custom_action_16 0x00000000
int id accessibility_custom_action_17 0x00000000
int id accessibility_custom_action_18 0x00000000
int id accessibility_custom_action_19 0x00000000
int id accessibility_custom_action_2 0x00000000
int id accessibility_custom_action_20 0x00000000
int id accessibility_custom_action_21 0x00000000
int id accessibility_custom_action_22 0x00000000
int id accessibility_custom_action_23 0x00000000
int id accessibility_custom_action_24 0x00000000
int id accessibility_custom_action_25 0x00000000
int id accessibility_custom_action_26 0x00000000
int id accessibility_custom_action_27 0x00000000
int id accessibility_custom_action_28 0x00000000
int id accessibility_custom_action_29 0x00000000
int id accessibility_custom_action_3 0x00000000
int id accessibility_custom_action_30 0x00000000
int id accessibility_custom_action_31 0x00000000
int id accessibility_custom_action_4 0x00000000
int id accessibility_custom_action_5 0x00000000
int id accessibility_custom_action_6 0x00000000
int id accessibility_custom_action_7 0x00000000
int id accessibility_custom_action_8 0x00000000
int id accessibility_custom_action_9 0x00000000
int id action0 0x00000000
int id action_container 0x00000000
int id action_divider 0x00000000
int id action_image 0x00000000
int id action_text 0x00000000
int id actions 0x00000000
int id all 0x00000000
int id async 0x00000000
int id blocking 0x00000000
int id bottom 0x00000000
int id cancel_action 0x00000000
int id center 0x00000000
int id center_horizontal 0x00000000
int id center_vertical 0x00000000
int id chronometer 0x00000000
int id clip_horizontal 0x00000000
int id clip_vertical 0x00000000
int id dialog_button 0x00000000
int id end 0x00000000
int id end_padder 0x00000000
int id fill 0x00000000
int id fill_horizontal 0x00000000
int id fill_vertical 0x00000000
int id forever 0x00000000
int id fragment_container_view_tag 0x00000000
int id icon 0x00000000
int id icon_group 0x00000000
int id info 0x00000000
int id italic 0x00000000
int id left 0x00000000
int id line1 0x00000000
int id line3 0x00000000
int id media_actions 0x00000000
int id none 0x00000000
int id normal 0x00000000
int id notification_background 0x00000000
int id notification_main_column 0x00000000
int id notification_main_column_container 0x00000000
int id right 0x00000000
int id right_icon 0x00000000
int id right_side 0x00000000
int id special_effects_controller_view_tag 0x00000000
int id start 0x00000000
int id status_bar_latest_event_content 0x00000000
int id tag_accessibility_actions 0x00000000
int id tag_accessibility_clickable_spans 0x00000000
int id tag_accessibility_heading 0x00000000
int id tag_accessibility_pane_title 0x00000000
int id tag_screen_reader_focusable 0x00000000
int id tag_transition_group 0x00000000
int id tag_unhandled_key_event_manager 0x00000000
int id tag_unhandled_key_listeners 0x00000000
int id text 0x00000000
int id text2 0x00000000
int id time 0x00000000
int id title 0x00000000
int id top 0x00000000
int id view_tree_lifecycle_owner 0x00000000
int id view_tree_saved_state_registry_owner 0x00000000
int id view_tree_view_model_store_owner 0x00000000
int id visible_removing_fragment_view_tag 0x00000000
int integer cancel_button_image_alpha 0x00000000
int integer status_bar_notification_info_maxnum 0x00000000
int layout custom_dialog 0x00000000
int layout notification_action 0x00000000
int layout notification_action_tombstone 0x00000000
int layout notification_media_action 0x00000000
int layout notification_media_cancel_action 0x00000000
int layout notification_template_big_media 0x00000000
int layout notification_template_big_media_custom 0x00000000
int layout notification_template_big_media_narrow 0x00000000
int layout notification_template_big_media_narrow_custom 0x00000000
int layout notification_template_custom_big 0x00000000
int layout notification_template_icon_group 0x00000000
int layout notification_template_lines_media 0x00000000
int layout notification_template_media 0x00000000
int layout notification_template_media_custom 0x00000000
int layout notification_template_part_chronometer 0x00000000
int layout notification_template_part_time 0x00000000
int string status_bar_notification_info_overflow 0x00000000
int style TextAppearance_Compat_Notification 0x00000000
int style TextAppearance_Compat_Notification_Info 0x00000000
int style TextAppearance_Compat_Notification_Info_Media 0x00000000
int style TextAppearance_Compat_Notification_Line2 0x00000000
int style TextAppearance_Compat_Notification_Line2_Media 0x00000000
int style TextAppearance_Compat_Notification_Media 0x00000000
int style TextAppearance_Compat_Notification_Time 0x00000000
int style TextAppearance_Compat_Notification_Time_Media 0x00000000
int style TextAppearance_Compat_Notification_Title 0x00000000
int style TextAppearance_Compat_Notification_Title_Media 0x00000000
int style Widget_Compat_NotificationActionContainer 0x00000000
int style Widget_Compat_NotificationActionText 0x00000000
int style Widget_Support_CoordinatorLayout 0x00000000
int[] styleable ColorStateListItem { 0x010101a5, 0x0101031f, 0x00000000 }
int styleable ColorStateListItem_android_color 0
int styleable ColorStateListItem_android_alpha 1
int styleable ColorStateListItem_alpha 2
int[] styleable CoordinatorLayout { 0x00000000, 0x00000000 }
int styleable CoordinatorLayout_keylines 0
int styleable CoordinatorLayout_statusBarBackground 1
int[] styleable CoordinatorLayout_Layout { 0x010100b3, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable CoordinatorLayout_Layout_android_layout_gravity 0
int styleable CoordinatorLayout_Layout_layout_anchor 1
int styleable CoordinatorLayout_Layout_layout_anchorGravity 2
int styleable CoordinatorLayout_Layout_layout_behavior 3
int styleable CoordinatorLayout_Layout_layout_dodgeInsetEdges 4
int styleable CoordinatorLayout_Layout_layout_insetEdge 5
int styleable CoordinatorLayout_Layout_layout_keyline 6
int[] styleable FontFamily { 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable FontFamily_fontProviderAuthority 0
int styleable FontFamily_fontProviderCerts 1
int styleable FontFamily_fontProviderFetchStrategy 2
int styleable FontFamily_fontProviderFetchTimeout 3
int styleable FontFamily_fontProviderPackage 4
int styleable FontFamily_fontProviderQuery 5
int[] styleable FontFamilyFont { 0x01010532, 0x01010533, 0x0101053f, 0x0101056f, 0x01010570, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 }
int styleable FontFamilyFont_android_font 0
int styleable FontFamilyFont_android_fontWeight 1
int styleable FontFamilyFont_android_fontStyle 2
int styleable FontFamilyFont_android_ttcIndex 3
int styleable FontFamilyFont_android_fontVariationSettings 4
int styleable FontFamilyFont_font 5
int styleable FontFamilyFont_fontStyle 6
int styleable FontFamilyFont_fontVariationSettings 7
int styleable FontFamilyFont_fontWeight 8
int styleable FontFamilyFont_ttcIndex 9
int[] styleable Fragment { 0x01010003, 0x010100d0, 0x010100d1 }
int styleable Fragment_android_name 0
int styleable Fragment_android_id 1
int styleable Fragment_android_tag 2
int[] styleable FragmentContainerView { 0x01010003, 0x010100d1 }
int styleable FragmentContainerView_android_name 0
int styleable FragmentContainerView_android_tag 1
int[] styleable GradientColor { 0x0101019d, 0x0101019e, 0x010101a1, 0x010101a2, 0x010101a3, 0x010101a4, 0x01010201, 0x0101020b, 0x01010510, 0x01010511, 0x01010512, 0x01010513 }
int styleable GradientColor_android_startColor 0
int styleable GradientColor_android_endColor 1
int styleable GradientColor_android_type 2
int styleable GradientColor_android_centerX 3
int styleable GradientColor_android_centerY 4
int styleable GradientColor_android_gradientRadius 5
int styleable GradientColor_android_tileMode 6
int styleable GradientColor_android_centerColor 7
int styleable GradientColor_android_startX 8
int styleable GradientColor_android_startY 9
int styleable GradientColor_android_endX 10
int styleable GradientColor_android_endY 11
int[] styleable GradientColorItem { 0x010101a5, 0x01010514 }
int styleable GradientColorItem_android_color 0
int styleable GradientColorItem_android_offset 1
Binary file not shown.
@@ -1,26 +0,0 @@
package com.hmwl.sample;
import android.content.Context;
import androidx.test.platform.app.InstrumentationRegistry;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
import static org.junit.Assert.*;
/**
* Instrumented test, which will execute on an Android device.
*
* @see <a href="http://d.android.com/tools/testing">Testing documentation</a>
*/
@RunWith(AndroidJUnit4.class)
public class ExampleInstrumentedTest {
@Test
public void useAppContext() {
// Context of the app under test.
Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext();
assertEquals("com.hmwl.sample", appContext.getPackageName());
}
}
+20 -19
View File
@@ -2,33 +2,34 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
<!-- Declare features -->
<uses-feature android:name="android.hardware.camera" />
<!-- 声明摄像头硬件特性 -->
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
<!-- Declare permissions -->
<!-- 声明摄像头权限 -->
<uses-permission android:name="android.permission.CAMERA" />
<application
android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/Theme.Sample">
<application>
<!-- 声明 MainActivity -->
<!-- 作为库,exported 通常设为 false,除非明确需要外部应用启动 -->
<activity
android:name=".MainActivity"
android:exported="true">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
android:name=".FaceActivity"
android:exported="true"
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize"
tools:ignore="MissingClass,LockedOrientationActivity">
<!-- 锁定竖屏:SDK 渲染按"屏幕中央正方形画布 + 上下黑边 letterbox"
的方式适配任意分辨率,目前只验证过竖屏。
configChanges 全包:避免软键盘/导航栏变化触发 Activity 重建——SDK
自己会通过 onWindowLost/onWindowInit 重建 swapchain,性能更好。 -->
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
<!-- 重要:移除 MAIN/LAUNCHER intent-filter -->
<!-- 让使用库的应用来决定哪个是主Activity -->
<meta-data
android:name="android.app.lib_name"
android:value="sample" />
android:value="face_sdk" />
</activity>
</application>
Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 47 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

File diff suppressed because it is too large Load Diff
Binary file not shown.

Before

Width:  |  Height:  |  Size: 139 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

+38 -7
View File
@@ -1,15 +1,46 @@
#version 450
// 输入变量(与顶点着色器输出对应)
layout(location = 0) in vec2 inTexCoord;
// 输出颜色
layout(location = 0) out vec4 outColor;
// 纹理采样器
layout(binding = 0) uniform sampler2D texSampler;
// ★ PushConstants 必须和 bg.vert / FaceApp.h 完全一致(包括我们没用到的
// 字段 zoom/ux/uy/offset_x/offset_y,因为 push constant 是按 offset 取的,
// 缺字段会导致后面 radius/screen_w/screen_h 落到错误偏移)。
layout(push_constant) uniform PushConstants {
float zoom;
float r;
float g;
float b;
float radius;
float ux;
float uy;
float offset_x;
float offset_y;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(fragment 不用,仅为对齐 push constant layout
float camera_aspect;
float camera_rotation;
float mirror_x;
} pc;
void main() {
// 采样纹理
outColor = texture(texSampler, inTexCoord);
}
// gl_FragCoord 是屏幕 framebuffer 里的像素坐标(左下原点)。圆形 mask 中心
// 锚定在屏幕真实中心 = (screen_w, screen_h)/2。这里不再用画布中心,因为
// letterbox 后画布中心和屏幕中心其实是同一个点(min 边居中),但用屏幕坐标
// 写法更直观也对 cover 模式更友好。
vec2 fragCoord = gl_FragCoord.xy;
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
float dist = length(fragCoord - center);
if (dist > pc.radius) {
// 画布外(含 letterbox 黑边)+ 圆外区域统一用业务给的纯色覆盖。
// 业务一般给 r=g=0 / b=1 之类,与原版语义保持一致。
outColor = vec4(pc.r, pc.g, pc.b, 1.0);
} else {
outColor = texture(texSampler, inTexCoord);
}
}
Binary file not shown.
+120 -47
View File
@@ -1,58 +1,131 @@
#version 450
// 硬编码的顶点数据 - 4个顶点组成三角形带覆盖整个屏幕
// const vec2 positions[6] = vec2[6](
// vec2( 1.0, 1.0), // 右上 - 三角形1
// vec2(-1.0, 1.0), // 左上 - 三角形1
// vec2(-1.0, -1.0), // 左下 - 三角形1
// vec2(-1.0, -1.0), // 左下 - 三角形2
// vec2( 1.0, -1.0), // 右下 - 三角形2
// vec2( 1.0, 1.0) // 右上 - 三角形2
// );
const float offset = (640-480)/(480.0);
// ★ bg quad 顶点放大到 ±10(而不是 ±1)的原因:
//
// bg 与 face mesh 的对齐依赖一个关键不变量——
// "屏幕上同一像素位置,bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe
// landmark 坐标"。要保住这个不变量,bg quad 必须经过和 face mesh 完全相
// 同的 zoom × offset × letterbox × mirror 链路 (见下方 gl_Position 计算)
// 而 outTexCoord 必须基于"未变换的 quad 顶点 pos"算 raw UV。这样屏幕 fragment
// 反算出的 quad pos 就等价于"对应的 landmark NDC"UV 完美对齐。
//
// 但是 zoom (>1) + 非零 offset + screen_h > screen_w 的 letterbox 会让 quad
// 在屏幕上呈非对称分布——之前 quad 顶点 ±1 时实测在 zoom=1.5 offset=(450,-450)
// 屏幕 1080×2340 下,quad 在 NDC.y 只覆盖 [-0.885, 0.500],屏幕底部
// [0.500, 1.0] 大片 letterbox 没有被 bg 覆盖,露出 RenderPass 的 clearColor
// 黑色——表现为"上面有蓝色填充,下面没有"。
//
// 解法:把 quad 顶点扩大到 ±10。GPU 会把超出 NDC [-1, 1] 的部分硬件 clip 掉,
// 但 clip 之后留在屏幕内的 fragment 上,outTexCoord 是顶点的"线性插值"**屏幕
// 内 fragment 反算出的 pos 与 quad 顶点取值范围无关**——所以 face mesh 对齐
// 完全不受影响。同时 ±10 在任何合理的 zoom/offset/letterbox 下都能保证 quad
// 在屏幕上完全覆盖 [-1, 1]² 含 letterbox。
//
// Worst-case 验证:zoom=1.5, offset=(450,-450), canvas=1080, screen=1080×2340
// pos.y = -10 → screen_ndc.y = (-10*1.5 + (-450/1080)) * 1080/2340 = -7.12 ✓
// pos.y = +10 → screen_ndc.y = (+10*1.5 + (-450/1080)) * 1080/2340 = +6.73 ✓
// 远超屏幕范围,clip 后保证整屏 letterbox 都被 bg.frag 的 r/g/b 蓝色覆盖。
const vec2 positions[6] = vec2[6](
vec2( 1.0, -1.0 -offset), // 右下
vec2(1.0, 1.0 + offset), // 右上 - 三角形1
vec2(-1.0, 1.0 + offset), // 左上 - 三角形1
vec2( 10.0, -10.0),
vec2( 10.0, 10.0),
vec2(-10.0, 10.0),
vec2(-1.0, 1.0 + offset), // 左上 - 三角形2
vec2( -1.0, -1.0-offset), // 左下 - 三角形2
vec2( 1.0, -1.0-offset) // 右下 - 三角形2
vec2(-10.0, 10.0),
vec2(-10.0, -10.0),
vec2( 10.0, -10.0)
);
const vec2 texCoords[6] = vec2[6](
vec2(1.0, 1.0), // 右上
vec2(0.0, 1.0), // 左上
vec2(0.0, 0.0), // 左下
vec2(0.0, 0.0), // 左下
vec2(1.0, 0.0), // 右下
vec2(1.0, 1.0) // 右上
);
// 定义 Push Constant,只有一个 float
// ★ PushConstants 必须和 C++ FaceApp.h 严格一致。前 9 个 float 是业务原始值
// SDK 已按 canvas_size/480 缩放到屏幕物理像素),后 5 个 float 是当前帧
// 的画布几何 + 相机帧元信息,由 FaceApp::render() 每帧写入。
layout(push_constant) uniform PushConstants {
float myValue;
} pushConstants;
float zoom;
float r;
float g;
float b;
float radius;
float ux;
float uy;
float offset_x;
float offset_y;
// 画布几何(每帧写入)
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(每次相机帧上传时写入)
float camera_aspect; // raw width / height(典型 4:3 = 1.333…)
float camera_rotation; // CameraX rotationDegrees: 0/90/180/270
// 镜子效果开关:1.0 = 前置摄像头,最终 NDC.x 翻转一次。
// 这与 FaceLandmarkerHelper 在前置时对 bitmap 做 postScale(-1,1) 的语义对偶:
// ① mediapipe 输入是"用户视角"图像 → landmark 也是用户视角;
// ② bg.vert 算出的 outTexCoord 还是 raw 帧 UV(旋转矩阵把 user 视角→raw);
// ③ 最后我们把 NDC.x 翻转,bg 在屏幕上呈现为水平镜像 = 镜子效果;
// ④ texture.vert 同样翻转一次,face 贴图与 bg 完美对齐。
float mirror_x;
} pc;
// 输出变量
layout(location = 0) out vec2 outTexCoord;
void main() {
// 获取顶点位置
vec2 position = positions[gl_VertexIndex];
position = position * pushConstants.myValue;
// 设置输出位置(Vulkan使用不同的坐标系)
gl_Position = vec4(position, 0.0, 1.0);
// 输出纹理坐标
outTexCoord = texCoords[gl_VertexIndex];
}
vec2 pos = positions[gl_VertexIndex];
// ===== outTexCoord:基于"未变换的 quad pos" 算 raw UV =====
//
// 重点:这里必须用未做 zoom/offset/letterbox/mirror 变换的原始 pos 来计算
// outTexCoord,而 gl_Position 在下面才做这些变换。这样:
// - 屏幕 fragment X 处对应的 quad pos = (X 反过 mirror、letterbox、offset、zoom)
// - face mesh 顶点的 mediapipe landmark 在屏幕上的位置 X' 经过相同链路
// - 当 X = X' 时(face mesh 顶点重叠某个屏幕像素),fragment 反算的 pos
// 刚好等于 face landmark 的画布 NDCbg 该 fragment 显示 landmark 对应
// raw UV → 与 face mesh 完美贴合。
//
// 公式链:画布 NDC ─[逆时针 rotation]→ 中央方形 NDC ─[scale & shift]→ raw UV
//
// 关于旋转方向:
// Android Matrix.postRotate(deg) 在 Bitmap canvasy 朝下)里是「视觉顺时针」。
// Vulkan NDC 也是 y 朝下。所以画布 NDC → 中央方形 NDC 是「视觉逆时针 rotation」。
// GLSL column-major mat2(c, -s, s, c) 对应矩阵 [c, s; -s, c]。
// 验证 rotation=0 时矩阵 = identity ✓
// 验证 rotation=90 时矩阵 = [0, 1; -1, 0]apply 到画布右上 (1,-1)
// (0*1 + 1*(-1), -1*1 + 0*(-1)) = (-1, -1) → 中央方形左上 ✓
// 物理:mediapipe 输入右上像素 = 中央方形顺时针 90° 后的右上 = 中央方形左上 ✓
float ang = radians(pc.camera_rotation);
float c = cos(ang);
float s = sin(ang);
mat2 R = mat2(c, -s, s, c); // 画布 NDC → 中央方形 NDC(视觉逆时针 rotation
vec2 cm = R * pos;
// 中央方形 NDC ∈ [-1, +1]² → raw 帧 UV。中央方形在 raw 帧 UV 上是中央
// min(W, H) x min(W, H) 的正方形,对应的 UV 半径:
// raw 横向长(W >= H):halfU = H/(2W) = 1/(2 * aspect)halfV = 0.5
// raw 纵向长(H > W):halfU = 0.5halfV = W/(2H) = aspect/2
float halfU, halfV;
if (pc.camera_aspect >= 1.0) {
halfU = 0.5 / pc.camera_aspect;
halfV = 0.5;
} else {
halfU = 0.5;
halfV = 0.5 * pc.camera_aspect;
}
outTexCoord = vec2(0.5 + halfU * cm.x, 0.5 + halfV * cm.y);
// ===== gl_Position:业务 zoom/offset + letterbox + mirror,与 texture.vert 严格一致 =====
//
// pos 仍按 quad 顶点 ±10 参与位置计算,pc.offset_x / pc.offset_y 已是屏幕物理
// 像素(SDK 端做过 canvas_size/480 缩放),/canvas_size 后是画布 NDC 单位;
// 最后 *canvas_size/screen_{w,h} 把画布缩到屏幕中央正方形,长边方向自然
// letterbox。quad 顶点放大到 ±10 让 GPU clip 后整屏覆盖。
pos *= pc.zoom;
pos.x += pc.offset_x / pc.canvas_size;
pos.y += pc.offset_y / pc.canvas_size;
pos.x *= pc.canvas_size / pc.screen_w;
pos.y *= pc.canvas_size / pc.screen_h;
// 镜子效果:前置时 mirror_x=1.0,把 NDC.x 翻转一次。无分支:
// 后置:(1 - 2*0) = +1 → 不变;
// 前置:(1 - 2*1) = -1 → 水平翻转。
// texture.vert 也做同样翻转,bg + face 整体水平镜像,face 贴图与 bg 完美对齐。
pos.x *= (1.0 - 2.0 * pc.mirror_x);
gl_Position = vec4(pos, 0.0, 1.0);
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
+39 -13
View File
@@ -1,26 +1,52 @@
#version 450
#version 450
layout (binding = 1) uniform sampler2D samplerColor;
layout (location = 0) in vec2 inUV;
layout (location = 1) in vec3 inNormal;
// ★ PushConstants 必须和 C++ FaceApp.h / texture.vert 严格一致。
// 关键:原版这里只声明到 uy,缺了 offset_x 之后的字段,那时之所以没报错是因为
// fragment 没读到那些字段。本次重构 fragment 要读 screen_w/screen_h,所以必须
// 把整个块声明完整,否则 layout offset 会错位。
layout(push_constant) uniform PushConstants {
float zoom;
float r;
float g;
float b;
float radius;
float ux;
float uy;
float offset_x;
float offset_y;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(fragment 不用,仅为对齐 push constant layout
float camera_aspect;
float camera_rotation;
float mirror_x;
} pc;
layout (location = 0) out vec4 outFragColor;
void main()
{
vec4 color = texture(samplerColor, inUV);
// gl_FragCoord 是屏幕 framebuffer 像素坐标。圆形 mask 中心锚定在屏幕真实中心,
// 跟随分辨率自动适配;不再写死 (240,240)。
vec2 fragCoord = gl_FragCoord.xy;
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
float dist = length(fragCoord - center);
vec3 N = normalize(inNormal);
vec4 textureColor = color;
if (textureColor.a < 0.1) {
//textureColor = vec4(0.2, 0.2, 0.2, 0.5);
discard;
if (dist > pc.radius) {
// 画布外(含 letterbox 黑边)+ 圆外区域用业务纯色覆盖,与 bg.frag 一致。
outFragColor = vec4(pc.r, pc.g, pc.b, 1.0);
} else {
// out.png 是 4096x2048 的 8x4 子图图集;ux/uy 选定当前 motion 帧子图,
// inUV 在子图内插值。这里和分辨率重构无关,保持原版逻辑不动。
vec2 pos = vec2(pc.ux / 4096.0, pc.uy / 2048.0);
vec2 uv = pos + vec2(inUV.x / 8.0, inUV.y / 4.0);
vec4 color = texture(samplerColor, uv);
outFragColor = color;
}
outFragColor = textureColor;
// outFragColor = vec4(1, 0, 0, 1);
}
}
Binary file not shown.
+51 -8
View File
@@ -1,6 +1,5 @@
#version 450
layout (location = 0) in vec3 inPos;
layout (location = 1) in vec2 inUV;
layout (location = 2) in vec3 inNormal;
@@ -16,9 +15,31 @@ layout (binding = 0) uniform UBO
layout (location = 0) out vec2 outUV;
layout (location = 1) out vec3 outNormal;
// ★ PushConstants 必须和 C++ FaceApp.h / bg.vert / bg.frag 严格一致。
// 前 9 个 float 是业务原始值(已被 SDK 在 render() 里按 canvas_size/480 缩放成
// 屏幕物理像素);后 3 个 float 是当前帧画布几何,由 SDK 每帧写入。
layout(push_constant) uniform PushConstants {
float myValue;
} pushConstants;
float zoom;
float r;
float g;
float b;
float radius;
float ux;
float uy;
float offset_x;
float offset_y;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息:camera_aspect / camera_rotation 在 face 渲染中不用——face
// landmark 已经是"正立中央方形"的归一化坐标,mediapipe 端转过;保留只是
// 为了对齐 push constant layout。
float camera_aspect;
float camera_rotation;
// mirror_x:1.0 表示前置摄像头镜子效果——把最终 gl_Position.x 翻转一次。
// bg.vert 也做同样翻转,所以 bg 与 face 同步水平镜像、互相对齐。
float mirror_x;
} pc;
out gl_PerVertex
{
@@ -29,11 +50,33 @@ void main()
{
outUV = inUV;
vec2 position = vec2(inPos.xy*2 -1);
position = position * pushConstants.myValue;
gl_Position = vec4(position, 0.5, 1.0);
// 模型顶点坐标 inPos.xy 来自 OBJ,已经是 [0,1]² 归一化范围。先映射到
// "画布 NDC ∈ [-1,+1]"。
vec2 position = vec2(inPos.xy * 2.0 - 1.0);
vec4 pos = ubo.model * vec4(inPos, 1.0);
// 画布坐标系:缩放 + 业务像素偏移。
// pc.offset_x / pc.offset_y 是屏幕物理像素(SDK 已做过 canvas_size/480 缩放),
// 除以 canvas_size 转成画布 NDC,跟原版 "/480" 在 480 屏上完全等价。
position = position * pc.zoom;
position.x = position.x + (pc.offset_x / pc.canvas_size);
position.y = position.y + (pc.offset_y / pc.canvas_size);
// 画布 NDC → 屏幕 NDC:屏幕长边方向乘以 canvas_size/screen_long < 1.0
// 让模型只占据屏幕中央正方形画布区域,长边方向的画布外是 letterbox。
position.x = position.x * (pc.canvas_size / pc.screen_w);
position.y = position.y * (pc.canvas_size / pc.screen_h);
// 镜子效果:前置摄像头时 mirror_x=1.0,把 NDC.x 翻转一次。
// 后置:(1 - 2*0) = +1 → 不变
// 前置:(1 - 2*1) = -1 → 水平翻转
// 无分支写法,配合 bg.vert 同样的翻转,让 bg + face 整体镜像、保持对齐。
position.x *= (1.0 - 2.0 * pc.mirror_x);
gl_Position = vec4(position, 0.5, 1.0);
// 注意:ubo.projection / ubo.model 在当前管线里没有真正参与 gl_Position 的
// 计算(顶点位置已直接给出 NDC)。这里仍按原版保留 normal 的世界变换,避免
// 影响其它依赖 outNormal 的 fragment 逻辑(例如 thick 版本)。
vec4 pos = ubo.model * vec4(inPos, 1.0);
outNormal = mat3(inverse(transpose(ubo.model))) * inNormal;
}
Binary file not shown.
@@ -0,0 +1,56 @@
#version 450
layout (binding = 1) uniform sampler2D samplerColor;
layout (binding = 2) uniform sampler2D samplerColor_ex;
layout (location = 0) in vec2 inUV;
layout (location = 1) in vec3 inNormal;
layout (location = 0) out vec4 outFragColor;
void main()
{
vec4 color = texture(samplerColor, inUV);
vec4 color_ex = texture(samplerColor_ex, inUV);
vec3 N = normalize(inNormal);
// 计算与 (0,0,1) 的点乘
float ndot = dot(N, vec3(0.0, 0.0, -1.0));
// 保证结果 >= 0
ndot = max(ndot, 0.0);
// 重新映射ndot值,让中间2/3部分进行融合
// 当ndot在[0.166, 0.833]范围内时进行混合,两端保持不变
float blendFactor;
if (ndot < 0.0) {
blendFactor = 0.0; // 完全使用color_ex
} else if (ndot > 0.1) {
blendFactor = 1.0; // 完全使用color
} else {
// 在中间2/3部分进行平滑过渡
blendFactor = (ndot - 0.0) / (1 - 0);
}
// 使用smoothstep创建中间2/3的融合区域
// 参数说明:smoothstep(edge0, edge1, x)
// 当x < edge0时返回0,当x > edge1时返回1,在中间时平滑过渡
//float blendFactor = smoothstep(0.166, 0.833, ndot);
// vec4 blendedColor = mix(color_ex, color, ndot);
// outFragColor = blendedColor;
if (ndot < 0.3){
outFragColor = color_ex;
}
else
{
outFragColor = color;
}
// outFragColor = vec4(1, 0, 0, 1);
}
Binary file not shown.
+512
View File
@@ -0,0 +1,512 @@
// CrashHandler.cpp
//
// Async-signal-safe crash dumper for the face_sdk native library. The goal
// is that the next time the app dies (e.g. another FORTIFY: pthread_mutex_lock
// called on a destroyed mutex from inside the Vulkan driver) we don't only
// have logcat — we also have a self-contained dump on the device's app data
// directory that the user can pull off and send back, even after a reboot.
//
// Why we need this:
// - DebugLog only writes "happy path" events. The Vulkan crash happens
// synchronously inside vkQueueSubmit / vkFreeCommandBuffers — there is
// no DebugLog call near the crash site, so the file log just stops.
// - logcat survives across the crash (debuggerd dumps the backtrace there)
// but logcat is volatile: a couple of reboots, a logcat -c, or a long
// idle period and it's gone. We want a persistent file.
// - tombstones in /data/tombstones/ are root-only on consumer devices.
//
// Implementation notes:
// - Inside a signal handler we MUST stick to async-signal-safe APIs
// (man 7 signal-safety). That rules out fprintf / snprintf / malloc.
// We use write(), our own integer-to-string conversion, and a single
// pre-opened fd. dladdr() is technically not on the POSIX safe list but
// bionic's implementation only takes one rwlock and is widely used in
// other crash dumpers (breakpad, crashpad, libunwindstack). We accept
// that risk because the alternative is no symbol at all.
// - We use <unwind.h> (_Unwind_Backtrace) instead of execinfo.h because
// bionic doesn't ship execinfo.h on all NDK levels, and _Unwind_Backtrace
// is the same primitive Android's own tombstoned uses.
// - We re-raise the original signal with the default handler at the end so
// the OS still produces a tombstone / ANR record for vendors that
// read /data/tombstones/.
#include "CrashHandler.h"
#ifndef _WIN32
#include <android/log.h>
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <signal.h>
#include <stdint.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <unwind.h>
#include <atomic>
namespace {
// ---------- Globals (touched from the handler -> only POD/atomics) ---------
constexpr size_t kCrashStackSize = 64 * 1024; // sigaltstack
constexpr size_t kMaxFrames = 64;
constexpr size_t kNoteCapacity = 256;
uint8_t g_sigStack[kCrashStackSize];
int g_crashFd = -1; // crash log fd (append, sync)
int g_debugFd = -1; // optional: also dup write to debug log
std::atomic<bool> g_installed{false};
std::atomic<bool> g_handlingCrash{false};
// Single writer of g_note: setNote() (uses memcpy under a tiny lock, but the
// handler reads byte-by-byte so a torn read at most produces a truncated
// note, never a deref of bad memory).
char g_note[kNoteCapacity] = {0};
pthread_mutex_t g_noteMtx = PTHREAD_MUTEX_INITIALIZER;
const int g_signals[] = { SIGSEGV, SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGSYS };
constexpr size_t kNumSignals = sizeof(g_signals) / sizeof(g_signals[0]);
// ----------------------- Async-signal-safe writers -------------------------
// Write a NUL-terminated string. Drops the trailing NUL.
void sigWrite(int fd, const char* s) {
if (fd < 0 || !s) return;
size_t n = 0;
while (s[n]) ++n;
if (n == 0) return;
// Loop until everything is written or we error out. Async-safe.
while (n > 0) {
ssize_t w = write(fd, s, n);
if (w <= 0) {
if (w < 0 && errno == EINTR) continue;
return;
}
s += w;
n -= (size_t)w;
}
}
// Write n bytes from buf. Used for the note (may contain anything).
void sigWriteN(int fd, const char* buf, size_t n) {
if (fd < 0 || !buf) return;
while (n > 0) {
ssize_t w = write(fd, buf, n);
if (w <= 0) {
if (w < 0 && errno == EINTR) continue;
return;
}
buf += w;
n -= (size_t)w;
}
}
// Write the same string to both the crash log and the debug log (if any).
void sigDump(const char* s) {
sigWrite(g_crashFd, s);
sigWrite(g_debugFd, s);
}
// Convert an unsigned integer to its decimal representation. Returns the
// number of characters written into buf (without a NUL).
size_t u64ToDec(uint64_t v, char* buf, size_t cap) {
if (cap == 0) return 0;
char tmp[32];
size_t i = 0;
if (v == 0) {
tmp[i++] = '0';
} else {
while (v && i < sizeof(tmp)) {
tmp[i++] = (char)('0' + (v % 10));
v /= 10;
}
}
size_t out = (i < cap) ? i : cap;
for (size_t k = 0; k < out; ++k) {
buf[k] = tmp[i - 1 - k];
}
return out;
}
// Convert an unsigned 64-bit value to a fixed-width 16-digit hex string.
// Useful for PC values.
size_t u64ToHex16(uint64_t v, char* buf, size_t cap) {
static const char digits[] = "0123456789abcdef";
if (cap < 16) return 0;
for (int i = 15; i >= 0; --i) {
buf[i] = digits[v & 0xF];
v >>= 4;
}
return 16;
}
// Write "<key>=<u64>\n".
void sigWriteKV_u64(int fd, const char* key, uint64_t v) {
sigWrite(fd, key);
sigWrite(fd, "=");
char num[24];
size_t n = u64ToDec(v, num, sizeof(num));
sigWriteN(fd, num, n);
sigWrite(fd, "\n");
}
// Write "<key>=0x<hex>\n".
void sigWriteKV_ptr(int fd, const char* key, uint64_t v) {
sigWrite(fd, key);
sigWrite(fd, "=0x");
char hx[16];
u64ToHex16(v, hx, sizeof(hx));
sigWriteN(fd, hx, 16);
sigWrite(fd, "\n");
}
// ------------------------- Signal name lookup ------------------------------
const char* signalName(int signo) {
switch (signo) {
case SIGSEGV: return "SIGSEGV";
case SIGABRT: return "SIGABRT";
case SIGBUS: return "SIGBUS";
case SIGFPE: return "SIGFPE";
case SIGILL: return "SIGILL";
case SIGSYS: return "SIGSYS";
case SIGTRAP: return "SIGTRAP";
default: return "SIG?";
}
}
// si_code to short string. Only the most common ones; everything else falls
// back to the numeric value via sigWriteKV_u64.
const char* siCodeName(int signo, int code) {
switch (signo) {
case SIGSEGV:
if (code == SEGV_MAPERR) return "SEGV_MAPERR";
if (code == SEGV_ACCERR) return "SEGV_ACCERR";
break;
case SIGBUS:
if (code == BUS_ADRALN) return "BUS_ADRALN";
if (code == BUS_ADRERR) return "BUS_ADRERR";
if (code == BUS_OBJERR) return "BUS_OBJERR";
break;
case SIGFPE:
if (code == FPE_INTDIV) return "FPE_INTDIV";
if (code == FPE_INTOVF) return "FPE_INTOVF";
if (code == FPE_FLTDIV) return "FPE_FLTDIV";
break;
case SIGILL:
if (code == ILL_ILLOPC) return "ILL_ILLOPC";
if (code == ILL_ILLOPN) return "ILL_ILLOPN";
break;
case SIGABRT:
if (code == SI_TKILL) return "SI_TKILL";
if (code == SI_USER) return "SI_USER";
break;
}
return "?";
}
// ------------------------- Backtrace via _Unwind_Backtrace -----------------
struct UnwindCtx {
uintptr_t* frames;
size_t count;
size_t cap;
};
_Unwind_Reason_Code unwindCallback(_Unwind_Context* ctx, void* arg) {
UnwindCtx* uc = static_cast<UnwindCtx*>(arg);
if (uc->count >= uc->cap) return _URC_END_OF_STACK;
uintptr_t pc = _Unwind_GetIP(ctx);
if (pc) {
// Trim Thumb bit on 32-bit ARM. No-op on aarch64/x86_64.
pc &= ~(uintptr_t)1;
uc->frames[uc->count++] = pc;
}
return _URC_NO_REASON;
}
size_t captureBacktrace(uintptr_t* out, size_t cap) {
UnwindCtx uc{out, 0, cap};
_Unwind_Backtrace(&unwindCallback, &uc);
return uc.count;
}
// Dump one frame: " #02 pc 000000000000abcd /path/lib.so (Symbol+0x10)"
void dumpFrame(int fd, size_t idx, uintptr_t pc) {
sigWrite(fd, " #");
char num[8];
if (idx < 10) {
num[0] = '0';
num[1] = (char)('0' + idx);
sigWriteN(fd, num, 2);
} else {
size_t n = u64ToDec(idx, num, sizeof(num));
sigWriteN(fd, num, n);
}
sigWrite(fd, " pc ");
char hx[16];
u64ToHex16((uint64_t)pc, hx, sizeof(hx));
sigWriteN(fd, hx, 16);
Dl_info info;
memset(&info, 0, sizeof(info));
if (dladdr(reinterpret_cast<void*>(pc), &info) && info.dli_fname) {
sigWrite(fd, " ");
sigWrite(fd, info.dli_fname);
if (info.dli_sname) {
uintptr_t sym = reinterpret_cast<uintptr_t>(info.dli_saddr);
uintptr_t off = (sym && pc >= sym) ? (pc - sym) : 0;
sigWrite(fd, " (");
sigWrite(fd, info.dli_sname);
sigWrite(fd, "+0x");
char ohx[16];
u64ToHex16((uint64_t)off, ohx, sizeof(ohx));
sigWriteN(fd, ohx, 16);
sigWrite(fd, ")");
} else if (info.dli_fbase) {
uintptr_t base = reinterpret_cast<uintptr_t>(info.dli_fbase);
uintptr_t off = (pc >= base) ? (pc - base) : 0;
sigWrite(fd, " (offset 0x");
char ohx[16];
u64ToHex16((uint64_t)off, ohx, sizeof(ohx));
sigWriteN(fd, ohx, 16);
sigWrite(fd, ")");
}
}
sigWrite(fd, "\n");
}
// ------------------------- Time + tid helpers ------------------------------
// Builds "YYYY-MM-DD HH:MM:SS.mmm UTC" into the given buffer (no NUL).
// Returns the number of bytes written. Async-signal-safe (no stdio, no
// localtime_r tz lookups).
size_t formatTimestamp(char* b, size_t cap) {
timespec ts{};
clock_gettime(CLOCK_REALTIME, &ts);
struct tm tm_info{};
time_t s = ts.tv_sec;
gmtime_r(&s, &tm_info);
size_t i = 0;
auto putUInt = [&](unsigned v, int width) {
char tmp[8];
size_t n = u64ToDec(v, tmp, sizeof(tmp));
while ((int)n < (size_t)width) {
if (i < cap) b[i++] = '0';
++n;
}
for (size_t k = 0; k < n; ++k) {
if (i < cap) b[i++] = tmp[k];
}
};
putUInt((unsigned)(tm_info.tm_year + 1900), 4); if (i < cap) b[i++] = '-';
putUInt((unsigned)(tm_info.tm_mon + 1), 2); if (i < cap) b[i++] = '-';
putUInt((unsigned)(tm_info.tm_mday), 2); if (i < cap) b[i++] = ' ';
putUInt((unsigned)(tm_info.tm_hour), 2); if (i < cap) b[i++] = ':';
putUInt((unsigned)(tm_info.tm_min), 2); if (i < cap) b[i++] = ':';
putUInt((unsigned)(tm_info.tm_sec), 2); if (i < cap) b[i++] = '.';
putUInt((unsigned)(ts.tv_nsec / 1000000), 3);
static const char kSuffix[] = " UTC";
for (size_t k = 0; k < sizeof(kSuffix) - 1; ++k) {
if (i < cap) b[i++] = kSuffix[k];
}
return i;
}
pid_t currentTid() {
return static_cast<pid_t>(syscall(SYS_gettid));
}
// ------------------------- Signal handler ----------------------------------
void crashHandler(int signo, siginfo_t* info, void* ucontext) {
(void)ucontext;
// Re-entry guard: if a second signal fires while we're dumping (e.g. our
// own dladdr trips a SIGSEGV) just chain to the default handler.
bool expected = false;
if (!g_handlingCrash.compare_exchange_strong(expected, true,
std::memory_order_acq_rel)) {
// Already in handler -> default + bail.
signal(signo, SIG_DFL);
raise(signo);
return;
}
sigDump("\n========== FACE_SDK CRASH ==========\n");
sigDump("time=");
{
char tsbuf[48];
size_t tn = formatTimestamp(tsbuf, sizeof(tsbuf));
sigWriteN(g_crashFd, tsbuf, tn);
sigWriteN(g_debugFd, tsbuf, tn);
}
sigDump("\n");
sigDump("signal=");
sigDump(signalName(signo));
sigDump(" code=");
sigDump(siCodeName(signo, info ? info->si_code : 0));
sigDump("\n");
if (info) {
sigWriteKV_u64(g_crashFd, "si_signo", (uint64_t)info->si_signo);
sigWriteKV_u64(g_debugFd, "si_signo", (uint64_t)info->si_signo);
sigWriteKV_u64(g_crashFd, "si_code", (uint64_t)info->si_code);
sigWriteKV_u64(g_debugFd, "si_code", (uint64_t)info->si_code);
sigWriteKV_ptr(g_crashFd, "si_addr", (uint64_t)(uintptr_t)info->si_addr);
sigWriteKV_ptr(g_debugFd, "si_addr", (uint64_t)(uintptr_t)info->si_addr);
}
sigWriteKV_u64(g_crashFd, "pid", (uint64_t)getpid());
sigWriteKV_u64(g_debugFd, "pid", (uint64_t)getpid());
sigWriteKV_u64(g_crashFd, "tid", (uint64_t)currentTid());
sigWriteKV_u64(g_debugFd, "tid", (uint64_t)currentTid());
// Note (current frame index, current motion, ...).
sigDump("note=");
sigWriteN(g_crashFd, g_note, strnlen(g_note, kNoteCapacity));
sigWriteN(g_debugFd, g_note, strnlen(g_note, kNoteCapacity));
sigDump("\n");
sigDump("backtrace:\n");
uintptr_t frames[kMaxFrames];
size_t nf = captureBacktrace(frames, kMaxFrames);
for (size_t i = 0; i < nf; ++i) {
dumpFrame(g_crashFd, i, frames[i]);
dumpFrame(g_debugFd, i, frames[i]);
}
sigDump("==================================\n");
// Make sure everything reaches disk before we self-destruct.
if (g_crashFd >= 0) fsync(g_crashFd);
if (g_debugFd >= 0) fsync(g_debugFd);
// Mirror to logcat too so a quick `adb logcat -d` after a reboot still
// shows the SIGNAL line (helps cross-checking the file).
__android_log_print(ANDROID_LOG_FATAL, "FACE_DBG_CRASH",
"fatal %s @ tid=%d, see face_sdk_crash.log",
signalName(signo), currentTid());
// Restore default handler and re-raise. This produces /data/tombstones/*
// on rooted devices and tells debuggerd to print the official Android
// backtrace into logcat (`crash_dump64 ... DEBUG`).
struct sigaction dfl{};
dfl.sa_handler = SIG_DFL;
sigemptyset(&dfl.sa_mask);
sigaction(signo, &dfl, nullptr);
raise(signo);
}
} // namespace
namespace CrashHandler {
void install(const std::string& internalDataPath,
const std::string& debugLogPath) {
bool expected = false;
if (!g_installed.compare_exchange_strong(expected, true,
std::memory_order_acq_rel)) {
return; // already installed
}
// Open the persistent crash log file in append mode. We keep it open
// forever so the signal handler doesn't have to call open() (which is
// safe but slow).
{
std::string path = internalDataPath.empty()
? std::string("/data/local/tmp/face_sdk_crash.log")
: internalDataPath + "/face_sdk_crash.log";
g_crashFd = open(path.c_str(),
O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
0644);
if (g_crashFd >= 0) {
// Header for the new run.
sigWrite(g_crashFd, "\n=== CrashHandler installed pid=");
char pidbuf[16];
size_t n = u64ToDec((uint64_t)getpid(), pidbuf, sizeof(pidbuf));
sigWriteN(g_crashFd, pidbuf, n);
sigWrite(g_crashFd, " ===\n");
fsync(g_crashFd);
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG",
"CrashHandler log file: %s", path.c_str());
} else {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler failed to open %s: %s",
path.c_str(), strerror(errno));
}
}
// Also keep a writable fd to the DebugLog file (if any) so dumps land
// next to the regular tail of the log. We don't touch g_fp inside
// DebugLog because that would need its mutex — not safe in handler.
if (!debugLogPath.empty()) {
g_debugFd = open(debugLogPath.c_str(),
O_WRONLY | O_APPEND | O_CLOEXEC,
0644);
if (g_debugFd < 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: cannot open debug log %s: %s",
debugLogPath.c_str(), strerror(errno));
}
}
// Set up an alternate stack so we still have stack space if the original
// thread ran out (very common for Vulkan crashes inside deep driver
// call chains).
stack_t ss{};
ss.ss_sp = g_sigStack;
ss.ss_size = sizeof(g_sigStack);
ss.ss_flags = 0;
if (sigaltstack(&ss, nullptr) != 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: sigaltstack failed: %s",
strerror(errno));
}
struct sigaction sa{};
sa.sa_sigaction = &crashHandler;
sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART;
sigemptyset(&sa.sa_mask);
for (size_t i = 0; i < kNumSignals; ++i) {
if (sigaction(g_signals[i], &sa, nullptr) != 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: sigaction(%d) failed: %s",
g_signals[i], strerror(errno));
}
}
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG",
"CrashHandler installed for SIGSEGV/SIGABRT/SIGBUS/SIGFPE/SIGILL/SIGSYS");
}
void setNote(const char* note) {
if (!note) note = "";
pthread_mutex_lock(&g_noteMtx);
size_t n = strnlen(note, kNoteCapacity - 1);
memcpy(g_note, note, n);
g_note[n] = '\0';
pthread_mutex_unlock(&g_noteMtx);
}
} // namespace CrashHandler
#else // _WIN32 — no-op on host build
namespace CrashHandler {
void install(const std::string&, const std::string&) {}
void setNote(const char*) {}
} // namespace CrashHandler
#endif
+33
View File
@@ -0,0 +1,33 @@
#ifndef FACE_SDK_CRASH_HANDLER_H
#define FACE_SDK_CRASH_HANDLER_H
#include <string>
namespace CrashHandler {
// Install signal handlers for SIGSEGV / SIGABRT / SIGBUS / SIGFPE / SIGILL.
//
// On a fatal signal we:
// 1) Switch to a pre-allocated 64KB sigaltstack so we still have stack
// space even if the original thread's stack was the cause.
// 2) Write a self-contained crash record to <internalDataPath>/face_sdk_crash.log
// (and also try to append it to the DebugLog file passed in).
// The record contains: signal name, si_code, si_addr, pid, tid,
// a synchronous unwound backtrace (PC + module + offset for each frame),
// and the value of any extra context the caller registered via setNote().
// 3) Re-raise the original signal with the default handler so that the
// Android tombstone pipeline still produces /data/tombstones/* files.
//
// Safe to call once. Subsequent calls are no-ops. Must be called AFTER
// DebugLog::init() so we know the log directory.
void install(const std::string& internalDataPath,
const std::string& debugLogPath = "");
// Optional short note (<= 256 chars) appended to every crash dump from now on.
// Useful to record "current frame index", "current motion", etc. so we know
// what was happening at the moment of the crash. Async-signal-safe to read.
void setNote(const char* note);
} // namespace CrashHandler
#endif // FACE_SDK_CRASH_HANDLER_H
+165
View File
@@ -0,0 +1,165 @@
#include "DebugLog.h"
#include <android/log.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <cstdarg>
#include <cstdio>
#include <ctime>
#include <cstring>
#include <mutex>
#include <string>
#include <unordered_map>
namespace {
std::mutex g_mtx;
FILE* g_fp = nullptr;
std::string g_path;
constexpr size_t kMaxSize = 2 * 1024 * 1024; // 2 MB before rotation
// Throttle bookkeeping: one slot per distinct key passed to log_throttled().
struct ThrottleSlot {
int64_t last_emit_ms = 0; // when we last actually wrote a line for this key
uint32_t suppressed = 0; // number of calls swallowed since last_emit_ms
};
std::unordered_map<std::string, ThrottleSlot> g_throttle;
constexpr int64_t kThrottleWindowMs = 2000; // 2 seconds
pid_t currentTid() {
return static_cast<pid_t>(syscall(SYS_gettid));
}
int64_t nowMs() {
timespec ts{};
clock_gettime(CLOCK_MONOTONIC, &ts);
return static_cast<int64_t>(ts.tv_sec) * 1000 + ts.tv_nsec / 1000000;
}
// Assumes g_mtx is held. Writes a single pre-formatted line to disk + logcat.
void writeLineLocked(const char* body) {
pid_t tid = currentTid();
timespec ts{};
clock_gettime(CLOCK_REALTIME, &ts);
struct tm tm_info{};
localtime_r(&ts.tv_sec, &tm_info);
char timebuf[32];
strftime(timebuf, sizeof(timebuf), "%H:%M:%S", &tm_info);
if (g_fp) {
std::fprintf(g_fp, "[%s.%03ld][tid=%d] %s\n",
timebuf, ts.tv_nsec / 1000000, tid, body);
std::fflush(g_fp);
}
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG", "[tid=%d] %s", tid, body);
}
} // namespace
namespace DebugLog {
void init(const char* internalDataPath) {
std::lock_guard<std::mutex> lk(g_mtx);
if (g_fp) {
return; // already initialized
}
const char* base = (internalDataPath && *internalDataPath) ? internalDataPath : "/data/local/tmp";
g_path = std::string(base) + "/face_sdk_debug.log";
// Rotate once if current file is too large.
struct stat st{};
if (stat(g_path.c_str(), &st) == 0 && static_cast<size_t>(st.st_size) > kMaxSize) {
std::string backup = g_path + ".old";
std::remove(backup.c_str());
std::rename(g_path.c_str(), backup.c_str());
}
g_fp = std::fopen(g_path.c_str(), "a");
if (g_fp) {
time_t now = time(nullptr);
struct tm tm_info{};
localtime_r(&now, &tm_info);
char tbuf[64];
strftime(tbuf, sizeof(tbuf), "%Y-%m-%d %H:%M:%S", &tm_info);
std::fprintf(g_fp,
"\n========== DebugLog opened @ %s (pid=%d) ==========\n",
tbuf, getpid());
std::fflush(g_fp);
}
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG",
"DebugLog file path: %s", g_path.c_str());
}
void log(const char* fmt, ...) {
char buf[1024];
va_list ap;
va_start(ap, fmt);
int n = std::vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
if (n < 0) return;
std::lock_guard<std::mutex> lk(g_mtx);
writeLineLocked(buf);
}
void log_throttled(const char* key, const char* fmt, ...) {
char buf[1024];
va_list ap;
va_start(ap, fmt);
int n = std::vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
if (n < 0) return;
const char* k = key ? key : "";
int64_t now = nowMs();
std::lock_guard<std::mutex> lk(g_mtx);
auto it = g_throttle.find(k);
if (it == g_throttle.end()) {
// First time we see this key -> always log.
ThrottleSlot slot;
slot.last_emit_ms = now;
slot.suppressed = 0;
g_throttle.emplace(k, slot);
writeLineLocked(buf);
return;
}
ThrottleSlot& slot = it->second;
int64_t elapsed = now - slot.last_emit_ms;
if (elapsed < kThrottleWindowMs) {
// Still inside the 2s window -> swallow.
++slot.suppressed;
return;
}
// Window elapsed: emit the line, annotated with how many we swallowed.
if (slot.suppressed > 0) {
char annotated[1200];
std::snprintf(annotated, sizeof(annotated),
"%s (repeated %u times in last %lldms, key=%s)",
buf, slot.suppressed, (long long)elapsed, k);
writeLineLocked(annotated);
} else {
writeLineLocked(buf);
}
slot.last_emit_ms = now;
slot.suppressed = 0;
}
void flush() {
std::lock_guard<std::mutex> lk(g_mtx);
if (g_fp) std::fflush(g_fp);
}
std::string getLogPath() {
std::lock_guard<std::mutex> lk(g_mtx);
return g_path;
}
} // namespace DebugLog
+33
View File
@@ -0,0 +1,33 @@
#ifndef FACE_SDK_DEBUGLOG_H
#define FACE_SDK_DEBUGLOG_H
#include <string>
namespace DebugLog {
// Call once (e.g. in android_main) with android_app->activity->internalDataPath.
// Safe to call multiple times; subsequent calls are no-ops.
void init(const char* internalDataPath);
// Thread-safe. Also mirrored to logcat with tag "FACE_DBG".
// Every line is prefixed with timestamp + thread id and flushed to disk immediately,
// so even a hard crash will preserve the tail.
void log(const char* fmt, ...) __attribute__((format(printf, 1, 2)));
// Throttled variant keyed by a caller-supplied string.
// - The first call with a given key is always written.
// - Subsequent calls with the same key within 2 seconds are suppressed.
// - When the throttle window elapses, the next matching call is written with a
// "(repeated N times in last Xms)" suffix describing the suppressed ones.
// Use stable, short keys (e.g. "drawFrame.acquire", "drawFrame.present").
void log_throttled(const char* key, const char* fmt, ...) __attribute__((format(printf, 2, 3)));
// Force flush to disk.
void flush();
// Absolute path of the log file (valid after init()).
std::string getLogPath();
} // namespace DebugLog
#endif // FACE_SDK_DEBUGLOG_H
+444 -30
View File
@@ -3,44 +3,112 @@
#include <game-activity/native_app_glue/android_native_app_glue.h>
#include <game-activity/GameActivity.h>
#include "AndroidOut.h"
#include "DebugLog.h"
#include "CrashHandler.h"
#include "FaceApp.h"
#include <android/asset_manager.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <thread>
using namespace std;
static inline pid_t dbg_tid() {
return static_cast<pid_t>(syscall(SYS_gettid));
}
static const char* appCmdName(int32_t cmd) {
switch (cmd) {
case APP_CMD_INIT_WINDOW: return "APP_CMD_INIT_WINDOW";
case APP_CMD_TERM_WINDOW: return "APP_CMD_TERM_WINDOW";
case APP_CMD_WINDOW_RESIZED: return "APP_CMD_WINDOW_RESIZED";
case APP_CMD_WINDOW_REDRAW_NEEDED: return "APP_CMD_WINDOW_REDRAW_NEEDED";
case APP_CMD_CONTENT_RECT_CHANGED: return "APP_CMD_CONTENT_RECT_CHANGED";
case APP_CMD_GAINED_FOCUS: return "APP_CMD_GAINED_FOCUS";
case APP_CMD_LOST_FOCUS: return "APP_CMD_LOST_FOCUS";
case APP_CMD_CONFIG_CHANGED: return "APP_CMD_CONFIG_CHANGED";
case APP_CMD_LOW_MEMORY: return "APP_CMD_LOW_MEMORY";
case APP_CMD_START: return "APP_CMD_START";
case APP_CMD_RESUME: return "APP_CMD_RESUME";
case APP_CMD_SAVE_STATE: return "APP_CMD_SAVE_STATE";
case APP_CMD_PAUSE: return "APP_CMD_PAUSE";
case APP_CMD_STOP: return "APP_CMD_STOP";
case APP_CMD_DESTROY: return "APP_CMD_DESTROY";
default: return "APP_CMD_???";
}
}
extern "C" {
android_app* g_android_app = nullptr;
Application* g_Application = nullptr;
FaceApp* g_Application = nullptr;
AAssetManager* g_assetManager = nullptr;
jobject g_callback = nullptr;
jobject g_callbackAnimationFinished = nullptr;
void handle_cmd(android_app *pApp, int32_t cmd) {
DebugLog::log(">> handle_cmd %s(%d) tid=%d", appCmdName(cmd), (int)cmd, dbg_tid());
switch (cmd) {
case APP_CMD_INIT_WINDOW:
aout << "APP_CMD_INIT_WINDOW" << std::endl;
g_Application->initVulkan();
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: calling onWindowInit()");
if (g_Application != nullptr) {
g_Application->onWindowInit();
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: onWindowInit() returned, isInited=%d",
(int)g_Application->isInited());
} else {
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: g_Application is null, skipping");
}
break;
case APP_CMD_TERM_WINDOW:
aout << "APP_CMD_TERM_WINDOW" << std::endl;
DebugLog::log("handle_cmd APP_CMD_TERM_WINDOW: calling onWindowLost()");
if (g_Application != nullptr) {
g_Application->onWindowLost();
}
DebugLog::log("handle_cmd APP_CMD_TERM_WINDOW: onWindowLost() returned");
break;
default:
break;
}
DebugLog::log("<< handle_cmd %s done", appCmdName(cmd));
}
const int ArgLen = 128*1024;
char g_InitArgString[ArgLen] = {0};
void android_main(struct android_app *pApp) {
DebugLog::init(pApp->activity->internalDataPath);
// 安装信号处理器越早越好:之前的 FORTIFY: pthread_mutex_lock 崩溃
// 是裸的 SIGABRT,没有任何 signal handler,所以 DebugLog 文件停在最后
// 一条业务日志、看不到任何崩溃栈。装上之后任何 fatal signal 都会把:
// * 信号名 / si_code / si_addr / pid / tid / 当前 note
// * 完整 backtracePC + 模块 + 符号 + 偏移)
// 同步落到 internalDataPath/face_sdk_crash.log(以及 DebugLog 文件尾),
// 然后再走默认 handler 让 debuggerd 产生 tombstone。
CrashHandler::install(pApp->activity->internalDataPath,
DebugLog::getLogPath());
DebugLog::log("android_main start, pid=%d tid=%d, logPath=%s",
getpid(), dbg_tid(), DebugLog::getLogPath().c_str());
aout << "Welcome to android_main" << std::endl;
g_android_app = pApp;
g_assetManager = pApp->activity->assetManager;
FaceApp application;
g_Application = &application;
//FaceApp application;
//g_Application = &application;
if(g_Application == nullptr) {
g_Application = new FaceApp();
}
g_Application->SetInitArg(g_InitArgString);
pApp->onAppCmd = handle_cmd;
long long last_update_time = 0;
android_app_set_motion_event_filter(pApp, nullptr);
long long _lastDrawFrameTime = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
do {
auto ms = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
auto start_time = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
// Process all pending events before running game logic.
bool done = false;
while (!done) {
@@ -66,32 +134,110 @@ void android_main(struct android_app *pApp) {
}
}
}
if(application.isInited())
// isInited() checks _applicationInited && _faceAppInited && _sceondInited,
// so while the window is gone (between TERM_WINDOW and the next
// INIT_WINDOW) this is false and we skip drawFrame entirely — no
// exceptions, no log spam. The outer sleep at the bottom already
// caps the loop frequency.
if(g_Application->isInited())
{
application.drawFrame();
auto frameTime = (start_time - _lastDrawFrameTime);
_lastDrawFrameTime = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
static uint64_t s_frameIdx = 0;
static uint64_t s_excCount = 0;
++s_frameIdx;
if (s_frameIdx % 600 == 0) {
DebugLog::log("heartbeat: frame=%llu exceptions_so_far=%llu",
(unsigned long long)s_frameIdx,
(unsigned long long)s_excCount);
}
// 每 60 帧刷新一次崩溃 note,崩溃时 dump 里能看到「最后一次活着」
// 的帧号和异常数,定位是不是在某一个固定帧附近卡死。
if (s_frameIdx % 60 == 0) {
char note[128];
std::snprintf(note, sizeof(note),
"drawFrame frame=%llu exc=%llu",
(unsigned long long)s_frameIdx,
(unsigned long long)s_excCount);
CrashHandler::setNote(note);
}
try {
g_Application->drawFrame(frameTime);
} catch (const std::exception& e) {
++s_excCount;
DebugLog::log_throttled("main.drawFrame.stdexc",
"!!! drawFrame std::exception (frame=%llu count=%llu): %s",
(unsigned long long)s_frameIdx,
(unsigned long long)s_excCount,
e.what());
} catch (...) {
++s_excCount;
DebugLog::log_throttled("main.drawFrame.unknown",
"!!! drawFrame unknown exception (frame=%llu count=%llu)",
(unsigned long long)s_frameIdx,
(unsigned long long)s_excCount);
}
}
auto end_time = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
auto frameTime = end_time - last_update_time;
last_update_time = end_time;
auto function_time = (end_time - start_time);
//aout << "function_time:" << function_time << " fps:" << (1000/frameTime) << std::endl;
if(function_time < 35)
{
this_thread::sleep_for(chrono::milliseconds((35-function_time)));
}
auto end_ms = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
this_thread::sleep_for(chrono::milliseconds((end_ms-ms)));
} while (!pApp->destroyRequested);
application.cleanup();
DebugLog::log("android_main: destroyRequested, running cleanup");
// 关键:这里只复位 FaceApp 的"second-init" 状态机标记,**不销毁**任何
// Vulkan 资源。
//
// - 窗口相关资源(surface/swapchain/framebuffers/imageViews/
// commandBuffers/sync)已经在 APP_CMD_TERM_WINDOW → onWindowLost()
// 里被 cleanupForWindowLost() 干净销毁;
// - renderPass / graphicsPipeline / pipelineLayout / VkDevice /
// VkInstance / VMA / 所有 textures / FaceApp 自身 pipelines 是 g_Application
// 单例的"全局资源",跨 NativeActivity 实例复用。下次 Activity 启动
// onWindowInit 会走 reinitForNewWindow() 重建窗口资源、复用全局资源。
//
// 历史教训:之前这里调过 g_Application->cleanupSecondInit(),那个函数会
// 把 renderPass / graphicsPipeline / pipelineLayout 全部 destroy 掉但既
// 不置 VK_NULL_HANDLE 也不复位 _applicationInited。结果用户从主界面切镜
// 头再进 MakeupActivity 时,onWindowInit 看到 _applicationInited=1 走 reinit
// 复用路径,引用悬空的 renderPass 调 vkCreateFramebuffervalidation
// layer 检测出 use-after-free 直接 SEGV。
//application.cleanup();
//g_Application->cleanupSecondInit();
g_Application->clearnSecondFaceApp();
DebugLog::log("android_main: exit");
}
}
extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize);
extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX);
extern void ReceiveFacePoint(float* pos, int count, int width, int height);
void processWithVulkan(uint8_t* data, int width, int height, int format,
int rowStride, size_t dataSize) {
TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize);
int rowStride, size_t dataSize, int rotation, bool mirrorX) {
TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize, rotation, mirrorX);
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer,
jint width, jint height, jint format,
jint row_stride, jint pixel_stride,
jint rotation) {
Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer,
jint width, jint height, jint format,
jint row_stride, jint pixel_stride,
jint rotation, jboolean mirror_x) {
static std::atomic<uint64_t> s_imgCount{0};
uint64_t n = s_imgCount.fetch_add(1) + 1;
if (n == 1 || n % 300 == 0) {
DebugLog::log("processImageNative #%llu tid=%d w=%d h=%d mirror=%d",
(unsigned long long)n, dbg_tid(), width, height, (int)mirror_x);
}
uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer));
jlong capacity = env->GetDirectBufferCapacity(buffer);
@@ -99,23 +245,291 @@ Java_com_hmwl_face_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jo
return;
}
// 在这里将图像数据传递给 Vulkan
// 创建 Vulkan 图像或更新现有图像
processWithVulkan(imageData, width, height, format, row_stride, capacity);
// rotation = ImageInfo.getRotationDegrees(),给 shader 用来做 UV 旋转,
// 让不同 sensor orientation 的设备显示方向一致。
// mirror_x = true 表示前置摄像头:所有顶点 shader 会把 gl_Position.x 翻转
// 一次,达到"镜子效果"——这是化妆/美妆类 app 的标准体验。
processWithVulkan(imageData, width, height, format, row_stride, capacity,
(int)rotation, mirror_x == JNI_TRUE);
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_FaceActivity_passDataToNative(JNIEnv *env, jobject thiz, jobject buffer,
jint point_count, jint width, jint height) {
Java_com_hmwl_face_1sdk_FaceActivity_passDataToNative(JNIEnv *env, jobject thiz, jobject buffer,
jint point_count, jint width, jint height) {
static std::atomic<uint64_t> s_ptCount{0};
uint64_t n = s_ptCount.fetch_add(1) + 1;
float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer));
if (pos == nullptr) {
// 处理错误
DebugLog::log_throttled("passDataToNative.nullBuffer",
"passDataToNative #%llu got NULL DirectBufferAddress!",
(unsigned long long)n);
return;
}
// 节流:每 120 次实际数据通路打一行(约每 4 秒一次,足够确认通路活着即可)。
if (n == 1 || n % 120 == 0) {
DebugLog::log("passDataToNative #%llu point_count=%d w=%d h=%d p0=(%.3f,%.3f,%.3f)",
(unsigned long long)n, point_count, width, height,
pos[0], pos[1], pos[2]);
}
ReceiveFacePoint(pos, point_count, width, height);
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_SetCppInitArg(JNIEnv *env, jobject thiz, jstring json) {
const char *nativeString = env->GetStringUTFChars(json, nullptr);
jsize len = env->GetStringUTFLength(json);
DebugLog::log("JNI SetCppInitArg tid=%d len=%d", dbg_tid(), (int)len);
if(len > ArgLen)
{
aout << "ArgLen to long:" << len << std::endl;
return;
}
memcpy(g_InitArgString, nativeString, len);
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_StopRunning(JNIEnv *env, jobject thiz) {
DebugLog::log("JNI StopRunning tid=%d", dbg_tid());
g_Application->Stop();
}
JavaVM* g_jvm = nullptr;
// 添加 JNI_OnLoad 函数
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) {
g_jvm = vm; // 保存 JavaVM 指针,供其他线程使用
return JNI_VERSION_1_6;
}
void CppCallback()
{
JNIEnv* env = nullptr;
bool needDetach = false;
// 获取当前线程的 JNIEnv
JavaVM* vm = g_jvm; // 你需要提前保存 JavaVM* 到全局变量 g_jvm
int status = vm->GetEnv((void**)&env, JNI_VERSION_1_6);
if (status == JNI_EDETACHED) {
// 当前线程未 attach,需 attach
if (vm->AttachCurrentThread(&env, nullptr) != 0) {
return;
}
needDetach = true;
} else if (status != JNI_OK) {
return;
}
// 或者直接将指针传递给Vulkan
ReceiveFacePoint(pos, point_count, width, height);
if (g_callback == nullptr) {
if (needDetach) vm->DetachCurrentThread();
return;
}
// 1. 获取 CallbackInterface 的 class
jclass callbackClass = env->GetObjectClass(g_callback);
if (callbackClass == nullptr) {
if (needDetach) vm->DetachCurrentThread();
return;
}
// 2. 获取 OnLoadActionFinished 方法 ID(注意方法名大小写必须一致!)
jmethodID methodId = env->GetMethodID(callbackClass, "OnLoadMotionListFinished", "(Ljava/lang/String;)V");
if (methodId == nullptr) {
env->ExceptionDescribe(); // 打印异常
env->ExceptionClear();
if (needDetach) vm->DetachCurrentThread();
return;
}
// 3. 构造 jstring 参数
jstring jResult = env->NewStringUTF("ok");
// 4. 调用 Java 回调
env->CallVoidMethod(g_callback, methodId, jResult);
// 5. 检查是否抛出异常
if (env->ExceptionCheck()) {
env->ExceptionDescribe();
env->ExceptionClear();
}
// 6. 释放局部引用(可选,但推荐)
env->DeleteLocalRef(jResult);
env->DeleteLocalRef(callbackClass);
// 7. 如果是临时 attach 的线程,要 detach
if (needDetach) {
vm->DetachCurrentThread();
}
}
void CppAnimationFinishedCallback()
{
JNIEnv* env = nullptr;
bool needDetach = false;
// 获取当前线程的 JNIEnv
JavaVM* vm = g_jvm; // 你需要提前保存 JavaVM* 到全局变量 g_jvm
int status = vm->GetEnv((void**)&env, JNI_VERSION_1_6);
if (status == JNI_EDETACHED) {
// 当前线程未 attach,需 attach
if (vm->AttachCurrentThread(&env, nullptr) != 0) {
return;
}
needDetach = true;
} else if (status != JNI_OK) {
return;
}
if (g_callbackAnimationFinished == nullptr) {
if (needDetach) vm->DetachCurrentThread();
return;
}
// 1. 获取 CallbackInterface 的 class
jclass callbackClass = env->GetObjectClass(g_callbackAnimationFinished);
if (callbackClass == nullptr) {
if (needDetach) vm->DetachCurrentThread();
return;
}
// 2. 获取 OnLoadActionFinished 方法 ID(注意方法名大小写必须一致!)
jmethodID methodId = env->GetMethodID(callbackClass, "OnPlayMotionListFinished", "()V");
if (methodId == nullptr) {
env->ExceptionDescribe(); // 打印异常
env->ExceptionClear();
if (needDetach) vm->DetachCurrentThread();
return;
}
// 4. 调用 Java 回调
env->CallVoidMethod(g_callbackAnimationFinished, methodId);
// 5. 检查是否抛出异常
if (env->ExceptionCheck()) {
env->ExceptionDescribe();
env->ExceptionClear();
}
env->DeleteLocalRef(callbackClass);
// 7. 如果是临时 attach 的线程,要 detach
if (needDetach) {
vm->DetachCurrentThread();
}
}
extern "C"
JNIEXPORT jstring JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_PreReadAction(JNIEnv *env, jobject thiz, jstring motion,
jobject callback) {
DebugLog::log("JNI PreReadAction tid=%d", dbg_tid());
// 清理旧的全局引用
if (g_callback != nullptr) {
env->DeleteGlobalRef(g_callback);
g_callback = nullptr;
}
// 保存新的回调
if (callback != nullptr) {
g_callback = env->NewGlobalRef(callback);
} else {
g_callback = nullptr;
}
const char *nativeString = env->GetStringUTFChars(motion, nullptr);
jsize len = env->GetStringUTFLength(motion);
if(len > ArgLen)
{
aout << "ArgLen to long:" << len << std::endl;
}
std::string ret = g_Application->preLoadMotionList(nativeString, CppCallback);
return env->NewStringUTF(ret.c_str());
}
const char DELIMITER = 0x1F; // same as \u001F
std::vector<std::string> splitMotionString(const std::string& input) {
std::vector<std::string> result;
if (input.empty()) return result;
size_t start = 0;
size_t pos = input.find(DELIMITER);
while (pos != std::string::npos) {
result.push_back(input.substr(start, pos - start));
start = pos + 1;
pos = input.find(DELIMITER, start);
}
// Add last part
result.push_back(input.substr(start));
return result;
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_ChangeMotionCpp(JNIEnv *env, jobject thiz, jstring json,
jobject callback, jboolean loop) {
const char *nativeString = env->GetStringUTFChars(json, nullptr);
jsize len = env->GetStringUTFLength(json);
DebugLog::log_throttled("JNI.ChangeMotionCpp",
"JNI ChangeMotionCpp tid=%d len=%d loop=%d",
dbg_tid(), (int)len, (int)loop);
if(len > ArgLen)
{
aout << "ArgLen to long:" << len << std::endl;
return;
}
// 保存新的回调
if (callback != nullptr) {
g_callbackAnimationFinished = env->NewGlobalRef(callback);
} else {
g_callbackAnimationFinished = nullptr;
}
if(g_Application->isInited())
{
std::vector<std::string> motions = splitMotionString(nativeString);
g_Application->changeMotionList(motions, CppAnimationFinishedCallback, loop);
}
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_StopMotionNative(JNIEnv *env, jobject thiz) {
DebugLog::log("JNI StopMotionNative tid=%d", dbg_tid());
g_Application->StopMotion();
}
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_ResumeMotionNative(JNIEnv *env, jobject thiz) {
DebugLog::log("JNI ResumeMotionNative tid=%d", dbg_tid());
g_Application->ResumeMotion();
}
// 给 Java/Kotlin 侧用的日志桥:把 Java 端关键事件写进 native 的
// face_sdk_debug.log,使 Java + native + Vulkan 渲染日志能在同一个
// 文件里按时间戳排好序,一次 adb pull 就能拿到完整时间线。
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_DebugLog_nativeLog(JNIEnv *env, jclass clazz,
jstring jtag, jstring jmsg) {
if (jmsg == nullptr) return;
const char* tag = jtag ? env->GetStringUTFChars(jtag, nullptr) : nullptr;
const char* msg = env->GetStringUTFChars(jmsg, nullptr);
if (tag) {
DebugLog::log("[J][%s] %s", tag, msg);
} else {
DebugLog::log("[J] %s", msg);
}
if (tag) env->ReleaseStringUTFChars(jtag, tag);
env->ReleaseStringUTFChars(jmsg, msg);
}
@@ -0,0 +1,38 @@
package com.hmwl.face_sdk;
import android.util.Log;
/**
* Java/Kotlin 端日志桥。所有调用最终通过 JNI 写到 native 端的
* face_sdk_debug.log 文件(与 native DebugLog::log 共用一份),
* 同时镜像到 logcat(tag = 调用方 tag),方便联机调试时一次 adb pull
* 就能拿到完整时间线(Java + native + Vulkan 渲染)。
*
* 使用约定:
* - 业务低频事件用 i(tag, msg)
* - 错误用 e(tag, msg)
* - 高频帧级日志请自行节流,本类不再做二次节流
*/
public final class DebugLog {
private DebugLog() {}
public static void i(String tag, String msg) {
Log.i(tag, msg);
try {
nativeLog(tag, msg);
} catch (UnsatisfiedLinkError e) {
// native 还没 attach 的极早期阶段,吞掉避免崩溃
}
}
public static void e(String tag, String msg) {
Log.e(tag, msg);
try {
nativeLog("[E]" + tag, msg);
} catch (UnsatisfiedLinkError e) {
// 同上
}
}
private static native void nativeLog(String tag, String msg);
}
@@ -1,14 +1,14 @@
package com.hmwl.face;
package com.hmwl.face_sdk;
import android.Manifest;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.util.Log;
import android.view.MotionEvent;
import android.view.View;
import androidx.annotation.NonNull;
import androidx.camera.core.AspectRatio;
import android.util.Size;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy;
@@ -31,15 +31,35 @@ import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener {
private static final String TAG = "FaceActivity";
static {
System.loadLibrary("sample");
System.loadLibrary("face_sdk");
}
/**
* Intent extra key调用方example/MainActivity 通过它告诉 FaceActivity
* 启动哪个朝向的摄像头值类型 int取自 androidx.camera.core.CameraSelector
* - CameraSelector.LENS_FACING_BACK (0)
* - CameraSelector.LENS_FACING_FRONT (1)
* 缺省未提供 extra时走 LENS_FACING_BACK保留旧调用方的行为
*
* 该值同时决定两件事
* CameraSelector 朝向
* mirrorX 镜像标志前置摄像头时 sensor 帧是"用户右手在画面右侧"
* 为了显示成"镜子效果"用户右手在屏幕左侧所有顶点 shader
* gl_Position 阶段对 NDC.x 做一次水平翻转这件事通过 push constant
* mirror_x 字段透传到 GPU对应地 FaceLandmarkerHelper 也会对
* bitmap postScale(-1, 1) mediapipe 输入与显示画面方向一致
* 确保 face 贴图与 bg 完美对齐
*/
public static final String EXTRA_LENS_FACING = "com.hmwl.face_sdk.LENS_FACING";
/** 当前 session 使用的摄像头朝向。在 onCreate 里从 Intent extra 读取并固定。 */
private int lensFacing = CameraSelector.LENS_FACING_BACK;
private ProcessCameraProvider cameraProvider;
private void initCamera(){
backgroundExecutor = Executors.newSingleThreadExecutor();
@@ -50,6 +70,21 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
}
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
if (requestCode == REQUEST_CAMERA_PERMISSION) {
// 检查权限是否被授予
if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
// 用户授予了权限启动相机
startCamera();
} else {
requestCameraPermission();
}
}
}
private void startCamera() {
ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
ProcessCameraProvider.getInstance(this);
@@ -59,58 +94,51 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
cameraProvider = cameraProviderFuture.get();
CameraSelector cameraSelector = new CameraSelector.Builder()
.requireLensFacing(CameraSelector.LENS_FACING_BACK)
.requireLensFacing(lensFacing)
.build();
// 显式锁定分析帧分辨率为 480x640CameraX UI 坐标系竖屏短边 x 长边
// 不同手机的相机硬件原生支持的分辨率差异很大旗舰机随便给 1920x1440
// 入门机可能是 320x240CameraX 默认会按"最贴近 setTargetAspectRatio
// supported size"挑一个,结果在不同设备上拿到的分析帧大小都不一样。
// 这会带来三个麻烦
// bg 纹理 GPU 内存占用波动很大高端机一帧 ~10MB staging buffer
// mediapipe 推理耗时随分辨率非线性升高Pixel 1920x1440
// 640x480 4-6 帧率掉得明显
// 我们的 SDK 用归一化 landmark + 中央方形 crop本来就不需要更高
// 分辨率多出来的像素是纯浪费
//
// 锁定 480x640 sensor 坐标系下分析帧固定 640x480 (rotation=90 )
// FaceLandmarkerHelper 中央方形裁切固定 480x480bg 上传纹理也固定
// 不同手机上行为一致
//
// 实际尺寸由 CameraX 决定会找最接近的硬件支持档位但设备实测
// 几乎都能精确给出 640x480
imageAnalyzer = new ImageAnalysis.Builder()
.setTargetAspectRatio(AspectRatio.RATIO_4_3)
.setTargetResolution(new Size(480, 640))
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build();
imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() {
private long lastCaptureTime = 0;
private int frameCount = 0;
@Override
public void analyze(@NonNull ImageProxy image) {
frameCount++;
long currentTime = System.currentTimeMillis();
try {
long captureTimeNs = image.getImageInfo().getTimestamp();
long analyzeTime = System.currentTimeMillis();
// 简单直接的计算方法
long captureTimeMs = TimeUnit.NANOSECONDS.toMillis(captureTimeNs);
long latency = analyzeTime - captureTimeMs;
// 帧率计算
if (lastCaptureTime > 0) {
long frameInterval = captureTimeMs - lastCaptureTime;
double fps = 1000.0 / frameInterval;
if (frameCount % 10 == 0) { // 每30帧打印一次
Log.i("TimeLatency", String.format(
"Frame %d - Latency: %d ms, FPS: %.1f",
frameCount, latency, fps
));
}
}
lastCaptureTime = captureTimeMs;
processImageForVulkan(image);
if(frameCount %2 == 0)
if(frameCount %3 == 0)
{
detectFace(image);
}
else
{
image.close();
}
// else
// {
// image.close();
// }
} finally {
//image.close(); // 确保在这里关闭
image.close(); // 确保在这里关闭
//Log.d("Camera", "Image closed, ready for next frame");
}
}
@@ -125,35 +153,56 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
}
private void processImageForVulkan(ImageProxy image) {
// 获取图像信息
int width = image.getWidth();
int height = image.getHeight();
int format = image.getFormat();
// 获取图像数据平面
ImageProxy.PlaneProxy[] planes = image.getPlanes();
// 对于 RGBA_8888 格式通常只有一个平面
if (planes.length > 0) {
ImageProxy.PlaneProxy plane = planes[0];
ByteBuffer buffer = plane.getBuffer();
int rowStride = plane.getRowStride();
int pixelStride = plane.getPixelStride();
// 调用 Native 方法处理图像
// mirrorX true 时所有顶点 shader 会把 gl_Position.x 翻转一次达到
// "镜子效果"前置摄像头唯一需要的额外处理后置不开镜像
boolean mirrorX = (lensFacing == CameraSelector.LENS_FACING_FRONT);
processImageNative(buffer, width, height, format,
rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
rowStride, pixelStride, image.getImageInfo().getRotationDegrees(),
mirrorX);
}
}
public void Stop(){
StopRunning();
}
private native void StopMotionNative();
public void StopMotion(){
StopMotionNative();
}
private native void ResumeMotionNative();
public void ResumeMotion(){
ResumeMotionNative();
}
private native void StopRunning();
// Native 方法
private native void processImageNative(ByteBuffer buffer, int width, int height,
int format, int rowStride, int pixelStride,
int rotation);
int rotation, boolean mirrorX);
private void detectFace(ImageProxy imageProxy) {
// 前置摄像头FaceLandmarkerHelper 内部会对 bitmap postScale(-1, 1)
// mediapipe 看到的是"用户视角"的图像这样 landmark 输出就是用户视角
// 归一化坐标 shader gl_Position.x 翻转一次后的 bg 画面方向一致
// face 贴图位置才能精确对齐
boolean isFrontCamera = (lensFacing == CameraSelector.LENS_FACING_FRONT);
faceLandmarkerHelper.detectLiveStream(
imageProxy,false
imageProxy, isFrontCamera
);
}
@@ -176,6 +225,15 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
@Override
protected void onCreate(Bundle savedInstanceState){
super.onCreate(savedInstanceState);
// 必须在 initCamera() 之前读CameraSelector 的朝向就靠这个 lensFacing
// 兼容旧调用方未提供 EXTRA_LENS_FACING 时默认后置
if (getIntent() != null) {
lensFacing = getIntent().getIntExtra(
EXTRA_LENS_FACING, CameraSelector.LENS_FACING_BACK);
}
Log.d(TAG, "onCreate lensFacing=" + lensFacing
+ " (front=" + CameraSelector.LENS_FACING_FRONT
+ " back=" + CameraSelector.LENS_FACING_BACK + ")");
initCamera();
backgroundExecutor.execute(new Runnable() {
@@ -186,7 +244,6 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
0.5f,
0.5f,
1,
0,
RunningMode.LIVE_STREAM,
FaceActivity.this,
FaceActivity.this
@@ -196,31 +253,6 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
}
@Override
public boolean onTouchEvent(MotionEvent event) {
int action = event.getActionMasked();
float x = event.getX();
float y = event.getY();
switch (action) {
case MotionEvent.ACTION_DOWN:
Log.d(TAG, "Java Layer: Touch DOWN at (" + x + ", " + y + ")");
// 你可以在这里做些事比如发消息给 C++ UI 反馈
break;
case MotionEvent.ACTION_MOVE:
Log.d(TAG, "Java Layer: Touch MOVE");
break;
case MotionEvent.ACTION_UP:
Log.d(TAG, "Java Layer: Touch UP");
break;
}
// 关键返回 false 才能让事件继续传递给 Native C++
// 如果返回 true表示事件已被消费C++ 层将收不到
return false;
}
@Override
public void onWindowFocusChanged(boolean hasFocus) {
super.onWindowFocusChanged(hasFocus);
@@ -301,11 +333,62 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
floatBuffer.put(points);
floatBuffer.position(0);
long cur_time = System.currentTimeMillis();
Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
// 节流 30 次调用打一次到 native 文件方便和 native 那边的
// passDataToNative.tick 对账
onResultsCallCount++;
if (onResultsCallCount % 30 == 1) {
DebugLog.i("FaceActivity",
"onResults call#" + onResultsCallCount
+ " index=" + index + " input=" + width + "x" + height
+ " p0=(" + points[0] + "," + points[1] + "," + points[2] + ")");
}
passDataToNative(nativeBuffer, index, width, height);
Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
}
private long onResultsCallCount = 0;
public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
protected native void SetCppInitArg(String json);
public void SetInitArg(String json)
{
SetCppInitArg(json);
}
public interface LoadMotionListFinishedCallback {
void OnLoadMotionListFinished(String result);
}
public interface PlayMotionListFinishedCallback {
void OnPlayMotionListFinished();
}
protected native String PreReadAction(String motion, LoadMotionListFinishedCallback callback);
public String PreLoadAction(String json, LoadMotionListFinishedCallback callback){
return PreReadAction(json, callback);
}
protected native void ChangeMotionCpp(String json, PlayMotionListFinishedCallback callback, boolean loop);
public void PlayMotionList(List<String> motionList, boolean loop, PlayMotionListFinishedCallback callback)
{
if(callback == null)
{
callback = new PlayMotionListFinishedCallback() {
@Override
public void OnPlayMotionListFinished() {
Log.i(TAG, "OnPlayMotionListFinished: ");
}
};
}
final String DELIMITER = "\u001F"; // ASCII Unit Separator
String motion_list_str = String.join(DELIMITER, motionList);
ChangeMotionCpp(motion_list_str, callback, loop);
}
// 添加这个缺失的方法
@Override
public void onEmpty() {
// 当没有检测到人脸时的处理逻辑
//Log.d(TAG, "No face detected in the current frame");
}
}
@@ -1,4 +1,4 @@
package com.hmwl.face
package com.hmwl.face_sdk
import android.content.Context
import android.graphics.Bitmap
@@ -20,8 +20,7 @@ class FaceLandmarkerHelper(
var minFaceTrackingConfidence: Float = DEFAULT_FACE_TRACKING_CONFIDENCE,
var minFacePresenceConfidence: Float = DEFAULT_FACE_PRESENCE_CONFIDENCE,
var maxNumFaces: Int = DEFAULT_NUM_FACES,
var currentDelegate: Int = DELEGATE_CPU,
var runningMode: RunningMode = RunningMode.IMAGE,
var runningMode: RunningMode = RunningMode.LIVE_STREAM,
val context: Context,
// this listener is only used when running in RunningMode.LIVE_STREAM
val faceLandmarkerHelperListener: LandmarkerListener? = null
@@ -50,14 +49,7 @@ class FaceLandmarkerHelper(
val baseOptionBuilder = BaseOptions.builder()
// Use the specified hardware for running the model. Default to CPU
when (currentDelegate) {
DELEGATE_CPU -> {
baseOptionBuilder.setDelegate(Delegate.CPU)
}
DELEGATE_GPU -> {
baseOptionBuilder.setDelegate(Delegate.GPU)
}
}
baseOptionBuilder.setDelegate(Delegate.GPU)
baseOptionBuilder.setModelAssetPath(MP_FACE_LANDMARKER_TASK)
@@ -75,6 +67,9 @@ class FaceLandmarkerHelper(
}
}
minFaceDetectionConfidence = 0.1f;
minFaceTrackingConfidence = 0.1f;
minFacePresenceConfidence = 0.1f;
try {
val baseOptions = baseOptionBuilder.build()
// Create an option builder with base options and specific
@@ -178,6 +173,13 @@ class FaceLandmarkerHelper(
" while not using RunningMode.LIVE_STREAM"
)
}
// 节流日志:每 ~30 次调用打一次(detectFace 在 FaceActivity 里只在
// frameCount % 3 == 0 时调用,所以 30 次大约对应 90 帧 ≈ 3 秒)。用来
// 确认 mediapipe 入口活着、isFrontCamera/rotation/分辨率有没有跑偏。
if ((frameId % 30L) == 0L) {
Log.i(TAG, "detectLiveStream tick frameId=$frameId src=${imageProxy.width}x${imageProxy.height}" +
" rot=${imageProxy.imageInfo.rotationDegrees} isFrontCamera=$isFrontCamera")
}
val frameTime = SystemClock.uptimeMillis()
val currentFrameId = frameId++
@@ -197,24 +199,42 @@ class FaceLandmarkerHelper(
Bitmap.Config.ARGB_8888
)
imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) }
imageProxy.close()
//imageProxy.close()
// 重要:这里 *不* 对前置摄像头做 postScale(-1, 1)!理由如下:
//
// 我们的渲染策略是 —— bg 与 face 顶点 shader 末尾统一对 gl_Position.x
// 乘 (1 - 2*mirror_x) 做镜子翻转。bg 是直接采样 sensor 帧、然后 shader
// 末尾翻一次;face 顶点用的是 mediapipe 输出的归一化坐标,shader 末尾
// 也翻一次。两者只有同时基于"sensor 视角"做输入、再一起在 shader 末尾
// 翻一次,face 才能精确对齐 bg 上的脸。
//
// 如果在这里给 mediapipe 喂"已镜像"的 bitmap(旧实现),mediapipe 输出
// 的 landmark 就是"用户视角"坐标,再被 shader 翻一次 → face 实际上被
// 翻了两次,跟只翻一次的 bg 错位(通常会在屏幕另一半,看起来就是"face
// 完全没渲染")。这正是我们最近调试看到的现象。
//
// mediapipe 对左右镜像不敏感(训练集本身覆盖各角度),sensor 视角下也
// 能稳定检测出 478 个 landmark,无需迎合"镜子视角"。因此 isFrontCamera
// 参数保留以备扩展,但当前不再据此修改输入图。
val matrix = Matrix().apply {
// Rotate the frame received from the camera to be in the same direction as it'll be shown
postRotate(imageProxy.imageInfo.rotationDegrees.toFloat())
// flip image if user use front camera
if (isFrontCamera) {
postScale(
-1f,
1f,
imageProxy.width.toFloat(),
imageProxy.height.toFloat()
)
}
}
// 旧代码硬编码了 (640-480)/2 = 80 和 bitmapBuffer.height=480),假设
// 相机帧是 640x480。在不同手机/不同 CameraX 配置下相机分辨率会变(比如
// 480x640 portrait sensor、1280x720 等),硬编码会越界裁切或丢一半画面。
//
// 改成:根据 imageProxy 的实际尺寸取中央 min(W,H) x min(W,H) 正方形作为
// mediapipe 输入。这与 bg.vert 的 UV 计算严格对齐 —— shader 那边也是从
// raw 帧的中央 min(W,H) 正方形采样,再按 rotation 旋转。两端语义一致才能
// 让 FaceLandmark 输出的归一化坐标 [0,1] 直接对应屏幕画布 NDC。
val srcW = imageProxy.width
val srcH = imageProxy.height
val side = minOf(srcW, srcH)
val cropX = (srcW - side) / 2
val cropY = (srcH - side) / 2
val rotatedBitmap = Bitmap.createBitmap(
bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height,
bitmapBuffer, cropX, cropY, side, side,
matrix, true
)
@@ -246,37 +266,43 @@ class FaceLandmarkerHelper(
val frameId = result.timestampMs()
if( result.faceLandmarks().size > 0 ) {
// 计算各种延时
val timings = frameTimings[frameId]
if (timings != null) {
val totalLatency = resultTime - timings.captureTime
val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime
val detectionLatency = resultTime - timings.detectionStartTime
val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime
Log.i("TimeLatency",
"总延时: ${totalLatency}ms | " +
"预处理: ${preprocessLatency}ms | " +
"检测: ${detectionLatency}ms | " +
"捕获到检测: ${captureToDetectionLatency}ms | " +
"FPS: ${String.format("%.1f", currentFps)}"
)
// val timings = frameTimings[frameId]
// if (timings != null) {
// val totalLatency = resultTime - timings.captureTime
// val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime
// val detectionLatency = resultTime - timings.detectionStartTime
// val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime
//
// Log.i("TimeLatency",
// "总延时: ${totalLatency}ms | " +
// "预处理: ${preprocessLatency}ms | " +
// "检测: ${detectionLatency}ms | " +
// "捕获到检测: ${captureToDetectionLatency}ms | " +
// "FPS: ${String.format("%.1f", currentFps)}"
// )
//
// val finishTimeMs = SystemClock.uptimeMillis()
// val inferenceTime = finishTimeMs - result.timestampMs()
// faceLandmarkerHelperListener?.onResults(
// ResultBundle(
// result,
// inferenceTime, // 主要是检测时间
// input.height,
// input.width,
// )
// )
// } else
//{
val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs()
faceLandmarkerHelperListener?.onResults(
ResultBundle(
result,
inferenceTime, // 主要是检测时间
input.height,
input.width,
)
)
} else {
val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs()
Log.i("TimeLatency",
"总延时: ${inferenceTime}ms | "
)
hitResultCount++
if (hitResultCount % 30L == 1L) {
// 节流:检测到人脸的频次。和 emptyResultCount 配合看就能知道
// 命中率(hitResultCount / (hitResultCount + emptyResultCount))。
Log.i(TAG, "returnLivestreamResult: HIT count=$hitResultCount" +
" landmarks=${result.faceLandmarks().firstOrNull()?.size}" +
" input=${input.width}x${input.height} infer=${inferenceTime}ms")
}
faceLandmarkerHelperListener?.onResults(
ResultBundle(
result,
@@ -285,12 +311,22 @@ class FaceLandmarkerHelper(
input.width
)
)
}
//}
}
else {
// 节流:mediapipe 拿到 frame 但没识别出人脸时,每 30 次空结果打一行。
// 只要这条日志在刷,就说明 mediapipe pipeline 正常活着,断点在"喂进去
// 的图本身没有可识别的脸"——常见于 isFrontCamera 路径下输入图被翻坏。
emptyResultCount++
if (emptyResultCount % 30L == 1L) {
Log.w(TAG, "returnLivestreamResult: faceLandmarks empty (count=$emptyResultCount)" +
" input=${input.width}x${input.height} ts=$frameId")
}
faceLandmarkerHelperListener?.onEmpty()
}
}
private var emptyResultCount: Long = 0
private var hitResultCount: Long = 0
// Return errors thrown during detection to this FaceLandmarkerHelper's
// caller
@@ -0,0 +1,27 @@
package com.hmwl.face_sdk;
import org.json.JSONObject;
public class Frame {
public String name;
public float x;
public float y;
public Frame(String name, float x, float y)
{
this.name = name;
this.x = x;
this.y = y;
}
public JSONObject toJsonObject(){
try {
JSONObject jsonObject = new JSONObject();
jsonObject.put("name", name);
jsonObject.put("x", x);
jsonObject.put("y", y);
return jsonObject;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
}
@@ -0,0 +1,30 @@
package com.hmwl.face_sdk;
import org.json.JSONObject;
public class InitArg {
public int action_fps;
public float zoom;
public float r;
public float g;
public float b;
public float radius;
public float offset_x;
public float offset_y;
public String toJson() {
try {
JSONObject jsonObject = new JSONObject();
jsonObject.put("action_fps", action_fps);
jsonObject.put("zoom", zoom);
jsonObject.put("r", r);
jsonObject.put("g", g);
jsonObject.put("b", b);
jsonObject.put("radius", radius);
jsonObject.put("offset_x", offset_x);
jsonObject.put("offset_y", offset_y);
return jsonObject.toString();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
}
@@ -0,0 +1,33 @@
package com.hmwl.face_sdk;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.List;
public class Motion {
public String name = "no";
public List<Frame> frames = new ArrayList<>();
public Motion(String name, List<Frame> frames)
{
this.name = name;
this.frames = frames;
}
public JSONObject toJsonObject() {
try {
JSONObject jsonObject = new JSONObject();
jsonObject.put("name", name);
JSONArray jframes = new JSONArray();
for(int i = 0; i < frames.size(); ++i){
jframes.put(frames.get(i).toJsonObject());
}
jsonObject.put("frames", jframes);
return jsonObject;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
};
@@ -0,0 +1,28 @@
package com.hmwl.face_sdk;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.List;
public class MotionList {
public List<Motion> motions = new ArrayList<>();
public JSONObject toJsonObject() {
try {
JSONObject jsonObject = new JSONObject();
JSONArray jMotions = new JSONArray();
for(int i = 0; i < motions.size(); ++i){
jMotions.put(motions.get(i).toJsonObject());
}
jsonObject.put("motions", jMotions);
return jsonObject;
} catch (Exception e) {
e.printStackTrace();
return new JSONObject();
}
}
public String toJsonString(){
return toJsonObject().toString();
}
}
@@ -1,17 +0,0 @@
package com.hmwl.sample;
import android.os.Bundle;
import com.hmwl.face.FaceActivity;
public class MainActivity extends FaceActivity {
private static final String TAG = "MainActivity";
@Override
protected void onCreate(Bundle savedInstanceState){
super.onCreate(savedInstanceState);
}
}
@@ -1,17 +0,0 @@
package com.hmwl.sample;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* Example local unit test, which will execute on the development machine (host).
*
* @see <a href="http://d.android.com/tools/testing">Testing documentation</a>
*/
public class ExampleUnitTest {
@Test
public void addition_isCorrect() {
assertEquals(4, 2 + 2);
}
}
+16 -2
View File
@@ -1,5 +1,19 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
google()
mavenCentral()
}
dependencies {
classpath 'com.android.tools.build:gradle:8.3.2'
}
}
plugins {
alias(libs.plugins.android.application) apply false
id 'org.jetbrains.kotlin.android' version '2.2.0' apply false
alias(libs.plugins.android.application) apply false
alias(libs.plugins.android.library) apply false
alias(libs.plugins.kotlin.android) apply false // 添加这行
//alias(libs.plugins.kotlin.android) apply false
//id 'org.jetbrains.kotlin.android' version '2.2.0' apply false
}
+12
View File
@@ -0,0 +1,12 @@
glslangValidator -V app/src/main/assets/shaders/bg.frag -o app/src/main/assets/shaders/bg.frag.spv
glslangValidator -V app/src/main/assets/shaders/bg.vert -o app/src/main/assets/shaders/bg.vert.spv
glslangValidator -V app/src/main/assets/shaders/texture.frag -o app/src/main/assets/shaders/texture.frag.spv
glslangValidator -V app/src/main/assets/shaders/texture.vert -o app/src/main/assets/shaders/texture.vert.spv
glslangValidator -V app/src/main/assets/shaders/simple_shader.frag -o app/src/main/assets/shaders/simple_shader.frag.spv
glslangValidator -V app/src/main/assets/shaders/simple_shader.vert -o app/src/main/assets/shaders/simple_shader.vert.spv
glslangValidator -V app/src/main/assets/shaders/texture_thick.frag -o app/src/main/assets/shaders/texture_thick.frag.spv
rem glslangValidator -V app/src/main/assets/shaders/texture_thick.vert -o app/src/main/assets/shaders/texture_thick.vert.spv
+1
View File
@@ -0,0 +1 @@
/build
+86
View File
@@ -0,0 +1,86 @@
plugins {
alias(libs.plugins.android.application)
alias(libs.plugins.kotlin.android)
//alias(libs.plugins.kotlin.android)
}
android {
namespace 'com.inewme.uvmirror'
compileSdk 36
defaultConfig {
applicationId "com.inewme.uvmirror.f20260127"
minSdk 29
targetSdk 36
versionCode 1
versionName "1.0"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}
buildTypes {
release {
signingConfig signingConfigs.debug
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
buildFeatures {
compose true
}
composeOptions {
kotlinCompilerExtensionVersion '1.5.1'
}
kotlinOptions {
jvmTarget = '17'
}
// kotlinOptions {
// jvmTarget = '17'
// }
}
dependencies {
implementation libs.appcompat
implementation libs.material
implementation libs.games.activity
implementation libs.core.ktx
implementation libs.androidx.core.ktx
testImplementation libs.junit
androidTestImplementation libs.ext.junit
androidTestImplementation libs.espresso.core
implementation project(':app')
// Jetpack Compose
implementation "androidx.compose.ui:ui:1.5.0"
implementation "androidx.compose.material3:material3:1.0.0"
implementation "androidx.compose.foundation:foundation:1.5.0"
implementation "androidx.compose.runtime:runtime:1.5.0"
implementation "androidx.activity:activity-compose:1.7.2" // ؼṩ setContent
implementation "androidx.compose.ui:ui-tooling:1.5.4"
def camerax_version = '1.4.2'
implementation "androidx.camera:camera-core:$camerax_version"
implementation "androidx.camera:camera-camera2:$camerax_version"
implementation "androidx.camera:camera-lifecycle:$camerax_version"
implementation "androidx.camera:camera-view:$camerax_version"
implementation 'com.google.protobuf:protobuf-javalite:3.21.12'
implementation 'com.google.android.datatransport:transport-runtime:3.1.9'
implementation 'com.google.android.datatransport:transport-backend-cct:3.1.9'
implementation 'com.google.flogger:flogger:0.7.4'
implementation 'com.google.flogger:flogger-system-backend:0.7.4'
implementation 'com.google.guava:guava:31.1-android'
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.7") // 提供 lifecycleScope
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0")
}
+21
View File
@@ -0,0 +1,21 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile
+33
View File
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<application
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/Theme.Sample" >
<activity
android:name=".MainActivity"
android:exported="true">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
<meta-data
android:name="android.app.lib_name"
android:value="face_sdk" />
</activity>
<activity
android:name=".MakeupActivity"
android:exported="false"
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
<!-- MakeupActivity 继承 FaceActivityface_sdk 渲染目前只支持竖屏。
configChanges 与 FaceActivity 保持一致,避免软键盘/导航栏触发重建。 -->
</application>
</manifest>
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Some files were not shown because too many files have changed in this diff Show More