1 Commits
27 changed files with 377 additions and 2299 deletions
+1 -6
View File
@@ -28,8 +28,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android")
app/src/main/cpp/AndroidOut.cpp app/src/main/cpp/AndroidOut.cpp
app/src/main/cpp/DebugLog.h app/src/main/cpp/DebugLog.h
app/src/main/cpp/DebugLog.cpp app/src/main/cpp/DebugLog.cpp
app/src/main/cpp/CrashHandler.h
app/src/main/cpp/CrashHandler.cpp
vulkan/AppBase.h vulkan/AppBase.h
vulkan/AppBase.cpp vulkan/AppBase.cpp
vulkan/Application.h vulkan/Application.h
@@ -53,10 +51,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android")
game-activity::game-activity_static game-activity::game-activity_static
Vulkan::Vulkan Vulkan::Vulkan
android android
log log)
# dl: needed by CrashHandler::dumpFrame -> dladdr() to map a
# PC back to "module + symbol + offset" in the crash log.
dl)
target_compile_definitions(face_sdk PRIVATE target_compile_definitions(face_sdk PRIVATE
VMA_STATIC_VULKAN_FUNCTIONS=0 VMA_STATIC_VULKAN_FUNCTIONS=0
-289
View File
@@ -1,289 +0,0 @@
# Face SDK 使用说明
基于 Vulkan + MediaPipe FaceLandmarker 的实时人脸贴图渲染 SDK,主要面向**美妆 / AR 试妆**场景:摄像头预览 → 人脸 landmark 检测 → 多帧 PNG 贴图按 motion 序列贴在人脸上。
支持任意分辨率手机(采用"短边正方形画布 + 长边 letterbox"布局)、前置 / 后置摄像头自动切换、镜子镜像效果。
---
## 目录结构
```
face_sdk/
├── app/ ← SDK 主体(可作为 AAR 给业务方接入)
│ ├── src/main/java/com/hmwl/face_sdk/
│ │ ├── FaceActivity.java ← 业务方继承的核心 Activity
│ │ ├── FaceLandmarkerHelper.kt ← MediaPipe 封装
│ │ ├── DebugLog.java ← Java/C++ 统一日志(写文件)
│ │ ├── InitArg.java / Motion.java / MotionList.java / Frame.java ← JSON 配置 POJO
│ │ └── ...
│ ├── src/main/cpp/main.cpp ← JNI + native 主循环
│ └── src/main/assets/shaders/ ← Vulkan GLSLbg/texture/...
├── vulkan/ ← 跨平台 Vulkan 渲染核心
│ ├── Application.{h,cpp} ← Vulkan 初始化 / 窗口生命周期
│ └── FaceApp.{h,cpp} ← 业务渲染(face mesh + bg quad + motion 调度)
├── example/ ← 示例 app,演示完整接入
│ ├── src/main/java/com/inewme/uvmirror/
│ │ ├── MainActivity.kt ← 入口(Compose UI,前/后置选择)
│ │ ├── MakeupActivity.kt ← 业务 Activity(继承 FaceActivity
│ │ └── MotionManager.kt ← motion 资源加载封装
│ ├── src/main/AndroidManifest.xml
│ └── src/main/assets/pic/
│ ├── motion_data_ex.json ← motion 元数据(帧文件名 + 锚点 x/y)
│ └── <motion_id>/000xxx.png ← motion 各帧贴图
├── compile_shader.bat ← 编译所有 .vert/.frag 为 .spv(必须在改 shader 后执行)
└── third_party/ ← VMA / glm / glfw / mediapipe …
```
---
## 快速上手
### 1. 编译 + 运行 example
```powershell
# Windows 上,确保 SDK 路径在 local.properties 里,以及 NDK 已安装
.\gradlew.bat :example:installDebug
```
启动后:
- 主界面有"前置摄像头 / 后置摄像头"两个 RadioButton(默认前置)
- 点击 "Go to Makeup" 进入 `MakeupActivity`
- `MakeupActivity` 自动加载 `assets/pic/motion_data_ex.json` 配置的 motion,按 `update()` 里的脚本播放
### 2. 改 shader 后必须重编
```powershell
.\compile_shader.bat
.\gradlew.bat :example:installDebug
```
---
## 在自己的 App 里集成
### Step 1:依赖 SDK 模块
在你的 `settings.gradle` / `build.gradle` 引入 `:app` 模块(或打包成 AAR)。
### Step 2:在 `AndroidManifest.xml` 声明业务 Activity
业务 Activity 必须**竖屏**,且**屏蔽常见 configChanges 不要重建**
```xml
<activity
android:name=".MakeupActivity"
android:exported="false"
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
```
### Step 3:继承 `FaceActivity` 写业务 Activity
```kotlin
class MakeupActivity : FaceActivity() {
lateinit var motionMgr: MotionManager
private var isMotionLoadFinished = false
override fun onCreate(savedInstanceState: Bundle?) {
// 在 super.onCreate 之前构造 MotionManager(它会调 SetInitArg
motionMgr = MotionManager(
this,
/* fps */ 10,
/* zoom */ 1.5f,
/* r,g,b */ 0f, 0f, 1f, // letterbox / 圆外纯色(蓝)
/* radius */ 240f, // 480 设计基线下的圆半径,SDK 自动按 canvas_size/480 缩放
/* offset */ 200f, -200f // 480 设计基线下的中心偏移
)
super.onCreate(savedInstanceState)
// 一次性加载所有要用到的 motion,加载完才能 PlayMotionList
motionMgr.LoadMotionList(listOf("4", "56")) {
isMotionLoadFinished = true
}
// 自己的 update 循环(生命周期感知,避免泄漏)
lifecycleScope.launch {
while (true) {
update()
delay(10)
}
}
}
private fun update() {
if (!isMotionLoadFinished) return
// 想播什么就调,调 SDK API(详见下面"API 参考"
}
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
if (keyCode == KeyEvent.KEYCODE_BACK) {
Stop() // ← 退出 Activity 前必须先 Stop(),否则 native 线程仍在跑
finish()
return true
}
return super.onKeyDown(keyCode, event)
}
}
```
### Step 4:从入口 Activity 启动业务 Activity,传入摄像头朝向
```kotlin
import androidx.camera.core.CameraSelector
import com.hmwl.face_sdk.FaceActivity
val intent = Intent(context, MakeupActivity::class.java).apply {
// 前置:CameraSelector.LENS_FACING_FRONT;后置:LENS_FACING_BACK
// 不传时 SDK 默认走后置(向后兼容)
putExtra(FaceActivity.EXTRA_LENS_FACING, CameraSelector.LENS_FACING_FRONT)
}
context.startActivity(intent)
```
前置摄像头 SDK 会自动:
- 在 GPU 端把 bg + face mesh 整体水平翻转 → 镜子效果
- mediapipe 输入仍是 raw(未镜像)帧,所以 landmark 与 raw 帧 UV 严格对齐
---
## API 参考(`FaceActivity` 公开方法)
| 方法 | 说明 |
| --- | --- |
| `static final String EXTRA_LENS_FACING` | Intent extra key,值为 `CameraSelector.LENS_FACING_FRONT``LENS_FACING_BACK` |
| `void SetInitArg(String json)` | 设置全局参数(zoom / r,g,b / radius / offset_x,offset_y / action_fps),JSON 格式见 `InitArg` |
| `String PreLoadAction(String json, LoadMotionListFinishedCallback cb)` | 异步预加载一组 motion 资源;加载完毕回调 cb |
| `void PlayMotionList(List<String> motionList, boolean loop, PlayMotionListFinishedCallback cb)` | 播放 motion 序列,`loop=true` 时无限循环,`cb` 为播完一轮的回调(loop 模式下不会触发,传 `null` 即可) |
| `void StopMotion()` | 暂停当前 motion 播放(停在当前帧) |
| `void ResumeMotion()` | 恢复 `StopMotion()` 之前的播放进度 |
| `void Stop()` | **完全停止 native 渲染**,退出 Activity 前必须调 |
**注意:** `LoadMotionList` / `PreLoadAction` 是异步的,回调里设置 `isMotionLoadFinished=true` 之后才能 `PlayMotionList`,否则播放命令会被静默丢弃。
---
## 资源准备
### Motion 帧图
每个 motion 用一个独立目录,PNG 帧按文件名顺序播放:
```
assets/pic/4/000000.png
assets/pic/4/000001.png
...
assets/pic/4/000031.png
```
PNG 必须是 **RGBA**(透明通道),SDK 直接采样后 alpha-blend 到 bg 上方。
### `motion_data_ex.json`
motion 元数据,描述每帧贴图的**锚点偏移**(贴在脸上的相对位置,画布坐标系,原点在屏幕中心):
```json
{
"4": {
"000000.png": { "x": 0.0, "y": 0.0 },
"000001.png": { "x": 0.0, "y": 0.0 },
...
},
"56": {
"000000.png": { "x": 0.0, "y": 0.0 },
...
}
}
```
---
## 设计要点(接入方可不读,但要改 shader / 渲染时必读)
### 1. 画布与 letterbox
- **画布**:屏幕短边的正方形,居中。屏幕长边方向自然 letterbox(上下或左右黑边)。
- 画布坐标系("画布 NDC")∈ [-1, +1]²,原点在画布中心。
- mediapipe FaceLandmarker 输出的归一化 landmark ∈ [0, 1]² 直接对应画布 NDC(经过 `*2-1`)。
- 业务参数 `zoom / offset_x / offset_y / radius` 都是**屏幕物理像素**(SDK 端按 `canvas_size / 480` 自动缩放,业务方用 480 设计基线即可)。
### 2. 摄像头帧 → 画布的几何流水线
```
raw 摄像头帧 (W×H, 4:3 典型)
└─① 中央裁切 min(W,H)×min(W,H) 正方形
└─② Matrix.postRotate(rotationDegrees) 顺时针旋转 → 正立 face crop
└─③ MediaPipe 输出 landmark ∈ [0,1]² ─────▶ 画布 NDC
└─ shader 做反向变换贴回 raw 帧 UV (bg)
└─ shader 直接当 face mesh 顶点 (face)
```
### 3. 镜子效果(前置摄像头)
`PushConstants.mirror_x ∈ {0.0, 1.0}`
- 前置时 = 1.0shader 里把最终 NDC.x 翻转 (`pos.x *= 1.0 - 2.0*mirror_x`)
- bg + face mesh 都做同样翻转 → 整体水平镜像,对齐保持
- mediapipe 仍接收**未镜像**的 raw 帧,避免双重镜像把 landmark 弄反
### 4. bg 与 face mesh 对齐不变量
bg 圆内显示摄像头视频,face mesh 把 motion png 贴到 landmark 位置——两者对齐依赖:
> **屏幕同一像素位置,bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe landmark 坐标**
实现方式:bg quad 与 face mesh **完全相同**地经过 `zoom × offset × letterbox × mirror` 几何链路;同时 bg 的 `outTexCoord` 基于"未变换的 quad 顶点 pos"算 raw UV——这样屏幕 fragment 反算的 pos 就等价于"对应 landmark 的画布 NDC"UV 完美贴合。
bg quad 顶点取 `±10`(而非 `±1`),保证任何 zoom/offset 下 quad 在屏幕上 GPU clip 后整屏覆盖,letterbox 也被 bg.frag 的 r/g/b 蓝色填充。
### 5. 摄像头分辨率适配
- CameraX 用 `setTargetResolution(640, 480)` 强制拿低分辨率帧(SDK 不需要高分辨率)
- shader 用 `camera_aspect = W/H``camera_rotation`CameraX 给的 rotationDegrees)做 UV 反变换
- 摄像头帧分辨率/朝向变化时,SDK 会在 `processCameraFrame` 里自动重建 bg 纹理
### 6. 窗口生命周期
- `APP_CMD_TERM_WINDOW``Application::cleanupForWindowLost()`:只销毁**窗口资源**surface / swapchain / framebuffers / imageViews / commandBuffers / sync),保留 renderPass / pipelines / device / VMA / textures
- `APP_CMD_INIT_WINDOW``Application::reinitForNewWindow()`:复用全局资源,只重建窗口资源
- `android_main` 退出时**不再额外销毁** Vulkan 资源(保留给下次 NativeActivity 复用),避免 use-after-free 崩溃
---
## 调试
### 日志文件
SDK 把 Java/Kotlin 日志(通过 `com.hmwl.face_sdk.DebugLog.i / e`)和 C++ 日志(`DebugLog::log`**统一写入**
```
/data/user/0/<your.package>/files/face_sdk_debug.log
```
```powershell
# 拉到本地分析
adb pull /data/user/0/com.inewme.uvmirror.f20260113/files/face_sdk_debug.log .
```
崩溃时还会自动追加一段含信号、寄存器、backtrace 的 dump`========== FACE_SDK CRASH ==========`)。
### 常见问题
| 现象 | 排查方向 |
| --- | --- |
| 黑屏 / 没有摄像头预览 | 检查相机权限是否授予;查看日志是否有 `processImageNative` 调用 |
| 人脸贴图不出现 | 检查 `motion_data_ex.json` 里的 motion id 是否在 `LoadMotionList` 里加载过;检查 `isMotionLoadFinished` 是否被回调置 true |
| 贴图不贴合人脸 | bg 与 face mesh 的 zoom/offset/mirror 必须严格一致——见"设计要点 #4 对齐不变量" |
| 切换摄像头后崩溃 | 见 `cleanupSecondInit` 注释;不要在 Android 路径调用它,仅 desktop 入口可用 |
| 改了 shader 不生效 | 必须先跑 `compile_shader.bat` 重新生成 `.spv` |
---
## 兼容性
- minSdk 29targetSdk 看 `app/build.gradle`
- 设备需支持 Vulkan 1.0+ 和 MediaPipe TFLite delegate
- 已在 MI 9Android 10、Adreno 640、Vulkan vendor `qglinternal`)上验证通过
+2 -1
View File
@@ -13,7 +13,7 @@ android {
compileSdk 36 compileSdk 36
defaultConfig { defaultConfig {
//applicationId "com.hmwl.face_sdk" //applicationId "com.hmwl.face_sdk"
minSdk 29 minSdk 30
targetSdk 36 targetSdk 36
versionCode 1 versionCode 1
versionName "1.0" versionName "1.0"
@@ -117,6 +117,7 @@ dependencies {
//implementation fileTree(dir: 'libs/tasks-vision', include: ['*.jar']) //implementation fileTree(dir: 'libs/tasks-vision', include: ['*.jar'])
//implementation fileTree(dir: 'libs/tasks-core', include: ['*.jar']) //implementation fileTree(dir: 'libs/tasks-core', include: ['*.jar'])
def camerax_version = '1.4.2' def camerax_version = '1.4.2'
implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-core:$camerax_version"
implementation "androidx.camera:camera-camera2:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version"
+1 -7
View File
@@ -16,13 +16,7 @@
<activity <activity
android:name=".FaceActivity" android:name=".FaceActivity"
android:exported="true" android:exported="true"
android:screenOrientation="portrait" tools:ignore="MissingClass"> <!-- 如果IDE报类找不到,可以添加这个 -->
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize"
tools:ignore="MissingClass,LockedOrientationActivity">
<!-- 锁定竖屏:SDK 渲染按"屏幕中央正方形画布 + 上下黑边 letterbox"
的方式适配任意分辨率,目前只验证过竖屏。
configChanges 全包:避免软键盘/导航栏变化触发 Activity 重建——SDK
自己会通过 onWindowLost/onWindowInit 重建 swapchain,性能更好。 -->
<!-- 重要:移除 MAIN/LAUNCHER intent-filter --> <!-- 重要:移除 MAIN/LAUNCHER intent-filter -->
<!-- 让使用库的应用来决定哪个是主Activity --> <!-- 让使用库的应用来决定哪个是主Activity -->
+24 -29
View File
@@ -1,46 +1,41 @@
#version 450 #version 450
// 输入变量(与顶点着色器输出对应)
layout(location = 0) in vec2 inTexCoord; layout(location = 0) in vec2 inTexCoord;
// 输出颜色
layout(location = 0) out vec4 outColor; layout(location = 0) out vec4 outColor;
// 纹理采样器
layout(binding = 0) uniform sampler2D texSampler; layout(binding = 0) uniform sampler2D texSampler;
// ★ PushConstants 必须和 bg.vert / FaceApp.h 完全一致(包括我们没用到的
// 字段 zoom/ux/uy/offset_x/offset_y,因为 push constant 是按 offset 取的,
// 缺字段会导致后面 radius/screen_w/screen_h 落到错误偏移)。
layout(push_constant) uniform PushConstants { layout(push_constant) uniform PushConstants {
float zoom; float zoom;
float r; float r;
float g; float g;
float b; float b;
float radius; float radius;
float ux; float ux;
float uy; float uy;
float offset_x; } pushConstants;
float offset_y;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(fragment 不用,仅为对齐 push constant layout
float camera_aspect;
float camera_rotation;
float mirror_x;
} pc;
void main() { void main() {
// gl_FragCoord 是屏幕 framebuffer 里的像素坐标(左下原点)。圆形 mask 中心 // outFragColor = color;
// 锚定在屏幕真实中心 = (screen_w, screen_h)/2。这里不再用画布中心,因为 // 获取当前片元的屏幕坐标(左下角为原点)
// letterbox 后画布中心和屏幕中心其实是同一个点(min 边居中),但用屏幕坐标
// 写法更直观也对 cover 模式更友好。
vec2 fragCoord = gl_FragCoord.xy; vec2 fragCoord = gl_FragCoord.xy;
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
// 屏幕中心(480x480 的中心是 (240, 240)
vec2 center = vec2(240.0, 240.0);
// 计算到中心的距离(欧氏距离,单位:像素)
float dist = length(fragCoord - center); float dist = length(fragCoord - center);
if (dist > pc.radius) { // 判断是否在半径之外
// 画布外(含 letterbox 黑边)+ 圆外区域统一用业务给的纯色覆盖。 if (dist > pushConstants.radius) {
// 业务一般给 r=g=0 / b=1 之类,与原版语义保持一致。 // 使用 push constant 中的 RGB 颜色(注意 alpha 设为 1.0
outColor = vec4(pc.r, pc.g, pc.b, 1.0); outColor = vec4(pushConstants.r, pushConstants.g, pushConstants.b, 1.0);
} else { } else {
// 采样纹理
outColor = texture(texSampler, inTexCoord); outColor = texture(texSampler, inTexCoord);
} }
} }
Binary file not shown.
+56 -119
View File
@@ -1,131 +1,68 @@
#version 450 #version 450
// ★ bg quad 顶点放大到 ±10(而不是 ±1)的原因: // 硬编码的顶点数据 - 4个顶点组成三角形带覆盖整个屏幕
// // const vec2 positions[6] = vec2[6](
// bg 与 face mesh 的对齐依赖一个关键不变量—— // vec2( 1.0, 1.0), // 右上 - 三角形1
// "屏幕上同一像素位置,bg 显示的 raw UV = 把这个屏幕位置反算回 mediapipe // vec2(-1.0, 1.0), // 左上 - 三角形1
// landmark 坐标"。要保住这个不变量,bg quad 必须经过和 face mesh 完全相 // vec2(-1.0, -1.0), // 左下 - 三角形1
// 同的 zoom × offset × letterbox × mirror 链路 (见下方 gl_Position 计算)
// 而 outTexCoord 必须基于"未变换的 quad 顶点 pos"算 raw UV。这样屏幕 fragment
// 反算出的 quad pos 就等价于"对应的 landmark NDC"UV 完美对齐。
//
// 但是 zoom (>1) + 非零 offset + screen_h > screen_w 的 letterbox 会让 quad
// 在屏幕上呈非对称分布——之前 quad 顶点 ±1 时实测在 zoom=1.5 offset=(450,-450)
// 屏幕 1080×2340 下,quad 在 NDC.y 只覆盖 [-0.885, 0.500],屏幕底部
// [0.500, 1.0] 大片 letterbox 没有被 bg 覆盖,露出 RenderPass 的 clearColor
// 黑色——表现为"上面有蓝色填充,下面没有"。
//
// 解法:把 quad 顶点扩大到 ±10。GPU 会把超出 NDC [-1, 1] 的部分硬件 clip 掉,
// 但 clip 之后留在屏幕内的 fragment 上,outTexCoord 是顶点的"线性插值"**屏幕
// 内 fragment 反算出的 pos 与 quad 顶点取值范围无关**——所以 face mesh 对齐
// 完全不受影响。同时 ±10 在任何合理的 zoom/offset/letterbox 下都能保证 quad
// 在屏幕上完全覆盖 [-1, 1]² 含 letterbox。
//
// Worst-case 验证:zoom=1.5, offset=(450,-450), canvas=1080, screen=1080×2340
// pos.y = -10 → screen_ndc.y = (-10*1.5 + (-450/1080)) * 1080/2340 = -7.12 ✓
// pos.y = +10 → screen_ndc.y = (+10*1.5 + (-450/1080)) * 1080/2340 = +6.73 ✓
// 远超屏幕范围,clip 后保证整屏 letterbox 都被 bg.frag 的 r/g/b 蓝色覆盖。
const vec2 positions[6] = vec2[6](
vec2( 10.0, -10.0),
vec2( 10.0, 10.0),
vec2(-10.0, 10.0),
vec2(-10.0, 10.0), // vec2(-1.0, -1.0), // 左下 - 三角形2
vec2(-10.0, -10.0), // vec2( 1.0, -1.0), // 右下 - 三角形2
vec2( 10.0, -10.0) // vec2( 1.0, 1.0) // 右上 - 三角形2
// );
const float offset = (640-480)/(480.0);
const vec2 positions[6] = vec2[6](
vec2( 1.0, -1.0 -offset), // 右下
vec2(1.0, 1.0 + offset), // 右上 - 三角形1
vec2(-1.0, 1.0 + offset), // 左上 - 三角形1
vec2(-1.0, 1.0 + offset), // 左上 - 三角形2
vec2( -1.0, -1.0-offset), // 左下 - 三角形2
vec2( 1.0, -1.0-offset) // 右下 - 三角形2
); );
// ★ PushConstants 必须和 C++ FaceApp.h 严格一致。前 9 个 float 是业务原始值
// SDK 已按 canvas_size/480 缩放到屏幕物理像素),后 5 个 float 是当前帧
// 的画布几何 + 相机帧元信息,由 FaceApp::render() 每帧写入。 const vec2 texCoords[6] = vec2[6](
vec2(1.0, 1.0), // 右上
vec2(0.0, 1.0), // 左上
vec2(0.0, 0.0), // 左下
vec2(0.0, 0.0), // 左下
vec2(1.0, 0.0), // 右下
vec2(1.0, 1.0) // 右上
);
// 定义 Push Constant,只有一个 float
layout(push_constant) uniform PushConstants { layout(push_constant) uniform PushConstants {
float zoom; float zoom;
float r; float r;
float g; float g;
float b; float b;
float radius; float radius;
float ux; float ux;
float uy; float uy;
float offset_x; float offset_x;
float offset_y; float offset_y;
// 画布几何(每帧写入) } pushConstants;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(每次相机帧上传时写入)
float camera_aspect; // raw width / height(典型 4:3 = 1.333…)
float camera_rotation; // CameraX rotationDegrees: 0/90/180/270
// 镜子效果开关:1.0 = 前置摄像头,最终 NDC.x 翻转一次。
// 这与 FaceLandmarkerHelper 在前置时对 bitmap 做 postScale(-1,1) 的语义对偶:
// ① mediapipe 输入是"用户视角"图像 → landmark 也是用户视角;
// ② bg.vert 算出的 outTexCoord 还是 raw 帧 UV(旋转矩阵把 user 视角→raw);
// ③ 最后我们把 NDC.x 翻转,bg 在屏幕上呈现为水平镜像 = 镜子效果;
// ④ texture.vert 同样翻转一次,face 贴图与 bg 完美对齐。
float mirror_x;
} pc;
// 输出变量
layout(location = 0) out vec2 outTexCoord; layout(location = 0) out vec2 outTexCoord;
void main() { void main() {
vec2 pos = positions[gl_VertexIndex]; // 获取顶点位置
vec2 position = positions[gl_VertexIndex];
// ===== outTexCoord:基于"未变换的 quad pos" 算 raw UV ===== position = position * pushConstants.zoom;
// position.x = position.x + (pushConstants.offset_x/480);
// 重点:这里必须用未做 zoom/offset/letterbox/mirror 变换的原始 pos 来计算 position.y = position.y + (pushConstants.offset_y/480);
// outTexCoord,而 gl_Position 在下面才做这些变换。这样:
// - 屏幕 fragment X 处对应的 quad pos = (X 反过 mirror、letterbox、offset、zoom) // 设置输出位置(Vulkan使用不同的坐标系)
// - face mesh 顶点的 mediapipe landmark 在屏幕上的位置 X' 经过相同链路 gl_Position = vec4(position, 0.0, 1.0);
// - 当 X = X' 时(face mesh 顶点重叠某个屏幕像素),fragment 反算的 pos
// 刚好等于 face landmark 的画布 NDCbg 该 fragment 显示 landmark 对应 // 输出纹理坐标
// raw UV → 与 face mesh 完美贴合。 outTexCoord = texCoords[gl_VertexIndex];
// }
// 公式链:画布 NDC ─[逆时针 rotation]→ 中央方形 NDC ─[scale & shift]→ raw UV
//
// 关于旋转方向:
// Android Matrix.postRotate(deg) 在 Bitmap canvasy 朝下)里是「视觉顺时针」。
// Vulkan NDC 也是 y 朝下。所以画布 NDC → 中央方形 NDC 是「视觉逆时针 rotation」。
// GLSL column-major mat2(c, -s, s, c) 对应矩阵 [c, s; -s, c]。
// 验证 rotation=0 时矩阵 = identity ✓
// 验证 rotation=90 时矩阵 = [0, 1; -1, 0]apply 到画布右上 (1,-1)
// (0*1 + 1*(-1), -1*1 + 0*(-1)) = (-1, -1) → 中央方形左上 ✓
// 物理:mediapipe 输入右上像素 = 中央方形顺时针 90° 后的右上 = 中央方形左上 ✓
float ang = radians(pc.camera_rotation);
float c = cos(ang);
float s = sin(ang);
mat2 R = mat2(c, -s, s, c); // 画布 NDC → 中央方形 NDC(视觉逆时针 rotation
vec2 cm = R * pos;
// 中央方形 NDC ∈ [-1, +1]² → raw 帧 UV。中央方形在 raw 帧 UV 上是中央
// min(W, H) x min(W, H) 的正方形,对应的 UV 半径:
// raw 横向长(W >= H):halfU = H/(2W) = 1/(2 * aspect)halfV = 0.5
// raw 纵向长(H > W):halfU = 0.5halfV = W/(2H) = aspect/2
float halfU, halfV;
if (pc.camera_aspect >= 1.0) {
halfU = 0.5 / pc.camera_aspect;
halfV = 0.5;
} else {
halfU = 0.5;
halfV = 0.5 * pc.camera_aspect;
}
outTexCoord = vec2(0.5 + halfU * cm.x, 0.5 + halfV * cm.y);
// ===== gl_Position:业务 zoom/offset + letterbox + mirror,与 texture.vert 严格一致 =====
//
// pos 仍按 quad 顶点 ±10 参与位置计算,pc.offset_x / pc.offset_y 已是屏幕物理
// 像素(SDK 端做过 canvas_size/480 缩放),/canvas_size 后是画布 NDC 单位;
// 最后 *canvas_size/screen_{w,h} 把画布缩到屏幕中央正方形,长边方向自然
// letterbox。quad 顶点放大到 ±10 让 GPU clip 后整屏覆盖。
pos *= pc.zoom;
pos.x += pc.offset_x / pc.canvas_size;
pos.y += pc.offset_y / pc.canvas_size;
pos.x *= pc.canvas_size / pc.screen_w;
pos.y *= pc.canvas_size / pc.screen_h;
// 镜子效果:前置时 mirror_x=1.0,把 NDC.x 翻转一次。无分支:
// 后置:(1 - 2*0) = +1 → 不变;
// 前置:(1 - 2*1) = -1 → 水平翻转。
// texture.vert 也做同样翻转,bg + face 整体水平镜像,face 贴图与 bg 完美对齐。
pos.x *= (1.0 - 2.0 * pc.mirror_x);
gl_Position = vec4(pos, 0.0, 1.0);
}
Binary file not shown.
+23 -31
View File
@@ -5,48 +5,40 @@ layout (binding = 1) uniform sampler2D samplerColor;
layout (location = 0) in vec2 inUV; layout (location = 0) in vec2 inUV;
layout (location = 1) in vec3 inNormal; layout (location = 1) in vec3 inNormal;
// ★ PushConstants 必须和 C++ FaceApp.h / texture.vert 严格一致。
// 关键:原版这里只声明到 uy,缺了 offset_x 之后的字段,那时之所以没报错是因为
// fragment 没读到那些字段。本次重构 fragment 要读 screen_w/screen_h,所以必须
// 把整个块声明完整,否则 layout offset 会错位。
layout(push_constant) uniform PushConstants { layout(push_constant) uniform PushConstants {
float zoom; float zoom;
float r; float r;
float g; float g;
float b; float b;
float radius; float radius;
float ux; float ux;
float uy; float uy;
float offset_x; } pushConstants;
float offset_y;
float canvas_size;
float screen_w;
float screen_h;
// 相机帧元信息(fragment 不用,仅为对齐 push constant layout
float camera_aspect;
float camera_rotation;
float mirror_x;
} pc;
layout (location = 0) out vec4 outFragColor; layout (location = 0) out vec4 outFragColor;
void main() void main()
{ {
// gl_FragCoord 是屏幕 framebuffer 像素坐标。圆形 mask 中心锚定在屏幕真实中心, // outFragColor = color;
// 跟随分辨率自动适配;不再写死 (240,240)。 // 获取当前片元的屏幕坐标(左下角为原点)
vec2 fragCoord = gl_FragCoord.xy; vec2 fragCoord = gl_FragCoord.xy;
vec2 center = vec2(pc.screen_w, pc.screen_h) * 0.5;
// 屏幕中心(480x480 的中心是 (240, 240)
vec2 center = vec2(240.0, 240.0);
// 计算到中心的距离(欧氏距离,单位:像素)
float dist = length(fragCoord - center); float dist = length(fragCoord - center);
if (dist > pc.radius) { // 判断是否在半径之外
// 画布外(含 letterbox 黑边)+ 圆外区域用业务纯色覆盖,与 bg.frag 一致。 if (dist > pushConstants.radius) {
outFragColor = vec4(pc.r, pc.g, pc.b, 1.0); // 使用 push constant 中的 RGB 颜色(注意 alpha 设为 1.0
outFragColor = vec4(pushConstants.r, pushConstants.g, pushConstants.b, 1.0);
} else { } else {
// out.png 是 4096x2048 的 8x4 子图图集;ux/uy 选定当前 motion 帧子图, vec2 pos = vec2(pushConstants.ux/4096.f, pushConstants.uy/2048.f);
// inUV 在子图内插值。这里和分辨率重构无关,保持原版逻辑不动。 vec2 uv = pos + vec2(inUV.x/8.f, inUV.y/4.f);
vec2 pos = vec2(pc.ux / 4096.0, pc.uy / 2048.0);
vec2 uv = pos + vec2(inUV.x / 8.0, inUV.y / 4.0);
vec4 color = texture(samplerColor, uv); vec4 color = texture(samplerColor, uv);
outFragColor = color; outFragColor = color;
} }
}
}
Binary file not shown.
+17 -50
View File
@@ -1,5 +1,6 @@
#version 450 #version 450
layout (location = 0) in vec3 inPos; layout (location = 0) in vec3 inPos;
layout (location = 1) in vec2 inUV; layout (location = 1) in vec2 inUV;
layout (location = 2) in vec3 inNormal; layout (location = 2) in vec3 inNormal;
@@ -15,31 +16,17 @@ layout (binding = 0) uniform UBO
layout (location = 0) out vec2 outUV; layout (location = 0) out vec2 outUV;
layout (location = 1) out vec3 outNormal; layout (location = 1) out vec3 outNormal;
// ★ PushConstants 必须和 C++ FaceApp.h / bg.vert / bg.frag 严格一致。
// 前 9 个 float 是业务原始值(已被 SDK 在 render() 里按 canvas_size/480 缩放成
// 屏幕物理像素);后 3 个 float 是当前帧画布几何,由 SDK 每帧写入。
layout(push_constant) uniform PushConstants { layout(push_constant) uniform PushConstants {
float zoom; float zoom;
float r; float r;
float g; float g;
float b; float b;
float radius; float radius;
float ux; float ux;
float uy; float uy;
float offset_x; float offset_x;
float offset_y; float offset_y;
float canvas_size; } pushConstants;
float screen_w;
float screen_h;
// 相机帧元信息:camera_aspect / camera_rotation 在 face 渲染中不用——face
// landmark 已经是"正立中央方形"的归一化坐标,mediapipe 端转过;保留只是
// 为了对齐 push constant layout。
float camera_aspect;
float camera_rotation;
// mirror_x:1.0 表示前置摄像头镜子效果——把最终 gl_Position.x 翻转一次。
// bg.vert 也做同样翻转,所以 bg 与 face 同步水平镜像、互相对齐。
float mirror_x;
} pc;
out gl_PerVertex out gl_PerVertex
{ {
@@ -50,33 +37,13 @@ void main()
{ {
outUV = inUV; outUV = inUV;
// 模型顶点坐标 inPos.xy 来自 OBJ,已经是 [0,1]² 归一化范围。先映射到 vec2 position = vec2(inPos.xy*2 -1);
// "画布 NDC ∈ [-1,+1]"。 position = position * pushConstants.zoom;
vec2 position = vec2(inPos.xy * 2.0 - 1.0); position.x = position.x + (pushConstants.offset_x/480);
position.y = position.y + (pushConstants.offset_y/480);
gl_Position = vec4(position, 0.5, 1.0);
// 画布坐标系:缩放 + 业务像素偏移。 vec4 pos = ubo.model * vec4(inPos, 1.0);
// pc.offset_x / pc.offset_y 是屏幕物理像素(SDK 已做过 canvas_size/480 缩放),
// 除以 canvas_size 转成画布 NDC,跟原版 "/480" 在 480 屏上完全等价。
position = position * pc.zoom;
position.x = position.x + (pc.offset_x / pc.canvas_size);
position.y = position.y + (pc.offset_y / pc.canvas_size);
// 画布 NDC → 屏幕 NDC:屏幕长边方向乘以 canvas_size/screen_long < 1.0
// 让模型只占据屏幕中央正方形画布区域,长边方向的画布外是 letterbox。
position.x = position.x * (pc.canvas_size / pc.screen_w);
position.y = position.y * (pc.canvas_size / pc.screen_h);
// 镜子效果:前置摄像头时 mirror_x=1.0,把 NDC.x 翻转一次。
// 后置:(1 - 2*0) = +1 → 不变
// 前置:(1 - 2*1) = -1 → 水平翻转
// 无分支写法,配合 bg.vert 同样的翻转,让 bg + face 整体镜像、保持对齐。
position.x *= (1.0 - 2.0 * pc.mirror_x);
gl_Position = vec4(position, 0.5, 1.0);
// 注意:ubo.projection / ubo.model 在当前管线里没有真正参与 gl_Position 的
// 计算(顶点位置已直接给出 NDC)。这里仍按原版保留 normal 的世界变换,避免
// 影响其它依赖 outNormal 的 fragment 逻辑(例如 thick 版本)。
vec4 pos = ubo.model * vec4(inPos, 1.0);
outNormal = mat3(inverse(transpose(ubo.model))) * inNormal; outNormal = mat3(inverse(transpose(ubo.model))) * inNormal;
} }
Binary file not shown.
-512
View File
@@ -1,512 +0,0 @@
// CrashHandler.cpp
//
// Async-signal-safe crash dumper for the face_sdk native library. The goal
// is that the next time the app dies (e.g. another FORTIFY: pthread_mutex_lock
// called on a destroyed mutex from inside the Vulkan driver) we don't only
// have logcat — we also have a self-contained dump on the device's app data
// directory that the user can pull off and send back, even after a reboot.
//
// Why we need this:
// - DebugLog only writes "happy path" events. The Vulkan crash happens
// synchronously inside vkQueueSubmit / vkFreeCommandBuffers — there is
// no DebugLog call near the crash site, so the file log just stops.
// - logcat survives across the crash (debuggerd dumps the backtrace there)
// but logcat is volatile: a couple of reboots, a logcat -c, or a long
// idle period and it's gone. We want a persistent file.
// - tombstones in /data/tombstones/ are root-only on consumer devices.
//
// Implementation notes:
// - Inside a signal handler we MUST stick to async-signal-safe APIs
// (man 7 signal-safety). That rules out fprintf / snprintf / malloc.
// We use write(), our own integer-to-string conversion, and a single
// pre-opened fd. dladdr() is technically not on the POSIX safe list but
// bionic's implementation only takes one rwlock and is widely used in
// other crash dumpers (breakpad, crashpad, libunwindstack). We accept
// that risk because the alternative is no symbol at all.
// - We use <unwind.h> (_Unwind_Backtrace) instead of execinfo.h because
// bionic doesn't ship execinfo.h on all NDK levels, and _Unwind_Backtrace
// is the same primitive Android's own tombstoned uses.
// - We re-raise the original signal with the default handler at the end so
// the OS still produces a tombstone / ANR record for vendors that
// read /data/tombstones/.
#include "CrashHandler.h"
#ifndef _WIN32
#include <android/log.h>
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <signal.h>
#include <stdint.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <unwind.h>
#include <atomic>
namespace {
// ---------- Globals (touched from the handler -> only POD/atomics) ---------
constexpr size_t kCrashStackSize = 64 * 1024; // sigaltstack
constexpr size_t kMaxFrames = 64;
constexpr size_t kNoteCapacity = 256;
uint8_t g_sigStack[kCrashStackSize];
int g_crashFd = -1; // crash log fd (append, sync)
int g_debugFd = -1; // optional: also dup write to debug log
std::atomic<bool> g_installed{false};
std::atomic<bool> g_handlingCrash{false};
// Single writer of g_note: setNote() (uses memcpy under a tiny lock, but the
// handler reads byte-by-byte so a torn read at most produces a truncated
// note, never a deref of bad memory).
char g_note[kNoteCapacity] = {0};
pthread_mutex_t g_noteMtx = PTHREAD_MUTEX_INITIALIZER;
const int g_signals[] = { SIGSEGV, SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGSYS };
constexpr size_t kNumSignals = sizeof(g_signals) / sizeof(g_signals[0]);
// ----------------------- Async-signal-safe writers -------------------------
// Write a NUL-terminated string. Drops the trailing NUL.
void sigWrite(int fd, const char* s) {
if (fd < 0 || !s) return;
size_t n = 0;
while (s[n]) ++n;
if (n == 0) return;
// Loop until everything is written or we error out. Async-safe.
while (n > 0) {
ssize_t w = write(fd, s, n);
if (w <= 0) {
if (w < 0 && errno == EINTR) continue;
return;
}
s += w;
n -= (size_t)w;
}
}
// Write n bytes from buf. Used for the note (may contain anything).
void sigWriteN(int fd, const char* buf, size_t n) {
if (fd < 0 || !buf) return;
while (n > 0) {
ssize_t w = write(fd, buf, n);
if (w <= 0) {
if (w < 0 && errno == EINTR) continue;
return;
}
buf += w;
n -= (size_t)w;
}
}
// Write the same string to both the crash log and the debug log (if any).
void sigDump(const char* s) {
sigWrite(g_crashFd, s);
sigWrite(g_debugFd, s);
}
// Convert an unsigned integer to its decimal representation. Returns the
// number of characters written into buf (without a NUL).
size_t u64ToDec(uint64_t v, char* buf, size_t cap) {
if (cap == 0) return 0;
char tmp[32];
size_t i = 0;
if (v == 0) {
tmp[i++] = '0';
} else {
while (v && i < sizeof(tmp)) {
tmp[i++] = (char)('0' + (v % 10));
v /= 10;
}
}
size_t out = (i < cap) ? i : cap;
for (size_t k = 0; k < out; ++k) {
buf[k] = tmp[i - 1 - k];
}
return out;
}
// Convert an unsigned 64-bit value to a fixed-width 16-digit hex string.
// Useful for PC values.
size_t u64ToHex16(uint64_t v, char* buf, size_t cap) {
static const char digits[] = "0123456789abcdef";
if (cap < 16) return 0;
for (int i = 15; i >= 0; --i) {
buf[i] = digits[v & 0xF];
v >>= 4;
}
return 16;
}
// Write "<key>=<u64>\n".
void sigWriteKV_u64(int fd, const char* key, uint64_t v) {
sigWrite(fd, key);
sigWrite(fd, "=");
char num[24];
size_t n = u64ToDec(v, num, sizeof(num));
sigWriteN(fd, num, n);
sigWrite(fd, "\n");
}
// Write "<key>=0x<hex>\n".
void sigWriteKV_ptr(int fd, const char* key, uint64_t v) {
sigWrite(fd, key);
sigWrite(fd, "=0x");
char hx[16];
u64ToHex16(v, hx, sizeof(hx));
sigWriteN(fd, hx, 16);
sigWrite(fd, "\n");
}
// ------------------------- Signal name lookup ------------------------------
const char* signalName(int signo) {
switch (signo) {
case SIGSEGV: return "SIGSEGV";
case SIGABRT: return "SIGABRT";
case SIGBUS: return "SIGBUS";
case SIGFPE: return "SIGFPE";
case SIGILL: return "SIGILL";
case SIGSYS: return "SIGSYS";
case SIGTRAP: return "SIGTRAP";
default: return "SIG?";
}
}
// si_code to short string. Only the most common ones; everything else falls
// back to the numeric value via sigWriteKV_u64.
const char* siCodeName(int signo, int code) {
switch (signo) {
case SIGSEGV:
if (code == SEGV_MAPERR) return "SEGV_MAPERR";
if (code == SEGV_ACCERR) return "SEGV_ACCERR";
break;
case SIGBUS:
if (code == BUS_ADRALN) return "BUS_ADRALN";
if (code == BUS_ADRERR) return "BUS_ADRERR";
if (code == BUS_OBJERR) return "BUS_OBJERR";
break;
case SIGFPE:
if (code == FPE_INTDIV) return "FPE_INTDIV";
if (code == FPE_INTOVF) return "FPE_INTOVF";
if (code == FPE_FLTDIV) return "FPE_FLTDIV";
break;
case SIGILL:
if (code == ILL_ILLOPC) return "ILL_ILLOPC";
if (code == ILL_ILLOPN) return "ILL_ILLOPN";
break;
case SIGABRT:
if (code == SI_TKILL) return "SI_TKILL";
if (code == SI_USER) return "SI_USER";
break;
}
return "?";
}
// ------------------------- Backtrace via _Unwind_Backtrace -----------------
struct UnwindCtx {
uintptr_t* frames;
size_t count;
size_t cap;
};
_Unwind_Reason_Code unwindCallback(_Unwind_Context* ctx, void* arg) {
UnwindCtx* uc = static_cast<UnwindCtx*>(arg);
if (uc->count >= uc->cap) return _URC_END_OF_STACK;
uintptr_t pc = _Unwind_GetIP(ctx);
if (pc) {
// Trim Thumb bit on 32-bit ARM. No-op on aarch64/x86_64.
pc &= ~(uintptr_t)1;
uc->frames[uc->count++] = pc;
}
return _URC_NO_REASON;
}
size_t captureBacktrace(uintptr_t* out, size_t cap) {
UnwindCtx uc{out, 0, cap};
_Unwind_Backtrace(&unwindCallback, &uc);
return uc.count;
}
// Dump one frame: " #02 pc 000000000000abcd /path/lib.so (Symbol+0x10)"
void dumpFrame(int fd, size_t idx, uintptr_t pc) {
sigWrite(fd, " #");
char num[8];
if (idx < 10) {
num[0] = '0';
num[1] = (char)('0' + idx);
sigWriteN(fd, num, 2);
} else {
size_t n = u64ToDec(idx, num, sizeof(num));
sigWriteN(fd, num, n);
}
sigWrite(fd, " pc ");
char hx[16];
u64ToHex16((uint64_t)pc, hx, sizeof(hx));
sigWriteN(fd, hx, 16);
Dl_info info;
memset(&info, 0, sizeof(info));
if (dladdr(reinterpret_cast<void*>(pc), &info) && info.dli_fname) {
sigWrite(fd, " ");
sigWrite(fd, info.dli_fname);
if (info.dli_sname) {
uintptr_t sym = reinterpret_cast<uintptr_t>(info.dli_saddr);
uintptr_t off = (sym && pc >= sym) ? (pc - sym) : 0;
sigWrite(fd, " (");
sigWrite(fd, info.dli_sname);
sigWrite(fd, "+0x");
char ohx[16];
u64ToHex16((uint64_t)off, ohx, sizeof(ohx));
sigWriteN(fd, ohx, 16);
sigWrite(fd, ")");
} else if (info.dli_fbase) {
uintptr_t base = reinterpret_cast<uintptr_t>(info.dli_fbase);
uintptr_t off = (pc >= base) ? (pc - base) : 0;
sigWrite(fd, " (offset 0x");
char ohx[16];
u64ToHex16((uint64_t)off, ohx, sizeof(ohx));
sigWriteN(fd, ohx, 16);
sigWrite(fd, ")");
}
}
sigWrite(fd, "\n");
}
// ------------------------- Time + tid helpers ------------------------------
// Builds "YYYY-MM-DD HH:MM:SS.mmm UTC" into the given buffer (no NUL).
// Returns the number of bytes written. Async-signal-safe (no stdio, no
// localtime_r tz lookups).
size_t formatTimestamp(char* b, size_t cap) {
timespec ts{};
clock_gettime(CLOCK_REALTIME, &ts);
struct tm tm_info{};
time_t s = ts.tv_sec;
gmtime_r(&s, &tm_info);
size_t i = 0;
auto putUInt = [&](unsigned v, int width) {
char tmp[8];
size_t n = u64ToDec(v, tmp, sizeof(tmp));
while ((int)n < (size_t)width) {
if (i < cap) b[i++] = '0';
++n;
}
for (size_t k = 0; k < n; ++k) {
if (i < cap) b[i++] = tmp[k];
}
};
putUInt((unsigned)(tm_info.tm_year + 1900), 4); if (i < cap) b[i++] = '-';
putUInt((unsigned)(tm_info.tm_mon + 1), 2); if (i < cap) b[i++] = '-';
putUInt((unsigned)(tm_info.tm_mday), 2); if (i < cap) b[i++] = ' ';
putUInt((unsigned)(tm_info.tm_hour), 2); if (i < cap) b[i++] = ':';
putUInt((unsigned)(tm_info.tm_min), 2); if (i < cap) b[i++] = ':';
putUInt((unsigned)(tm_info.tm_sec), 2); if (i < cap) b[i++] = '.';
putUInt((unsigned)(ts.tv_nsec / 1000000), 3);
static const char kSuffix[] = " UTC";
for (size_t k = 0; k < sizeof(kSuffix) - 1; ++k) {
if (i < cap) b[i++] = kSuffix[k];
}
return i;
}
pid_t currentTid() {
return static_cast<pid_t>(syscall(SYS_gettid));
}
// ------------------------- Signal handler ----------------------------------
void crashHandler(int signo, siginfo_t* info, void* ucontext) {
(void)ucontext;
// Re-entry guard: if a second signal fires while we're dumping (e.g. our
// own dladdr trips a SIGSEGV) just chain to the default handler.
bool expected = false;
if (!g_handlingCrash.compare_exchange_strong(expected, true,
std::memory_order_acq_rel)) {
// Already in handler -> default + bail.
signal(signo, SIG_DFL);
raise(signo);
return;
}
sigDump("\n========== FACE_SDK CRASH ==========\n");
sigDump("time=");
{
char tsbuf[48];
size_t tn = formatTimestamp(tsbuf, sizeof(tsbuf));
sigWriteN(g_crashFd, tsbuf, tn);
sigWriteN(g_debugFd, tsbuf, tn);
}
sigDump("\n");
sigDump("signal=");
sigDump(signalName(signo));
sigDump(" code=");
sigDump(siCodeName(signo, info ? info->si_code : 0));
sigDump("\n");
if (info) {
sigWriteKV_u64(g_crashFd, "si_signo", (uint64_t)info->si_signo);
sigWriteKV_u64(g_debugFd, "si_signo", (uint64_t)info->si_signo);
sigWriteKV_u64(g_crashFd, "si_code", (uint64_t)info->si_code);
sigWriteKV_u64(g_debugFd, "si_code", (uint64_t)info->si_code);
sigWriteKV_ptr(g_crashFd, "si_addr", (uint64_t)(uintptr_t)info->si_addr);
sigWriteKV_ptr(g_debugFd, "si_addr", (uint64_t)(uintptr_t)info->si_addr);
}
sigWriteKV_u64(g_crashFd, "pid", (uint64_t)getpid());
sigWriteKV_u64(g_debugFd, "pid", (uint64_t)getpid());
sigWriteKV_u64(g_crashFd, "tid", (uint64_t)currentTid());
sigWriteKV_u64(g_debugFd, "tid", (uint64_t)currentTid());
// Note (current frame index, current motion, ...).
sigDump("note=");
sigWriteN(g_crashFd, g_note, strnlen(g_note, kNoteCapacity));
sigWriteN(g_debugFd, g_note, strnlen(g_note, kNoteCapacity));
sigDump("\n");
sigDump("backtrace:\n");
uintptr_t frames[kMaxFrames];
size_t nf = captureBacktrace(frames, kMaxFrames);
for (size_t i = 0; i < nf; ++i) {
dumpFrame(g_crashFd, i, frames[i]);
dumpFrame(g_debugFd, i, frames[i]);
}
sigDump("==================================\n");
// Make sure everything reaches disk before we self-destruct.
if (g_crashFd >= 0) fsync(g_crashFd);
if (g_debugFd >= 0) fsync(g_debugFd);
// Mirror to logcat too so a quick `adb logcat -d` after a reboot still
// shows the SIGNAL line (helps cross-checking the file).
__android_log_print(ANDROID_LOG_FATAL, "FACE_DBG_CRASH",
"fatal %s @ tid=%d, see face_sdk_crash.log",
signalName(signo), currentTid());
// Restore default handler and re-raise. This produces /data/tombstones/*
// on rooted devices and tells debuggerd to print the official Android
// backtrace into logcat (`crash_dump64 ... DEBUG`).
struct sigaction dfl{};
dfl.sa_handler = SIG_DFL;
sigemptyset(&dfl.sa_mask);
sigaction(signo, &dfl, nullptr);
raise(signo);
}
} // namespace
namespace CrashHandler {
void install(const std::string& internalDataPath,
const std::string& debugLogPath) {
bool expected = false;
if (!g_installed.compare_exchange_strong(expected, true,
std::memory_order_acq_rel)) {
return; // already installed
}
// Open the persistent crash log file in append mode. We keep it open
// forever so the signal handler doesn't have to call open() (which is
// safe but slow).
{
std::string path = internalDataPath.empty()
? std::string("/data/local/tmp/face_sdk_crash.log")
: internalDataPath + "/face_sdk_crash.log";
g_crashFd = open(path.c_str(),
O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
0644);
if (g_crashFd >= 0) {
// Header for the new run.
sigWrite(g_crashFd, "\n=== CrashHandler installed pid=");
char pidbuf[16];
size_t n = u64ToDec((uint64_t)getpid(), pidbuf, sizeof(pidbuf));
sigWriteN(g_crashFd, pidbuf, n);
sigWrite(g_crashFd, " ===\n");
fsync(g_crashFd);
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG",
"CrashHandler log file: %s", path.c_str());
} else {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler failed to open %s: %s",
path.c_str(), strerror(errno));
}
}
// Also keep a writable fd to the DebugLog file (if any) so dumps land
// next to the regular tail of the log. We don't touch g_fp inside
// DebugLog because that would need its mutex — not safe in handler.
if (!debugLogPath.empty()) {
g_debugFd = open(debugLogPath.c_str(),
O_WRONLY | O_APPEND | O_CLOEXEC,
0644);
if (g_debugFd < 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: cannot open debug log %s: %s",
debugLogPath.c_str(), strerror(errno));
}
}
// Set up an alternate stack so we still have stack space if the original
// thread ran out (very common for Vulkan crashes inside deep driver
// call chains).
stack_t ss{};
ss.ss_sp = g_sigStack;
ss.ss_size = sizeof(g_sigStack);
ss.ss_flags = 0;
if (sigaltstack(&ss, nullptr) != 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: sigaltstack failed: %s",
strerror(errno));
}
struct sigaction sa{};
sa.sa_sigaction = &crashHandler;
sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART;
sigemptyset(&sa.sa_mask);
for (size_t i = 0; i < kNumSignals; ++i) {
if (sigaction(g_signals[i], &sa, nullptr) != 0) {
__android_log_print(ANDROID_LOG_WARN, "FACE_DBG",
"CrashHandler: sigaction(%d) failed: %s",
g_signals[i], strerror(errno));
}
}
__android_log_print(ANDROID_LOG_INFO, "FACE_DBG",
"CrashHandler installed for SIGSEGV/SIGABRT/SIGBUS/SIGFPE/SIGILL/SIGSYS");
}
void setNote(const char* note) {
if (!note) note = "";
pthread_mutex_lock(&g_noteMtx);
size_t n = strnlen(note, kNoteCapacity - 1);
memcpy(g_note, note, n);
g_note[n] = '\0';
pthread_mutex_unlock(&g_noteMtx);
}
} // namespace CrashHandler
#else // _WIN32 — no-op on host build
namespace CrashHandler {
void install(const std::string&, const std::string&) {}
void setNote(const char*) {}
} // namespace CrashHandler
#endif
-33
View File
@@ -1,33 +0,0 @@
#ifndef FACE_SDK_CRASH_HANDLER_H
#define FACE_SDK_CRASH_HANDLER_H
#include <string>
namespace CrashHandler {
// Install signal handlers for SIGSEGV / SIGABRT / SIGBUS / SIGFPE / SIGILL.
//
// On a fatal signal we:
// 1) Switch to a pre-allocated 64KB sigaltstack so we still have stack
// space even if the original thread's stack was the cause.
// 2) Write a self-contained crash record to <internalDataPath>/face_sdk_crash.log
// (and also try to append it to the DebugLog file passed in).
// The record contains: signal name, si_code, si_addr, pid, tid,
// a synchronous unwound backtrace (PC + module + offset for each frame),
// and the value of any extra context the caller registered via setNote().
// 3) Re-raise the original signal with the default handler so that the
// Android tombstone pipeline still produces /data/tombstones/* files.
//
// Safe to call once. Subsequent calls are no-ops. Must be called AFTER
// DebugLog::init() so we know the log directory.
void install(const std::string& internalDataPath,
const std::string& debugLogPath = "");
// Optional short note (<= 256 chars) appended to every crash dump from now on.
// Useful to record "current frame index", "current motion", etc. so we know
// what was happening at the moment of the crash. Async-signal-safe to read.
void setNote(const char* note);
} // namespace CrashHandler
#endif // FACE_SDK_CRASH_HANDLER_H
+21 -87
View File
@@ -4,14 +4,12 @@
#include <game-activity/GameActivity.h> #include <game-activity/GameActivity.h>
#include "AndroidOut.h" #include "AndroidOut.h"
#include "DebugLog.h" #include "DebugLog.h"
#include "CrashHandler.h"
#include "FaceApp.h" #include "FaceApp.h"
#include <android/asset_manager.h> #include <android/asset_manager.h>
#include <sys/syscall.h> #include <sys/syscall.h>
#include <unistd.h> #include <unistd.h>
#include <atomic> #include <atomic>
#include <chrono> #include <chrono>
#include <cstdio>
#include <thread> #include <thread>
using namespace std; using namespace std;
@@ -56,13 +54,9 @@ void handle_cmd(android_app *pApp, int32_t cmd) {
case APP_CMD_INIT_WINDOW: case APP_CMD_INIT_WINDOW:
aout << "APP_CMD_INIT_WINDOW" << std::endl; aout << "APP_CMD_INIT_WINDOW" << std::endl;
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: calling onWindowInit()"); DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: calling onWindowInit()");
if (g_Application != nullptr) { g_Application->onWindowInit();
g_Application->onWindowInit(); DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: onWindowInit() returned, isInited=%d",
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: onWindowInit() returned, isInited=%d", (int)g_Application->isInited());
(int)g_Application->isInited());
} else {
DebugLog::log("handle_cmd APP_CMD_INIT_WINDOW: g_Application is null, skipping");
}
break; break;
case APP_CMD_TERM_WINDOW: case APP_CMD_TERM_WINDOW:
aout << "APP_CMD_TERM_WINDOW" << std::endl; aout << "APP_CMD_TERM_WINDOW" << std::endl;
@@ -83,15 +77,6 @@ char g_InitArgString[ArgLen] = {0};
void android_main(struct android_app *pApp) { void android_main(struct android_app *pApp) {
DebugLog::init(pApp->activity->internalDataPath); DebugLog::init(pApp->activity->internalDataPath);
// 安装信号处理器越早越好:之前的 FORTIFY: pthread_mutex_lock 崩溃
// 是裸的 SIGABRT,没有任何 signal handler,所以 DebugLog 文件停在最后
// 一条业务日志、看不到任何崩溃栈。装上之后任何 fatal signal 都会把:
// * 信号名 / si_code / si_addr / pid / tid / 当前 note
// * 完整 backtracePC + 模块 + 符号 + 偏移)
// 同步落到 internalDataPath/face_sdk_crash.log(以及 DebugLog 文件尾),
// 然后再走默认 handler 让 debuggerd 产生 tombstone。
CrashHandler::install(pApp->activity->internalDataPath,
DebugLog::getLogPath());
DebugLog::log("android_main start, pid=%d tid=%d, logPath=%s", DebugLog::log("android_main start, pid=%d tid=%d, logPath=%s",
getpid(), dbg_tid(), DebugLog::getLogPath().c_str()); getpid(), dbg_tid(), DebugLog::getLogPath().c_str());
aout << "Welcome to android_main" << std::endl; aout << "Welcome to android_main" << std::endl;
@@ -152,16 +137,6 @@ void android_main(struct android_app *pApp) {
(unsigned long long)s_frameIdx, (unsigned long long)s_frameIdx,
(unsigned long long)s_excCount); (unsigned long long)s_excCount);
} }
// 每 60 帧刷新一次崩溃 note,崩溃时 dump 里能看到「最后一次活着」
// 的帧号和异常数,定位是不是在某一个固定帧附近卡死。
if (s_frameIdx % 60 == 0) {
char note[128];
std::snprintf(note, sizeof(note),
"drawFrame frame=%llu exc=%llu",
(unsigned long long)s_frameIdx,
(unsigned long long)s_excCount);
CrashHandler::setNote(note);
}
try { try {
g_Application->drawFrame(frameTime); g_Application->drawFrame(frameTime);
@@ -191,37 +166,19 @@ void android_main(struct android_app *pApp) {
} }
} while (!pApp->destroyRequested); } while (!pApp->destroyRequested);
DebugLog::log("android_main: destroyRequested, running cleanup"); DebugLog::log("android_main: destroyRequested, running cleanup");
// 关键:这里只复位 FaceApp 的"second-init" 状态机标记,**不销毁**任何
// Vulkan 资源。
//
// - 窗口相关资源(surface/swapchain/framebuffers/imageViews/
// commandBuffers/sync)已经在 APP_CMD_TERM_WINDOW → onWindowLost()
// 里被 cleanupForWindowLost() 干净销毁;
// - renderPass / graphicsPipeline / pipelineLayout / VkDevice /
// VkInstance / VMA / 所有 textures / FaceApp 自身 pipelines 是 g_Application
// 单例的"全局资源",跨 NativeActivity 实例复用。下次 Activity 启动
// onWindowInit 会走 reinitForNewWindow() 重建窗口资源、复用全局资源。
//
// 历史教训:之前这里调过 g_Application->cleanupSecondInit(),那个函数会
// 把 renderPass / graphicsPipeline / pipelineLayout 全部 destroy 掉但既
// 不置 VK_NULL_HANDLE 也不复位 _applicationInited。结果用户从主界面切镜
// 头再进 MakeupActivity 时,onWindowInit 看到 _applicationInited=1 走 reinit
// 复用路径,引用悬空的 renderPass 调 vkCreateFramebuffervalidation
// layer 检测出 use-after-free 直接 SEGV。
//application.cleanup(); //application.cleanup();
//g_Application->cleanupSecondInit(); g_Application->cleanupSecondInit();
g_Application->clearnSecondFaceApp(); g_Application->clearnSecondFaceApp();
DebugLog::log("android_main: exit"); DebugLog::log("android_main: exit");
} }
} }
extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX); extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize);
extern void ReceiveFacePoint(float* pos, int count, int width, int height); extern void ReceiveFacePoint(float* pos, int count, int width, int height);
void processWithVulkan(uint8_t* data, int width, int height, int format, void processWithVulkan(uint8_t* data, int width, int height, int format,
int rowStride, size_t dataSize, int rotation, bool mirrorX) { int rowStride, size_t dataSize) {
TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize, rotation, mirrorX); TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize);
} }
@@ -231,13 +188,14 @@ JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer, Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer,
jint width, jint height, jint format, jint width, jint height, jint format,
jint row_stride, jint pixel_stride, jint row_stride, jint pixel_stride,
jint rotation, jboolean mirror_x) { jint rotation) {
static std::atomic<uint64_t> s_imgCount{0}; static std::atomic<uint64_t> s_imgCount{0};
uint64_t n = s_imgCount.fetch_add(1) + 1; uint64_t n = s_imgCount.fetch_add(1) + 1;
if (n == 1 || n % 300 == 0) { if (n == 1 || n % 300 == 0) {
DebugLog::log("processImageNative #%llu tid=%d w=%d h=%d mirror=%d", DebugLog::log("processImageNative #%llu tid=%d w=%d h=%d",
(unsigned long long)n, dbg_tid(), width, height, (int)mirror_x); (unsigned long long)n, dbg_tid(), width, height);
} }
// TODO: implement processImageNative()
uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer)); uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer));
jlong capacity = env->GetDirectBufferCapacity(buffer); jlong capacity = env->GetDirectBufferCapacity(buffer);
@@ -245,12 +203,9 @@ Java_com_hmwl_face_1sdk_FaceActivity_processImageNative(JNIEnv *env, jobject thi
return; return;
} }
// rotation = ImageInfo.getRotationDegrees(),给 shader 用来做 UV 旋转, // 在这里将图像数据传递给 Vulkan
// 让不同 sensor orientation 的设备显示方向一致。 // 创建 Vulkan 图像或更新现有图像
// mirror_x = true 表示前置摄像头:所有顶点 shader 会把 gl_Position.x 翻转 processWithVulkan(imageData, width, height, format, row_stride, capacity);
// 一次,达到"镜子效果"——这是化妆/美妆类 app 的标准体验。
processWithVulkan(imageData, width, height, format, row_stride, capacity,
(int)rotation, mirror_x == JNI_TRUE);
} }
extern "C" extern "C"
JNIEXPORT void JNICALL JNIEXPORT void JNICALL
@@ -258,21 +213,19 @@ Java_com_hmwl_face_1sdk_FaceActivity_passDataToNative(JNIEnv *env, jobject thiz,
jint point_count, jint width, jint height) { jint point_count, jint width, jint height) {
static std::atomic<uint64_t> s_ptCount{0}; static std::atomic<uint64_t> s_ptCount{0};
uint64_t n = s_ptCount.fetch_add(1) + 1; uint64_t n = s_ptCount.fetch_add(1) + 1;
if (n == 1 || n % 300 == 0) {
DebugLog::log("passDataToNative #%llu tid=%d point_count=%d w=%d h=%d",
(unsigned long long)n, dbg_tid(), point_count, width, height);
}
// TODO: implement passDataToNative()
float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer)); float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer));
if (pos == nullptr) { if (pos == nullptr) {
DebugLog::log_throttled("passDataToNative.nullBuffer", // 处理错误
"passDataToNative #%llu got NULL DirectBufferAddress!",
(unsigned long long)n);
return; return;
} }
// 节流:每 120 次实际数据通路打一行(约每 4 秒一次,足够确认通路活着即可)。
if (n == 1 || n % 120 == 0) {
DebugLog::log("passDataToNative #%llu point_count=%d w=%d h=%d p0=(%.3f,%.3f,%.3f)",
(unsigned long long)n, point_count, width, height,
pos[0], pos[1], pos[2]);
}
// 或者直接将指针传递给Vulkan
ReceiveFacePoint(pos, point_count, width, height); ReceiveFacePoint(pos, point_count, width, height);
} }
@@ -513,23 +466,4 @@ JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_FaceActivity_ResumeMotionNative(JNIEnv *env, jobject thiz) { Java_com_hmwl_face_1sdk_FaceActivity_ResumeMotionNative(JNIEnv *env, jobject thiz) {
DebugLog::log("JNI ResumeMotionNative tid=%d", dbg_tid()); DebugLog::log("JNI ResumeMotionNative tid=%d", dbg_tid());
g_Application->ResumeMotion(); g_Application->ResumeMotion();
}
// 给 Java/Kotlin 侧用的日志桥:把 Java 端关键事件写进 native 的
// face_sdk_debug.log,使 Java + native + Vulkan 渲染日志能在同一个
// 文件里按时间戳排好序,一次 adb pull 就能拿到完整时间线。
extern "C"
JNIEXPORT void JNICALL
Java_com_hmwl_face_1sdk_DebugLog_nativeLog(JNIEnv *env, jclass clazz,
jstring jtag, jstring jmsg) {
if (jmsg == nullptr) return;
const char* tag = jtag ? env->GetStringUTFChars(jtag, nullptr) : nullptr;
const char* msg = env->GetStringUTFChars(jmsg, nullptr);
if (tag) {
DebugLog::log("[J][%s] %s", tag, msg);
} else {
DebugLog::log("[J] %s", msg);
}
if (tag) env->ReleaseStringUTFChars(jtag, tag);
env->ReleaseStringUTFChars(jmsg, msg);
} }
@@ -1,38 +0,0 @@
package com.hmwl.face_sdk;
import android.util.Log;
/**
* Java/Kotlin 端日志桥。所有调用最终通过 JNI 写到 native 端的
* face_sdk_debug.log 文件(与 native DebugLog::log 共用一份),
* 同时镜像到 logcat(tag = 调用方 tag),方便联机调试时一次 adb pull
* 就能拿到完整时间线(Java + native + Vulkan 渲染)。
*
* 使用约定:
* - 业务低频事件用 i(tag, msg)
* - 错误用 e(tag, msg)
* - 高频帧级日志请自行节流,本类不再做二次节流
*/
public final class DebugLog {
private DebugLog() {}
public static void i(String tag, String msg) {
Log.i(tag, msg);
try {
nativeLog(tag, msg);
} catch (UnsatisfiedLinkError e) {
// native 还没 attach 的极早期阶段,吞掉避免崩溃
}
}
public static void e(String tag, String msg) {
Log.e(tag, msg);
try {
nativeLog("[E]" + tag, msg);
} catch (UnsatisfiedLinkError e) {
// 同上
}
}
private static native void nativeLog(String tag, String msg);
}
@@ -7,8 +7,7 @@ import android.util.Log;
import android.view.View; import android.view.View;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import android.util.Size; import androidx.camera.core.AspectRatio;
import androidx.camera.core.CameraSelector; import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis; import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy; import androidx.camera.core.ImageProxy;
@@ -39,27 +38,6 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
System.loadLibrary("face_sdk"); System.loadLibrary("face_sdk");
} }
/**
* Intent extra key:调用方(example/MainActivity 等)通过它告诉 FaceActivity
* 启动哪个朝向的摄像头。值类型 int,取自 androidx.camera.core.CameraSelector
* - CameraSelector.LENS_FACING_BACK (0)
* - CameraSelector.LENS_FACING_FRONT (1)
* 缺省(未提供 extra)时走 LENS_FACING_BACK,保留旧调用方的行为。
*
* 该值同时决定两件事:
* ① CameraSelector 朝向;
* ② mirrorX 镜像标志:前置摄像头时 sensor 帧是"用户右手在画面右侧",
* 为了显示成"镜子效果"(用户右手在屏幕左侧),所有顶点 shader 在
* gl_Position 阶段对 NDC.x 做一次水平翻转。这件事通过 push constant
* 的 mirror_x 字段透传到 GPU;对应地 FaceLandmarkerHelper 也会对
* bitmap 做 postScale(-1, 1) 让 mediapipe 输入与显示画面方向一致,
* 确保 face 贴图与 bg 完美对齐。
*/
public static final String EXTRA_LENS_FACING = "com.hmwl.face_sdk.LENS_FACING";
/** 当前 session 使用的摄像头朝向。在 onCreate 里从 Intent extra 读取并固定。 */
private int lensFacing = CameraSelector.LENS_FACING_BACK;
private ProcessCameraProvider cameraProvider; private ProcessCameraProvider cameraProvider;
private void initCamera(){ private void initCamera(){
backgroundExecutor = Executors.newSingleThreadExecutor(); backgroundExecutor = Executors.newSingleThreadExecutor();
@@ -94,28 +72,11 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
cameraProvider = cameraProviderFuture.get(); cameraProvider = cameraProviderFuture.get();
CameraSelector cameraSelector = new CameraSelector.Builder() CameraSelector cameraSelector = new CameraSelector.Builder()
.requireLensFacing(lensFacing) .requireLensFacing(CameraSelector.LENS_FACING_BACK)
.build(); .build();
// 显式锁定分析帧分辨率为 480x640(CameraX UI 坐标系,竖屏:短边 x 长边)。
// 不同手机的相机硬件原生支持的分辨率差异很大(旗舰机随便给 1920x1440、
// 入门机可能是 320x240),CameraX 默认会按"最贴近 setTargetAspectRatio
// 的 supported size"挑一个,结果在不同设备上拿到的分析帧大小都不一样。
// 这会带来三个麻烦:
// ① bg 纹理 GPU 内存占用波动很大(高端机一帧 ~10MB staging buffer);
// ② mediapipe 推理耗时随分辨率非线性升高(Pixel 上 1920x1440 比
// 640x480 慢 4-6 倍),帧率掉得明显;
// ③ 我们的 SDK 用归一化 landmark + 中央方形 crop,本来就不需要更高
// 分辨率,多出来的像素是纯浪费。
//
// 锁定 480x640 后:sensor 坐标系下分析帧固定 640x480 (rotation=90 时)
// FaceLandmarkerHelper 中央方形裁切固定 480x480,bg 上传纹理也固定,
// 不同手机上行为一致。
//
// 实际尺寸由 CameraX 决定(会找最接近的硬件支持档位),但设备实测
// 几乎都能精确给出 640x480。
imageAnalyzer = new ImageAnalysis.Builder() imageAnalyzer = new ImageAnalysis.Builder()
.setTargetResolution(new Size(480, 640)) .setTargetAspectRatio(AspectRatio.RATIO_4_3)
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build(); .build();
@@ -153,24 +114,24 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
} }
private void processImageForVulkan(ImageProxy image) { private void processImageForVulkan(ImageProxy image) {
// 获取图像信息
int width = image.getWidth(); int width = image.getWidth();
int height = image.getHeight(); int height = image.getHeight();
int format = image.getFormat(); int format = image.getFormat();
// 获取图像数据平面
ImageProxy.PlaneProxy[] planes = image.getPlanes(); ImageProxy.PlaneProxy[] planes = image.getPlanes();
// 对于 RGBA_8888 格式,通常只有一个平面
if (planes.length > 0) { if (planes.length > 0) {
ImageProxy.PlaneProxy plane = planes[0]; ImageProxy.PlaneProxy plane = planes[0];
ByteBuffer buffer = plane.getBuffer(); ByteBuffer buffer = plane.getBuffer();
int rowStride = plane.getRowStride(); int rowStride = plane.getRowStride();
int pixelStride = plane.getPixelStride(); int pixelStride = plane.getPixelStride();
// mirrorX 为 true 时所有顶点 shader 会把 gl_Position.x 翻转一次,达到 // 调用 Native 方法处理图像
// "镜子效果"——前置摄像头唯一需要的额外处理。后置不开镜像。
boolean mirrorX = (lensFacing == CameraSelector.LENS_FACING_FRONT);
processImageNative(buffer, width, height, format, processImageNative(buffer, width, height, format,
rowStride, pixelStride, image.getImageInfo().getRotationDegrees(), rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
mirrorX);
} }
} }
@@ -193,16 +154,11 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
// Native 方法 // Native 方法
private native void processImageNative(ByteBuffer buffer, int width, int height, private native void processImageNative(ByteBuffer buffer, int width, int height,
int format, int rowStride, int pixelStride, int format, int rowStride, int pixelStride,
int rotation, boolean mirrorX); int rotation);
private void detectFace(ImageProxy imageProxy) { private void detectFace(ImageProxy imageProxy) {
// 前置摄像头:FaceLandmarkerHelper 内部会对 bitmap 做 postScale(-1, 1)
// 让 mediapipe 看到的是"用户视角"的图像。这样 landmark 输出就是用户视角
// 归一化坐标,跟 shader 里 gl_Position.x 翻转一次后的 bg 画面方向一致,
// face 贴图位置才能精确对齐。
boolean isFrontCamera = (lensFacing == CameraSelector.LENS_FACING_FRONT);
faceLandmarkerHelper.detectLiveStream( faceLandmarkerHelper.detectLiveStream(
imageProxy, isFrontCamera imageProxy,false
); );
} }
@@ -225,15 +181,6 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
@Override @Override
protected void onCreate(Bundle savedInstanceState){ protected void onCreate(Bundle savedInstanceState){
super.onCreate(savedInstanceState); super.onCreate(savedInstanceState);
// 必须在 initCamera() 之前读,CameraSelector 的朝向就靠这个 lensFacing。
// 兼容旧调用方:未提供 EXTRA_LENS_FACING 时默认后置。
if (getIntent() != null) {
lensFacing = getIntent().getIntExtra(
EXTRA_LENS_FACING, CameraSelector.LENS_FACING_BACK);
}
Log.d(TAG, "onCreate lensFacing=" + lensFacing
+ " (front=" + CameraSelector.LENS_FACING_FRONT
+ " back=" + CameraSelector.LENS_FACING_BACK + ")");
initCamera(); initCamera();
backgroundExecutor.execute(new Runnable() { backgroundExecutor.execute(new Runnable() {
@@ -333,18 +280,11 @@ public class FaceActivity extends GameActivity implements FaceLandmarkerHelper.L
FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
floatBuffer.put(points); floatBuffer.put(points);
floatBuffer.position(0); floatBuffer.position(0);
// 节流:每 30 次调用打一次到 native 文件,方便和 native 那边的 long cur_time = System.currentTimeMillis();
// passDataToNative.tick 对账。 Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
onResultsCallCount++;
if (onResultsCallCount % 30 == 1) {
DebugLog.i("FaceActivity",
"onResults call#" + onResultsCallCount
+ " index=" + index + " input=" + width + "x" + height
+ " p0=(" + points[0] + "," + points[1] + "," + points[2] + ")");
}
passDataToNative(nativeBuffer, index, width, height); passDataToNative(nativeBuffer, index, width, height);
Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
} }
private long onResultsCallCount = 0;
public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height); public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
@@ -173,13 +173,6 @@ class FaceLandmarkerHelper(
" while not using RunningMode.LIVE_STREAM" " while not using RunningMode.LIVE_STREAM"
) )
} }
// 节流日志:每 ~30 次调用打一次(detectFace 在 FaceActivity 里只在
// frameCount % 3 == 0 时调用,所以 30 次大约对应 90 帧 ≈ 3 秒)。用来
// 确认 mediapipe 入口活着、isFrontCamera/rotation/分辨率有没有跑偏。
if ((frameId % 30L) == 0L) {
Log.i(TAG, "detectLiveStream tick frameId=$frameId src=${imageProxy.width}x${imageProxy.height}" +
" rot=${imageProxy.imageInfo.rotationDegrees} isFrontCamera=$isFrontCamera")
}
val frameTime = SystemClock.uptimeMillis() val frameTime = SystemClock.uptimeMillis()
val currentFrameId = frameId++ val currentFrameId = frameId++
@@ -201,40 +194,22 @@ class FaceLandmarkerHelper(
imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) } imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) }
//imageProxy.close() //imageProxy.close()
// 重要:这里 *不* 对前置摄像头做 postScale(-1, 1)!理由如下:
//
// 我们的渲染策略是 —— bg 与 face 顶点 shader 末尾统一对 gl_Position.x
// 乘 (1 - 2*mirror_x) 做镜子翻转。bg 是直接采样 sensor 帧、然后 shader
// 末尾翻一次;face 顶点用的是 mediapipe 输出的归一化坐标,shader 末尾
// 也翻一次。两者只有同时基于"sensor 视角"做输入、再一起在 shader 末尾
// 翻一次,face 才能精确对齐 bg 上的脸。
//
// 如果在这里给 mediapipe 喂"已镜像"的 bitmap(旧实现),mediapipe 输出
// 的 landmark 就是"用户视角"坐标,再被 shader 翻一次 → face 实际上被
// 翻了两次,跟只翻一次的 bg 错位(通常会在屏幕另一半,看起来就是"face
// 完全没渲染")。这正是我们最近调试看到的现象。
//
// mediapipe 对左右镜像不敏感(训练集本身覆盖各角度),sensor 视角下也
// 能稳定检测出 478 个 landmark,无需迎合"镜子视角"。因此 isFrontCamera
// 参数保留以备扩展,但当前不再据此修改输入图。
val matrix = Matrix().apply { val matrix = Matrix().apply {
// Rotate the frame received from the camera to be in the same direction as it'll be shown
postRotate(imageProxy.imageInfo.rotationDegrees.toFloat()) postRotate(imageProxy.imageInfo.rotationDegrees.toFloat())
// flip image if user use front camera
if (isFrontCamera) {
postScale(
-1f,
1f,
imageProxy.width.toFloat(),
imageProxy.height.toFloat()
)
}
} }
// 旧代码硬编码了 (640-480)/2 = 80 和 bitmapBuffer.height=480),假设
// 相机帧是 640x480。在不同手机/不同 CameraX 配置下相机分辨率会变(比如
// 480x640 portrait sensor、1280x720 等),硬编码会越界裁切或丢一半画面。
//
// 改成:根据 imageProxy 的实际尺寸取中央 min(W,H) x min(W,H) 正方形作为
// mediapipe 输入。这与 bg.vert 的 UV 计算严格对齐 —— shader 那边也是从
// raw 帧的中央 min(W,H) 正方形采样,再按 rotation 旋转。两端语义一致才能
// 让 FaceLandmark 输出的归一化坐标 [0,1] 直接对应屏幕画布 NDC。
val srcW = imageProxy.width
val srcH = imageProxy.height
val side = minOf(srcW, srcH)
val cropX = (srcW - side) / 2
val cropY = (srcH - side) / 2
val rotatedBitmap = Bitmap.createBitmap( val rotatedBitmap = Bitmap.createBitmap(
bitmapBuffer, cropX, cropY, side, side, bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height,
matrix, true matrix, true
) )
@@ -295,14 +270,9 @@ class FaceLandmarkerHelper(
//{ //{
val finishTimeMs = SystemClock.uptimeMillis() val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs() val inferenceTime = finishTimeMs - result.timestampMs()
hitResultCount++ Log.i("TimeLatency",
if (hitResultCount % 30L == 1L) { "总延时: ${inferenceTime}ms | "
// 节流:检测到人脸的频次。和 emptyResultCount 配合看就能知道 )
// 命中率(hitResultCount / (hitResultCount + emptyResultCount))。
Log.i(TAG, "returnLivestreamResult: HIT count=$hitResultCount" +
" landmarks=${result.faceLandmarks().firstOrNull()?.size}" +
" input=${input.width}x${input.height} infer=${inferenceTime}ms")
}
faceLandmarkerHelperListener?.onResults( faceLandmarkerHelperListener?.onResults(
ResultBundle( ResultBundle(
result, result,
@@ -314,19 +284,9 @@ class FaceLandmarkerHelper(
//} //}
} }
else { else {
// 节流:mediapipe 拿到 frame 但没识别出人脸时,每 30 次空结果打一行。
// 只要这条日志在刷,就说明 mediapipe pipeline 正常活着,断点在"喂进去
// 的图本身没有可识别的脸"——常见于 isFrontCamera 路径下输入图被翻坏。
emptyResultCount++
if (emptyResultCount % 30L == 1L) {
Log.w(TAG, "returnLivestreamResult: faceLandmarks empty (count=$emptyResultCount)" +
" input=${input.width}x${input.height} ts=$frameId")
}
faceLandmarkerHelperListener?.onEmpty() faceLandmarkerHelperListener?.onEmpty()
} }
} }
private var emptyResultCount: Long = 0
private var hitResultCount: Long = 0
// Return errors thrown during detection to this FaceLandmarkerHelper's // Return errors thrown during detection to this FaceLandmarkerHelper's
// caller // caller
+2 -2
View File
@@ -9,8 +9,8 @@ android {
compileSdk 36 compileSdk 36
defaultConfig { defaultConfig {
applicationId "com.inewme.uvmirror.f20260127" applicationId "com.inewme.uvmirror.f20260113"
minSdk 29 minSdk 30
targetSdk 36 targetSdk 36
versionCode 1 versionCode 1
versionName "1.0" versionName "1.0"
+1 -5
View File
@@ -23,11 +23,7 @@
<activity <activity
android:name=".MakeupActivity" android:name=".MakeupActivity"
android:exported="false" android:exported="false" />
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|keyboard|navigation|smallestScreenSize" />
<!-- MakeupActivity 继承 FaceActivityface_sdk 渲染目前只支持竖屏。
configChanges 与 FaceActivity 保持一致,避免软键盘/导航栏触发重建。 -->
</application> </application>
</manifest> </manifest>
@@ -5,27 +5,16 @@ import android.os.Bundle
import androidx.activity.ComponentActivity import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent import androidx.activity.compose.setContent
import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.selection.selectable
import androidx.compose.material3.Button import androidx.compose.material3.Button
import androidx.compose.material3.MaterialTheme import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Surface import androidx.compose.material3.Surface
import androidx.compose.material3.Text import androidx.compose.material3.Text
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.tooling.preview.Preview
import androidx.camera.core.CameraSelector
import com.hmwl.face_sdk.FaceActivity
class MainActivity : ComponentActivity() { class MainActivity : ComponentActivity() {
override fun onCreate(savedInstanceState: Bundle?) { override fun onCreate(savedInstanceState: Bundle?) {
@@ -47,30 +36,17 @@ fun MyApp(modifier: Modifier = Modifier) {
@Composable @Composable
fun Greeting(name: String, modifier: Modifier = Modifier) { fun Greeting(name: String, modifier: Modifier = Modifier) {
val context = LocalContext.current val context = LocalContext.current // 获取上下文用于启动 Activity
// 默认前置:化妆/美妆类典型场景。
// 用 CameraSelector.LENS_FACING_FRONT / LENS_FACING_BACK 直接当 state,省去
// 中间枚举类型,最后通过 Intent extra 透传给 FaceActivity。
var lensFacing by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) }
Column( Column(
modifier = modifier.padding(16.dp) modifier = modifier.padding(16.dp)
) { ) {
Text(text = "Hello $name!") Text(text = "Hello $name!")
LensFacingRadioGroup(
selected = lensFacing,
onSelected = { lensFacing = it }
)
Button( Button(
onClick = { onClick = {
val intent = Intent(context, MakeupActivity::class.java).apply { // 跳转到 MakeupActivity
// FaceActivity 在 onCreate 里读这个 extra 来决定 CameraSelector 朝向 val intent = Intent(context, MakeupActivity::class.java)
// 与 isFrontCamera 镜像逻辑;缺省时(旧调用方)走 LENS_FACING_BACK
// 向后兼容。
putExtra(FaceActivity.EXTRA_LENS_FACING, lensFacing)
}
context.startActivity(intent) context.startActivity(intent)
} }
) { ) {
@@ -79,43 +55,10 @@ fun Greeting(name: String, modifier: Modifier = Modifier) {
} }
} }
@Composable
private fun LensFacingRadioGroup(
selected: Int,
onSelected: (Int) -> Unit,
modifier: Modifier = Modifier
) {
val options = listOf(
CameraSelector.LENS_FACING_FRONT to "前置摄像头",
CameraSelector.LENS_FACING_BACK to "后置摄像头"
)
Column(modifier = modifier.padding(vertical = 8.dp)) {
Text(text = "摄像头朝向")
options.forEach { (value, label) ->
Row(
modifier = Modifier
.selectable(
selected = (value == selected),
onClick = { onSelected(value) },
role = Role.RadioButton
)
.padding(vertical = 4.dp),
verticalAlignment = Alignment.CenterVertically
) {
RadioButton(
selected = (value == selected),
onClick = null
)
Text(text = label, modifier = Modifier.padding(start = 8.dp))
}
}
}
}
@Preview(showBackground = true) @Preview(showBackground = true)
@Composable @Composable
fun GreetingPreview() { fun GreetingPreview() {
MaterialTheme { MaterialTheme {
Greeting("Android") Greeting("Android")
} }
} }
@@ -1,19 +1,11 @@
package com.inewme.uvmirror package com.inewme.uvmirror
import android.graphics.Color
import android.os.Bundle import android.os.Bundle
import android.util.Log import android.util.Log
import android.view.Gravity
import android.view.KeyEvent import android.view.KeyEvent
import android.view.KeyEvent.KEYCODE_BACK import android.view.KeyEvent.KEYCODE_BACK
import android.view.MotionEvent import android.view.MotionEvent
import android.view.View
import android.view.ViewGroup
import android.widget.FrameLayout
import android.widget.LinearLayout
import android.widget.ProgressBar
import android.widget.TextView
import androidx.lifecycle.lifecycleScope import androidx.lifecycle.lifecycleScope
import com.hmwl.face_sdk.FaceActivity import com.hmwl.face_sdk.FaceActivity
@@ -26,12 +18,9 @@ class MakeupActivity : FaceActivity()
{ {
lateinit var motionMgr: MotionManager lateinit var motionMgr: MotionManager
private var updateJob: Job? = null private var updateJob: Job? = null
private var loadingOverlay: View? = null
override fun onCreate(savedInstanceState: Bundle?) { override fun onCreate(savedInstanceState: Bundle?) {
motionMgr = MotionManager(this, 10, 1.5f, 0f,0f,1f,240f, 200f, -200f) motionMgr = MotionManager(this, 10, 1.5f, 0f,0f,1f,240f, 200f, -200f)
super.onCreate(savedInstanceState) super.onCreate(savedInstanceState)
showLoadingOverlay()
// 启动一个生命周期感知的协程, 每10ms 调用一次,可以通过update 来处理业务的逻辑 // 启动一个生命周期感知的协程, 每10ms 调用一次,可以通过update 来处理业务的逻辑
updateJob = lifecycleScope.launch { updateJob = lifecycleScope.launch {
while (true) { while (true) {
@@ -40,71 +29,90 @@ class MakeupActivity : FaceActivity()
} }
} }
// 首先一次性加载当前业务逻辑所要求的所有动画资源, 之后就可以任意跳转,和播放,不用考虑状态 // 首先一次性加载当前业务逻辑所要求的所有动画资源, 之后就可以任意跳转,和播放,不用考虑状态
motionMgr.LoadMotionList(sequenceString){ motionMgr.LoadMotionList(sequenceAll){
//当加载完成了所有资源后会回调 //当加载完成了所有资源后会回调
isMotionLoadFinished = true isMotionLoadFinished = true
hideLoadingOverlay()
} }
} }
private fun showLoadingOverlay() {
val overlay = FrameLayout(this).apply {
setBackgroundColor(0x99000000.toInt())
}
val container = LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
gravity = Gravity.CENTER
layoutParams = FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT
)
}
container.addView(ProgressBar(this).apply {
layoutParams = LinearLayout.LayoutParams(
LinearLayout.LayoutParams.WRAP_CONTENT,
LinearLayout.LayoutParams.WRAP_CONTENT
)
})
container.addView(TextView(this).apply {
text = "加载中..."
setTextColor(Color.WHITE)
textSize = 16f
layoutParams = LinearLayout.LayoutParams(
LinearLayout.LayoutParams.WRAP_CONTENT,
LinearLayout.LayoutParams.WRAP_CONTENT
).apply { topMargin = 24 }
})
overlay.addView(container)
addContentView(overlay, ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT
))
loadingOverlay = overlay
}
private fun hideLoadingOverlay() {
runOnUiThread { loadingOverlay?.visibility = View.GONE }
}
//定义加载完成的变量,通过回调函数设置,加载完成后才可以进行动画播放的操作 //定义加载完成的变量,通过回调函数设置,加载完成后才可以进行动画播放的操作
var isMotionLoadFinished: Boolean = false var isMotionLoadFinished: Boolean = false
private var hasStartedPlaying: Boolean = false
private var currentIndex: Int = 0
private val sequence = listOf(4,5,6,7,11,12,13,14,21,56,59,62,65,68,71,78,81)
private val sequenceString = sequence.map { it.toString() }
//这里是模拟业务逻辑的地方
// 模拟问题1, 打断当前的动画,然后跳转到指定步骤
// 首先播放动画序列 粉底(4、5、6、7)、腮红(55、59)、口红由(76、77、78) 一直循环播放 "4", "5", "6", "7", "55", "59", "76", "77", "78"
// 然后 当播放到 10秒的时候,也可以是其他事件,需要暂停动画
// 然后 暂停5秒,这个时候需要继续播放动画
// 然后 又播放了8秒钟,这个时候要却换到 (11, 13, 21 动画序列了), 同时设置序列播放完成回调函数
// 然后 播放完这个序列又要切换为原来的序列播放,通过序列播放完成回调函数实现
private var elapsedTime = 0 // 总经过时间(ms
private val sequenceA = listOf("4")
private val sequenceB = listOf("56")
private val sequenceAll = listOf("4", "56")
private fun update() private fun update()
{ {
if(isMotionLoadFinished && !hasStartedPlaying) if(isMotionLoadFinished)
{ {
hasStartedPlaying = true if(elapsedTime == 0){
PlayMotionList(listOf(sequenceString[0]), true, null) //这次播放是一直循环播放,我们不需要判断动画是否播放完成, 回调函数直接传入null
PlayMotionList(sequenceA, true, null)
}
elapsedTime += 10 // 动画加载完成后开始计时,每次调用增加 10ms
if(elapsedTime == 10*1000){
//暂停动画
StopMotion()
}
if(elapsedTime == 10*1000 + 5*1000){
//恢复播放动画
ResumeMotion()
}
if(elapsedTime == 10*1000 + 5*1000 + 8*1000){
PlayMotionList(sequenceB, true){
//这里是播放完动画后的回调,再次切换到原来的 sequenceA
PlayMotionList(sequenceA, true, null)
}
}
} }
} }
// 模拟问题2, 当语音播报结束的时候,打断当前的动画, 跳转到另外一个动画
// 首先播放动画序列 粉底(4、5、6、7)、
// 然后 当播放到 10秒的时候,表示语音播报结束, 条装到 "55", "59" 动画
// private var elapsedTime = 0 // 总经过时间(ms
// private val sequenceA = listOf("4", "5", "6")
// private val sequenceB = listOf("7", "55", "59")
// private val sequenceAll = listOf("4", "5", "6", "7", "55", "59")
//
// private fun update()
// {
// if(isMotionLoadFinished)
// {
// if(elapsedTime == 0){
// //这次播放是一直循环播放,我们不需要判断动画是否播放完成, 回调函数直接传入null
// PlayMotionList(sequenceA, true, null)
// }
//
// elapsedTime += 10 // 动画加载完成后开始计时,每次调用增加 10ms
//
// if(elapsedTime == 10*1000){
// //暂停动画
// PlayMotionList(sequenceB, true, null)
// }
// }
//
// }
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
if (keyCode == KEYCODE_BACK) { if (keyCode == KEYCODE_BACK) {
Log.d("MakeupActivity", "Back key pressed") Log.d("MakeupActivity", "Back key pressed")
@@ -129,10 +137,7 @@ class MakeupActivity : FaceActivity()
override fun onTouchEvent(event: MotionEvent): Boolean { override fun onTouchEvent(event: MotionEvent): Boolean {
when (event.action) { when (event.action) {
MotionEvent.ACTION_UP -> { MotionEvent.ACTION_UP -> {
if (isMotionLoadFinished) {
currentIndex = (currentIndex + 1) % sequenceString.size
PlayMotionList(listOf(sequenceString[currentIndex]), true, null)
}
} }
} }
return true return true
+1 -1
View File
@@ -1,3 +1,3 @@
<resources> <resources>
<string name="app_name">f20260127</string> <string name="app_name">f20260113</string>
</resources> </resources>
+44 -349
View File
@@ -60,10 +60,7 @@ void Application::initWindow()
glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API);
glfwWindowHint(GLFW_RESIZABLE, GLFW_FALSE); glfwWindowHint(GLFW_RESIZABLE, GLFW_FALSE);
// Windows 端仅用于本机调试。改成接近手机的竖屏比例(540x960), window = glfwCreateWindow(480, 480, "Vulkan", nullptr, nullptr);
// 这样能直观验证「画布居中正方形 + 上下黑边 letterbox」的效果。
// Android 端走另一条分支,窗口尺寸由系统 surface 决定。
window = glfwCreateWindow(540, 960, "Vulkan", nullptr, nullptr);
#else #else
#endif #endif
@@ -113,19 +110,11 @@ void Application::initVulkan()
createCommandPool(); // 创建命令池 createCommandPool(); // 创建命令池
createCommandBuffer(); // 创建命令缓冲区 createCommandBuffer(); // 创建命令缓冲区
createSyncObjects(); // 创建同步对象 createSyncObjects(); // 创建同步对象
// 复用式 single-time-command 资源:从 commandPool_ex 预分配
// kTransferSlotCount 个 cmdbuf + 同数 signaled fence
// 之后所有 copyBuffer / updateTexture 走 runTransferCommand
// 不再每次 vkAllocate/vkFree。
createTransferResources();
_lastDrawFrameTime = getCurrentTimeMillis(); _lastDrawFrameTime = getCurrentTimeMillis();
_applicationInited = true; _applicationInited = true;
// The first branch above already built all of the window-dependent
// objects. Mark _sceondInited so we don't fall into the recovery
// branch below and leak a second copy of surface/swapChain/etc.
_sceondInited = true;
} }
else if (!_sceondInited) {
if (!_sceondInited) {
createSurface(); createSurface();
createSwapChain(); createSwapChain();
createImageViews(); createImageViews();
@@ -137,68 +126,16 @@ void Application::initVulkan()
} }
} }
// ⚠️ Android 路径不要调这个!这个函数销毁的是「第二阶段 init」期间创建的所有
// 资源(含 renderPass / graphicsPipeline / pipelineLayout),但**没有**重置
// _applicationInited / _sceondInited,也没有把销毁的句柄置 VK_NULL_HANDLE。
//
// 之前 main.cpp android_main 退出时调过它,结果用户在主界面切换镜头再进入
// MakeupActivity 时 onWindowInit 看到 _applicationInited=1 走 reinit 复用路径
// 引用悬空 renderPassvalidation layer 检测出 use-after-free 触发 SEGV。
//
// Android 上的窗口生命周期是 cleanupForWindowLost() + reinitForNewWindow()
// renderPass / pipeline 跨 NativeActivity 实例由 g_Application 单例持有。
// 这个函数目前只保留给 vulkan/main.cpp 那个独立 desktop 入口的进程退出路径。
void Application::cleanupSecondInit() void Application::cleanupSecondInit()
{ {
// ★ 幂等:销毁后立刻把句柄置 VK_NULL_HANDLE / 清空 vector。 vkDestroySwapchainKHR(device, swapChain, nullptr);
// vkDestroySurfaceKHR(instance, surface, nullptr);
// 这个函数过去在 Android 路径 (android_main 退出) 上被错误地调用过; vkDestroyPipeline(device, graphicsPipeline, nullptr);
// 由于销毁后没有清零句柄,紧接着重新进入的 reinitForNewWindow() 在 vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
// extent/format 不变时会跳过重建,让 framebuffer / drawFrame 在 vkDestroyRenderPass(device, renderPass, nullptr);
// 已经销毁的 renderPass 上继续工作,最终在
// vkCmdBeginRenderPass 内部 (libGLES_mali) 触发 SIGSEGV。
//
// 现在 android_main 退出路径已经不再调本函数;本函数只剩 Windows
// 路径 (mainLoop → cleanupSecondInit → initVulkan 的"二次 session"
// 复用) 在用。把它写成幂等的可以兜住将来任何对它的误用。
FACE_DBG_LOG("cleanupSecondInit: begin (swapChain=%s surface=%s renderPass=%s pipelineLayout=%s graphicsPipeline=%s imageViews=%zu framebuffers=%zu)",
swapChain == VK_NULL_HANDLE ? "null" : "live",
surface == VK_NULL_HANDLE ? "null" : "live",
renderPass == VK_NULL_HANDLE ? "null" : "live",
pipelineLayout == VK_NULL_HANDLE ? "null" : "live",
graphicsPipeline == VK_NULL_HANDLE ? "null" : "live",
swapChainImageViews.size(), swapChainFramebuffers.size());
if (device != VK_NULL_HANDLE) {
vkDeviceWaitIdle(device);
}
if (swapChain != VK_NULL_HANDLE) {
vkDestroySwapchainKHR(device, swapChain, nullptr);
swapChain = VK_NULL_HANDLE;
}
if (surface != VK_NULL_HANDLE) {
vkDestroySurfaceKHR(instance, surface, nullptr);
surface = VK_NULL_HANDLE;
}
if (graphicsPipeline != VK_NULL_HANDLE) {
vkDestroyPipeline(device, graphicsPipeline, nullptr);
graphicsPipeline = VK_NULL_HANDLE;
}
if (pipelineLayout != VK_NULL_HANDLE) {
vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
pipelineLayout = VK_NULL_HANDLE;
}
if (renderPass != VK_NULL_HANDLE) {
vkDestroyRenderPass(device, renderPass, nullptr);
renderPass = VK_NULL_HANDLE;
}
for (auto imageView : swapChainImageViews) { for (auto imageView : swapChainImageViews) {
if (imageView != VK_NULL_HANDLE) { vkDestroyImageView(device, imageView, nullptr);
vkDestroyImageView(device, imageView, nullptr);
}
} }
swapChainImageViews.clear();
for (auto framebuffer : swapChainFramebuffers) for (auto framebuffer : swapChainFramebuffers)
{ {
if (framebuffer != VK_NULL_HANDLE) if (framebuffer != VK_NULL_HANDLE)
@@ -207,8 +144,7 @@ void Application::cleanupSecondInit()
} }
} }
swapChainFramebuffers.clear(); swapChainFramebuffers.clear();
swapChainImages.clear();
_sceondInited = false; _sceondInited = false;
} }
@@ -219,14 +155,8 @@ void Application::cleanupForWindowLost()
(int)_applicationInited, (int)_sceondInited); (int)_applicationInited, (int)_sceondInited);
return; return;
} }
FACE_DBG_LOG("cleanupForWindowLost: begin (imageCount=%zu MAX_FRAMES_IN_FLIGHT=%d extent=%ux%u format=%d)", FACE_DBG_LOG("cleanupForWindowLost: begin (imageCount=%zu MAX_FRAMES_IN_FLIGHT=%d)",
swapChainImages.size(), MAX_FRAMES_IN_FLIGHT, swapChainImages.size(), MAX_FRAMES_IN_FLIGHT);
swapChainExtent.width, swapChainExtent.height, (int)swapChainImageFormat);
// Snapshot before we destroy the swapchain so reinitForNewWindow() can
// decide whether the new swapchain needs renderPass / pipelines rebuilt.
_prevSwapChainExtent = swapChainExtent;
_prevSwapChainImageFormat = swapChainImageFormat;
// Block until GPU is no longer using any of the resources we are about // Block until GPU is no longer using any of the resources we are about
// to destroy. Anything weaker than this risks hitting VK_ERROR_DEVICE_LOST // to destroy. Anything weaker than this risks hitting VK_ERROR_DEVICE_LOST
@@ -292,73 +222,29 @@ void Application::cleanupForWindowLost()
FACE_DBG_LOG("cleanupForWindowLost: done"); FACE_DBG_LOG("cleanupForWindowLost: done");
} }
bool Application::reinitForNewWindow() void Application::reinitForNewWindow()
{ {
if (!_applicationInited) { if (!_applicationInited) {
FACE_DBG_LOG("reinitForNewWindow: skip, _applicationInited=0 (must go through initVulkan first)"); FACE_DBG_LOG("reinitForNewWindow: skip, _applicationInited=0 (must go through initVulkan first)");
return false; return;
} }
if (_sceondInited) { if (_sceondInited) {
FACE_DBG_LOG("reinitForNewWindow: skip, already _sceondInited=1"); FACE_DBG_LOG("reinitForNewWindow: skip, already _sceondInited=1");
return false; return;
} }
FACE_DBG_LOG("reinitForNewWindow: begin (prev extent=%ux%u format=%d)", FACE_DBG_LOG("reinitForNewWindow: begin");
_prevSwapChainExtent.width, _prevSwapChainExtent.height,
(int)_prevSwapChainImageFormat);
createSurface(); createSurface();
createSwapChain(); createSwapChain();
const bool extentChanged = (swapChainExtent.width != _prevSwapChainExtent.width) ||
(swapChainExtent.height != _prevSwapChainExtent.height);
const bool formatChanged = (swapChainImageFormat != _prevSwapChainImageFormat);
const bool swapchainIncompatible = extentChanged || formatChanged;
if (formatChanged) {
// renderPass bakes in swapChainImageFormat, so it must be rebuilt
// when the surface chose a different format. The old Application
// graphicsPipeline is tied to the old renderPass, so destroy it
// first to make the handle invalidation explicit.
FACE_DBG_LOG("reinitForNewWindow: format changed (%d -> %d), rebuilding renderPass",
(int)_prevSwapChainImageFormat, (int)swapChainImageFormat);
if (graphicsPipeline != VK_NULL_HANDLE) {
vkDestroyPipeline(device, graphicsPipeline, nullptr);
graphicsPipeline = VK_NULL_HANDLE;
}
if (pipelineLayout != VK_NULL_HANDLE) {
vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
pipelineLayout = VK_NULL_HANDLE;
}
if (renderPass != VK_NULL_HANDLE) {
vkDestroyRenderPass(device, renderPass, nullptr);
renderPass = VK_NULL_HANDLE;
}
createRenderPass();
createPipelineLayout();
createGraphicsPipeline();
} else if (extentChanged) {
// renderPass is still fine, but the Application pipeline has a static
// viewport baked to the old extent and must be rebuilt.
FACE_DBG_LOG("reinitForNewWindow: extent changed (%ux%u -> %ux%u), rebuilding graphicsPipeline",
_prevSwapChainExtent.width, _prevSwapChainExtent.height,
swapChainExtent.width, swapChainExtent.height);
if (graphicsPipeline != VK_NULL_HANDLE) {
vkDestroyPipeline(device, graphicsPipeline, nullptr);
graphicsPipeline = VK_NULL_HANDLE;
}
createGraphicsPipeline();
}
createImageViews(); createImageViews();
createFramebuffers(); createFramebuffers();
createCommandBuffer(); createCommandBuffer();
createSyncObjects(); createSyncObjects();
_sceondInited = true; _sceondInited = true;
FACE_DBG_LOG("reinitForNewWindow: done (imageCount=%zu extent=%ux%u format=%d MAX_FRAMES_IN_FLIGHT=%d swapchainIncompatible=%d)", FACE_DBG_LOG("reinitForNewWindow: done (imageCount=%zu extent=%ux%u MAX_FRAMES_IN_FLIGHT=%d)",
swapChainImages.size(), swapChainExtent.width, swapChainExtent.height, swapChainImages.size(), swapChainExtent.width, swapChainExtent.height,
(int)swapChainImageFormat, MAX_FRAMES_IN_FLIGHT, (int)swapchainIncompatible); MAX_FRAMES_IN_FLIGHT);
return swapchainIncompatible;
} }
@@ -851,14 +737,7 @@ void Application::drawFrame(long long frameTime)
submitInfo.signalSemaphoreCount = 1; submitInfo.signalSemaphoreCount = 1;
submitInfo.pSignalSemaphores = signalSemaphores; submitInfo.pSignalSemaphores = signalSemaphores;
// 串行化 graphicsQueue/presentQueue 的 submit 与 present VkResult submitRes = vkQueueSubmit(graphicsQueue, 1, &submitInfo, inFlightFences[currentFrame]);
// 确保与 copyBuffer / updateTexture 等其它线程的队列提交互斥,
// 避免驱动内部 pthread_mutex 在长时间并发下被破坏。
VkResult submitRes;
{
std::lock_guard<std::mutex> poolLock(poolQueueMtx);
submitRes = vkQueueSubmit(graphicsQueue, 1, &submitInfo, inFlightFences[currentFrame]);
}
if (submitRes != VK_SUCCESS) { if (submitRes != VK_SUCCESS) {
#ifndef _WIN32 #ifndef _WIN32
DebugLog::log_throttled("drawFrame.submit", DebugLog::log_throttled("drawFrame.submit",
@@ -878,10 +757,7 @@ void Application::drawFrame(long long frameTime)
presentInfo.swapchainCount = 1; presentInfo.swapchainCount = 1;
presentInfo.pSwapchains = swapChains; presentInfo.pSwapchains = swapChains;
presentInfo.pImageIndices = &imageIndex; presentInfo.pImageIndices = &imageIndex;
{ result = vkQueuePresentKHR(presentQueue, &presentInfo);
std::lock_guard<std::mutex> poolLock(poolQueueMtx);
result = vkQueuePresentKHR(presentQueue, &presentInfo);
}
if (result != VK_SUCCESS) { if (result != VK_SUCCESS) {
#ifndef _WIN32 #ifndef _WIN32
@@ -1192,184 +1068,6 @@ void Application::endSingleTimeCommands(VkDevice device, VkCommandPool commandPo
vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer); vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer);
} }
void Application::createTransferResources()
{
if (m_xferInited) {
#ifndef _WIN32
DebugLog::log("createTransferResources: already inited, skip");
#endif
return;
}
if (commandPool_ex == VK_NULL_HANDLE) {
#ifndef _WIN32
DebugLog::log("createTransferResources: commandPool_ex is null, abort");
#endif
throw std::runtime_error("createTransferResources: commandPool_ex not created yet");
}
VkCommandBufferAllocateInfo allocInfo{};
allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocInfo.commandPool = commandPool_ex;
allocInfo.commandBufferCount = kTransferSlotCount;
if (vkAllocateCommandBuffers(device, &allocInfo, m_xferCmd) != VK_SUCCESS) {
throw std::runtime_error("createTransferResources: vkAllocateCommandBuffers failed");
}
// fence 创建为 SIGNALED:第一次 runTransferCommand 的 vkWaitForFences
// 会立刻返回,避免冷启动卡顿。
VkFenceCreateInfo fenceInfo{};
fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT;
for (uint32_t i = 0; i < kTransferSlotCount; ++i) {
if (vkCreateFence(device, &fenceInfo, nullptr, &m_xferFence[i]) != VK_SUCCESS) {
for (uint32_t j = 0; j < i; ++j) {
vkDestroyFence(device, m_xferFence[j], nullptr);
m_xferFence[j] = VK_NULL_HANDLE;
}
vkFreeCommandBuffers(device, commandPool_ex, kTransferSlotCount, m_xferCmd);
for (uint32_t j = 0; j < kTransferSlotCount; ++j) m_xferCmd[j] = VK_NULL_HANDLE;
throw std::runtime_error("createTransferResources: vkCreateFence failed");
}
}
m_xferIdx = 0;
m_xferInited = true;
#ifndef _WIN32
DebugLog::log("createTransferResources: done, slots=%u pool=commandPool_ex",
(unsigned)kTransferSlotCount);
#endif
}
void Application::destroyTransferResources()
{
if (!m_xferInited) {
return;
}
// 调用方必须保证 GPU 已 idle 且没有线程正在 runTransferCommand。
// 这里再加一道 m_xferMtx,串行化潜在的最后一次 transfer。
std::lock_guard<std::mutex> xferLock(m_xferMtx);
for (uint32_t i = 0; i < kTransferSlotCount; ++i) {
if (m_xferFence[i] != VK_NULL_HANDLE) {
vkDestroyFence(device, m_xferFence[i], nullptr);
m_xferFence[i] = VK_NULL_HANDLE;
}
}
if (m_xferCmd[0] != VK_NULL_HANDLE && commandPool_ex != VK_NULL_HANDLE) {
vkFreeCommandBuffers(device, commandPool_ex, kTransferSlotCount, m_xferCmd);
}
for (uint32_t i = 0; i < kTransferSlotCount; ++i) {
m_xferCmd[i] = VK_NULL_HANDLE;
}
m_xferIdx = 0;
m_xferInited = false;
#ifndef _WIN32
DebugLog::log("destroyTransferResources: done");
#endif
}
void Application::runTransferCommand(const std::function<void(VkCommandBuffer)>& record)
{
if (!m_xferInited) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.notInited",
"runTransferCommand: not inited, skip");
#endif
return;
}
// 串行化所有 transfer 调用:
// - 保证 m_xferIdx 推进 / m_xferCmd[idx] 录制 / m_xferFence[idx] 等待
// 形成一组原子动作;
// - 同时也保证 commandPool_ex 的「外部同步」语义(同一时刻只允许一个
// 线程对它做 record/reset)。
std::lock_guard<std::mutex> xferLock(m_xferMtx);
const uint32_t idx = m_xferIdx;
VkFence fence = m_xferFence[idx];
VkCommandBuffer cmd = m_xferCmd[idx];
// 等上一次该 slot 的提交真正完成。fence 是独立同步对象,不需要持
// poolQueueMtx 就可以等待,drawFrame 的 submit 不会被阻塞。
VkResult wr = vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX);
if (wr != VK_SUCCESS) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.wait",
"runTransferCommand: vkWaitForFences slot=%u -> %d",
idx, (int)wr);
#endif
}
vkResetFences(device, 1, &fence);
vkResetCommandBuffer(cmd, 0);
VkCommandBufferBeginInfo beginInfo{};
beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if (vkBeginCommandBuffer(cmd, &beginInfo) != VK_SUCCESS) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.begin",
"runTransferCommand: vkBeginCommandBuffer slot=%u failed", idx);
#endif
return;
}
// 调用方在这里 record 命令:vkCmdCopyBuffer / vkCmdCopyBufferToImage /
// transitionImageLayout 等。这些是纯 record 操作,在 m_xferMtx 持有
// 且 cmd 独占的前提下不需要再额外加锁。
record(cmd);
if (vkEndCommandBuffer(cmd) != VK_SUCCESS) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.end",
"runTransferCommand: vkEndCommandBuffer slot=%u failed", idx);
#endif
return;
}
VkSubmitInfo submitInfo{};
submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &cmd;
// vkQueueSubmit 必须和 drawFrame 的 vkQueueSubmit / vkQueuePresentKHR
// 互斥(队列要求外部同步)。其它步骤只占 m_xferMtx 即可。
{
std::lock_guard<std::mutex> poolLock(poolQueueMtx);
VkResult sr = vkQueueSubmit(graphicsQueue, 1, &submitInfo, fence);
if (sr != VK_SUCCESS) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.submit",
"runTransferCommand: vkQueueSubmit slot=%u -> %d",
idx, (int)sr);
#endif
}
}
// ★ 关键:调用方代码(FaceApp::uploadVertexData / Application::updateTexture
// 在 runTransferCommand 之后会立刻 vkMapMemory + memcpy 覆写共享的
// staging buffer,去做下一次拷贝。所以本接口必须像旧的 vkQueueWaitIdle
// 一样保证 GPU **已读完** staging 才能返回,否则覆写会 race 上 GPU
// 还在执行的 vkCmdCopyBuffer / vkCmdCopyBufferToImage,导致顶点 / 纹理
// 数据被错位拼接(外观就是模型畸形 / 纹理花屏)。
//
// 这里只等自己这一次的 fence,不像旧实现 vkQueueWaitIdle 那样等整个
// graphicsQueue(包括 drawFrame 的提交),所以不会拖慢渲染主路径。
//
// 不持 poolQueueMtxfence 是独立同步对象,等它不需要外部互斥。
VkResult er = vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX);
if (er != VK_SUCCESS) {
#ifndef _WIN32
DebugLog::log_throttled("runTransferCommand.endWait",
"runTransferCommand: end vkWaitForFences slot=%u -> %d",
idx, (int)er);
#endif
}
m_xferIdx = (idx + 1) % kTransferSlotCount;
}
void Application::updateTexture(VkDevice device, VkPhysicalDevice physicalDevice, void Application::updateTexture(VkDevice device, VkPhysicalDevice physicalDevice,
VkCommandPool commandPool, VkQueue queue, VkCommandPool commandPool, VkQueue queue,
@@ -1402,36 +1100,33 @@ void Application::updateTexture(VkDevice device, VkPhysicalDevice physicalDevice
vkUnmapMemory(device, texture.stagingBufferMemory); vkUnmapMemory(device, texture.stagingBufferMemory);
// 走 runTransferCommand:复用 commandPool_ex 上预分配的 cmdbuf + fence VkCommandBuffer commandBuffer = beginSingleTimeCommands(device, commandPool);
// 不再每帧 vkAllocate/vkFree,也不再 vkQueueWaitIdle。
// 注意:传入的 commandPool / queue 参数保留只是为了兼容旧接口,
// 实际命令池一律换成 commandPool_ex(与渲染主管线物理隔离)。
(void)commandPool;
(void)queue;
runTransferCommand([&](VkCommandBuffer commandBuffer) {
transitionImageLayout(commandBuffer, texture.image,
VK_IMAGE_LAYOUT_UNDEFINED,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
VkBufferImageCopy region = {}; transitionImageLayout(commandBuffer, texture.image,
region.bufferOffset = 0; VK_IMAGE_LAYOUT_UNDEFINED,
region.bufferRowLength = 0; VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
region.bufferImageHeight = 0;
region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
region.imageSubresource.mipLevel = 0;
region.imageSubresource.baseArrayLayer = 0;
region.imageSubresource.layerCount = 1;
region.imageOffset = { 0, 0, 0 };
region.imageExtent = { static_cast<uint32_t>(width),
static_cast<uint32_t>(height), 1 };
vkCmdCopyBufferToImage(commandBuffer, texture.stagingBuffer, texture.image, VkBufferImageCopy region = {};
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region); region.bufferOffset = 0;
region.bufferRowLength = 0;
region.bufferImageHeight = 0;
region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
region.imageSubresource.mipLevel = 0;
region.imageSubresource.baseArrayLayer = 0;
region.imageSubresource.layerCount = 1;
region.imageOffset = { 0, 0, 0 };
region.imageExtent = { static_cast<uint32_t>(width),
static_cast<uint32_t>(height), 1 };
vkCmdCopyBufferToImage(commandBuffer, texture.stagingBuffer, texture.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
transitionImageLayout(commandBuffer, texture.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
endSingleTimeCommands(device, commandPool, queue, commandBuffer);
transitionImageLayout(commandBuffer, texture.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
});
texture.image_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; texture.image_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
} }
+1 -89
View File
@@ -5,7 +5,6 @@
#include "AppBase.h" #include "AppBase.h"
#include <thread> #include <thread>
#include <mutex> #include <mutex>
#include <functional>
struct Texture struct Texture
{ {
@@ -49,68 +48,8 @@ public:
virtual void render(VkCommandBuffer commandBuffer, long long frameTime); virtual void render(VkCommandBuffer commandBuffer, long long frameTime);
virtual bool isInited() { return _applicationInited; } virtual bool isInited() { return _applicationInited; }
std::mutex createTextureMtx; std::mutex createTextureMtx;
// 全局 GPU 提交互斥锁:任何对 graphicsQueue / presentQueue 的提交
// vkQueueSubmit / vkQueuePresentKHR / vkQueueWaitIdle)以及共享
// commandPool 的 vkAllocateCommandBuffers / vkFreeCommandBuffers
// 都必须在持有此锁的期间执行,否则驱动内部维护命令池与队列的
// pthread_mutex 在长时间多线程并发下会被破坏(FORTIFY: pthread_mutex_lock
// called on a destroyed mutex)。
// 需要覆盖的 3 条并发线路:
// 1) 渲染线程 drawFrame
// 2) processImageNative → updateTexture (single-time commands)
// 3) passDataToNative → update_face_vertex_buffer → copyBuffer
std::mutex poolQueueMtx;
void processWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& texture, bool srgb, VkCommandPool pool, std::string tex_path); void processWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& texture, bool srgb, VkCommandPool pool, std::string tex_path);
// ---------------------------------------------------------------------
// 复用式 single-time-command 接口(替代每帧 allocate+submit+waitIdle+free
//
// ★ 同步语义(重要):
// 本接口 **同步**,返回时 GPU 已经读完 record 里访问的源数据。
// 行为上等价于旧的 vkQueueSubmit + vkQueueWaitIdle,但只等自己这次
// 提交的 fence,不阻塞渲染线程在 graphicsQueue 上的其它提交。
//
// 为什么必须同步:调用方 (FaceApp::uploadVertexData /
// Application::updateTexture) 紧接着会复用同一个 staging buffer
//
// vkMapMemory(staging); memcpy(staging, A); vkUnmapMemory;
// runTransferCommand([&](cmd){ vkCmdCopyBuffer(cmd, staging, dstA); });
// vkMapMemory(staging); memcpy(staging, B); ← 必须等 dstA 拷完!
// vkUnmapMemory;
// runTransferCommand([&](cmd){ vkCmdCopyBuffer(cmd, staging, dstB); });
//
// 如果 runTransferCommand 异步返回,第二次 memcpy 会在 GPU 还没读完
// staging 里的 A 时就把它覆盖成 B —— 顶点 / 纹理立刻畸形。
//
// 行为:
// - 共 kTransferSlotCount 个 VkCommandBuffer + 同数 VkFence
// 从 commandPool_ex 一次性分配,在 cleanup 时一次性释放。
// - 每次 runTransferCommand:
// 1) m_xferMtx.lock() (串行所有 transfer 提交)
// 2) 当前 slot 上 vkWaitForFences (等上一次该 slot 的提交完成)
// 3) vkResetFences + vkResetCommandBuffer + vkBegin
// 4) 调用 record(cmd) 录制命令 (vkCmdCopy* 等)
// 5) vkEndCommandBuffer
// 6) poolQueueMtx 内 vkQueueSubmit(graphicsQueue, fence)
// 7) vkWaitForFences(fence) (★ 等本次 GPU 完成才返回)
// 8) m_xferIdx 推进到下一个 slot
//
// 为什么这样设计:
// - 完全消除每帧 vkAllocateCommandBuffers / vkFreeCommandBuffers
// 根除驱动 per-pool mutex 在长时间高频压力下被破坏导致的
// FORTIFY: pthread_mutex_lock called on a destroyed mutex 崩溃。
// - 用 fence 替代 vkQueueWaitIdle,等待粒度只到自己这一次提交,
// 不会 stall 整条 graphicsQueue(包括渲染主路径的提交)。
// - 固定使用 commandPool_ex(与渲染主用的 commandPool 物理隔离),
// 即使驱动还有内部 contention,也不会波及渲染主管线。
//
// 调用约束:
// - 调用方 **绝不能** 提前持有 poolQueueMtx;本接口内部按需短暂获取。
// - record 函数体内只允许 vkCmd*(命令录制)这类操作,不要在里面调
// queue / pool 级别的 APIsubmit / present / pool reset 等)。
static constexpr uint32_t kTransferSlotCount = 3;
void runTransferCommand(const std::function<void(VkCommandBuffer)>& record);
protected: protected:
VkPhysicalDevice physicalDevice = VK_NULL_HANDLE; // 物理设备 VkPhysicalDevice physicalDevice = VK_NULL_HANDLE; // 物理设备
@@ -151,34 +90,7 @@ public:
// Rebuild the window-dependent Vulkan objects torn down above. // Rebuild the window-dependent Vulkan objects torn down above.
// Safe to call only after cleanupForWindowLost(). // Safe to call only after cleanupForWindowLost().
// Returns true when the new swapchain's extent or format differs from the void reinitForNewWindow();
// one in use before cleanupForWindowLost(). When true, any pipeline baked
// with a static viewport/scissor from swapChainExtent -- including the
// FaceApp pipelines -- must be destroyed and recreated against the new
// renderPass / extent. Application's own renderPass + graphicsPipeline are
// already handled internally.
bool reinitForNewWindow();
// Extent / format captured by the most recent cleanupForWindowLost().
// Used by reinitForNewWindow() to decide whether renderPass / pipelines
// must be rebuilt against the new swapchain.
VkExtent2D _prevSwapChainExtent = {0, 0};
VkFormat _prevSwapChainImageFormat = VK_FORMAT_UNDEFINED;
protected:
// Transfer command resources(详见 runTransferCommand 上方注释)。
// 命令缓冲来自 commandPool_ex,与渲染主用的 commandPool 物理隔离。
VkCommandBuffer m_xferCmd[kTransferSlotCount] = { VK_NULL_HANDLE, VK_NULL_HANDLE, VK_NULL_HANDLE };
VkFence m_xferFence[kTransferSlotCount] = { VK_NULL_HANDLE, VK_NULL_HANDLE, VK_NULL_HANDLE };
uint32_t m_xferIdx = 0;
bool m_xferInited = false;
std::mutex m_xferMtx;
// 创建/销毁 transfer 资源。createTransferResources 必须在 createCommandPool
// 之后调用;destroyTransferResources 必须在销毁 commandPool_ex 之前调用,
// 且 GPU 已 idlevkDeviceWaitIdle 或确认所有 fence 已 signaled)。
void createTransferResources();
void destroyTransferResources();
protected: protected:
void loadTexture(std::string path, Texture& tex, bool srgb, VkCommandPool pool); void loadTexture(std::string path, Texture& tex, bool srgb, VkCommandPool pool);
+78 -301
View File
@@ -1,4 +1,4 @@
#include "FaceApp.h" #include "FaceApp.h"
#include "hardcode_data.h" #include "hardcode_data.h"
#include "lodepng.h" #include "lodepng.h"
#include <sstream> #include <sstream>
@@ -9,10 +9,8 @@
#ifndef _WIN32 #ifndef _WIN32
#include "../app/src/main/cpp/DebugLog.h" #include "../app/src/main/cpp/DebugLog.h"
#define FACE_DBG_LOG(...) DebugLog::log(__VA_ARGS__) #define FACE_DBG_LOG(...) DebugLog::log(__VA_ARGS__)
#define FACE_DBG_LOG_THROTTLED(key, ...) DebugLog::log_throttled(key, __VA_ARGS__)
#else #else
#define FACE_DBG_LOG(...) ((void)0) #define FACE_DBG_LOG(...) ((void)0)
#define FACE_DBG_LOG_THROTTLED(key, ...) ((void)0)
#endif #endif
@@ -47,13 +45,10 @@ void FaceApp::Stop()
void ReceiveFacePoint(float* pos, int pointCount, int width, int height) void ReceiveFacePoint(float* pos, int pointCount, int width, int height)
{ {
FaceApp* self = FaceApp::Get(); FaceApp* self = FaceApp::Get();
if (self == nullptr) if (self != nullptr)
{ {
FACE_DBG_LOG_THROTTLED("ReceiveFacePoint.noFaceApp", FaceApp::Get()->update_face_vertex_buffer(pos, pointCount);
"ReceiveFacePoint dropped: FaceApp::Get() == nullptr");
return;
} }
self->update_face_vertex_buffer(pos, pointCount);
} }
// 定义帧数据结构 // 定义帧数据结构
@@ -63,12 +58,10 @@ struct FrameData {
int height; int height;
int rowStride; int rowStride;
size_t dataSize; size_t dataSize;
int rotation; // CameraX 给的 rotationDegrees,用于 shader 端 UV 旋转
bool mirrorX; // 前置摄像头时为 trueshader 在 gl_Position.x 翻转一次
FrameData(uint8_t* d, int w, int h, int rs, size_t ds, int rot, bool mx) FrameData(uint8_t* d, int w, int h, int rs, size_t ds)
: data(nullptr), width(w), height(h), rowStride(rs), dataSize(ds), : data(nullptr), width(w), height(h), rowStride(rs), dataSize(ds) {
rotation(rot), mirrorX(mx) { // 深拷贝数据
if (d && ds > 0) { if (d && ds > 0) {
data = new uint8_t[dataSize]; data = new uint8_t[dataSize];
memcpy(data, d, dataSize); memcpy(data, d, dataSize);
@@ -82,16 +75,18 @@ struct FrameData {
} }
} }
// 禁止拷贝构造和赋值(使用移动语义)
FrameData(const FrameData&) = delete; FrameData(const FrameData&) = delete;
FrameData& operator=(const FrameData&) = delete; FrameData& operator=(const FrameData&) = delete;
// 移动构造
FrameData(FrameData&& other) noexcept FrameData(FrameData&& other) noexcept
: data(other.data), width(other.width), height(other.height), : data(other.data), width(other.width), height(other.height),
rowStride(other.rowStride), dataSize(other.dataSize), rowStride(other.rowStride), dataSize(other.dataSize) {
rotation(other.rotation), mirrorX(other.mirrorX) {
other.data = nullptr; other.data = nullptr;
} }
// 移动赋值
FrameData& operator=(FrameData&& other) noexcept { FrameData& operator=(FrameData&& other) noexcept {
if (this != &other) { if (this != &other) {
if (data) delete[] data; if (data) delete[] data;
@@ -100,8 +95,6 @@ struct FrameData {
height = other.height; height = other.height;
rowStride = other.rowStride; rowStride = other.rowStride;
dataSize = other.dataSize; dataSize = other.dataSize;
rotation = other.rotation;
mirrorX = other.mirrorX;
other.data = nullptr; other.data = nullptr;
} }
return *this; return *this;
@@ -113,7 +106,7 @@ std::queue<FrameData> frameQueue;
std::mutex queueMutex; std::mutex queueMutex;
const int MAX_QUEUE_SIZE = 4; const int MAX_QUEUE_SIZE = 4;
void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX) void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize)
{ {
if (!FaceApp::Get()->isInited()) { if (!FaceApp::Get()->isInited()) {
return; return;
@@ -121,25 +114,24 @@ void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowS
std::lock_guard<std::mutex> lock(queueMutex); std::lock_guard<std::mutex> lock(queueMutex);
frameQueue.emplace(data, width, height, rowStride, dataSize, rotation, mirrorX); // 添加新帧数据到队列
frameQueue.emplace(data, width, height, rowStride, dataSize);
// 如果队列大小达到阈值,开始处理最旧的一帧 // 如果队列大小达到阈值,开始处理最旧的一帧
if (frameQueue.size() >= MAX_QUEUE_SIZE) { if (frameQueue.size() >= MAX_QUEUE_SIZE) {
Texture& tex_bg = FaceApp::Get()->tex_bg;
FrameData& oldestFrame = frameQueue.front(); FrameData& oldestFrame = frameQueue.front();
// 走 FaceApp::processCameraFrame:相比 base::processWithVulkan,它多做两件 FaceApp::Get()->processWithVulkan(
// 事 —— ① 检测帧尺寸变化时 destroy/recreate tex_bg(不同手机或 CameraX
// 重建 session 后第一帧尺寸可能与之前不同);② 缓存 raw_aspect / rotation
// / mirrorX 到 m_cameraAspect / m_cameraRotation / m_mirrorX,由 render()
// 推到 push constant。
FaceApp::Get()->processCameraFrame(
oldestFrame.data, oldestFrame.data,
oldestFrame.width, oldestFrame.width,
oldestFrame.height, oldestFrame.height,
oldestFrame.rowStride, oldestFrame.rowStride,
oldestFrame.dataSize, oldestFrame.dataSize,
oldestFrame.rotation, tex_bg,
oldestFrame.mirrorX false,
FaceApp::Get()->commandPool,
"data_from_camera"
); );
frameQueue.pop(); frameQueue.pop();
@@ -304,21 +296,15 @@ void FaceApp::create_face_pipelines()
VkPipelineRasterizationStateCreateInfo rasterization_state{}; VkPipelineRasterizationStateCreateInfo rasterization_state{};
rasterization_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; rasterization_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
rasterization_state.polygonMode = VK_POLYGON_MODE_FILL; rasterization_state.polygonMode = VK_POLYGON_MODE_FILL;
// ★ 关键:face 必须 NONE。原因:
// 1. face mesh 是 2D 平面网格(texture.vert 让所有顶点 z=0.5),不存在
// 自遮挡,没必要 cull;
// 2. 前置摄像头镜子效果是在 vertex shader 里通过 NDC.x 翻转实现的
// `position.x *= (1.0 - 2.0 * pc.mirror_x)`)。X 翻转会把所有三角
// 形的卷绕方向从 CCW 变 CW。如果这里 cullMode=BACK_BIT + frontFace=
// CCW,前置时整张脸会被 cull,face 完全不可见——这正是历史踩坑点。
// 所以请勿改回 BACK_BIT。如果未来要再加 cull,必须配两套 pipeline(前置
// 用 CW、后置用 CCW),或在前置时改成 frontFace=CW,否则一定会复现这个 bug。
rasterization_state.cullMode = VK_CULL_MODE_NONE; rasterization_state.cullMode = VK_CULL_MODE_NONE;
rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
rasterization_state.flags = 0; rasterization_state.flags = 0;
rasterization_state.depthClampEnable = VK_FALSE; rasterization_state.depthClampEnable = VK_FALSE;
rasterization_state.lineWidth = 1.0f; rasterization_state.lineWidth = 1.0f;
rasterization_state.cullMode = VK_CULL_MODE_BACK_BIT;
rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
VkPipelineColorBlendAttachmentState colorBlendAttachment{}; VkPipelineColorBlendAttachmentState colorBlendAttachment{};
@@ -345,16 +331,13 @@ void FaceApp::create_face_pipelines()
colorBlending.attachmentCount = 1; colorBlending.attachmentCount = 1;
colorBlending.pAttachments = &colorBlendAttachment; colorBlending.pAttachments = &colorBlendAttachment;
// face 是 2D 平面网格(texture.vert 让顶点 z=0.5 一致),完全不需要 depth // Note: Using reversed depth-buffer for increased precision, so Greater depth values are kept
// test/write。bg 管线本来也是关 depth 的;保持两边一致,让"先 bg 再 face"
// 的覆盖顺序由 draw 顺序而非 depth 决定,避免 reversed-depthGREATER+
// renderpass 没显式 clear depth attachment 时引发的"face fragment 全部
// 被 depth test fail"的隐性 bug。
VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {}; VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {};
depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
depth_stencil_state.depthTestEnable = VK_FALSE; depth_stencil_state.depthTestEnable = VK_TRUE;
depth_stencil_state.depthWriteEnable = VK_FALSE; depth_stencil_state.depthWriteEnable = VK_TRUE;
depth_stencil_state.depthCompareOp = VK_COMPARE_OP_ALWAYS; depth_stencil_state.depthCompareOp = VK_COMPARE_OP_GREATER;
depth_stencil_state.front = depth_stencil_state.back; depth_stencil_state.front = depth_stencil_state.back;
depth_stencil_state.back.compareOp = VK_COMPARE_OP_ALWAYS; depth_stencil_state.back.compareOp = VK_COMPARE_OP_ALWAYS;
@@ -632,8 +615,6 @@ void FaceApp::setup_descriptor_set()
void FaceApp::update_descriptor_set(vector<Texture>& texs, vector<VkDescriptorSet>& descriptSet) void FaceApp::update_descriptor_set(vector<Texture>& texs, vector<VkDescriptorSet>& descriptSet)
{ {
FACE_DBG_LOG("update_descriptor_set: texs.size=%zu descriptSet.size=%zu",
texs.size(), descriptSet.size());
for (int i = 0; i < texs.size(); ++i) for (int i = 0; i < texs.size(); ++i)
{ {
if (texs[i].image == VK_NULL_HANDLE) if (texs[i].image == VK_NULL_HANDLE)
@@ -714,84 +695,21 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
//Application::render(commandBuffer); //Application::render(commandBuffer);
// === 分辨率适配:每帧把"业务参数"缩放到当前屏幕物理像素 ===
// 业务层(MotionManager / InitArg)传进来的 radius / offset_x / offset_y 一直
// 都是按"480x480 设计画布"给的,比如 radius=240 表示在 480 屏上半径 240px
// (即 50% 短边)。现在屏幕变成任意分辨率后,要保持视觉一致:
// 实际像素 = 业务值 * (画布短边 / 480)
// 同时把 canvas/screen 几何写进 push constantshader 用它们:
// 1) 把"画布 NDC ∈ [-1,+1]" 缩到屏幕中央正方形(letterbox);
// 2) 把 fragment 里的圆心从写死的 (240,240) 改成屏幕真实中心。
//
// 注意 pushConstants 本身保留业务原始值不动(SetInitArg 设进来的),
// 这里只把缩放后的副本 pc 推给 GPU。下面 ux/uy 仍然写回 pushConstants
// 是因为它们和分辨率无关,是 motion 帧的纹理坐标。
updateCanvasMetrics();
PushConstants pc = pushConstants;
const float k = m_canvasSize / 480.0f;
pc.radius *= k;
pc.offset_x *= k;
pc.offset_y *= k;
pc.canvas_size = m_canvasSize;
pc.screen_w = m_screenW;
pc.screen_h = m_screenH;
// 相机帧元信息:由 processCameraFrame 在每帧上传纹理之前更新。第一帧之前
// (还没收到相机数据时)m_cameraAspect/m_cameraRotation 是构造函数设的
// 4:3 + 90° 默认值 —— 这两个值正是 CameraX RATIO_4_3 + 大部分 Android 后置
// 主摄竖屏的典型组合,所以即使首帧 bg 用默认值渲染也不会出现明显错位。
pc.camera_aspect = m_cameraAspect;
pc.camera_rotation = m_cameraRotation;
// mirror_x: 前置摄像头时 = 1.0,所有顶点 shader 在最后输出 gl_Position 前
// 把 NDC.x 翻转一次,达到镜子效果(用户右手 → 屏幕左侧)。bg 与 face 都
// 同步翻转,对齐保持。后置时 = 0.0,shader 公式 (1 - 2 * mirror_x) = 1 短路。
pc.mirror_x = m_mirrorX;
vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout_bg, 0, 1, &m_descriptor_set_bg, 0, NULL); vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout_bg, 0, 1, &m_descriptor_set_bg, 0, NULL);
vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_graphicsPipeline_bg); vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_graphicsPipeline_bg);
// bg.frag 也读 push constantr/g/b/radius/screen_w/screen_h)所以 stage 必须 vkCmdPushConstants(commandBuffer, m_pipelineLayout_bg, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(pushConstants), &pushConstants);
// 同时包含 fragment;老代码这里只写了 VERTEX_BIT,是个隐藏的 spec 违规。
vkCmdPushConstants(commandBuffer, m_pipelineLayout_bg, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc);
vkCmdDraw(commandBuffer, 6, 1, 0, 0); vkCmdDraw(commandBuffer, 6, 1, 0, 0);
// 诊断:把 face 渲染所有"前置门"状态打成一行节流日志。一旦 face 没出现,
// 直接看这条就能知道断在哪一关:
// _isChangeMostion=0 → 业务还没调 PlayMotionList / changeMotionList
// _curMotions=0 → motion list 是空的(getMotionByName 全 miss
// _curMotionIndex 越界 → 业务逻辑或 callback 状态机错了
// delta_last_update >= 2000 → mediapipe / passDataToNative 链路 2s 没动静
const long long now_ms = getCurrentTimeMillis();
const long long delta_lu = now_ms - (long long)last_update_time;
FACE_DBG_LOG_THROTTLED("render.face.gate",
"render.face gate _isChangeMostion=%d _curMotions=%zu"
" _curMotionIndex=%d _curFrameIndex=%d delta_lastUpdate=%lldms"
" _playMotion=%d",
(int)_isChangeMostion, _curMotions.size(),
(int)_curMotionIndex, (int)_curFrameIndex,
delta_lu, (int)_playMotion);
if (!_isChangeMostion) if (!_isChangeMostion)
{ {
return; return;
} }
if (_curMotions.empty())
{
FACE_DBG_LOG_THROTTLED("render.face.emptyMotions",
"render.face dropped: _isChangeMostion=1 but _curMotions is empty");
return;
}
string curMotionName = _curMotions[_curMotionIndex].name; string curMotionName = _curMotions[_curMotionIndex].name;
int loadMotionIndex = motion_list_map[curMotionName]; int loadMotionIndex = motion_list_map[curMotionName];
// face draw 前关键参数节流日志,保留少量"任何时候挂了能立刻判断断点"的字段。
FACE_DBG_LOG_THROTTLED("render.face.preDraw",
"render.face preDraw motion='%s' loadMotionIdx=%d obj_idx=%zu mirror=%.1f",
curMotionName.c_str(), loadMotionIndex, obj_indices.size(), pc.mirror_x);
//if (cur_left) //if (cur_left)
//{ //{
vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, &m_descriptor_sets_left[loadMotionIndex], 0, NULL); vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, &m_descriptor_sets_left[loadMotionIndex], 0, NULL);
@@ -810,14 +728,12 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
float uy = _curMotions[_curMotionIndex].frames[_curFrameIndex].y; float uy = _curMotions[_curMotionIndex].frames[_curFrameIndex].y;
pushConstants.ux = ux; pushConstants.ux = ux;
pushConstants.uy = uy; pushConstants.uy = uy;
pc.ux = ux;
pc.uy = uy;
} }
} }
//fsValues[1] = 0; //fsValues[1] = 0;
//fsValues[2] = 360; //fsValues[2] = 360;
vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc); vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pushConstants), &pushConstants);
//vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(fsValues), fsValues); //vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(fsValues), fsValues);
@@ -833,16 +749,7 @@ void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime)
#else #else
if (getCurrentTimeMillis() - last_update_time < 2000) if (getCurrentTimeMillis() - last_update_time < 2000)
{ {
FACE_DBG_LOG_THROTTLED("render.face.draw", vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0);
"render.face vkCmdDrawIndexed indices=%zu motion=%s frame=%d",
obj_indices.size(), curMotionName.c_str(), (int)_curFrameIndex);
vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0);
}
else
{
FACE_DBG_LOG_THROTTLED("render.face.heartbeatStale",
"render.face SKIP vkCmdDrawIndexed: delta_last_update=%lldms (>= 2000ms)",
(long long)(getCurrentTimeMillis() - last_update_time));
} }
#endif #endif
} }
@@ -980,12 +887,6 @@ void FaceApp::onWindowLost()
std::unique_lock<std::mutex> lk_point(mtx_point); std::unique_lock<std::mutex> lk_point(mtx_point);
std::unique_lock<std::mutex> lk_motion(changeMotionMtx); std::unique_lock<std::mutex> lk_motion(changeMotionMtx);
std::unique_lock<std::mutex> lk_tex(createTextureMtx); std::unique_lock<std::mutex> lk_tex(createTextureMtx);
// 同时阻塞所有并发的 GPU 提交(drawFrame / copyBuffer / updateTexture)。
// cleanupForWindowLost() 会执行 vkDeviceWaitIdle 并销毁 swapchain / surface /
// semaphores / fences 等窗口相关资源,如果此时有其它线程正在 vkQueueSubmit
// 或 vkQueuePresentKHR,会触发 FORTIFY: pthread_mutex_lock called on a
// destroyed mutex。这里持锁确保销毁与提交是互斥的。
std::unique_lock<std::mutex> lk_pool(poolQueueMtx);
Application::cleanupForWindowLost(); Application::cleanupForWindowLost();
_secondfaceAppInited = false; _secondfaceAppInited = false;
@@ -1005,18 +906,12 @@ void FaceApp::onWindowInit()
} else { } else {
// Recovery path after an earlier onWindowLost. Rebuild only the // Recovery path after an earlier onWindowLost. Rebuild only the
// window-dependent Vulkan objects; keep renderPass / pipelines / // window-dependent Vulkan objects; keep renderPass / pipelines /
// FaceApp GPU resources intact unless the new swapchain is // FaceApp GPU resources intact.
// incompatible (e.g. rotation changed extent).
std::unique_lock<std::mutex> lk_point(mtx_point); std::unique_lock<std::mutex> lk_point(mtx_point);
std::unique_lock<std::mutex> lk_motion(changeMotionMtx); std::unique_lock<std::mutex> lk_motion(changeMotionMtx);
std::unique_lock<std::mutex> lk_tex(createTextureMtx); std::unique_lock<std::mutex> lk_tex(createTextureMtx);
bool swapchainIncompatible = Application::reinitForNewWindow(); Application::reinitForNewWindow();
if (swapchainIncompatible && _faceAppInited) {
FACE_DBG_LOG("FaceApp::onWindowInit: swapchain incompatible, rebuilding FaceApp pipelines");
recreatePipelinesForSwapchain();
}
if (!_secondfaceAppInited) { if (!_secondfaceAppInited) {
_secondfaceAppInited = true; _secondfaceAppInited = true;
@@ -1030,39 +925,13 @@ void FaceApp::onWindowInit()
(int)_secondfaceAppInited, (int)_running); (int)_secondfaceAppInited, (int)_running);
} }
void FaceApp::recreatePipelinesForSwapchain()
{
// Destroy the two FaceApp pipelines. Layouts / descriptor set layouts are
// swapchain-independent and kept as-is so existing descriptor sets keep
// pointing at the same texture/uniform resources.
if (m_graphicsPipeline != VK_NULL_HANDLE) {
vkDestroyPipeline(device, m_graphicsPipeline, nullptr);
m_graphicsPipeline = VK_NULL_HANDLE;
}
if (m_graphicsPipeline_bg != VK_NULL_HANDLE) {
vkDestroyPipeline(device, m_graphicsPipeline_bg, nullptr);
m_graphicsPipeline_bg = VK_NULL_HANDLE;
}
// Recreate against the fresh renderPass (if format changed) and the new
// swapChainExtent (viewport/scissor are baked statically into these).
create_face_pipelines();
create_pipelines_bg();
FACE_DBG_LOG("FaceApp::recreatePipelinesForSwapchain: rebuilt for extent=%ux%u",
swapChainExtent.width, swapChainExtent.height);
}
void FaceApp::update_uniform_buffers() void FaceApp::update_uniform_buffers()
{ {
// 注意:当前 vertex shader (texture.vert) 实际并没有使用 ubo.projection 把模型 uint32_t width = 480;
// 投影到屏幕——它直接拿 inPos.xy*2-1 当 NDC 用。所以下面 aspect 取 1.0 只是 uint32_t height = 480;
// 为了让 ubo 不再依赖原本写死的 480x480 输入分辨率;要真正影响屏幕显示比例的 float zoom = 2;
// 是 push constant 里的 canvas_size / screen_w / screen_h,由 render() 写入。
const float aspect = 1.0f;
const float zoom = 2.0f;
// Vertex shader // Vertex shader
ubo_vs.projection = glm::perspective(glm::radians(60.0f), aspect, 0.001f, 256.0f); ubo_vs.projection = glm::perspective(glm::radians(60.0f), static_cast<float>(width) / static_cast<float>(height), 0.001f, 256.0f);
glm::mat4 view_matrix = glm::translate(glm::mat4(1.0f), glm::vec3(0.0f, 0.0f, zoom)); glm::mat4 view_matrix = glm::translate(glm::mat4(1.0f), glm::vec3(0.0f, 0.0f, zoom));
ubo_vs.model = view_matrix * glm::translate(glm::mat4(1.0f), camera_pos); ubo_vs.model = view_matrix * glm::translate(glm::mat4(1.0f), camera_pos);
@@ -1075,26 +944,6 @@ void FaceApp::update_uniform_buffers()
memcpy(uniform_buffer_mapped, &ubo_vs, sizeof(ubo_vs)); memcpy(uniform_buffer_mapped, &ubo_vs, sizeof(ubo_vs));
} }
void FaceApp::updateCanvasMetrics()
{
// 把"屏幕中央 min(w,h) 的正方形"作为设计画布。画布外的部分会落在
// shader 计算的 NDC ∈ [-1,+1] 之外,被 GPU 自动裁剪为黑色 letterbox。
//
// swapChainExtent 来自 Application::createSwapChain 里的
// surface_capabilities.currentExtent,即当前窗口/Surface 的真实分辨率。
// 任何 swapchain 重建(onWindowLost -> onWindowInit)都会刷新它,所以
// 这里每帧重算是廉价且自洽的。
m_screenW = (float)swapChainExtent.width;
m_screenH = (float)swapChainExtent.height;
m_canvasSize = (m_screenW < m_screenH) ? m_screenW : m_screenH;
if (m_canvasSize <= 0.0f) {
// swapchain 还没初始化 / 已销毁时给个安全值,避免 shader 除零。
m_canvasSize = 480.0f;
m_screenW = 480.0f;
m_screenH = 480.0f;
}
}
void FaceApp::createVertexBuffer() void FaceApp::createVertexBuffer()
{ {
VkDeviceSize vertexBufferSize = sizeof(TextureLoadingVertexStructure) * obj_vertices.size(); VkDeviceSize vertexBufferSize = sizeof(TextureLoadingVertexStructure) * obj_vertices.size();
@@ -1143,85 +992,17 @@ void FaceApp::uploadVertexData() {
void FaceApp::processCameraFrame(uint8_t* data, int width, int height,
int rowStride, size_t dataSize, int rotation,
bool mirrorX)
{
if (!isInited() || data == nullptr || width <= 0 || height <= 0) {
return;
}
// 1) rotation 收敛到 0/90/180/270。CameraX 文档保证只会给这四个值,
// 但理论上 ((rotation % 360) + 360) % 360 更鲁棒。
int rotNorm = ((rotation % 360) + 360) % 360;
if (rotNorm != 0 && rotNorm != 90 && rotNorm != 180 && rotNorm != 270) {
// 非法值时保守用 90(与原 hardcode 一致)。
rotNorm = 90;
}
// 2) 元信息更新(无锁单写者:仅相机分析线程写,render 线程读)。
// 放在 sizeChanged 判断前后都安全:render() 每帧重新拷到 push constant。
m_cameraAspect = (float)width / (float)height;
m_cameraRotation = (float)rotNorm;
m_mirrorX = mirrorX ? 1.0f : 0.0f;
// 3) 检查帧尺寸是否与现有 tex_bg 一致。
// 场景:CameraX session 重建(如 Activity onResume)后第一帧分辨率可能
// 不同;或者 setTargetResolution 在不同设备上落地为不同尺寸。
bool sizeChanged = (tex_bg.image != VK_NULL_HANDLE)
&& (tex_bg.width != width || tex_bg.height != height);
if (!sizeChanged) {
// 快路径(每帧):直接走 base 标准上传路径。第一次 image == VK_NULL_HANDLE
// base 会自动 createTexture(width, height, ...) 自适应分辨率;之后每帧
// image 已存在,直接走 staging buffer 拷贝。
processWithVulkan(data, width, height, rowStride, dataSize,
tex_bg, false, commandPool, "data_from_camera");
return;
}
// 慢路径(罕见,整个 session 通常 1 次):尺寸变化 → 重建 image + 刷新
// descriptor set。整个 ceremony 必须严格串行:
// ① createTextureMtx 与 drawFrame / 其它 processWithVulkan 互斥
// FaceApp::drawFrame 进 Application::drawFrame 之前会持此锁,所以
// 我们持锁期间渲染线程被阻塞,不会有新的 cmdbuf 绑定 m_descriptor_set_bg
// 并 submit)。
// ② vkDeviceWaitIdle 等待所有"已 submit 但未完成"的 cmdbuf 跑完,确保
// 销毁旧 image/view 时 GPU 已经不再采样它。
// ③ destroy → create → updateTexture → refresh_descriptor_set_bg
// 全部在锁内完成,期间 m_descriptor_set_bg 处于"指向无效 view"的中间
// 状态,但因为渲染线程被锁阻塞,这个中间状态对 GPU 不可见。
// 注意不能调 base::processWithVulkan,那会再次 lock createTextureMtx 死锁;
// 直接调 createTexture + updateTexture,这两个 base 方法不持 createTextureMtx。
#ifndef _WIN32
DebugLog::log("processCameraFrame: bg size changed %dx%d -> %dx%d, recreate texture",
tex_bg.width, tex_bg.height, width, height);
#endif
std::unique_lock<std::mutex> lk_tex(createTextureMtx);
{
std::lock_guard<std::mutex> lk_pool(poolQueueMtx);
vkDeviceWaitIdle(device);
}
destroyTexture(device, tex_bg);
createTexture(device, physicalDevice, width, height, tex_bg, false,
"data_from_camera", dataSize);
updateTexture(device, physicalDevice, commandPool, graphicsQueue,
data, width, height, rowStride, dataSize, tex_bg);
refresh_descriptor_set_bg();
}
void FaceApp::update_face_vertex_buffer(float* pos, int pointCount) void FaceApp::update_face_vertex_buffer(float* pos, int pointCount)
{ {
if(!isInited()) if(!isInited())
{ {
FACE_DBG_LOG_THROTTLED("update_face_vertex_buffer.notInited",
"update_face_vertex_buffer dropped: !isInited()");
return; return;
} }
std::lock_guard<std::mutex> lock(mtx_point); std::lock_guard<std::mutex> lock(mtx_point);
last_update_time = getCurrentTimeMillis(); last_update_time = getCurrentTimeMillis();
for (int i = 0; i < obj_vertices.size(); ++i) for (int i = 0; i < obj_vertices.size(); ++i)
{ {
@@ -1241,20 +1022,41 @@ void FaceApp::update_face_vertex_buffer(float* pos, int pointCount)
void FaceApp::copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size) void FaceApp::copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size)
{ {
// 改造说明: VkCommandBuffer commandBuffer = beginSingleTimeCommands();
// 旧实现是 vkAllocate(commandPool) + vkBegin + vkCmdCopyBuffer + vkEnd +
// vkQueueSubmit(graphicsQueue) + vkQueueWaitIdle + vkFree(commandPool) VkBufferCopy copyRegion = {};
// 每帧 update_face_vertex_buffer 会调两次 copyBuffer(顶点 + 索引), copyRegion.size = size;
// 长期高频 allocate/free 会把驱动 per-pool mutex 玩坏,触发 vkCmdCopyBuffer(commandBuffer, srcBuffer, dstBuffer, 1, &copyRegion);
// FORTIFY: pthread_mutex_lock called on a destroyed mutex。
// endSingleTimeCommands(commandBuffer);
// 现在改走基类的 runTransferCommand,复用 commandPool_ex 上预分配的 }
// 3 个 cmdbuf + fence,并发安全由 m_xferMtx + poolQueueMtx 联合保证。
runTransferCommand([&](VkCommandBuffer commandBuffer) { VkCommandBuffer FaceApp::beginSingleTimeCommands() {
VkBufferCopy copyRegion = {}; VkCommandBufferAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO };
copyRegion.size = size; allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
vkCmdCopyBuffer(commandBuffer, srcBuffer, dstBuffer, 1, &copyRegion); allocInfo.commandPool = commandPool;
}); allocInfo.commandBufferCount = 1;
VkCommandBuffer commandBuffer;
vkAllocateCommandBuffers(device, &allocInfo, &commandBuffer);
VkCommandBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
vkBeginCommandBuffer(commandBuffer, &beginInfo);
return commandBuffer;
}
void FaceApp::endSingleTimeCommands(VkCommandBuffer commandBuffer) {
vkEndCommandBuffer(commandBuffer);
VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &commandBuffer;
vkQueueSubmit(graphicsQueue, 1, &submitInfo, VK_NULL_HANDLE);
vkQueueWaitIdle(graphicsQueue);
vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer);
} }
void FaceApp::createUniformBuffer() void FaceApp::createUniformBuffer()
@@ -1433,12 +1235,8 @@ void FaceApp::setup_descriptor_set_layout_bg()
pipeline_layout_create_info.pSetLayouts = &m_descriptorSetLayout_bg; pipeline_layout_create_info.pSetLayouts = &m_descriptorSetLayout_bg;
// bg.frag 同样要读 push constantr/g/b/radius,以及分辨率适配重构后的
// screen_w/screen_h),所以 stage 必须包含 FRAGMENT_BIT。老代码这里只写了
// VERTEX_BIT,按 Vulkan spec 是非法的(验证层会 warning,部分驱动会读到
// 未定义内容),借这次重构修掉。
VkPushConstantRange pushConstantRanges[1]; VkPushConstantRange pushConstantRanges[1];
pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT; pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
pushConstantRanges[0].offset = 0; pushConstantRanges[0].offset = 0;
pushConstantRanges[0].size = sizeof(PushConstants); pushConstantRanges[0].size = sizeof(PushConstants);
@@ -1459,18 +1257,7 @@ void FaceApp::setup_descriptor_set_bg()
VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, &m_descriptor_set_bg)); VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, &m_descriptor_set_bg));
refresh_descriptor_set_bg();
}
void FaceApp::refresh_descriptor_set_bg()
{
if (m_descriptor_set_bg == VK_NULL_HANDLE) {
return;
}
if (tex_bg.view == VK_NULL_HANDLE || tex_bg.sampler == VK_NULL_HANDLE) {
// 还没 loadTexture / processCameraFrame 过,没法绑定。
return;
}
VkDescriptorImageInfo image_descriptor; VkDescriptorImageInfo image_descriptor;
image_descriptor.imageView = tex_bg.view; image_descriptor.imageView = tex_bg.view;
@@ -1485,7 +1272,9 @@ void FaceApp::refresh_descriptor_set_bg()
write_descriptor_set.pImageInfo = &image_descriptor; write_descriptor_set.pImageInfo = &image_descriptor;
write_descriptor_set.descriptorCount = 1; write_descriptor_set.descriptorCount = 1;
vkUpdateDescriptorSets(device, 1, &write_descriptor_set, 0, NULL); std::vector<VkWriteDescriptorSet> write_descriptor_sets = { write_descriptor_set };
vkUpdateDescriptorSets(device, static_cast<uint32_t>(write_descriptor_sets.size()), write_descriptor_sets.data(), 0, NULL);
} }
void FaceApp::destroyTexture(VkDevice device, Texture& texture) { void FaceApp::destroyTexture(VkDevice device, Texture& texture) {
@@ -1638,9 +1427,6 @@ void FaceApp::cleanup()
//} //}
destroyTexture(device, tex_bg); destroyTexture(device, tex_bg);
// 必须在销毁 commandPool_ex 之前释放从它分配的 transfer cmdbuf + fence。
// vkDeviceWaitIdle 已在本函数开头调过,提交不会再有 in-flight。
destroyTransferResources();
vkDestroyCommandPool(device, commandPool, nullptr); vkDestroyCommandPool(device, commandPool, nullptr);
vkDestroyCommandPool(device, commandPool_ex, nullptr); vkDestroyCommandPool(device, commandPool_ex, nullptr);
Application::cleanup(); Application::cleanup();
@@ -1799,7 +1585,6 @@ void FaceApp::loadMotionThread()
string name = m.name; string name = m.name;
if (motion_list_map.find(name) != motion_list_map.end()) if (motion_list_map.find(name) != motion_list_map.end())
{ {
FACE_DBG_LOG("loadMotionThread: skip already-loaded motion='%s'", name.c_str());
continue; continue;
} }
m_texs_left.push_back(Texture()); m_texs_left.push_back(Texture());
@@ -1816,10 +1601,6 @@ void FaceApp::loadMotionThread()
#endif // _WIN32 #endif // _WIN32
motion_list_map[name] = m_texs_left.size() - 1; motion_list_map[name] = m_texs_left.size() - 1;
_loadMotionMap[name] = m; _loadMotionMap[name] = m;
FACE_DBG_LOG("loadMotionThread: loaded motion='%s' idx=%zu tex_image=%p tex_view=%p tex_sampler=%p frames=%zu",
name.c_str(), m_texs_left.size() - 1,
(void*)newTex.image, (void*)newTex.view, (void*)newTex.sampler,
m.frames.size());
} }
// for (int i = 0; i < _loadMotions.size(); ++i) // for (int i = 0; i < _loadMotions.size(); ++i)
@@ -1856,7 +1637,7 @@ Motion FaceApp::getMotionByName(string name)
void FaceApp::changeMotionList(vector<string> motions, AnimationFinishedCallback callback, bool loop) void FaceApp::changeMotionList(vector<string> motions, AnimationFinishedCallback callback, bool loop)
{ {
FACE_DBG_LOG_THROTTLED("FaceApp.changeMotionList", DebugLog::log_throttled("FaceApp.changeMotionList",
"FaceApp::changeMotionList called count=%zu loop=%d _isLoadMotion=%d", "FaceApp::changeMotionList called count=%zu loop=%d _isLoadMotion=%d",
motions.size(), (int)loop, (int)_isLoadMotion); motions.size(), (int)loop, (int)_isLoadMotion);
if (_isLoadMotion) if (_isLoadMotion)
@@ -1880,11 +1661,7 @@ void FaceApp::changeMotionList(vector<string> motions, AnimationFinishedCallback
vector<Motion> motion_list; vector<Motion> motion_list;
for (auto ms : motions) { for (auto ms : motions) {
Motion m = getMotionByName(ms); Motion m = getMotionByName(ms);
auto it = motion_list_map.find(ms); motion_list.push_back(m);
int idx = (it == motion_list_map.end()) ? -1 : (int)it->second;
FACE_DBG_LOG("changeMotionList: requested='%s' idx=%d frames=%zu (-1=NOT FOUND)",
ms.c_str(), idx, m.frames.size());
motion_list.push_back(m);
} }
_curMotions = motion_list; _curMotions = motion_list;
+3 -96
View File
@@ -49,15 +49,6 @@ struct InitArg
}; };
// 推送给 shader 的常量。前 9 个 float 是「业务原始值」,单位以 480x480 设计画布
// 为基准(向后兼容):业务给 radius=240 / offset_x=200 / offset_y=-200 等都按这个画布
// 思考。后 3 个 float 是 SDK 在每次 render() 时根据当前 swapchain 尺寸计算并写入的
// 「画布几何」,shader 通过它把「画布坐标」映射到「屏幕坐标」并把圆形 mask 中心
// 锚定在屏幕中心。
//
// ★ 字段顺序和命名必须和 GLSL 中的 layout(push_constant) 块严格一致,否则会读到
// 错位数据。修改时四个 shader (bg.vert / bg.frag / texture.vert / texture.frag)
// 都要同步更新。
struct PushConstants { struct PushConstants {
float zoom = 0.5f; float zoom = 0.5f;
float r = 0; float r = 0;
@@ -68,32 +59,6 @@ struct PushConstants {
float uy = 0; float uy = 0;
float offset_x = 0; float offset_x = 0;
float offset_y = 0; float offset_y = 0;
// 由 SDK 在 FaceApp::render() 内每帧写入,业务无感。
// canvas_size = min(swapchain.w, swapchain.h),单位:物理像素
// screen_w / screen_h = swapchain 当前实际尺寸,单位:物理像素
// 设计画布始终居中铺在屏幕中央正方形区域;外围是 letterbox 黑边。
float canvas_size = 480.0f;
float screen_w = 480.0f;
float screen_h = 480.0f;
// 相机帧元信息,由 SDK 在收到第一帧 / 帧尺寸变化时更新。shader 用来:
// 1) 把相机帧(任意 raw aspect)正确 cover 到 1:1 画布;
// 2) 按 camera_rotation 旋转 UV,让画面在屏幕上正立——这是不同手机的
// sensor orientation 不同导致画面侧着 / 倒着的根因。
// camera_aspect = camera_raw_w / camera_raw_h(典型 4:3 = 1.333…)
// camera_rotation = CameraX 的 ImageInfo.getRotationDegrees()
// 表示「图像顺时针旋转多少度才能正立」,离散 0/90/180/270。
// 默认值 4/3 + 90° 是兼容老 hardcode 行为(大部分 Android 手机后置主摄竖屏)。
float camera_aspect = 4.0f / 3.0f;
float camera_rotation = 90.0f;
// 镜像标志:1.0 表示前置摄像头需要"镜子效果",所有顶点 shader 会把
// gl_Position.x 翻转一次(NDC 水平翻转)。配合 FaceLandmarkerHelper 在
// bitmap 阶段做的 postScale(-1,1)bg 与 face 同步翻转、互相对齐。
// 后置摄像头此值为 0.0,shader 短路无开销。
// 用 float 而不是 bool/int 是为了和 Vulkan push constant 4-byte 对齐 + GLSL
// 端 layout 简单一致;shader 里 `1.0 - 2.0 * mirror_x` 实现无分支翻转。
float mirror_x = 0.0f;
}; };
using Callback = std::function<void()>; using Callback = std::function<void()>;
@@ -147,6 +112,8 @@ private:
const bool kThick = false; const bool kThick = false;
void copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size); void copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size);
VkCommandBuffer beginSingleTimeCommands();
void endSingleTimeCommands(VkCommandBuffer commandBuffer);
// 顶点缓冲区相关 // 顶点缓冲区相关
VkBuffer m_vertexBuffer = VK_NULL_HANDLE; VkBuffer m_vertexBuffer = VK_NULL_HANDLE;
@@ -198,64 +165,13 @@ private:
//float myFloatValue = 1.5f; //float myFloatValue = 1.5f;
// 「业务原始值」缓存:SetInitArg 把 InitArg 拷到这里,render() 每帧读出来再
// 缩放 + 注入屏幕几何。注意 ux/uy 由 motion 帧每帧写入,也保留在这里。
PushConstants pushConstants; PushConstants pushConstants;
// ===== 屏幕分辨率适配 =====
// 原 SDK 把屏幕写死 480x480,所有业务参数都按 480 像素画布给。现在改成支持
// 任意分辨率:把"屏幕中央 min(w,h) 边长的正方形"作为设计画布,画布外是黑边。
// 业务参数语义不变,SDK 在 render() 里按 (canvas_size / 480) 缩放成屏幕物理
// 像素后再推给 shader。
// m_canvasSize = min(swapChainExtent.w, swapChainExtent.h)
// m_screenW / m_screenH = swapChainExtent.w / swapChainExtent.h
// updateCanvasMetrics() 由 render() 在每次 vkCmdPushConstants 之前调用。
// 因为 swapchain 重建会走 onWindowLost -> onWindowInit ->
// recreatePipelinesForSwapchain,而 render() 永远在 drawFrame 里被调,所以
// 这套数值天然会跟随 swapchain 变化。
float m_canvasSize = 480.0f;
float m_screenW = 480.0f;
float m_screenH = 480.0f;
void updateCanvasMetrics();
// ===== 相机帧元信息适配 =====
// 不同手机摄像头物理分辨率和 sensor orientation 都不一样,CameraX 给的图像
// 既不一定是精确 4:3,也不一定是「正立」朝向。SDK 在这里缓存最近一帧的
// (raw_aspect, rotation)render() 时塞进 push constant 让 shader 自己处理:
// - bg.vert 按 m_cameraRotation 旋转 UV,让画面在屏幕上方向正确
// - bg.vert 按 m_cameraAspect 自适应 cover 画布,避免拉扁/拉长
// 由 processCameraFrame() 根据每次相机帧实际 width / height / rotation 更新。
// 第一帧之前用安全默认值(4:3 + 90°,对应大部分 Android 后置主摄竖屏行为)。
float m_cameraAspect = 4.0f / 3.0f;
float m_cameraRotation = 90.0f;
// 镜像标志缓存:与 m_cameraAspect 同写者(相机分析线程),由 render() 拷到
// push constant 的 mirror_x 字段。前置=1.0,后置=0.0。
float m_mirrorX = 0.0f;
public:
// JNI 入口(main.cpp)调用,用来在每帧上传相机纹理之前同步元信息。
// 内部会:
// 1) 检测 width × height 与已有 tex_bg 是否一致,不一致就 destroy + recreate
// 2) 更新 m_cameraAspect / m_cameraRotation / m_mirrorX
// 3) 调用 base 的 processWithVulkan 走 staging buffer 上传。
// mirrorX 来自 Java 层 lensFacing == LENS_FACING_FRONT,表示是否需要镜子效果。
// 必须由唯一的相机分析线程调用(FaceActivity backgroundExecutor),
// 内部依赖 createTextureMtx 序列化与 drawFrame / cleanup 的并发。
void processCameraFrame(uint8_t* data, int width, int height,
int rowStride, size_t dataSize, int rotation,
bool mirrorX);
private:
void create_pipelines_bg(); void create_pipelines_bg();
void setup_descriptor_set_layout_bg(); void setup_descriptor_set_layout_bg();
void setup_descriptor_set_bg(); void setup_descriptor_set_bg();
// 把 m_descriptor_set_bg 重新绑定到当前的 tex_bg.view / sampler。
// 当相机分辨率变化导致 tex_bg 被 destroy + recreate 后,新建的 image
// 有新的 VkImageView 句柄,旧的 descriptor set 里缓存的 handle 失效,
// 必须 vkUpdateDescriptorSets 让 descriptor 指向新 view。该函数不重新
// 分配 descriptor set,仅刷新绑定。
void refresh_descriptor_set_bg();
VkPipeline m_graphicsPipeline_bg = VK_NULL_HANDLE; VkPipeline m_graphicsPipeline_bg = VK_NULL_HANDLE;
VkPipelineLayout m_pipelineLayout_bg = VK_NULL_HANDLE; VkPipelineLayout m_pipelineLayout_bg = VK_NULL_HANDLE;
VkDescriptorSetLayout m_descriptorSetLayout_bg = VK_NULL_HANDLE; VkDescriptorSetLayout m_descriptorSetLayout_bg = VK_NULL_HANDLE;
@@ -275,17 +191,8 @@ public:
// Called from main thread in response to APP_CMD_INIT_WINDOW. // Called from main thread in response to APP_CMD_INIT_WINDOW.
// Does the full first-time initVulkan() on the very first call, and a // Does the full first-time initVulkan() on the very first call, and a
// lightweight swapchain/surface rebuild on subsequent calls. When the // lightweight swapchain/surface rebuild on subsequent calls.
// new swapchain's extent or format differs from the previous one, the
// FaceApp pipelines (baked with static viewport / old renderPass) are
// also destroyed and recreated via recreatePipelinesForSwapchain().
void onWindowInit(); void onWindowInit();
// Destroy and recreate m_graphicsPipeline / m_graphicsPipeline_bg so they
// match the current renderPass and swapChainExtent. Descriptor set layouts,
// pipeline layouts, vertex/index buffers, uniform buffers and textures are
// all kept. Must be called with device idle and the FaceApp mutexes held.
void recreatePipelinesForSwapchain();
void loadMotionThread(); void loadMotionThread();
std::thread worker_; std::thread worker_;
bool _isLoadMotion = false; bool _isLoadMotion = false;