339 lines
13 KiB
C++
339 lines
13 KiB
C++
#ifndef __FaceApp_H__
|
||
#define __FaceApp_H__
|
||
|
||
#include "Application.h"
|
||
#include <functional>
|
||
#include "../third_party/vma/include/vk_mem_alloc.h"
|
||
#include <map>
|
||
#include <glm/glm.hpp>
|
||
#include <glm/gtc/matrix_transform.hpp>
|
||
#include <glm/gtc/matrix_transform.inl>
|
||
#include "nlohmann/json.hpp"
|
||
using namespace std;
|
||
using json = nlohmann::json;
|
||
|
||
struct Frame {
|
||
string name;
|
||
float x;
|
||
float y;
|
||
NLOHMANN_DEFINE_TYPE_INTRUSIVE(Frame, name, x, y)
|
||
};
|
||
|
||
struct Motion {
|
||
string name;
|
||
vector< Frame> frames;
|
||
string getMotionId() {
|
||
return name;
|
||
}
|
||
NLOHMANN_DEFINE_TYPE_INTRUSIVE(Motion, name, frames)
|
||
};
|
||
|
||
struct MotionList
|
||
{
|
||
vector<Motion> motions;
|
||
NLOHMANN_DEFINE_TYPE_INTRUSIVE(MotionList, motions)
|
||
};
|
||
|
||
struct InitArg
|
||
{
|
||
int action_fps;
|
||
float zoom;
|
||
float r;
|
||
float g;
|
||
float b;
|
||
float radius;
|
||
float offset_x;
|
||
float offset_y;
|
||
//Motion motion;
|
||
NLOHMANN_DEFINE_TYPE_INTRUSIVE(InitArg, action_fps, zoom, r, g, b, radius, offset_x, offset_y);
|
||
};
|
||
|
||
|
||
// 推送给 shader 的常量。前 9 个 float 是「业务原始值」，单位以 480x480 设计画布
|
||
// 为基准（向后兼容）：业务给 radius=240 / offset_x=200 / offset_y=-200 等都按这个画布
|
||
// 思考。后 3 个 float 是 SDK 在每次 render() 时根据当前 swapchain 尺寸计算并写入的
|
||
// 「画布几何」，shader 通过它把「画布坐标」映射到「屏幕坐标」并把圆形 mask 中心
|
||
// 锚定在屏幕中心。
|
||
//
|
||
// ★ 字段顺序和命名必须和 GLSL 中的 layout(push_constant) 块严格一致，否则会读到
|
||
// 错位数据。修改时四个 shader (bg.vert / bg.frag / texture.vert / texture.frag)
|
||
// 都要同步更新。
|
||
struct PushConstants {
|
||
float zoom = 0.5f;
|
||
float r = 0;
|
||
float g = 0;
|
||
float b = 0;
|
||
float radius = 240;
|
||
float ux = 0;
|
||
float uy = 0;
|
||
float offset_x = 0;
|
||
float offset_y = 0;
|
||
// 由 SDK 在 FaceApp::render() 内每帧写入，业务无感。
|
||
// canvas_size = min(swapchain.w, swapchain.h)，单位：物理像素
|
||
// screen_w / screen_h = swapchain 当前实际尺寸，单位：物理像素
|
||
// 设计画布始终居中铺在屏幕中央正方形区域；外围是 letterbox 黑边。
|
||
float canvas_size = 480.0f;
|
||
float screen_w = 480.0f;
|
||
float screen_h = 480.0f;
|
||
|
||
// 相机帧元信息，由 SDK 在收到第一帧 / 帧尺寸变化时更新。shader 用来：
|
||
// 1) 把相机帧（任意 raw aspect）正确 cover 到 1:1 画布；
|
||
// 2) 按 camera_rotation 旋转 UV，让画面在屏幕上正立——这是不同手机的
|
||
// sensor orientation 不同导致画面侧着 / 倒着的根因。
|
||
// camera_aspect = camera_raw_w / camera_raw_h（典型 4:3 = 1.333…）
|
||
// camera_rotation = CameraX 的 ImageInfo.getRotationDegrees()，
|
||
// 表示「图像顺时针旋转多少度才能正立」，离散 0/90/180/270。
|
||
// 默认值 4/3 + 90° 是兼容老 hardcode 行为（大部分 Android 手机后置主摄竖屏）。
|
||
float camera_aspect = 4.0f / 3.0f;
|
||
float camera_rotation = 90.0f;
|
||
|
||
// 镜像标志：1.0 表示前置摄像头需要"镜子效果"，所有顶点 shader 会把
|
||
// gl_Position.x 翻转一次（NDC 水平翻转）。配合 FaceLandmarkerHelper 在
|
||
// bitmap 阶段做的 postScale(-1,1)，bg 与 face 同步翻转、互相对齐。
|
||
// 后置摄像头此值为 0.0，shader 短路无开销。
|
||
// 用 float 而不是 bool/int 是为了和 Vulkan push constant 4-byte 对齐 + GLSL
|
||
// 端 layout 简单一致；shader 里 `1.0 - 2.0 * mirror_x` 实现无分支翻转。
|
||
float mirror_x = 0.0f;
|
||
};
|
||
|
||
using Callback = std::function<void()>;
|
||
using AnimationFinishedCallback = std::function<void()>;
|
||
|
||
class FaceApp :public Application
|
||
{
|
||
public:
|
||
FaceApp(/* args */);
|
||
~FaceApp();
|
||
|
||
struct
|
||
{
|
||
glm::mat4 projection;
|
||
glm::mat4 model;
|
||
glm::vec4 view_pos;
|
||
float lod_bias = 0.0f;
|
||
} ubo_vs;
|
||
|
||
void initVulkan() override;
|
||
|
||
|
||
void render(VkCommandBuffer commandBuffer, long long frameTime) override;
|
||
static FaceApp* Get() { return faceIns; }
|
||
void update_face_vertex_buffer(float* pos, int pointCount);
|
||
|
||
virtual bool isInited() override { return _applicationInited && _faceAppInited && _sceondInited; }
|
||
virtual void cleanup() override;
|
||
|
||
Texture tex_bg = {};
|
||
|
||
private:
|
||
glm::vec3 rotation = glm::vec3();
|
||
glm::vec3 camera_pos = glm::vec3();
|
||
private:
|
||
static FaceApp* faceIns;
|
||
VmaAllocator allocator = nullptr;
|
||
bool LoadOBJ(const std::string& filename,std::vector<TextureLoadingVertexStructure>& vertices, std::vector<uint32_t>& indices);
|
||
|
||
void createVmaAllocator();
|
||
void update_uniform_buffers();
|
||
|
||
void createVertexBuffer();
|
||
void uploadVertexData();
|
||
VkDescriptorPool descriptor_pool = VK_NULL_HANDLE;
|
||
void create_face_pipelines();
|
||
void setup_descriptor_pool();
|
||
void setup_descriptor_set_layout();
|
||
void setup_descriptor_set();
|
||
void update_descriptor_set(vector<Texture>& texs, vector<VkDescriptorSet>& descriptSet);
|
||
const bool kThick = false;
|
||
|
||
void copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size);
|
||
|
||
// 顶点缓冲区相关
|
||
VkBuffer m_vertexBuffer = VK_NULL_HANDLE;
|
||
VmaAllocation m_vertexBufferAllocation = VK_NULL_HANDLE;
|
||
VkBuffer m_stagingBuffer = VK_NULL_HANDLE;
|
||
VmaAllocation m_stagingBufferAllocation = VK_NULL_HANDLE;
|
||
|
||
// 索引缓冲区相关
|
||
VkBuffer m_indexBuffer = VK_NULL_HANDLE;
|
||
VmaAllocation m_indexBufferAllocation = VK_NULL_HANDLE;
|
||
|
||
// 渲染管线和描述符
|
||
VkPipeline m_graphicsPipeline = VK_NULL_HANDLE;
|
||
VkPipelineLayout m_pipelineLayout = VK_NULL_HANDLE;
|
||
VkDescriptorSetLayout m_descriptorSetLayout = VK_NULL_HANDLE;
|
||
vector<VkDescriptorSet> m_descriptor_sets_left;
|
||
//vector<VkDescriptorSet> m_descriptor_sets_right;
|
||
const uint32_t kTextureInit = 1;
|
||
const uint32_t kTextureMax = 20;
|
||
vector<Texture> m_texs_left;
|
||
|
||
map<string, int> motion_list_map;
|
||
|
||
|
||
//vector<Texture> m_texs_right;
|
||
//bool cur_left = true;
|
||
|
||
//vector<Texture>* _cur_texs;
|
||
vector<Texture> m_texs_thick;
|
||
std::vector<unsigned char> dummy_data;
|
||
unsigned dummy_w = 4096;
|
||
unsigned dummy_h = 2048;
|
||
|
||
std::vector<TextureLoadingVertexStructure> obj_vertices;
|
||
std::vector<uint32_t> obj_indices;
|
||
std::map<int, int> obj_vertices_map;
|
||
std::map<int, int> vertices_map_3dmax;
|
||
|
||
std::mutex mtx_point;
|
||
std::mutex changeMotionMtx;
|
||
long long last_update_time;
|
||
|
||
VkBuffer uniform_buffer_vs;
|
||
VmaAllocation uniform_buffer_allocation; // 需要这个来管理内存
|
||
void* uniform_buffer_mapped = nullptr; // 映射的内存指针
|
||
void createUniformBuffer();
|
||
|
||
|
||
|
||
|
||
//float myFloatValue = 1.5f;
|
||
// 「业务原始值」缓存：SetInitArg 把 InitArg 拷到这里，render() 每帧读出来再
|
||
// 缩放 + 注入屏幕几何。注意 ux/uy 由 motion 帧每帧写入，也保留在这里。
|
||
PushConstants pushConstants;
|
||
|
||
// ===== 屏幕分辨率适配 =====
|
||
// 原 SDK 把屏幕写死 480x480，所有业务参数都按 480 像素画布给。现在改成支持
|
||
// 任意分辨率：把"屏幕中央 min(w,h) 边长的正方形"作为设计画布，画布外是黑边。
|
||
// 业务参数语义不变，SDK 在 render() 里按 (canvas_size / 480) 缩放成屏幕物理
|
||
// 像素后再推给 shader。
|
||
// m_canvasSize = min(swapChainExtent.w, swapChainExtent.h)
|
||
// m_screenW / m_screenH = swapChainExtent.w / swapChainExtent.h
|
||
// updateCanvasMetrics() 由 render() 在每次 vkCmdPushConstants 之前调用。
|
||
// 因为 swapchain 重建会走 onWindowLost -> onWindowInit ->
|
||
// recreatePipelinesForSwapchain，而 render() 永远在 drawFrame 里被调，所以
|
||
// 这套数值天然会跟随 swapchain 变化。
|
||
float m_canvasSize = 480.0f;
|
||
float m_screenW = 480.0f;
|
||
float m_screenH = 480.0f;
|
||
void updateCanvasMetrics();
|
||
|
||
// ===== 相机帧元信息适配 =====
|
||
// 不同手机摄像头物理分辨率和 sensor orientation 都不一样，CameraX 给的图像
|
||
// 既不一定是精确 4:3，也不一定是「正立」朝向。SDK 在这里缓存最近一帧的
|
||
// (raw_aspect, rotation)，render() 时塞进 push constant 让 shader 自己处理：
|
||
// - bg.vert 按 m_cameraRotation 旋转 UV，让画面在屏幕上方向正确
|
||
// - bg.vert 按 m_cameraAspect 自适应 cover 画布，避免拉扁/拉长
|
||
// 由 processCameraFrame() 根据每次相机帧实际 width / height / rotation 更新。
|
||
// 第一帧之前用安全默认值（4:3 + 90°，对应大部分 Android 后置主摄竖屏行为）。
|
||
float m_cameraAspect = 4.0f / 3.0f;
|
||
float m_cameraRotation = 90.0f;
|
||
// 镜像标志缓存：与 m_cameraAspect 同写者（相机分析线程），由 render() 拷到
|
||
// push constant 的 mirror_x 字段。前置=1.0，后置=0.0。
|
||
float m_mirrorX = 0.0f;
|
||
public:
|
||
// JNI 入口（main.cpp）调用，用来在每帧上传相机纹理之前同步元信息。
|
||
// 内部会：
|
||
// 1) 检测 width × height 与已有 tex_bg 是否一致，不一致就 destroy + recreate；
|
||
// 2) 更新 m_cameraAspect / m_cameraRotation / m_mirrorX；
|
||
// 3) 调用 base 的 processWithVulkan 走 staging buffer 上传。
|
||
// mirrorX 来自 Java 层 lensFacing == LENS_FACING_FRONT，表示是否需要镜子效果。
|
||
// 必须由唯一的相机分析线程调用（FaceActivity backgroundExecutor），
|
||
// 内部依赖 createTextureMtx 序列化与 drawFrame / cleanup 的并发。
|
||
void processCameraFrame(uint8_t* data, int width, int height,
|
||
int rowStride, size_t dataSize, int rotation,
|
||
bool mirrorX);
|
||
private:
|
||
|
||
|
||
|
||
void create_pipelines_bg();
|
||
void setup_descriptor_set_layout_bg();
|
||
void setup_descriptor_set_bg();
|
||
// 把 m_descriptor_set_bg 重新绑定到当前的 tex_bg.view / sampler。
|
||
// 当相机分辨率变化导致 tex_bg 被 destroy + recreate 后，新建的 image
|
||
// 有新的 VkImageView 句柄，旧的 descriptor set 里缓存的 handle 失效，
|
||
// 必须 vkUpdateDescriptorSets 让 descriptor 指向新 view。该函数不重新
|
||
// 分配 descriptor set，仅刷新绑定。
|
||
void refresh_descriptor_set_bg();
|
||
VkPipeline m_graphicsPipeline_bg = VK_NULL_HANDLE;
|
||
VkPipelineLayout m_pipelineLayout_bg = VK_NULL_HANDLE;
|
||
VkDescriptorSetLayout m_descriptorSetLayout_bg = VK_NULL_HANDLE;
|
||
VkDescriptorSet m_descriptor_set_bg = VK_NULL_HANDLE;
|
||
|
||
bool _secondfaceAppInited = true;
|
||
public:
|
||
void clearnSecondFaceApp();
|
||
void Start();
|
||
void Stop();
|
||
|
||
// Called from main thread in response to APP_CMD_TERM_WINDOW.
|
||
// Serializes against JNI callbacks (processImageNative / passDataToNative /
|
||
// changeMotionList) by taking all three FaceApp mutexes + createTextureMtx,
|
||
// then tears down window-dependent Vulkan objects via Application.
|
||
void onWindowLost();
|
||
|
||
// Called from main thread in response to APP_CMD_INIT_WINDOW.
|
||
// Does the full first-time initVulkan() on the very first call, and a
|
||
// lightweight swapchain/surface rebuild on subsequent calls. When the
|
||
// new swapchain's extent or format differs from the previous one, the
|
||
// FaceApp pipelines (baked with static viewport / old renderPass) are
|
||
// also destroyed and recreated via recreatePipelinesForSwapchain().
|
||
void onWindowInit();
|
||
|
||
// Destroy and recreate m_graphicsPipeline / m_graphicsPipeline_bg so they
|
||
// match the current renderPass and swapChainExtent. Descriptor set layouts,
|
||
// pipeline layouts, vertex/index buffers, uniform buffers and textures are
|
||
// all kept. Must be called with device idle and the FaceApp mutexes held.
|
||
void recreatePipelinesForSwapchain();
|
||
void loadMotionThread();
|
||
std::thread worker_;
|
||
bool _isLoadMotion = false;
|
||
bool _isChangeMostion = false;
|
||
Callback _callback_loadfinish;
|
||
AnimationFinishedCallback _animationFinishedCallback;
|
||
bool _animationLoop = true;
|
||
string preLoadMotionList(string motion_list_str, Callback callback);
|
||
Motion getMotionByName(string name);
|
||
map<string, Motion>_loadMotionMap;
|
||
vector<Motion> _curLoadMotionList;
|
||
vector<Motion>_curMotions;
|
||
void changeMotionList(vector<string> motions, AnimationFinishedCallback callback, bool loop);
|
||
|
||
//void changeMotion(Motion& motion);
|
||
InitArg _initArg;
|
||
//Motion _curMotion;
|
||
//string _curMotion_type;
|
||
//int _curMotion_png_size;
|
||
//Motion _nextMotion;
|
||
//int _nextMotionIndex;
|
||
|
||
//enum MotionState {
|
||
// ready,
|
||
// loading_next_motion,
|
||
// load_next_motion_finished,
|
||
//};
|
||
|
||
//MotionState _motionState = ready;
|
||
|
||
int _curFrameIndex = 0;
|
||
int _curMotionIndex = 0;
|
||
void SetInitArg(const char* arg);
|
||
|
||
void drawFrame(long long frameTime)override;
|
||
|
||
void destroyTexture(VkDevice device, Texture& texture);
|
||
void cleanupResources(VkDevice device, VmaAllocator allocator);
|
||
|
||
bool _running = false;
|
||
|
||
bool _playMotion = true;
|
||
void StopMotion();
|
||
|
||
void ResumeMotion();
|
||
bool _faceAppInited = false;
|
||
|
||
};
|
||
|
||
#endif
|