#ifndef __FaceApp_H__ #define __FaceApp_H__ #include "Application.h" #include #include "../third_party/vma/include/vk_mem_alloc.h" #include #include #include #include #include "nlohmann/json.hpp" using namespace std; using json = nlohmann::json; struct Frame { string name; float x; float y; NLOHMANN_DEFINE_TYPE_INTRUSIVE(Frame, name, x, y) }; struct Motion { string name; vector< Frame> frames; string getMotionId() { return name; } NLOHMANN_DEFINE_TYPE_INTRUSIVE(Motion, name, frames) }; struct MotionList { vector motions; NLOHMANN_DEFINE_TYPE_INTRUSIVE(MotionList, motions) }; struct InitArg { int action_fps; float zoom; float r; float g; float b; float radius; float offset_x; float offset_y; //Motion motion; NLOHMANN_DEFINE_TYPE_INTRUSIVE(InitArg, action_fps, zoom, r, g, b, radius, offset_x, offset_y); }; // 推送给 shader 的常量。前 9 个 float 是「业务原始值」,单位以 480x480 设计画布 // 为基准(向后兼容):业务给 radius=240 / offset_x=200 / offset_y=-200 等都按这个画布 // 思考。后 3 个 float 是 SDK 在每次 render() 时根据当前 swapchain 尺寸计算并写入的 // 「画布几何」,shader 通过它把「画布坐标」映射到「屏幕坐标」并把圆形 mask 中心 // 锚定在屏幕中心。 // // ★ 字段顺序和命名必须和 GLSL 中的 layout(push_constant) 块严格一致,否则会读到 // 错位数据。修改时四个 shader (bg.vert / bg.frag / texture.vert / texture.frag) // 都要同步更新。 struct PushConstants { float zoom = 0.5f; float r = 0; float g = 0; float b = 0; float radius = 240; float ux = 0; float uy = 0; float offset_x = 0; float offset_y = 0; // 由 SDK 在 FaceApp::render() 内每帧写入,业务无感。 // canvas_size = min(swapchain.w, swapchain.h),单位:物理像素 // screen_w / screen_h = swapchain 当前实际尺寸,单位:物理像素 // 设计画布始终居中铺在屏幕中央正方形区域;外围是 letterbox 黑边。 float canvas_size = 480.0f; float screen_w = 480.0f; float screen_h = 480.0f; // 相机帧元信息,由 SDK 在收到第一帧 / 帧尺寸变化时更新。shader 用来: // 1) 把相机帧(任意 raw aspect)正确 cover 到 1:1 画布; // 2) 按 camera_rotation 旋转 UV,让画面在屏幕上正立——这是不同手机的 // sensor orientation 不同导致画面侧着 / 倒着的根因。 // camera_aspect = camera_raw_w / camera_raw_h(典型 4:3 = 1.333…) // camera_rotation = CameraX 的 ImageInfo.getRotationDegrees(), // 表示「图像顺时针旋转多少度才能正立」,离散 0/90/180/270。 // 默认值 4/3 + 90° 是兼容老 hardcode 行为(大部分 Android 手机后置主摄竖屏)。 float camera_aspect = 4.0f / 3.0f; float camera_rotation = 90.0f; // 镜像标志:1.0 表示前置摄像头需要"镜子效果",所有顶点 shader 会把 // gl_Position.x 翻转一次(NDC 水平翻转)。配合 FaceLandmarkerHelper 在 // bitmap 阶段做的 postScale(-1,1),bg 与 face 同步翻转、互相对齐。 // 后置摄像头此值为 0.0,shader 短路无开销。 // 用 float 而不是 bool/int 是为了和 Vulkan push constant 4-byte 对齐 + GLSL // 端 layout 简单一致;shader 里 `1.0 - 2.0 * mirror_x` 实现无分支翻转。 float mirror_x = 0.0f; }; using Callback = std::function; using AnimationFinishedCallback = std::function; class FaceApp :public Application { public: FaceApp(/* args */); ~FaceApp(); struct { glm::mat4 projection; glm::mat4 model; glm::vec4 view_pos; float lod_bias = 0.0f; } ubo_vs; void initVulkan() override; void render(VkCommandBuffer commandBuffer, long long frameTime) override; static FaceApp* Get() { return faceIns; } void update_face_vertex_buffer(float* pos, int pointCount); virtual bool isInited() override { return _applicationInited && _faceAppInited && _sceondInited; } virtual void cleanup() override; Texture tex_bg = {}; private: glm::vec3 rotation = glm::vec3(); glm::vec3 camera_pos = glm::vec3(); private: static FaceApp* faceIns; VmaAllocator allocator = nullptr; bool LoadOBJ(const std::string& filename,std::vector& vertices, std::vector& indices); void createVmaAllocator(); void update_uniform_buffers(); void createVertexBuffer(); void uploadVertexData(); VkDescriptorPool descriptor_pool = VK_NULL_HANDLE; void create_face_pipelines(); void setup_descriptor_pool(); void setup_descriptor_set_layout(); void setup_descriptor_set(); void update_descriptor_set(vector& texs, vector& descriptSet); const bool kThick = false; void copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size); // 顶点缓冲区相关 VkBuffer m_vertexBuffer = VK_NULL_HANDLE; VmaAllocation m_vertexBufferAllocation = VK_NULL_HANDLE; VkBuffer m_stagingBuffer = VK_NULL_HANDLE; VmaAllocation m_stagingBufferAllocation = VK_NULL_HANDLE; // 索引缓冲区相关 VkBuffer m_indexBuffer = VK_NULL_HANDLE; VmaAllocation m_indexBufferAllocation = VK_NULL_HANDLE; // 渲染管线和描述符 VkPipeline m_graphicsPipeline = VK_NULL_HANDLE; VkPipelineLayout m_pipelineLayout = VK_NULL_HANDLE; VkDescriptorSetLayout m_descriptorSetLayout = VK_NULL_HANDLE; vector m_descriptor_sets_left; //vector m_descriptor_sets_right; const uint32_t kTextureInit = 1; const uint32_t kTextureMax = 20; vector m_texs_left; map motion_list_map; //vector m_texs_right; //bool cur_left = true; //vector* _cur_texs; vector m_texs_thick; std::vector dummy_data; unsigned dummy_w = 4096; unsigned dummy_h = 2048; std::vector obj_vertices; std::vector obj_indices; std::map obj_vertices_map; std::map vertices_map_3dmax; std::mutex mtx_point; std::mutex changeMotionMtx; long long last_update_time; VkBuffer uniform_buffer_vs; VmaAllocation uniform_buffer_allocation; // 需要这个来管理内存 void* uniform_buffer_mapped = nullptr; // 映射的内存指针 void createUniformBuffer(); //float myFloatValue = 1.5f; // 「业务原始值」缓存:SetInitArg 把 InitArg 拷到这里,render() 每帧读出来再 // 缩放 + 注入屏幕几何。注意 ux/uy 由 motion 帧每帧写入,也保留在这里。 PushConstants pushConstants; // ===== 屏幕分辨率适配 ===== // 原 SDK 把屏幕写死 480x480,所有业务参数都按 480 像素画布给。现在改成支持 // 任意分辨率:把"屏幕中央 min(w,h) 边长的正方形"作为设计画布,画布外是黑边。 // 业务参数语义不变,SDK 在 render() 里按 (canvas_size / 480) 缩放成屏幕物理 // 像素后再推给 shader。 // m_canvasSize = min(swapChainExtent.w, swapChainExtent.h) // m_screenW / m_screenH = swapChainExtent.w / swapChainExtent.h // updateCanvasMetrics() 由 render() 在每次 vkCmdPushConstants 之前调用。 // 因为 swapchain 重建会走 onWindowLost -> onWindowInit -> // recreatePipelinesForSwapchain,而 render() 永远在 drawFrame 里被调,所以 // 这套数值天然会跟随 swapchain 变化。 float m_canvasSize = 480.0f; float m_screenW = 480.0f; float m_screenH = 480.0f; void updateCanvasMetrics(); // ===== 相机帧元信息适配 ===== // 不同手机摄像头物理分辨率和 sensor orientation 都不一样,CameraX 给的图像 // 既不一定是精确 4:3,也不一定是「正立」朝向。SDK 在这里缓存最近一帧的 // (raw_aspect, rotation),render() 时塞进 push constant 让 shader 自己处理: // - bg.vert 按 m_cameraRotation 旋转 UV,让画面在屏幕上方向正确 // - bg.vert 按 m_cameraAspect 自适应 cover 画布,避免拉扁/拉长 // 由 processCameraFrame() 根据每次相机帧实际 width / height / rotation 更新。 // 第一帧之前用安全默认值(4:3 + 90°,对应大部分 Android 后置主摄竖屏行为)。 float m_cameraAspect = 4.0f / 3.0f; float m_cameraRotation = 90.0f; // 镜像标志缓存:与 m_cameraAspect 同写者(相机分析线程),由 render() 拷到 // push constant 的 mirror_x 字段。前置=1.0,后置=0.0。 float m_mirrorX = 0.0f; public: // JNI 入口(main.cpp)调用,用来在每帧上传相机纹理之前同步元信息。 // 内部会: // 1) 检测 width × height 与已有 tex_bg 是否一致,不一致就 destroy + recreate; // 2) 更新 m_cameraAspect / m_cameraRotation / m_mirrorX; // 3) 调用 base 的 processWithVulkan 走 staging buffer 上传。 // mirrorX 来自 Java 层 lensFacing == LENS_FACING_FRONT,表示是否需要镜子效果。 // 必须由唯一的相机分析线程调用(FaceActivity backgroundExecutor), // 内部依赖 createTextureMtx 序列化与 drawFrame / cleanup 的并发。 void processCameraFrame(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX); private: void create_pipelines_bg(); void setup_descriptor_set_layout_bg(); void setup_descriptor_set_bg(); // 把 m_descriptor_set_bg 重新绑定到当前的 tex_bg.view / sampler。 // 当相机分辨率变化导致 tex_bg 被 destroy + recreate 后,新建的 image // 有新的 VkImageView 句柄,旧的 descriptor set 里缓存的 handle 失效, // 必须 vkUpdateDescriptorSets 让 descriptor 指向新 view。该函数不重新 // 分配 descriptor set,仅刷新绑定。 void refresh_descriptor_set_bg(); VkPipeline m_graphicsPipeline_bg = VK_NULL_HANDLE; VkPipelineLayout m_pipelineLayout_bg = VK_NULL_HANDLE; VkDescriptorSetLayout m_descriptorSetLayout_bg = VK_NULL_HANDLE; VkDescriptorSet m_descriptor_set_bg = VK_NULL_HANDLE; bool _secondfaceAppInited = true; public: void clearnSecondFaceApp(); void Start(); void Stop(); // Called from main thread in response to APP_CMD_TERM_WINDOW. // Serializes against JNI callbacks (processImageNative / passDataToNative / // changeMotionList) by taking all three FaceApp mutexes + createTextureMtx, // then tears down window-dependent Vulkan objects via Application. void onWindowLost(); // Called from main thread in response to APP_CMD_INIT_WINDOW. // Does the full first-time initVulkan() on the very first call, and a // lightweight swapchain/surface rebuild on subsequent calls. When the // new swapchain's extent or format differs from the previous one, the // FaceApp pipelines (baked with static viewport / old renderPass) are // also destroyed and recreated via recreatePipelinesForSwapchain(). void onWindowInit(); // Destroy and recreate m_graphicsPipeline / m_graphicsPipeline_bg so they // match the current renderPass and swapChainExtent. Descriptor set layouts, // pipeline layouts, vertex/index buffers, uniform buffers and textures are // all kept. Must be called with device idle and the FaceApp mutexes held. void recreatePipelinesForSwapchain(); void loadMotionThread(); std::thread worker_; bool _isLoadMotion = false; bool _isChangeMostion = false; Callback _callback_loadfinish; AnimationFinishedCallback _animationFinishedCallback; bool _animationLoop = true; string preLoadMotionList(string motion_list_str, Callback callback); Motion getMotionByName(string name); map_loadMotionMap; vector _curLoadMotionList; vector_curMotions; void changeMotionList(vector motions, AnimationFinishedCallback callback, bool loop); //void changeMotion(Motion& motion); InitArg _initArg; //Motion _curMotion; //string _curMotion_type; //int _curMotion_png_size; //Motion _nextMotion; //int _nextMotionIndex; //enum MotionState { // ready, // loading_next_motion, // load_next_motion_finished, //}; //MotionState _motionState = ready; int _curFrameIndex = 0; int _curMotionIndex = 0; void SetInitArg(const char* arg); void drawFrame(long long frameTime)override; void destroyTexture(VkDevice device, Texture& texture); void cleanupResources(VkDevice device, VmaAllocator allocator); bool _running = false; bool _playMotion = true; void StopMotion(); void ResumeMotion(); bool _faceAppInited = false; }; #endif