#include "FaceApp.h" #include "hardcode_data.h" #include "lodepng.h" #include #include #include #include #ifndef _WIN32 #include "../app/src/main/cpp/DebugLog.h" #define FACE_DBG_LOG(...) DebugLog::log(__VA_ARGS__) #define FACE_DBG_LOG_THROTTLED(key, ...) DebugLog::log_throttled(key, __VA_ARGS__) #else #define FACE_DBG_LOG(...) ((void)0) #define FACE_DBG_LOG_THROTTLED(key, ...) ((void)0) #endif FaceApp* FaceApp::faceIns = nullptr; FaceApp::FaceApp(/* args */) { faceIns = this; } FaceApp::~FaceApp() { if(worker_.joinable()) { worker_.join(); } } void FaceApp::Stop() { FACE_DBG_LOG("FaceApp::Stop called (_running=%d worker_joinable=%d)", (int)_running, (int)worker_.joinable()); _running = false; if(worker_.joinable()) { worker_.join(); } FACE_DBG_LOG("FaceApp::Stop done"); } void ReceiveFacePoint(float* pos, int pointCount, int width, int height) { FaceApp* self = FaceApp::Get(); if (self == nullptr) { FACE_DBG_LOG_THROTTLED("ReceiveFacePoint.noFaceApp", "ReceiveFacePoint dropped: FaceApp::Get() == nullptr"); return; } self->update_face_vertex_buffer(pos, pointCount); } // 定义帧数据结构 struct FrameData { uint8_t* data; int width; int height; int rowStride; size_t dataSize; int rotation; // CameraX 给的 rotationDegrees,用于 shader 端 UV 旋转 bool mirrorX; // 前置摄像头时为 true,shader 在 gl_Position.x 翻转一次 FrameData(uint8_t* d, int w, int h, int rs, size_t ds, int rot, bool mx) : data(nullptr), width(w), height(h), rowStride(rs), dataSize(ds), rotation(rot), mirrorX(mx) { if (d && ds > 0) { data = new uint8_t[dataSize]; memcpy(data, d, dataSize); } } ~FrameData() { if (data) { delete[] data; data = nullptr; } } FrameData(const FrameData&) = delete; FrameData& operator=(const FrameData&) = delete; FrameData(FrameData&& other) noexcept : data(other.data), width(other.width), height(other.height), rowStride(other.rowStride), dataSize(other.dataSize), rotation(other.rotation), mirrorX(other.mirrorX) { other.data = nullptr; } FrameData& operator=(FrameData&& other) noexcept { if (this != &other) { if (data) delete[] data; data = other.data; width = other.width; height = other.height; rowStride = other.rowStride; dataSize = other.dataSize; rotation = other.rotation; mirrorX = other.mirrorX; other.data = nullptr; } return *this; } }; // 全局队列和互斥锁 std::queue frameQueue; std::mutex queueMutex; const int MAX_QUEUE_SIZE = 4; void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX) { if (!FaceApp::Get()->isInited()) { return; } std::lock_guard lock(queueMutex); frameQueue.emplace(data, width, height, rowStride, dataSize, rotation, mirrorX); // 如果队列大小达到阈值,开始处理最旧的一帧 if (frameQueue.size() >= MAX_QUEUE_SIZE) { FrameData& oldestFrame = frameQueue.front(); // 走 FaceApp::processCameraFrame:相比 base::processWithVulkan,它多做两件 // 事 —— ① 检测帧尺寸变化时 destroy/recreate tex_bg(不同手机或 CameraX // 重建 session 后第一帧尺寸可能与之前不同);② 缓存 raw_aspect / rotation // / mirrorX 到 m_cameraAspect / m_cameraRotation / m_mirrorX,由 render() // 推到 push constant。 FaceApp::Get()->processCameraFrame( oldestFrame.data, oldestFrame.width, oldestFrame.height, oldestFrame.rowStride, oldestFrame.dataSize, oldestFrame.rotation, oldestFrame.mirrorX ); frameQueue.pop(); } } bool FaceApp::LoadOBJ(const std::string& filename, std::vector& vertices, std::vector& indices) { // 临时存储从OBJ文件读取的原始数据 std::vector temp_positions; std::vector temp_texcoords; std::vector temp_normals; // 用于处理顶点索引 std::vector vertexIndices, uvIndices, normalIndices; std::vector data = readFile(filename); // 将 vector 转换为以 null 结尾的字符串(安全做法) std::string content(data.begin(), data.end()); std::istringstream iss(content); // 用字符串创建字符串流 std::string line; while (std::getline(iss, line)) { // 跳过空行和注释行 if (line.empty() || line[0] == '#') { continue; } std::istringstream iss(line); std::string type; iss >> type; if (type == "v") { // 顶点位置 float x, y, z; iss >> x >> y >> z; temp_positions.push_back(x); temp_positions.push_back(y); temp_positions.push_back(z); } else if (type == "vt") { // 纹理坐标 float u, v; iss >> u >> v; temp_texcoords.push_back(u); temp_texcoords.push_back(1 - v); } else if (type == "vn") { // 法线 float nx, ny, nz; iss >> nx >> ny >> nz; temp_normals.push_back(nx); temp_normals.push_back(ny); temp_normals.push_back(nz); } else if (type == "f") { // 面(三角形) std::string vertex1, vertex2, vertex3; iss >> vertex1 >> vertex2 >> vertex3; // 处理每个顶点的索引 for (const std::string& vertex : { vertex1, vertex2, vertex3 }) { std::istringstream viss(vertex); std::string v, vt, vn; // 解析顶点索引格式:v/vt/vn 或 v//vn 或 v std::getline(viss, v, '/'); std::getline(viss, vt, '/'); std::getline(viss, vn, '/'); int posIndex = std::stoi(v) - 1; // OBJ索引从1开始 int texIndex = -1, normIndex = -1; if (!vt.empty()) texIndex = std::stoi(vt) - 1; if (!vn.empty()) normIndex = std::stoi(vn) - 1; vertexIndices.push_back(posIndex); uvIndices.push_back(texIndex); normalIndices.push_back(normIndex); } } } // 创建顶点数据 vertices.clear(); indices.clear(); // 用于去重的哈希映射 std::map vertexMap; for (size_t i = 0; i < vertexIndices.size(); i++) { int posIndex = vertexIndices[i]; int texIndex = uvIndices[i]; int normIndex = normalIndices[i]; // 创建唯一标识符 std::string vertexKey = std::to_string(posIndex) + "/" + std::to_string(texIndex) + "/" + std::to_string(normIndex); // 检查是否已经存在相同的顶点 if (vertexMap.find(vertexKey) != vertexMap.end()) { // 使用现有顶点的索引 indices.push_back(vertexMap[vertexKey]); } else { // 创建新顶点 TextureLoadingVertexStructure vertex; // 设置位置 if (posIndex >= 0 && posIndex * 3 + 2 < temp_positions.size()) { vertex.pos[0] = temp_positions[posIndex * 3]; vertex.pos[1] = temp_positions[posIndex * 3 + 1]; vertex.pos[2] = temp_positions[posIndex * 3 + 2]; } else { vertex.pos[0] = vertex.pos[1] = vertex.pos[2] = 0.0f; } // 设置纹理坐标 if (texIndex >= 0 && texIndex * 2 + 1 < temp_texcoords.size()) { vertex.uv[0] = temp_texcoords[texIndex * 2]; vertex.uv[1] = temp_texcoords[texIndex * 2 + 1]; } else { vertex.uv[0] = vertex.uv[1] = 0.0f; } // 设置法线 if (normIndex >= 0 && normIndex * 3 + 2 < temp_normals.size()) { vertex.normal[0] = temp_normals[normIndex * 3]; vertex.normal[1] = temp_normals[normIndex * 3 + 1]; vertex.normal[2] = temp_normals[normIndex * 3 + 2]; } else { vertex.normal[0] = vertex.normal[1] = 0.0f; vertex.normal[2] = 1.0f; // 默认法线 } // 添加新顶点并记录索引 uint32_t newIndex = static_cast(vertices.size()); vertices.push_back(vertex); indices.push_back(newIndex); obj_vertices_map[newIndex] = posIndex; vertexMap[vertexKey] = newIndex; } } return true; } void FaceApp::create_face_pipelines() { VkPipelineInputAssemblyStateCreateInfo input_assembly_state{}; input_assembly_state.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; input_assembly_state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; input_assembly_state.flags = 0; input_assembly_state.primitiveRestartEnable = VK_FALSE; VkPipelineRasterizationStateCreateInfo rasterization_state{}; rasterization_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; rasterization_state.polygonMode = VK_POLYGON_MODE_FILL; // ★ 关键:face 必须 NONE。原因: // 1. face mesh 是 2D 平面网格(texture.vert 让所有顶点 z=0.5),不存在 // 自遮挡,没必要 cull; // 2. 前置摄像头镜子效果是在 vertex shader 里通过 NDC.x 翻转实现的 // (`position.x *= (1.0 - 2.0 * pc.mirror_x)`)。X 翻转会把所有三角 // 形的卷绕方向从 CCW 变 CW。如果这里 cullMode=BACK_BIT + frontFace= // CCW,前置时整张脸会被 cull,face 完全不可见——这正是历史踩坑点。 // 所以请勿改回 BACK_BIT。如果未来要再加 cull,必须配两套 pipeline(前置 // 用 CW、后置用 CCW),或在前置时改成 frontFace=CW,否则一定会复现这个 bug。 rasterization_state.cullMode = VK_CULL_MODE_NONE; rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; rasterization_state.flags = 0; rasterization_state.depthClampEnable = VK_FALSE; rasterization_state.lineWidth = 1.0f; VkPipelineColorBlendAttachmentState colorBlendAttachment{}; colorBlendAttachment.blendEnable = VK_TRUE; colorBlendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; // 常用的Alpha混合公式 colorBlendAttachment.srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA; colorBlendAttachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; colorBlendAttachment.colorBlendOp = VK_BLEND_OP_ADD; colorBlendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE; colorBlendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; colorBlendAttachment.alphaBlendOp = VK_BLEND_OP_ADD; VkPipelineColorBlendStateCreateInfo colorBlending{}; colorBlending.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; colorBlending.logicOpEnable = VK_FALSE; colorBlending.attachmentCount = 1; colorBlending.pAttachments = &colorBlendAttachment; // face 是 2D 平面网格(texture.vert 让顶点 z=0.5 一致),完全不需要 depth // test/write。bg 管线本来也是关 depth 的;保持两边一致,让"先 bg 再 face" // 的覆盖顺序由 draw 顺序而非 depth 决定,避免 reversed-depth(GREATER)+ // renderpass 没显式 clear depth attachment 时引发的"face fragment 全部 // 被 depth test fail"的隐性 bug。 VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {}; depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; depth_stencil_state.depthTestEnable = VK_FALSE; depth_stencil_state.depthWriteEnable = VK_FALSE; depth_stencil_state.depthCompareOp = VK_COMPARE_OP_ALWAYS; depth_stencil_state.front = depth_stencil_state.back; depth_stencil_state.back.compareOp = VK_COMPARE_OP_ALWAYS; VkPipelineViewportStateCreateInfo viewport_state{}; viewport_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; viewport_state.viewportCount = 1; viewport_state.scissorCount = 1; viewport_state.flags = 0; VkPipelineMultisampleStateCreateInfo multisample_state{}; multisample_state.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; multisample_state.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; multisample_state.flags = 0; // 动态定义视口和剪裁,暂时用不到 //std::vector dynamic_state_enables = { // VK_DYNAMIC_STATE_VIEWPORT, // VK_DYNAMIC_STATE_SCISSOR }; //VkPipelineDynamicStateCreateInfo dynamic_state{}; //dynamic_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; //dynamic_state.pDynamicStates = dynamic_state_enables.data(); //dynamic_state.dynamicStateCount = static_cast(dynamic_state_enables.size()); //dynamic_state.flags = 0; // 视口状态 VkViewport viewport{}; viewport.x = 0.0f; viewport.y = 0.0f; viewport.width = (float)swapChainExtent.width; viewport.height = (float)swapChainExtent.height; viewport.minDepth = 0.0f; viewport.maxDepth = 1.0f; VkRect2D scissor{}; scissor.offset = { 0, 0 }; scissor.extent = swapChainExtent; VkPipelineViewportStateCreateInfo viewportState{}; viewportState.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; viewportState.viewportCount = 1; viewportState.pViewports = &viewport; viewportState.scissorCount = 1; viewportState.pScissors = &scissor; std::vector vertShaderCode; std::vector fragShaderCode; if (kThick) { vertShaderCode = readFile("shaders/texture_thick.vert.spv"); fragShaderCode = readFile("shaders/texture_thick.frag.spv"); } else { vertShaderCode = readFile("shaders/texture.vert.spv"); fragShaderCode = readFile("shaders/texture.frag.spv"); } VkShaderModule vertShaderModule = createShaderModule(this->device, vertShaderCode); VkShaderModule fragShaderModule = createShaderModule(this->device, fragShaderCode); VkPipelineShaderStageCreateInfo vertShaderStageInfo{}; vertShaderStageInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; vertShaderStageInfo.stage = VK_SHADER_STAGE_VERTEX_BIT; vertShaderStageInfo.module = vertShaderModule; vertShaderStageInfo.pName = "main"; VkPipelineShaderStageCreateInfo fragShaderStageInfo{}; fragShaderStageInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; fragShaderStageInfo.stage = VK_SHADER_STAGE_FRAGMENT_BIT; fragShaderStageInfo.module = fragShaderModule; fragShaderStageInfo.pName = "main"; VkPipelineShaderStageCreateInfo shader_stages[] = { vertShaderStageInfo, fragShaderStageInfo }; // Vertex bindings and attributes std::vector vertex_input_bindings{};; VkVertexInputBindingDescription vertex_input_binding_description{}; vertex_input_binding_description.binding = 0; vertex_input_binding_description.stride = sizeof(TextureLoadingVertexStructure); vertex_input_binding_description.inputRate = VK_VERTEX_INPUT_RATE_VERTEX; vertex_input_bindings.push_back(vertex_input_binding_description); std::vector vertex_input_attributes{}; VkVertexInputAttributeDescription viaPos{}; viaPos.location = 0; viaPos.binding = 0; viaPos.format = VK_FORMAT_R32G32B32_SFLOAT; viaPos.offset = offsetof(TextureLoadingVertexStructure, pos); vertex_input_attributes.push_back(viaPos); VkVertexInputAttributeDescription viaUv{}; viaUv.location = 1; viaUv.binding = 0; viaUv.format = VK_FORMAT_R32G32_SFLOAT; viaUv.offset = offsetof(TextureLoadingVertexStructure, uv); vertex_input_attributes.push_back(viaUv); VkVertexInputAttributeDescription viaNormal{}; viaNormal.location = 2; viaNormal.binding = 0; viaNormal.format = VK_FORMAT_R32G32B32_SFLOAT; viaNormal.offset = offsetof(TextureLoadingVertexStructure, normal); vertex_input_attributes.push_back(viaNormal); VkPipelineVertexInputStateCreateInfo vertex_input_state{}; vertex_input_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); VkGraphicsPipelineCreateInfo pipeline_create_info{}; pipeline_create_info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; pipeline_create_info.layout = m_pipelineLayout; pipeline_create_info.renderPass = renderPass; pipeline_create_info.flags = 0; pipeline_create_info.basePipelineIndex = -1; pipeline_create_info.basePipelineHandle = VK_NULL_HANDLE; pipeline_create_info.pVertexInputState = &vertex_input_state; pipeline_create_info.pInputAssemblyState = &input_assembly_state; pipeline_create_info.pRasterizationState = &rasterization_state; pipeline_create_info.pColorBlendState = &colorBlending; //&color_blend_state; pipeline_create_info.pMultisampleState = &multisample_state; pipeline_create_info.pViewportState = &viewport_state; pipeline_create_info.pDepthStencilState = &depth_stencil_state; //pipeline_create_info.pDynamicState = &dynamic_state; pipeline_create_info.pViewportState = &viewportState; pipeline_create_info.stageCount = 2; pipeline_create_info.pStages = shader_stages; VK_CHECK(vkCreateGraphicsPipelines(device, VK_NULL_HANDLE, 1, &pipeline_create_info, nullptr, &m_graphicsPipeline)); // 销毁着色器模块 vkDestroyShaderModule(device, fragShaderModule, nullptr); vkDestroyShaderModule(device, vertShaderModule, nullptr); } void FaceApp::setup_descriptor_set_layout() { VkDescriptorSetLayoutBinding set_layout_binding_vertex{}; set_layout_binding_vertex.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; set_layout_binding_vertex.stageFlags = VK_SHADER_STAGE_VERTEX_BIT; set_layout_binding_vertex.binding = 0; set_layout_binding_vertex.descriptorCount = 1; VkDescriptorSetLayoutBinding set_layout_binding_fragment{}; set_layout_binding_fragment.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; set_layout_binding_fragment.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; set_layout_binding_fragment.binding = 1; set_layout_binding_fragment.descriptorCount = 1; if (kThick) { // 添加第二个纹理绑定点 VkDescriptorSetLayoutBinding set_layout_binding_fragment1{}; set_layout_binding_fragment1.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; set_layout_binding_fragment1.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; set_layout_binding_fragment1.binding = 2; set_layout_binding_fragment1.descriptorCount = 1; std::vector set_layout_bindings{ set_layout_binding_vertex , set_layout_binding_fragment, set_layout_binding_fragment1 }; VkDescriptorSetLayoutCreateInfo descriptor_layout{}; descriptor_layout.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; descriptor_layout.pBindings = set_layout_bindings.data(); descriptor_layout.bindingCount = static_cast(set_layout_bindings.size()); VK_CHECK(vkCreateDescriptorSetLayout(device, &descriptor_layout, nullptr, &m_descriptorSetLayout)); } else { std::vector set_layout_bindings{ set_layout_binding_vertex , set_layout_binding_fragment }; VkDescriptorSetLayoutCreateInfo descriptor_layout{}; descriptor_layout.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; descriptor_layout.pBindings = set_layout_bindings.data(); descriptor_layout.bindingCount = static_cast(set_layout_bindings.size()); VK_CHECK(vkCreateDescriptorSetLayout(device, &descriptor_layout, nullptr, &m_descriptorSetLayout)); } VkPipelineLayoutCreateInfo pipeline_layout_create_info{}; pipeline_layout_create_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; pipeline_layout_create_info.setLayoutCount = 1; pipeline_layout_create_info.pSetLayouts = &m_descriptorSetLayout; //VkPushConstantRange pushConstantRange{}; //pushConstantRange.stageFlags = VK_SHADER_STAGE_VERTEX_BIT; //pushConstantRange.offset = 0; //pushConstantRange.size = sizeof(float); //pipeline_layout_create_info.pushConstantRangeCount = 1; //pipeline_layout_create_info.pPushConstantRanges = &pushConstantRange; VkPushConstantRange pushConstantRanges[1]; pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT; pushConstantRanges[0].offset = 0; pushConstantRanges[0].size = sizeof(PushConstants); //pushConstantRanges[1].stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; //pushConstantRanges[1].offset = sizeof(float); //pushConstantRanges[1].size = 2*sizeof(float); pipeline_layout_create_info.pushConstantRangeCount = 1; pipeline_layout_create_info.pPushConstantRanges = &pushConstantRanges[0]; VK_CHECK(vkCreatePipelineLayout(device, &pipeline_layout_create_info, nullptr, &m_pipelineLayout)); } void FaceApp::setup_descriptor_pool() { std::vector pool_sizes = { { .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, .descriptorCount = (kTextureMax *2) }, { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .descriptorCount = ((kTextureMax + 1)*2) }, // 如果需要其他类型的描述符,在这里添加 }; VkDescriptorPoolCreateInfo descriptor_pool_info{}; descriptor_pool_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; descriptor_pool_info.poolSizeCount = static_cast(pool_sizes.size()); descriptor_pool_info.pPoolSizes = pool_sizes.data(); descriptor_pool_info.maxSets = kTextureMax + (kTextureMax + 1 + kTextureMax); descriptor_pool_info.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; VK_CHECK(vkCreateDescriptorPool(device, &descriptor_pool_info, nullptr, &descriptor_pool)); } void FaceApp::setup_descriptor_set() { m_descriptor_sets_left.resize(kTextureMax); std::vector layouts(kTextureMax, m_descriptorSetLayout); VkDescriptorSetAllocateInfo alloc_info{}; alloc_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; alloc_info.descriptorPool = descriptor_pool; alloc_info.pSetLayouts = layouts.data(); alloc_info.descriptorSetCount = m_descriptor_sets_left.size(); VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, m_descriptor_sets_left.data())); update_descriptor_set(m_texs_left, m_descriptor_sets_left); //m_descriptor_sets_right.resize(kTextureMax); //VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, m_descriptor_sets_right.data())); //update_descriptor_set(m_texs_left, m_descriptor_sets_right); } void FaceApp::update_descriptor_set(vector& texs, vector& descriptSet) { FACE_DBG_LOG("update_descriptor_set: texs.size=%zu descriptSet.size=%zu", texs.size(), descriptSet.size()); for (int i = 0; i < texs.size(); ++i) { if (texs[i].image == VK_NULL_HANDLE) { break; } VkDescriptorBufferInfo buffer_descriptor{}; buffer_descriptor.buffer = uniform_buffer_vs; buffer_descriptor.range = VK_WHOLE_SIZE; buffer_descriptor.offset = 0; VkDescriptorImageInfo image_descriptor; image_descriptor.imageView = texs[i].view; image_descriptor.sampler = texs[i].sampler; image_descriptor.imageLayout = texs[i].image_layout; VkWriteDescriptorSet write_descriptor_set_uniform{}; write_descriptor_set_uniform.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; write_descriptor_set_uniform.dstSet = descriptSet[i]; write_descriptor_set_uniform.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; write_descriptor_set_uniform.dstBinding = 0; write_descriptor_set_uniform.pBufferInfo = &buffer_descriptor; write_descriptor_set_uniform.descriptorCount = 1; VkWriteDescriptorSet write_descriptor_set_image{}; write_descriptor_set_image.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; write_descriptor_set_image.dstSet = descriptSet[i]; write_descriptor_set_image.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; write_descriptor_set_image.dstBinding = 1; write_descriptor_set_image.pImageInfo = &image_descriptor; write_descriptor_set_image.descriptorCount = 1; if (kThick) { // 第二个纹理描述符 - 假设你的第二个纹理变量名为 tex_demo1 VkDescriptorImageInfo image_descriptor1; image_descriptor1.imageView = m_texs_thick[i].view; image_descriptor1.sampler = m_texs_thick[i].sampler; image_descriptor1.imageLayout = m_texs_thick[i].image_layout; // 添加第二个纹理的写入描述符 VkWriteDescriptorSet write_descriptor_set_image1{}; write_descriptor_set_image1.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; write_descriptor_set_image1.dstSet = descriptSet[i]; write_descriptor_set_image1.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; write_descriptor_set_image1.dstBinding = 2; // 绑定到位置2 write_descriptor_set_image1.pImageInfo = &image_descriptor1; write_descriptor_set_image1.descriptorCount = 1; std::vector write_descriptor_sets = { write_descriptor_set_uniform, write_descriptor_set_image, write_descriptor_set_image1, // 添加第二个纹理 }; vkUpdateDescriptorSets(device, static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0, NULL); } else { std::vector write_descriptor_sets = { write_descriptor_set_uniform, write_descriptor_set_image, }; vkUpdateDescriptorSets(device, static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0, NULL); } } } void FaceApp::render(VkCommandBuffer commandBuffer, long long frameTime) { //Application::render(commandBuffer); // === 分辨率适配:每帧把"业务参数"缩放到当前屏幕物理像素 === // 业务层(MotionManager / InitArg)传进来的 radius / offset_x / offset_y 一直 // 都是按"480x480 设计画布"给的,比如 radius=240 表示在 480 屏上半径 240px // (即 50% 短边)。现在屏幕变成任意分辨率后,要保持视觉一致: // 实际像素 = 业务值 * (画布短边 / 480) // 同时把 canvas/screen 几何写进 push constant,shader 用它们: // 1) 把"画布 NDC ∈ [-1,+1]" 缩到屏幕中央正方形(letterbox); // 2) 把 fragment 里的圆心从写死的 (240,240) 改成屏幕真实中心。 // // 注意 pushConstants 本身保留业务原始值不动(SetInitArg 设进来的), // 这里只把缩放后的副本 pc 推给 GPU。下面 ux/uy 仍然写回 pushConstants, // 是因为它们和分辨率无关,是 motion 帧的纹理坐标。 updateCanvasMetrics(); PushConstants pc = pushConstants; const float k = m_canvasSize / 480.0f; pc.radius *= k; pc.offset_x *= k; pc.offset_y *= k; pc.canvas_size = m_canvasSize; pc.screen_w = m_screenW; pc.screen_h = m_screenH; // 相机帧元信息:由 processCameraFrame 在每帧上传纹理之前更新。第一帧之前 // (还没收到相机数据时)m_cameraAspect/m_cameraRotation 是构造函数设的 // 4:3 + 90° 默认值 —— 这两个值正是 CameraX RATIO_4_3 + 大部分 Android 后置 // 主摄竖屏的典型组合,所以即使首帧 bg 用默认值渲染也不会出现明显错位。 pc.camera_aspect = m_cameraAspect; pc.camera_rotation = m_cameraRotation; // mirror_x: 前置摄像头时 = 1.0,所有顶点 shader 在最后输出 gl_Position 前 // 把 NDC.x 翻转一次,达到镜子效果(用户右手 → 屏幕左侧)。bg 与 face 都 // 同步翻转,对齐保持。后置时 = 0.0,shader 公式 (1 - 2 * mirror_x) = 1 短路。 pc.mirror_x = m_mirrorX; vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout_bg, 0, 1, &m_descriptor_set_bg, 0, NULL); vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_graphicsPipeline_bg); // bg.frag 也读 push constant(r/g/b/radius/screen_w/screen_h)所以 stage 必须 // 同时包含 fragment;老代码这里只写了 VERTEX_BIT,是个隐藏的 spec 违规。 vkCmdPushConstants(commandBuffer, m_pipelineLayout_bg, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc); vkCmdDraw(commandBuffer, 6, 1, 0, 0); // 诊断:把 face 渲染所有"前置门"状态打成一行节流日志。一旦 face 没出现, // 直接看这条就能知道断在哪一关: // _isChangeMostion=0 → 业务还没调 PlayMotionList / changeMotionList // _curMotions=0 → motion list 是空的(getMotionByName 全 miss) // _curMotionIndex 越界 → 业务逻辑或 callback 状态机错了 // delta_last_update >= 2000 → mediapipe / passDataToNative 链路 2s 没动静 const long long now_ms = getCurrentTimeMillis(); const long long delta_lu = now_ms - (long long)last_update_time; FACE_DBG_LOG_THROTTLED("render.face.gate", "render.face gate _isChangeMostion=%d _curMotions=%zu" " _curMotionIndex=%d _curFrameIndex=%d delta_lastUpdate=%lldms" " _playMotion=%d", (int)_isChangeMostion, _curMotions.size(), (int)_curMotionIndex, (int)_curFrameIndex, delta_lu, (int)_playMotion); if (!_isChangeMostion) { return; } if (_curMotions.empty()) { FACE_DBG_LOG_THROTTLED("render.face.emptyMotions", "render.face dropped: _isChangeMostion=1 but _curMotions is empty"); return; } string curMotionName = _curMotions[_curMotionIndex].name; int loadMotionIndex = motion_list_map[curMotionName]; // face draw 前关键参数节流日志,保留少量"任何时候挂了能立刻判断断点"的字段。 FACE_DBG_LOG_THROTTLED("render.face.preDraw", "render.face preDraw motion='%s' loadMotionIdx=%d obj_idx=%zu mirror=%.1f", curMotionName.c_str(), loadMotionIndex, obj_indices.size(), pc.mirror_x); //if (cur_left) //{ vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, &m_descriptor_sets_left[loadMotionIndex], 0, NULL); //} //else //{ // vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, &m_descriptor_sets_right[_curMotionIndex], 0, NULL); //} //float fsValues[3] = { myFloatValue, 0 , 0}; if (_curMotionIndex < _curMotions.size()) { if (_curFrameIndex < _curMotions[_curMotionIndex].frames.size()) { float ux = _curMotions[_curMotionIndex].frames[_curFrameIndex].x; float uy = _curMotions[_curMotionIndex].frames[_curFrameIndex].y; pushConstants.ux = ux; pushConstants.uy = uy; pc.ux = ux; pc.uy = uy; } } //fsValues[1] = 0; //fsValues[2] = 360; vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT| VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(pc), &pc); //vkCmdPushConstants(commandBuffer, m_pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(fsValues), fsValues); vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_graphicsPipeline); VkDeviceSize offsets[1] = { 0 }; VkBuffer vertexBuffers[] = { m_vertexBuffer }; vkCmdBindVertexBuffers(commandBuffer, 0, 1, vertexBuffers, offsets); vkCmdBindIndexBuffer(commandBuffer, m_indexBuffer, 0, VK_INDEX_TYPE_UINT32); #ifdef _WIN32 vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0); #else if (getCurrentTimeMillis() - last_update_time < 2000) { FACE_DBG_LOG_THROTTLED("render.face.draw", "render.face vkCmdDrawIndexed indices=%zu motion=%s frame=%d", obj_indices.size(), curMotionName.c_str(), (int)_curFrameIndex); vkCmdDrawIndexed(commandBuffer, obj_indices.size(), 1, 0, 0, 0); } else { FACE_DBG_LOG_THROTTLED("render.face.heartbeatStale", "render.face SKIP vkCmdDrawIndexed: delta_last_update=%lldms (>= 2000ms)", (long long)(getCurrentTimeMillis() - last_update_time)); } #endif } void FaceApp::createVmaAllocator() { //VmaAllocatorCreateInfo allocatorInfo = {}; //allocatorInfo.physicalDevice = physicalDevice; //allocatorInfo.device = device; //allocatorInfo.instance = instance; //allocatorInfo.vulkanApiVersion = VK_API_VERSION_1_0; //vmaCreateAllocator(&allocatorInfo, &allocator); // 1. 设置 Vulkan 函数指针 VmaVulkanFunctions vulkanFunctions{}; vulkanFunctions.vkGetInstanceProcAddr = vkGetInstanceProcAddr; vulkanFunctions.vkGetDeviceProcAddr = vkGetDeviceProcAddr; // 2. 配置 Allocator VmaAllocatorCreateInfo allocatorInfo{}; allocatorInfo.physicalDevice = physicalDevice; allocatorInfo.device = device; allocatorInfo.instance = instance; allocatorInfo.vulkanApiVersion = VK_API_VERSION_1_0; allocatorInfo.pVulkanFunctions = &vulkanFunctions; VkResult result = vmaCreateAllocator(&allocatorInfo, &allocator); } void FaceApp::initVulkan() { static int s_initVulkanCallCount = 0; ++s_initVulkanCallCount; FACE_DBG_LOG("FaceApp::initVulkan enter, call#%d _applicationInited=%d _faceAppInited=%d _secondfaceAppInited=%d", s_initVulkanCallCount, (int)_applicationInited, (int)_faceAppInited, (int)_secondfaceAppInited); Application::initVulkan(); if (!_faceAppInited) { createVmaAllocator(); LoadOBJ("face_picture_3dmax.obj", obj_vertices, obj_indices); //m_texs_left.resize(kTextureMax); //m_texs_right.resize(kTextureMax); if (kThick) { m_texs_thick.resize(kTextureMax); } std::vector data = readFileUnsignedChar("dummy.png", false); unsigned error = lodepng::decode(dummy_data, dummy_w, dummy_h, data, LCT_RGBA, 8); for (int i = 0; i < kTextureInit; ++i) { //string id_str = std::to_string(i); //loadTexture(dummy_data, dummy_data.size(), dummy_w, dummy_h, m_texs_left[i], true, commandPool, "dummy.png_left_" + id_str); //loadTexture(dummy_data, dummy_data.size(), dummy_w, dummy_h, m_texs_right[i], true, commandPool, "dummy.png_right_" + id_str); if (kThick) { //loadTexture(dummy_data, dummy_data.size(), dummy_w, dummy_h, m_texs_thick[i], true, commandPool, "dummy.png_thick_" + id_str); } } loadTexture("out.png", tex_bg, true, commandPool); createVertexBuffer(); createUniformBuffer(); setup_descriptor_pool(); setup_descriptor_set_layout(); setup_descriptor_set(); create_face_pipelines(); setup_descriptor_set_layout_bg(); setup_descriptor_set_bg(); create_pipelines_bg(); uploadVertexData(); last_update_time = getCurrentTimeMillis(); //changeMotion(_initArg.motion); Start(); _faceAppInited = true; #if _WIN32 std::vector floatArray; std::string& str = HardCodeData::Get().face_result_point_str; std::stringstream ss(str); std::string token; while (std::getline(ss, token, ',')) { floatArray.push_back(std::stof(token)); } //ReceiveFacePoint(floatArray.data(), floatArray.size() / 3, 480, 480); #endif } if (!_secondfaceAppInited) { Start(); _playMotion = true; _secondfaceAppInited = true; } FACE_DBG_LOG("FaceApp::initVulkan exit, call#%d _applicationInited=%d _faceAppInited=%d _secondfaceAppInited=%d", s_initVulkanCallCount, (int)_applicationInited, (int)_faceAppInited, (int)_secondfaceAppInited); } void FaceApp::clearnSecondFaceApp() { FACE_DBG_LOG("FaceApp::clearnSecondFaceApp: _secondfaceAppInited %d -> 0", (int)_secondfaceAppInited); _secondfaceAppInited = false; } void FaceApp::Start() { FACE_DBG_LOG("FaceApp::Start: _running %d -> 1", (int)_running); _running = true; } void FaceApp::onWindowLost() { FACE_DBG_LOG("FaceApp::onWindowLost enter _applicationInited=%d _faceAppInited=%d _sceondInited=%d _secondfaceAppInited=%d _running=%d", (int)_applicationInited, (int)_faceAppInited, (int)_sceondInited, (int)_secondfaceAppInited, (int)_running); // Stop the render loop from touching Vulkan while we tear down. _running = false; // Serialize against JNI callbacks that may concurrently submit GPU work // via commandPool / commandPool_ex (processImageNative -> update texture, // passDataToNative -> update vertex buffer, changeMotionList). std::unique_lock lk_point(mtx_point); std::unique_lock lk_motion(changeMotionMtx); std::unique_lock lk_tex(createTextureMtx); // 同时阻塞所有并发的 GPU 提交(drawFrame / copyBuffer / updateTexture)。 // cleanupForWindowLost() 会执行 vkDeviceWaitIdle 并销毁 swapchain / surface / // semaphores / fences 等窗口相关资源,如果此时有其它线程正在 vkQueueSubmit // 或 vkQueuePresentKHR,会触发 FORTIFY: pthread_mutex_lock called on a // destroyed mutex。这里持锁确保销毁与提交是互斥的。 std::unique_lock lk_pool(poolQueueMtx); Application::cleanupForWindowLost(); _secondfaceAppInited = false; FACE_DBG_LOG("FaceApp::onWindowLost done"); } void FaceApp::onWindowInit() { FACE_DBG_LOG("FaceApp::onWindowInit enter _applicationInited=%d _faceAppInited=%d _sceondInited=%d _secondfaceAppInited=%d", (int)_applicationInited, (int)_faceAppInited, (int)_sceondInited, (int)_secondfaceAppInited); if (!_applicationInited) { // First-time path: go through the full initVulkan pipeline. initVulkan(); } else { // Recovery path after an earlier onWindowLost. Rebuild only the // window-dependent Vulkan objects; keep renderPass / pipelines / // FaceApp GPU resources intact unless the new swapchain is // incompatible (e.g. rotation changed extent). std::unique_lock lk_point(mtx_point); std::unique_lock lk_motion(changeMotionMtx); std::unique_lock lk_tex(createTextureMtx); bool swapchainIncompatible = Application::reinitForNewWindow(); if (swapchainIncompatible && _faceAppInited) { FACE_DBG_LOG("FaceApp::onWindowInit: swapchain incompatible, rebuilding FaceApp pipelines"); recreatePipelinesForSwapchain(); } if (!_secondfaceAppInited) { _secondfaceAppInited = true; _playMotion = true; _running = true; } } FACE_DBG_LOG("FaceApp::onWindowInit done _applicationInited=%d _faceAppInited=%d _sceondInited=%d _secondfaceAppInited=%d _running=%d", (int)_applicationInited, (int)_faceAppInited, (int)_sceondInited, (int)_secondfaceAppInited, (int)_running); } void FaceApp::recreatePipelinesForSwapchain() { // Destroy the two FaceApp pipelines. Layouts / descriptor set layouts are // swapchain-independent and kept as-is so existing descriptor sets keep // pointing at the same texture/uniform resources. if (m_graphicsPipeline != VK_NULL_HANDLE) { vkDestroyPipeline(device, m_graphicsPipeline, nullptr); m_graphicsPipeline = VK_NULL_HANDLE; } if (m_graphicsPipeline_bg != VK_NULL_HANDLE) { vkDestroyPipeline(device, m_graphicsPipeline_bg, nullptr); m_graphicsPipeline_bg = VK_NULL_HANDLE; } // Recreate against the fresh renderPass (if format changed) and the new // swapChainExtent (viewport/scissor are baked statically into these). create_face_pipelines(); create_pipelines_bg(); FACE_DBG_LOG("FaceApp::recreatePipelinesForSwapchain: rebuilt for extent=%ux%u", swapChainExtent.width, swapChainExtent.height); } void FaceApp::update_uniform_buffers() { // 注意:当前 vertex shader (texture.vert) 实际并没有使用 ubo.projection 把模型 // 投影到屏幕——它直接拿 inPos.xy*2-1 当 NDC 用。所以下面 aspect 取 1.0 只是 // 为了让 ubo 不再依赖原本写死的 480x480 输入分辨率;要真正影响屏幕显示比例的 // 是 push constant 里的 canvas_size / screen_w / screen_h,由 render() 写入。 const float aspect = 1.0f; const float zoom = 2.0f; // Vertex shader ubo_vs.projection = glm::perspective(glm::radians(60.0f), aspect, 0.001f, 256.0f); glm::mat4 view_matrix = glm::translate(glm::mat4(1.0f), glm::vec3(0.0f, 0.0f, zoom)); ubo_vs.model = view_matrix * glm::translate(glm::mat4(1.0f), camera_pos); ubo_vs.model = glm::rotate(ubo_vs.model, glm::radians(rotation.x), glm::vec3(1.0f, 0.0f, 0.0f)); ubo_vs.model = glm::rotate(ubo_vs.model, glm::radians(rotation.y), glm::vec3(0.0f, 1.0f, 0.0f)); ubo_vs.model = glm::rotate(ubo_vs.model, glm::radians(rotation.z), glm::vec3(0.0f, 0.0f, 1.0f)); ubo_vs.view_pos = glm::vec4(0.0f, 0.0f, -zoom, 0.0f); memcpy(uniform_buffer_mapped, &ubo_vs, sizeof(ubo_vs)); } void FaceApp::updateCanvasMetrics() { // 把"屏幕中央 min(w,h) 的正方形"作为设计画布。画布外的部分会落在 // shader 计算的 NDC ∈ [-1,+1] 之外,被 GPU 自动裁剪为黑色 letterbox。 // // swapChainExtent 来自 Application::createSwapChain 里的 // surface_capabilities.currentExtent,即当前窗口/Surface 的真实分辨率。 // 任何 swapchain 重建(onWindowLost -> onWindowInit)都会刷新它,所以 // 这里每帧重算是廉价且自洽的。 m_screenW = (float)swapChainExtent.width; m_screenH = (float)swapChainExtent.height; m_canvasSize = (m_screenW < m_screenH) ? m_screenW : m_screenH; if (m_canvasSize <= 0.0f) { // swapchain 还没初始化 / 已销毁时给个安全值,避免 shader 除零。 m_canvasSize = 480.0f; m_screenW = 480.0f; m_screenH = 480.0f; } } void FaceApp::createVertexBuffer() { VkDeviceSize vertexBufferSize = sizeof(TextureLoadingVertexStructure) * obj_vertices.size(); VkDeviceSize indexBufferSize = sizeof(uint32_t) * obj_indices.size(); VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; bufferInfo.size = vertexBufferSize; bufferInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; VmaAllocationCreateInfo allocInfo = {}; allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &m_vertexBuffer, &m_vertexBufferAllocation, nullptr); bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; allocInfo.usage = VMA_MEMORY_USAGE_CPU_TO_GPU; vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &m_stagingBuffer, &m_stagingBufferAllocation, nullptr); bufferInfo.size = indexBufferSize; bufferInfo.usage = VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &m_indexBuffer, &m_indexBufferAllocation, nullptr); } void FaceApp::uploadVertexData() { void* data; vmaMapMemory(allocator, m_stagingBufferAllocation, &data); memcpy(data, obj_vertices.data(), sizeof(TextureLoadingVertexStructure) * obj_vertices.size()); vmaUnmapMemory(allocator, m_stagingBufferAllocation); copyBuffer(m_stagingBuffer, m_vertexBuffer, sizeof(TextureLoadingVertexStructure) * obj_vertices.size()); vmaMapMemory(allocator, m_stagingBufferAllocation, &data); memcpy(data, obj_indices.data(), sizeof(uint32_t) * obj_indices.size()); vmaUnmapMemory(allocator, m_stagingBufferAllocation); copyBuffer(m_stagingBuffer, m_indexBuffer, sizeof(uint32_t) * obj_indices.size()); } void FaceApp::processCameraFrame(uint8_t* data, int width, int height, int rowStride, size_t dataSize, int rotation, bool mirrorX) { if (!isInited() || data == nullptr || width <= 0 || height <= 0) { return; } // 1) rotation 收敛到 0/90/180/270。CameraX 文档保证只会给这四个值, // 但理论上 ((rotation % 360) + 360) % 360 更鲁棒。 int rotNorm = ((rotation % 360) + 360) % 360; if (rotNorm != 0 && rotNorm != 90 && rotNorm != 180 && rotNorm != 270) { // 非法值时保守用 90(与原 hardcode 一致)。 rotNorm = 90; } // 2) 元信息更新(无锁单写者:仅相机分析线程写,render 线程读)。 // 放在 sizeChanged 判断前后都安全:render() 每帧重新拷到 push constant。 m_cameraAspect = (float)width / (float)height; m_cameraRotation = (float)rotNorm; m_mirrorX = mirrorX ? 1.0f : 0.0f; // 3) 检查帧尺寸是否与现有 tex_bg 一致。 // 场景:CameraX session 重建(如 Activity onResume)后第一帧分辨率可能 // 不同;或者 setTargetResolution 在不同设备上落地为不同尺寸。 bool sizeChanged = (tex_bg.image != VK_NULL_HANDLE) && (tex_bg.width != width || tex_bg.height != height); if (!sizeChanged) { // 快路径(每帧):直接走 base 标准上传路径。第一次 image == VK_NULL_HANDLE // base 会自动 createTexture(width, height, ...) 自适应分辨率;之后每帧 // image 已存在,直接走 staging buffer 拷贝。 processWithVulkan(data, width, height, rowStride, dataSize, tex_bg, false, commandPool, "data_from_camera"); return; } // 慢路径(罕见,整个 session 通常 1 次):尺寸变化 → 重建 image + 刷新 // descriptor set。整个 ceremony 必须严格串行: // ① createTextureMtx 与 drawFrame / 其它 processWithVulkan 互斥 // (FaceApp::drawFrame 进 Application::drawFrame 之前会持此锁,所以 // 我们持锁期间渲染线程被阻塞,不会有新的 cmdbuf 绑定 m_descriptor_set_bg // 并 submit)。 // ② vkDeviceWaitIdle 等待所有"已 submit 但未完成"的 cmdbuf 跑完,确保 // 销毁旧 image/view 时 GPU 已经不再采样它。 // ③ destroy → create → updateTexture → refresh_descriptor_set_bg // 全部在锁内完成,期间 m_descriptor_set_bg 处于"指向无效 view"的中间 // 状态,但因为渲染线程被锁阻塞,这个中间状态对 GPU 不可见。 // 注意不能调 base::processWithVulkan,那会再次 lock createTextureMtx 死锁; // 直接调 createTexture + updateTexture,这两个 base 方法不持 createTextureMtx。 #ifndef _WIN32 DebugLog::log("processCameraFrame: bg size changed %dx%d -> %dx%d, recreate texture", tex_bg.width, tex_bg.height, width, height); #endif std::unique_lock lk_tex(createTextureMtx); { std::lock_guard lk_pool(poolQueueMtx); vkDeviceWaitIdle(device); } destroyTexture(device, tex_bg); createTexture(device, physicalDevice, width, height, tex_bg, false, "data_from_camera", dataSize); updateTexture(device, physicalDevice, commandPool, graphicsQueue, data, width, height, rowStride, dataSize, tex_bg); refresh_descriptor_set_bg(); } void FaceApp::update_face_vertex_buffer(float* pos, int pointCount) { if(!isInited()) { FACE_DBG_LOG_THROTTLED("update_face_vertex_buffer.notInited", "update_face_vertex_buffer dropped: !isInited()"); return; } std::lock_guard lock(mtx_point); last_update_time = getCurrentTimeMillis(); for (int i = 0; i < obj_vertices.size(); ++i) { int face_index = obj_vertices_map[HardCodeData::Get().indexMap[i]]; float x = pos[face_index * 3 + 0]; float y = pos[face_index * 3 + 1]; float z = pos[face_index * 3 + 2]; obj_vertices[i].pos[0] = x; obj_vertices[i].pos[1] = y; obj_vertices[i].pos[2] = z; } calculateVertexNormals(obj_vertices, obj_indices); uploadVertexData(); } void FaceApp::copyBuffer(VkBuffer srcBuffer, VkBuffer dstBuffer, VkDeviceSize size) { // 改造说明: // 旧实现是 vkAllocate(commandPool) + vkBegin + vkCmdCopyBuffer + vkEnd + // vkQueueSubmit(graphicsQueue) + vkQueueWaitIdle + vkFree(commandPool) // 每帧 update_face_vertex_buffer 会调两次 copyBuffer(顶点 + 索引), // 长期高频 allocate/free 会把驱动 per-pool mutex 玩坏,触发 // FORTIFY: pthread_mutex_lock called on a destroyed mutex。 // // 现在改走基类的 runTransferCommand,复用 commandPool_ex 上预分配的 // 3 个 cmdbuf + fence,并发安全由 m_xferMtx + poolQueueMtx 联合保证。 runTransferCommand([&](VkCommandBuffer commandBuffer) { VkBufferCopy copyRegion = {}; copyRegion.size = size; vkCmdCopyBuffer(commandBuffer, srcBuffer, dstBuffer, 1, ©Region); }); } void FaceApp::createUniformBuffer() { VkDeviceSize bufferSize = sizeof(ubo_vs); VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; bufferInfo.size = bufferSize; bufferInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; VmaAllocationCreateInfo allocInfo = {}; allocInfo.usage = VMA_MEMORY_USAGE_CPU_TO_GPU; allocInfo.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; // 创建缓冲区和内存分配 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &uniform_buffer_vs, &uniform_buffer_allocation, nullptr); // 映射内存以便直接写入 vmaMapMemory(allocator, uniform_buffer_allocation, &uniform_buffer_mapped); update_uniform_buffers(); } void FaceApp::create_pipelines_bg() { VkPipelineInputAssemblyStateCreateInfo input_assembly_state{}; input_assembly_state.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; input_assembly_state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; input_assembly_state.flags = 0; input_assembly_state.primitiveRestartEnable = VK_FALSE; VkPipelineRasterizationStateCreateInfo rasterization_state{}; rasterization_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; rasterization_state.polygonMode = VK_POLYGON_MODE_FILL; rasterization_state.cullMode = VK_CULL_MODE_NONE; rasterization_state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; rasterization_state.flags = 0; rasterization_state.depthClampEnable = VK_FALSE; rasterization_state.lineWidth = 1.0f; VkPipelineColorBlendAttachmentState colorBlendAttachment{}; colorBlendAttachment.blendEnable = VK_FALSE; colorBlendAttachment.colorWriteMask = 0xf; VkPipelineColorBlendStateCreateInfo colorBlending{}; colorBlending.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; colorBlending.attachmentCount = 1; colorBlending.pAttachments = &colorBlendAttachment; VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {}; depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; depth_stencil_state.depthTestEnable = VK_FALSE; depth_stencil_state.depthWriteEnable = VK_FALSE; depth_stencil_state.depthCompareOp = VK_COMPARE_OP_GREATER; depth_stencil_state.front = depth_stencil_state.back; depth_stencil_state.back.compareOp = VK_COMPARE_OP_ALWAYS; VkPipelineViewportStateCreateInfo viewport_state{}; viewport_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; viewport_state.viewportCount = 1; viewport_state.scissorCount = 1; viewport_state.flags = 0; VkPipelineMultisampleStateCreateInfo multisample_state{}; multisample_state.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; multisample_state.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; multisample_state.flags = 0; // 视口状态 VkViewport viewport{}; viewport.x = 0.0f; viewport.y = 0.0f; viewport.width = (float)swapChainExtent.width; viewport.height = (float)swapChainExtent.height; viewport.minDepth = 0.0f; viewport.maxDepth = 1.0f; VkRect2D scissor{}; scissor.offset = { 0, 0 }; scissor.extent = swapChainExtent; VkPipelineViewportStateCreateInfo viewportState{}; viewportState.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; viewportState.viewportCount = 1; viewportState.pViewports = &viewport; viewportState.scissorCount = 1; viewportState.pScissors = &scissor; auto vertShaderCode = readFile("shaders/bg.vert.spv"); auto fragShaderCode = readFile("shaders/bg.frag.spv"); VkShaderModule vertShaderModule = createShaderModule(this->device, vertShaderCode); VkShaderModule fragShaderModule = createShaderModule(this->device, fragShaderCode); VkPipelineShaderStageCreateInfo vertShaderStageInfo{}; vertShaderStageInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; vertShaderStageInfo.stage = VK_SHADER_STAGE_VERTEX_BIT; vertShaderStageInfo.module = vertShaderModule; vertShaderStageInfo.pName = "main"; VkPipelineShaderStageCreateInfo fragShaderStageInfo{}; fragShaderStageInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; fragShaderStageInfo.stage = VK_SHADER_STAGE_FRAGMENT_BIT; fragShaderStageInfo.module = fragShaderModule; fragShaderStageInfo.pName = "main"; VkPipelineShaderStageCreateInfo shader_stages[] = { vertShaderStageInfo, fragShaderStageInfo }; VkPipelineVertexInputStateCreateInfo vertex_input_state{}; vertex_input_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; VkGraphicsPipelineCreateInfo pipeline_create_info{}; pipeline_create_info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; pipeline_create_info.layout = m_pipelineLayout_bg; pipeline_create_info.renderPass = renderPass; pipeline_create_info.flags = 0; pipeline_create_info.basePipelineIndex = -1; pipeline_create_info.basePipelineHandle = VK_NULL_HANDLE; pipeline_create_info.pVertexInputState = &vertex_input_state; pipeline_create_info.pInputAssemblyState = &input_assembly_state; pipeline_create_info.pRasterizationState = &rasterization_state; pipeline_create_info.pColorBlendState = &colorBlending; //&color_blend_state; pipeline_create_info.pMultisampleState = &multisample_state; pipeline_create_info.pViewportState = &viewport_state; pipeline_create_info.pDepthStencilState = &depth_stencil_state; //pipeline_create_info.pDynamicState = &dynamic_state; pipeline_create_info.pViewportState = &viewportState; pipeline_create_info.stageCount = 2; pipeline_create_info.pStages = shader_stages; VK_CHECK(vkCreateGraphicsPipelines(device, VK_NULL_HANDLE, 1, &pipeline_create_info, nullptr, &m_graphicsPipeline_bg)); // 销毁着色器模块 vkDestroyShaderModule(device, fragShaderModule, nullptr); vkDestroyShaderModule(device, vertShaderModule, nullptr); } void FaceApp::setup_descriptor_set_layout_bg() { VkDescriptorSetLayoutBinding set_layout_binding{}; set_layout_binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; set_layout_binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; set_layout_binding.binding = 0; set_layout_binding.descriptorCount = 1; std::vector set_layout_bindings ={set_layout_binding }; VkDescriptorSetLayoutCreateInfo descriptor_layout{}; descriptor_layout.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; descriptor_layout.pBindings = set_layout_bindings.data(); descriptor_layout.bindingCount = static_cast(set_layout_bindings.size()); VK_CHECK(vkCreateDescriptorSetLayout(device, &descriptor_layout, nullptr, &m_descriptorSetLayout_bg)); VkPipelineLayoutCreateInfo pipeline_layout_create_info{}; pipeline_layout_create_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; pipeline_layout_create_info.setLayoutCount = 1; pipeline_layout_create_info.pSetLayouts = &m_descriptorSetLayout_bg; // bg.frag 同样要读 push constant(r/g/b/radius,以及分辨率适配重构后的 // screen_w/screen_h),所以 stage 必须包含 FRAGMENT_BIT。老代码这里只写了 // VERTEX_BIT,按 Vulkan spec 是非法的(验证层会 warning,部分驱动会读到 // 未定义内容),借这次重构修掉。 VkPushConstantRange pushConstantRanges[1]; pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT; pushConstantRanges[0].offset = 0; pushConstantRanges[0].size = sizeof(PushConstants); pipeline_layout_create_info.pushConstantRangeCount = 1; pipeline_layout_create_info.pPushConstantRanges = &pushConstantRanges[0]; VK_CHECK(vkCreatePipelineLayout(device, &pipeline_layout_create_info, nullptr, &m_pipelineLayout_bg)); } void FaceApp::setup_descriptor_set_bg() { VkDescriptorSetAllocateInfo alloc_info{}; alloc_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; alloc_info.descriptorPool = descriptor_pool; alloc_info.pSetLayouts = &m_descriptorSetLayout_bg; alloc_info.descriptorSetCount = 1; VK_CHECK(vkAllocateDescriptorSets(device, &alloc_info, &m_descriptor_set_bg)); refresh_descriptor_set_bg(); } void FaceApp::refresh_descriptor_set_bg() { if (m_descriptor_set_bg == VK_NULL_HANDLE) { return; } if (tex_bg.view == VK_NULL_HANDLE || tex_bg.sampler == VK_NULL_HANDLE) { // 还没 loadTexture / processCameraFrame 过,没法绑定。 return; } VkDescriptorImageInfo image_descriptor; image_descriptor.imageView = tex_bg.view; image_descriptor.sampler = tex_bg.sampler; image_descriptor.imageLayout = tex_bg.image_layout; VkWriteDescriptorSet write_descriptor_set{}; write_descriptor_set.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; write_descriptor_set.dstSet = m_descriptor_set_bg; write_descriptor_set.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; write_descriptor_set.dstBinding = 0; write_descriptor_set.pImageInfo = &image_descriptor; write_descriptor_set.descriptorCount = 1; vkUpdateDescriptorSets(device, 1, &write_descriptor_set, 0, NULL); } void FaceApp::destroyTexture(VkDevice device, Texture& texture) { // 注意销毁顺序:先销毁依赖对象,后销毁被依赖对象 // 1. 销毁采样器 if (texture.sampler != VK_NULL_HANDLE) { vkDestroySampler(device, texture.sampler, nullptr); texture.sampler = VK_NULL_HANDLE; } // 2. 销毁图像视图 if (texture.view != VK_NULL_HANDLE) { vkDestroyImageView(device, texture.view, nullptr); texture.view = VK_NULL_HANDLE; } // 3. 销毁图像 if (texture.image != VK_NULL_HANDLE) { vkDestroyImage(device, texture.image, nullptr); texture.image = VK_NULL_HANDLE; } // 4. 释放设备内存 if (texture.device_memory != VK_NULL_HANDLE) { vkFreeMemory(device, texture.device_memory, nullptr); texture.device_memory = VK_NULL_HANDLE; //std::cout << "vkFreeMemory device_memory " << texture.texture_path << std::endl; } if (texture.stagingBuffer != VK_NULL_HANDLE) { vkDestroyBuffer(device, texture.stagingBuffer, nullptr); texture.stagingBuffer = VK_NULL_HANDLE; } if (texture.stagingBufferMemory != VK_NULL_HANDLE) { vkFreeMemory(device, texture.stagingBufferMemory, nullptr); //std::cout << "vkFreeMemory stagingBufferMemory " << texture.texture_path << std::endl; texture.stagingBufferMemory = VK_NULL_HANDLE; } } void FaceApp::cleanupResources(VkDevice device, VmaAllocator allocator) { // 销毁图形管线 if (m_graphicsPipeline != VK_NULL_HANDLE) { vkDestroyPipeline(device, m_graphicsPipeline, nullptr); m_graphicsPipeline = VK_NULL_HANDLE; } // 销毁管线布局 if (m_pipelineLayout != VK_NULL_HANDLE) { vkDestroyPipelineLayout(device, m_pipelineLayout, nullptr); m_pipelineLayout = VK_NULL_HANDLE; } // 销毁描述符集布局 if (m_descriptorSetLayout != VK_NULL_HANDLE) { vkDestroyDescriptorSetLayout(device, m_descriptorSetLayout, nullptr); m_descriptorSetLayout = VK_NULL_HANDLE; } // 销毁图形管线 if (m_graphicsPipeline_bg != VK_NULL_HANDLE) { vkDestroyPipeline(device, m_graphicsPipeline_bg, nullptr); m_graphicsPipeline_bg = VK_NULL_HANDLE; } // 销毁管线布局 if (m_pipelineLayout_bg != VK_NULL_HANDLE) { vkDestroyPipelineLayout(device, m_pipelineLayout_bg, nullptr); m_pipelineLayout_bg = VK_NULL_HANDLE; } // 销毁描述符集布局 if (m_descriptorSetLayout_bg != VK_NULL_HANDLE) { vkDestroyDescriptorSetLayout(device, m_descriptorSetLayout_bg, nullptr); m_descriptorSetLayout_bg = VK_NULL_HANDLE; } // 销毁顶点缓冲区 if (m_vertexBuffer != VK_NULL_HANDLE) { vkDestroyBuffer(device, m_vertexBuffer, nullptr); m_vertexBuffer = VK_NULL_HANDLE; } if (m_vertexBufferAllocation != VK_NULL_HANDLE) { vmaFreeMemory(allocator, m_vertexBufferAllocation); m_vertexBufferAllocation = VK_NULL_HANDLE; } // 销毁暂存缓冲区 if (m_stagingBuffer != VK_NULL_HANDLE) { vkDestroyBuffer(device, m_stagingBuffer, nullptr); m_stagingBuffer = VK_NULL_HANDLE; } if (m_stagingBufferAllocation != VK_NULL_HANDLE) { vmaFreeMemory(allocator, m_stagingBufferAllocation); m_stagingBufferAllocation = VK_NULL_HANDLE; } // 销毁索引缓冲区 if (m_indexBuffer != VK_NULL_HANDLE) { vkDestroyBuffer(device, m_indexBuffer, nullptr); m_indexBuffer = VK_NULL_HANDLE; } if (m_indexBufferAllocation != VK_NULL_HANDLE) { vmaFreeMemory(allocator, m_indexBufferAllocation); m_indexBufferAllocation = VK_NULL_HANDLE; } if (descriptor_pool != VK_NULL_HANDLE) { vkDestroyDescriptorPool(device, descriptor_pool, nullptr); descriptor_pool = VK_NULL_HANDLE; } // 清空描述符集列表(不需要单独销毁,由描述符池管理) m_descriptor_sets_left.clear(); //m_descriptor_sets_right.clear(); } void FaceApp::cleanup() { FACE_DBG_LOG("FaceApp::cleanup enter"); vkDeviceWaitIdle(device); if (uniform_buffer_mapped != nullptr) { vmaUnmapMemory(allocator, uniform_buffer_allocation); uniform_buffer_mapped = nullptr; } if (uniform_buffer_vs != VK_NULL_HANDLE) { vmaDestroyBuffer(allocator, uniform_buffer_vs, uniform_buffer_allocation); uniform_buffer_vs = VK_NULL_HANDLE; uniform_buffer_allocation = VK_NULL_HANDLE; // 可选,但推荐重置 } cleanupResources(device, allocator); for (int i = 0; i < this->m_texs_left.size(); ++i) { destroyTexture(device, m_texs_left[i]); } //for (int i = 0; i < this->m_texs_right.size(); ++i) //{ // destroyTexture(device, m_texs_right[i]); //} destroyTexture(device, tex_bg); // 必须在销毁 commandPool_ex 之前释放从它分配的 transfer cmdbuf + fence。 // vkDeviceWaitIdle 已在本函数开头调过,提交不会再有 in-flight。 destroyTransferResources(); vkDestroyCommandPool(device, commandPool, nullptr); vkDestroyCommandPool(device, commandPool_ex, nullptr); Application::cleanup(); //if (allocator != VK_NULL_HANDLE) { // vmaDestroyAllocator(allocator); // allocator = VK_NULL_HANDLE; //} FACE_DBG_LOG("FaceApp::cleanup done"); } void FaceApp::SetInitArg(const char* arg) { _initArg = json::parse(arg); pushConstants.zoom = _initArg.zoom; pushConstants.r = _initArg.r; pushConstants.g = _initArg.g; pushConstants.b = _initArg.b; pushConstants.radius = _initArg.radius; pushConstants.offset_x = _initArg.offset_x; pushConstants.offset_y = _initArg.offset_y; } void FaceApp::drawFrame(long long frameTime) { if (!_running) { return; } // if (_motionState == loading_next_motion) // { // int i = _nextMotionIndex; // //for (int i = 0; i < _curMotion.png_names.size(); ++i) // { // //#ifdef _WIN32 // string path = "pic"; // string path_name = path + "/" + _nextMotion.name + "/" + _nextMotion.png_names[i];// +std::to_string(i) + ".png"; // loadTextureExample(UTF8ToWideString(path_name), m_texs_next[i], true); //#else // //string path = _curMotion.type + "/" + _curMotion.technique + "/" + _curMotion.step + "/" + _curMotion.show_type; // string path = "pic"; // string path_name = path + "/" + _nextMotion.type + "/" + _nextMotion.png_names[i];// +std::to_string(i) + ".png"; // loadTexture(path_name, m_texs_next[i], true); //#endif // _WIN32 // // _nextMotionIndex++; // if (_nextMotionIndex >= _nextMotion.png_names.size()) // { // _motionState = load_next_motion_finished; // _callback_loadfinish(_nextMotion.name); // } // } // } if (_curMotions.size() > 0) { static long long game_time = 0; if (_playMotion) { game_time += frameTime; } long long actionTime = (1000 / _initArg.action_fps); if (game_time > actionTime) { if (_curFrameIndex < _curMotions[_curMotionIndex].frames.size() - 1) { _curFrameIndex += 1; } else { if (_curMotionIndex < _curMotions.size() - 1) { _curMotionIndex++; _curFrameIndex = 0; } else { if (_animationFinishedCallback != nullptr) { _animationFinishedCallback(); } if (_animationLoop) { _curMotionIndex = 0; _curFrameIndex = 0; } else { _animationFinishedCallback = nullptr; } } } game_time = game_time - actionTime; } } std::unique_lock lock_point(mtx_point); std::unique_lock lock_changeMotion(changeMotionMtx); std::unique_lock lock_Texture(createTextureMtx); Application::drawFrame(frameTime); } //void FaceApp::changeMotion(const char* motion_type) //{ // //Motion motion = json::parse(json); // changeMotion(motion); //} string FaceApp::preLoadMotionList(string motion_list_str, Callback callback) { FACE_DBG_LOG("FaceApp::preLoadMotionList called str_len=%zu _isLoadMotion=%d", motion_list_str.size(), (int)_isLoadMotion); if (_isLoadMotion) { return "failue load not finished"; } _isLoadMotion = true; _callback_loadfinish = callback; _curLoadMotionList.clear(); json j = json::parse(motion_list_str); MotionList motion_list = j.get(); _curLoadMotionList = motion_list.motions; if (worker_.joinable()) { worker_.join(); } worker_ = std::thread(&FaceApp::loadMotionThread, this); FACE_DBG_LOG("FaceApp::preLoadMotionList worker_ started, todo_count=%zu", _curLoadMotionList.size()); return "ok"; } void FaceApp::loadMotionThread() { FACE_DBG_LOG("FaceApp::loadMotionThread enter, todo_count=%zu", _curLoadMotionList.size()); while (!isInited()) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } //vector* load_text = nullptr; //if (cur_left) //{ // load_text = &m_texs_right; //} //else //{ // load_text = &m_texs_left; //} //vector& pre_texs = *load_text; for (auto m : _curLoadMotionList) { string name = m.name; if (motion_list_map.find(name) != motion_list_map.end()) { FACE_DBG_LOG("loadMotionThread: skip already-loaded motion='%s'", name.c_str()); continue; } m_texs_left.push_back(Texture()); Texture& newTex = m_texs_left[m_texs_left.size() - 1]; loadTexture(dummy_data, dummy_data.size(), dummy_w, dummy_h, newTex, true, commandPool, ""); #ifdef _WIN32 string path = "pic"; string path_name = path + "/" + name + "ex.png"; loadTextureExample(UTF8ToWideString(path_name), newTex, true, commandPool_ex); #else string path = "pic"; string path_name = path + "/" + name + "ex.png"; loadTexture(path_name, newTex, true, commandPool_ex); #endif // _WIN32 motion_list_map[name] = m_texs_left.size() - 1; _loadMotionMap[name] = m; FACE_DBG_LOG("loadMotionThread: loaded motion='%s' idx=%zu tex_image=%p tex_view=%p tex_sampler=%p frames=%zu", name.c_str(), m_texs_left.size() - 1, (void*)newTex.image, (void*)newTex.view, (void*)newTex.sampler, m.frames.size()); } // for (int i = 0; i < _loadMotions.size(); ++i) // { // if (pre_texs[i].image == VK_NULL_HANDLE) // { // loadTexture(dummy_data, dummy_data.size(), dummy_w, dummy_h, pre_texs[i], true, commandPool, ""); // } // // //#ifdef _WIN32 // string path = "pic"; // string path_name = path + "/" + _loadMotions[i].name + "ex.png"; // loadTextureExample(UTF8ToWideString(path_name), pre_texs[i], true, commandPool_ex); //#else // string path = "pic"; // string path_name = path + "/" + _loadMotions[i].name + "ex.png"; // loadTexture(path_name, pre_texs[i], true, commandPool_ex); //#endif // _WIN32 // // motion_list_map[_loadMotions[i].name] = i; // } update_descriptor_set(m_texs_left, m_descriptor_sets_left); _isLoadMotion = false; FACE_DBG_LOG("FaceApp::loadMotionThread done, loaded_total=%zu", motion_list_map.size()); _callback_loadfinish(); } Motion FaceApp::getMotionByName(string name) { return _loadMotionMap[name]; } void FaceApp::changeMotionList(vector motions, AnimationFinishedCallback callback, bool loop) { FACE_DBG_LOG_THROTTLED("FaceApp.changeMotionList", "FaceApp::changeMotionList called count=%zu loop=%d _isLoadMotion=%d", motions.size(), (int)loop, (int)_isLoadMotion); if (_isLoadMotion) { return; } std::unique_lock lock_changeMotion(changeMotionMtx); //for (int i = 0; i < _preLoadMotions.size(); ++i) //{ // destroyTexture(device, m_texs[i]); //} //for (int i = 0; i < _preLoadMotions.size(); ++i) //{ // m_texs[i] = m_texs_next[i]; // //m_texs_next[i].reset(); //} vector motion_list; for (auto ms : motions) { Motion m = getMotionByName(ms); auto it = motion_list_map.find(ms); int idx = (it == motion_list_map.end()) ? -1 : (int)it->second; FACE_DBG_LOG("changeMotionList: requested='%s' idx=%d frames=%zu (-1=NOT FOUND)", ms.c_str(), idx, m.frames.size()); motion_list.push_back(m); } _curMotions = motion_list; //if (cur_left) //{ // update_descriptor_set(m_texs_right, m_descriptor_sets_right); // cur_left = false; //} //else //{ // update_descriptor_set(m_texs_left, m_descriptor_sets_left); // cur_left = true; //} _curFrameIndex = 0; _curMotionIndex = 0; _animationFinishedCallback = callback; _animationLoop = loop; _isChangeMostion = true; } void FaceApp::StopMotion() { FACE_DBG_LOG("FaceApp::StopMotion: _playMotion %d -> 0", (int)_playMotion); _playMotion = false; } void FaceApp::ResumeMotion() { FACE_DBG_LOG("FaceApp::ResumeMotion: _playMotion %d -> 1", (int)_playMotion); _playMotion = true; }