diff --git a/app/build.gradle b/app/build.gradle index 21af407..f7eb601 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -1,5 +1,6 @@ plugins { alias(libs.plugins.android.application) + id 'org.jetbrains.kotlin.android' } ext.vvl_version='1.4.321.0' @@ -57,4 +58,10 @@ dependencies { testImplementation libs.junit androidTestImplementation libs.ext.junit androidTestImplementation libs.espresso.core + implementation 'com.google.mediapipe:tasks-vision:0.10.26' + def camerax_version = '1.4.2' + implementation "androidx.camera:camera-core:$camerax_version" + implementation "androidx.camera:camera-camera2:$camerax_version" + implementation "androidx.camera:camera-lifecycle:$camerax_version" + implementation "androidx.camera:camera-view:$camerax_version" } \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index ac5f87b..4c5a35b 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,6 +2,12 @@ + + + + + + destroyRequested); application.cleanup(); } +} + +extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize); +extern void ReceiveFacePoint(float* pos, int count, int width, int height); + +void processWithVulkan(uint8_t* data, int width, int height, int format, + int rowStride, size_t dataSize) { + TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_hmwl_sample_MainActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer, + jint width, jint height, jint format, + jint row_stride, jint pixel_stride, + jint rotation) { + + uint8_t* imageData = static_cast(env->GetDirectBufferAddress(buffer)); + jlong capacity = env->GetDirectBufferCapacity(buffer); + + if (imageData == nullptr) { + return; + } + + // 在这里将图像数据传递给 Vulkan + // 创建 Vulkan 图像或更新现有图像 + processWithVulkan(imageData, width, height, format, row_stride, capacity); +} + + +extern "C" +JNIEXPORT void JNICALL +Java_com_hmwl_sample_MainActivity_passDataToNative(JNIEnv *env, jobject thiz, jobject buffer, + jint point_count, jint width, jint height) { + float* pos = static_cast(env->GetDirectBufferAddress(buffer)); + + if (pos == nullptr) { + // 处理错误 + return; + } + + // 或者直接将指针传递给Vulkan + ReceiveFacePoint(pos, point_count, width, height); } \ No newline at end of file diff --git a/app/src/main/java/com/hmwl/sample/FaceLandmarkerHelper.kt b/app/src/main/java/com/hmwl/sample/FaceLandmarkerHelper.kt new file mode 100644 index 0000000..6a5e01c --- /dev/null +++ b/app/src/main/java/com/hmwl/sample/FaceLandmarkerHelper.kt @@ -0,0 +1,334 @@ + +package com.hmwl.sample; + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.os.SystemClock +import android.util.Log +import androidx.annotation.VisibleForTesting +import androidx.camera.core.ImageProxy +import com.google.mediapipe.framework.image.BitmapImageBuilder +import com.google.mediapipe.framework.image.MPImage +import com.google.mediapipe.tasks.core.BaseOptions +import com.google.mediapipe.tasks.core.Delegate +import com.google.mediapipe.tasks.vision.core.RunningMode +import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarker +import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult + +class FaceLandmarkerHelper( + var minFaceDetectionConfidence: Float = DEFAULT_FACE_DETECTION_CONFIDENCE, + var minFaceTrackingConfidence: Float = DEFAULT_FACE_TRACKING_CONFIDENCE, + var minFacePresenceConfidence: Float = DEFAULT_FACE_PRESENCE_CONFIDENCE, + var maxNumFaces: Int = DEFAULT_NUM_FACES, + var currentDelegate: Int = DELEGATE_CPU, + var runningMode: RunningMode = RunningMode.IMAGE, + val context: Context, + // this listener is only used when running in RunningMode.LIVE_STREAM + val faceLandmarkerHelperListener: LandmarkerListener? = null +) { + + // For this example this needs to be a var so it can be reset on changes. + // If the Face Landmarker will not change, a lazy val would be preferable. + private var faceLandmarker: FaceLandmarker? = null + + init { + setupFaceLandmarker() + } + + fun clearFaceLandmarker() { + faceLandmarker?.close() + faceLandmarker = null + } + + // Return running status of FaceLandmarkerHelper + fun isClose(): Boolean { + return faceLandmarker == null + } + + fun setupFaceLandmarker() { + // Set general face landmarker options + val baseOptionBuilder = BaseOptions.builder() + + // Use the specified hardware for running the model. Default to CPU + when (currentDelegate) { + DELEGATE_CPU -> { + baseOptionBuilder.setDelegate(Delegate.CPU) + } + DELEGATE_GPU -> { + baseOptionBuilder.setDelegate(Delegate.GPU) + } + } + + baseOptionBuilder.setModelAssetPath(MP_FACE_LANDMARKER_TASK) + + // Check if runningMode is consistent with faceLandmarkerHelperListener + when (runningMode) { + RunningMode.LIVE_STREAM -> { + if (faceLandmarkerHelperListener == null) { + throw IllegalStateException( + "faceLandmarkerHelperListener must be set when runningMode is LIVE_STREAM." + ) + } + } + else -> { + // no-op + } + } + + try { + val baseOptions = baseOptionBuilder.build() + // Create an option builder with base options and specific + // options only use for Face Landmarker. + val optionsBuilder = + FaceLandmarker.FaceLandmarkerOptions.builder() + .setBaseOptions(baseOptions) + .setMinFaceDetectionConfidence(minFaceDetectionConfidence) + .setMinTrackingConfidence(minFaceTrackingConfidence) + .setMinFacePresenceConfidence(minFacePresenceConfidence) + .setNumFaces(maxNumFaces) + .setOutputFaceBlendshapes(false) + .setOutputFacialTransformationMatrixes(false) + .setRunningMode(runningMode) + + // The ResultListener and ErrorListener only use for LIVE_STREAM mode. + if (runningMode == RunningMode.LIVE_STREAM) { + optionsBuilder + .setResultListener(this::returnLivestreamResult) + .setErrorListener(this::returnLivestreamError) + } + + val options = optionsBuilder.build() + faceLandmarker = + FaceLandmarker.createFromOptions(context, options) + } catch (e: IllegalStateException) { + faceLandmarkerHelperListener?.onError( + "Face Landmarker failed to initialize. See error logs for " + + "details" + ) + Log.e( + TAG, "MediaPipe failed to load the task with error: " + e + .message + ) + } catch (e: RuntimeException) { + // This occurs if the model being used does not support GPU + faceLandmarkerHelperListener?.onError( + "Face Landmarker failed to initialize. See error logs for " + + "details", GPU_ERROR + ) + Log.e( + TAG, + "Face Landmarker failed to load model with error: " + e.message + ) + } + } + + private var frameId: Long = 0 + // 简化的延时跟踪 + private val frameTimings = mutableMapOf() + private var lastFpsUpdateTime: Long = 0 + private var frameCount: Int = 0 + private var currentFps: Double = 0.0 + + // 内部数据类用于存储时间信息 + private data class FrameTiming( + var captureTime: Long = 0, + var preprocessStartTime: Long = 0, + var preprocessEndTime: Long = 0, + var detectionStartTime: Long = 0 + ) + + // 辅助方法：记录帧时间信息 + private fun recordFrameTiming(frameId: Long, block: (FrameTiming) -> Unit) { + val timing = frameTimings.getOrPut(frameId) { FrameTiming() } + block(timing) + } + + // 辅助方法：更新FPS计算 + private fun updateFps() { + frameCount++ + val currentTime = SystemClock.uptimeMillis() + + if (lastFpsUpdateTime == 0L) { + lastFpsUpdateTime = currentTime + return + } + + val timeDiff = currentTime - lastFpsUpdateTime + if (timeDiff >= 1000) { // 每秒更新一次FPS + currentFps = frameCount * 1000.0 / timeDiff + frameCount = 0 + lastFpsUpdateTime = currentTime + } + } + + // 辅助方法：清理旧的时间记录 + private fun cleanupOldTimings(currentFrameId: Long) { + val framesToRemove = frameTimings.keys.filter { it < currentFrameId - 10 } + framesToRemove.forEach { frameTimings.remove(it) } + } + + // Convert the ImageProxy to MP Image and feed it to FacelandmakerHelper. + fun detectLiveStream( + imageProxy: ImageProxy, + isFrontCamera: Boolean + ) { + if (runningMode != RunningMode.LIVE_STREAM) { + throw IllegalArgumentException( + "Attempting to call detectLiveStream" + + " while not using RunningMode.LIVE_STREAM" + ) + } + val frameTime = SystemClock.uptimeMillis() + + val currentFrameId = frameId++ + val captureTime = SystemClock.uptimeMillis() + + // 记录捕获时间 + recordFrameTiming(currentFrameId) { + it.captureTime = captureTime + it.preprocessStartTime = SystemClock.uptimeMillis() + } + + // Copy out RGB bits from the frame to a bitmap buffer + val bitmapBuffer = + Bitmap.createBitmap( + imageProxy.width, + imageProxy.height, + Bitmap.Config.ARGB_8888 + ) + imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) } + imageProxy.close() + + val matrix = Matrix().apply { + // Rotate the frame received from the camera to be in the same direction as it'll be shown + postRotate(imageProxy.imageInfo.rotationDegrees.toFloat()) + + // flip image if user use front camera + if (isFrontCamera) { + postScale( + -1f, + 1f, + imageProxy.width.toFloat(), + imageProxy.height.toFloat() + ) + } + } + val rotatedBitmap = Bitmap.createBitmap( + bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height, + matrix, true + ) + + //val bitmap = BitmapFactory.decodeFile("/sdcard/Android/data/com.khronos.vulkan_samples/files/assets/face480.png") + //val mpImage = BitmapImageBuilder(bitmap).build() + + // Convert the input Bitmap object to an MPImage object to run inference + val mpImage = BitmapImageBuilder(rotatedBitmap).build() + + detectAsync(mpImage, frameTime) + } + + // Run face face landmark using MediaPipe Face Landmarker API + @VisibleForTesting + fun detectAsync(mpImage: MPImage, frameTime: Long) { + faceLandmarker?.detectAsync(mpImage, frameTime) + // As we're using running mode LIVE_STREAM, the landmark result will + // be returned in returnLivestreamResult function + } + + + + // Return the landmark result to this FaceLandmarkerHelper's caller + private fun returnLivestreamResult( + result: FaceLandmarkerResult, + input: MPImage + ) { + val resultTime = SystemClock.uptimeMillis() + val frameId = result.timestampMs() + if( result.faceLandmarks().size > 0 ) { + // 计算各种延时 + val timings = frameTimings[frameId] + if (timings != null) { + val totalLatency = resultTime - timings.captureTime + val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime + val detectionLatency = resultTime - timings.detectionStartTime + val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime + + Log.i("TimeLatency", + "总延时: ${totalLatency}ms | " + + "预处理: ${preprocessLatency}ms | " + + "检测: ${detectionLatency}ms | " + + "捕获到检测: ${captureToDetectionLatency}ms | " + + "FPS: ${String.format("%.1f", currentFps)}" + ) + + val finishTimeMs = SystemClock.uptimeMillis() + val inferenceTime = finishTimeMs - result.timestampMs() + faceLandmarkerHelperListener?.onResults( + ResultBundle( + result, + inferenceTime, // 主要是检测时间 + input.height, + input.width, + ) + ) + } else { + val finishTimeMs = SystemClock.uptimeMillis() + val inferenceTime = finishTimeMs - result.timestampMs() + Log.i("TimeLatency", + "总延时: ${inferenceTime}ms | " + ) + faceLandmarkerHelperListener?.onResults( + ResultBundle( + result, + inferenceTime, + input.height, + input.width + ) + ) + } + } + else { + faceLandmarkerHelperListener?.onEmpty() + } + } + + // Return errors thrown during detection to this FaceLandmarkerHelper's + // caller + private fun returnLivestreamError(error: RuntimeException) { + faceLandmarkerHelperListener?.onError( + error.message ?: "An unknown error has occurred" + ) + } + + companion object { + const val TAG = "FaceLandmarkerHelper" + private const val MP_FACE_LANDMARKER_TASK = "face_landmarker.task" + + const val DELEGATE_CPU = 0 + const val DELEGATE_GPU = 1 + const val DEFAULT_FACE_DETECTION_CONFIDENCE = 0.5F + const val DEFAULT_FACE_TRACKING_CONFIDENCE = 0.5F + const val DEFAULT_FACE_PRESENCE_CONFIDENCE = 0.5F + const val DEFAULT_NUM_FACES = 1 + const val OTHER_ERROR = 0 + const val GPU_ERROR = 1 + } + + data class ResultBundle( + val result: FaceLandmarkerResult, + val inferenceTime: Long, + val inputImageHeight: Int, + val inputImageWidth: Int, + ) + + interface LandmarkerListener { + fun onError(error: String, errorCode: Int = OTHER_ERROR) + fun onResults(resultBundle: ResultBundle) + + fun onEmpty() {} + } +} diff --git a/app/src/main/java/com/hmwl/sample/MainActivity.java b/app/src/main/java/com/hmwl/sample/MainActivity.java index 1548ec4..bb5e328 100644 --- a/app/src/main/java/com/hmwl/sample/MainActivity.java +++ b/app/src/main/java/com/hmwl/sample/MainActivity.java @@ -1,18 +1,202 @@ package com.hmwl.sample; +import android.Manifest; +import android.content.pm.PackageManager; +import android.os.Bundle; import android.util.Log; import android.view.MotionEvent; import android.view.View; -import com.google.androidgamesdk.GameActivity; +import androidx.annotation.NonNull; +import androidx.camera.core.AspectRatio; +import androidx.camera.core.CameraSelector; +import androidx.camera.core.ImageAnalysis; +import androidx.camera.core.ImageProxy; +import androidx.camera.lifecycle.ProcessCameraProvider; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; -public class MainActivity extends GameActivity{ +import com.google.androidgamesdk.GameActivity; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult; + +import org.jetbrains.annotations.NotNull; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener { private static final String TAG = "MainActivity"; static { System.loadLibrary("sample"); } + private ProcessCameraProvider cameraProvider; + private void initCamera(){ + backgroundExecutor = Executors.newSingleThreadExecutor(); + if (hasCameraPermission()) { + startCamera(); + } else { + requestCameraPermission(); + } + } + + private void startCamera() { + ListenableFuture cameraProviderFuture = + ProcessCameraProvider.getInstance(this); + + cameraProviderFuture.addListener(() -> { + try { + cameraProvider = cameraProviderFuture.get(); + + CameraSelector cameraSelector = new CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build(); + + imageAnalyzer = new ImageAnalysis.Builder() + .setTargetAspectRatio(AspectRatio.RATIO_4_3) + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) + .build(); + + + imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() { + private long lastCaptureTime = 0; + private int frameCount = 0; + + @Override + public void analyze(@NonNull ImageProxy image) { + frameCount++; + long currentTime = System.currentTimeMillis(); + try { + long captureTimeNs = image.getImageInfo().getTimestamp(); + long analyzeTime = System.currentTimeMillis(); + + // 简单直接的计算方法 + long captureTimeMs = TimeUnit.NANOSECONDS.toMillis(captureTimeNs); + long latency = analyzeTime - captureTimeMs; + + // 帧率计算 + if (lastCaptureTime > 0) { + long frameInterval = captureTimeMs - lastCaptureTime; + double fps = 1000.0 / frameInterval; + + if (frameCount % 10 == 0) { // 每30帧打印一次 + Log.i("TimeLatency", String.format( + "Frame %d - Latency: %d ms, FPS: %.1f", + frameCount, latency, fps + )); + } + } + + lastCaptureTime = captureTimeMs; + + processImageForVulkan(image); + if(frameCount %2 == 0) + { + detectFace(image); + } + else + { + image.close(); + } + } finally { + //image.close(); // 确保在这里关闭 + //Log.d("Camera", "Image closed, ready for next frame"); + } + } + }); + + cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer); + + } catch (ExecutionException | InterruptedException e) { + Log.e("MainActivity", "Error starting camera: " + e.getMessage()); + } + }, ContextCompat.getMainExecutor(this)); + } + + private void processImageForVulkan(ImageProxy image) { + // 获取图像信息 + int width = image.getWidth(); + int height = image.getHeight(); + int format = image.getFormat(); + + // 获取图像数据平面 + ImageProxy.PlaneProxy[] planes = image.getPlanes(); + + // 对于 RGBA_8888 格式，通常只有一个平面 + if (planes.length > 0) { + ImageProxy.PlaneProxy plane = planes[0]; + ByteBuffer buffer = plane.getBuffer(); + int rowStride = plane.getRowStride(); + int pixelStride = plane.getPixelStride(); + + // 调用 Native 方法处理图像 + processImageNative(buffer, width, height, format, + rowStride, pixelStride, image.getImageInfo().getRotationDegrees()); + } + } + + // Native 方法 + private native void processImageNative(ByteBuffer buffer, int width, int height, + int format, int rowStride, int pixelStride, + int rotation); + + private void detectFace(ImageProxy imageProxy) { + faceLandmarkerHelper.detectLiveStream( + imageProxy,false + ); + } + + private boolean hasCameraPermission() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) + == PackageManager.PERMISSION_GRANTED; + } + + private void requestCameraPermission() { + ActivityCompat.requestPermissions(this, + new String[]{Manifest.permission.CAMERA}, + REQUEST_CAMERA_PERMISSION); + } + + private static final int REQUEST_CAMERA_PERMISSION = 1001; + ImageAnalysis imageAnalyzer; + ExecutorService backgroundExecutor; + FaceLandmarkerHelper faceLandmarkerHelper; + + @Override + protected void onCreate(Bundle savedInstanceState){ + super.onCreate(savedInstanceState); + initCamera(); + + backgroundExecutor.execute(new Runnable() { + @Override + public void run() { + faceLandmarkerHelper = new FaceLandmarkerHelper( + 0.5f, + 0.5f, + 0.5f, + 1, + 0, + RunningMode.LIVE_STREAM, + MainActivity.this, + MainActivity.this + ); + } + }); + } + + + @Override public boolean onTouchEvent(MotionEvent event) { int action = event.getActionMasked(); @@ -57,4 +241,71 @@ public class MainActivity extends GameActivity{ | View.SYSTEM_UI_FLAG_FULLSCREEN ); } + + @Override + public void onError(@NotNull String error, int errorCode) { + + } + + private ByteBuffer nativeBuffer = null; + float[] points = new float[480 * 3]; + + + public class Vector { + public float x, y, z; + + public Vector(float x, float y, float z) { + this.x = x; + this.y = y; + this.z = z; + } + } + + public float z_rate = -1.0f; + + @Override + public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) { + FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult(); + int height = resultBundle.getInputImageHeight(); + int width = resultBundle.getInputImageWidth(); + int bufferSize = 480 * 3 * 4; + + if(nativeBuffer == null) + { + // 创建直接ByteBuffer（在native内存中） + nativeBuffer = ByteBuffer.allocateDirect(bufferSize); + nativeBuffer.order(ByteOrder.nativeOrder()); + } + + long start_time = System.currentTimeMillis(); + + int index = 0; + // Iterate through each detected face + for (List faceLandmarks : faceLandmarkerResult.faceLandmarks()) { + //should be 1 + + int arrayIndex = 0; + for (NormalizedLandmark p : faceLandmarks) + { + //sb.append("["); + points[arrayIndex++] = p.x(); + //sb.append(""+p.x()+","); + points[arrayIndex++] = p.y(); + //sb.append(""+p.y()+","); + points[arrayIndex++] = p.z() * z_rate; + index++; + } + } + + + FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); + floatBuffer.put(points); + floatBuffer.position(0); + long cur_time = System.currentTimeMillis(); + Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time)); + passDataToNative(nativeBuffer, index, width, height); + Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time)); + } + + public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height); } \ No newline at end of file diff --git a/build.gradle b/build.gradle index 565f8c2..ceed304 100644 --- a/build.gradle +++ b/build.gradle @@ -1,4 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { alias(libs.plugins.android.application) apply false + id 'org.jetbrains.kotlin.android' version '2.2.0' apply false } \ No newline at end of file diff --git a/vulkan/Application.cpp b/vulkan/Application.cpp index b8ed56e..4749ecf 100644 --- a/vulkan/Application.cpp +++ b/vulkan/Application.cpp @@ -626,13 +626,13 @@ uint32_t Application::findMemoryType(VkPhysicalDevice physicalDevice, uint32_t t throw std::runtime_error("Failed to find suitable memory type!"); } -Texture Application::loadTexture(std::string path, Texture& tex) +Texture Application::loadTexture(std::string path, Texture& tex, bool srgb) { std::vector data = readFileEx(path); std::vector image; unsigned w, h; unsigned error = lodepng::decode(image, w, h, data, LCT_RGBA, 8); - processWithVulkan(image.data(), w, h, w * 4, image.size(), tex, true); + processWithVulkan(image.data(), w, h, w * 4, image.size(), tex, srgb); return tex; } diff --git a/vulkan/Application.h b/vulkan/Application.h index f1eb3f6..30ff4be 100644 --- a/vulkan/Application.h +++ b/vulkan/Application.h @@ -43,6 +43,8 @@ public: virtual void render(VkCommandBuffer commandBuffer); virtual bool isInited() { return inited; } + void processWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& out_texture, bool srgb); + protected: VkPhysicalDevice physicalDevice = VK_NULL_HANDLE; // 物理设备 @@ -69,10 +71,11 @@ protected: int currentFrame = 0; int MAX_FRAMES_IN_FLIGHT = 2; bool inited = false; + protected: - Texture loadTexture(std::string path, Texture& tex); - void processWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& out_texture, bool srgb); + Texture loadTexture(std::string path, Texture& tex, bool srgb); + uint32_t findMemoryType(VkPhysicalDevice physicalDevice, uint32_t typeFilter, VkMemoryPropertyFlags properties); void createTexture(VkDevice device, VkPhysicalDevice physicalDevice, int width, int height, Texture& texture, bool srgb); diff --git a/vulkan/FaceApp.cpp b/vulkan/FaceApp.cpp index 8e290b1..4781a37 100644 --- a/vulkan/FaceApp.cpp +++ b/vulkan/FaceApp.cpp @@ -24,6 +24,14 @@ void ReceiveFacePoint(float* pos, int pointCount, int width, int height) } } + + +void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize) +{ + Texture& tex_bg = FaceApp::Get()->tex_bg; + FaceApp::Get()->processWithVulkan(data, width, height, rowStride, dataSize, tex_bg, false); +} + bool FaceApp::LoadOBJ(const std::string& filename, std::vector& vertices, std::vector& indices) { @@ -477,6 +485,8 @@ void FaceApp::setup_descriptor_set() void FaceApp::render(VkCommandBuffer commandBuffer) { + std::unique_lock lock(mtx); + std::unique_lock lock_point(mtx_point); //Application::render(commandBuffer); vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout_bg, 0, 1, &m_descriptor_set_bg, 0, NULL); @@ -539,8 +549,8 @@ void FaceApp::initVulkan() Application::initVulkan(); createVmaAllocator(); LoadOBJ("face_picture_3dmax.obj", obj_vertices, obj_indices); - loadTexture("demo0.png", tex_demo0); - loadTexture("out.png", tex_bg); + loadTexture("demo0.png", tex_demo0, true); + loadTexture("out.png", tex_bg, false); createVertexBuffer(); createUniformBuffer(); setup_descriptor_pool(); diff --git a/vulkan/FaceApp.h b/vulkan/FaceApp.h index b04986b..0584700 100644 --- a/vulkan/FaceApp.h +++ b/vulkan/FaceApp.h @@ -44,6 +44,9 @@ public: virtual bool isInited() override { return inited && faceAppInited; } virtual void cleanup() override; + + Texture tex_bg = {}; + private: glm::vec3 rotation = glm::vec3(); glm::vec3 camera_pos = glm::vec3(); @@ -103,7 +106,7 @@ private: float myFloatValue = 1.5f; - Texture tex_bg = {}; + void create_pipelines_bg(); void setup_descriptor_set_layout_bg(); void setup_descriptor_set_bg();