设备上跑通人脸检测和线条绘制

2025-10-20 22:51:44 +08:00
parent 361708bd11
commit 220e1873f9
11 changed files with 667 additions and 9 deletions
@@ -2,6 +2,12 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools">

+    <!-- Declare features -->
+    <uses-feature android:name="android.hardware.camera" />
+
+    <!-- Declare permissions -->
+    <uses-permission android:name="android.permission.CAMERA" />
+
    <application
        android:allowBackup="true"
        android:dataExtractionRules="@xml/data_extraction_rules"
@@ -75,4 +75,47 @@ void android_main(struct android_app *pApp) {
    } while (!pApp->destroyRequested);
    application.cleanup();
 }
+}
+
+extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize);
+extern void ReceiveFacePoint(float* pos, int count, int width, int height);
+
+void processWithVulkan(uint8_t* data, int width, int height, int format,
+                       int rowStride, size_t dataSize) {
+    TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_hmwl_sample_MainActivity_processImageNative(JNIEnv *env, jobject thiz, jobject buffer,
+                                                     jint width, jint height, jint format,
+                                                     jint row_stride, jint pixel_stride,
+                                                     jint rotation) {
+
+    uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer));
+    jlong capacity = env->GetDirectBufferCapacity(buffer);
+
+    if (imageData == nullptr) {
+        return;
+    }
+
+    // 在这里将图像数据传递给 Vulkan
+    // 创建 Vulkan 图像或更新现有图像
+    processWithVulkan(imageData, width, height, format, row_stride, capacity);
+}
+
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_hmwl_sample_MainActivity_passDataToNative(JNIEnv *env, jobject thiz, jobject buffer,
+                                                   jint point_count, jint width, jint height) {
+    float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer));
+
+    if (pos == nullptr) {
+        // 处理错误
+        return;
+    }
+
+    // 或者直接将指针传递给Vulkan
+    ReceiveFacePoint(pos, point_count, width, height);
 }
@@ -0,0 +1,334 @@
+
+package com.hmwl.sample;
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.Matrix
+import android.media.MediaMetadataRetriever
+import android.net.Uri
+import android.os.SystemClock
+import android.util.Log
+import androidx.annotation.VisibleForTesting
+import androidx.camera.core.ImageProxy
+import com.google.mediapipe.framework.image.BitmapImageBuilder
+import com.google.mediapipe.framework.image.MPImage
+import com.google.mediapipe.tasks.core.BaseOptions
+import com.google.mediapipe.tasks.core.Delegate
+import com.google.mediapipe.tasks.vision.core.RunningMode
+import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarker
+import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult
+
+class FaceLandmarkerHelper(
+    var minFaceDetectionConfidence: Float = DEFAULT_FACE_DETECTION_CONFIDENCE,
+    var minFaceTrackingConfidence: Float = DEFAULT_FACE_TRACKING_CONFIDENCE,
+    var minFacePresenceConfidence: Float = DEFAULT_FACE_PRESENCE_CONFIDENCE,
+    var maxNumFaces: Int = DEFAULT_NUM_FACES,
+    var currentDelegate: Int = DELEGATE_CPU,
+    var runningMode: RunningMode = RunningMode.IMAGE,
+    val context: Context,
+    // this listener is only used when running in RunningMode.LIVE_STREAM
+    val faceLandmarkerHelperListener: LandmarkerListener? = null
+) {
+
+    // For this example this needs to be a var so it can be reset on changes.
+    // If the Face Landmarker will not change, a lazy val would be preferable.
+    private var faceLandmarker: FaceLandmarker? = null
+
+    init {
+        setupFaceLandmarker()
+    }
+
+    fun clearFaceLandmarker() {
+        faceLandmarker?.close()
+        faceLandmarker = null
+    }
+
+    // Return running status of FaceLandmarkerHelper
+    fun isClose(): Boolean {
+        return faceLandmarker == null
+    }
+
+    fun setupFaceLandmarker() {
+        // Set general face landmarker options
+        val baseOptionBuilder = BaseOptions.builder()
+
+        // Use the specified hardware for running the model. Default to CPU
+        when (currentDelegate) {
+            DELEGATE_CPU -> {
+                baseOptionBuilder.setDelegate(Delegate.CPU)
+            }
+            DELEGATE_GPU -> {
+                baseOptionBuilder.setDelegate(Delegate.GPU)
+            }
+        }
+
+        baseOptionBuilder.setModelAssetPath(MP_FACE_LANDMARKER_TASK)
+
+        // Check if runningMode is consistent with faceLandmarkerHelperListener
+        when (runningMode) {
+            RunningMode.LIVE_STREAM -> {
+                if (faceLandmarkerHelperListener == null) {
+                    throw IllegalStateException(
+                        "faceLandmarkerHelperListener must be set when runningMode is LIVE_STREAM."
+                    )
+                }
+            }
+            else -> {
+                // no-op
+            }
+        }
+
+        try {
+            val baseOptions = baseOptionBuilder.build()
+            // Create an option builder with base options and specific
+            // options only use for Face Landmarker.
+            val optionsBuilder =
+                FaceLandmarker.FaceLandmarkerOptions.builder()
+                    .setBaseOptions(baseOptions)
+                    .setMinFaceDetectionConfidence(minFaceDetectionConfidence)
+                    .setMinTrackingConfidence(minFaceTrackingConfidence)
+                    .setMinFacePresenceConfidence(minFacePresenceConfidence)
+                    .setNumFaces(maxNumFaces)
+                    .setOutputFaceBlendshapes(false)
+                    .setOutputFacialTransformationMatrixes(false)
+                    .setRunningMode(runningMode)
+
+            // The ResultListener and ErrorListener only use for LIVE_STREAM mode.
+            if (runningMode == RunningMode.LIVE_STREAM) {
+                optionsBuilder
+                    .setResultListener(this::returnLivestreamResult)
+                    .setErrorListener(this::returnLivestreamError)
+            }
+
+            val options = optionsBuilder.build()
+            faceLandmarker =
+                FaceLandmarker.createFromOptions(context, options)
+        } catch (e: IllegalStateException) {
+            faceLandmarkerHelperListener?.onError(
+                "Face Landmarker failed to initialize. See error logs for " +
+                        "details"
+            )
+            Log.e(
+                TAG, "MediaPipe failed to load the task with error: " + e
+                    .message
+            )
+        } catch (e: RuntimeException) {
+            // This occurs if the model being used does not support GPU
+            faceLandmarkerHelperListener?.onError(
+                "Face Landmarker failed to initialize. See error logs for " +
+                        "details", GPU_ERROR
+            )
+            Log.e(
+                TAG,
+                "Face Landmarker failed to load model with error: " + e.message
+            )
+        }
+    }
+
+    private var frameId: Long = 0
+    // 简化的延时跟踪
+    private val frameTimings = mutableMapOf<Long, FrameTiming>()
+    private var lastFpsUpdateTime: Long = 0
+    private var frameCount: Int = 0
+    private var currentFps: Double = 0.0
+
+    // 内部数据类用于存储时间信息
+    private data class FrameTiming(
+        var captureTime: Long = 0,
+        var preprocessStartTime: Long = 0,
+        var preprocessEndTime: Long = 0,
+        var detectionStartTime: Long = 0
+    )
+
+    // 辅助方法：记录帧时间信息
+    private fun recordFrameTiming(frameId: Long, block: (FrameTiming) -> Unit) {
+        val timing = frameTimings.getOrPut(frameId) { FrameTiming() }
+        block(timing)
+    }
+
+    // 辅助方法：更新FPS计算
+    private fun updateFps() {
+        frameCount++
+        val currentTime = SystemClock.uptimeMillis()
+
+        if (lastFpsUpdateTime == 0L) {
+            lastFpsUpdateTime = currentTime
+            return
+        }
+
+        val timeDiff = currentTime - lastFpsUpdateTime
+        if (timeDiff >= 1000) { // 每秒更新一次FPS
+            currentFps = frameCount * 1000.0 / timeDiff
+            frameCount = 0
+            lastFpsUpdateTime = currentTime
+        }
+    }
+
+    // 辅助方法：清理旧的时间记录
+    private fun cleanupOldTimings(currentFrameId: Long) {
+        val framesToRemove = frameTimings.keys.filter { it < currentFrameId - 10 }
+        framesToRemove.forEach { frameTimings.remove(it) }
+    }
+
+    // Convert the ImageProxy to MP Image and feed it to FacelandmakerHelper.
+    fun detectLiveStream(
+        imageProxy: ImageProxy,
+        isFrontCamera: Boolean
+    ) {
+        if (runningMode != RunningMode.LIVE_STREAM) {
+            throw IllegalArgumentException(
+                "Attempting to call detectLiveStream" +
+                        " while not using RunningMode.LIVE_STREAM"
+            )
+        }
+        val frameTime = SystemClock.uptimeMillis()
+
+        val currentFrameId = frameId++
+        val captureTime = SystemClock.uptimeMillis()
+
+        // 记录捕获时间
+        recordFrameTiming(currentFrameId) {
+            it.captureTime = captureTime
+            it.preprocessStartTime = SystemClock.uptimeMillis()
+        }
+
+        // Copy out RGB bits from the frame to a bitmap buffer
+        val bitmapBuffer =
+            Bitmap.createBitmap(
+                imageProxy.width,
+                imageProxy.height,
+                Bitmap.Config.ARGB_8888
+            )
+        imageProxy.use { bitmapBuffer.copyPixelsFromBuffer(imageProxy.planes[0].buffer) }
+        imageProxy.close()
+
+        val matrix = Matrix().apply {
+            // Rotate the frame received from the camera to be in the same direction as it'll be shown
+            postRotate(imageProxy.imageInfo.rotationDegrees.toFloat())
+
+            // flip image if user use front camera
+            if (isFrontCamera) {
+                postScale(
+                    -1f,
+                    1f,
+                    imageProxy.width.toFloat(),
+                    imageProxy.height.toFloat()
+                )
+            }
+        }
+        val rotatedBitmap = Bitmap.createBitmap(
+            bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height,
+            matrix, true
+        )
+
+        //val bitmap = BitmapFactory.decodeFile("/sdcard/Android/data/com.khronos.vulkan_samples/files/assets/face480.png")
+        //val mpImage = BitmapImageBuilder(bitmap).build()
+
+        // Convert the input Bitmap object to an MPImage object to run inference
+        val mpImage = BitmapImageBuilder(rotatedBitmap).build()
+
+        detectAsync(mpImage, frameTime)
+    }
+
+    // Run face face landmark using MediaPipe Face Landmarker API
+    @VisibleForTesting
+    fun detectAsync(mpImage: MPImage, frameTime: Long) {
+        faceLandmarker?.detectAsync(mpImage, frameTime)
+        // As we're using running mode LIVE_STREAM, the landmark result will
+        // be returned in returnLivestreamResult function
+    }
+
+
+
+    // Return the landmark result to this FaceLandmarkerHelper's caller
+    private fun returnLivestreamResult(
+        result: FaceLandmarkerResult,
+        input: MPImage
+    ) {
+        val resultTime = SystemClock.uptimeMillis()
+        val frameId = result.timestampMs()
+        if( result.faceLandmarks().size > 0 ) {
+            // 计算各种延时
+            val timings = frameTimings[frameId]
+            if (timings != null) {
+                val totalLatency = resultTime - timings.captureTime
+                val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime
+                val detectionLatency = resultTime - timings.detectionStartTime
+                val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime
+
+                Log.i("TimeLatency",
+                    "总延时: ${totalLatency}ms | " +
+                            "预处理: ${preprocessLatency}ms | " +
+                            "检测: ${detectionLatency}ms | " +
+                            "捕获到检测: ${captureToDetectionLatency}ms | " +
+                            "FPS: ${String.format("%.1f", currentFps)}"
+                )
+
+                val finishTimeMs = SystemClock.uptimeMillis()
+                val inferenceTime = finishTimeMs - result.timestampMs()
+                faceLandmarkerHelperListener?.onResults(
+                    ResultBundle(
+                        result,
+                        inferenceTime, // 主要是检测时间
+                        input.height,
+                        input.width,
+                    )
+                )
+            } else {
+                val finishTimeMs = SystemClock.uptimeMillis()
+                val inferenceTime = finishTimeMs - result.timestampMs()
+                Log.i("TimeLatency",
+                    "总延时: ${inferenceTime}ms | "
+                )
+                faceLandmarkerHelperListener?.onResults(
+                    ResultBundle(
+                        result,
+                        inferenceTime,
+                        input.height,
+                        input.width
+                    )
+                )
+            }
+        }
+        else {
+            faceLandmarkerHelperListener?.onEmpty()
+        }
+    }
+
+    // Return errors thrown during detection to this FaceLandmarkerHelper's
+    // caller
+    private fun returnLivestreamError(error: RuntimeException) {
+        faceLandmarkerHelperListener?.onError(
+            error.message ?: "An unknown error has occurred"
+        )
+    }
+
+    companion object {
+        const val TAG = "FaceLandmarkerHelper"
+        private const val MP_FACE_LANDMARKER_TASK = "face_landmarker.task"
+
+        const val DELEGATE_CPU = 0
+        const val DELEGATE_GPU = 1
+        const val DEFAULT_FACE_DETECTION_CONFIDENCE = 0.5F
+        const val DEFAULT_FACE_TRACKING_CONFIDENCE = 0.5F
+        const val DEFAULT_FACE_PRESENCE_CONFIDENCE = 0.5F
+        const val DEFAULT_NUM_FACES = 1
+        const val OTHER_ERROR = 0
+        const val GPU_ERROR = 1
+    }
+
+    data class ResultBundle(
+        val result: FaceLandmarkerResult,
+        val inferenceTime: Long,
+        val inputImageHeight: Int,
+        val inputImageWidth: Int,
+    )
+
+    interface LandmarkerListener {
+        fun onError(error: String, errorCode: Int = OTHER_ERROR)
+        fun onResults(resultBundle: ResultBundle)
+
+        fun onEmpty() {}
+    }
+}
@@ -1,18 +1,202 @@
 package com.hmwl.sample;

+import android.Manifest;
+import android.content.pm.PackageManager;
+import android.os.Bundle;
 import android.util.Log;
 import android.view.MotionEvent;
 import android.view.View;

-import com.google.androidgamesdk.GameActivity;
+import androidx.annotation.NonNull;
+import androidx.camera.core.AspectRatio;
+import androidx.camera.core.CameraSelector;
+import androidx.camera.core.ImageAnalysis;
+import androidx.camera.core.ImageProxy;
+import androidx.camera.lifecycle.ProcessCameraProvider;
+import androidx.core.app.ActivityCompat;
+import androidx.core.content.ContextCompat;

-public class MainActivity extends GameActivity{
+import com.google.androidgamesdk.GameActivity;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
+import com.google.mediapipe.tasks.vision.core.RunningMode;
+import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener {
    private static final String TAG = "MainActivity";

    static {
        System.loadLibrary("sample");
    }

+    private ProcessCameraProvider cameraProvider;
+    private void initCamera(){
+        backgroundExecutor = Executors.newSingleThreadExecutor();
+        if (hasCameraPermission()) {
+            startCamera();
+        } else {
+            requestCameraPermission();
+        }
+    }
+
+    private void startCamera() {
+        ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
+                ProcessCameraProvider.getInstance(this);
+
+        cameraProviderFuture.addListener(() -> {
+            try {
+                cameraProvider = cameraProviderFuture.get();
+
+                CameraSelector cameraSelector = new CameraSelector.Builder()
+                        .requireLensFacing(CameraSelector.LENS_FACING_BACK)
+                        .build();
+
+                imageAnalyzer = new ImageAnalysis.Builder()
+                        .setTargetAspectRatio(AspectRatio.RATIO_4_3)
+                        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+                        .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
+                        .build();
+
+
+                imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() {
+                    private long lastCaptureTime = 0;
+                    private int frameCount = 0;
+
+                    @Override
+                    public void analyze(@NonNull ImageProxy image) {
+                        frameCount++;
+                        long currentTime = System.currentTimeMillis();
+                        try {
+                            long captureTimeNs = image.getImageInfo().getTimestamp();
+                            long analyzeTime = System.currentTimeMillis();
+
+                            // 简单直接的计算方法
+                            long captureTimeMs = TimeUnit.NANOSECONDS.toMillis(captureTimeNs);
+                            long latency = analyzeTime - captureTimeMs;
+
+                            // 帧率计算
+                            if (lastCaptureTime > 0) {
+                                long frameInterval = captureTimeMs - lastCaptureTime;
+                                double fps = 1000.0 / frameInterval;
+
+                                if (frameCount % 10 == 0) { // 每30帧打印一次
+                                    Log.i("TimeLatency", String.format(
+                                            "Frame %d - Latency: %d ms, FPS: %.1f",
+                                            frameCount, latency, fps
+                                    ));
+                                }
+                            }
+
+                            lastCaptureTime = captureTimeMs;
+
+                            processImageForVulkan(image);
+                            if(frameCount %2 == 0)
+                            {
+                                detectFace(image);
+                            }
+                            else
+                            {
+                                image.close();
+                            }
+                        } finally {
+                            //image.close(); // 确保在这里关闭
+                            //Log.d("Camera", "Image closed, ready for next frame");
+                        }
+                    }
+                });
+
+                cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer);
+
+            } catch (ExecutionException | InterruptedException e) {
+                Log.e("MainActivity", "Error starting camera: " + e.getMessage());
+            }
+        }, ContextCompat.getMainExecutor(this));
+    }
+
+    private void processImageForVulkan(ImageProxy image) {
+        // 获取图像信息
+        int width = image.getWidth();
+        int height = image.getHeight();
+        int format = image.getFormat();
+
+        // 获取图像数据平面
+        ImageProxy.PlaneProxy[] planes = image.getPlanes();
+
+        // 对于 RGBA_8888 格式，通常只有一个平面
+        if (planes.length > 0) {
+            ImageProxy.PlaneProxy plane = planes[0];
+            ByteBuffer buffer = plane.getBuffer();
+            int rowStride = plane.getRowStride();
+            int pixelStride = plane.getPixelStride();
+
+            // 调用 Native 方法处理图像
+            processImageNative(buffer, width, height, format,
+                    rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
+        }
+    }
+
+    // Native 方法
+    private native void processImageNative(ByteBuffer buffer, int width, int height,
+                                           int format, int rowStride, int pixelStride,
+                                           int rotation);
+
+    private void detectFace(ImageProxy imageProxy) {
+        faceLandmarkerHelper.detectLiveStream(
+                imageProxy,false
+        );
+    }
+
+    private boolean hasCameraPermission() {
+        return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
+                == PackageManager.PERMISSION_GRANTED;
+    }
+
+    private void requestCameraPermission() {
+        ActivityCompat.requestPermissions(this,
+                new String[]{Manifest.permission.CAMERA},
+                REQUEST_CAMERA_PERMISSION);
+    }
+
+    private static final int REQUEST_CAMERA_PERMISSION = 1001;
+    ImageAnalysis imageAnalyzer;
+    ExecutorService backgroundExecutor;
+    FaceLandmarkerHelper faceLandmarkerHelper;
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState){
+        super.onCreate(savedInstanceState);
+        initCamera();
+
+        backgroundExecutor.execute(new Runnable() {
+            @Override
+            public void run() {
+                faceLandmarkerHelper = new FaceLandmarkerHelper(
+                        0.5f,
+                        0.5f,
+                        0.5f,
+                        1,
+                        0,
+                        RunningMode.LIVE_STREAM,
+                        MainActivity.this,
+                        MainActivity.this
+                );
+            }
+        });
+    }
+
+
+
    @Override
    public boolean onTouchEvent(MotionEvent event) {
        int action = event.getActionMasked();
@@ -57,4 +241,71 @@ public class MainActivity extends GameActivity{
                        | View.SYSTEM_UI_FLAG_FULLSCREEN
        );
    }
+
+    @Override
+    public void onError(@NotNull String error, int errorCode) {
+
+    }
+
+    private ByteBuffer nativeBuffer = null;
+    float[] points = new float[480 * 3];
+
+
+    public class Vector {
+        public float x, y, z;
+
+        public Vector(float x, float y, float z) {
+            this.x = x;
+            this.y = y;
+            this.z = z;
+        }
+    }
+
+    public float z_rate = -1.0f;
+
+    @Override
+    public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) {
+        FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult();
+        int height = resultBundle.getInputImageHeight();
+        int width = resultBundle.getInputImageWidth();
+        int bufferSize = 480 * 3 * 4;
+
+        if(nativeBuffer == null)
+        {
+            // 创建直接ByteBuffer（在native内存中）
+            nativeBuffer = ByteBuffer.allocateDirect(bufferSize);
+            nativeBuffer.order(ByteOrder.nativeOrder());
+        }
+
+        long start_time =  System.currentTimeMillis();
+
+        int index = 0;
+        // Iterate through each detected face
+        for (List<NormalizedLandmark> faceLandmarks : faceLandmarkerResult.faceLandmarks()) {
+            //should be 1
+
+            int arrayIndex = 0;
+            for (NormalizedLandmark p : faceLandmarks)
+            {
+                //sb.append("[");
+                points[arrayIndex++] = p.x();
+                //sb.append(""+p.x()+",");
+                points[arrayIndex++] = p.y();
+                //sb.append(""+p.y()+",");
+                points[arrayIndex++] = p.z() * z_rate;
+                index++;
+            }
+        }
+
+
+        FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
+        floatBuffer.put(points);
+        floatBuffer.position(0);
+        long cur_time =  System.currentTimeMillis();
+        Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
+        passDataToNative(nativeBuffer, index, width, height);
+        Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
+    }
+
+    public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
 }