package com.khronos.vulkan_samples; import android.Manifest; import android.content.pm.PackageManager; import android.os.Bundle; import android.util.Log; import android.widget.Toast; import androidx.annotation.NonNull; import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.ImageProxy; import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import com.google.androidgamesdk.GameActivity; import com.google.common.util.concurrent.ListenableFuture; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; import com.google.mediapipe.tasks.vision.core.RunningMode; import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult; import org.jetbrains.annotations.NotNull; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener{ private ProcessCameraProvider cameraProvider; private void initCamera(){ backgroundExecutor = Executors.newSingleThreadExecutor(); if (hasCameraPermission()) { startCamera(); } else { requestCameraPermission(); } } private void detectFace(ImageProxy imageProxy) { faceLandmarkerHelper.detectLiveStream( imageProxy,true ); } @Override protected void onCreate(Bundle savedInstanceState) { String native_lib_name = getResources().getString(R.string.native_lib_name); try { System.loadLibrary(native_lib_name); } catch (UnsatisfiedLinkError e) { Toast.makeText(getApplicationContext(), "Native code library failed to load.", Toast.LENGTH_SHORT).show(); } String[] argArray = new String[2]; argArray[0] = "sample"; argArray[1] = "texture_loading";//"hello_triangle"; sendArgumentsToPlatform(argArray); super.onCreate(savedInstanceState); initCamera(); backgroundExecutor.execute(new Runnable() { @Override public void run() { faceLandmarkerHelper = new FaceLandmarkerHelper( 0.5f, 0.5f, 0.5f, 1, 0, RunningMode.LIVE_STREAM, MainActivity.this, MainActivity.this ); } }); Log.i("MainActivity", "onCreate: "); } private boolean hasCameraPermission() { return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } private void requestCameraPermission() { ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, REQUEST_CAMERA_PERMISSION); } private static final int REQUEST_CAMERA_PERMISSION = 1001; ImageAnalysis imageAnalyzer; ExecutorService backgroundExecutor; FaceLandmarkerHelper faceLandmarkerHelper; private void startCamera() { ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(this); cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); CameraSelector cameraSelector = new CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_FRONT) .build(); imageAnalyzer = new ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build(); imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() { @Override public void analyze(@NonNull ImageProxy image) { try { Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight()); processImageForVulkan(image); detectFace(image); } finally { //image.close(); // 确保在这里关闭 Log.d("Camera", "Image closed, ready for next frame"); } } }); cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer); } catch (ExecutionException | InterruptedException e) { Log.e("MainActivity", "Error starting camera: " + e.getMessage()); } }, ContextCompat.getMainExecutor(this)); } private void processImageForVulkan(ImageProxy image) { // 获取图像信息 int width = image.getWidth(); int height = image.getHeight(); int format = image.getFormat(); // 获取图像数据平面 ImageProxy.PlaneProxy[] planes = image.getPlanes(); // 对于 RGBA_8888 格式,通常只有一个平面 if (planes.length > 0) { ImageProxy.PlaneProxy plane = planes[0]; ByteBuffer buffer = plane.getBuffer(); int rowStride = plane.getRowStride(); int pixelStride = plane.getPixelStride(); // 调用 Native 方法处理图像 processImageNative(buffer, width, height, format, rowStride, pixelStride, image.getImageInfo().getRotationDegrees()); } } // Native 方法 private native void processImageNative(ByteBuffer buffer, int width, int height, int format, int rowStride, int pixelStride, int rotation); private static final int REQUEST_CODE_PERMISSIONS = 1001; private static final String[] REQUIRED_PERMISSIONS = new String[]{Manifest.permission.CAMERA}; private static final String TAG = "MainActivity"; private boolean allPermissionsGranted() { for (String permission : REQUIRED_PERMISSIONS) { if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) { return false; } } return true; } @Override public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (requestCode == REQUEST_CODE_PERMISSIONS) { if (allPermissionsGranted()) { startCamera(); } else { Log.e(TAG, "Permissions not granted by the user."); finish(); // Close the activity if permission is denied } } } private native void sendArgumentsToPlatform(String[] args); @Override public void onError(@NotNull String error, int errorCode) { Log.e(TAG, "onError: Face Error"); } private ByteBuffer nativeBuffer = null; float[] points = new float[480 * 3]; @Override public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) { FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult(); int height = resultBundle.getInputImageHeight(); int width = resultBundle.getInputImageWidth(); int bufferSize = 480 * 3 * 4; if(nativeBuffer == null) { // 创建直接ByteBuffer(在native内存中) nativeBuffer = ByteBuffer.allocateDirect(bufferSize); nativeBuffer.order(ByteOrder.nativeOrder()); } int index = 0; // Iterate through each detected face for (List faceLandmarks : faceLandmarkerResult.faceLandmarks()) { //should be 1 int arrayIndex = 0; for (NormalizedLandmark p : faceLandmarks) { //Log.i(TAG, "Index" + index + " x:"+p.x() + " y:"+p.y() + " z:"+p.z()); points[arrayIndex++] = p.x(); points[arrayIndex++] = p.y(); points[arrayIndex++] = p.z(); index++; } } // 转换为FloatBuffer并填充数据 FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); floatBuffer.put(points); floatBuffer.position(0); passDataToNative(nativeBuffer, index, width, height); } public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height); }