package com.khronos.vulkan_samples; import android.Manifest; import android.content.pm.PackageManager; import android.os.Bundle; import android.os.Handler; import android.os.Looper; import android.util.Log; import android.view.WindowManager; import android.widget.Toast; import androidx.annotation.NonNull; import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.ImageProxy; import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import com.google.androidgamesdk.GameActivity; import com.google.common.util.concurrent.ListenableFuture; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; import com.google.mediapipe.tasks.vision.core.RunningMode; import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult; import org.jetbrains.annotations.NotNull; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener, CameraDataFetcher.CameraDataCallback { private ProcessCameraProvider cameraProvider; private void initCamera(){ backgroundExecutor = Executors.newSingleThreadExecutor(); if (hasCameraPermission()) { startCamera(); } else { requestCameraPermission(); } } private void detectFace(ImageProxy imageProxy) { faceLandmarkerHelper.detectLiveStream( imageProxy,false ); } private Handler handler = new Handler(Looper.getMainLooper()); private Runnable cameraDataRunnable; private static final long INTERVAL = 1000; // 5秒间隔 @Override protected void onCreate(Bundle savedInstanceState) { String native_lib_name = getResources().getString(R.string.native_lib_name); try { System.loadLibrary(native_lib_name); } catch (UnsatisfiedLinkError e) { Toast.makeText(getApplicationContext(), "Native code library failed to load.", Toast.LENGTH_SHORT).show(); } String[] argArray = new String[2]; argArray[0] = "sample"; argArray[1] = "texture_loading";//"hello_triangle"; sendArgumentsToPlatform(argArray); super.onCreate(savedInstanceState); // 隐藏状态栏 getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); initCamera(); backgroundExecutor.execute(new Runnable() { @Override public void run() { faceLandmarkerHelper = new FaceLandmarkerHelper( 0.5f, 0.5f, 0.5f, 1, 0, RunningMode.LIVE_STREAM, MainActivity.this, MainActivity.this ); } }); // 初始化Runnable cameraDataRunnable = new Runnable() { @Override public void run() { fetchCameraData(); handler.postDelayed(this, INTERVAL); // 再次调度 } }; // 开始周期性调用 startPeriodicFetching(); Log.i("MainActivity", "onCreate: "); } boolean canFetch = true; private void fetchCameraData() { if(canFetch) { CameraDataFetcher.fetchCameraData("http://175.178.100.240:5000/data", MainActivity.this); canFetch = false; } } private void startPeriodicFetching() { handler.postDelayed(cameraDataRunnable, INTERVAL); } private void stopPeriodicFetching() { handler.removeCallbacks(cameraDataRunnable); } @Override protected void onDestroy() { super.onDestroy(); stopPeriodicFetching(); // 避免内存泄漏 } private boolean hasCameraPermission() { return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } private void requestCameraPermission() { ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, REQUEST_CAMERA_PERMISSION); } private static final int REQUEST_CAMERA_PERMISSION = 1001; ImageAnalysis imageAnalyzer; ExecutorService backgroundExecutor; FaceLandmarkerHelper faceLandmarkerHelper; private void startCamera() { ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(this); cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); CameraSelector cameraSelector = new CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_BACK) .build(); imageAnalyzer = new ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build(); imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() { @Override public void analyze(@NonNull ImageProxy image) { try { //Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight()); processImageForVulkan(image); detectFace(image); } finally { //image.close(); // 确保在这里关闭 //Log.d("Camera", "Image closed, ready for next frame"); } } }); cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer); } catch (ExecutionException | InterruptedException e) { Log.e("MainActivity", "Error starting camera: " + e.getMessage()); } }, ContextCompat.getMainExecutor(this)); } private void processImageForVulkan(ImageProxy image) { // 获取图像信息 int width = image.getWidth(); int height = image.getHeight(); int format = image.getFormat(); // 获取图像数据平面 ImageProxy.PlaneProxy[] planes = image.getPlanes(); // 对于 RGBA_8888 格式,通常只有一个平面 if (planes.length > 0) { ImageProxy.PlaneProxy plane = planes[0]; ByteBuffer buffer = plane.getBuffer(); int rowStride = plane.getRowStride(); int pixelStride = plane.getPixelStride(); // 调用 Native 方法处理图像 processImageNative(buffer, width, height, format, rowStride, pixelStride, image.getImageInfo().getRotationDegrees()); } } // Native 方法 private native void processImageNative(ByteBuffer buffer, int width, int height, int format, int rowStride, int pixelStride, int rotation); private static final int REQUEST_CODE_PERMISSIONS = 1001; private static final String[] REQUIRED_PERMISSIONS = new String[]{Manifest.permission.CAMERA}; private static final String TAG = "MainActivity"; private boolean allPermissionsGranted() { for (String permission : REQUIRED_PERMISSIONS) { if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) { return false; } } return true; } @Override public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (requestCode == REQUEST_CODE_PERMISSIONS) { if (allPermissionsGranted()) { startCamera(); } else { Log.e(TAG, "Permissions not granted by the user."); finish(); // Close the activity if permission is denied } } } private native void sendArgumentsToPlatform(String[] args); @Override public void onError(@NotNull String error, int errorCode) { Log.e(TAG, "onError: Face Error"); } private ByteBuffer nativeBuffer = null; float[] points = new float[480 * 3]; @Override public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) { FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult(); int height = resultBundle.getInputImageHeight(); int width = resultBundle.getInputImageWidth(); int bufferSize = 480 * 3 * 4; if(nativeBuffer == null) { // 创建直接ByteBuffer(在native内存中) nativeBuffer = ByteBuffer.allocateDirect(bufferSize); nativeBuffer.order(ByteOrder.nativeOrder()); } int index = 0; // Iterate through each detected face for (List faceLandmarks : faceLandmarkerResult.faceLandmarks()) { //should be 1 int arrayIndex = 0; for (NormalizedLandmark p : faceLandmarks) { //Log.i(TAG, "Index" + index + " x:"+p.x() + " y:"+p.y() + " z:"+p.z()); points[arrayIndex++] = p.x(); points[arrayIndex++] = p.y(); points[arrayIndex++] = p.z() * z_rate; index++; } } // 转换为FloatBuffer并填充数据 FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); floatBuffer.put(points); floatBuffer.position(0); passDataToNative(nativeBuffer, index, width, height); } public float z_rate = 1.0f; public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height); public native void upDateCameraData(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz, float camx, float camy, float camz, float dashSize, float gapSize, float dashOffset, float uAASize); @Override public void onSuccess(CameraDataFetcher.CameraData data) { z_rate = data.z_rate; upDateCameraData(data.fov, data.x, data.y, data.z, data.sx, data.sy, data.sz, data.rotx, data.roty, data.rotz, data.camx, data.camy, data.camz, data.dashSize, data.gapSize, data.dashOffset, data.uAASize ); canFetch = true; Log.e(TAG, "GetCameraData " + data.toString()); } @Override public void onError(String errorMessage) { Log.e(TAG, "GetCameraData Error"); canFetch = true; } }