package com.khronos.vulkan_samples; import android.Manifest; import android.content.pm.PackageManager; import android.os.Bundle; import android.os.Handler; import android.os.Looper; import android.util.Log; import android.view.WindowManager; import android.widget.Toast; import androidx.annotation.NonNull; import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.ImageProxy; import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import com.google.androidgamesdk.GameActivity; import com.google.common.util.concurrent.ListenableFuture; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; import com.google.mediapipe.tasks.vision.core.RunningMode; import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarker; import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult; import org.jetbrains.annotations.NotNull; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener, CameraDataFetcher.CameraDataCallback { private ProcessCameraProvider cameraProvider; private void initCamera(){ backgroundExecutor = Executors.newSingleThreadExecutor(); if (hasCameraPermission()) { startCamera(); } else { requestCameraPermission(); } } private void detectFace(ImageProxy imageProxy) { faceLandmarkerHelper.detectLiveStream( imageProxy,false ); } private Handler handler = new Handler(Looper.getMainLooper()); private Runnable cameraDataRunnable; private static final long INTERVAL = 1000; // 5秒间隔 @Override protected void onCreate(Bundle savedInstanceState) { // if (!AssetsCopyUtil.isExternalStorageAvailable()) { // Log.w(TAG, "External storage not available, skipping assets copy"); // return; // } //AssetsCopyUtil.copyAssetsToAppFiles(this, "assets"); //AssetsCopyUtil.copyAssetsToAppFiles(this, "shaders"); String native_lib_name = getResources().getString(R.string.native_lib_name); try { System.loadLibrary(native_lib_name); } catch (UnsatisfiedLinkError e) { Toast.makeText(getApplicationContext(), "Native code library failed to load.", Toast.LENGTH_SHORT).show(); } String[] argArray = new String[2]; argArray[0] = "sample"; argArray[1] = "texture_loading";//"hello_triangle"; sendArgumentsToPlatform(argArray); super.onCreate(savedInstanceState); // 隐藏状态栏 getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); initCamera(); backgroundExecutor.execute(new Runnable() { @Override public void run() { faceLandmarkerHelper = new FaceLandmarkerHelper( 0.5f, 0.5f, 0.5f, 1, 0, RunningMode.LIVE_STREAM, MainActivity.this, MainActivity.this ); } }); // 初始化Runnable // cameraDataRunnable = new Runnable() { // @Override // public void run() { // fetchCameraData(); // handler.postDelayed(this, INTERVAL); // 再次调度 // } // }; // 开始周期性调用 // startPeriodicFetching(); Log.i("MainActivity", "onCreate: "); } boolean canFetch = true; private void fetchCameraData() { if(canFetch) { CameraDataFetcher.fetchCameraData("http://175.178.100.240:5000/data", MainActivity.this); canFetch = false; } } private void startPeriodicFetching() { handler.postDelayed(cameraDataRunnable, INTERVAL); } private void stopPeriodicFetching() { handler.removeCallbacks(cameraDataRunnable); } @Override protected void onDestroy() { super.onDestroy(); stopPeriodicFetching(); // 避免内存泄漏 } private boolean hasCameraPermission() { return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } private void requestCameraPermission() { ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, REQUEST_CAMERA_PERMISSION); } private static final int REQUEST_CAMERA_PERMISSION = 1001; ImageAnalysis imageAnalyzer; ExecutorService backgroundExecutor; FaceLandmarkerHelper faceLandmarkerHelper; private void startCamera() { ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(this); cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); CameraSelector cameraSelector = new CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_BACK) .build(); imageAnalyzer = new ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build(); imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() { @Override public void analyze(@NonNull ImageProxy image) { try { //Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight()); processImageForVulkan(image); detectFace(image); } finally { //image.close(); // 确保在这里关闭 //Log.d("Camera", "Image closed, ready for next frame"); } } }); cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer); } catch (ExecutionException | InterruptedException e) { Log.e("MainActivity", "Error starting camera: " + e.getMessage()); } }, ContextCompat.getMainExecutor(this)); } private void processImageForVulkan(ImageProxy image) { // 获取图像信息 int width = image.getWidth(); int height = image.getHeight(); int format = image.getFormat(); // 获取图像数据平面 ImageProxy.PlaneProxy[] planes = image.getPlanes(); // 对于 RGBA_8888 格式,通常只有一个平面 if (planes.length > 0) { ImageProxy.PlaneProxy plane = planes[0]; ByteBuffer buffer = plane.getBuffer(); int rowStride = plane.getRowStride(); int pixelStride = plane.getPixelStride(); // 调用 Native 方法处理图像 processImageNative(buffer, width, height, format, rowStride, pixelStride, image.getImageInfo().getRotationDegrees()); } } // Native 方法 private native void processImageNative(ByteBuffer buffer, int width, int height, int format, int rowStride, int pixelStride, int rotation); private static final int REQUEST_CODE_PERMISSIONS = 1001; private static final String[] REQUIRED_PERMISSIONS = new String[]{Manifest.permission.CAMERA}; private static final String TAG = "MainActivity"; private boolean allPermissionsGranted() { for (String permission : REQUIRED_PERMISSIONS) { if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) { return false; } } return true; } @Override public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (requestCode == REQUEST_CODE_PERMISSIONS) { if (allPermissionsGranted()) { startCamera(); } else { Log.e(TAG, "Permissions not granted by the user."); finish(); // Close the activity if permission is denied } } } private native void sendArgumentsToPlatform(String[] args); @Override public void onError(@NotNull String error, int errorCode) { Log.e(TAG, "onError: Face Error"); } private ByteBuffer nativeBuffer = null; float[] points = new float[480 * 3]; public class Vector { public float x, y, z; public Vector(float x, float y, float z) { this.x = x; this.y = y; this.z = z; } } public static String toObjString(ArrayList vertices, ArrayList triangleIndices) { StringBuilder sb = new StringBuilder(); // 写入顶点数据 (v x y z) for (Vector v : vertices) { sb.append(String.format("v %.6f %.6f %.6f\n", v.x, v.y, v.z)); } // 写入面数据 (f i1 i2 i3),OBJ索引从1开始,因此每个索引+1 int size = triangleIndices.size(); for (int i = 0; i < size; i += 3) { if (i + 2 < size) { int i1 = triangleIndices.get(i) + 1; int i2 = triangleIndices.get(i + 1) + 1; int i3 = triangleIndices.get(i + 2) + 1; sb.append(String.format("f %d %d %d\n", i1, i2, i3)); } else { // 如果剩余索引不足3个,说明数据不完整 System.err.println("Warning: Incomplete triangle face at the end of index list."); } } return sb.toString(); } @Override public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) { FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult(); int height = resultBundle.getInputImageHeight(); int width = resultBundle.getInputImageWidth(); int bufferSize = 480 * 3 * 4; if(nativeBuffer == null) { // 创建直接ByteBuffer(在native内存中) nativeBuffer = ByteBuffer.allocateDirect(bufferSize); nativeBuffer.order(ByteOrder.nativeOrder()); } // ArrayList vertexs = new ArrayList<>(); // ArrayList triangle_index = new ArrayList<>(); // // for (var conn : FaceLandmarker.FACE_LANDMARKS_TESSELATION) { // triangle_index.add(conn.start()); // } //StringBuilder sb = new StringBuilder(); //sb.append("["); int index = 0; // Iterate through each detected face for (List faceLandmarks : faceLandmarkerResult.faceLandmarks()) { //should be 1 int arrayIndex = 0; for (NormalizedLandmark p : faceLandmarks) { //sb.append("["); points[arrayIndex++] = p.x(); //sb.append(""+p.x()+","); points[arrayIndex++] = p.y(); //sb.append(""+p.y()+","); points[arrayIndex++] = p.z() * z_rate; //sb.append(""+p.z()); //vertexs.add(new Vector(p.x(), p.y(), p.z())); //sb.append("],"); index++; } } //sb.append("]"); //String objStr = sb.toString(); //String objStr = toObjString(vertexs, triangle_index); // 转换为FloatBuffer并填充数据 FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); floatBuffer.put(points); floatBuffer.position(0); passDataToNative(nativeBuffer, index, width, height); } public float z_rate = -1.0f; public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height); public native void upDateCameraData(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz, float camx, float camy, float camz, float dashSize, float gapSize, float dashOffset, float uAASize); @Override public void onSuccess(CameraDataFetcher.CameraData data) { z_rate = data.z_rate; upDateCameraData(data.fov, data.x, data.y, data.z, data.sx, data.sy, data.sz, data.rotx, data.roty, data.rotz, data.camx, data.camy, data.camz, data.dashSize, data.gapSize, data.dashOffset, data.uAASize ); canFetch = true; Log.e(TAG, "GetCameraData " + data.toString()); } @Override public void onError(String errorMessage) { Log.e(TAG, "GetCameraData Error"); canFetch = true; } }