397 lines
14 KiB
Java
397 lines
14 KiB
Java
package com.khronos.vulkan_samples;
|
|
|
|
|
|
import android.Manifest;
|
|
import android.content.pm.PackageManager;
|
|
import android.os.Bundle;
|
|
import android.os.Handler;
|
|
import android.os.Looper;
|
|
import android.os.SystemClock;
|
|
import android.util.Log;
|
|
import android.view.WindowManager;
|
|
import android.widget.Toast;
|
|
|
|
import androidx.annotation.NonNull;
|
|
import androidx.camera.core.AspectRatio;
|
|
import androidx.camera.core.CameraSelector;
|
|
import androidx.camera.core.ImageAnalysis;
|
|
import androidx.camera.core.ImageProxy;
|
|
import androidx.camera.lifecycle.ProcessCameraProvider;
|
|
import androidx.core.app.ActivityCompat;
|
|
import androidx.core.content.ContextCompat;
|
|
|
|
import com.google.androidgamesdk.GameActivity;
|
|
import com.google.common.util.concurrent.ListenableFuture;
|
|
import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
|
|
import com.google.mediapipe.tasks.vision.core.RunningMode;
|
|
import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarker;
|
|
import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult;
|
|
|
|
import org.jetbrains.annotations.NotNull;
|
|
|
|
import java.nio.ByteBuffer;
|
|
import java.nio.ByteOrder;
|
|
import java.nio.FloatBuffer;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.concurrent.ExecutionException;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
|
|
public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener, CameraDataFetcher.CameraDataCallback {
|
|
private ProcessCameraProvider cameraProvider;
|
|
private void initCamera(){
|
|
backgroundExecutor = Executors.newSingleThreadExecutor();
|
|
if (hasCameraPermission()) {
|
|
startCamera();
|
|
} else {
|
|
requestCameraPermission();
|
|
}
|
|
}
|
|
|
|
private void detectFace(ImageProxy imageProxy) {
|
|
faceLandmarkerHelper.detectLiveStream(
|
|
imageProxy,false
|
|
);
|
|
}
|
|
|
|
private Handler handler = new Handler(Looper.getMainLooper());
|
|
private Runnable cameraDataRunnable;
|
|
private static final long INTERVAL = 1000; // 5秒间隔
|
|
|
|
@Override
|
|
protected void onCreate(Bundle savedInstanceState) {
|
|
|
|
// if (!AssetsCopyUtil.isExternalStorageAvailable()) {
|
|
// Log.w(TAG, "External storage not available, skipping assets copy");
|
|
// return;
|
|
// }
|
|
|
|
//AssetsCopyUtil.copyAssetsToAppFiles(this, "assets");
|
|
//AssetsCopyUtil.copyAssetsToAppFiles(this, "shaders");
|
|
|
|
|
|
String native_lib_name = getResources().getString(R.string.native_lib_name);
|
|
try {
|
|
System.loadLibrary(native_lib_name);
|
|
} catch (UnsatisfiedLinkError e) {
|
|
Toast.makeText(getApplicationContext(), "Native code library failed to load.", Toast.LENGTH_SHORT).show();
|
|
}
|
|
String[] argArray = new String[2];
|
|
argArray[0] = "sample";
|
|
argArray[1] = "texture_loading";//"hello_triangle";
|
|
sendArgumentsToPlatform(argArray);
|
|
super.onCreate(savedInstanceState);
|
|
|
|
// 隐藏状态栏
|
|
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
|
|
WindowManager.LayoutParams.FLAG_FULLSCREEN);
|
|
|
|
initCamera();
|
|
|
|
backgroundExecutor.execute(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
faceLandmarkerHelper = new FaceLandmarkerHelper(
|
|
0.5f,
|
|
0.5f,
|
|
0.5f,
|
|
1,
|
|
0,
|
|
RunningMode.LIVE_STREAM,
|
|
MainActivity.this,
|
|
MainActivity.this
|
|
);
|
|
}
|
|
});
|
|
|
|
// 初始化Runnable
|
|
// cameraDataRunnable = new Runnable() {
|
|
// @Override
|
|
// public void run() {
|
|
// fetchCameraData();
|
|
// handler.postDelayed(this, INTERVAL); // 再次调度
|
|
// }
|
|
// };
|
|
|
|
// 开始周期性调用
|
|
// startPeriodicFetching();
|
|
|
|
|
|
Log.i("MainActivity", "onCreate: ");
|
|
}
|
|
|
|
boolean canFetch = true;
|
|
private void fetchCameraData() {
|
|
if(canFetch)
|
|
{
|
|
CameraDataFetcher.fetchCameraData("http://175.178.100.240:5000/data", MainActivity.this);
|
|
canFetch = false;
|
|
}
|
|
}
|
|
private void startPeriodicFetching() {
|
|
handler.postDelayed(cameraDataRunnable, INTERVAL);
|
|
}
|
|
|
|
private void stopPeriodicFetching() {
|
|
handler.removeCallbacks(cameraDataRunnable);
|
|
}
|
|
|
|
@Override
|
|
protected void onDestroy() {
|
|
super.onDestroy();
|
|
stopPeriodicFetching(); // 避免内存泄漏
|
|
}
|
|
|
|
private boolean hasCameraPermission() {
|
|
return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
|
|
== PackageManager.PERMISSION_GRANTED;
|
|
}
|
|
|
|
private void requestCameraPermission() {
|
|
ActivityCompat.requestPermissions(this,
|
|
new String[]{Manifest.permission.CAMERA},
|
|
REQUEST_CAMERA_PERMISSION);
|
|
}
|
|
|
|
private static final int REQUEST_CAMERA_PERMISSION = 1001;
|
|
ImageAnalysis imageAnalyzer;
|
|
ExecutorService backgroundExecutor;
|
|
FaceLandmarkerHelper faceLandmarkerHelper;
|
|
|
|
private void startCamera() {
|
|
ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
|
|
ProcessCameraProvider.getInstance(this);
|
|
|
|
cameraProviderFuture.addListener(() -> {
|
|
try {
|
|
cameraProvider = cameraProviderFuture.get();
|
|
|
|
CameraSelector cameraSelector = new CameraSelector.Builder()
|
|
.requireLensFacing(CameraSelector.LENS_FACING_BACK)
|
|
.build();
|
|
|
|
imageAnalyzer = new ImageAnalysis.Builder()
|
|
.setTargetAspectRatio(AspectRatio.RATIO_4_3)
|
|
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
|
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
|
|
.build();
|
|
|
|
|
|
imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() {
|
|
|
|
@Override
|
|
public void analyze(@NonNull ImageProxy image) {
|
|
try {
|
|
long analyzeTime = SystemClock.elapsedRealtimeNanos();
|
|
long captureTime = image.getImageInfo().getTimestamp();
|
|
long latencyNs = analyzeTime - captureTime;
|
|
double latencyMs = latencyNs / 1_000_000.0;
|
|
|
|
Log.d("CameraLatency", String.format("Image capture to analyze latency: %.2f ms", latencyMs));
|
|
//Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight());
|
|
processImageForVulkan(image);
|
|
detectFace(image);
|
|
} finally {
|
|
//image.close(); // 确保在这里关闭
|
|
//Log.d("Camera", "Image closed, ready for next frame");
|
|
}
|
|
}
|
|
});
|
|
|
|
cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer);
|
|
|
|
} catch (ExecutionException | InterruptedException e) {
|
|
Log.e("MainActivity", "Error starting camera: " + e.getMessage());
|
|
}
|
|
}, ContextCompat.getMainExecutor(this));
|
|
}
|
|
|
|
|
|
private void processImageForVulkan(ImageProxy image) {
|
|
// 获取图像信息
|
|
int width = image.getWidth();
|
|
int height = image.getHeight();
|
|
int format = image.getFormat();
|
|
|
|
// 获取图像数据平面
|
|
ImageProxy.PlaneProxy[] planes = image.getPlanes();
|
|
|
|
// 对于 RGBA_8888 格式，通常只有一个平面
|
|
if (planes.length > 0) {
|
|
ImageProxy.PlaneProxy plane = planes[0];
|
|
ByteBuffer buffer = plane.getBuffer();
|
|
int rowStride = plane.getRowStride();
|
|
int pixelStride = plane.getPixelStride();
|
|
|
|
// 调用 Native 方法处理图像
|
|
processImageNative(buffer, width, height, format,
|
|
rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
|
|
}
|
|
}
|
|
|
|
// Native 方法
|
|
private native void processImageNative(ByteBuffer buffer, int width, int height,
|
|
int format, int rowStride, int pixelStride,
|
|
int rotation);
|
|
|
|
|
|
private static final int REQUEST_CODE_PERMISSIONS = 1001;
|
|
private static final String[] REQUIRED_PERMISSIONS = new String[]{Manifest.permission.CAMERA};
|
|
private static final String TAG = "MainActivity";
|
|
|
|
private boolean allPermissionsGranted() {
|
|
for (String permission : REQUIRED_PERMISSIONS) {
|
|
if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
@Override
|
|
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
|
|
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
|
|
if (requestCode == REQUEST_CODE_PERMISSIONS) {
|
|
if (allPermissionsGranted()) {
|
|
startCamera();
|
|
} else {
|
|
Log.e(TAG, "Permissions not granted by the user.");
|
|
finish(); // Close the activity if permission is denied
|
|
}
|
|
}
|
|
}
|
|
|
|
private native void sendArgumentsToPlatform(String[] args);
|
|
|
|
@Override
|
|
public void onError(@NotNull String error, int errorCode) {
|
|
Log.e(TAG, "onError: Face Error");
|
|
}
|
|
|
|
private ByteBuffer nativeBuffer = null;
|
|
float[] points = new float[480 * 3];
|
|
|
|
|
|
public class Vector {
|
|
public float x, y, z;
|
|
|
|
public Vector(float x, float y, float z) {
|
|
this.x = x;
|
|
this.y = y;
|
|
this.z = z;
|
|
}
|
|
}
|
|
|
|
public static String toObjString(ArrayList<Vector> vertices, ArrayList<Integer> triangleIndices) {
|
|
StringBuilder sb = new StringBuilder();
|
|
|
|
// 写入顶点数据 (v x y z)
|
|
for (Vector v : vertices) {
|
|
sb.append(String.format("v %.6f %.6f %.6f\n", v.x, v.y, v.z));
|
|
}
|
|
|
|
// 写入面数据 (f i1 i2 i3)，OBJ索引从1开始，因此每个索引+1
|
|
int size = triangleIndices.size();
|
|
for (int i = 0; i < size; i += 3) {
|
|
if (i + 2 < size) {
|
|
int i1 = triangleIndices.get(i) + 1;
|
|
int i2 = triangleIndices.get(i + 1) + 1;
|
|
int i3 = triangleIndices.get(i + 2) + 1;
|
|
sb.append(String.format("f %d %d %d\n", i1, i2, i3));
|
|
} else {
|
|
// 如果剩余索引不足3个，说明数据不完整
|
|
System.err.println("Warning: Incomplete triangle face at the end of index list.");
|
|
}
|
|
}
|
|
|
|
return sb.toString();
|
|
}
|
|
|
|
@Override
|
|
public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) {
|
|
FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult();
|
|
int height = resultBundle.getInputImageHeight();
|
|
int width = resultBundle.getInputImageWidth();
|
|
int bufferSize = 480 * 3 * 4;
|
|
|
|
if(nativeBuffer == null)
|
|
{
|
|
// 创建直接ByteBuffer（在native内存中）
|
|
nativeBuffer = ByteBuffer.allocateDirect(bufferSize);
|
|
nativeBuffer.order(ByteOrder.nativeOrder());
|
|
}
|
|
|
|
long start_time = System.currentTimeMillis();
|
|
|
|
// ArrayList<Vector> vertexs = new ArrayList<>();
|
|
// ArrayList<Integer> triangle_index = new ArrayList<>();
|
|
//
|
|
// for (var conn : FaceLandmarker.FACE_LANDMARKS_TESSELATION) {
|
|
// triangle_index.add(conn.start());
|
|
// }
|
|
|
|
|
|
//StringBuilder sb = new StringBuilder();
|
|
//sb.append("[");
|
|
|
|
int index = 0;
|
|
// Iterate through each detected face
|
|
for (List<NormalizedLandmark> faceLandmarks : faceLandmarkerResult.faceLandmarks()) {
|
|
//should be 1
|
|
|
|
int arrayIndex = 0;
|
|
for (NormalizedLandmark p : faceLandmarks)
|
|
{
|
|
//sb.append("[");
|
|
points[arrayIndex++] = p.x();
|
|
//sb.append(""+p.x()+",");
|
|
points[arrayIndex++] = p.y();
|
|
//sb.append(""+p.y()+",");
|
|
points[arrayIndex++] = p.z() * z_rate;
|
|
//sb.append(""+p.z());
|
|
//vertexs.add(new Vector(p.x(), p.y(), p.z()));
|
|
//sb.append("],");
|
|
index++;
|
|
}
|
|
}
|
|
|
|
//sb.append("]");
|
|
//String objStr = sb.toString();
|
|
|
|
//String objStr = toObjString(vertexs, triangle_index);
|
|
|
|
// 转换为FloatBuffer并填充数据
|
|
FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
|
|
floatBuffer.put(points);
|
|
floatBuffer.position(0);
|
|
long cur_time = System.currentTimeMillis();
|
|
Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
|
|
passDataToNative(nativeBuffer, index, width, height);
|
|
Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
|
|
}
|
|
|
|
public float z_rate = -1.0f;
|
|
|
|
public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
|
|
public native void upDateCameraData(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz,
|
|
float camx, float camy, float camz, float dashSize, float gapSize, float dashOffset, float uAASize);
|
|
|
|
@Override
|
|
public void onSuccess(CameraDataFetcher.CameraData data) {
|
|
z_rate = data.z_rate;
|
|
upDateCameraData(data.fov, data.x, data.y, data.z, data.sx, data.sy, data.sz,
|
|
data.rotx, data.roty, data.rotz,
|
|
data.camx, data.camy, data.camz,
|
|
data.dashSize, data.gapSize, data.dashOffset, data.uAASize
|
|
);
|
|
canFetch = true;
|
|
Log.e(TAG, "GetCameraData " + data.toString());
|
|
}
|
|
|
|
@Override
|
|
public void onError(String errorMessage) {
|
|
Log.e(TAG, "GetCameraData Error");
|
|
canFetch = true;
|
|
}
|
|
} |