xsl/Vulkan-Samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

删除video检测的代码，添加延时输出的代码。

Browse Source
This commit is contained in:
xsl
2025-10-14 17:17:29 +08:00
parent 5b9ca62976
commit 5e851505bb
2 changed files with 103 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt
+92 -143
View File
@@ -139,6 +139,51 @@ class FaceLandmarkerHelper(
}
}
private var frameId: Long = 0
// 简化的延时跟踪
private val frameTimings = mutableMapOf<Long, FrameTiming>()
private var lastFpsUpdateTime: Long = 0
private var frameCount: Int = 0
private var currentFps: Double = 0.0
// 内部数据类用于存储时间信息
private data class FrameTiming(
var captureTime: Long = 0,
var preprocessStartTime: Long = 0,
var preprocessEndTime: Long = 0,
var detectionStartTime: Long = 0
)
// 辅助方法：记录帧时间信息
private fun recordFrameTiming(frameId: Long, block: (FrameTiming) -> Unit) {
val timing = frameTimings.getOrPut(frameId) { FrameTiming() }
block(timing)
}
// 辅助方法：更新FPS计算
private fun updateFps() {
frameCount++
val currentTime = SystemClock.uptimeMillis()
if (lastFpsUpdateTime == 0L) {
lastFpsUpdateTime = currentTime
return
}
val timeDiff = currentTime - lastFpsUpdateTime
if (timeDiff >= 1000) { // 每秒更新一次FPS
currentFps = frameCount * 1000.0 / timeDiff
frameCount = 0
lastFpsUpdateTime = currentTime
}
}
// 辅助方法：清理旧的时间记录
private fun cleanupOldTimings(currentFrameId: Long) {
val framesToRemove = frameTimings.keys.filter { it < currentFrameId - 10 }
framesToRemove.forEach { frameTimings.remove(it) }
}
// Convert the ImageProxy to MP Image and feed it to FacelandmakerHelper.
fun detectLiveStream(
imageProxy: ImageProxy,
@@ -152,6 +197,15 @@ class FaceLandmarkerHelper(
}
val frameTime = SystemClock.uptimeMillis()
val currentFrameId = frameId++
val captureTime = SystemClock.uptimeMillis()
// 记录捕获时间
recordFrameTiming(currentFrameId) {
it.captureTime = captureTime
it.preprocessStartTime = SystemClock.uptimeMillis()
}
// Copy out RGB bits from the frame to a bitmap buffer
val bitmapBuffer =
Bitmap.createBitmap(
@@ -198,152 +252,54 @@ class FaceLandmarkerHelper(
// be returned in returnLivestreamResult function
}
// Accepts the URI for a video file loaded from the user's gallery and attempts to run
// face landmarker inference on the video. This process will evaluate every
// frame in the video and attach the results to a bundle that will be
// returned.
fun detectVideoFile(
videoUri: Uri,
inferenceIntervalMs: Long
): VideoResultBundle? {
if (runningMode != RunningMode.VIDEO) {
throw IllegalArgumentException(
"Attempting to call detectVideoFile" +
" while not using RunningMode.VIDEO"
)
}
// Inference time is the difference between the system time at the start and finish of the
// process
val startTime = SystemClock.uptimeMillis()
var didErrorOccurred = false
// Load frames from the video and run the face landmarker.
val retriever = MediaMetadataRetriever()
retriever.setDataSource(context, videoUri)
val videoLengthMs =
retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)
?.toLong()
// Note: We need to read width/height from frame instead of getting the width/height
// of the video directly because MediaRetriever returns frames that are smaller than the
// actual dimension of the video file.
val firstFrame = retriever.getFrameAtTime(0)
val width = firstFrame?.width
val height = firstFrame?.height
// If the video is invalid, returns a null detection result
if ((videoLengthMs == null) || (width == null) || (height == null)) return null
// Next, we'll get one frame every frameInterval ms, then run detection on these frames.
val resultList = mutableListOf<FaceLandmarkerResult>()
val numberOfFrameToRead = videoLengthMs.div(inferenceIntervalMs)
for (i in 0..numberOfFrameToRead) {
val timestampMs = i * inferenceIntervalMs // ms
retriever
.getFrameAtTime(
timestampMs * 1000, // convert from ms to micro-s
MediaMetadataRetriever.OPTION_CLOSEST
)
?.let { frame ->
// Convert the video frame to ARGB_8888 which is required by the MediaPipe
val argb8888Frame =
if (frame.config == Bitmap.Config.ARGB_8888) frame
else frame.copy(Bitmap.Config.ARGB_8888, false)
// Convert the input Bitmap object to an MPImage object to run inference
val mpImage = BitmapImageBuilder(argb8888Frame).build()
// Run face landmarker using MediaPipe Face Landmarker API
faceLandmarker?.detectForVideo(mpImage, timestampMs)
?.let { detectionResult ->
resultList.add(detectionResult)
} ?: {
didErrorOccurred = true
faceLandmarkerHelperListener?.onError(
"ResultBundle could not be returned" +
" in detectVideoFile"
)
}
}
?: run {
didErrorOccurred = true
faceLandmarkerHelperListener?.onError(
"Frame at specified time could not be" +
" retrieved when detecting in video."
)
}
}
retriever.release()
val inferenceTimePerFrameMs =
(SystemClock.uptimeMillis() - startTime).div(numberOfFrameToRead)
return if (didErrorOccurred) {
null
} else {
VideoResultBundle(resultList, inferenceTimePerFrameMs, height, width)
}
}
// Accepted a Bitmap and runs face landmarker inference on it to return
// results back to the caller
fun detectImage(image: Bitmap): ResultBundle? {
if (runningMode != RunningMode.IMAGE) {
throw IllegalArgumentException(
"Attempting to call detectImage" +
" while not using RunningMode.IMAGE"
)
}
// Inference time is the difference between the system time at the
// start and finish of the process
val startTime = SystemClock.uptimeMillis()
// Convert the input Bitmap object to an MPImage object to run inference
val mpImage = BitmapImageBuilder(image).build()
// Run face landmarker using MediaPipe Face Landmarker API
faceLandmarker?.detect(mpImage)?.also { landmarkResult ->
val inferenceTimeMs = SystemClock.uptimeMillis() - startTime
return ResultBundle(
landmarkResult,
inferenceTimeMs,
image.height,
image.width
)
}
// If faceLandmarker?.detect() returns null, this is likely an error. Returning null
// to indicate this.
faceLandmarkerHelperListener?.onError(
"Face Landmarker failed to detect."
)
return null
}
// Return the landmark result to this FaceLandmarkerHelper's caller
private fun returnLivestreamResult(
result: FaceLandmarkerResult,
input: MPImage
) {
val resultTime = SystemClock.uptimeMillis()
val frameId = result.timestampMs()
if( result.faceLandmarks().size > 0 ) {
val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs()
// 计算各种延时
val timings = frameTimings[frameId]
if (timings != null) {
val totalLatency = resultTime - timings.captureTime
val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime
val detectionLatency = resultTime - timings.detectionStartTime
val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime
faceLandmarkerHelperListener?.onResults(
ResultBundle(
result,
inferenceTime,
input.height,
input.width
Log.d("TimeLatency",
"总延时: ${totalLatency}ms | " +
"预处理: ${preprocessLatency}ms | " +
"检测: ${detectionLatency}ms | " +
"捕获到检测: ${captureToDetectionLatency}ms | " +
"FPS: ${String.format("%.1f", currentFps)}"
)
)
val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs()
faceLandmarkerHelperListener?.onResults(
ResultBundle(
result,
inferenceTime, // 主要是检测时间
input.height,
input.width,
)
)
} else {
val finishTimeMs = SystemClock.uptimeMillis()
val inferenceTime = finishTimeMs - result.timestampMs()
faceLandmarkerHelperListener?.onResults(
ResultBundle(
result,
inferenceTime,
input.height,
input.width
)
)
}
}
else {
faceLandmarkerHelperListener?.onEmpty()
@@ -379,13 +335,6 @@ class FaceLandmarkerHelper(
val inputImageWidth: Int,
)
data class VideoResultBundle(
val results: List<FaceLandmarkerResult>,
val inferenceTime: Long,
val inputImageHeight: Int,
val inputImageWidth: Int,
)
interface LandmarkerListener {
fun onError(error: String, errorCode: Int = OTHER_ERROR)
fun onResults(resultBundle: ResultBundle)
app/android/java/com/khronos/vulkan_samples/MainActivity.java
+11 -1
View File
@@ -6,6 +6,7 @@ import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import android.view.WindowManager;
import android.widget.Toast;
@@ -182,6 +183,12 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L
@Override
public void analyze(@NonNull ImageProxy image) {
try {
long analyzeTime = SystemClock.elapsedRealtimeNanos();
long captureTime = image.getImageInfo().getTimestamp();
long latencyNs = analyzeTime - captureTime;
double latencyMs = latencyNs / 1_000_000.0;
Log.d("CameraLatency", String.format("Image capture to analyze latency: %.2f ms", latencyMs));
//Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight());
processImageForVulkan(image);
detectFace(image);
@@ -315,7 +322,7 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L
nativeBuffer.order(ByteOrder.nativeOrder());
}
long start_time = System.currentTimeMillis();
// ArrayList<Vector> vertexs = new ArrayList<>();
// ArrayList<Integer> triangle_index = new ArrayList<>();
@@ -358,7 +365,10 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L
FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
floatBuffer.put(points);
floatBuffer.position(0);
long cur_time = System.currentTimeMillis();
Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time));
passDataToNative(nativeBuffer, index, width, height);
Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time));
}
public float z_rate = -1.0f;