From 5e851505bbbc92bca4d074e919cb31a1b33625b6 Mon Sep 17 00:00:00 2001 From: Xiang Silian Date: Tue, 14 Oct 2025 17:17:29 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4video=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=EF=BC=8C=E6=B7=BB=E5=8A=A0=E5=BB=B6?= =?UTF-8?q?=E6=97=B6=E8=BE=93=E5=87=BA=E7=9A=84=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vulkan_samples/FaceLandmarkerHelper.kt | 235 +++++++----------- .../khronos/vulkan_samples/MainActivity.java | 12 +- 2 files changed, 103 insertions(+), 144 deletions(-) diff --git a/app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt b/app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt index 6ba7817..3f359c7 100644 --- a/app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt +++ b/app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt @@ -139,6 +139,51 @@ class FaceLandmarkerHelper( } } + private var frameId: Long = 0 + // 简化的延时跟踪 + private val frameTimings = mutableMapOf() + private var lastFpsUpdateTime: Long = 0 + private var frameCount: Int = 0 + private var currentFps: Double = 0.0 + + // 内部数据类用于存储时间信息 + private data class FrameTiming( + var captureTime: Long = 0, + var preprocessStartTime: Long = 0, + var preprocessEndTime: Long = 0, + var detectionStartTime: Long = 0 + ) + + // 辅助方法：记录帧时间信息 + private fun recordFrameTiming(frameId: Long, block: (FrameTiming) -> Unit) { + val timing = frameTimings.getOrPut(frameId) { FrameTiming() } + block(timing) + } + + // 辅助方法：更新FPS计算 + private fun updateFps() { + frameCount++ + val currentTime = SystemClock.uptimeMillis() + + if (lastFpsUpdateTime == 0L) { + lastFpsUpdateTime = currentTime + return + } + + val timeDiff = currentTime - lastFpsUpdateTime + if (timeDiff >= 1000) { // 每秒更新一次FPS + currentFps = frameCount * 1000.0 / timeDiff + frameCount = 0 + lastFpsUpdateTime = currentTime + } + } + + // 辅助方法：清理旧的时间记录 + private fun cleanupOldTimings(currentFrameId: Long) { + val framesToRemove = frameTimings.keys.filter { it < currentFrameId - 10 } + framesToRemove.forEach { frameTimings.remove(it) } + } + // Convert the ImageProxy to MP Image and feed it to FacelandmakerHelper. fun detectLiveStream( imageProxy: ImageProxy, @@ -152,6 +197,15 @@ class FaceLandmarkerHelper( } val frameTime = SystemClock.uptimeMillis() + val currentFrameId = frameId++ + val captureTime = SystemClock.uptimeMillis() + + // 记录捕获时间 + recordFrameTiming(currentFrameId) { + it.captureTime = captureTime + it.preprocessStartTime = SystemClock.uptimeMillis() + } + // Copy out RGB bits from the frame to a bitmap buffer val bitmapBuffer = Bitmap.createBitmap( @@ -198,152 +252,54 @@ class FaceLandmarkerHelper( // be returned in returnLivestreamResult function } - // Accepts the URI for a video file loaded from the user's gallery and attempts to run - // face landmarker inference on the video. This process will evaluate every - // frame in the video and attach the results to a bundle that will be - // returned. - fun detectVideoFile( - videoUri: Uri, - inferenceIntervalMs: Long - ): VideoResultBundle? { - if (runningMode != RunningMode.VIDEO) { - throw IllegalArgumentException( - "Attempting to call detectVideoFile" + - " while not using RunningMode.VIDEO" - ) - } - // Inference time is the difference between the system time at the start and finish of the - // process - val startTime = SystemClock.uptimeMillis() - - var didErrorOccurred = false - - // Load frames from the video and run the face landmarker. - val retriever = MediaMetadataRetriever() - retriever.setDataSource(context, videoUri) - val videoLengthMs = - retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) - ?.toLong() - - // Note: We need to read width/height from frame instead of getting the width/height - // of the video directly because MediaRetriever returns frames that are smaller than the - // actual dimension of the video file. - val firstFrame = retriever.getFrameAtTime(0) - val width = firstFrame?.width - val height = firstFrame?.height - - // If the video is invalid, returns a null detection result - if ((videoLengthMs == null) || (width == null) || (height == null)) return null - - // Next, we'll get one frame every frameInterval ms, then run detection on these frames. - val resultList = mutableListOf() - val numberOfFrameToRead = videoLengthMs.div(inferenceIntervalMs) - - for (i in 0..numberOfFrameToRead) { - val timestampMs = i * inferenceIntervalMs // ms - - retriever - .getFrameAtTime( - timestampMs * 1000, // convert from ms to micro-s - MediaMetadataRetriever.OPTION_CLOSEST - ) - ?.let { frame -> - // Convert the video frame to ARGB_8888 which is required by the MediaPipe - val argb8888Frame = - if (frame.config == Bitmap.Config.ARGB_8888) frame - else frame.copy(Bitmap.Config.ARGB_8888, false) - - // Convert the input Bitmap object to an MPImage object to run inference - val mpImage = BitmapImageBuilder(argb8888Frame).build() - - // Run face landmarker using MediaPipe Face Landmarker API - faceLandmarker?.detectForVideo(mpImage, timestampMs) - ?.let { detectionResult -> - resultList.add(detectionResult) - } ?: { - didErrorOccurred = true - faceLandmarkerHelperListener?.onError( - "ResultBundle could not be returned" + - " in detectVideoFile" - ) - } - } - ?: run { - didErrorOccurred = true - faceLandmarkerHelperListener?.onError( - "Frame at specified time could not be" + - " retrieved when detecting in video." - ) - } - } - - retriever.release() - - val inferenceTimePerFrameMs = - (SystemClock.uptimeMillis() - startTime).div(numberOfFrameToRead) - - return if (didErrorOccurred) { - null - } else { - VideoResultBundle(resultList, inferenceTimePerFrameMs, height, width) - } - } - - // Accepted a Bitmap and runs face landmarker inference on it to return - // results back to the caller - fun detectImage(image: Bitmap): ResultBundle? { - if (runningMode != RunningMode.IMAGE) { - throw IllegalArgumentException( - "Attempting to call detectImage" + - " while not using RunningMode.IMAGE" - ) - } - - - // Inference time is the difference between the system time at the - // start and finish of the process - val startTime = SystemClock.uptimeMillis() - - // Convert the input Bitmap object to an MPImage object to run inference - val mpImage = BitmapImageBuilder(image).build() - - // Run face landmarker using MediaPipe Face Landmarker API - faceLandmarker?.detect(mpImage)?.also { landmarkResult -> - val inferenceTimeMs = SystemClock.uptimeMillis() - startTime - return ResultBundle( - landmarkResult, - inferenceTimeMs, - image.height, - image.width - ) - } - - // If faceLandmarker?.detect() returns null, this is likely an error. Returning null - // to indicate this. - faceLandmarkerHelperListener?.onError( - "Face Landmarker failed to detect." - ) - return null - } // Return the landmark result to this FaceLandmarkerHelper's caller private fun returnLivestreamResult( result: FaceLandmarkerResult, input: MPImage ) { + val resultTime = SystemClock.uptimeMillis() + val frameId = result.timestampMs() if( result.faceLandmarks().size > 0 ) { - val finishTimeMs = SystemClock.uptimeMillis() - val inferenceTime = finishTimeMs - result.timestampMs() + // 计算各种延时 + val timings = frameTimings[frameId] + if (timings != null) { + val totalLatency = resultTime - timings.captureTime + val preprocessLatency = timings.preprocessEndTime - timings.preprocessStartTime + val detectionLatency = resultTime - timings.detectionStartTime + val captureToDetectionLatency = timings.detectionStartTime - timings.captureTime - faceLandmarkerHelperListener?.onResults( - ResultBundle( - result, - inferenceTime, - input.height, - input.width + Log.d("TimeLatency", + "总延时: ${totalLatency}ms | " + + "预处理: ${preprocessLatency}ms | " + + "检测: ${detectionLatency}ms | " + + "捕获到检测: ${captureToDetectionLatency}ms | " + + "FPS: ${String.format("%.1f", currentFps)}" ) - ) + + val finishTimeMs = SystemClock.uptimeMillis() + val inferenceTime = finishTimeMs - result.timestampMs() + faceLandmarkerHelperListener?.onResults( + ResultBundle( + result, + inferenceTime, // 主要是检测时间 + input.height, + input.width, + ) + ) + } else { + val finishTimeMs = SystemClock.uptimeMillis() + val inferenceTime = finishTimeMs - result.timestampMs() + faceLandmarkerHelperListener?.onResults( + ResultBundle( + result, + inferenceTime, + input.height, + input.width + ) + ) + } } else { faceLandmarkerHelperListener?.onEmpty() @@ -379,13 +335,6 @@ class FaceLandmarkerHelper( val inputImageWidth: Int, ) - data class VideoResultBundle( - val results: List, - val inferenceTime: Long, - val inputImageHeight: Int, - val inputImageWidth: Int, - ) - interface LandmarkerListener { fun onError(error: String, errorCode: Int = OTHER_ERROR) fun onResults(resultBundle: ResultBundle) diff --git a/app/android/java/com/khronos/vulkan_samples/MainActivity.java b/app/android/java/com/khronos/vulkan_samples/MainActivity.java index 6633d81..e466314 100644 --- a/app/android/java/com/khronos/vulkan_samples/MainActivity.java +++ b/app/android/java/com/khronos/vulkan_samples/MainActivity.java @@ -6,6 +6,7 @@ import android.content.pm.PackageManager; import android.os.Bundle; import android.os.Handler; import android.os.Looper; +import android.os.SystemClock; import android.util.Log; import android.view.WindowManager; import android.widget.Toast; @@ -182,6 +183,12 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L @Override public void analyze(@NonNull ImageProxy image) { try { + long analyzeTime = SystemClock.elapsedRealtimeNanos(); + long captureTime = image.getImageInfo().getTimestamp(); + long latencyNs = analyzeTime - captureTime; + double latencyMs = latencyNs / 1_000_000.0; + + Log.d("CameraLatency", String.format("Image capture to analyze latency: %.2f ms", latencyMs)); //Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight()); processImageForVulkan(image); detectFace(image); @@ -315,7 +322,7 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L nativeBuffer.order(ByteOrder.nativeOrder()); } - + long start_time = System.currentTimeMillis(); // ArrayList vertexs = new ArrayList<>(); // ArrayList triangle_index = new ArrayList<>(); @@ -358,7 +365,10 @@ public class MainActivity extends GameActivity implements FaceLandmarkerHelper.L FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer(); floatBuffer.put(points); floatBuffer.position(0); + long cur_time = System.currentTimeMillis(); + Log.i("TimeLatency","Result Data ProcessTime:" + (cur_time-start_time)); passDataToNative(nativeBuffer, index, width, height); + Log.i("TimeLatency","passDataToNative ProcessTime:" + (System.currentTimeMillis() - cur_time)); } public float z_rate = -1.0f;