阿斯蒂芬

and_proj/app/build.gradle

@@ -97,5 +97,5 @@ dependencies {
 	implementation "androidx.camera:camera-lifecycle:$camerax_version"
 	implementation "androidx.camera:camera-view:$camerax_version"
 	//noinspection Aligned16KB
-	implementation 'com.google.mediapipe:tasks-vision:0.20230731'
+	implementation 'com.google.mediapipe:tasks-vision:0.10.26'
 }

and_proj/app/download_tasks.gradle

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *             http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+task downloadTaskFile(type: Download) {
+    src 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task'
+    dest project.ext.ASSET_DIR + '/face_landmarker.task'
+    overwrite false
+}
+
+preBuild.dependsOn downloadTaskFile

app/android/AndroidManifest.xml

@@ -28,12 +28,15 @@
 	<!-- Declare permissions -->
 	<uses-permission android:name="android.permission.CAMERA" />
 	<uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
+	<uses-permission android:name="android.permission.INTERNET" />
+	<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 
 	<application android:label="@string/app_name"
 		android:debuggable="true"
 		android:icon="@mipmap/ic_launcher"
 		android:roundIcon="@mipmap/ic_launcher_round"
 		android:theme="@style/AppTheme"
+		android:networkSecurityConfig="@xml/network_security_config"
 		tools:ignore="GoogleAppIndexingWarning,HardcodedDebugMode">
 
 		<activity android:name="com.khronos.vulkan_samples.MainActivity"

app/android/java/com/khronos/vulkan_samples/CameraDataFetcher.java

@@ -0,0 +1,154 @@
+package com.khronos.vulkan_samples;
+
+import android.os.AsyncTask;
+import android.util.Log;
+import org.json.JSONObject;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class CameraDataFetcher {
+
+    public interface CameraDataCallback {
+        void onSuccess(CameraData data);
+        void onError(String errorMessage);
+    }
+
+    // 数据模型类
+    public static class CameraData {
+        public float fov;
+        public float x;
+        public float y;
+        public float z;
+        public float sx;
+        public float sy;
+        public float sz;
+        public float rotx;
+        public float roty;
+        public float rotz;
+        public float camx;
+        public float camy;
+        public float camz;
+        public float z_rate;
+
+        // 从JSONObject解析数据
+        public static CameraData fromJson(JSONObject json) {
+            CameraData data = new CameraData();
+            try {
+                data.fov = (float) json.getDouble("fov");
+                data.x = (float)json.getDouble("x");
+                data.y = (float)json.getDouble("y");
+                data.z = (float)json.getDouble("z");
+                data.sx = (float)json.getDouble("sx");
+                data.sy = (float)json.getDouble("sy");
+                data.sz = (float)json.getDouble("sz");
+                data.rotx = (float)json.getDouble("rotx");
+                data.roty = (float)json.getDouble("roty");
+                data.rotz = (float)json.getDouble("rotz");
+                data.camx = (float)json.getDouble("camx");
+                data.camy = (float)json.getDouble("camy");
+                data.camz = (float)json.getDouble("camz");
+                data.z_rate = (float)json.getDouble("z_rate");
+            } catch (Exception e) {
+                Log.e("CameraData", "解析JSON失败: " + e.getMessage());
+            }
+            return data;
+        }
+
+
+        @Override
+        public String toString() {
+            return String.format(
+                    "CameraData{fov=%.2f, x=%.2f, y=%.2f, z=%.2f," +
+                            "rotx=%.2f, roty=%.2f, rotz=%.2f, camx=%.2f, camy=%.2f, camz=%.2f}",
+                    fov, x, y, z, rotx, roty, rotz, camx, camy, camz
+            );
+        }
+    }
+
+    /**
+     * 异步获取相机数据
+     * @param apiUrl 接口URL
+     * @param callback 回调接口
+     */
+    public static void fetchCameraData(String apiUrl, CameraDataCallback callback) {
+        new FetchCameraDataTask(callback).execute(apiUrl);
+    }
+
+    // 异步任务类
+    private static class FetchCameraDataTask extends AsyncTask<String, Void, CameraData> {
+        private CameraDataCallback callback;
+        private String errorMessage;
+
+        public FetchCameraDataTask(CameraDataCallback callback) {
+            this.callback = callback;
+        }
+
+        @Override
+        protected CameraData doInBackground(String... urls) {
+            if (urls.length == 0) {
+                errorMessage = "URL不能为空";
+                return null;
+            }
+
+            String apiUrl = urls[0];
+            HttpURLConnection connection = null;
+            BufferedReader reader = null;
+
+            try {
+                URL url = new URL(apiUrl);
+                connection = (HttpURLConnection) url.openConnection();
+                connection.setRequestMethod("GET");
+                connection.setConnectTimeout(10000); // 10秒连接超时
+                connection.setReadTimeout(10000);    // 10秒读取超时
+                connection.setRequestProperty("Accept", "application/json");
+
+                int responseCode = connection.getResponseCode();
+                if (responseCode == HttpURLConnection.HTTP_OK) {
+                    InputStream inputStream = connection.getInputStream();
+                    reader = new BufferedReader(new InputStreamReader(inputStream));
+
+                    StringBuilder response = new StringBuilder();
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        response.append(line);
+                    }
+
+                    JSONObject jsonObject = new JSONObject(response.toString());
+                    return CameraData.fromJson(jsonObject);
+                } else {
+                    errorMessage = "HTTP请求失败，响应码: " + responseCode;
+                    return null;
+                }
+            } catch (Exception e) {
+                errorMessage = "获取相机数据时发生错误: " + e.getMessage();
+                Log.e("CameraDataFetcher", errorMessage, e);
+                return null;
+            } finally {
+                if (connection != null) {
+                    connection.disconnect();
+                }
+                try {
+                    if (reader != null) {
+                        reader.close();
+                    }
+                } catch (Exception e) {
+                    Log.e("CameraDataFetcher", "关闭流时出错", e);
+                }
+            }
+        }
+
+        @Override
+        protected void onPostExecute(CameraData result) {
+            if (callback != null) {
+                if (result != null) {
+                    callback.onSuccess(result);
+                } else {
+                    callback.onError(errorMessage != null ? errorMessage : "未知错误");
+                }
+            }
+        }
+    }
+}

app/android/java/com/khronos/vulkan_samples/FaceLandmarkerHelper.kt

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.mediapipe.examples.facelandmarker
+package com.khronos.vulkan_samples
 
 import android.content.Context
 import android.graphics.Bitmap
@@ -176,7 +176,7 @@ class FaceLandmarkerHelper(
             }
         }
         val rotatedBitmap = Bitmap.createBitmap(
-            bitmapBuffer, 0, 0, bitmapBuffer.width, bitmapBuffer.height,
+            bitmapBuffer, (640-480)/2, 0, bitmapBuffer.height, bitmapBuffer.height,
             matrix, true
         )
 

app/android/java/com/khronos/vulkan_samples/MainActivity.java

@@ -3,61 +3,42 @@ package com.khronos.vulkan_samples;
 
 import android.Manifest;
 import android.content.pm.PackageManager;
-import android.graphics.Color;
-import android.graphics.PixelFormat;
-import android.graphics.SurfaceTexture;
 import android.os.Bundle;
+import android.os.Handler;
+import android.os.Looper;
 import android.util.Log;
-import android.view.Surface;
+import android.view.WindowManager;
-import android.view.SurfaceView;
-import android.view.TextureView;
-import android.view.View;
-import android.view.ViewGroup;
-import android.widget.FrameLayout;
 import android.widget.Toast;
 
 import androidx.annotation.NonNull;
+import androidx.camera.core.AspectRatio;
 import androidx.camera.core.CameraSelector;
-import androidx.camera.core.Preview;
+import androidx.camera.core.ImageAnalysis;
-import androidx.camera.core.SurfaceRequest;
+import androidx.camera.core.ImageProxy;
 import androidx.camera.lifecycle.ProcessCameraProvider;
 import androidx.core.app.ActivityCompat;
 import androidx.core.content.ContextCompat;
 
 import com.google.androidgamesdk.GameActivity;
 import com.google.common.util.concurrent.ListenableFuture;
+import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
+import com.google.mediapipe.tasks.vision.core.RunningMode;
+import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarkerResult;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+import java.util.List;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
-public class MainActivity extends GameActivity {
+public class MainActivity extends GameActivity implements FaceLandmarkerHelper.LandmarkerListener, CameraDataFetcher.CameraDataCallback {
-    private TextureView textureView; // 使用 TextureView 替代 PreviewView
     private ProcessCameraProvider cameraProvider;
-    private FrameLayout containerLayout;
     private void initCamera(){
-        // 获取 GameActivity 的根视图
+        backgroundExecutor = Executors.newSingleThreadExecutor();
-        ViewGroup rootView = (ViewGroup) getWindow().getDecorView();
-
-        // 创建容器布局
-        containerLayout = new FrameLayout(this);
-        containerLayout.setLayoutParams(new ViewGroup.LayoutParams(
-                ViewGroup.LayoutParams.MATCH_PARENT,
-                ViewGroup.LayoutParams.MATCH_PARENT
-        ));
-
-        // 创建 TextureView 用于摄像头预览
-        textureView = new TextureView(this);
-        textureView.setLayoutParams(new ViewGroup.LayoutParams(
-                ViewGroup.LayoutParams.MATCH_PARENT,
-                ViewGroup.LayoutParams.MATCH_PARENT
-        ));
-
-        // 将 TextureView 添加到容器
-        containerLayout.addView(textureView);
-
-        // 将容器添加到根视图的底部（作为背景）
-        //rootView.addView(containerLayout); // 添加到索引0（最底层）
-        rootView.addView(containerLayout, 0);
-
-
         if (hasCameraPermission()) {
             startCamera();
         } else {
@@ -65,6 +46,15 @@ public class MainActivity extends GameActivity {
         }
     }
 
+    private void detectFace(ImageProxy imageProxy) {
+        faceLandmarkerHelper.detectLiveStream(
+                imageProxy,false
+        );
+    }
+
+    private Handler handler = new Handler(Looper.getMainLooper());
+    private Runnable cameraDataRunnable;
+    private static final long INTERVAL = 1000; // 5秒间隔
 
     @Override
     protected void onCreate(Bundle savedInstanceState) {
@@ -76,15 +66,70 @@ public class MainActivity extends GameActivity {
         }
         String[] argArray = new String[2];
         argArray[0] = "sample";
-        argArray[1] = "hello_triangle";
+        argArray[1] = "texture_loading";//"hello_triangle";
         sendArgumentsToPlatform(argArray);
         super.onCreate(savedInstanceState);
 
+        // 隐藏状态栏
+        getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
+                WindowManager.LayoutParams.FLAG_FULLSCREEN);
+
         initCamera();
 
+        backgroundExecutor.execute(new Runnable() {
+            @Override
+            public void run() {
+                faceLandmarkerHelper = new FaceLandmarkerHelper(
+                        0.5f,
+                        0.5f,
+                         0.5f,
+                        1,
+                        0,
+                        RunningMode.LIVE_STREAM,
+                        MainActivity.this,
+                        MainActivity.this
+                        );
+            }
+        });
+
+        // 初始化Runnable
+        cameraDataRunnable = new Runnable() {
+            @Override
+            public void run() {
+                fetchCameraData();
+                handler.postDelayed(this, INTERVAL); // 再次调度
+            }
+        };
+
+        // 开始周期性调用
+        startPeriodicFetching();
+
+
         Log.i("MainActivity", "onCreate: ");
     }
 
+    boolean canFetch = true;
+    private void fetchCameraData() {
+        if(canFetch)
+        {
+            CameraDataFetcher.fetchCameraData("http://175.178.100.240:5000/data", MainActivity.this);
+            canFetch = false;
+        }
+    }
+    private void startPeriodicFetching() {
+        handler.postDelayed(cameraDataRunnable, INTERVAL);
+    }
+
+    private void stopPeriodicFetching() {
+        handler.removeCallbacks(cameraDataRunnable);
+    }
+
+    @Override
+    protected void onDestroy() {
+        super.onDestroy();
+        stopPeriodicFetching(); // 避免内存泄漏
+    }
+
     private boolean hasCameraPermission() {
         return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
                 == PackageManager.PERMISSION_GRANTED;
@@ -97,6 +142,9 @@ public class MainActivity extends GameActivity {
     }
 
     private static final int REQUEST_CAMERA_PERMISSION = 1001;
+    ImageAnalysis imageAnalyzer;
+    ExecutorService backgroundExecutor;
+    FaceLandmarkerHelper faceLandmarkerHelper;
 
     private void startCamera() {
         ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
@@ -106,26 +154,33 @@ public class MainActivity extends GameActivity {
             try {
                 cameraProvider = cameraProviderFuture.get();
 
-                Preview preview = new Preview.Builder().build();
-
                 CameraSelector cameraSelector = new CameraSelector.Builder()
                         .requireLensFacing(CameraSelector.LENS_FACING_BACK)
                         .build();
 
-                // 使用 TextureView 的 SurfaceTexture
+                imageAnalyzer = new ImageAnalysis.Builder()
-                SurfaceTexture surfaceTexture = textureView.getSurfaceTexture();
+                        .setTargetAspectRatio(AspectRatio.RATIO_4_3)
-                if (surfaceTexture != null) {
+                        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
-                    Surface surface = new Surface(surfaceTexture);
+                        .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
-                    preview.setSurfaceProvider(new Preview.SurfaceProvider() {
+                        .build();
-                        @Override
-                        public void onSurfaceRequested(@NonNull SurfaceRequest request) {
-                            request.provideSurface(surface, ContextCompat.getMainExecutor(MainActivity.this),
-                                    result -> {});
-                        }
-                    });
-                }
 
-                cameraProvider.bindToLifecycle(this, cameraSelector, preview);
+
+                imageAnalyzer.setAnalyzer(backgroundExecutor, new ImageAnalysis.Analyzer() {
+
+                    @Override
+                    public void analyze(@NonNull ImageProxy image) {
+                        try {
+                            //Log.d("Camera", "Received image: " + image.getWidth() + "x" + image.getHeight());
+                            processImageForVulkan(image);
+                            detectFace(image);
+                        } finally {
+                            //image.close(); // 确保在这里关闭
+                            //Log.d("Camera", "Image closed, ready for next frame");
+                        }
+                    }
+                });
+
+                cameraProvider.bindToLifecycle(this, cameraSelector, imageAnalyzer);
 
             } catch (ExecutionException | InterruptedException e) {
                 Log.e("MainActivity", "Error starting camera: " + e.getMessage());
@@ -134,6 +189,34 @@ public class MainActivity extends GameActivity {
     }
 
 
+    private void processImageForVulkan(ImageProxy image) {
+        // 获取图像信息
+        int width = image.getWidth();
+        int height = image.getHeight();
+        int format = image.getFormat();
+
+        // 获取图像数据平面
+        ImageProxy.PlaneProxy[] planes = image.getPlanes();
+
+        // 对于 RGBA_8888 格式，通常只有一个平面
+        if (planes.length > 0) {
+            ImageProxy.PlaneProxy plane = planes[0];
+            ByteBuffer buffer = plane.getBuffer();
+            int rowStride = plane.getRowStride();
+            int pixelStride = plane.getPixelStride();
+
+            // 调用 Native 方法处理图像
+            processImageNative(buffer, width, height, format,
+                    rowStride, pixelStride, image.getImageInfo().getRotationDegrees());
+        }
+    }
+
+    // Native 方法
+    private native void processImageNative(ByteBuffer buffer, int width, int height,
+                                           int format, int rowStride, int pixelStride,
+                                           int rotation);
+
+
     private static final int REQUEST_CODE_PERMISSIONS = 1001;
     private static final String[] REQUIRED_PERMISSIONS = new String[]{Manifest.permission.CAMERA};
     private static final String TAG = "MainActivity";
@@ -161,4 +244,72 @@ public class MainActivity extends GameActivity {
     }
 
     private native void sendArgumentsToPlatform(String[] args);
+
+    @Override
+    public void onError(@NotNull String error, int errorCode) {
+        Log.e(TAG, "onError: Face Error");
+    }
+
+    private ByteBuffer nativeBuffer = null;
+    float[] points = new float[480 * 3];
+
+
+    @Override
+    public void onResults(FaceLandmarkerHelper.@NotNull ResultBundle resultBundle) {
+        FaceLandmarkerResult faceLandmarkerResult = resultBundle.getResult();
+        int height = resultBundle.getInputImageHeight();
+        int width = resultBundle.getInputImageWidth();
+        int bufferSize = 480 * 3 * 4;
+
+        if(nativeBuffer == null)
+        {
+            // 创建直接ByteBuffer（在native内存中）
+            nativeBuffer = ByteBuffer.allocateDirect(bufferSize);
+            nativeBuffer.order(ByteOrder.nativeOrder());
+        }
+
+        int index = 0;
+        // Iterate through each detected face
+        for (List<NormalizedLandmark> faceLandmarks : faceLandmarkerResult.faceLandmarks()) {
+            //should be 1
+
+            int arrayIndex = 0;
+            for (NormalizedLandmark p : faceLandmarks)
+            {
+                //Log.i(TAG, "Index" + index + " x:"+p.x() + " y:"+p.y() + " z:"+p.z());
+                points[arrayIndex++] = p.x();
+                points[arrayIndex++] = p.y();
+                points[arrayIndex++] = p.z() * z_rate;
+                index++;
+            }
+        }
+
+        // 转换为FloatBuffer并填充数据
+        FloatBuffer floatBuffer = nativeBuffer.asFloatBuffer();
+        floatBuffer.put(points);
+        floatBuffer.position(0);
+        passDataToNative(nativeBuffer, index, width, height);
+    }
+
+    public float z_rate = 1.0f;
+
+    public native void passDataToNative(ByteBuffer buffer, int pointCount, int width, int height);
+    public native void upDateCameraData(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz,  float camx, float camy, float camz);
+
+    @Override
+    public void onSuccess(CameraDataFetcher.CameraData data) {
+        z_rate = data.z_rate;
+        upDateCameraData(data.fov, data.x, data.y, data.z, data.sx, data.sy, data.sz,
+                data.rotx, data.roty, data.rotz,
+                data.camx, data.camy, data.camz
+        );
+        canFetch = true;
+        Log.e(TAG, "GetCameraData " + data.toString());
+    }
+
+    @Override
+    public void onError(String errorMessage) {
+        Log.e(TAG, "GetCameraData Error");
+        canFetch = true;
+    }
 }

app/android/res/xml/network_security_config.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<network-security-config>
+    <base-config cleartextTrafficPermitted="true">
+        <trust-anchors>
+            <certificates src="system" />
+            <!-- 如果需要兼容非常旧的Android版本（API 23及以下），可以加上 user -->
+            <!-- <certificates src="user" /> -->
+        </trust-anchors>
+    </base-config>
+</network-security-config>

components/android/src/context.cpp

@@ -107,4 +107,69 @@ JNIEXPORT void JNICALL
 Java_com_khronos_vulkan_1samples_MainActivity_nativeButtonClicked(JNIEnv *env, jobject thiz) {
     // TODO: implement nativeButtonClicked()
 	LOGI("call MainActivity_nativeButtonClicked.");
+}
+
+extern void TextureLoadProcessWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize);
+extern void ReceiveFacePoint(float* pos, int count, int width, int height);
+
+void processWithVulkan(uint8_t* data, int width, int height, int format,
+                       int rowStride, size_t dataSize) {
+    // Vulkan 处理逻辑
+    // 1. 创建 Vulkan 图像（如果尚未创建）
+    // 2. 将数据复制到 Vulkan 图像内存
+    // 3. 执行 Vulkan 渲染或计算操作
+
+    //LOGI("ProcessingImage: %dx%d, stride: %d, size: %zu", width, height, rowStride, dataSize);
+    TextureLoadProcessWithVulkan(data, width, height, rowStride, dataSize);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_khronos_vulkan_1samples_MainActivity_processImageNative(JNIEnv *env, jobject thiz,
+                                                                 jobject buffer, jint width,
+                                                                 jint height, jint format,
+                                                                 jint row_stride, jint pixel_stride,
+                                                                 jint rotation) {
+    // TODO: implement processImageNative()
+    // 获取直接字节缓冲区指针
+    uint8_t* imageData = static_cast<uint8_t*>(env->GetDirectBufferAddress(buffer));
+    jlong capacity = env->GetDirectBufferCapacity(buffer);
+
+    if (imageData == nullptr) {
+        LOGI("Failed to get direct buffer address");
+        return;
+    }
+
+    // 在这里将图像数据传递给 Vulkan
+    // 创建 Vulkan 图像或更新现有图像
+    processWithVulkan(imageData, width, height, format, row_stride, capacity);
+}
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_khronos_vulkan_1samples_MainActivity_passDataToNative(JNIEnv *env, jobject thiz,
+                                                               jobject buffer, jint pointCount, jint width, jint height) {
+    // TODO: implement passDataToNative()
+
+    // 获取直接缓冲区的指针
+    float* pos = static_cast<float*>(env->GetDirectBufferAddress(buffer));
+
+    if (pos == nullptr) {
+        // 处理错误
+        return;
+    }
+
+    // 或者直接将指针传递给Vulkan
+    ReceiveFacePoint(pos, pointCount, width, height);
+}
+
+extern void global_update_mvp(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz, float camx, float camy, float camz);
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_khronos_vulkan_1samples_MainActivity_upDateCameraData(JNIEnv *env, jobject thiz,
+                                                               jfloat fov, jfloat x, jfloat y,jfloat z,
+                                                               jfloat sx, jfloat sy,jfloat sz,
+                                                               jfloat rotx, jfloat roty,jfloat rotz,
+                                                               jfloat camx,jfloat camy, jfloat camz) {
+    global_update_mvp(fov, x, y, z, sx, sy, sz, rotx, roty, rotz, camx, camy, camz);
 }

framework/api_vulkan_sample.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2025, Sascha Willems
+/* Copyright (c) 2019-2025, Sascha Willems
  * Copyright (c) 2024-2025, Arm Limited and Contributors
  *
  * SPDX-License-Identifier: Apache-2.0
@@ -644,6 +644,9 @@ void ApiVulkanSample::create_command_pool()
 {
 	VkCommandPoolCreateInfo command_pool_info = {};
 	command_pool_info.sType                   = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+	// --- 新增：设置允许重置命令缓冲区的标志 ---
+	command_pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
+	// --- 新增结束 ---
 	command_pool_info.queueFamilyIndex        = get_device().get_queue_by_flags(VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT, 0).get_family_index();
 	VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_info, nullptr, &cmd_pool));
 }

framework/api_vulkan_sample.h

@@ -409,8 +409,8 @@ class ApiVulkanSample : public vkb::VulkanSampleC
 
   public:
 	bool     prepared = false;
-	uint32_t width    = 1280;
+	uint32_t width    = 480;
-	uint32_t height   = 720;
+	uint32_t height   = 480;
 
 	VkClearColorValue default_clear_color = {{0.002f, 0.002f, 0.002f, 1.0f}};
 

framework/platform/window.h

@@ -72,7 +72,7 @@ class Window
 		Mode        mode      = Mode::Default;
 		bool        resizable = true;
 		Vsync       vsync     = Vsync::Default;
-		Extent      extent    = {1280, 720};
+		Extent      extent    = {480, 480};
 	};
 
 	/**

samples/api/texture_loading/CMakeLists.txt

@@ -28,6 +28,10 @@ add_sample_with_tags(
     SHADER_FILES_GLSL
         "texture_loading/glsl/texture.vert"
         "texture_loading/glsl/texture.frag"
+        "texture_loading/glsl/bg.vert"
+        "texture_loading/glsl/bg.frag"
+        "texture_loading/glsl/pointcloud.vert"
+        "texture_loading/glsl/pointcloud.frag"
     SHADER_FILES_HLSL
         "texture_loading/hlsl/texture.vert.hlsl"
         "texture_loading/hlsl/texture.frag.hlsl")

samples/api/texture_loading/texture_loading.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2024, Sascha Willems
+/* Copyright (c) 2019-2024, Sascha Willems
  *
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -15,17 +15,19 @@
  * limitations under the License.
  */
 
-/*
+ /*
- * Texture loading (and display) example (including mip maps)
+  * Texture loading (and display) example (including mip maps)
- */
+  */
 
 #pragma once
 
 #include <ktx.h>
 
 #include "api_vulkan_sample.h"
+#include <thread>
+#include <mutex>
 
-// Vertex layout for this example
+  // Vertex layout for this example
 struct TextureLoadingVertexStructure
 {
 	float pos[3];
@@ -33,11 +35,14 @@ struct TextureLoadingVertexStructure
 	float normal[3];
 };
 
+struct PointVertex {
+	float x, y, z; // 位置
+	float r, g, b; // 颜色 (可以根据需要修改)
+};
+
 class TextureLoading : public ApiVulkanSample
 {
-  public:
+public:
-	// Contains all Vulkan objects that are required to store and use a texture
-	// Note that this repository contains a texture class (vulkan_texture.h) that encapsulates texture loading functionality in a class that is used in subsequent demos
 	struct Texture
 	{
 		VkSampler      sampler;
@@ -47,13 +52,18 @@ class TextureLoading : public ApiVulkanSample
 		VkImageView    view;
 		uint32_t       width, height;
 		uint32_t       mip_levels;
-	} texture;
+	};
+
+	Texture texture = { 0 };
+	Texture texture_point = { 0 };
+	Texture cam_text = { 0 };
 
 	std::unique_ptr<vkb::core::BufferC> vertex_buffer;
 	std::unique_ptr<vkb::core::BufferC> index_buffer;
 	uint32_t                            index_count;
 
-	std::unique_ptr<vkb::core::BufferC> uniform_buffer_vs;
+	std::unique_ptr<vkb::core::BufferC> vertex_buffer_point;
+	uint32_t                            point_count = 0;
 
 	struct
 	{
@@ -63,33 +73,108 @@ class TextureLoading : public ApiVulkanSample
 		float     lod_bias = 0.0f;
 	} ubo_vs;
 
+	struct
+	{
+		glm::mat4 projection;
+		glm::mat4 model;
+		glm::mat4 view;
+	} ubo_vs_point;
+
+	std::unique_ptr<vkb::core::BufferC> uniform_buffer_vs;
+	std::unique_ptr<vkb::core::BufferC> uniform_buffer_vs_point;
+
+
+	//glm::vec3 camear_pos_point = {0, 0, -2.f};
+	//glm::vec3 face_pos_point;
+
 	struct
 	{
 		VkPipeline solid;
+		VkPipeline background;
+		VkPipeline point;
 	} pipelines;
 
+
 	VkPipelineLayout      pipeline_layout;
+	VkPipelineLayout      pipeline_layout_bg;
+	VkPipelineLayout      pipeline_layout_point;
+
 	VkDescriptorSet       descriptor_set;
+	VkDescriptorSet       descriptor_set_bg;
+	VkDescriptorSet       descriptor_set_point;
+
 	VkDescriptorSetLayout descriptor_set_layout;
+	VkDescriptorSetLayout descriptor_set_layout_bg;
+	VkDescriptorSetLayout descriptor_set_layout_point;
+
+	static TextureLoading* Get() {
+		return this_instance;
+	}
+
+	static TextureLoading* this_instance;
 
 	TextureLoading();
 	~TextureLoading();
-	virtual void request_gpu_features(vkb::PhysicalDevice &gpu) override;
+	virtual void request_gpu_features(vkb::PhysicalDevice& gpu) override;
 	void         load_texture();
 	void         destroy_texture(Texture texture);
 	void         build_command_buffers() override;
+
 	void         draw();
+	void		 draw_point_cloud(VkCommandBuffer command_buffer);
 	void         generate_quad();
+
 	void         setup_descriptor_pool();
+
 	void         setup_descriptor_set_layout();
 	void         setup_descriptor_set();
-	void         prepare_pipelines();
+
+	void         setup_descriptor_set_layout_bg();
+	void         setup_descriptor_set_bg();
+
+	void         setup_descriptor_set_layout_point();
+	void		 setup_descriptor_set_point();
+
 	void         prepare_uniform_buffers();
 	void         update_uniform_buffers();
-	bool         prepare(const vkb::ApplicationOptions &options) override;
+
+	void         prepare_uniform_buffers_point();
+	void         update_uniform_buffers_point(float fov, float x, float y, float z, float sx, float sy, float sz, float rotx, float roty, float rotz, float camx, float camy, float camz);
+
+
+	void         prepare_pipelines();
+	void		 prepare_pipeline_bg();
+	void		 prepare_pipeline_point();
+
+
+	bool         prepare(const vkb::ApplicationOptions& options) override;
 	virtual void render(float delta_time) override;
 	virtual void view_changed() override;
-	virtual void on_update_ui_overlay(vkb::Drawer &drawer) override;
+	virtual void on_update_ui_overlay(vkb::Drawer& drawer) override;
+
+	void            processWithVulkan(uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& out_texture);
+	void            createTexture(VkDevice device, VkPhysicalDevice physicalDevice, int width, int height, Texture& texture);
+	void            updateTexture(VkDevice device, VkPhysicalDevice physicalDevice, VkCommandPool commandPool, VkQueue queue, uint8_t* data, int width, int height, int rowStride, size_t dataSize, Texture& texture);
+	uint32_t        findMemoryType(VkPhysicalDevice physicalDevice, uint32_t typeFilter, VkMemoryPropertyFlags properties);
+	VkCommandBuffer beginSingleTimeCommands(VkDevice device, VkCommandPool commandPool);
+	void            endSingleTimeCommands(VkDevice device, VkCommandPool commandPool, VkQueue queue, VkCommandBuffer commandBuffer);
+	void            transitionImageLayout(VkCommandBuffer commandBuffer, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout);
+
+public:
+
+	void update_point_vertex_buffer(float* pos, int pointCount);
+private:
+	std::thread workerThread;
+	bool running = false;
+
+public:
+	//void run();
+	//void start();
+	std::mutex mtx;
+	std::mutex mtx_point;
+	std::mutex mtx_mvp;
+	//void stop();
+	//void updateTexture();
 };
 
 std::unique_ptr<vkb::Application> create_texture_loading();

shaders/texture_loading/glsl/bg.frag

@@ -0,0 +1,15 @@
+#version 450
+
+// 输入变量（与顶点着色器输出对应）
+layout(location = 0) in vec2 inTexCoord;
+
+// 输出颜色
+layout(location = 0) out vec4 outColor;
+
+// 纹理采样器
+layout(binding = 0) uniform sampler2D texSampler;
+
+void main() {
+    // 采样纹理
+    outColor = texture(texSampler, inTexCoord);
+}

shaders/texture_loading/glsl/bg.vert

@@ -0,0 +1,65 @@
+#version 450
+
+// 硬编码的顶点数据 - 4个顶点组成三角形带覆盖整个屏幕
+// const vec2 positions[6] = vec2[6](
+//     vec2( 1.0,  1.0),  // 右上 - 三角形1
+//     vec2(-1.0,  1.0),  // 左上 - 三角形1
+//     vec2(-1.0, -1.0),  // 左下 - 三角形1
+
+//     vec2(-1.0, -1.0),  // 左下 - 三角形2  
+//     vec2( 1.0, -1.0),  // 右下 - 三角形2
+//     vec2( 1.0,  1.0)   // 右上 - 三角形2
+// );
+
+const float offset = (640-480)/(480.0);
+
+
+const vec2 positions[6] = vec2[6](
+    vec2( 1.0,  -1.0 -offset),  // 右下
+    vec2(1.0,  1.0 + offset),  // 右上 - 三角形1
+    vec2(-1.0, 1.0 + offset),  // 左上 - 三角形1
+
+    vec2(-1.0, 1.0 + offset),  // 左上 - 三角形2  
+    vec2( -1.0, -1.0-offset),  // 左下 - 三角形2
+    vec2( 1.0,  -1.0-offset)   // 右下 - 三角形2
+);
+
+
+//const float offset = (640-480)/(640.0*2); // 假设宽高比为4:3
+// const float offset = 0;
+
+// const vec2 positions[6] = vec2[6](
+//     vec2( -1.0 - offset,  1.0),  // 右上 -> 左上
+//     vec2(-1.0 - offset,  -1.0),  // 左上 - 左下
+//     vec2(1.0 + offset, -1.0),  // 左下 - 右下
+
+//     vec2(1.0 + offset, -1.0),  // 左下 - 右下  
+//     vec2( 1.0 + offset, 1.0),  // 右下 - 右上
+//     vec2( -1.0 - offset,  1.0)   // 右上 - 左上
+// );
+
+
+
+const vec2 texCoords[6] = vec2[6](
+    vec2(1.0, 1.0),  // 右上
+    vec2(0.0, 1.0),  // 左上
+    vec2(0.0, 0.0),  // 左下
+    
+    vec2(0.0, 0.0),  // 左下
+    vec2(1.0, 0.0),  // 右下
+    vec2(1.0, 1.0)   // 右上
+);
+
+// 输出变量
+layout(location = 0) out vec2 outTexCoord;
+
+void main() {
+    // 获取顶点位置
+    vec2 position = positions[gl_VertexIndex];
+    
+    // 设置输出位置（Vulkan使用不同的坐标系）
+    gl_Position = vec4(position, 0.0, 1.0);
+    
+    // 输出纹理坐标
+    outTexCoord = texCoords[gl_VertexIndex];
+}

shaders/texture_loading/glsl/pointcloud.frag

@@ -0,0 +1,11 @@
+#version 450
+
+// 从顶点着色器接收的颜色
+layout(location = 0) in vec3 fragColor;
+
+// 输出颜色
+layout(location = 0) out vec4 outColor;
+
+void main() {
+    outColor = vec4(fragColor, 1.0); // 使用传入的颜色
+}

shaders/texture_loading/glsl/pointcloud.vert

@@ -0,0 +1,21 @@
+#version 450
+
+layout(location = 0) in vec3 inPosition;
+layout(location = 1) in vec3 inColor;
+
+layout(location = 0) out vec3 fragColor;
+
+layout(binding = 0) uniform UniformBufferObject {
+    mat4 proj;
+    mat4 model;
+    mat4 view;
+} ubo;
+
+void main() {
+    // 预计算MVP矩阵以减少乘法次数
+    mat4 mvp = ubo.proj * ubo.view * ubo.model;
+    //vec3 pos = vec3(2.0, inPosition.y, inPosition.z);
+    gl_Position = mvp * vec4(inPosition, 1.0);
+    fragColor = inColor;
+    gl_PointSize = 2.0;
+}