xsl/Vulkan-Samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
xsl
2025-09-04 10:54:47 +08:00
commit 6bc8f61b18
1808 changed files with 208268 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/general/mobile_nerf_rayquery/CMakeLists.txt
+31
View File
@@ -0,0 +1,31 @@
# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 the "License";
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
get_filename_component(FOLDER_NAME ${CMAKE_CURRENT_LIST_DIR} NAME)
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_LIST_DIR} PATH)
get_filename_component(CATEGORY_NAME ${PARENT_DIR} NAME)
add_sample_with_tags(
ID ${FOLDER_NAME}
CATEGORY ${CATEGORY_NAME}
AUTHOR "Qualcomm"
NAME "Mobile NeRF Ray Query"
DESCRIPTION "A Mobile Neural Radiance Field synthesizer sample using ray query, based on textured polygons."
SHADER_FILES_GLSL
"mobile_nerf_rayquery/quad.vert"
"mobile_nerf_rayquery/rayquery_morpheus_combo.frag"
"mobile_nerf_rayquery/rayquery_morpheus.frag")
samples/general/mobile_nerf_rayquery/README.adoc
+33
View File
@@ -0,0 +1,33 @@
////
- Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved
-
- SPDX-License-Identifier: Apache-2.0
-
- Licensed under the Apache License, Version 2.0 the "License";
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
////
= Mobile NeRF Ray Query
ifdef::site-gen-antora[]
TIP: The source for this sample can be found in the https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/general/mobile_nerf_rayquery[Khronos Vulkan samples github repository].
endif::[]
NeRF is a new 3D representation method in Computer Vision that creates images of a 3D scene using several 2D pictures taken from different viewpoints.
This method constructs a representation of the 3D volume. Various adaptations of NeRF target different use cases, including MobileNeRF, which focuses on rendering NeRF efficiently on mobile phones by leveraging existing traditional graphic hardware.
This version enhances the https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/general/mobile_nerf[previous MobileNeRF implementation] by using the Vulkan Ray Query feature, which leverages the hardware ray tracing capabilities of mobile GPUs that support it.
This enhancement greatly boosts performance in most use cases. Additionally, the Vulkan API provides great flexibility for modifying and optimizing the rendering pipeline and shaders, enabling more functionalities while delivering optimal performance.
== Notes
The original source code is also licensed under Apache-2.0, all shader files used by the sample have comments to indicate changes, when applicable.
samples/general/mobile_nerf_rayquery/mobile_nerf_rayquery.cpp
+1235
View File
File diff suppressed because it is too large Load Diff
samples/general/mobile_nerf_rayquery/mobile_nerf_rayquery.h
+195
View File
@@ -0,0 +1,195 @@
/* Copyright (c) 2023-2025, Qualcomm Innovation Center, Inc. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 the "License";
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "api_vulkan_sample.h"
#include <core/acceleration_structure.h>
#include <json.hpp>
using json = nlohmann::json;
namespace vkb
{
namespace sg
{
class Scene;
class Node;
class Mesh;
class SubMesh;
class Camera;
} // namespace sg
} // namespace vkb
class MobileNerfRayQuery : public ApiVulkanSample
{
public:
MobileNerfRayQuery();
~MobileNerfRayQuery() override;
void request_gpu_features(vkb::PhysicalDevice &gpu) override;
void render(float delta_time) override;
bool prepare(const vkb::ApplicationOptions &options) override;
private:
struct GlobalUniform
{
alignas(16) glm::mat4 view_inverse;
alignas(16) glm::mat4 proj_inverse;
alignas(8) glm::vec2 img_dim;
alignas(4) float tan_half_fov{};
} global_uniform;
#define WEIGHTS_0_COUNT (176)
#define WEIGHTS_1_COUNT (256)
// The third layer weights' size is changed from 48 to 64 to make sure a 16 bytes alignement
// #define WEIGHTS_2_COUNT (48)
#define WEIGHTS_2_COUNT (64)
#define BIAS_0_COUNT (16)
#define BIAS_1_COUNT (16)
// The third layer bias' size is changed from 3 to 4 to make sure a 16 bytes alignement
#define BIAS_2_COUNT (4)
// some typedef for each model
struct MLP_Weights
{
float data[WEIGHTS_0_COUNT + WEIGHTS_1_COUNT + WEIGHTS_2_COUNT +
BIAS_0_COUNT + BIAS_1_COUNT + BIAS_2_COUNT]; // Array of floats
};
struct Vertex
{
glm::vec3 position;
glm::vec2 tex_coord;
};
struct InstancingInfo
{
glm::ivec3 dim;
glm::vec3 interval;
};
struct FrameBufferAttachment
{
VkSampler sampler;
VkDeviceMemory memory;
VkImage image = VK_NULL_HANDLE;
VkImageView view;
VkFormat format;
uint32_t width;
uint32_t height;
};
struct Model
{
int model_index{};
int sub_model_num{};
std::vector<Vertex> vertices;
std::vector<std::array<uint32_t, 3>> indices;
// Feature maps
Texture texture_input_0, texture_input_1;
// Each model has its vertex buffer and index buffer. In ray query, they are storage buffers.
std::unique_ptr<vkb::core::BufferC> vertex_buffer{nullptr};
std::unique_ptr<vkb::core::BufferC> index_buffer{nullptr};
// Each model has its BLAS
std::unique_ptr<vkb::core::AccelerationStructure> bottom_level_acceleration_structure{nullptr};
} model;
std::vector<Model> models;
// MLPs for each model
std::vector<MLP_Weights> mlp_weight_vector;
std::vector<std::unique_ptr<vkb::core::BufferC>> weights_buffers;
// Global uniform buffer
std::unique_ptr<vkb::core::BufferC> uniform_buffer;
std::array<VkPipelineShaderStageCreateInfo, 2> shader_stages{};
VkPipeline pipeline{VK_NULL_HANDLE};
VkPipelineLayout pipeline_layout{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_common{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_vertices{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_indices{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_feature1{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_feature2{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_common{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_vertices{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_indices{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_feature1{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_feature2{VK_NULL_HANDLE};
// Ray tracing structures
std::unique_ptr<vkb::core::AccelerationStructure> top_level_acceleration_structure{nullptr};
// For loading mobile nerf assets map
json asset_map;
int num_models = 0;
bool combo_mode = false;
bool do_rotation = false;
std::vector<std::string> model_path;
glm::vec3 camera_pos = glm::vec3(-2.2f, 2.2f, 2.2f);
// Currently combo mode translation are hard-coded
glm::mat4x4 combo_model_transform[4] = {
glm::translate(glm::vec3(0.5, 0.75, 0)), glm::translate(glm::vec3(0.5, 0.25, 0)),
glm::translate(glm::vec3(0, -0.25, 0.5)), glm::translate(glm::vec3(0, -0.75, -0.5))};
// For instancing
InstancingInfo instancing_info;
// Viewport Setting
float fov = 60.0f;
uint32_t view_port_width = width;
uint32_t view_port_height = height;
bool use_native_screen_size = false;
// Feature map format
VkFormat feature_map_format = VK_FORMAT_R16G16B16A16_SFLOAT;
void read_json_map();
void load_shaders();
void create_uniforms();
void create_static_object_buffers(int models_entry);
void load_scene(int model_index, int sub_model_index, int models_entry);
void initialize_mlp_uniform_buffers(int model_index);
void update_uniform_buffer();
void update_weights_buffers();
void create_pipeline_layout();
void create_descriptor_pool();
void create_descriptor_sets();
void prepare_pipelines();
void build_command_buffers() override;
void draw();
uint64_t get_buffer_device_address(VkBuffer buffer);
void create_top_level_acceleration_structure();
void create_bottom_level_acceleration_structure(int model_entry);
void create_texture(int model_index, int sub_model_index, int models_entry);
void create_texture_helper(std::string const &texturePath, Texture &texture_input);
};
std::unique_ptr<vkb::VulkanSampleC> create_mobile_nerf_rayquery();