xsl/Vulkan-Samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
xsl
2025-09-04 10:54:47 +08:00
commit 6bc8f61b18
1808 changed files with 208268 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/general/README.adoc
+31
View File
@@ -0,0 +1,31 @@
////
- Copyright (c) 2024, Sascha Willems
-
- SPDX-License-Identifier: Apache-2.0
-
- Licensed under the Apache License, Version 2.0 the "License";
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
////
ifndef::general_samplespath[:general_samplespath:]
== General Samples
The goal of these samples is to demonstrate different techniques or showcase complex scenarios that doesn't necessarily fit any of the main categories.
=== xref:./{general_samplespath}mobile_nerf/README.adoc[Mobile NeRF]
A Neural Radiance Field synthesizer sample, based on textured polygons.
=== xref:./{general_samplespath}mobile_nerf_rayquery/README.adoc[Mobile NeRF Ray Query]
A Mobile Neural Radiance Field synthesizer sample using ray query, based on textured polygons.
samples/general/mobile_nerf/CMakeLists.txt
+40
View File
@@ -0,0 +1,40 @@
# Copyright (c) 2023-2025, Qualcomm Innovation Center, Inc. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 the "License";
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
get_filename_component(FOLDER_NAME ${CMAKE_CURRENT_LIST_DIR} NAME)
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_LIST_DIR} PATH)
get_filename_component(CATEGORY_NAME ${PARENT_DIR} NAME)
add_sample_with_tags(
ID ${FOLDER_NAME}
CATEGORY ${CATEGORY_NAME}
AUTHOR "Qualcomm"
NAME "Mobile NeRF"
DESCRIPTION "A Mobile Neural Radiance Field synthesizer sample, based on textured polygons"
SHADER_FILES_GLSL
"mobile_nerf/merged.frag"
"mobile_nerf/merged_morpheus.frag"
"mobile_nerf/mlp.frag"
"mobile_nerf/mlp_combo.frag"
"mobile_nerf/mlp_morpheus.frag"
"mobile_nerf/mlp_morpheus_combo.frag"
"mobile_nerf/quad.vert"
"mobile_nerf/raster.frag"
"mobile_nerf/raster.vert"
"mobile_nerf/raster_combo.frag"
"mobile_nerf/raster_morpheus.frag"
"mobile_nerf/raster_morpheus_combo.frag")
samples/general/mobile_nerf/README.adoc
+40
View File
@@ -0,0 +1,40 @@
////
- Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved
-
- SPDX-License-Identifier: Apache-2.0
-
- Licensed under the Apache License, Version 2.0 the "License";
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
////
= Mobile NeRF
ifdef::site-gen-antora[]
TIP: The source for this sample can be found in the https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/general/mobile_nerf[Khronos Vulkan samples github repository].
endif::[]
This sample is a modified version of the https://mobile-nerf.github.io/[Mobile Nerf] developed by Google.
It's based on its original https://github.com/google-research/jax3d/tree/main/jax3d/projects/mobilenerf[source code] but optimized for Vulkan.
This is a different version from traditional NeRF rendering, which normally requires tracing rays (usually done via ray-marching) and querying a MLP multiple times for each ray. These many queries result in non-interactive frame rates on most of the GPUs.
The mobile version uses the rasterization pipeline to render the final image; this is done via a triangle mesh and a feature texture, where each of its visible pixels are run through a small MLP (executed in the fragment shader) that converts the feature data and view direction to the corresponding output pixel color. This technique enables interactive FPS even on mobile GPUs (thus the name).
== Description: [https://mobile-nerf.github.io/[Mobile Nerf]]
Neural Radiance Fields (NeRFs) have demonstrated amazing ability to synthesize images of 3D scenes from novel views.
However, they rely upon specialized volumetric rendering algorithms based on ray marching that are mismatched to the capabilities of widely deployed graphics hardware.
This paper introduces a new NeRF representation based on textured polygons that can synthesize novel images efficiently with standard rendering pipelines.
The NeRF is represented as a set of polygons with textures representing binary opacities and feature vectors.
Traditional rendering of the polygons with a z-buffer yields an image with features at every pixel, which are interpreted by a small, view-dependent MLP running in a fragment shader to produce a final pixel color.
This approach enables NeRFs to be rendered with the traditional polygon rasterization pipeline, which provides massive pixel-level parallelism, achieving interactive frame rates on a wide range of compute platforms, including mobile phones.
== Notes
The original source code is also licensed under Apache-2.0, all shader files used by the sample have comments to indicate changes, when applicable.
samples/general/mobile_nerf/mobile_nerf.cpp
+1853
View File
File diff suppressed because it is too large Load Diff
samples/general/mobile_nerf/mobile_nerf.h
+237
View File
@@ -0,0 +1,237 @@
/* Copyright (c) 2023-2025, Qualcomm Innovation Center, Inc. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 the "License";
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "api_vulkan_sample.h"
#include <core/acceleration_structure.h>
#include <json.hpp>
using json = nlohmann::json;
namespace vkb
{
namespace sg
{
class Scene;
class Node;
class Mesh;
class SubMesh;
class Camera;
} // namespace sg
} // namespace vkb
class MobileNerf : public ApiVulkanSample
{
public:
MobileNerf();
~MobileNerf() override;
void request_gpu_features(vkb::PhysicalDevice &gpu) override;
void render(float delta_time) override;
bool prepare(const vkb::ApplicationOptions &options) override;
bool resize(const uint32_t width, const uint32_t height) override;
private:
struct GlobalUniform
{
glm::mat4x4 model;
glm::mat4x4 view;
glm::mat4x4 proj;
alignas(16) glm::vec3 camera_position;
alignas(16) glm::vec3 camera_side;
alignas(16) glm::vec3 camera_up;
alignas(16) glm::vec3 camera_lookat;
alignas(8) glm::vec2 img_dim;
alignas(4) float tan_half_fov;
} global_uniform;
struct PushConstants
{
unsigned int weight_idx;
} push_constants;
#define WEIGHTS_0_COUNT (176)
#define WEIGHTS_1_COUNT (256)
// The third layer weights' size is changed from 48 to 64 to make sure a 16 bytes alignement
#define WEIGHTS_2_COUNT (64)
#define BIAS_0_COUNT (16)
#define BIAS_1_COUNT (16)
// The third layer bias' size is changed from 3 to 4 to make sure a 16 bytes alignement
#define BIAS_2_COUNT (4)
struct MLP_Weights
{
float data[WEIGHTS_0_COUNT + WEIGHTS_1_COUNT + WEIGHTS_2_COUNT +
BIAS_0_COUNT + BIAS_1_COUNT + BIAS_2_COUNT]; // Array of floats
};
struct Vertex
{
glm::vec3 position;
glm::vec2 tex_coord;
};
struct InstancingInfo
{
glm::ivec3 dim;
glm::vec3 interval;
};
struct InstanceData
{
glm::vec3 pos_offset;
};
struct FrameBufferAttachment
{
using ImagePtr = std::unique_ptr<vkb::core::Image>;
ImagePtr image;
VkSampler sampler;
VkImageView view;
operator bool() const
{
return image && image->get_handle() != VK_NULL_HANDLE;
}
void destroy();
};
struct Model
{
std::vector<Vertex> vertices;
std::vector<std::array<uint32_t, 3>> indices;
Texture texture_input_0, texture_input_1;
// Vulkan Buffers for each model
std::unique_ptr<vkb::core::BufferC> vertex_buffer{nullptr};
std::unique_ptr<vkb::core::BufferC> index_buffer{nullptr};
// Each model will have its own pipeline and descriptor set
VkPipeline pipeline_first_pass{VK_NULL_HANDLE};
// We make the descriptor set a vector for the forward mode
// Deferred mode will only have one set of descriptor per model
std::vector<VkDescriptorSet> descriptor_set_first_pass{VK_NULL_HANDLE};
int sub_model_num;
int model_index;
};
std::vector<Model> models;
// MLPs for each model
std::vector<MLP_Weights> mlp_weight_vector;
std::vector<std::unique_ptr<vkb::core::BufferC>> weights_buffers;
// Uniform buffer for each model
std::vector<std::unique_ptr<vkb::core::BufferC>> uniform_buffers;
// Buffer to store instance data
std::unique_ptr<vkb::core::BufferC> instance_buffer{nullptr};
// Common
void read_json_map();
void load_shaders();
void build_command_buffers() override;
void create_uniforms();
void create_static_object_buffers(int model_index, int sub_model_index, int models_entry);
void prepare_instance_data();
void load_scene(int model_index, int sub_model_index, int models_entry);
void initialize_mlp_uniform_buffers(int model_index);
void update_uniform_buffers();
void update_weights_buffers();
void create_texture(int model_index, int sub_model_index, int models_entry);
void create_texture_helper(std::string const &texturePath, Texture &texture);
VkFormat feature_map_format = VK_FORMAT_R16G16B16A16_SFLOAT;
// Helper function
void setup_attachment(VkFormat format, VkImageUsageFlags usage, FrameBufferAttachment &attachment);
void draw();
// First pass
VkDescriptorSetLayout descriptor_set_first_pass_layout{VK_NULL_HANDLE};
VkPipelineLayout pipeline_first_pass_layout{VK_NULL_HANDLE};
std::array<VkPipelineShaderStageCreateInfo, 2> shader_stages_first_pass;
std::array<VkPipelineShaderStageCreateInfo, 2> shader_stages_second_pass;
void create_descriptor_pool();
void create_pipeline_layout_fist_pass();
void create_descriptor_sets_first_pass(Model &model);
void prepare_pipelines();
unsigned int color_attach_0_idx;
unsigned int color_attach_1_idx;
unsigned int color_attach_2_idx;
unsigned int color_attach_3_idx;
unsigned int depth_attach_idx;
unsigned int swapchain_attach_idx;
struct Attachments_baseline
{
FrameBufferAttachment feature_0, feature_1, feature_2, weights_idx;
};
std::vector<Attachments_baseline> frameAttachments;
std::vector<VkFramebuffer> nerf_framebuffers;
VkRenderPass render_pass_nerf{VK_NULL_HANDLE};
// For the baseline (mlp in frag shader) second pass
VkPipeline pipeline_baseline{VK_NULL_HANDLE};
VkPipelineLayout pipeline_layout_baseline{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_baseline{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_baseline{VK_NULL_HANDLE};
void update_render_pass_nerf_baseline();
void create_pipeline_layout_baseline();
void create_descriptor_sets_baseline();
void setup_nerf_framebuffer_baseline();
void update_descriptor_sets_baseline();
void build_command_buffers_baseline();
// For creating the forward mode rendepass
void update_render_pass_nerf_forward();
// For loading nerf assets map
json asset_map;
std::vector<std::string> model_path;
std::vector<bool> using_original_nerf_models;
bool combo_mode = false;
bool use_deferred = false;
bool do_rotation = false;
glm::vec3 camera_pos = glm::vec3(-2.2f, 2.2f, 2.2f);
// Currently combo mode translation are hard-coded
glm::mat4x4 combo_model_transform[4] = {
glm::translate(glm::vec3(0.5, 0.75, 0)), glm::translate(glm::vec3(0.5, 0.25, 0)),
glm::translate(glm::vec3(0, -0.25, 0.5)), glm::translate(glm::vec3(0, -0.75, -0.5))};
// For instancing
InstancingInfo instancing_info;
// Viewport Setting
float fov = 60.0f;
uint32_t view_port_width = width;
uint32_t view_port_height = height;
bool use_native_screen_size = false;
};
std::unique_ptr<vkb::VulkanSampleC> create_mobile_nerf();
samples/general/mobile_nerf_rayquery/CMakeLists.txt
+31
View File
@@ -0,0 +1,31 @@
# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 the "License";
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
get_filename_component(FOLDER_NAME ${CMAKE_CURRENT_LIST_DIR} NAME)
get_filename_component(PARENT_DIR ${CMAKE_CURRENT_LIST_DIR} PATH)
get_filename_component(CATEGORY_NAME ${PARENT_DIR} NAME)
add_sample_with_tags(
ID ${FOLDER_NAME}
CATEGORY ${CATEGORY_NAME}
AUTHOR "Qualcomm"
NAME "Mobile NeRF Ray Query"
DESCRIPTION "A Mobile Neural Radiance Field synthesizer sample using ray query, based on textured polygons."
SHADER_FILES_GLSL
"mobile_nerf_rayquery/quad.vert"
"mobile_nerf_rayquery/rayquery_morpheus_combo.frag"
"mobile_nerf_rayquery/rayquery_morpheus.frag")
samples/general/mobile_nerf_rayquery/README.adoc
+33
View File
@@ -0,0 +1,33 @@
////
- Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved
-
- SPDX-License-Identifier: Apache-2.0
-
- Licensed under the Apache License, Version 2.0 the "License";
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
////
= Mobile NeRF Ray Query
ifdef::site-gen-antora[]
TIP: The source for this sample can be found in the https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/general/mobile_nerf_rayquery[Khronos Vulkan samples github repository].
endif::[]
NeRF is a new 3D representation method in Computer Vision that creates images of a 3D scene using several 2D pictures taken from different viewpoints.
This method constructs a representation of the 3D volume. Various adaptations of NeRF target different use cases, including MobileNeRF, which focuses on rendering NeRF efficiently on mobile phones by leveraging existing traditional graphic hardware.
This version enhances the https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/general/mobile_nerf[previous MobileNeRF implementation] by using the Vulkan Ray Query feature, which leverages the hardware ray tracing capabilities of mobile GPUs that support it.
This enhancement greatly boosts performance in most use cases. Additionally, the Vulkan API provides great flexibility for modifying and optimizing the rendering pipeline and shaders, enabling more functionalities while delivering optimal performance.
== Notes
The original source code is also licensed under Apache-2.0, all shader files used by the sample have comments to indicate changes, when applicable.
samples/general/mobile_nerf_rayquery/mobile_nerf_rayquery.cpp
+1235
View File
File diff suppressed because it is too large Load Diff
samples/general/mobile_nerf_rayquery/mobile_nerf_rayquery.h
+195
View File
@@ -0,0 +1,195 @@
/* Copyright (c) 2023-2025, Qualcomm Innovation Center, Inc. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 the "License";
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "api_vulkan_sample.h"
#include <core/acceleration_structure.h>
#include <json.hpp>
using json = nlohmann::json;
namespace vkb
{
namespace sg
{
class Scene;
class Node;
class Mesh;
class SubMesh;
class Camera;
} // namespace sg
} // namespace vkb
class MobileNerfRayQuery : public ApiVulkanSample
{
public:
MobileNerfRayQuery();
~MobileNerfRayQuery() override;
void request_gpu_features(vkb::PhysicalDevice &gpu) override;
void render(float delta_time) override;
bool prepare(const vkb::ApplicationOptions &options) override;
private:
struct GlobalUniform
{
alignas(16) glm::mat4 view_inverse;
alignas(16) glm::mat4 proj_inverse;
alignas(8) glm::vec2 img_dim;
alignas(4) float tan_half_fov{};
} global_uniform;
#define WEIGHTS_0_COUNT (176)
#define WEIGHTS_1_COUNT (256)
// The third layer weights' size is changed from 48 to 64 to make sure a 16 bytes alignement
// #define WEIGHTS_2_COUNT (48)
#define WEIGHTS_2_COUNT (64)
#define BIAS_0_COUNT (16)
#define BIAS_1_COUNT (16)
// The third layer bias' size is changed from 3 to 4 to make sure a 16 bytes alignement
#define BIAS_2_COUNT (4)
// some typedef for each model
struct MLP_Weights
{
float data[WEIGHTS_0_COUNT + WEIGHTS_1_COUNT + WEIGHTS_2_COUNT +
BIAS_0_COUNT + BIAS_1_COUNT + BIAS_2_COUNT]; // Array of floats
};
struct Vertex
{
glm::vec3 position;
glm::vec2 tex_coord;
};
struct InstancingInfo
{
glm::ivec3 dim;
glm::vec3 interval;
};
struct FrameBufferAttachment
{
VkSampler sampler;
VkDeviceMemory memory;
VkImage image = VK_NULL_HANDLE;
VkImageView view;
VkFormat format;
uint32_t width;
uint32_t height;
};
struct Model
{
int model_index{};
int sub_model_num{};
std::vector<Vertex> vertices;
std::vector<std::array<uint32_t, 3>> indices;
// Feature maps
Texture texture_input_0, texture_input_1;
// Each model has its vertex buffer and index buffer. In ray query, they are storage buffers.
std::unique_ptr<vkb::core::BufferC> vertex_buffer{nullptr};
std::unique_ptr<vkb::core::BufferC> index_buffer{nullptr};
// Each model has its BLAS
std::unique_ptr<vkb::core::AccelerationStructure> bottom_level_acceleration_structure{nullptr};
} model;
std::vector<Model> models;
// MLPs for each model
std::vector<MLP_Weights> mlp_weight_vector;
std::vector<std::unique_ptr<vkb::core::BufferC>> weights_buffers;
// Global uniform buffer
std::unique_ptr<vkb::core::BufferC> uniform_buffer;
std::array<VkPipelineShaderStageCreateInfo, 2> shader_stages{};
VkPipeline pipeline{VK_NULL_HANDLE};
VkPipelineLayout pipeline_layout{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_common{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_vertices{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_indices{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_feature1{VK_NULL_HANDLE};
std::vector<VkDescriptorSet> descriptor_set_feature2{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_common{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_vertices{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_indices{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_feature1{VK_NULL_HANDLE};
VkDescriptorSetLayout descriptor_set_layout_feature2{VK_NULL_HANDLE};
// Ray tracing structures
std::unique_ptr<vkb::core::AccelerationStructure> top_level_acceleration_structure{nullptr};
// For loading mobile nerf assets map
json asset_map;
int num_models = 0;
bool combo_mode = false;
bool do_rotation = false;
std::vector<std::string> model_path;
glm::vec3 camera_pos = glm::vec3(-2.2f, 2.2f, 2.2f);
// Currently combo mode translation are hard-coded
glm::mat4x4 combo_model_transform[4] = {
glm::translate(glm::vec3(0.5, 0.75, 0)), glm::translate(glm::vec3(0.5, 0.25, 0)),
glm::translate(glm::vec3(0, -0.25, 0.5)), glm::translate(glm::vec3(0, -0.75, -0.5))};
// For instancing
InstancingInfo instancing_info;
// Viewport Setting
float fov = 60.0f;
uint32_t view_port_width = width;
uint32_t view_port_height = height;
bool use_native_screen_size = false;
// Feature map format
VkFormat feature_map_format = VK_FORMAT_R16G16B16A16_SFLOAT;
void read_json_map();
void load_shaders();
void create_uniforms();
void create_static_object_buffers(int models_entry);
void load_scene(int model_index, int sub_model_index, int models_entry);
void initialize_mlp_uniform_buffers(int model_index);
void update_uniform_buffer();
void update_weights_buffers();
void create_pipeline_layout();
void create_descriptor_pool();
void create_descriptor_sets();
void prepare_pipelines();
void build_command_buffers() override;
void draw();
uint64_t get_buffer_device_address(VkBuffer buffer);
void create_top_level_acceleration_structure();
void create_bottom_level_acceleration_structure(int model_entry);
void create_texture(int model_index, int sub_model_index, int models_entry);
void create_texture_helper(std::string const &texturePath, Texture &texture_input);
};
std::unique_ptr<vkb::VulkanSampleC> create_mobile_nerf_rayquery();