/* Copyright (c) 2023-2025, Qualcomm Innovation Center, Inc. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 the "License"; * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include "api_vulkan_sample.h" #include #include using json = nlohmann::json; namespace vkb { namespace sg { class Scene; class Node; class Mesh; class SubMesh; class Camera; } // namespace sg } // namespace vkb class MobileNerf : public ApiVulkanSample { public: MobileNerf(); ~MobileNerf() override; void request_gpu_features(vkb::PhysicalDevice &gpu) override; void render(float delta_time) override; bool prepare(const vkb::ApplicationOptions &options) override; bool resize(const uint32_t width, const uint32_t height) override; private: struct GlobalUniform { glm::mat4x4 model; glm::mat4x4 view; glm::mat4x4 proj; alignas(16) glm::vec3 camera_position; alignas(16) glm::vec3 camera_side; alignas(16) glm::vec3 camera_up; alignas(16) glm::vec3 camera_lookat; alignas(8) glm::vec2 img_dim; alignas(4) float tan_half_fov; } global_uniform; struct PushConstants { unsigned int weight_idx; } push_constants; #define WEIGHTS_0_COUNT (176) #define WEIGHTS_1_COUNT (256) // The third layer weights' size is changed from 48 to 64 to make sure a 16 bytes alignement #define WEIGHTS_2_COUNT (64) #define BIAS_0_COUNT (16) #define BIAS_1_COUNT (16) // The third layer bias' size is changed from 3 to 4 to make sure a 16 bytes alignement #define BIAS_2_COUNT (4) struct MLP_Weights { float data[WEIGHTS_0_COUNT + WEIGHTS_1_COUNT + WEIGHTS_2_COUNT + BIAS_0_COUNT + BIAS_1_COUNT + BIAS_2_COUNT]; // Array of floats }; struct Vertex { glm::vec3 position; glm::vec2 tex_coord; }; struct InstancingInfo { glm::ivec3 dim; glm::vec3 interval; }; struct InstanceData { glm::vec3 pos_offset; }; struct FrameBufferAttachment { using ImagePtr = std::unique_ptr; ImagePtr image; VkSampler sampler; VkImageView view; operator bool() const { return image && image->get_handle() != VK_NULL_HANDLE; } void destroy(); }; struct Model { std::vector vertices; std::vector> indices; Texture texture_input_0, texture_input_1; // Vulkan Buffers for each model std::unique_ptr vertex_buffer{nullptr}; std::unique_ptr index_buffer{nullptr}; // Each model will have its own pipeline and descriptor set VkPipeline pipeline_first_pass{VK_NULL_HANDLE}; // We make the descriptor set a vector for the forward mode // Deferred mode will only have one set of descriptor per model std::vector descriptor_set_first_pass{VK_NULL_HANDLE}; int sub_model_num; int model_index; }; std::vector models; // MLPs for each model std::vector mlp_weight_vector; std::vector> weights_buffers; // Uniform buffer for each model std::vector> uniform_buffers; // Buffer to store instance data std::unique_ptr instance_buffer{nullptr}; // Common void read_json_map(); void load_shaders(); void build_command_buffers() override; void create_uniforms(); void create_static_object_buffers(int model_index, int sub_model_index, int models_entry); void prepare_instance_data(); void load_scene(int model_index, int sub_model_index, int models_entry); void initialize_mlp_uniform_buffers(int model_index); void update_uniform_buffers(); void update_weights_buffers(); void create_texture(int model_index, int sub_model_index, int models_entry); void create_texture_helper(std::string const &texturePath, Texture &texture); VkFormat feature_map_format = VK_FORMAT_R16G16B16A16_SFLOAT; // Helper function void setup_attachment(VkFormat format, VkImageUsageFlags usage, FrameBufferAttachment &attachment); void draw(); // First pass VkDescriptorSetLayout descriptor_set_first_pass_layout{VK_NULL_HANDLE}; VkPipelineLayout pipeline_first_pass_layout{VK_NULL_HANDLE}; std::array shader_stages_first_pass; std::array shader_stages_second_pass; void create_descriptor_pool(); void create_pipeline_layout_fist_pass(); void create_descriptor_sets_first_pass(Model &model); void prepare_pipelines(); unsigned int color_attach_0_idx; unsigned int color_attach_1_idx; unsigned int color_attach_2_idx; unsigned int color_attach_3_idx; unsigned int depth_attach_idx; unsigned int swapchain_attach_idx; struct Attachments_baseline { FrameBufferAttachment feature_0, feature_1, feature_2, weights_idx; }; std::vector frameAttachments; std::vector nerf_framebuffers; VkRenderPass render_pass_nerf{VK_NULL_HANDLE}; // For the baseline (mlp in frag shader) second pass VkPipeline pipeline_baseline{VK_NULL_HANDLE}; VkPipelineLayout pipeline_layout_baseline{VK_NULL_HANDLE}; VkDescriptorSetLayout descriptor_set_layout_baseline{VK_NULL_HANDLE}; std::vector descriptor_set_baseline{VK_NULL_HANDLE}; void update_render_pass_nerf_baseline(); void create_pipeline_layout_baseline(); void create_descriptor_sets_baseline(); void setup_nerf_framebuffer_baseline(); void update_descriptor_sets_baseline(); void build_command_buffers_baseline(); // For creating the forward mode rendepass void update_render_pass_nerf_forward(); // For loading nerf assets map json asset_map; std::vector model_path; std::vector using_original_nerf_models; bool combo_mode = false; bool use_deferred = false; bool do_rotation = false; glm::vec3 camera_pos = glm::vec3(-2.2f, 2.2f, 2.2f); // Currently combo mode translation are hard-coded glm::mat4x4 combo_model_transform[4] = { glm::translate(glm::vec3(0.5, 0.75, 0)), glm::translate(glm::vec3(0.5, 0.25, 0)), glm::translate(glm::vec3(0, -0.25, 0.5)), glm::translate(glm::vec3(0, -0.75, -0.5))}; // For instancing InstancingInfo instancing_info; // Viewport Setting float fov = 60.0f; uint32_t view_port_width = width; uint32_t view_port_height = height; bool use_native_screen_size = false; }; std::unique_ptr create_mobile_nerf();