#include #include #include "kompute/Manager.hpp" namespace kp { #if DEBUG #ifndef KOMPUTE_DISABLE_VK_DEBUG_LAYERS static VKAPI_ATTR VkBool32 VKAPI_CALL debugMessageCallback(VkDebugReportFlagsEXT flags, VkDebugReportObjectTypeEXT objectType, uint64_t object, size_t location, int32_t messageCode, const char* pLayerPrefix, const char* pMessage, void* pUserData) { SPDLOG_DEBUG("[VALIDATION]: {} - {}", pLayerPrefix, pMessage); return VK_FALSE; } #endif #endif Manager::Manager() : Manager(0) {} Manager::Manager(uint32_t physicalDeviceIndex, const std::vector& familyQueueIndices) { this->mPhysicalDeviceIndex = physicalDeviceIndex; this->createInstance(); this->createDevice(familyQueueIndices); } Manager::Manager(std::shared_ptr instance, std::shared_ptr physicalDevice, std::shared_ptr device, uint32_t physicalDeviceIndex) { this->mInstance = instance; this->mPhysicalDevice = physicalDevice; this->mDevice = device; this->mPhysicalDeviceIndex = physicalDeviceIndex; } Manager::~Manager() { SPDLOG_DEBUG("Kompute Manager Destructor started"); if (this->mDevice == nullptr) { SPDLOG_ERROR( "Kompute Manager destructor reached with null Device pointer"); return; } if (this->mManagedSequences.size()) { SPDLOG_DEBUG("Kompute Manager explicitly running destructor for " "managed sequences"); for (const std::pair>& sqPair : this->mManagedSequences) { sqPair.second->freeMemoryDestroyGPUResources(); } this->mManagedSequences.clear(); } if (this->mFreeDevice) { SPDLOG_INFO("Destroying device"); this->mDevice->destroy( (vk::Optional)nullptr); SPDLOG_DEBUG("Kompute Manager Destroyed Device"); } if (this->mInstance == nullptr) { SPDLOG_ERROR( "Kompute Manager destructor reached with null Instance pointer"); return; } #if DEBUG #ifndef KOMPUTE_DISABLE_VK_DEBUG_LAYERS if (this->mDebugReportCallback) { this->mInstance->destroyDebugReportCallbackEXT( this->mDebugReportCallback, nullptr, this->mDebugDispatcher); SPDLOG_DEBUG("Kompute Manager Destroyed Debug Report Callback"); } #endif #endif if (this->mFreeInstance) { this->mInstance->destroy( (vk::Optional)nullptr); SPDLOG_DEBUG("Kompute Manager Destroyed Instance"); } } std::shared_ptr Manager::getOrCreateManagedSequence(std::string sequenceName) { SPDLOG_DEBUG("Kompute Manager creating Sequence object"); std::unordered_map>::iterator found = this->mManagedSequences.find(sequenceName); if (found == this->mManagedSequences.end()) { return this->createManagedSequence(sequenceName); } else { return found->second; } } std::shared_ptr Manager::createManagedSequence(std::string sequenceName, uint32_t queueIndex) { SPDLOG_DEBUG("Kompute Manager createManagedSequence with sequenceName: {} " "and queueIndex: {}", sequenceName, queueIndex); std::shared_ptr sq = std::make_shared(this->mPhysicalDevice, this->mDevice, this->mComputeQueues[queueIndex], this->mComputeQueueFamilyIndices[queueIndex]); sq->init(); if (sequenceName.empty()) { this->mCurrentSequenceIndex++; this->mManagedSequences.insert( { KP_DEFAULT_SESSION + std::to_string(this->mCurrentSequenceIndex), sq }); } else { // TODO: Check if sequence doesn't already exist this->mManagedSequences.insert({ sequenceName, sq }); } return sq; } void Manager::createInstance() { SPDLOG_DEBUG("Kompute Manager creating instance"); this->mFreeInstance = true; vk::ApplicationInfo applicationInfo; applicationInfo.pApplicationName = "Vulkan Kompute"; applicationInfo.pEngineName = "VulkanKompute"; applicationInfo.apiVersion = KOMPUTE_VK_API_VERSION; applicationInfo.engineVersion = KOMPUTE_VK_API_VERSION; applicationInfo.applicationVersion = KOMPUTE_VK_API_VERSION; std::vector applicationExtensions; applicationExtensions.push_back(VK_EXT_DEBUG_REPORT_EXTENSION_NAME); vk::InstanceCreateInfo computeInstanceCreateInfo; computeInstanceCreateInfo.pApplicationInfo = &applicationInfo; if (!applicationExtensions.empty()) { computeInstanceCreateInfo.enabledExtensionCount = (uint32_t)applicationExtensions.size(); computeInstanceCreateInfo.ppEnabledExtensionNames = applicationExtensions.data(); } #if DEBUG #ifndef KOMPUTE_DISABLE_VK_DEBUG_LAYERS SPDLOG_DEBUG("Kompute Manager adding debug validation layers"); // We'll identify the layers that are supported std::vector validLayerNames; std::vector desiredLayerNames = { "VK_LAYER_LUNARG_assistant_layer", "VK_LAYER_LUNARG_standard_validation" }; // Identify the valid layer names based on the desiredLayerNames { std::set uniqueLayerNames; std::vector availableLayerProperties = vk::enumerateInstanceLayerProperties(); for (vk::LayerProperties layerProperties : availableLayerProperties) { std::string layerName(layerProperties.layerName.data()); uniqueLayerNames.insert(layerName); } for (const char* desiredLayerName : desiredLayerNames) { if (uniqueLayerNames.count(desiredLayerName) != 0) { validLayerNames.push_back(desiredLayerName); } } } if (validLayerNames.size() > 0) { computeInstanceCreateInfo.enabledLayerCount = (uint32_t)validLayerNames.size(); computeInstanceCreateInfo.ppEnabledLayerNames = validLayerNames.data(); } #endif #endif this->mInstance = std::make_shared(); vk::createInstance( &computeInstanceCreateInfo, nullptr, this->mInstance.get()); SPDLOG_DEBUG("Kompute Manager Instance Created"); #if DEBUG #ifndef KOMPUTE_DISABLE_VK_DEBUG_LAYERS SPDLOG_DEBUG("Kompute Manager adding debug callbacks"); if (validLayerNames.size() > 0) { vk::DebugReportFlagsEXT debugFlags = vk::DebugReportFlagBitsEXT::eError | vk::DebugReportFlagBitsEXT::eWarning; vk::DebugReportCallbackCreateInfoEXT debugCreateInfo = {}; debugCreateInfo.pfnCallback = (PFN_vkDebugReportCallbackEXT)debugMessageCallback; debugCreateInfo.flags = debugFlags; this->mDebugDispatcher.init(*this->mInstance, &vkGetInstanceProcAddr); this->mDebugReportCallback = this->mInstance->createDebugReportCallbackEXT( debugCreateInfo, nullptr, this->mDebugDispatcher); } #endif #endif } void Manager::createDevice(const std::vector& familyQueueIndices) { SPDLOG_DEBUG("Kompute Manager creating Device"); if (this->mInstance == nullptr) { throw std::runtime_error("Kompute Manager instance is null"); } if (this->mPhysicalDeviceIndex < 0) { throw std::runtime_error( "Kompute Manager physical device index not provided"); } this->mFreeDevice = true; std::vector physicalDevices = this->mInstance->enumeratePhysicalDevices(); vk::PhysicalDevice physicalDevice = physicalDevices[this->mPhysicalDeviceIndex]; this->mPhysicalDevice = std::make_shared(physicalDevice); vk::PhysicalDeviceProperties physicalDeviceProperties = physicalDevice.getProperties(); SPDLOG_INFO("Using physical device index {} found {}", this->mPhysicalDeviceIndex, physicalDeviceProperties.deviceName); if (!familyQueueIndices.size()) { // Find compute queue std::vector allQueueFamilyProperties = physicalDevice.getQueueFamilyProperties(); uint32_t computeQueueFamilyIndex = -1; for (uint32_t i = 0; i < allQueueFamilyProperties.size(); i++) { vk::QueueFamilyProperties queueFamilyProperties = allQueueFamilyProperties[i]; if (queueFamilyProperties.queueFlags & vk::QueueFlagBits::eCompute) { computeQueueFamilyIndex = i; break; } } if (computeQueueFamilyIndex < 0) { throw std::runtime_error("Compute queue is not supported"); } this->mComputeQueueFamilyIndices.push_back(computeQueueFamilyIndex); } else { this->mComputeQueueFamilyIndices = familyQueueIndices; } std::unordered_map familyQueueCounts; std::unordered_map> familyQueuePriorities; for (const auto& value : this->mComputeQueueFamilyIndices) { familyQueueCounts[value]++; familyQueuePriorities[value].push_back(1.0f); } std::unordered_map familyQueueIndexCount; std::vector deviceQueueCreateInfos; for (const auto& familyQueueInfo : familyQueueCounts) { // Setting the device count to 0 familyQueueIndexCount[familyQueueInfo.first] = 0; // Creating the respective device queue vk::DeviceQueueCreateInfo deviceQueueCreateInfo( vk::DeviceQueueCreateFlags(), familyQueueInfo.first, familyQueueInfo.second, familyQueuePriorities[familyQueueInfo.first].data()); deviceQueueCreateInfos.push_back(deviceQueueCreateInfo); } vk::DeviceCreateInfo deviceCreateInfo(vk::DeviceCreateFlags(), deviceQueueCreateInfos.size(), deviceQueueCreateInfos.data()); this->mDevice = std::make_shared(); physicalDevice.createDevice( &deviceCreateInfo, nullptr, this->mDevice.get()); SPDLOG_DEBUG("Kompute Manager device created"); for (const uint32_t& familyQueueIndex : this->mComputeQueueFamilyIndices) { std::shared_ptr currQueue = std::make_shared(); this->mDevice->getQueue(familyQueueIndex, familyQueueIndexCount[familyQueueIndex], currQueue.get()); familyQueueIndexCount[familyQueueIndex]++; this->mComputeQueues.push_back(currQueue); } SPDLOG_DEBUG("Kompute Manager compute queue obtained"); } }