#include "kompute/Tensor.hpp" namespace kp { Tensor::Tensor(std::shared_ptr physicalDevice, std::shared_ptr device, void* data, uint32_t elementTotalCount, uint32_t elementMemorySize, const TensorDataTypes& dataType, const TensorTypes& tensorType) { KP_LOG_DEBUG("Kompute Tensor constructor data length: {}, and type: {}", elementTotalCount, tensorType); this->mPhysicalDevice = physicalDevice; this->mDevice = device; this->mDataType = dataType; this->mTensorType = tensorType; this->rebuild(data, elementTotalCount, elementMemorySize); } Tensor::~Tensor() { KP_LOG_DEBUG("Kompute Tensor destructor started. Type: {}", this->tensorType()); this->destroy(); KP_LOG_DEBUG("Kompute Tensor destructor success"); } void Tensor::rebuild(void* data, uint32_t elementTotalCount, uint32_t elementMemorySize) { KP_LOG_DEBUG("Kompute Tensor rebuilding with size {}", elementTotalCount); this->mSize = elementTotalCount; this->mDataTypeMemorySize = elementMemorySize; if (this->mPrimaryBuffer || this->mPrimaryMemory) { KP_LOG_DEBUG( "Kompute Tensor destroying existing resources before rebuild"); this->destroy(); } this->allocateMemoryCreateGPUResources(); this->mapRawData(); memcpy(this->mRawData, data, this->memorySize()); } Tensor::TensorTypes Tensor::tensorType() { return this->mTensorType; } bool Tensor::isInit() { return this->mDevice && this->mPrimaryBuffer && this->mPrimaryMemory && this->mRawData; } void Tensor::recordCopyFrom(const vk::CommandBuffer& commandBuffer, std::shared_ptr copyFromTensor, bool createBarrier) { vk::DeviceSize bufferSize(this->memorySize()); vk::BufferCopy copyRegion(0, 0, bufferSize); KP_LOG_DEBUG("Kompute Tensor recordCopyFrom data size {}.", bufferSize); this->recordCopyBuffer(commandBuffer, copyFromTensor->mPrimaryBuffer, this->mPrimaryBuffer, bufferSize, copyRegion, createBarrier); } void Tensor::recordCopyFromStagingToDevice(const vk::CommandBuffer& commandBuffer, bool createBarrier) { vk::DeviceSize bufferSize(this->memorySize()); vk::BufferCopy copyRegion(0, 0, bufferSize); KP_LOG_DEBUG("Kompute Tensor copying data size {}.", bufferSize); this->recordCopyBuffer(commandBuffer, this->mStagingBuffer, this->mPrimaryBuffer, bufferSize, copyRegion, createBarrier); } void Tensor::recordCopyFromDeviceToStaging(const vk::CommandBuffer& commandBuffer, bool createBarrier) { vk::DeviceSize bufferSize(this->memorySize()); vk::BufferCopy copyRegion(0, 0, bufferSize); KP_LOG_DEBUG("Kompute Tensor copying data size {}.", bufferSize); this->recordCopyBuffer(commandBuffer, this->mPrimaryBuffer, this->mStagingBuffer, bufferSize, copyRegion, createBarrier); } void Tensor::recordCopyBuffer(const vk::CommandBuffer& commandBuffer, std::shared_ptr bufferFrom, std::shared_ptr bufferTo, vk::DeviceSize bufferSize, vk::BufferCopy copyRegion, bool createBarrier) { commandBuffer.copyBuffer(*bufferFrom, *bufferTo, copyRegion); if (createBarrier) { // Buffer to ensure wait until data is copied to staging buffer this->recordBufferMemoryBarrier(commandBuffer, vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eHostRead, vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost); } } void Tensor::recordBufferMemoryBarrier(const vk::CommandBuffer& commandBuffer, vk::AccessFlagBits srcAccessMask, vk::AccessFlagBits dstAccessMask, vk::PipelineStageFlagBits srcStageMask, vk::PipelineStageFlagBits dstStageMask) { KP_LOG_DEBUG("Kompute Tensor recording buffer memory barrier"); vk::DeviceSize bufferSize = this->memorySize(); vk::BufferMemoryBarrier bufferMemoryBarrier; bufferMemoryBarrier.buffer = *this->mPrimaryBuffer; bufferMemoryBarrier.size = bufferSize; bufferMemoryBarrier.srcAccessMask = srcAccessMask; bufferMemoryBarrier.dstAccessMask = dstAccessMask; bufferMemoryBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; bufferMemoryBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; commandBuffer.pipelineBarrier(srcStageMask, dstStageMask, vk::DependencyFlags(), nullptr, bufferMemoryBarrier, nullptr); } vk::DescriptorBufferInfo Tensor::constructDescriptorBufferInfo() { KP_LOG_DEBUG("Kompute Tensor construct descriptor buffer info size {}", this->memorySize()); vk::DeviceSize bufferSize = this->memorySize(); return vk::DescriptorBufferInfo(*this->mPrimaryBuffer, 0, // offset bufferSize); } vk::BufferUsageFlags Tensor::getPrimaryBufferUsageFlags() { switch (this->mTensorType) { case TensorTypes::eDevice: return vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; break; case TensorTypes::eHost: return vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; break; case TensorTypes::eStorage: return vk::BufferUsageFlagBits::eStorageBuffer; break; default: throw std::runtime_error("Kompute Tensor invalid tensor type"); } } vk::MemoryPropertyFlags Tensor::getPrimaryMemoryPropertyFlags() { switch (this->mTensorType) { case TensorTypes::eDevice: return vk::MemoryPropertyFlagBits::eDeviceLocal; break; case TensorTypes::eHost: return vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent; break; case TensorTypes::eStorage: return vk::MemoryPropertyFlagBits::eDeviceLocal; break; default: throw std::runtime_error("Kompute Tensor invalid tensor type"); } } vk::BufferUsageFlags Tensor::getStagingBufferUsageFlags() { switch (this->mTensorType) { case TensorTypes::eDevice: return vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; break; default: throw std::runtime_error("Kompute Tensor invalid tensor type"); } } vk::MemoryPropertyFlags Tensor::getStagingMemoryPropertyFlags() { switch (this->mTensorType) { case TensorTypes::eDevice: return vk::MemoryPropertyFlagBits::eHostVisible; break; default: throw std::runtime_error("Kompute Tensor invalid tensor type"); } } void Tensor::allocateMemoryCreateGPUResources() { KP_LOG_DEBUG("Kompute Tensor creating buffer"); if (!this->mPhysicalDevice) { throw std::runtime_error("Kompute Tensor phyisical device is null"); } if (!this->mDevice) { throw std::runtime_error("Kompute Tensor device is null"); } KP_LOG_DEBUG("Kompute Tensor creating primary buffer and memory"); this->mPrimaryBuffer = std::make_shared(); this->createBuffer(this->mPrimaryBuffer, this->getPrimaryBufferUsageFlags()); this->mFreePrimaryBuffer = true; this->mPrimaryMemory = std::make_shared(); this->allocateBindMemory(this->mPrimaryBuffer, this->mPrimaryMemory, this->getPrimaryMemoryPropertyFlags()); this->mFreePrimaryMemory = true; if (this->mTensorType == TensorTypes::eDevice) { KP_LOG_DEBUG("Kompute Tensor creating staging buffer and memory"); this->mStagingBuffer = std::make_shared(); this->createBuffer(this->mStagingBuffer, this->getStagingBufferUsageFlags()); this->mFreeStagingBuffer = true; this->mStagingMemory = std::make_shared(); this->allocateBindMemory(this->mStagingBuffer, this->mStagingMemory, this->getStagingMemoryPropertyFlags()); this->mFreeStagingMemory = true; } KP_LOG_DEBUG("Kompute Tensor buffer & memory creation successful"); } void Tensor::createBuffer(std::shared_ptr buffer, vk::BufferUsageFlags bufferUsageFlags) { vk::DeviceSize bufferSize = this->memorySize(); if (bufferSize < 1) { throw std::runtime_error( "Kompute Tensor attempted to create a zero-sized buffer"); } KP_LOG_DEBUG("Kompute Tensor creating buffer with memory size: {}, and " "usage flags: {}", bufferSize, vk::to_string(bufferUsageFlags)); // TODO: Explore having concurrent sharing mode (with option) vk::BufferCreateInfo bufferInfo(vk::BufferCreateFlags(), bufferSize, bufferUsageFlags, vk::SharingMode::eExclusive); this->mDevice->createBuffer(&bufferInfo, nullptr, buffer.get()); } void Tensor::allocateBindMemory(std::shared_ptr buffer, std::shared_ptr memory, vk::MemoryPropertyFlags memoryPropertyFlags) { KP_LOG_DEBUG("Kompute Tensor allocating and binding memory"); vk::PhysicalDeviceMemoryProperties memoryProperties = this->mPhysicalDevice->getMemoryProperties(); vk::MemoryRequirements memoryRequirements = this->mDevice->getBufferMemoryRequirements(*buffer); uint32_t memoryTypeIndex = -1; for (uint32_t i = 0; i < memoryProperties.memoryTypeCount; i++) { if (memoryRequirements.memoryTypeBits & (1 << i)) { if (((memoryProperties.memoryTypes[i]).propertyFlags & memoryPropertyFlags) == memoryPropertyFlags) { memoryTypeIndex = i; break; } } } if (memoryTypeIndex < 0) { throw std::runtime_error( "Memory type index for buffer creation not found"); } KP_LOG_DEBUG( "Kompute Tensor allocating memory index: {}, size {}, flags: {}", memoryTypeIndex, memoryRequirements.size, vk::to_string(memoryPropertyFlags)); vk::MemoryAllocateInfo memoryAllocateInfo(memoryRequirements.size, memoryTypeIndex); this->mDevice->allocateMemory(&memoryAllocateInfo, nullptr, memory.get()); this->mDevice->bindBufferMemory(*buffer, *memory, 0); } void Tensor::destroy() { KP_LOG_DEBUG("Kompute Tensor started destroy()"); // Setting raw data to null regardless whether device is available to invalidate Tensor this->mRawData = nullptr; this->mSize = 0; this->mDataTypeMemorySize = 0; if (!this->mDevice) { KP_LOG_WARN( "Kompute Tensor destructor reached with null Device pointer"); return; } // Unmap the current memory data this->unmapRawData(); if (this->mFreePrimaryBuffer) { if (!this->mPrimaryBuffer) { KP_LOG_WARN("Kompose Tensor expected to destroy primary buffer " "but got null buffer"); } else { KP_LOG_DEBUG("Kompose Tensor destroying primary buffer"); this->mDevice->destroy( *this->mPrimaryBuffer, (vk::Optional)nullptr); this->mPrimaryBuffer = nullptr; this->mFreePrimaryBuffer = false; } } if (this->mFreeStagingBuffer) { if (!this->mStagingBuffer) { KP_LOG_WARN("Kompose Tensor expected to destroy staging buffer " "but got null buffer"); } else { KP_LOG_DEBUG("Kompose Tensor destroying staging buffer"); this->mDevice->destroy( *this->mStagingBuffer, (vk::Optional)nullptr); this->mStagingBuffer = nullptr; this->mFreeStagingBuffer = false; } } if (this->mFreePrimaryMemory) { if (!this->mPrimaryMemory) { KP_LOG_WARN("Kompose Tensor expected to free primary memory but " "got null memory"); } else { KP_LOG_DEBUG("Kompose Tensor freeing primary memory"); this->mDevice->freeMemory( *this->mPrimaryMemory, (vk::Optional)nullptr); this->mPrimaryMemory = nullptr; this->mFreePrimaryMemory = false; } } if (this->mFreeStagingMemory) { if (!this->mStagingMemory) { KP_LOG_WARN("Kompose Tensor expected to free staging memory but " "got null memory"); } else { KP_LOG_DEBUG("Kompose Tensor freeing staging memory"); this->mDevice->freeMemory( *this->mStagingMemory, (vk::Optional)nullptr); this->mStagingMemory = nullptr; this->mFreeStagingMemory = false; } } if (this->mDevice) { this->mDevice = nullptr; } KP_LOG_DEBUG("Kompute Tensor successful destroy()"); } template<> Tensor::TensorDataTypes TensorT::dataType() { return Tensor::TensorDataTypes::eBool; } template<> Tensor::TensorDataTypes TensorT::dataType() { return Tensor::TensorDataTypes::eInt; } template<> Tensor::TensorDataTypes TensorT::dataType() { return Tensor::TensorDataTypes::eUnsignedInt; } template<> Tensor::TensorDataTypes TensorT::dataType() { return Tensor::TensorDataTypes::eFloat; } template<> Tensor::TensorDataTypes TensorT::dataType() { return Tensor::TensorDataTypes::eDouble; } }