thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Removed OpCreateTensor in favour of manager memory ownership

Browse source
This commit is contained in:
Alejandro Saucedo 2021-02-08 19:15:48 +00:00
parent bf401019c9
commit fc3d23d3f9
10 changed files with 64 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/include/kompute/Manager.hpp
View file

@ -1,13 +1,12 @@
#pragma once
#include <unordered_map>
#include <set>
#include "kompute/Core.hpp"
#include "kompute/Sequence.hpp"
#include "kompute/operations/OpTensorCreate.hpp"
#define KP_DEFAULT_SESSION "DEFAULT"
namespace kp {
@ -231,8 +230,8 @@ class Manager
/**
* Function that simplifies the common workflow of tensor creation and
* initialization. It will take the constructor parameters for a Tensor
* and will will us it to create a new Tensor and then create it using
* the OpCreateTensor command.
* and will will us it to create a new Tensor and then create it. The
* tensor memory will then be managed and owned by the manager.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
@ -242,17 +241,49 @@ class Manager
const std::vector<float>& data,
Tensor::TensorTypes tensorType = Tensor::TensorTypes::eDevice)
{
SPDLOG_DEBUG("Kompute Manager createInitTensor triggered");
SPDLOG_DEBUG("Kompute Manager buildTensor triggered");
SPDLOG_DEBUG("Kompute Manager creating new tensor shared ptr");
std::shared_ptr<Tensor> tensor =
std::make_shared<Tensor>(kp::Tensor(data, tensorType));
this->evalOpDefault<OpTensorCreate>({ tensor });
tensor->init(this->mPhysicalDevice, this->mDevice);
if (tensor->tensorType() != Tensor::TensorTypes::eStorage) {
tensor->mapDataIntoHostMemory();
}
this->mManagedTensors.insert(tensor);
return tensor;
}
/**
* Function that simplifies the common workflow of tensor initialisation. It will take the constructor parameters for a Tensor and will will us it to create a new Tensor. The tensor memory will then be managed and owned by the manager.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
* @returns Initialized Tensor with memory Syncd to GPU device
*/
void rebuildTensors(std::vector<std::shared_ptr<kp::Tensor>> tensors)
{
SPDLOG_DEBUG("Kompute Manager rebuildTensors triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
if (tensor->tensorType() != Tensor::TensorTypes::eStorage) {
tensor->mapDataIntoHostMemory();
}
std::set<std::shared_ptr<Tensor>>::iterator it = this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
}
}
private:
// -------------- OPTIONALLY OWNED RESOURCES
std::shared_ptr<vk::Instance> mInstance = nullptr;
@ -263,6 +294,8 @@ class Manager
bool mFreeDevice = false;
// -------------- ALWAYS OWNED RESOURCES
std::set<std::shared_ptr<Tensor>> mManagedTensors;
std::unordered_map<std::string, std::shared_ptr<Sequence>>
mManagedSequences;