thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Renamed tensor and rebuild functions

Browse source
This commit is contained in:
Alejandro Saucedo 2021-02-09 21:29:24 +00:00
parent 91252201ce
commit 0d9a9758da
2 changed files with 62 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

46
src/Manager.cpp
View file

@ -111,46 +111,34 @@ Manager::~Manager()
}
std::shared_ptr<Sequence>
Manager::getOrCreateManagedSequence(std::string sequenceName)
Manager::sequence(std::string sequenceName, uint32_t queueIndex)
{
SPDLOG_DEBUG("Kompute Manager creating Sequence object");
SPDLOG_DEBUG("Kompute Manager sequence() with sequenceName: {} "
"and queueIndex: {}",
sequenceName,
queueIndex);
std::shared_ptr<Sequence> sq = nullptr;
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator found =
this->mManagedSequences.find(sequenceName);
if (found == this->mManagedSequences.end()) {
return this->createManagedSequence(sequenceName);
std::shared_ptr<Sequence> sq =
std::make_shared<Sequence>(this->mPhysicalDevice,
this->mDevice,
this->mComputeQueues[queueIndex],
this->mComputeQueueFamilyIndices[queueIndex]);
sq->init();
this->mManagedSequences.insert({ sequenceName, sq });
return sq;
} else {
return found->second;
}
}
std::shared_ptr<Sequence>
Manager::createManagedSequence(std::string sequenceName, uint32_t queueIndex)
{
SPDLOG_DEBUG("Kompute Manager createManagedSequence with sequenceName: {} "
"and queueIndex: {}",
sequenceName,
queueIndex);
std::shared_ptr<Sequence> sq =
std::make_shared<Sequence>(this->mPhysicalDevice,
this->mDevice,
this->mComputeQueues[queueIndex],
this->mComputeQueueFamilyIndices[queueIndex]);
sq->init();
if (sequenceName.empty()) {
this->mCurrentSequenceIndex++;
this->mManagedSequences.insert({ KP_DEFAULT_SESSION, sq });
} else {
// TODO: Check if sequence doesn't already exist
this->mManagedSequences.insert({ sequenceName, sq });
}
return sq;
}
void
Manager::createInstance()
{

79
src/include/kompute/Manager.hpp
View file

@ -64,23 +64,12 @@ class Manager
*
* @param sequenceName The name for the named sequence to be retrieved or
* created
* @param queueIndex The queue to use from the available queues
* @return Shared pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> getOrCreateManagedSequence(
std::string sequenceName);
/**
* Create a new managed Kompute sequence so it's available within the
* manager.
*
* @param sequenceName The name for the named sequence to be created, if
* empty then default indexed value is used
* @param queueIndex The queue to use from the available queues
* @return Weak pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> createManagedSequence(
std::string sequenceName = "",
uint32_t queueIndex = 0);
std::shared_ptr<Sequence> sequence(
std::string sequenceName = KP_DEFAULT_SESSION,
uint32_t queueIndex = 0);
/**
* Function that evaluates operation against named sequence.
@ -97,7 +86,7 @@ class Manager
{
SPDLOG_DEBUG("Kompute Manager evalOp triggered");
std::shared_ptr<kp::Sequence> sq =
this->getOrCreateManagedSequence(sequenceName);
this->sequence(sequenceName);
SPDLOG_DEBUG("Kompute Manager evalOp running sequence BEGIN");
sq->begin();
@ -147,7 +136,7 @@ class Manager
SPDLOG_DEBUG("Kompute Manager evalOpAsync triggered");
std::shared_ptr<kp::Sequence> sq =
this->getOrCreateManagedSequence(sequenceName);
this->sequence(sequenceName);
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence BEGIN");
sq->begin();
@ -234,12 +223,12 @@ class Manager
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
std::shared_ptr<Tensor> buildTensor(
std::shared_ptr<Tensor> tensor(
const std::vector<float>& data,
Tensor::TensorTypes tensorType = Tensor::TensorTypes::eDevice,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager buildTensor triggered");
SPDLOG_DEBUG("Kompute Manager tensor triggered");
SPDLOG_DEBUG("Kompute Manager creating new tensor shared ptr");
std::shared_ptr<Tensor> tensor =
@ -261,28 +250,18 @@ class Manager
* create a new Tensor. The tensor memory will then be managed and owned by
* the manager.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
* @param tensors Array of tensors to rebuild
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
void rebuildTensors(std::vector<std::shared_ptr<kp::Tensor>> tensors,
void rebuild(std::vector<std::shared_ptr<kp::Tensor>> tensors,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuildTensors triggered");
SPDLOG_DEBUG("Kompute Manager rebuild triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
// False syncData to run all tensors at once instead one by one
this->rebuild(tensor, false);
}
if (syncDataToGPU) {
@ -290,6 +269,38 @@ class Manager
}
}
/**
* Function that simplifies the common workflow of tensor initialisation. It
* will take the constructor parameters for a Tensor and will will us it to
* create a new Tensor. The tensor memory will then be managed and owned by
* the manager.
*
* @param tensors Single tensor to rebuild
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
void rebuild(std::shared_ptr<kp::Tensor> tensor,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>({ tensor });
}
}
private:
// -------------- OPTIONALLY OWNED RESOURCES
std::shared_ptr<vk::Instance> mInstance = nullptr;