thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Single include

Browse source
This commit is contained in:
Alejandro Saucedo 2021-02-09 21:30:52 +00:00
parent 4e9888e7d6
commit 1edcb425ce
1 changed files with 45 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

79
single_include/kompute/Kompute.hpp
View file

@ -1351,23 +1351,12 @@ class Manager
*
* @param sequenceName The name for the named sequence to be retrieved or
* created
* @param queueIndex The queue to use from the available queues
* @return Shared pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> getOrCreateManagedSequence(
std::string sequenceName);
/**
* Create a new managed Kompute sequence so it's available within the
* manager.
*
* @param sequenceName The name for the named sequence to be created, if
* empty then default indexed value is used
* @param queueIndex The queue to use from the available queues
* @return Weak pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> createManagedSequence(
std::string sequenceName = "",
uint32_t queueIndex = 0);
std::shared_ptr<Sequence> sequence(
std::string sequenceName = KP_DEFAULT_SESSION,
uint32_t queueIndex = 0);
/**
* Function that evaluates operation against named sequence.
@ -1384,7 +1373,7 @@ class Manager
{
SPDLOG_DEBUG("Kompute Manager evalOp triggered");
std::shared_ptr<kp::Sequence> sq =
this->getOrCreateManagedSequence(sequenceName);
this->sequence(sequenceName);
SPDLOG_DEBUG("Kompute Manager evalOp running sequence BEGIN");
sq->begin();
@ -1434,7 +1423,7 @@ class Manager
SPDLOG_DEBUG("Kompute Manager evalOpAsync triggered");
std::shared_ptr<kp::Sequence> sq =
this->getOrCreateManagedSequence(sequenceName);
this->sequence(sequenceName);
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence BEGIN");
sq->begin();
@ -1521,12 +1510,12 @@ class Manager
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
std::shared_ptr<Tensor> buildTensor(
std::shared_ptr<Tensor> tensor(
const std::vector<float>& data,
Tensor::TensorTypes tensorType = Tensor::TensorTypes::eDevice,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager buildTensor triggered");
SPDLOG_DEBUG("Kompute Manager tensor triggered");
SPDLOG_DEBUG("Kompute Manager creating new tensor shared ptr");
std::shared_ptr<Tensor> tensor =
@ -1548,28 +1537,18 @@ class Manager
* create a new Tensor. The tensor memory will then be managed and owned by
* the manager.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
* @param tensors Array of tensors to rebuild
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
void rebuildTensors(std::vector<std::shared_ptr<kp::Tensor>> tensors,
void rebuild(std::vector<std::shared_ptr<kp::Tensor>> tensors,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuildTensors triggered");
SPDLOG_DEBUG("Kompute Manager rebuild triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
// False syncData to run all tensors at once instead one by one
this->rebuild(tensor, false);
}
if (syncDataToGPU) {
@ -1577,6 +1556,38 @@ class Manager
}
}
/**
* Function that simplifies the common workflow of tensor initialisation. It
* will take the constructor parameters for a Tensor and will will us it to
* create a new Tensor. The tensor memory will then be managed and owned by
* the manager.
*
* @param tensors Single tensor to rebuild
* @param syncDataToGPU Whether to sync the data to GPU memory
* @returns Initialized Tensor with memory Syncd to GPU device
*/
void rebuild(std::shared_ptr<kp::Tensor> tensor,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>({ tensor });
}
}
private:
// -------------- OPTIONALLY OWNED RESOURCES
std::shared_ptr<vk::Instance> mInstance = nullptr;